From 5342a43318afee1aa82ff2cf98db3f64cf21d1aa Mon Sep 17 00:00:00 2001
From: nyatinte <takumi.062757@gmail.com>
Date: Sun, 7 Apr 2024 21:42:19 +0900
Subject: [PATCH 01/74] Added resize functionality to notebooks

---
 .../templates/article/HighlightsLayer.tsx     | 31 +++++++++++++++++--
 packages/web/package.json                     |  1 +
 2 files changed, 29 insertions(+), 3 deletions(-)

diff --git a/packages/web/components/templates/article/HighlightsLayer.tsx b/packages/web/components/templates/article/HighlightsLayer.tsx
index c0066bc58..39242e188 100644
--- a/packages/web/components/templates/article/HighlightsLayer.tsx
+++ b/packages/web/components/templates/article/HighlightsLayer.tsx
@@ -32,6 +32,7 @@ import { NotebookContent } from './Notebook'
 import { NotebookHeader } from './NotebookHeader'
 import useGetWindowDimensions from '../../../lib/hooks/useGetWindowDimensions'
 import { ConfirmationModal } from '../../patterns/ConfirmationModal'
+import { Resizable } from 're-resizable'
 
 type HighlightsLayerProps = {
   viewer: UserBasicData
@@ -850,7 +851,7 @@ export function HighlightsLayer(props: HighlightsLayerProps): JSX.Element {
       <SlidingPane
         className="sliding-pane-class"
         isOpen={props.showHighlightsModal}
-        width={windowDimensions.width < 600 ? '100%' : '420px'}
+        width="fit-content"
         hideHeader={true}
         from="right"
         overlayClassName="slide-panel-overlay"
@@ -858,7 +859,31 @@ export function HighlightsLayer(props: HighlightsLayerProps): JSX.Element {
           props.setShowHighlightsModal(false)
         }}
       >
-        <>
+        <Resizable
+          onResize={(_e, _direction, ref) => {
+            if (parseInt(ref.style.width) < 210) {
+              props.setShowHighlightsModal(false)
+            }
+          }}
+          defaultSize={{
+            width: windowDimensions.width < 600 ? '100%' : '420px',
+            height: '100%',
+          }}
+          enable={
+            windowDimensions.width < 600
+              ? false
+              : {
+                  top: false,
+                  right: false,
+                  bottom: false,
+                  left: true,
+                  topRight: false,
+                  bottomRight: false,
+                  bottomLeft: false,
+                  topLeft: false,
+                }
+          }
+        >
           <NotebookHeader
             viewer={props.viewer}
             item={props.item}
@@ -887,7 +912,7 @@ export function HighlightsLayer(props: HighlightsLayerProps): JSX.Element {
               )
             }}
           />
-        </>
+        </Resizable>
       </SlidingPane>
     </>
   )
diff --git a/packages/web/package.json b/packages/web/package.json
index c10a0573c..a067e1b51 100644
--- a/packages/web/package.json
+++ b/packages/web/package.json
@@ -49,6 +49,7 @@
     "phosphor-react": "^1.4.0",
     "posthog-js": "^1.78.2",
     "pspdfkit": "^2023.4.6",
+    "re-resizable": "^6.9.11",
     "react": "^18.2.0",
     "react-color": "^2.19.3",
     "react-colorful": "^5.5.1",

From ab81d357aeb77722861f0c4863e5234daad5eecc Mon Sep 17 00:00:00 2001
From: Jackson Harper <jacksonh@gmail.com>
Date: Mon, 8 Apr 2024 18:47:51 +0800
Subject: [PATCH 02/74] Update version code

---
 android/Omnivore/app/build.gradle.kts | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/android/Omnivore/app/build.gradle.kts b/android/Omnivore/app/build.gradle.kts
index ea62ccd25..9030972dd 100644
--- a/android/Omnivore/app/build.gradle.kts
+++ b/android/Omnivore/app/build.gradle.kts
@@ -27,8 +27,8 @@ android {
         applicationId = "app.omnivore.omnivore"
         minSdk = 26
         targetSdk = 34
-        versionCode = 200004
-        versionName = "0.200.4"
+        versionCode = 2000050
+        versionName = "0.200.5"
 
         testInstrumentationRunner = "androidx.test.runner.AndroidJUnitRunner"
         vectorDrawables {

From dd235602d69a0f95f3e927d934fa3bde4227ee04 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E8=92=8B=E5=B0=8F=E9=9C=95?= <Holton.Jiang@Gmail.com>
Date: Mon, 8 Apr 2024 20:33:44 +0800
Subject: [PATCH 03/74] Add readerSettingsGeneric

---
 .../Sources/Views/Resources/zh-Hans.lproj/Localizable.strings    | 1 +
 1 file changed, 1 insertion(+)

diff --git a/apple/OmnivoreKit/Sources/Views/Resources/zh-Hans.lproj/Localizable.strings b/apple/OmnivoreKit/Sources/Views/Resources/zh-Hans.lproj/Localizable.strings
index 8d223097a..251667073 100644
--- a/apple/OmnivoreKit/Sources/Views/Resources/zh-Hans.lproj/Localizable.strings
+++ b/apple/OmnivoreKit/Sources/Views/Resources/zh-Hans.lproj/Localizable.strings
@@ -198,6 +198,7 @@
 "clubsGeneric" = "阅读俱乐部";
 "filterGeneric" = "特色功能";
 "errorGeneric" = "哦！出现了小问题，请您重试。";
+"readerSettingsGeneric" = "阅读设置";
 "pushNotificationsGeneric" = "推送通知";
 "dismissButton" = "返回或撤销";
 "errorNetwork" = "我们在连接到互联网时遇到问题。";

From 90fe5e8109bcc3f7c5ae434daa6b247bf1a26c84 Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Tue, 9 Apr 2024 10:30:40 +0800
Subject: [PATCH 04/74] ignore expiration of jwt token for tts streaming

---
 packages/text-to-speech/src/index.ts | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/packages/text-to-speech/src/index.ts b/packages/text-to-speech/src/index.ts
index 506ebf22f..d148204c5 100644
--- a/packages/text-to-speech/src/index.ts
+++ b/packages/text-to-speech/src/index.ts
@@ -232,8 +232,9 @@ export const textToSpeechStreamingHandler = Sentry.GCPFunction.wrapHttpFunction(
 
     let claim: Claim
     try {
-      jwt.verify(token, process.env.JWT_SECRET)
-      claim = jwt.decode(token) as Claim
+      claim = jwt.verify(token, process.env.JWT_SECRET, {
+        ignoreExpiration: true,
+      }) as Claim
     } catch (e) {
       console.error('Authentication error:', e)
       return res.status(401).send({ errorCode: 'UNAUTHENTICATED' })

From c4882c25427e9a016cc214e15dbad72e9f50f526 Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Tue, 9 Apr 2024 10:43:33 +0800
Subject: [PATCH 05/74] update dev cli

---
 packages/text-to-speech/package.json | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/packages/text-to-speech/package.json b/packages/text-to-speech/package.json
index c240ea2bc..f8e669fc9 100644
--- a/packages/text-to-speech/package.json
+++ b/packages/text-to-speech/package.json
@@ -18,6 +18,7 @@
     "start": "functions-framework --target=textToSpeechHandler",
     "start_streaming": "functions-framework --target=textToSpeechStreamingHandler",
     "dev": "concurrently \"tsc -w\" \"nodemon --watch ./build/ --exec npm run start\"",
+    "dev_streaming": "concurrently \"tsc -w\" \"nodemon --watch ./build/ --exec npm run start_streaming\"",
     "gcloud-deploy": "gcloud functions deploy text-to-speech --gen2 --entry-point=textToSpeechHandler --trigger-http --allow-unauthenticated --region=us-west2 --runtime nodejs14",
     "deploy": "yarn build && yarn gcloud-deploy"
   },
@@ -45,6 +46,7 @@
     "linkedom": "^0.14.12",
     "microsoft-cognitiveservices-speech-sdk": "1.30",
     "natural": "^6.2.0",
+    "nodemon": "^2.0.15",
     "underscore": "^1.13.4"
   },
   "volta": {

From 70b1a1d3c7c76716f0551b92990a48219637ca72 Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Tue, 9 Apr 2024 10:47:23 +0800
Subject: [PATCH 06/74] add comment

---
 packages/text-to-speech/src/index.ts | 1 +
 1 file changed, 1 insertion(+)

diff --git a/packages/text-to-speech/src/index.ts b/packages/text-to-speech/src/index.ts
index d148204c5..08686e8dc 100644
--- a/packages/text-to-speech/src/index.ts
+++ b/packages/text-to-speech/src/index.ts
@@ -232,6 +232,7 @@ export const textToSpeechStreamingHandler = Sentry.GCPFunction.wrapHttpFunction(
 
     let claim: Claim
     try {
+      // ignore expiration for now and verify function will also decode the token
       claim = jwt.verify(token, process.env.JWT_SECRET, {
         ignoreExpiration: true,
       }) as Claim

From e19d4357c88653539778fe142551b9baf54096de Mon Sep 17 00:00:00 2001
From: Jackson Harper <jacksonh@gmail.com>
Date: Tue, 9 Apr 2024 10:50:40 +0800
Subject: [PATCH 07/74] Add rules and email addresses to admin

---
 pkg/admin/src/db.ts    | 145 +++++++++++++++++++++++++++++------------
 pkg/admin/src/index.ts |  12 ++--
 2 files changed, 111 insertions(+), 46 deletions(-)

diff --git a/pkg/admin/src/db.ts b/pkg/admin/src/db.ts
index 6a56e8e90..eb4ce7dd9 100644
--- a/pkg/admin/src/db.ts
+++ b/pkg/admin/src/db.ts
@@ -44,7 +44,6 @@ export const registerDatabase = async (secrets: any): Promise<Connection> => {
       AdminUser,
       User,
       UserProfile,
-      UserArticle,
       ReceivedEmail,
       ContentDisplayReport,
       Group,
@@ -55,6 +54,8 @@ export const registerDatabase = async (secrets: any): Promise<Connection> => {
       Recommendation,
       GroupMembership,
       Features,
+      EmailAddress,
+      Rule,
     ],
   })
 
@@ -113,9 +114,6 @@ export class User extends BaseEntity {
   @Column({ type: 'enum', enum: StatusType })
   status!: StatusType
 
-  @OneToMany(() => UserArticle, (ua) => ua.user)
-  articles!: UserArticle[]
-
   @Column({ type: 'text' })
   source_user_id!: string
 
@@ -132,42 +130,17 @@ export class UserProfile extends BaseEntity {
   public username!: string
 
   @JoinColumn({ name: 'user_id' })
-  @ManyToOne(() => User, (user) => user.articles, { eager: true })
+  @ManyToOne(() => User, { eager: true })
   user!: User
 }
 
-@Entity({ name: 'user_articles' })
-export class UserArticle extends BaseEntity {
-  @PrimaryGeneratedColumn('uuid')
-  id!: string
-
-  @JoinColumn({ name: 'user_id' })
-  @ManyToOne(() => User, (user) => user.articles, { eager: true })
-  user!: User
-
-  @Column({ type: 'text', name: 'article_id' })
-  articleId!: string
-
-  @Column({ type: 'text' })
-  slug!: string
-
-  @Column({ type: 'timestamp', name: 'created_at' })
-  createdAt!: Date
-
-  @Column({ type: 'timestamp', name: 'updated_at' })
-  updatedAt!: Date
-
-  @Column({ type: 'timestamp', name: 'saved_at' })
-  savedAt!: Date
-}
-
 @Entity({ name: 'content_display_report' })
 export class ContentDisplayReport extends BaseEntity {
   @PrimaryGeneratedColumn('uuid')
   id!: string
 
   @JoinColumn({ name: 'user_id' })
-  @ManyToOne(() => User, (user) => user.articles, { eager: true })
+  @ManyToOne(() => User, { eager: true })
   user!: User
 
   @Column({ type: 'text', name: 'original_url' })
@@ -189,7 +162,7 @@ export class ReceivedEmail extends BaseEntity {
   id!: string
 
   @JoinColumn({ name: 'user_id' })
-  @ManyToOne(() => User, (user) => user.articles, { eager: true })
+  @ManyToOne(() => User, { eager: true })
   user!: User
 
   @Column('text')
@@ -254,7 +227,7 @@ export class Integration extends BaseEntity {
   id!: string
 
   @JoinColumn({ name: 'user_id' })
-  @ManyToOne(() => User, (user) => user.articles, { eager: true })
+  @ManyToOne(() => User, { eager: true })
   user!: User
 
   @Column('varchar', { length: 40 })
@@ -293,7 +266,7 @@ export class Subscription extends BaseEntity {
   id!: string
 
   @JoinColumn({ name: 'user_id' })
-  @ManyToOne(() => User, (user) => user.articles, { eager: true })
+  @ManyToOne(() => User, { eager: true })
   user!: User
 
   @Column('text')
@@ -335,7 +308,7 @@ export class LibraryItem extends BaseEntity {
   id!: string
 
   @JoinColumn({ name: 'user_id' })
-  @ManyToOne(() => User, (user) => user.articles, { eager: true })
+  @ManyToOne(() => User, { eager: true })
   user!: User
 
   @Column({ type: 'enum', enum: LibraryItemState })
@@ -382,7 +355,7 @@ export class UploadFile extends BaseEntity {
   id!: string
 
   @JoinColumn({ name: 'user_id' })
-  @ManyToOne(() => User, (user) => user.articles, { eager: true })
+  @ManyToOne(() => User, { eager: true })
   user!: User
 
   @Column('text')
@@ -410,15 +383,15 @@ export class Recommendation extends BaseEntity {
   id!: string
 
   @JoinColumn({ name: 'recommender_id' })
-  @ManyToOne(() => User, (user) => user.articles, { eager: true })
+  @ManyToOne(() => User, { eager: true })
   recommender!: User
 
   @JoinColumn({ name: 'library_item_id' })
-  @ManyToOne(() => User, (user) => user.articles, { eager: true })
+  @ManyToOne(() => User, { eager: true })
   libraryItem!: LibraryItem
 
   @JoinColumn({ name: 'group_id' })
-  @ManyToOne(() => User, (user) => user.articles, { eager: true })
+  @ManyToOne(() => User, { eager: true })
   group!: Group
 
   @Column('text', { nullable: true })
@@ -434,7 +407,7 @@ export class GroupMembership extends BaseEntity {
   id!: string
 
   @JoinColumn({ name: 'user_id' })
-  @ManyToOne(() => User, (user) => user.articles, { eager: true })
+  @ManyToOne(() => User, { eager: true })
   user!: User
 
   @JoinColumn({ name: 'group_id' })
@@ -456,7 +429,7 @@ export class Features extends BaseEntity {
   id!: string
 
   @JoinColumn({ name: 'user_id' })
-  @ManyToOne(() => User, (user) => user.articles, { eager: true })
+  @ManyToOne(() => User, { eager: true })
   user!: User
 
   @Column('text')
@@ -474,3 +447,93 @@ export class Features extends BaseEntity {
   @Column({ type: 'timestamp', name: 'updated_at' })
   updatedAt!: Date
 }
+
+@Entity({ name: 'newsletter_emails' })
+export class EmailAddress extends BaseEntity {
+  @PrimaryGeneratedColumn('uuid')
+  id!: string
+
+  @JoinColumn({ name: 'user_id' })
+  @ManyToOne(() => User)
+  user!: User
+
+  @Column('varchar')
+  address!: string
+
+  @Column('text')
+  folder?: string | null
+
+  @Column('text')
+  name?: string | null
+
+  @Column('text')
+  description?: string | null
+
+  @Column({ type: 'timestamp', name: 'created_at' })
+  createdAt!: Date
+
+  @Column({ type: 'timestamp', name: 'updated_at' })
+  updatedAt!: Date
+}
+
+@Entity({ name: 'rules' })
+export class Rule extends BaseEntity {
+  @PrimaryGeneratedColumn('uuid')
+  id!: string
+
+  @JoinColumn({ name: 'user_id' })
+  @ManyToOne(() => User)
+  user!: User
+
+  @Column('varchar')
+  name!: string
+
+  @Column('text')
+  filter!: string
+
+  @Column({ type: 'timestamp', name: 'created_at' })
+  createdAt!: Date
+
+  @Column({ type: 'timestamp', name: 'updated_at' })
+  updatedAt!: Date
+
+  @Column({ type: 'timestamp', name: 'failed_at' })
+  failedAt?: Date
+}
+
+// @Entity({ name: 'rules' })
+// export class Rule {
+//   @PrimaryGeneratedColumn('uuid')
+//   id!: string
+
+//   // @ManyToOne(() => User, { onDelete: 'CASCADE' })
+//   // @JoinColumn({ name: 'user_id' })
+//   // user!: User
+
+//   @Column('text')
+//   name!: string
+
+//   // @Column('text')
+//   // filter!: string
+
+//   // @Column('simple-json')
+//   // actions!: RuleAction[]
+
+//   @Column('text', { nullable: true })
+//   description?: string | null
+
+//   // @Column('text', { array: true })
+//   // eventTypes!: RuleEventType[]
+
+//   // @Column('boolean', { default: true })
+//   // enabled!: boolean
+
+//   // @Column({ type: 'timestamp', name: 'created_at' })
+//   // createdAt!: Date
+
+//   // @Column({ type: 'timestamp', name: 'updated_at' })
+//   // updatedAt!: Date
+
+//   // @Column({ type: 'timestamp', name: 'failed_at' })
+//   // failedAt?: Date
+// }
diff --git a/pkg/admin/src/index.ts b/pkg/admin/src/index.ts
index d6b5fcd07..662cb8180 100644
--- a/pkg/admin/src/index.ts
+++ b/pkg/admin/src/index.ts
@@ -5,7 +5,6 @@ import {
   registerDatabase,
   AdminUser,
   User,
-  UserArticle,
   UserProfile,
   ReceivedEmail,
   Group,
@@ -16,6 +15,8 @@ import {
   Recommendation,
   GroupMembership,
   Features,
+  EmailAddress,
+  Rule,
 } from './db'
 import { compare, hashSync } from 'bcryptjs'
 const readYamlFile = require('read-yaml-file')
@@ -33,6 +34,9 @@ const ADMIN_USER_EMAIL =
     databases: [db],
     rootPath: '/admin',
     resources: [
+      {
+        resource: ContentDisplayReport,
+      },
       {
         resource: User,
         options: {
@@ -40,18 +44,16 @@ const ADMIN_USER_EMAIL =
         },
       },
       { resource: UserProfile, options: { parent: { name: 'Users' } } },
-      { resource: UserArticle, options: { parent: { name: 'Users' } } },
       { resource: ReceivedEmail, options: { parent: { name: 'Users' } } },
       { resource: Group, options: { parent: { name: 'Users' } } },
       { resource: Subscription, options: { parent: { name: 'Users' } } },
       { resource: Integration, options: { parent: { name: 'Users' } } },
       { resource: LibraryItem, options: { parent: { name: 'Users' } } },
-      {
-        resource: ContentDisplayReport,
-      },
       { resource: Recommendation, options: { parent: { name: 'Users' } } },
       { resource: GroupMembership, options: { parent: { name: 'Users' } } },
       { resource: Features, options: { parent: { name: 'Users' } } },
+      { resource: EmailAddress, options: { parent: { name: 'Users' } } },
+      { resource: Rule, options: { parent: { name: 'Users' } } },
     ],
   })
 

From 5d6c2ba7f8ee12160a72404584a72e2ea261ed95 Mon Sep 17 00:00:00 2001
From: Jackson Harper <jacksonh@gmail.com>
Date: Tue, 9 Apr 2024 11:16:47 +0800
Subject: [PATCH 08/74] Remove commented out items

---
 pkg/admin/src/db.ts | 37 -------------------------------------
 1 file changed, 37 deletions(-)

diff --git a/pkg/admin/src/db.ts b/pkg/admin/src/db.ts
index eb4ce7dd9..78f55c4c7 100644
--- a/pkg/admin/src/db.ts
+++ b/pkg/admin/src/db.ts
@@ -500,40 +500,3 @@ export class Rule extends BaseEntity {
   @Column({ type: 'timestamp', name: 'failed_at' })
   failedAt?: Date
 }
-
-// @Entity({ name: 'rules' })
-// export class Rule {
-//   @PrimaryGeneratedColumn('uuid')
-//   id!: string
-
-//   // @ManyToOne(() => User, { onDelete: 'CASCADE' })
-//   // @JoinColumn({ name: 'user_id' })
-//   // user!: User
-
-//   @Column('text')
-//   name!: string
-
-//   // @Column('text')
-//   // filter!: string
-
-//   // @Column('simple-json')
-//   // actions!: RuleAction[]
-
-//   @Column('text', { nullable: true })
-//   description?: string | null
-
-//   // @Column('text', { array: true })
-//   // eventTypes!: RuleEventType[]
-
-//   // @Column('boolean', { default: true })
-//   // enabled!: boolean
-
-//   // @Column({ type: 'timestamp', name: 'created_at' })
-//   // createdAt!: Date
-
-//   // @Column({ type: 'timestamp', name: 'updated_at' })
-//   // updatedAt!: Date
-
-//   // @Column({ type: 'timestamp', name: 'failed_at' })
-//   // failedAt?: Date
-// }

From e7fc2c4567569e7cdcd51efd86d25ccd4273ccaa Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Tue, 9 Apr 2024 18:39:17 +0800
Subject: [PATCH 09/74] reset failed_at to null if a feed is refreshed
 successfully after failure

---
 packages/api/src/jobs/rss/refreshFeed.ts         | 2 +-
 packages/api/src/services/update_subscription.ts | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/packages/api/src/jobs/rss/refreshFeed.ts b/packages/api/src/jobs/rss/refreshFeed.ts
index 76078a8f4..8fec18c20 100644
--- a/packages/api/src/jobs/rss/refreshFeed.ts
+++ b/packages/api/src/jobs/rss/refreshFeed.ts
@@ -511,7 +511,7 @@ const processSubscription = async (
 
   // fetch feed
   let itemCount = 0,
-    failedAt: Date | undefined
+    failedAt: Date | null = null
 
   const feedLastBuildDate = feed.lastBuildDate
   logger.info(`Feed last build date ${feedLastBuildDate || 'N/A'}`)
diff --git a/packages/api/src/services/update_subscription.ts b/packages/api/src/services/update_subscription.ts
index 9d0970ba5..d333f58ba 100644
--- a/packages/api/src/services/update_subscription.ts
+++ b/packages/api/src/services/update_subscription.ts
@@ -49,7 +49,7 @@ export const updateSubscription = async (
     lastFetchedChecksum: newData.lastFetchedChecksum || undefined,
     status: newData.status || undefined,
     scheduledAt: newData.scheduledAt || undefined,
-    failedAt: newData.failedAt || undefined,
+    failedAt: newData.failedAt,
     autoAddToLibrary: newData.autoAddToLibrary ?? undefined,
     isPrivate: newData.isPrivate ?? undefined,
     fetchContentType: newData.fetchContentType || undefined,

From a59c850c5f6f1116e12cd166274547949610f11b Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Tue, 9 Apr 2024 19:44:16 +0800
Subject: [PATCH 10/74] fix telegram channel

---
 packages/api/src/jobs/rss/refreshFeed.ts | 34 +++++++++++++++++++-----
 1 file changed, 28 insertions(+), 6 deletions(-)

diff --git a/packages/api/src/jobs/rss/refreshFeed.ts b/packages/api/src/jobs/rss/refreshFeed.ts
index 8fec18c20..5afdab813 100644
--- a/packages/api/src/jobs/rss/refreshFeed.ts
+++ b/packages/api/src/jobs/rss/refreshFeed.ts
@@ -192,22 +192,40 @@ export const fetchAndChecksum = async (url: string) => {
 
 const parseFeed = async (url: string, content: string) => {
   try {
-    // check if url is a telegram channel
-    const telegramRegex = /https:\/\/t\.me\/([a-zA-Z0-9_]+)/
+    // check if url is a telegram channel or preview
+    const telegramRegex = /t\.me\/([^/]+)/
     const telegramMatch = url.match(telegramRegex)
     if (telegramMatch) {
+      let channel = telegramMatch[1]
+      if (channel.startsWith('s/')) {
+        channel = channel.slice(2)
+      } else {
+        // open the preview page to get the data
+        const fetchResult = await fetchAndChecksum(`https://t.me/s/${channel}`)
+        if (!fetchResult) {
+          return null
+        }
+
+        content = fetchResult.content
+      }
+
       const dom = parseHTML(content).document
-      const title = dom.querySelector('meta[property="og:title"]')
+      const title =
+        dom
+          .querySelector('meta[property="og:title"]')
+          ?.getAttribute('content') || dom.title
       // post has attribute data-post
       const posts = dom.querySelectorAll('[data-post]')
       const items = Array.from(posts)
         .map((post) => {
-          const id = post.getAttribute('data-post')
+          const id = post.getAttribute('data-post')?.split('/')[1]
           if (!id) {
             return null
           }
 
-          const url = `https://t.me/${telegramMatch[1]}/${id}`
+          const url = `https://t.me/s/${channel}/${id}`
+          const content = post.outerHTML
+
           // find the <time> element
           const time = post.querySelector('time')
           const dateTime = time?.getAttribute('datetime') || undefined
@@ -215,12 +233,16 @@ const parseFeed = async (url: string, content: string) => {
           return {
             link: url,
             isoDate: dateTime,
+            title: `${title} - ${id}`,
+            creator: title,
+            content,
+            links: [url],
           }
         })
         .filter((item) => !!item) as RssFeedItem[]
 
       return {
-        title: title?.getAttribute('content') || dom.title,
+        title,
         items,
       }
     }

From 5cc5c559ac80b2cdeca3c253261b2b6b2f5dcb1b Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Thu, 4 Apr 2024 15:53:25 +0800
Subject: [PATCH 11/74] add reply to email API which replies Okay to the sender

---
 packages/api/src/generated/graphql.ts         |  48 +++++
 packages/api/src/generated/schema.graphql     |  15 ++
 .../api/src/resolvers/function_resolvers.ts   |   8 +-
 .../api/src/resolvers/recent_emails/index.ts  | 204 ++++++++++--------
 packages/api/src/schema.ts                    |  15 ++
 5 files changed, 196 insertions(+), 94 deletions(-)

diff --git a/packages/api/src/generated/graphql.ts b/packages/api/src/generated/graphql.ts
index 93afb8908..1343a034a 100644
--- a/packages/api/src/generated/graphql.ts
+++ b/packages/api/src/generated/graphql.ts
@@ -1616,6 +1616,7 @@ export type Mutation = {
   optInFeature: OptInFeatureResult;
   recommend: RecommendResult;
   recommendHighlights: RecommendHighlightsResult;
+  replyToEmail: ReplyToEmailResult;
   reportItem: ReportItemResult;
   revokeApiKey: RevokeApiKeyResult;
   saveArticleReadingProgress: SaveArticleReadingProgressResult;
@@ -1836,6 +1837,12 @@ export type MutationRecommendHighlightsArgs = {
 };
 
 
+export type MutationReplyToEmailArgs = {
+  recentEmailId: Scalars['ID'];
+  reply: Scalars['String'];
+};
+
+
 export type MutationReportItemArgs = {
   input: ReportItemInput;
 };
@@ -2430,6 +2437,22 @@ export type ReminderSuccess = {
   reminder: Reminder;
 };
 
+export type ReplyToEmailError = {
+  __typename?: 'ReplyToEmailError';
+  errorCodes: Array<ReplyToEmailErrorCode>;
+};
+
+export enum ReplyToEmailErrorCode {
+  Unauthorized = 'UNAUTHORIZED'
+}
+
+export type ReplyToEmailResult = ReplyToEmailError | ReplyToEmailSuccess;
+
+export type ReplyToEmailSuccess = {
+  __typename?: 'ReplyToEmailSuccess';
+  success: Scalars['Boolean'];
+};
+
 export type ReportItemInput = {
   itemUrl: Scalars['String'];
   pageId: Scalars['ID'];
@@ -4245,6 +4268,10 @@ export type ResolversTypes = {
   ReminderErrorCode: ReminderErrorCode;
   ReminderResult: ResolversTypes['ReminderError'] | ResolversTypes['ReminderSuccess'];
   ReminderSuccess: ResolverTypeWrapper<ReminderSuccess>;
+  ReplyToEmailError: ResolverTypeWrapper<ReplyToEmailError>;
+  ReplyToEmailErrorCode: ReplyToEmailErrorCode;
+  ReplyToEmailResult: ResolversTypes['ReplyToEmailError'] | ResolversTypes['ReplyToEmailSuccess'];
+  ReplyToEmailSuccess: ResolverTypeWrapper<ReplyToEmailSuccess>;
   ReportItemInput: ReportItemInput;
   ReportItemResult: ResolverTypeWrapper<ReportItemResult>;
   ReportType: ReportType;
@@ -4763,6 +4790,9 @@ export type ResolversParentTypes = {
   ReminderError: ReminderError;
   ReminderResult: ResolversParentTypes['ReminderError'] | ResolversParentTypes['ReminderSuccess'];
   ReminderSuccess: ReminderSuccess;
+  ReplyToEmailError: ReplyToEmailError;
+  ReplyToEmailResult: ResolversParentTypes['ReplyToEmailError'] | ResolversParentTypes['ReplyToEmailSuccess'];
+  ReplyToEmailSuccess: ReplyToEmailSuccess;
   ReportItemInput: ReportItemInput;
   ReportItemResult: ReportItemResult;
   RevokeApiKeyError: RevokeApiKeyError;
@@ -6128,6 +6158,7 @@ export type MutationResolvers<ContextType = ResolverContext, ParentType extends
   optInFeature?: Resolver<ResolversTypes['OptInFeatureResult'], ParentType, ContextType, RequireFields<MutationOptInFeatureArgs, 'input'>>;
   recommend?: Resolver<ResolversTypes['RecommendResult'], ParentType, ContextType, RequireFields<MutationRecommendArgs, 'input'>>;
   recommendHighlights?: Resolver<ResolversTypes['RecommendHighlightsResult'], ParentType, ContextType, RequireFields<MutationRecommendHighlightsArgs, 'input'>>;
+  replyToEmail?: Resolver<ResolversTypes['ReplyToEmailResult'], ParentType, ContextType, RequireFields<MutationReplyToEmailArgs, 'recentEmailId' | 'reply'>>;
   reportItem?: Resolver<ResolversTypes['ReportItemResult'], ParentType, ContextType, RequireFields<MutationReportItemArgs, 'input'>>;
   revokeApiKey?: Resolver<ResolversTypes['RevokeApiKeyResult'], ParentType, ContextType, RequireFields<MutationRevokeApiKeyArgs, 'id'>>;
   saveArticleReadingProgress?: Resolver<ResolversTypes['SaveArticleReadingProgressResult'], ParentType, ContextType, RequireFields<MutationSaveArticleReadingProgressArgs, 'input'>>;
@@ -6417,6 +6448,20 @@ export type ReminderSuccessResolvers<ContextType = ResolverContext, ParentType e
   __isTypeOf?: IsTypeOfResolverFn<ParentType, ContextType>;
 };
 
+export type ReplyToEmailErrorResolvers<ContextType = ResolverContext, ParentType extends ResolversParentTypes['ReplyToEmailError'] = ResolversParentTypes['ReplyToEmailError']> = {
+  errorCodes?: Resolver<Array<ResolversTypes['ReplyToEmailErrorCode']>, ParentType, ContextType>;
+  __isTypeOf?: IsTypeOfResolverFn<ParentType, ContextType>;
+};
+
+export type ReplyToEmailResultResolvers<ContextType = ResolverContext, ParentType extends ResolversParentTypes['ReplyToEmailResult'] = ResolversParentTypes['ReplyToEmailResult']> = {
+  __resolveType: TypeResolveFn<'ReplyToEmailError' | 'ReplyToEmailSuccess', ParentType, ContextType>;
+};
+
+export type ReplyToEmailSuccessResolvers<ContextType = ResolverContext, ParentType extends ResolversParentTypes['ReplyToEmailSuccess'] = ResolversParentTypes['ReplyToEmailSuccess']> = {
+  success?: Resolver<ResolversTypes['Boolean'], ParentType, ContextType>;
+  __isTypeOf?: IsTypeOfResolverFn<ParentType, ContextType>;
+};
+
 export type ReportItemResultResolvers<ContextType = ResolverContext, ParentType extends ResolversParentTypes['ReportItemResult'] = ResolversParentTypes['ReportItemResult']> = {
   message?: Resolver<ResolversTypes['String'], ParentType, ContextType>;
   __isTypeOf?: IsTypeOfResolverFn<ParentType, ContextType>;
@@ -7488,6 +7533,9 @@ export type Resolvers<ContextType = ResolverContext> = {
   ReminderError?: ReminderErrorResolvers<ContextType>;
   ReminderResult?: ReminderResultResolvers<ContextType>;
   ReminderSuccess?: ReminderSuccessResolvers<ContextType>;
+  ReplyToEmailError?: ReplyToEmailErrorResolvers<ContextType>;
+  ReplyToEmailResult?: ReplyToEmailResultResolvers<ContextType>;
+  ReplyToEmailSuccess?: ReplyToEmailSuccessResolvers<ContextType>;
   ReportItemResult?: ReportItemResultResolvers<ContextType>;
   RevokeApiKeyError?: RevokeApiKeyErrorResolvers<ContextType>;
   RevokeApiKeyResult?: RevokeApiKeyResultResolvers<ContextType>;
diff --git a/packages/api/src/generated/schema.graphql b/packages/api/src/generated/schema.graphql
index ba442c3a7..90bd58980 100644
--- a/packages/api/src/generated/schema.graphql
+++ b/packages/api/src/generated/schema.graphql
@@ -1454,6 +1454,7 @@ type Mutation {
   optInFeature(input: OptInFeatureInput!): OptInFeatureResult!
   recommend(input: RecommendInput!): RecommendResult!
   recommendHighlights(input: RecommendHighlightsInput!): RecommendHighlightsResult!
+  replyToEmail(recentEmailId: ID!, reply: String!): ReplyToEmailResult!
   reportItem(input: ReportItemInput!): ReportItemResult!
   revokeApiKey(id: ID!): RevokeApiKeyResult!
   saveArticleReadingProgress(input: SaveArticleReadingProgressInput!): SaveArticleReadingProgressResult!
@@ -1811,6 +1812,20 @@ type ReminderSuccess {
   reminder: Reminder!
 }
 
+type ReplyToEmailError {
+  errorCodes: [ReplyToEmailErrorCode!]!
+}
+
+enum ReplyToEmailErrorCode {
+  UNAUTHORIZED
+}
+
+union ReplyToEmailResult = ReplyToEmailError | ReplyToEmailSuccess
+
+type ReplyToEmailSuccess {
+  success: Boolean!
+}
+
 input ReportItemInput {
   itemUrl: String!
   pageId: ID!
diff --git a/packages/api/src/resolvers/function_resolvers.ts b/packages/api/src/resolvers/function_resolvers.ts
index 9a8fcfa58..95bd3e219 100644
--- a/packages/api/src/resolvers/function_resolvers.ts
+++ b/packages/api/src/resolvers/function_resolvers.ts
@@ -150,7 +150,11 @@ import {
   webhookResolver,
   webhooksResolver,
 } from './index'
-import { markEmailAsItemResolver, recentEmailsResolver } from './recent_emails'
+import {
+  markEmailAsItemResolver,
+  recentEmailsResolver,
+  replyToEmailResolver,
+} from './recent_emails'
 import { recentSearchesResolver } from './recent_searches'
 import { WithDataSourcesContext } from './types'
 import { updateEmailResolver } from './user'
@@ -316,6 +320,7 @@ export const functionResolvers = {
     emptyTrash: emptyTrashResolver,
     fetchContent: fetchContentResolver,
     exportToIntegration: exportToIntegrationResolver,
+    replyToEmail: replyToEmailResolver,
   },
   Query: {
     me: getMeUserResolver,
@@ -680,4 +685,5 @@ export const functionResolvers = {
   ...resultResolveTypeResolver('FetchContent'),
   ...resultResolveTypeResolver('Integration'),
   ...resultResolveTypeResolver('ExportToIntegration'),
+  ...resultResolveTypeResolver('ReplyToEmail'),
 }
diff --git a/packages/api/src/resolvers/recent_emails/index.ts b/packages/api/src/resolvers/recent_emails/index.ts
index a4b86a929..0241b5424 100644
--- a/packages/api/src/resolvers/recent_emails/index.ts
+++ b/packages/api/src/resolvers/recent_emails/index.ts
@@ -7,10 +7,14 @@ import {
   MarkEmailAsItemErrorCode,
   MarkEmailAsItemSuccess,
   MutationMarkEmailAsItemArgs,
+  MutationReplyToEmailArgs,
   RecentEmailsError,
-  RecentEmailsErrorCode,
   RecentEmailsSuccess,
+  ReplyToEmailError,
+  ReplyToEmailErrorCode,
+  ReplyToEmailSuccess,
 } from '../../generated/graphql'
+import { getRepository } from '../../repository'
 import { updateReceivedEmail } from '../../services/received_emails'
 import { saveNewsletter } from '../../services/save_newsletter_email'
 import { authorized } from '../../utils/gql-utils'
@@ -20,27 +24,19 @@ import { sendEmail } from '../../utils/sendEmail'
 export const recentEmailsResolver = authorized<
   RecentEmailsSuccess,
   RecentEmailsError
->(async (_, __, { authTrx, log, uid }) => {
-  try {
-    const recentEmails = await authTrx((t) =>
-      t.getRepository(ReceivedEmail).find({
-        where: {
-          user: { id: uid },
-        },
-        order: { createdAt: 'DESC' },
-        take: 20,
-      })
-    )
+>(async (_, __, { authTrx, uid }) => {
+  const recentEmails = await authTrx((t) =>
+    t.getRepository(ReceivedEmail).find({
+      where: {
+        user: { id: uid },
+      },
+      order: { createdAt: 'DESC' },
+      take: 20,
+    })
+  )
 
-    return {
-      recentEmails,
-    }
-  } catch (error) {
-    log.error('Error getting recent emails', error)
-
-    return {
-      errorCodes: [RecentEmailsErrorCode.BadRequest],
-    }
+  return {
+    recentEmails,
   }
 })
 
@@ -49,87 +45,109 @@ export const markEmailAsItemResolver = authorized<
   MarkEmailAsItemError,
   MutationMarkEmailAsItemArgs
 >(async (_, { recentEmailId }, { authTrx, uid, log }) => {
-  try {
-    const recentEmail = await authTrx((t) =>
-      t.getRepository(ReceivedEmail).findOneBy({
-        id: recentEmailId,
+  const recentEmail = await authTrx((t) =>
+    t.getRepository(ReceivedEmail).findOneBy({
+      id: recentEmailId,
+      user: { id: uid },
+      type: 'non-article',
+    })
+  )
+  if (!recentEmail) {
+    log.info('no recent email', recentEmailId)
+
+    return {
+      errorCodes: [MarkEmailAsItemErrorCode.Unauthorized],
+    }
+  }
+
+  const newsletterEmail = await authTrx((t) =>
+    t.getRepository(NewsletterEmail).findOne({
+      where: {
         user: { id: uid },
-        type: 'non-article',
-      })
-    )
-    if (!recentEmail) {
-      log.info('no recent email', recentEmailId)
-
-      return {
-        errorCodes: [MarkEmailAsItemErrorCode.Unauthorized],
-      }
-    }
-
-    const newsletterEmail = await authTrx((t) =>
-      t.getRepository(NewsletterEmail).findOne({
-        where: {
-          user: { id: uid },
-          address: ILike(recentEmail.to),
-        },
-        relations: ['user'],
-      })
-    )
-    if (!newsletterEmail) {
-      log.info('no newsletter email for', {
-        id: recentEmail.id,
-        to: recentEmail.to,
-        from: recentEmail.from,
-      })
-
-      return {
-        errorCodes: [MarkEmailAsItemErrorCode.NotFound],
-      }
-    }
-
-    const success = await saveNewsletter(
-      {
-        from: recentEmail.from,
-        email: recentEmail.to,
-        title: recentEmail.subject,
-        content: recentEmail.html,
-        url: generateUniqueUrl(),
-        author: parseEmailAddress(recentEmail.from).name,
-        receivedEmailId: recentEmail.id,
+        address: ILike(recentEmail.to),
       },
-      newsletterEmail
-    )
-    if (!success) {
-      log.info('newsletter not created', recentEmail.id)
-
-      return {
-        errorCodes: [MarkEmailAsItemErrorCode.BadRequest],
-      }
-    }
-
-    // update received email type
-    await updateReceivedEmail(recentEmail.id, 'article', uid)
-
-    const text = `A recent email marked as a library item
-                    by: ${uid}
-                    from: ${recentEmail.from}
-                    subject: ${recentEmail.subject}`
-
-    // email us to let us know that an email failed to parse as an article
-    await sendEmail({
-      to: env.sender.feedback,
-      subject: 'A recent email marked as a library item',
-      text,
-      from: env.sender.message,
+      relations: ['user'],
+    })
+  )
+  if (!newsletterEmail) {
+    log.info('no newsletter email for', {
+      id: recentEmail.id,
+      to: recentEmail.to,
+      from: recentEmail.from,
     })
 
     return {
-      success,
+      errorCodes: [MarkEmailAsItemErrorCode.NotFound],
     }
-  } catch (error) {
-    log.error('Error marking email as item', error)
+  }
+
+  const success = await saveNewsletter(
+    {
+      from: recentEmail.from,
+      email: recentEmail.to,
+      title: recentEmail.subject,
+      content: recentEmail.html,
+      url: generateUniqueUrl(),
+      author: parseEmailAddress(recentEmail.from).name,
+      receivedEmailId: recentEmail.id,
+    },
+    newsletterEmail
+  )
+  if (!success) {
+    log.info('newsletter not created', recentEmail.id)
 
     return {
       errorCodes: [MarkEmailAsItemErrorCode.BadRequest],
     }
   }
+
+  // update received email type
+  await updateReceivedEmail(recentEmail.id, 'article', uid)
+
+  const text = `A recent email marked as a library item
+                    by: ${uid}
+                    from: ${recentEmail.from}
+                    subject: ${recentEmail.subject}`
+
+  // email us to let us know that an email failed to parse as an article
+  await sendEmail({
+    to: env.sender.feedback,
+    subject: 'A recent email marked as a library item',
+    text,
+    from: env.sender.message,
+  })
+
+  return {
+    success,
+  }
+})
+
+export const replyToEmailResolver = authorized<
+  ReplyToEmailSuccess,
+  ReplyToEmailError,
+  MutationReplyToEmailArgs
+>(async (_, { recentEmailId }, { uid, log }) => {
+  const recentEmail = await getRepository(ReceivedEmail).findOneBy({
+    id: recentEmailId,
+    user: { id: uid },
+  })
+
+  if (!recentEmail) {
+    log.info('no recent email', recentEmailId)
+
+    return {
+      errorCodes: [ReplyToEmailErrorCode.Unauthorized],
+    }
+  }
+
+  const result = await sendEmail({
+    to: recentEmail.from,
+    subject: 'Re: ' + recentEmail.subject,
+    text: 'Okay',
+    from: recentEmail.to,
+  })
+
+  return {
+    success: result,
+  }
 })
diff --git a/packages/api/src/schema.ts b/packages/api/src/schema.ts
index 6b0c3cb01..0c64a00e5 100755
--- a/packages/api/src/schema.ts
+++ b/packages/api/src/schema.ts
@@ -3066,6 +3066,20 @@ const schema = gql`
     FAILED_TO_CREATE_TASK
   }
 
+  union ReplyToEmailResult = ReplyToEmailSuccess | ReplyToEmailError
+
+  type ReplyToEmailSuccess {
+    success: Boolean!
+  }
+
+  type ReplyToEmailError {
+    errorCodes: [ReplyToEmailErrorCode!]!
+  }
+
+  enum ReplyToEmailErrorCode {
+    UNAUTHORIZED
+  }
+
   # Mutations
   type Mutation {
     googleLogin(input: GoogleLoginInput!): LoginResult!
@@ -3165,6 +3179,7 @@ const schema = gql`
       contentType: String!
     ): UploadImportFileResult!
     markEmailAsItem(recentEmailId: ID!): MarkEmailAsItemResult!
+    replyToEmail(recentEmailId: ID!, reply: String!): ReplyToEmailResult!
     bulkAction(
       query: String!
       action: BulkActionType!

From 289d750a057415ac17c898c19df4e03104f541c6 Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Mon, 8 Apr 2024 22:59:04 +0800
Subject: [PATCH 12/74] add columns for reply_to and reply and create tables
 for tracking service usage and enforce limit according to the subscription
 plan

---
 .../db/migrations/0172.do.service_usage.sql   | 37 +++++++++++++++++++
 .../db/migrations/0172.undo.service_usage.sql | 15 ++++++++
 2 files changed, 52 insertions(+)
 create mode 100755 packages/db/migrations/0172.do.service_usage.sql
 create mode 100755 packages/db/migrations/0172.undo.service_usage.sql

diff --git a/packages/db/migrations/0172.do.service_usage.sql b/packages/db/migrations/0172.do.service_usage.sql
new file mode 100755
index 000000000..f0beeb316
--- /dev/null
+++ b/packages/db/migrations/0172.do.service_usage.sql
@@ -0,0 +1,37 @@
+-- Type: DO
+-- Name: service_usage
+-- Description: Create table for tracking service usage and enforce limit
+
+BEGIN;
+
+ALTER TABLE omnivore.received_emails
+    ADD COLUMN reply_to TEXT,
+    ADD COLUMN reply TEXT;
+
+CREATE TABLE omnivore.subscription_plan (
+    id SERIAL PRIMARY KEY,
+    name VARCHAR(255) NOT NULL,
+    description TEXT,
+    max_emails_sent_per_day INT NOT NULL,
+    created_at timestamptz NOT NULL default current_timestamp
+);
+
+INSERT INTO omnivore.subscription_plan (id, name, description, max_emails_sent_per_day)
+VALUES (1, 'Basic', 'Basic plan', 3);
+
+CREATE TABLE omnivore.service_usage (
+	id uuid PRIMARY KEY DEFAULT uuid_generate_v1mc(),
+    user_id uuid NOT NULL REFERENCES omnivore.user,
+    action VARCHAR(255) NOT NULL,
+    created_at timestamptz NOT NULL default current_timestamp
+);
+
+CREATE INDEX ON omnivore.service_usage (user_id);
+
+CREATE POLICY create_service_usage on omnivore.service_usage
+    FOR INSERT TO omnivore_user
+    WITH CHECK (true);
+
+GRANT SELECT, INSERT ON omnivore.service_usage TO omnivore_user;
+
+COMMIT;
diff --git a/packages/db/migrations/0172.undo.service_usage.sql b/packages/db/migrations/0172.undo.service_usage.sql
new file mode 100755
index 000000000..99d324323
--- /dev/null
+++ b/packages/db/migrations/0172.undo.service_usage.sql
@@ -0,0 +1,15 @@
+-- Type: UNDO
+-- Name: service_usage
+-- Description: Create table for tracking service usage and enforce limit
+
+BEGIN;
+
+DROP TABLE IF EXISTS omnivore.service_usage;
+
+DROP TABLE IF EXISTS omnivore.subscription_plan;
+
+ATLER TABLE omnivore.received_emails
+    DROP COLUMN IF EXISTS reply_to,
+    DROP COLUMN IF EXISTS reply;
+
+COMMIT;

From 764b05fff4a98e32ba0f3d343f88b1f273878147 Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Mon, 8 Apr 2024 23:05:50 +0800
Subject: [PATCH 13/74] save reply-to in recent emails

---
 packages/api/src/entity/received_email.ts    | 6 ++++++
 packages/api/src/routers/svc/emails.ts       | 5 ++++-
 packages/api/src/schema.ts                   | 2 ++
 packages/api/src/services/received_emails.ts | 4 +++-
 packages/inbound-email-handler/src/index.ts  | 2 ++
 5 files changed, 17 insertions(+), 2 deletions(-)

diff --git a/packages/api/src/entity/received_email.ts b/packages/api/src/entity/received_email.ts
index b68289989..cb5e5937c 100644
--- a/packages/api/src/entity/received_email.ts
+++ b/packages/api/src/entity/received_email.ts
@@ -33,6 +33,12 @@ export class ReceivedEmail {
   @Column('text')
   html!: string
 
+  @Column('text')
+  replyTo?: string
+
+  @Column('text')
+  reply?: string
+
   @Column('text')
   type!: 'article' | 'non-article'
 
diff --git a/packages/api/src/routers/svc/emails.ts b/packages/api/src/routers/svc/emails.ts
index 3d5075cd1..eda01a541 100644
--- a/packages/api/src/routers/svc/emails.ts
+++ b/packages/api/src/routers/svc/emails.ts
@@ -27,6 +27,7 @@ interface EmailMessage {
   text: string
   forwardedFrom?: string
   receivedEmailId: string
+  replyTo?: string
 }
 
 function isEmailMessage(data: any): data is EmailMessage {
@@ -165,7 +166,9 @@ export function emailsServiceRouter() {
         req.body.subject,
         req.body.text,
         req.body.html,
-        user.id
+        user.id,
+        'non-article',
+        req.body.replyTo
       )
 
       analytics.capture({
diff --git a/packages/api/src/schema.ts b/packages/api/src/schema.ts
index 0c64a00e5..0d1f3d501 100755
--- a/packages/api/src/schema.ts
+++ b/packages/api/src/schema.ts
@@ -2547,6 +2547,8 @@ const schema = gql`
     type: String!
     text: String!
     html: String
+    replyTo: String
+    reply: String
     createdAt: Date!
   }
 
diff --git a/packages/api/src/services/received_emails.ts b/packages/api/src/services/received_emails.ts
index b1ea031af..003c82d9d 100644
--- a/packages/api/src/services/received_emails.ts
+++ b/packages/api/src/services/received_emails.ts
@@ -8,7 +8,8 @@ export const saveReceivedEmail = async (
   text: string,
   html: string,
   userId: string,
-  type: 'article' | 'non-article' = 'non-article'
+  type: 'article' | 'non-article' = 'non-article',
+  replyTo?: string
 ): Promise<ReceivedEmail> => {
   return authTrx(
     (t) =>
@@ -20,6 +21,7 @@ export const saveReceivedEmail = async (
         html,
         type,
         user: { id: userId },
+        replyTo,
       }),
     undefined,
     userId
diff --git a/packages/inbound-email-handler/src/index.ts b/packages/inbound-email-handler/src/index.ts
index eefadb6a3..391244e35 100644
--- a/packages/inbound-email-handler/src/index.ts
+++ b/packages/inbound-email-handler/src/index.ts
@@ -120,6 +120,7 @@ export const inboundEmailHandler = Sentry.GCPFunction.wrapHttpFunction(
 
       // original sender email address
       const from = parsed['from']
+      const replyTo = parsed['reply-to']
       const subject = parsed['subject']
       const html = parsed['html']
       const text = parsed['text']
@@ -140,6 +141,7 @@ export const inboundEmailHandler = Sentry.GCPFunction.wrapHttpFunction(
         subject,
         html,
         text,
+        replyTo,
       })
 
       try {

From dce0b15f4fd570937f21716e85846b87d57358e5 Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Mon, 8 Apr 2024 23:09:23 +0800
Subject: [PATCH 14/74] send to the reply-to address if it exists or the from
 address

---
 packages/api/src/resolvers/recent_emails/index.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/api/src/resolvers/recent_emails/index.ts b/packages/api/src/resolvers/recent_emails/index.ts
index 0241b5424..e7d3330fa 100644
--- a/packages/api/src/resolvers/recent_emails/index.ts
+++ b/packages/api/src/resolvers/recent_emails/index.ts
@@ -141,7 +141,7 @@ export const replyToEmailResolver = authorized<
   }
 
   const result = await sendEmail({
-    to: recentEmail.from,
+    to: recentEmail.replyTo || recentEmail.from, // send to the reply-to address if it exists or the from address
     subject: 'Re: ' + recentEmail.subject,
     text: 'Okay',
     from: recentEmail.to,

From 74f5b922dd989f6759a914eba2aa675eb86dd9df Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Mon, 8 Apr 2024 23:11:57 +0800
Subject: [PATCH 15/74] update received email reply

---
 packages/api/src/resolvers/recent_emails/index.ts | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/packages/api/src/resolvers/recent_emails/index.ts b/packages/api/src/resolvers/recent_emails/index.ts
index e7d3330fa..a21cd496a 100644
--- a/packages/api/src/resolvers/recent_emails/index.ts
+++ b/packages/api/src/resolvers/recent_emails/index.ts
@@ -127,7 +127,8 @@ export const replyToEmailResolver = authorized<
   ReplyToEmailError,
   MutationReplyToEmailArgs
 >(async (_, { recentEmailId }, { uid, log }) => {
-  const recentEmail = await getRepository(ReceivedEmail).findOneBy({
+  const repo = getRepository(ReceivedEmail)
+  const recentEmail = await repo.findOneBy({
     id: recentEmailId,
     user: { id: uid },
   })
@@ -140,13 +141,18 @@ export const replyToEmailResolver = authorized<
     }
   }
 
+  const reply = 'Okay'
+
   const result = await sendEmail({
     to: recentEmail.replyTo || recentEmail.from, // send to the reply-to address if it exists or the from address
     subject: 'Re: ' + recentEmail.subject,
-    text: 'Okay',
+    text: reply,
     from: recentEmail.to,
   })
 
+  // update received email reply
+  await repo.update(recentEmailId, { reply })
+
   return {
     success: result,
   }

From e7fc1316ddf577dba7212f08828f837330353425 Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Tue, 9 Apr 2024 11:09:02 +0800
Subject: [PATCH 16/74] enqueue reply email job

---
 packages/api/src/jobs/send_email.ts               | 8 ++++----
 packages/api/src/resolvers/recent_emails/index.ts | 5 +++--
 packages/api/src/services/send_emails.ts          | 6 +++---
 3 files changed, 10 insertions(+), 9 deletions(-)

diff --git a/packages/api/src/jobs/send_email.ts b/packages/api/src/jobs/send_email.ts
index 8831fd970..1c94174f1 100644
--- a/packages/api/src/jobs/send_email.ts
+++ b/packages/api/src/jobs/send_email.ts
@@ -9,7 +9,8 @@ export const SEND_EMAIL_JOB = 'send-email'
 type ContentType = { html: string } | { text: string } | { templateId: string }
 export type SendEmailJobData = Merge<
   {
-    emailAddress: string
+    to: string
+    from?: string
     subject?: string
     html?: string
     text?: string
@@ -21,7 +22,7 @@ export type SendEmailJobData = Merge<
 
 export const sendEmailJob = async (data: SendEmailJobData) => {
   if (process.env.USE_MAILJET && data.dynamicTemplateData) {
-    return sendWithMailJet(data.emailAddress, data.dynamicTemplateData.link)
+    return sendWithMailJet(data.to, data.dynamicTemplateData.link)
   }
 
   if (!data.html && !data.text && !data.templateId) {
@@ -31,7 +32,6 @@ export const sendEmailJob = async (data: SendEmailJobData) => {
 
   return sendEmail({
     ...data,
-    from: env.sender.message,
-    to: data.emailAddress,
+    from: data.from || env.sender.message,
   })
 }
diff --git a/packages/api/src/resolvers/recent_emails/index.ts b/packages/api/src/resolvers/recent_emails/index.ts
index a21cd496a..30f69e979 100644
--- a/packages/api/src/resolvers/recent_emails/index.ts
+++ b/packages/api/src/resolvers/recent_emails/index.ts
@@ -17,6 +17,7 @@ import {
 import { getRepository } from '../../repository'
 import { updateReceivedEmail } from '../../services/received_emails'
 import { saveNewsletter } from '../../services/save_newsletter_email'
+import { enqueueSendEmail } from '../../utils/createTask'
 import { authorized } from '../../utils/gql-utils'
 import { generateUniqueUrl, parseEmailAddress } from '../../utils/parser'
 import { sendEmail } from '../../utils/sendEmail'
@@ -143,7 +144,7 @@ export const replyToEmailResolver = authorized<
 
   const reply = 'Okay'
 
-  const result = await sendEmail({
+  const result = await enqueueSendEmail({
     to: recentEmail.replyTo || recentEmail.from, // send to the reply-to address if it exists or the from address
     subject: 'Re: ' + recentEmail.subject,
     text: reply,
@@ -154,6 +155,6 @@ export const replyToEmailResolver = authorized<
   await repo.update(recentEmailId, { reply })
 
   return {
-    success: result,
+    success: !!result,
   }
 })
diff --git a/packages/api/src/services/send_emails.ts b/packages/api/src/services/send_emails.ts
index 70dfb019f..69a537bdb 100644
--- a/packages/api/src/services/send_emails.ts
+++ b/packages/api/src/services/send_emails.ts
@@ -18,7 +18,7 @@ export const sendNewAccountVerificationEmail = async (user: {
   }
 
   const result = await enqueueSendEmail({
-    emailAddress: user.email,
+    to: user.email,
     dynamicTemplateData: dynamicTemplateData,
     templateId: env.sendgrid.confirmationTemplateId,
   })
@@ -78,7 +78,7 @@ export const sendAccountChangeEmail = async (user: {
   }
 
   const result = await enqueueSendEmail({
-    emailAddress: user.email,
+    to: user.email,
     dynamicTemplateData: dynamicTemplateData,
     templateId: env.sendgrid.verificationTemplateId,
   })
@@ -100,7 +100,7 @@ export const sendPasswordResetEmail = async (user: {
   }
 
   const result = await enqueueSendEmail({
-    emailAddress: user.email,
+    to: user.email,
     dynamicTemplateData: dynamicTemplateData,
     templateId: env.sendgrid.resetPasswordTemplateId,
   })

From f2f18eb12b5d04e8449797c76997836fb5371e4e Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Tue, 9 Apr 2024 11:16:20 +0800
Subject: [PATCH 17/74] reply only allowed text

---
 packages/api/src/generated/graphql.ts             | 14 +++++++++++++-
 packages/api/src/generated/schema.graphql         | 11 ++++++++++-
 packages/api/src/resolvers/recent_emails/index.ts | 14 ++++++++------
 packages/api/src/schema.ts                        |  9 ++++++++-
 4 files changed, 39 insertions(+), 9 deletions(-)

diff --git a/packages/api/src/generated/graphql.ts b/packages/api/src/generated/graphql.ts
index 1343a034a..cd4b67b31 100644
--- a/packages/api/src/generated/graphql.ts
+++ b/packages/api/src/generated/graphql.ts
@@ -58,6 +58,13 @@ export type AddPopularReadSuccess = {
   pageId: Scalars['String'];
 };
 
+export enum AllowedReply {
+  Confirm = 'CONFIRM',
+  Okay = 'OKAY',
+  Subscribe = 'SUBSCRIBE',
+  Yes = 'YES'
+}
+
 export type ApiKey = {
   __typename?: 'ApiKey';
   createdAt: Scalars['Date'];
@@ -1839,7 +1846,7 @@ export type MutationRecommendHighlightsArgs = {
 
 export type MutationReplyToEmailArgs = {
   recentEmailId: Scalars['ID'];
-  reply: Scalars['String'];
+  reply: AllowedReply;
 };
 
 
@@ -2282,6 +2289,8 @@ export type RecentEmail = {
   from: Scalars['String'];
   html?: Maybe<Scalars['String']>;
   id: Scalars['ID'];
+  reply?: Maybe<Scalars['String']>;
+  replyTo?: Maybe<Scalars['String']>;
   subject: Scalars['String'];
   text: Scalars['String'];
   to: Scalars['String'];
@@ -3931,6 +3940,7 @@ export type ResolversTypes = {
   AddPopularReadErrorCode: AddPopularReadErrorCode;
   AddPopularReadResult: ResolversTypes['AddPopularReadError'] | ResolversTypes['AddPopularReadSuccess'];
   AddPopularReadSuccess: ResolverTypeWrapper<AddPopularReadSuccess>;
+  AllowedReply: AllowedReply;
   ApiKey: ResolverTypeWrapper<ApiKey>;
   ApiKeysError: ResolverTypeWrapper<ApiKeysError>;
   ApiKeysErrorCode: ApiKeysErrorCode;
@@ -6324,6 +6334,8 @@ export type RecentEmailResolvers<ContextType = ResolverContext, ParentType exten
   from?: Resolver<ResolversTypes['String'], ParentType, ContextType>;
   html?: Resolver<Maybe<ResolversTypes['String']>, ParentType, ContextType>;
   id?: Resolver<ResolversTypes['ID'], ParentType, ContextType>;
+  reply?: Resolver<Maybe<ResolversTypes['String']>, ParentType, ContextType>;
+  replyTo?: Resolver<Maybe<ResolversTypes['String']>, ParentType, ContextType>;
   subject?: Resolver<ResolversTypes['String'], ParentType, ContextType>;
   text?: Resolver<ResolversTypes['String'], ParentType, ContextType>;
   to?: Resolver<ResolversTypes['String'], ParentType, ContextType>;
diff --git a/packages/api/src/generated/schema.graphql b/packages/api/src/generated/schema.graphql
index 90bd58980..e48fe3cff 100644
--- a/packages/api/src/generated/schema.graphql
+++ b/packages/api/src/generated/schema.graphql
@@ -37,6 +37,13 @@ type AddPopularReadSuccess {
   pageId: String!
 }
 
+enum AllowedReply {
+  CONFIRM
+  OKAY
+  SUBSCRIBE
+  YES
+}
+
 type ApiKey {
   createdAt: Date!
   expiresAt: Date!
@@ -1454,7 +1461,7 @@ type Mutation {
   optInFeature(input: OptInFeatureInput!): OptInFeatureResult!
   recommend(input: RecommendInput!): RecommendResult!
   recommendHighlights(input: RecommendHighlightsInput!): RecommendHighlightsResult!
-  replyToEmail(recentEmailId: ID!, reply: String!): ReplyToEmailResult!
+  replyToEmail(recentEmailId: ID!, reply: AllowedReply!): ReplyToEmailResult!
   reportItem(input: ReportItemInput!): ReportItemResult!
   revokeApiKey(id: ID!): RevokeApiKeyResult!
   saveArticleReadingProgress(input: SaveArticleReadingProgressInput!): SaveArticleReadingProgressResult!
@@ -1672,6 +1679,8 @@ type RecentEmail {
   from: String!
   html: String
   id: ID!
+  reply: String
+  replyTo: String
   subject: String!
   text: String!
   to: String!
diff --git a/packages/api/src/resolvers/recent_emails/index.ts b/packages/api/src/resolvers/recent_emails/index.ts
index 30f69e979..84aca707a 100644
--- a/packages/api/src/resolvers/recent_emails/index.ts
+++ b/packages/api/src/resolvers/recent_emails/index.ts
@@ -127,7 +127,7 @@ export const replyToEmailResolver = authorized<
   ReplyToEmailSuccess,
   ReplyToEmailError,
   MutationReplyToEmailArgs
->(async (_, { recentEmailId }, { uid, log }) => {
+>(async (_, { recentEmailId, reply }, { uid, log }) => {
   const repo = getRepository(ReceivedEmail)
   const recentEmail = await repo.findOneBy({
     id: recentEmailId,
@@ -142,8 +142,6 @@ export const replyToEmailResolver = authorized<
     }
   }
 
-  const reply = 'Okay'
-
   const result = await enqueueSendEmail({
     to: recentEmail.replyTo || recentEmail.from, // send to the reply-to address if it exists or the from address
     subject: 'Re: ' + recentEmail.subject,
@@ -151,10 +149,14 @@ export const replyToEmailResolver = authorized<
     from: recentEmail.to,
   })
 
-  // update received email reply
-  await repo.update(recentEmailId, { reply })
+  const success = !!result
+
+  if (success) {
+    // update received email reply
+    await repo.update(recentEmailId, { reply })
+  }
 
   return {
-    success: !!result,
+    success,
   }
 })
diff --git a/packages/api/src/schema.ts b/packages/api/src/schema.ts
index 0d1f3d501..acae6f520 100755
--- a/packages/api/src/schema.ts
+++ b/packages/api/src/schema.ts
@@ -3082,6 +3082,13 @@ const schema = gql`
     UNAUTHORIZED
   }
 
+  enum AllowedReply {
+    YES
+    OKAY
+    CONFIRM
+    SUBSCRIBE
+  }
+
   # Mutations
   type Mutation {
     googleLogin(input: GoogleLoginInput!): LoginResult!
@@ -3181,7 +3188,7 @@ const schema = gql`
       contentType: String!
     ): UploadImportFileResult!
     markEmailAsItem(recentEmailId: ID!): MarkEmailAsItemResult!
-    replyToEmail(recentEmailId: ID!, reply: String!): ReplyToEmailResult!
+    replyToEmail(recentEmailId: ID!, reply: AllowedReply!): ReplyToEmailResult!
     bulkAction(
       query: String!
       action: BulkActionType!

From 5f239d2dcbb10dcfdf2baa4230bfe23e54ecff27 Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Tue, 9 Apr 2024 16:49:03 +0800
Subject: [PATCH 18/74] update graceful shutdown

---
 packages/api/src/apollo.ts           | 14 ++++++++++-
 packages/api/src/queue-processor.ts  | 11 +++++++++
 packages/api/src/server.ts           | 36 ++++++++++------------------
 packages/api/test/global-teardown.ts | 12 +++++-----
 packages/api/test/util.ts            |  4 +++-
 5 files changed, 46 insertions(+), 31 deletions(-)

diff --git a/packages/api/src/apollo.ts b/packages/api/src/apollo.ts
index 2559197ba..38420daa1 100644
--- a/packages/api/src/apollo.ts
+++ b/packages/api/src/apollo.ts
@@ -28,6 +28,7 @@ import { SetClaimsRole } from './utils/dictionary'
 import { logger } from './utils/logger'
 import { ReadingProgressDataSource } from './datasources/reading_progress_data_source'
 import { createPrometheusExporterPlugin } from '@bmatei/apollo-prometheus-exporter'
+import { ApolloServerPlugin } from 'apollo-server-plugin-base'
 
 const signToken = promisify(jwt.sign)
 const pubsub = createPubSubClient()
@@ -112,10 +113,20 @@ export function makeApolloServer(app: Express): ApolloServer {
     },
   })
 
+  // enforce usage limits for the API
+  const usageLimitPlugin = (): ApolloServerPlugin<RequestContext> => {
+    return {
+      async requestDidStart(contextValue) {
+        // get graphql query from the request
+        console.log(contextValue)
+      },
+    }
+  }
+
   const apollo = new ApolloServer({
     schema: schema,
     context: contextFunc,
-    plugins: [promExporter],
+    plugins: [promExporter, usageLimitPlugin],
     formatError: (err) => {
       logger.info('server error', err)
       Sentry.captureException(err)
@@ -124,6 +135,7 @@ export function makeApolloServer(app: Express): ApolloServer {
     },
     introspection: env.dev.isLocal,
     persistedQueries: false,
+    stopOnTerminationSignals: false, // we handle this ourselves
   })
 
   return apollo
diff --git a/packages/api/src/queue-processor.ts b/packages/api/src/queue-processor.ts
index aab69d3dc..d13e934de 100644
--- a/packages/api/src/queue-processor.ts
+++ b/packages/api/src/queue-processor.ts
@@ -304,8 +304,19 @@ const main = async () => {
 
   const gracefulShutdown = async (signal: string) => {
     console.log(`[queue-processor]: Received ${signal}, closing server...`)
+    await new Promise<void>((resolve) => {
+      server.close((err) => {
+        console.log('[queue-processor]: Express server closed')
+        if (err) {
+          console.log('[queue-processor]: error stopping server', { err })
+        }
+
+        resolve()
+      })
+    })
     await worker.close()
     await redisDataSource.shutdown()
+    await appDataSource.destroy()
     process.exit(0)
   }
 
diff --git a/packages/api/src/server.ts b/packages/api/src/server.ts
index 473df7a79..2c945581c 100755
--- a/packages/api/src/server.ts
+++ b/packages/api/src/server.ts
@@ -47,11 +47,7 @@ import { apiLimiter, authLimiter } from './utils/rate_limit'
 
 const PORT = process.env.PORT || 4000
 
-export const createApp = (): {
-  app: Express
-  apollo: ApolloServer
-  httpServer: Server
-} => {
+export const createApp = (): Express => {
   const app = express()
 
   // eslint-disable-next-line @typescript-eslint/ban-ts-comment
@@ -136,10 +132,7 @@ export const createApp = (): {
     res.end(await prom.register.metrics())
   })
 
-  const apollo = makeApolloServer(app)
-  const httpServer = createServer(app)
-
-  return { app, apollo, httpServer }
+  return app
 }
 
 const main = async (): Promise<void> => {
@@ -154,19 +147,17 @@ const main = async (): Promise<void> => {
     await redisDataSource.initialize()
   }
 
-  const { app, apollo, httpServer } = createApp()
-
+  const app = createApp()
+  const apollo = makeApolloServer(app)
   await apollo.start()
   apollo.applyMiddleware({ app, path: '/api/graphql', cors: corsConfig })
 
-  if (!env.dev.isLocal) {
-    const mwLogger = loggers.get('express', { levels: config.syslog.levels })
-    const transport = buildLoggerTransport('express')
-    const mw = await lw.express.makeMiddleware(mwLogger, transport)
-    app.use(mw)
-  }
+  const mwLogger = loggers.get('express', { levels: config.syslog.levels })
+  const transport = buildLoggerTransport('express')
+  const mw = await lw.express.makeMiddleware(mwLogger, transport)
+  app.use(mw)
 
-  const listener = httpServer.listen({ port: PORT }, async () => {
+  const listener = app.listen({ port: PORT }, async () => {
     const logger = buildLogger('app.dispatch')
     logger.notice(`🚀 Server ready at ${apollo.graphqlPath}`)
   })
@@ -181,15 +172,14 @@ const main = async (): Promise<void> => {
   listener.timeout = 640 * 1000 // match headersTimeout
 
   const gracefulShutdown = async (signal: string) => {
+    console.log(`[api]: Received ${signal}, closing server...`)
+    await apollo.stop()
+    console.log('[api]: Apollo server stopped')
+
     console.log('[posthog]: flushing events')
     await analytics.shutdownAsync()
     console.log('[posthog]: events flushed')
 
-    console.log(`[api]: Received ${signal}, closing server...`)
-
-    await apollo.stop()
-    console.log('[api]: Apollo server stopped')
-
     await new Promise<void>((resolve) => {
       listener.close((err) => {
         console.log('[api]: Express listener closed')
diff --git a/packages/api/test/global-teardown.ts b/packages/api/test/global-teardown.ts
index 922d5db46..383fd6e36 100644
--- a/packages/api/test/global-teardown.ts
+++ b/packages/api/test/global-teardown.ts
@@ -7,16 +7,16 @@ export const mochaGlobalTeardown = async () => {
   await stopApolloServer()
   console.log('apollo server stopped')
 
-  await appDataSource.destroy()
-  console.log('db connection closed')
-
   if (env.redis.cache.url) {
-    await redisDataSource.shutdown()
-    console.log('redis connection closed')
-
     if (redisDataSource.workerRedisClient) {
       await stopWorker()
       console.log('worker closed')
     }
+
+    await redisDataSource.shutdown()
+    console.log('redis connection closed')
   }
+
+  await appDataSource.destroy()
+  console.log('db connection closed')
 }
diff --git a/packages/api/test/util.ts b/packages/api/test/util.ts
index 6717fdbe1..ee1c30bdc 100644
--- a/packages/api/test/util.ts
+++ b/packages/api/test/util.ts
@@ -2,11 +2,13 @@ import { ConnectionOptions, Job, QueueEvents, Worker } from 'bullmq'
 import { nanoid } from 'nanoid'
 import supertest from 'supertest'
 import { v4 } from 'uuid'
+import { makeApolloServer } from '../src/apollo'
 import { createWorker, QUEUE_NAME } from '../src/queue-processor'
 import { createApp } from '../src/server'
 import { corsConfig } from '../src/utils/corsConfig'
 
-const { app, apollo } = createApp()
+const app = createApp()
+const apollo = makeApolloServer(app)
 export const request = supertest(app)
 let worker: Worker
 let queueEvents: QueueEvents

From cd6f3e6bbe2496dbf0f2e0ae1f5e2625402c16b3 Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Tue, 9 Apr 2024 18:13:38 +0800
Subject: [PATCH 19/74] enforce usage limits for the API

---
 packages/api/src/apollo.ts                    | 18 ++++++++++++-
 packages/api/src/entity/service_usage.ts      | 25 +++++++++++++++++++
 packages/api/src/services/service_usage.ts    | 22 ++++++++++++++++
 .../db/migrations/0172.do.service_usage.sql   |  8 +++---
 4 files changed, 69 insertions(+), 4 deletions(-)
 create mode 100644 packages/api/src/entity/service_usage.ts
 create mode 100644 packages/api/src/services/service_usage.ts

diff --git a/packages/api/src/apollo.ts b/packages/api/src/apollo.ts
index 38420daa1..0326ec95a 100644
--- a/packages/api/src/apollo.ts
+++ b/packages/api/src/apollo.ts
@@ -29,6 +29,7 @@ import { logger } from './utils/logger'
 import { ReadingProgressDataSource } from './datasources/reading_progress_data_source'
 import { createPrometheusExporterPlugin } from '@bmatei/apollo-prometheus-exporter'
 import { ApolloServerPlugin } from 'apollo-server-plugin-base'
+import { countDailyServiceUsage } from './services/service_usage'
 
 const signToken = promisify(jwt.sign)
 const pubsub = createPubSubClient()
@@ -115,10 +116,25 @@ export function makeApolloServer(app: Express): ApolloServer {
 
   // enforce usage limits for the API
   const usageLimitPlugin = (): ApolloServerPlugin<RequestContext> => {
+    // TODO: load the limit from the DB into memory when the server starts
+    // hardcode the limit for now
+    const MAX_SENT_EMAIL_PER_DAY = 3
+
     return {
       async requestDidStart(contextValue) {
         // get graphql query from the request
-        console.log(contextValue)
+        const query = contextValue.request.query
+        // get the user id from the claims
+        const userId = contextValue.context.claims?.uid
+        const action = 'replyToEmail'
+        if (userId && query?.includes(action)) {
+          // get the user's email sent count from the DB
+          const emailSentCount = await countDailyServiceUsage(userId, action)
+          if (emailSentCount >= MAX_SENT_EMAIL_PER_DAY) {
+            // if the user has reached the limit, throw an error
+            throw new Error('You have reached the daily email limit')
+          }
+        }
       },
     }
   }
diff --git a/packages/api/src/entity/service_usage.ts b/packages/api/src/entity/service_usage.ts
new file mode 100644
index 000000000..056a51fc0
--- /dev/null
+++ b/packages/api/src/entity/service_usage.ts
@@ -0,0 +1,25 @@
+import {
+  Column,
+  CreateDateColumn,
+  Entity,
+  JoinColumn,
+  ManyToOne,
+  PrimaryGeneratedColumn,
+} from 'typeorm'
+import { User } from './user'
+
+@Entity('service_usage')
+export class ServiceUsage {
+  @PrimaryGeneratedColumn('uuid')
+  id!: string
+
+  @ManyToOne(() => User)
+  @JoinColumn({ name: 'user_id' })
+  user!: User
+
+  @Column('varchar')
+  action!: string
+
+  @CreateDateColumn({ default: () => 'CURRENT_TIMESTAMP' })
+  createdAt!: Date
+}
diff --git a/packages/api/src/services/service_usage.ts b/packages/api/src/services/service_usage.ts
new file mode 100644
index 000000000..9d4ada1d8
--- /dev/null
+++ b/packages/api/src/services/service_usage.ts
@@ -0,0 +1,22 @@
+import { Between } from 'typeorm'
+import { ServiceUsage } from '../entity/service_usage'
+import { authTrx, getRepository } from '../repository'
+import { DateTime } from 'luxon'
+
+const repo = getRepository(ServiceUsage)
+
+export const countDailyServiceUsage = async (
+  userId: string,
+  action: string
+) => {
+  return authTrx((tx) =>
+    tx.withRepository(repo).countBy({
+      user: { id: userId },
+      action,
+      createdAt: Between(
+        DateTime.now().startOf('day').toJSDate(),
+        DateTime.now().endOf('day').toJSDate()
+      ),
+    })
+  )
+}
diff --git a/packages/db/migrations/0172.do.service_usage.sql b/packages/db/migrations/0172.do.service_usage.sql
index f0beeb316..fa02e6214 100755
--- a/packages/db/migrations/0172.do.service_usage.sql
+++ b/packages/db/migrations/0172.do.service_usage.sql
@@ -28,9 +28,11 @@ CREATE TABLE omnivore.service_usage (
 
 CREATE INDEX ON omnivore.service_usage (user_id);
 
-CREATE POLICY create_service_usage on omnivore.service_usage
-    FOR INSERT TO omnivore_user
-    WITH CHECK (true);
+ALTER TABLE omnivore.service_usage ENABLE ROW LEVEL SECURITY;
+
+CREATE POLICY service_usage_policy on omnivore.service_usage
+    USING (user_id = omnivore.get_current_user_id())
+    WITH CHECK (user_id = omnivore.get_current_user_id());
 
 GRANT SELECT, INSERT ON omnivore.service_usage TO omnivore_user;
 

From 67657773cf4c82e083aad58050ad594fbb54db2c Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Tue, 9 Apr 2024 18:26:03 +0800
Subject: [PATCH 20/74] track the usage of the api

---
 packages/api/src/apollo.ts                 | 20 +++++++++++++++++++-
 packages/api/src/services/service_usage.ts |  9 +++++++++
 2 files changed, 28 insertions(+), 1 deletion(-)

diff --git a/packages/api/src/apollo.ts b/packages/api/src/apollo.ts
index 0326ec95a..b2c748921 100644
--- a/packages/api/src/apollo.ts
+++ b/packages/api/src/apollo.ts
@@ -29,7 +29,10 @@ import { logger } from './utils/logger'
 import { ReadingProgressDataSource } from './datasources/reading_progress_data_source'
 import { createPrometheusExporterPlugin } from '@bmatei/apollo-prometheus-exporter'
 import { ApolloServerPlugin } from 'apollo-server-plugin-base'
-import { countDailyServiceUsage } from './services/service_usage'
+import {
+  countDailyServiceUsage,
+  createServiceUsage,
+} from './services/service_usage'
 
 const signToken = promisify(jwt.sign)
 const pubsub = createPubSubClient()
@@ -135,6 +138,21 @@ export function makeApolloServer(app: Express): ApolloServer {
             throw new Error('You have reached the daily email limit')
           }
         }
+
+        return {
+          // track usage of the API
+          async willSendResponse(requestContext) {
+            // if the request was successful, increment the user's email sent count
+            if (
+              userId &&
+              query?.includes(action) &&
+              !requestContext.response.errors &&
+              !requestContext.response.data?.replyToEmail?.errorCodes
+            ) {
+              await createServiceUsage(userId, action)
+            }
+          },
+        }
       },
     }
   }
diff --git a/packages/api/src/services/service_usage.ts b/packages/api/src/services/service_usage.ts
index 9d4ada1d8..454f3d9a9 100644
--- a/packages/api/src/services/service_usage.ts
+++ b/packages/api/src/services/service_usage.ts
@@ -20,3 +20,12 @@ export const countDailyServiceUsage = async (
     })
   )
 }
+
+export const createServiceUsage = async (userId: string, action: string) => {
+  return authTrx((tx) =>
+    tx.withRepository(repo).save({
+      user: { id: userId },
+      action,
+    })
+  )
+}

From cd6594a2fd4d595c49f5e8d64ee1a606d0ff27aa Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Tue, 9 Apr 2024 18:30:26 +0800
Subject: [PATCH 21/74] add logs

---
 packages/api/src/apollo.ts | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/packages/api/src/apollo.ts b/packages/api/src/apollo.ts
index b2c748921..26d255e5d 100644
--- a/packages/api/src/apollo.ts
+++ b/packages/api/src/apollo.ts
@@ -131,9 +131,14 @@ export function makeApolloServer(app: Express): ApolloServer {
         const userId = contextValue.context.claims?.uid
         const action = 'replyToEmail'
         if (userId && query?.includes(action)) {
+          logger.info('checking usage limit for user', { userId, action })
           // get the user's email sent count from the DB
           const emailSentCount = await countDailyServiceUsage(userId, action)
           if (emailSentCount >= MAX_SENT_EMAIL_PER_DAY) {
+            logger.info('user has reached the daily email limit', {
+              userId,
+              action,
+            })
             // if the user has reached the limit, throw an error
             throw new Error('You have reached the daily email limit')
           }
@@ -149,6 +154,10 @@ export function makeApolloServer(app: Express): ApolloServer {
               !requestContext.response.errors &&
               !requestContext.response.data?.replyToEmail?.errorCodes
             ) {
+              logger.info('incrementing usage count for user', {
+                userId,
+                action,
+              })
               await createServiceUsage(userId, action)
             }
           },

From f66d3fa36cd83054c82e09b6e491a1182a6dde00 Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Thu, 11 Apr 2024 10:44:57 +0800
Subject: [PATCH 22/74] migrate inbound-email tasks from pubsub to bullmq

---
 packages/api/src/jobs/email/inbound_emails.ts | 306 ++++++++++++++++++
 .../api/src/jobs/{ => email}/send_email.ts    |  11 +-
 packages/api/src/queue-processor.ts           |  20 +-
 .../api/src/resolvers/recent_emails/index.ts  |   2 +-
 packages/api/src/routers/svc/emails.ts        |   2 +-
 .../api/src/services/save_newsletter_email.ts |  54 ----
 packages/api/src/utils/createTask.ts          |   2 +-
 packages/inbound-email-handler/.eslintrc      |  10 +-
 packages/inbound-email-handler/package.json   |  11 +-
 .../inbound-email-handler/src/attachment.ts   | 141 +++-----
 packages/inbound-email-handler/src/index.ts   | 159 +++------
 packages/inbound-email-handler/src/job.ts     |  72 +++++
 .../inbound-email-handler/src/newsletter.ts   |  16 +-
 .../src/redis_data_source.ts                  | 102 ++++++
 yarn.lock                                     |   3 +-
 15 files changed, 608 insertions(+), 303 deletions(-)
 create mode 100644 packages/api/src/jobs/email/inbound_emails.ts
 rename packages/api/src/jobs/{ => email}/send_email.ts (75%)
 create mode 100644 packages/inbound-email-handler/src/job.ts
 create mode 100644 packages/inbound-email-handler/src/redis_data_source.ts

diff --git a/packages/api/src/jobs/email/inbound_emails.ts b/packages/api/src/jobs/email/inbound_emails.ts
new file mode 100644
index 000000000..db8fc02eb
--- /dev/null
+++ b/packages/api/src/jobs/email/inbound_emails.ts
@@ -0,0 +1,306 @@
+import { handleNewsletter } from '@omnivore/content-handler'
+import { Converter } from 'showdown'
+import { ContentReaderType, LibraryItemState } from '../../entity/library_item'
+import { SubscriptionStatus } from '../../entity/subscription'
+import { UploadFile } from '../../entity/upload_file'
+import { env } from '../../env'
+import { PageType, UploadFileStatus } from '../../generated/graphql'
+import { authTrx } from '../../repository'
+import { createOrUpdateLibraryItem } from '../../services/library_item'
+import {
+  findNewsletterEmailByAddress,
+  updateConfirmationCode,
+} from '../../services/newsletters'
+import {
+  saveReceivedEmail,
+  updateReceivedEmail,
+} from '../../services/received_emails'
+import { saveNewsletter } from '../../services/save_newsletter_email'
+import { saveUrlFromEmail } from '../../services/save_url'
+import { getSubscriptionByName } from '../../services/subscriptions'
+import { analytics } from '../../utils/analytics'
+import { enqueueSendEmail } from '../../utils/createTask'
+import { generateSlug, isUrl } from '../../utils/helpers'
+import { logger } from '../../utils/logger'
+import {
+  parseEmailAddress,
+  isProbablyArticle,
+  getTitleFromEmailSubject,
+  generateUniqueUrl,
+} from '../../utils/parser'
+import {
+  generateUploadFilePathName,
+  getStorageFileDetails,
+} from '../../utils/uploads'
+
+interface EmailJobData {
+  from: string
+  to: string
+  subject: string
+  html: string
+  text: string
+  headers: Record<string, string | string[]>
+  unsubMailTo?: string
+  unsubHttpUrl?: string
+  forwardedFrom?: string
+  replyTo?: string
+  confirmationCode?: string
+  uploadFile?: {
+    fileName: string
+    contentType: string
+    id: string
+  }
+}
+
+const converter = new Converter()
+export const FORWARD_EMAIL_JOB = 'forward-email'
+export const SAVE_NEWSLETTER_JOB = 'save-newsletter'
+export const CONFIRM_EMAIL_JOB = 'confirmation-email'
+export const SAVE_ATTACHMENT_JOB = 'save-attachment'
+
+export const plainTextToHtml = (text: string): string => {
+  return converter.makeHtml(text)
+}
+
+export const forwardEmailJob = async (data: EmailJobData) => {
+  const { from, to, subject, html, text, replyTo, forwardedFrom } = data
+
+  // get user from newsletter email
+  const newsletterEmail = await findNewsletterEmailByAddress(to)
+
+  if (!newsletterEmail) {
+    logger.error(`newsletter email not found: ${to}`)
+    return false
+  }
+
+  const user = newsletterEmail.user
+  const parsedFrom = parseEmailAddress(from)
+
+  const { id: receivedEmailId } = await saveReceivedEmail(
+    from,
+    to,
+    subject,
+    text,
+    html,
+    user.id,
+    'non-article',
+    replyTo
+  )
+
+  if (
+    await isProbablyArticle(
+      forwardedFrom || parsedFrom.address || from,
+      subject
+    )
+  ) {
+    logger.info('handling as article')
+    const savedNewsletter = await saveNewsletter(
+      {
+        title: getTitleFromEmailSubject(subject),
+        author: parsedFrom.name || from,
+        url: generateUniqueUrl(),
+        content: html || text,
+        receivedEmailId,
+        email: newsletterEmail.address,
+      },
+      newsletterEmail
+    )
+    if (!savedNewsletter) {
+      logger.error('Failed to save email', { from, to, subject })
+      return false
+    }
+
+    // update received email type
+    await updateReceivedEmail(receivedEmailId, 'article', user.id)
+
+    return true
+  }
+
+  analytics.capture({
+    distinctId: user.id,
+    event: 'non_newsletter_email_received',
+    properties: {
+      env: env.server.apiEnv,
+    },
+  })
+
+  // forward non-newsletter emails to the registered email address
+  const result = await enqueueSendEmail({
+    from: env.sender.message,
+    to: user.email,
+    subject: `Fwd: ${subject}`,
+    html,
+    text,
+    replyTo: replyTo || from,
+  })
+
+  return !!result
+}
+
+export const saveNewsletterJob = async (data: EmailJobData) => {
+  const {
+    from,
+    to,
+    subject,
+    html,
+    text,
+    replyTo,
+    headers,
+    unsubMailTo,
+    unsubHttpUrl,
+  } = data
+
+  // get user from newsletter email
+  const newsletterEmail = await findNewsletterEmailByAddress(to)
+  if (!newsletterEmail) {
+    logger.error(`newsletter email not found: ${to}`)
+
+    return false
+  }
+
+  const user = newsletterEmail.user
+  const { id: receivedEmailId } = await saveReceivedEmail(
+    from,
+    to,
+    subject,
+    text,
+    html,
+    user.id,
+    'article',
+    replyTo
+  )
+
+  if (isUrl(subject)) {
+    // save url if the title is a parsable url
+    const result = await saveUrlFromEmail(
+      subject,
+      receivedEmailId,
+      newsletterEmail.user.id
+    )
+
+    return result
+  }
+
+  // convert text to html if html is not available
+  const content = html || plainTextToHtml(text)
+  const newsletter = await handleNewsletter({
+    from,
+    to,
+    subject,
+    html: content,
+    headers,
+  })
+
+  const parsedFrom = parseEmailAddress(from)
+  const author = parsedFrom.name || from
+
+  // do not subscribe if subscription already exists and is unsubscribed
+  const existingSubscription = await getSubscriptionByName(
+    author,
+    newsletterEmail.user.id
+  )
+  if (existingSubscription?.status === SubscriptionStatus.Unsubscribed) {
+    logger.info(`newsletter already unsubscribed: ${from}`)
+    return false
+  }
+
+  // save newsletter instead
+  const result = await saveNewsletter(
+    {
+      email: newsletterEmail.address,
+      content,
+      url: generateUniqueUrl(),
+      title: subject,
+      author,
+      unsubMailTo,
+      unsubHttpUrl,
+      receivedEmailId,
+      ...newsletter,
+    },
+    newsletterEmail
+  )
+
+  return result
+}
+
+export const saveAttachmentJob = async (data: EmailJobData) => {
+  const { from, to, subject, html, text, replyTo, uploadFile } = data
+
+  // get user from newsletter email
+  const newsletterEmail = await findNewsletterEmailByAddress(to)
+  if (!newsletterEmail) {
+    logger.error(`newsletter email not found: ${to}`)
+
+    return false
+  }
+
+  const user = newsletterEmail.user
+  await saveReceivedEmail(
+    from,
+    to,
+    subject,
+    text,
+    html,
+    user.id,
+    'article',
+    replyTo
+  )
+
+  const uploadFileData = await authTrx(
+    (tx) =>
+      tx.getRepository(UploadFile).save({
+        ...uploadFile,
+        status: UploadFileStatus.Completed,
+        user: { id: user.id },
+      }),
+    undefined,
+    user.id
+  )
+
+  const uploadFileDetails = await getStorageFileDetails(
+    uploadFileData.id,
+    uploadFileData.fileName
+  )
+
+  const uploadFilePathName = generateUploadFilePathName(
+    uploadFileData.id,
+    uploadFileData.fileName
+  )
+
+  const uploadFileUrlOverride = `https://omnivore.app/attachments/${uploadFilePathName}`
+  const uploadFileHash = uploadFileDetails.md5Hash
+  const itemType =
+    uploadFileData.contentType === 'application/pdf'
+      ? PageType.File
+      : PageType.Book
+  const title = subject || uploadFileData.fileName
+  const itemToCreate = {
+    originalUrl: uploadFileUrlOverride,
+    itemType,
+    textContentHash: uploadFileHash,
+    uploadFile: { id: uploadFileData.id },
+    title,
+    readableContent: '',
+    slug: generateSlug(title),
+    state: LibraryItemState.Succeeded,
+    user: { id: user.id },
+    contentReader:
+      itemType === PageType.File
+        ? ContentReaderType.PDF
+        : ContentReaderType.EPUB,
+  }
+
+  await createOrUpdateLibraryItem(itemToCreate, user.id)
+
+  return true
+}
+
+export const confirmEmailJob = async (data: EmailJobData) => {
+  const { confirmationCode, to } = data
+  if (!confirmationCode) {
+    logger.error('confirmation code not provided')
+    return false
+  }
+
+  return updateConfirmationCode(to, confirmationCode)
+}
diff --git a/packages/api/src/jobs/send_email.ts b/packages/api/src/jobs/email/send_email.ts
similarity index 75%
rename from packages/api/src/jobs/send_email.ts
rename to packages/api/src/jobs/email/send_email.ts
index 1c94174f1..130e2b14a 100644
--- a/packages/api/src/jobs/send_email.ts
+++ b/packages/api/src/jobs/email/send_email.ts
@@ -1,8 +1,8 @@
-import { env } from '../env'
-import { sendWithMailJet } from '../services/send_emails'
-import { Merge } from '../util'
-import { logger } from '../utils/logger'
-import { sendEmail } from '../utils/sendEmail'
+import { env } from '../../env'
+import { sendWithMailJet } from '../../services/send_emails'
+import { Merge } from '../../util'
+import { logger } from '../../utils/logger'
+import { sendEmail } from '../../utils/sendEmail'
 
 export const SEND_EMAIL_JOB = 'send-email'
 
@@ -16,6 +16,7 @@ export type SendEmailJobData = Merge<
     text?: string
     templateId?: string
     dynamicTemplateData?: Record<string, any>
+    replyTo?: string
   },
   ContentType
 >
diff --git a/packages/api/src/queue-processor.ts b/packages/api/src/queue-processor.ts
index d13e934de..136a854f4 100644
--- a/packages/api/src/queue-processor.ts
+++ b/packages/api/src/queue-processor.ts
@@ -36,7 +36,7 @@ import {
 import { refreshAllFeeds } from './jobs/rss/refreshAllFeeds'
 import { refreshFeed } from './jobs/rss/refreshFeed'
 import { savePageJob } from './jobs/save_page'
-import { sendEmailJob, SEND_EMAIL_JOB } from './jobs/send_email'
+import { sendEmailJob, SEND_EMAIL_JOB } from './jobs/email/send_email'
 import {
   syncReadPositionsJob,
   SYNC_READ_POSITIONS_JOB_NAME,
@@ -53,6 +53,16 @@ import { redisDataSource } from './redis_data_source'
 import { CACHED_READING_POSITION_PREFIX } from './services/cached_reading_position'
 import { getJobPriority } from './utils/createTask'
 import { logger } from './utils/logger'
+import {
+  confirmEmailJob,
+  CONFIRM_EMAIL_JOB,
+  forwardEmailJob,
+  FORWARD_EMAIL_JOB,
+  saveAttachmentJob,
+  saveNewsletterJob,
+  SAVE_ATTACHMENT_JOB,
+  SAVE_NEWSLETTER_JOB,
+} from './jobs/email/inbound_emails'
 
 export const QUEUE_NAME = 'omnivore-backend-queue'
 export const JOB_VERSION = 'v001'
@@ -160,6 +170,14 @@ export const createWorker = (connection: ConnectionOptions) =>
           return exportAllItems(job.data)
         case SEND_EMAIL_JOB:
           return sendEmailJob(job.data)
+        case CONFIRM_EMAIL_JOB:
+          return confirmEmailJob(job.data)
+        case SAVE_ATTACHMENT_JOB:
+          return saveAttachmentJob(job.data)
+        case SAVE_NEWSLETTER_JOB:
+          return saveNewsletterJob(job.data)
+        case FORWARD_EMAIL_JOB:
+          return forwardEmailJob(job.data)
         default:
           logger.warning(`[queue-processor] unhandled job: ${job.name}`)
       }
diff --git a/packages/api/src/resolvers/recent_emails/index.ts b/packages/api/src/resolvers/recent_emails/index.ts
index 84aca707a..b8bbd275c 100644
--- a/packages/api/src/resolvers/recent_emails/index.ts
+++ b/packages/api/src/resolvers/recent_emails/index.ts
@@ -89,7 +89,7 @@ export const markEmailAsItemResolver = authorized<
       title: recentEmail.subject,
       content: recentEmail.html,
       url: generateUniqueUrl(),
-      author: parseEmailAddress(recentEmail.from).name,
+      author: parseEmailAddress(recentEmail.from).name || recentEmail.from,
       receivedEmailId: recentEmail.id,
     },
     newsletterEmail
diff --git a/packages/api/src/routers/svc/emails.ts b/packages/api/src/routers/svc/emails.ts
index eda01a541..47f8f0b77 100644
--- a/packages/api/src/routers/svc/emails.ts
+++ b/packages/api/src/routers/svc/emails.ts
@@ -83,7 +83,7 @@ export function emailsServiceRouter() {
         const savedNewsletter = await saveNewsletter(
           {
             title: getTitleFromEmailSubject(data.subject),
-            author: parsedFrom.name,
+            author: parsedFrom.name || data.from,
             url: generateUniqueUrl(),
             content: data.html || data.text,
             receivedEmailId: data.receivedEmailId,
diff --git a/packages/api/src/services/save_newsletter_email.ts b/packages/api/src/services/save_newsletter_email.ts
index 08d6fa67b..f11601288 100644
--- a/packages/api/src/services/save_newsletter_email.ts
+++ b/packages/api/src/services/save_newsletter_email.ts
@@ -76,59 +76,5 @@ export const saveNewsletter = async (
     return false
   }
 
-  // sends push notification
-  // const deviceTokens = await getDeviceTokensByUserId(newsletterEmail.user.id)
-  // if (!deviceTokens) {
-  //   logger.info('Device tokens not set:', newsletterEmail.user.id)
-  //   return true
-  // }
-
-  // const multicastMessage = messageForLink(page, deviceTokens)
-  // await sendMulticastPushNotifications(
-  //   newsletterEmail.user.id,
-  //   multicastMessage,
-  //   'newsletter'
-  // )
-
   return true
 }
-
-// const messageForLink = (
-//   link: Page,
-//   deviceTokens: UserDeviceToken[]
-// ): MulticastMessage => {
-//   let title = '📫 - An article was added to your Omnivore Inbox'
-
-//   if (link.author) {
-//     title = `📫 - ${link.author} has published a new article`
-//   }
-
-//   const pushData = !link
-//     ? undefined
-//     : {
-//         link: Buffer.from(
-//           JSON.stringify({
-//             id: link.id,
-//             url: link.url,
-//             slug: link.slug,
-//             title: link.title,
-//             image: link.image,
-//             author: link.author,
-//             isArchived: !!link.archivedAt,
-//             contentReader: ContentReader.Web,
-//             readingProgressPercent: link.readingProgressPercent,
-//             readingProgressAnchorIndex: link.readingProgressAnchorIndex,
-//           })
-//         ).toString('base64'),
-//       }
-
-//   return {
-//     notification: {
-//       title: title,
-//       body: link.title,
-//       imageUrl: link.image || undefined,
-//     },
-//     data: pushData,
-//     tokens: deviceTokens.map((token) => token.token),
-//   }
-// }
diff --git a/packages/api/src/utils/createTask.ts b/packages/api/src/utils/createTask.ts
index c2ce23fa9..001daee64 100644
--- a/packages/api/src/utils/createTask.ts
+++ b/packages/api/src/utils/createTask.ts
@@ -35,7 +35,7 @@ import {
   REFRESH_ALL_FEEDS_JOB_NAME,
   REFRESH_FEED_JOB_NAME,
 } from '../jobs/rss/refreshAllFeeds'
-import { SendEmailJobData, SEND_EMAIL_JOB } from '../jobs/send_email'
+import { SendEmailJobData, SEND_EMAIL_JOB } from '../jobs/email/send_email'
 import { SYNC_READ_POSITIONS_JOB_NAME } from '../jobs/sync_read_positions'
 import { TriggerRuleJobData, TRIGGER_RULE_JOB_NAME } from '../jobs/trigger_rule'
 import {
diff --git a/packages/inbound-email-handler/.eslintrc b/packages/inbound-email-handler/.eslintrc
index e006282a6..301be9795 100644
--- a/packages/inbound-email-handler/.eslintrc
+++ b/packages/inbound-email-handler/.eslintrc
@@ -2,5 +2,13 @@
   "extends": "../../.eslintrc",
   "parserOptions": {
     "project": "tsconfig.json"
+  },
+  "rules": {
+    "@typescript-eslint/no-floating-promises": [
+      "error",
+      {
+        "ignoreIIFE": true
+      }
+    ]
   }
-}
\ No newline at end of file
+}
diff --git a/packages/inbound-email-handler/package.json b/packages/inbound-email-handler/package.json
index 2330a4308..b2aa9ba52 100644
--- a/packages/inbound-email-handler/package.json
+++ b/packages/inbound-email-handler/package.json
@@ -32,17 +32,16 @@
   },
   "dependencies": {
     "@google-cloud/functions-framework": "3.1.2",
-    "@google-cloud/pubsub": "^4.0.0",
-    "@omnivore/content-handler": "1.0.0",
-    "@sendgrid/client": "^7.6.0",
+    "@google-cloud/storage": "^7.0.1",
     "@sentry/serverless": "^7.77.0",
     "addressparser": "^1.0.1",
-    "axios": "^0.27.2",
-    "jsonwebtoken": "^8.5.1",
+    "bullmq": "^5.1.1",
+    "dotenv": "^8.2.0",
+    "ioredis": "^5.3.2",
     "parse-headers": "^2.0.4",
     "parse-multipart-data": "^1.2.1",
     "rfc2047": "^4.0.1",
-    "showdown": "^2.1.0"
+    "uuid": "^8.3.1"
   },
   "volta": {
     "extends": "../../package.json"
diff --git a/packages/inbound-email-handler/src/attachment.ts b/packages/inbound-email-handler/src/attachment.ts
index 273555bae..db3d0d094 100644
--- a/packages/inbound-email-handler/src/attachment.ts
+++ b/packages/inbound-email-handler/src/attachment.ts
@@ -1,8 +1,11 @@
-import axios, { AxiosResponse } from 'axios'
-import * as jwt from 'jsonwebtoken'
-import { promisify } from 'util'
+import { Storage } from '@google-cloud/storage'
+import { v4 as uuid } from 'uuid'
+import { EmailJobType, queueEmailJob } from './job'
 
-const signToken = promisify(jwt.sign)
+const storage = process.env.GCS_UPLOAD_SA_KEY_FILE_PATH
+  ? new Storage({ keyFilename: process.env.GCS_UPLOAD_SA_KEY_FILE_PATH })
+  : new Storage()
+const bucketName = process.env.GCS_UPLOAD_BUCKET || 'omnivore-files'
 
 export interface Attachment {
   contentType: string
@@ -10,11 +13,6 @@ export interface Attachment {
   filename: string | undefined
 }
 
-type UploadResponse = {
-  id: string
-  url: string
-}
-
 export const isAttachment = (contentType: string, data: Buffer): boolean => {
   return (
     (contentType === 'application/pdf' ||
@@ -23,11 +21,26 @@ export const isAttachment = (contentType: string, data: Buffer): boolean => {
   )
 }
 
+export const uploadToBucket = async (
+  fileName: string,
+  data: Buffer,
+  options?: { contentType?: string; public?: boolean }
+) => {
+  const uploadFileId = uuid()
+
+  await storage
+    .bucket(bucketName)
+    .file(`u/${uploadFileId}/${fileName}`)
+    .save(data, { ...options, timeout: 30000 })
+
+  return uploadFileId
+}
+
 export const handleAttachments = async (
-  email: string,
+  from: string,
+  to: string,
   subject: string,
-  attachments: Attachment[],
-  receivedEmailId: string
+  attachments: Attachment[]
 ): Promise<void> => {
   for await (const attachment of attachments) {
     const { contentType, data } = attachment
@@ -36,98 +49,20 @@ export const handleAttachments = async (
         ? 'attachment.pdf'
         : 'attachment.epub'
 
-    try {
-      const uploadResult = await getUploadIdAndSignedUrl(
-        email,
-        filename,
-        contentType
-      )
-      if (!uploadResult.url || !uploadResult.id) {
-        console.log('failed to create upload request', uploadResult)
-        return
-      }
-      await uploadToSignedUrl(uploadResult.url, data, contentType)
-      await createArticle(email, uploadResult.id, subject, receivedEmailId)
-    } catch (error) {
-      console.error('handleAttachments error', error)
-    }
-  }
-}
+    const uploadFileId = await uploadToBucket(filename, data, {
+      contentType,
+      public: false,
+    })
 
-const getUploadIdAndSignedUrl = async (
-  email: string,
-  fileName: string,
-  contentType: string
-): Promise<UploadResponse> => {
-  if (process.env.JWT_SECRET === undefined) {
-    throw new Error('JWT_SECRET is not defined')
-  }
-  const auth = await signToken(email, process.env.JWT_SECRET)
-  const data = {
-    fileName,
-    email,
-    contentType,
-  }
-
-  if (process.env.INTERNAL_SVC_ENDPOINT === undefined) {
-    throw new Error('REST_BACKEND_ENDPOINT is not defined')
-  }
-  const response = await axios.post(
-    `${process.env.INTERNAL_SVC_ENDPOINT}svc/email-attachment/upload`,
-    data,
-    {
-      headers: {
-        Authorization: `${auth as string}`,
-        'Content-Type': 'application/json',
+    await queueEmailJob(EmailJobType.SaveAttachment, {
+      from,
+      to,
+      uploadFile: {
+        fileName: filename,
+        contentType,
+        id: uploadFileId,
       },
-    }
-  )
-  return response.data as UploadResponse
-}
-
-const uploadToSignedUrl = async (
-  uploadUrl: string,
-  data: Buffer,
-  contentType: string
-): Promise<AxiosResponse> => {
-  return axios.put(uploadUrl, data, {
-    headers: {
-      'Content-Type': contentType,
-    },
-    maxBodyLength: 1000000000,
-    maxContentLength: 100000000,
-  })
-}
-
-const createArticle = async (
-  email: string,
-  uploadFileId: string,
-  subject: string,
-  receivedEmailId: string
-): Promise<AxiosResponse> => {
-  const data = {
-    email,
-    uploadFileId,
-    subject,
-    receivedEmailId,
+      subject,
+    })
   }
-
-  if (process.env.JWT_SECRET === undefined) {
-    throw new Error('JWT_SECRET is not defined')
-  }
-  const auth = await signToken(email, process.env.JWT_SECRET)
-
-  if (process.env.INTERNAL_SVC_ENDPOINT === undefined) {
-    throw new Error('REST_BACKEND_ENDPOINT is not defined')
-  }
-  return axios.post(
-    `${process.env.INTERNAL_SVC_ENDPOINT}svc/email-attachment/create-article`,
-    data,
-    {
-      headers: {
-        Authorization: `${auth as string}`,
-        'Content-Type': 'application/json',
-      },
-    }
-  )
 }
diff --git a/packages/inbound-email-handler/src/index.ts b/packages/inbound-email-handler/src/index.ts
index 391244e35..0abd80dac 100644
--- a/packages/inbound-email-handler/src/index.ts
+++ b/packages/inbound-email-handler/src/index.ts
@@ -2,92 +2,29 @@
 /* eslint-disable @typescript-eslint/no-explicit-any */
 /* eslint-disable @typescript-eslint/no-unsafe-argument */
 /* eslint-disable @typescript-eslint/no-unused-vars */
-
-import { PubSub } from '@google-cloud/pubsub'
-import { handleNewsletter } from '@omnivore/content-handler'
-import { generateUniqueUrl } from '@omnivore/content-handler/build/src/content-handler'
 import * as Sentry from '@sentry/serverless'
-import axios from 'axios'
-import * as jwt from 'jsonwebtoken'
 import parseHeaders from 'parse-headers'
 import * as multipart from 'parse-multipart-data'
 import rfc2047 from 'rfc2047'
-import { Converter } from 'showdown'
-import { promisify } from 'util'
 import { Attachment, handleAttachments, isAttachment } from './attachment'
+import { EmailJobType, queueEmailJob } from './job'
 import {
   handleGoogleConfirmationEmail,
   isGoogleConfirmationEmail,
   isSubscriptionConfirmationEmail,
-  parseAuthor,
   parseUnsubscribe,
 } from './newsletter'
 
-interface SaveReceivedEmailResponse {
-  id: string
-}
-
 interface Envelope {
   to: string[]
   from: string
 }
 
-const signToken = promisify(jwt.sign)
-
 Sentry.GCPFunction.init({
   dsn: process.env.SENTRY_DSN,
   tracesSampleRate: 0,
 })
 
-const NEWSLETTER_EMAIL_RECEIVED_TOPIC = 'newsletterEmailReceived'
-const NON_NEWSLETTER_EMAIL_TOPIC = 'nonNewsletterEmailReceived'
-const pubsub = new PubSub()
-const converter = new Converter()
-
-export const plainTextToHtml = (text: string): string => {
-  return converter.makeHtml(text)
-}
-
-export const publishMessage = async (
-  topic: string,
-  message: any
-): Promise<string | undefined> => {
-  return pubsub
-    .topic(topic)
-    .publishMessage({ json: message })
-    .catch((err) => {
-      console.log('error publishing message:', err)
-      return undefined
-    })
-}
-
-const saveReceivedEmail = async (
-  email: string,
-  data: any
-): Promise<SaveReceivedEmailResponse> => {
-  if (process.env.JWT_SECRET === undefined) {
-    throw new Error('JWT_SECRET is not defined')
-  }
-  const auth = await signToken(email, process.env.JWT_SECRET)
-
-  if (process.env.INTERNAL_SVC_ENDPOINT === undefined) {
-    throw new Error('REST_BACKEND_ENDPOINT is not defined')
-  }
-
-  const response = await axios.post(
-    `${process.env.INTERNAL_SVC_ENDPOINT}svc/pubsub/emails/save`,
-    data,
-    {
-      headers: {
-        Authorization: `${auth as string}`,
-        'Content-Type': 'application/json',
-      },
-    }
-  )
-
-  return response.data as SaveReceivedEmailResponse
-}
-
 export const parsedTo = (parsed: Record<string, string>): string => {
   // envelope to contains the real recipient email address
   try {
@@ -135,97 +72,77 @@ export const inboundEmailHandler = Sentry.GCPFunction.wrapHttpFunction(
         ? parseUnsubscribe(unSubHeader)
         : undefined
 
-      const { id: receivedEmailId } = await saveReceivedEmail(to, {
-        from,
-        to,
-        subject,
-        html,
-        text,
-        replyTo,
-      })
-
       try {
         // check if it is a subscription or google confirmation email
         const isGoogleConfirmation = isGoogleConfirmationEmail(from, subject)
         if (isGoogleConfirmation || isSubscriptionConfirmationEmail(subject)) {
           console.debug('handleConfirmation', from, subject)
           // we need to parse the confirmation code from the email
-          isGoogleConfirmation &&
-            (await handleGoogleConfirmationEmail(to, subject))
-          // queue non-newsletter emails
-          await pubsub.topic(NON_NEWSLETTER_EMAIL_TOPIC).publishMessage({
-            json: {
-              from,
-              to,
-              subject,
-              html,
-              text,
-              unsubMailTo: unsubscribe?.mailTo,
-              unsubHttpUrl: unsubscribe?.httpUrl,
-              forwardedFrom,
-              receivedEmailId,
-            },
+          if (isGoogleConfirmation) {
+            await handleGoogleConfirmationEmail(from, to, subject)
+          }
+
+          // forward emails
+          await queueEmailJob(EmailJobType.ForwardEmail, {
+            from,
+            to,
+            subject,
+            html,
+            text,
+            headers,
+            forwardedFrom,
+            replyTo,
           })
           return res.send('ok')
         }
         if (attachments.length > 0) {
           console.debug('handle attachments', from, to, subject)
           // save the attachments as articles
-          await handleAttachments(to, subject, attachments, receivedEmailId)
+          await handleAttachments(from, to, subject, attachments)
           return res.send('ok')
         }
 
-        // convert text to html if html is not available
-        const content = html || plainTextToHtml(text)
-
         // all other emails are considered newsletters
-        const newsletterMessage = await handleNewsletter({
+        // queue newsletter emails
+        await queueEmailJob(EmailJobType.SaveNewsletter, {
           from,
           to,
           subject,
-          html: content,
+          html,
+          text,
           headers,
+          unsubMailTo: unsubscribe?.mailTo,
+          unsubHttpUrl: unsubscribe?.httpUrl,
+          forwardedFrom,
+          replyTo,
         })
 
-        // queue newsletter emails
-        await pubsub.topic(NEWSLETTER_EMAIL_RECEIVED_TOPIC).publishMessage({
-          json: {
-            email: to,
-            content,
-            url: generateUniqueUrl(),
-            title: subject,
-            author: parseAuthor(from),
-            unsubMailTo: unsubscribe?.mailTo,
-            unsubHttpUrl: unsubscribe?.httpUrl,
-            receivedEmailId,
-            ...newsletterMessage,
-          },
-        })
         res.send('newsletter received')
       } catch (error) {
-        console.log(
+        console.error(
           'error handling emails, will forward.',
           from,
           to,
           subject,
           error
         )
-        // queue error emails
-        await pubsub.topic(NON_NEWSLETTER_EMAIL_TOPIC).publishMessage({
-          json: {
-            from,
-            to,
-            subject,
-            html,
-            text,
-            forwardedFrom,
-            receivedEmailId,
-          },
+
+        // fallback to forward the email
+        await queueEmailJob(EmailJobType.ForwardEmail, {
+          from,
+          to,
+          subject,
+          html,
+          text,
+          headers,
+          forwardedFrom,
+          replyTo,
         })
+
         res.send('ok')
       }
     } catch (e) {
-      console.log(e)
+      console.error(e)
       res.send(e)
     }
   }
diff --git a/packages/inbound-email-handler/src/job.ts b/packages/inbound-email-handler/src/job.ts
new file mode 100644
index 000000000..89a2fea40
--- /dev/null
+++ b/packages/inbound-email-handler/src/job.ts
@@ -0,0 +1,72 @@
+import { BulkJobOptions, Queue } from 'bullmq'
+import { redisDataSource } from './redis_data_source'
+
+const QUEUE_NAME = 'omnivore-backend-queue'
+export enum EmailJobType {
+  ForwardEmail = 'forward-email',
+  SaveNewsletter = 'save-newsletter',
+  ConfirmationEmail = 'confirmation-email',
+  SaveAttachment = 'save-attachment',
+}
+
+interface EmailJobData {
+  from: string
+  to: string
+  subject: string
+  html?: string
+  text?: string
+  headers?: Record<string, string | string[]>
+  unsubMailTo?: string
+  unsubHttpUrl?: string
+  forwardedFrom?: string
+  replyTo?: string
+  uploadFile?: {
+    fileName: string
+    contentType: string
+    id: string
+  }
+  confirmationCode?: string
+}
+
+const queue = new Queue(QUEUE_NAME, {
+  connection: redisDataSource.queueRedisClient,
+})
+
+const getPriority = (jobType: EmailJobType): number => {
+  // we want to prioritized jobs by the expected time to complete
+  // lower number means higher priority
+  // priority 1: jobs that are expected to finish immediately
+  // priority 5: jobs that are expected to finish in less than 10 second
+  // priority 10: jobs that are expected to finish in less than 10 minutes
+  // priority 100: jobs that are expected to finish in less than 1 hour
+  switch (jobType) {
+    case EmailJobType.ForwardEmail:
+    case EmailJobType.ConfirmationEmail:
+      return 1
+    case EmailJobType.SaveAttachment:
+    case EmailJobType.SaveNewsletter:
+      return 5
+    default:
+      throw new Error(`unknown job type: ${jobType as string}`)
+  }
+}
+
+const getOpts = (jobType: EmailJobType): BulkJobOptions => {
+  return {
+    removeOnComplete: true,
+    removeOnFail: true,
+    attempts: 3,
+    priority: getPriority(jobType),
+    backoff: {
+      type: 'exponential',
+      delay: 2000,
+    },
+  }
+}
+
+export const queueEmailJob = async (
+  jobType: EmailJobType,
+  data: EmailJobData
+) => {
+  await queue.add(jobType, data, getOpts(jobType))
+}
diff --git a/packages/inbound-email-handler/src/newsletter.ts b/packages/inbound-email-handler/src/newsletter.ts
index 54be5d3dd..b31103ada 100644
--- a/packages/inbound-email-handler/src/newsletter.ts
+++ b/packages/inbound-email-handler/src/newsletter.ts
@@ -1,12 +1,11 @@
 import addressparser from 'addressparser'
-import { publishMessage } from './index'
+import { EmailJobType, queueEmailJob } from './job'
 
 interface Unsubscribe {
   mailTo?: string
   httpUrl?: string
 }
 
-const GOOGLE_CONFIRMATION_CODE_RECEIVED_TOPIC = 'emailConfirmationCodeReceived'
 const GOOGLE_CONFIRMATION_EMAIL_SENDER_ADDRESS = 'forwarding-noreply@google.com'
 // check unicode parentheses too
 const GOOGLE_CONFIRMATION_CODE_PATTERN = /\d+/u
@@ -41,24 +40,25 @@ export const parseAuthor = (address: string): string => {
 }
 
 export const handleGoogleConfirmationEmail = async (
-  email: string,
+  from: string,
+  to: string,
   subject: string
 ) => {
-  console.log('confirmation email', email, subject)
+  console.log('confirmation email', from, to, subject)
 
   const confirmationCode = getConfirmationCode(subject)
-  if (!email || !confirmationCode) {
+  if (!to || !confirmationCode) {
     console.log(
       'confirmation email error, user email:',
-      email,
+      to,
       'confirmationCode',
       confirmationCode
     )
     throw new Error('invalid confirmation email')
   }
 
-  const message = { emailAddress: email, confirmationCode: confirmationCode }
-  return publishMessage(GOOGLE_CONFIRMATION_CODE_RECEIVED_TOPIC, message)
+  const message = { from, to, confirmationCode, subject }
+  return queueEmailJob(EmailJobType.ConfirmationEmail, message)
 }
 
 export const getConfirmationCode = (subject: string): string | undefined => {
diff --git a/packages/inbound-email-handler/src/redis_data_source.ts b/packages/inbound-email-handler/src/redis_data_source.ts
new file mode 100644
index 000000000..b8ca0ce6b
--- /dev/null
+++ b/packages/inbound-email-handler/src/redis_data_source.ts
@@ -0,0 +1,102 @@
+import Redis, { RedisOptions } from 'ioredis'
+import 'dotenv/config'
+
+type RedisClientType = 'cache' | 'mq'
+type RedisDataSourceOption = {
+  url?: string
+  cert?: string
+}
+export type RedisDataSourceOptions = {
+  [key in RedisClientType]: RedisDataSourceOption
+}
+
+export class RedisDataSource {
+  options: RedisDataSourceOptions
+
+  cacheClient: Redis
+  queueRedisClient: Redis
+
+  constructor(options: RedisDataSourceOptions) {
+    this.options = options
+
+    const cacheClient = createIORedisClient('cache', this.options)
+    if (!cacheClient) throw 'Error initializing cache redis client'
+
+    this.cacheClient = cacheClient
+    this.queueRedisClient =
+      createIORedisClient('mq', this.options) || this.cacheClient // if mq is not defined, use cache
+  }
+
+  async shutdown(): Promise<void> {
+    try {
+      await this.queueRedisClient?.quit()
+      await this.cacheClient?.quit()
+    } catch (err) {
+      console.error('error while shutting down redis', err)
+    }
+  }
+}
+
+const createIORedisClient = (
+  name: RedisClientType,
+  options: RedisDataSourceOptions
+): Redis | undefined => {
+  const option = options[name]
+  const redisURL = option.url
+  if (!redisURL) {
+    console.log(`no redisURL supplied: ${name}`)
+    return undefined
+  }
+
+  const redisCert = option.cert
+  const tls =
+    redisURL.startsWith('rediss://') && redisCert
+      ? {
+          ca: redisCert,
+          rejectUnauthorized: false,
+        }
+      : undefined
+
+  const redisOptions: RedisOptions = {
+    tls,
+    name,
+    connectTimeout: 10000,
+    maxRetriesPerRequest: null,
+    offlineQueue: false,
+  }
+  return new Redis(redisURL, redisOptions)
+}
+
+export const redisDataSource = new RedisDataSource({
+  cache: {
+    url: process.env.REDIS_URL,
+    cert: process.env.REDIS_CERT,
+  },
+  mq: {
+    url: process.env.MQ_REDIS_URL,
+    cert: process.env.MQ_REDIS_CERT,
+  },
+})
+
+const gracefulShutdown = async (signal: string) => {
+  console.log(`Received ${signal}, shutting down gracefully...`)
+
+  try {
+    await redisDataSource.shutdown()
+    console.log('redis shutdown successfully')
+  } catch (error) {
+    console.error('error while shutting down redis', error)
+  }
+  process.exit(0)
+}
+
+process.on('SIGINT', () => {
+  ;(async () => {
+    await gracefulShutdown('SIGINT')
+  })()
+})
+process.on('SIGTERM', () => {
+  ;(async () => {
+    await gracefulShutdown('SIGTERM')
+  })()
+})
diff --git a/yarn.lock b/yarn.lock
index 074fd4c76..46b7be829 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -5783,7 +5783,7 @@
     lodash "^4.17.4"
     read-pkg-up "^7.0.0"
 
-"@sendgrid/client@^7.6.0", "@sendgrid/client@^7.7.0":
+"@sendgrid/client@^7.7.0":
   version "7.7.0"
   resolved "https://registry.yarnpkg.com/@sendgrid/client/-/client-7.7.0.tgz#f8f67abd604205a0d0b1af091b61517ef465fdbf"
   integrity sha512-SxH+y8jeAQSnDavrTD0uGDXYIIkFylCo+eDofVmZLQ0f862nnqbC3Vd1ej6b7Le7lboyzQF6F7Fodv02rYspuA==
@@ -11679,6 +11679,7 @@ caniuse-lite@^1.0.30001109, caniuse-lite@^1.0.30001251, caniuse-lite@^1.0.300012
   version "1.0.30001600"
   resolved "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001600.tgz"
   integrity sha512-+2S9/2JFhYmYaDpZvo0lKkfvuKIglrx68MwOBqMGHhQsNkLjB5xtc/TGoEPs+MxjSyN/72qer2g97nzR641mOQ==
+
 capital-case@^1.0.4:
   version "1.0.4"
   resolved "https://registry.yarnpkg.com/capital-case/-/capital-case-1.0.4.tgz#9d130292353c9249f6b00fa5852bee38a717e669"

From 07a14e0a7f81bd548927ae452a4604cc4f0a5503 Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Thu, 11 Apr 2024 11:00:18 +0800
Subject: [PATCH 23/74] fix tests

---
 .../test/newsletter.test.ts                   | 28 +------------------
 packages/inbound-email-handler/tsconfig.json  |  2 +-
 2 files changed, 2 insertions(+), 28 deletions(-)

diff --git a/packages/inbound-email-handler/test/newsletter.test.ts b/packages/inbound-email-handler/test/newsletter.test.ts
index cf9bff45c..b32a7f504 100644
--- a/packages/inbound-email-handler/test/newsletter.test.ts
+++ b/packages/inbound-email-handler/test/newsletter.test.ts
@@ -2,7 +2,7 @@ import { expect } from 'chai'
 import 'mocha'
 import parseHeaders from 'parse-headers'
 import rfc2047 from 'rfc2047'
-import { parsedTo, plainTextToHtml } from '../src'
+import { parsedTo } from '../src'
 import {
   getConfirmationCode,
   isGoogleConfirmationEmail,
@@ -138,29 +138,3 @@ describe('decode and parse headers', () => {
     })
   })
 })
-
-describe('plainTextToHtml', () => {
-  it('converts text to html', () => {
-    const text =
-      'DEVOPS WEEKLY\r\n' +
-      'ISSUE #665 - 24th September 2023\r\n' +
-      '\r\n' +
-      'A few posts on CI tooling this week, along with a good introduction to developer portals/platforms and other topics.\r\n' +
-      '\r\n' +
-      'StackHawk sponsors Devops Weekly\r\n' +
-      '============================\r\n' +
-      '\r\n' +
-      'Experience automated security testing without the hassle of connecting your own app or configuring an environment! Follow the Tutorial to try out StackHawk and explore a world where security becomes an accelerator, not a blocker\r\n' +
-      '\r\n' +
-      'https://sthwk.com/tutorial\r\n' +
-      '\r\n'
-    expect(plainTextToHtml(text)).to.eql(
-      `<p>DEVOPS WEEKLY
-ISSUE #665 - 24th September 2023</p>
-<p>A few posts on CI tooling this week, along with a good introduction to developer portals/platforms and other topics.</p>
-<h1 id="stackhawksponsorsdevopsweekly">StackHawk sponsors Devops Weekly</h1>
-<p>Experience automated security testing without the hassle of connecting your own app or configuring an environment! Follow the Tutorial to try out StackHawk and explore a world where security becomes an accelerator, not a blocker</p>
-<p>https://sthwk.com/tutorial</p>`
-    )
-  })
-})
diff --git a/packages/inbound-email-handler/tsconfig.json b/packages/inbound-email-handler/tsconfig.json
index 5220d6b3f..f450acf38 100644
--- a/packages/inbound-email-handler/tsconfig.json
+++ b/packages/inbound-email-handler/tsconfig.json
@@ -5,5 +5,5 @@
     "rootDir": ".",
     "lib": ["dom"]
   },
-  "include": ["src"]
+  "include": ["src", "test"]
 }

From a062152d4a08b09007200d76e20279a0285a1838 Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Thu, 11 Apr 2024 11:08:07 +0800
Subject: [PATCH 24/74] fix tests

---
 packages/inbound-email-handler/mocha-config.json        | 4 +++-
 packages/inbound-email-handler/src/redis_data_source.ts | 9 +++------
 packages/inbound-email-handler/test/global-teardown.ts  | 5 +++++
 3 files changed, 11 insertions(+), 7 deletions(-)
 create mode 100644 packages/inbound-email-handler/test/global-teardown.ts

diff --git a/packages/inbound-email-handler/mocha-config.json b/packages/inbound-email-handler/mocha-config.json
index 5a2631f42..897a3bb1e 100644
--- a/packages/inbound-email-handler/mocha-config.json
+++ b/packages/inbound-email-handler/mocha-config.json
@@ -1,4 +1,6 @@
 {
     "extension": ["ts"],
-    "spec": "test/**/*.test.ts"
+    "spec": "test/**/*.test.ts",
+    "require": ["test/global-teardown.ts"],
+    "timeout": 10000
   }
diff --git a/packages/inbound-email-handler/src/redis_data_source.ts b/packages/inbound-email-handler/src/redis_data_source.ts
index b8ca0ce6b..05bf8d26c 100644
--- a/packages/inbound-email-handler/src/redis_data_source.ts
+++ b/packages/inbound-email-handler/src/redis_data_source.ts
@@ -81,12 +81,9 @@ export const redisDataSource = new RedisDataSource({
 const gracefulShutdown = async (signal: string) => {
   console.log(`Received ${signal}, shutting down gracefully...`)
 
-  try {
-    await redisDataSource.shutdown()
-    console.log('redis shutdown successfully')
-  } catch (error) {
-    console.error('error while shutting down redis', error)
-  }
+  await redisDataSource.shutdown()
+  console.log('redis shutdown successfully')
+
   process.exit(0)
 }
 
diff --git a/packages/inbound-email-handler/test/global-teardown.ts b/packages/inbound-email-handler/test/global-teardown.ts
new file mode 100644
index 000000000..cb733d056
--- /dev/null
+++ b/packages/inbound-email-handler/test/global-teardown.ts
@@ -0,0 +1,5 @@
+import { redisDataSource } from '../src/redis_data_source'
+
+export const mochaGlobalTeardown = async () => {
+  await redisDataSource.shutdown()
+}

From 66c1f86ab5179d0e460b62a1e25c91e785db8cc7 Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Thu, 11 Apr 2024 11:28:40 +0800
Subject: [PATCH 25/74] fix dependecies

---
 packages/inbound-email-handler/package.json |  1 +
 yarn.lock                                   | 10 ++++++++++
 2 files changed, 11 insertions(+)

diff --git a/packages/inbound-email-handler/package.json b/packages/inbound-email-handler/package.json
index b2aa9ba52..892f4040e 100644
--- a/packages/inbound-email-handler/package.json
+++ b/packages/inbound-email-handler/package.json
@@ -22,6 +22,7 @@
   },
   "devDependencies": {
     "@types/addressparser": "^1.0.1",
+    "@types/chai": "^4.3.6",
     "@types/json-bigint": "^1.0.1",
     "@types/node": "^14.11.2",
     "@types/rfc2047": "^2.0.1",
diff --git a/yarn.lock b/yarn.lock
index 46b7be829..ea4826013 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -7781,6 +7781,11 @@
   resolved "https://registry.yarnpkg.com/@types/chai/-/chai-4.3.4.tgz#e913e8175db8307d78b4e8fa690408ba6b65dee4"
   integrity sha512-KnRanxnpfpjUTqTCXslZSEdLfXExwgNxYPdiO2WGUj8+HDjFi8R3k5RVKPeSCzLjCcshCAtVO2QBbVuAV4kTnw==
 
+"@types/chai@^4.3.6":
+  version "4.3.14"
+  resolved "https://registry.yarnpkg.com/@types/chai/-/chai-4.3.14.tgz#ae3055ea2be43c91c9fd700a36d67820026d96e6"
+  integrity sha512-Wj71sXE4Q4AkGdG9Tvq1u/fquNz9EdG4LIJMwVVII7ashjD/8cf8fyIfJAjRr6YcsXnSE8cOGQPq1gqeR8z+3w==
+
 "@types/chrome@^0.0.197":
   version "0.0.197"
   resolved "https://registry.yarnpkg.com/@types/chrome/-/chrome-0.0.197.tgz#c1b50cdb72ee40f9bc1411506031a9f8a925ab35"
@@ -26576,6 +26581,11 @@ rc@^1.2.7, rc@^1.2.8:
     minimist "^1.2.0"
     strip-json-comments "~2.0.1"
 
+re-resizable@^6.9.11:
+  version "6.9.11"
+  resolved "https://registry.yarnpkg.com/re-resizable/-/re-resizable-6.9.11.tgz#f356e27877f12d926d076ab9ad9ff0b95912b475"
+  integrity sha512-a3hiLWck/NkmyLvGWUuvkAmN1VhwAz4yOhS6FdMTaxCUVN9joIWkT11wsO68coG/iEYuwn+p/7qAmfQzRhiPLQ==
+
 react-color@^2.19.3:
   version "2.19.3"
   resolved "https://registry.yarnpkg.com/react-color/-/react-color-2.19.3.tgz#ec6c6b4568312a3c6a18420ab0472e146aa5683d"

From fd2f60ae3d4c95cf78dd118f8a12c9bb5374657e Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Thu, 11 Apr 2024 11:48:50 +0800
Subject: [PATCH 26/74] fix dependecies

---
 packages/inbound-email-handler/package.json | 1 +
 yarn.lock                                   | 5 +++++
 2 files changed, 6 insertions(+)

diff --git a/packages/inbound-email-handler/package.json b/packages/inbound-email-handler/package.json
index 892f4040e..2581a61a2 100644
--- a/packages/inbound-email-handler/package.json
+++ b/packages/inbound-email-handler/package.json
@@ -24,6 +24,7 @@
     "@types/addressparser": "^1.0.1",
     "@types/chai": "^4.3.6",
     "@types/json-bigint": "^1.0.1",
+    "@types/mocha": "^10.0.0",
     "@types/node": "^14.11.2",
     "@types/rfc2047": "^2.0.1",
     "@types/showdown": "^2.0.1",
diff --git a/yarn.lock b/yarn.lock
index ea4826013..1e9489503 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -8324,6 +8324,11 @@
   resolved "https://registry.yarnpkg.com/@types/minimist/-/minimist-1.2.5.tgz#ec10755e871497bcd83efe927e43ec46e8c0747e"
   integrity sha512-hov8bUuiLiyFPGyFPE1lwWhmzYbirOXQNNo40+y3zow8aFVTeyn3VWL0VFFfdNddA8S4Vf0Tc062rzyNr7Paag==
 
+"@types/mocha@^10.0.0":
+  version "10.0.6"
+  resolved "https://registry.yarnpkg.com/@types/mocha/-/mocha-10.0.6.tgz#818551d39113081048bdddbef96701b4e8bb9d1b"
+  integrity sha512-dJvrYWxP/UcXm36Qn36fxhUKu8A/xMRXVT2cliFF1Z7UA9liG5Psj3ezNSZw+5puH2czDXRLcXQxf8JbJt0ejg==
+
 "@types/mocha@^10.0.1":
   version "10.0.1"
   resolved "https://registry.yarnpkg.com/@types/mocha/-/mocha-10.0.1.tgz#2f4f65bb08bc368ac39c96da7b2f09140b26851b"

From 6278f5fa98cc3e3380646ec9753a797755ff8fc6 Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Thu, 11 Apr 2024 16:51:42 +0800
Subject: [PATCH 27/74] remove ffmpeg dependency

---
 packages/text-to-speech/package.json |  3 -
 yarn.lock                            | 82 +++-------------------------
 2 files changed, 7 insertions(+), 78 deletions(-)

diff --git a/packages/text-to-speech/package.json b/packages/text-to-speech/package.json
index f8e669fc9..bd97a0260 100644
--- a/packages/text-to-speech/package.json
+++ b/packages/text-to-speech/package.json
@@ -23,7 +23,6 @@
     "deploy": "yarn build && yarn gcloud-deploy"
   },
   "devDependencies": {
-    "@types/fluent-ffmpeg": "^2.1.20",
     "@types/html-to-text": "^8.1.1",
     "@types/natural": "^5.1.1",
     "@types/node": "^14.11.2",
@@ -33,13 +32,11 @@
     "mocha": "^10.0.0"
   },
   "dependencies": {
-    "@ffmpeg-installer/ffmpeg": "^1.1.0",
     "@google-cloud/functions-framework": "3.1.2",
     "@google-cloud/storage": "^7.0.1",
     "@sentry/serverless": "^7.77.0",
     "axios": "^0.27.2",
     "dotenv": "^16.0.1",
-    "fluent-ffmpeg": "^2.1.2",
     "html-to-text": "^8.2.1",
     "ioredis": "^5.3.2",
     "jsonwebtoken": "^8.5.1",
diff --git a/yarn.lock b/yarn.lock
index 074fd4c76..71a8f77bc 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -2633,60 +2633,6 @@
   dependencies:
     text-decoding "^1.0.0"
 
-"@ffmpeg-installer/darwin-arm64@4.1.5":
-  version "4.1.5"
-  resolved "https://registry.yarnpkg.com/@ffmpeg-installer/darwin-arm64/-/darwin-arm64-4.1.5.tgz#b7b5c262dd96d1aea4807514e1cdcf6e11f82743"
-  integrity sha512-hYqTiP63mXz7wSQfuqfFwfLOfwwFChUedeCVKkBtl/cliaTM7/ePI9bVzfZ2c+dWu3TqCwLDRWNSJ5pqZl8otA==
-
-"@ffmpeg-installer/darwin-x64@4.1.0":
-  version "4.1.0"
-  resolved "https://registry.yarnpkg.com/@ffmpeg-installer/darwin-x64/-/darwin-x64-4.1.0.tgz#48e1706c690e628148482bfb64acb67472089aaa"
-  integrity sha512-Z4EyG3cIFjdhlY8wI9aLUXuH8nVt7E9SlMVZtWvSPnm2sm37/yC2CwjUzyCQbJbySnef1tQwGG2Sx+uWhd9IAw==
-
-"@ffmpeg-installer/ffmpeg@^1.1.0":
-  version "1.1.0"
-  resolved "https://registry.yarnpkg.com/@ffmpeg-installer/ffmpeg/-/ffmpeg-1.1.0.tgz#87fdb9e7d180e8d78f7903f9441e36f978938a90"
-  integrity sha512-Uq4rmwkdGxIa9A6Bd/VqqYbT7zqh1GrT5/rFwCwKM70b42W5gIjWeVETq6SdcL0zXqDtY081Ws/iJWhr1+xvQg==
-  optionalDependencies:
-    "@ffmpeg-installer/darwin-arm64" "4.1.5"
-    "@ffmpeg-installer/darwin-x64" "4.1.0"
-    "@ffmpeg-installer/linux-arm" "4.1.3"
-    "@ffmpeg-installer/linux-arm64" "4.1.4"
-    "@ffmpeg-installer/linux-ia32" "4.1.0"
-    "@ffmpeg-installer/linux-x64" "4.1.0"
-    "@ffmpeg-installer/win32-ia32" "4.1.0"
-    "@ffmpeg-installer/win32-x64" "4.1.0"
-
-"@ffmpeg-installer/linux-arm64@4.1.4":
-  version "4.1.4"
-  resolved "https://registry.yarnpkg.com/@ffmpeg-installer/linux-arm64/-/linux-arm64-4.1.4.tgz#7219f3f901bb67f7926cb060b56b6974a6cad29f"
-  integrity sha512-dljEqAOD0oIM6O6DxBW9US/FkvqvQwgJ2lGHOwHDDwu/pX8+V0YsDL1xqHbj1DMX/+nP9rxw7G7gcUvGspSoKg==
-
-"@ffmpeg-installer/linux-arm@4.1.3":
-  version "4.1.3"
-  resolved "https://registry.yarnpkg.com/@ffmpeg-installer/linux-arm/-/linux-arm-4.1.3.tgz#c554f105ed5f10475ec25d7bec94926ce18db4c1"
-  integrity sha512-NDf5V6l8AfzZ8WzUGZ5mV8O/xMzRag2ETR6+TlGIsMHp81agx51cqpPItXPib/nAZYmo55Bl2L6/WOMI3A5YRg==
-
-"@ffmpeg-installer/linux-ia32@4.1.0":
-  version "4.1.0"
-  resolved "https://registry.yarnpkg.com/@ffmpeg-installer/linux-ia32/-/linux-ia32-4.1.0.tgz#adad70b0d0d9d8d813983d6e683c5a338a75e442"
-  integrity sha512-0LWyFQnPf+Ij9GQGD034hS6A90URNu9HCtQ5cTqo5MxOEc7Rd8gLXrJvn++UmxhU0J5RyRE9KRYstdCVUjkNOQ==
-
-"@ffmpeg-installer/linux-x64@4.1.0":
-  version "4.1.0"
-  resolved "https://registry.yarnpkg.com/@ffmpeg-installer/linux-x64/-/linux-x64-4.1.0.tgz#b4a5d89c4e12e6d9306dbcdc573df716ec1c4323"
-  integrity sha512-Y5BWhGLU/WpQjOArNIgXD3z5mxxdV8c41C+U15nsE5yF8tVcdCGet5zPs5Zy3Ta6bU7haGpIzryutqCGQA/W8A==
-
-"@ffmpeg-installer/win32-ia32@4.1.0":
-  version "4.1.0"
-  resolved "https://registry.yarnpkg.com/@ffmpeg-installer/win32-ia32/-/win32-ia32-4.1.0.tgz#6eac4fb691b64c02e7a116c1e2d167f3e9b40638"
-  integrity sha512-FV2D7RlaZv/lrtdhaQ4oETwoFUsUjlUiasiZLDxhEUPdNDWcH1OU9K1xTvqz+OXLdsmYelUDuBS/zkMOTtlUAw==
-
-"@ffmpeg-installer/win32-x64@4.1.0":
-  version "4.1.0"
-  resolved "https://registry.yarnpkg.com/@ffmpeg-installer/win32-x64/-/win32-x64-4.1.0.tgz#17e8699b5798d4c60e36e2d6326a8ebe5e95a2c5"
-  integrity sha512-Drt5u2vzDnIONf4ZEkKtFlbvwj6rI3kxw1Ck9fpudmtgaZIHD4ucsWB2lCZBXRxJgXR+2IMSti+4rtM4C4rXgg==
-
 "@firebase/app-types@0.9.0":
   version "0.9.0"
   resolved "https://registry.yarnpkg.com/@firebase/app-types/-/app-types-0.9.0.tgz#35b5c568341e9e263b29b3d2ba0e9cfc9ec7f01e"
@@ -7986,13 +7932,6 @@
   resolved "https://registry.yarnpkg.com/@types/firefox-webext-browser/-/firefox-webext-browser-94.0.1.tgz#52afb975253dc0fd350d5d58c7fe9fd1a01f64a1"
   integrity sha512-I6iHRQJSTZ+gYt2IxdH2RRAMvcUyK8v5Ig7fHQR0IwUNYP7hz9+cziBVIKxLCO6XI7fiyRsNOWObfl3/4Js2Lg==
 
-"@types/fluent-ffmpeg@^2.1.20":
-  version "2.1.20"
-  resolved "https://registry.yarnpkg.com/@types/fluent-ffmpeg/-/fluent-ffmpeg-2.1.20.tgz#3b5f42fc8263761d58284fa46ee6759a64ce54ac"
-  integrity sha512-B+OvhCdJ3LgEq2PhvWNOiB/EfwnXLElfMCgc4Z1K5zXgSfo9I6uGKwR/lqmNPFQuebNnes7re3gqkV77SyypLg==
-  dependencies:
-    "@types/node" "*"
-
 "@types/fs-extra@^11.0.1":
   version "11.0.1"
   resolved "https://registry.yarnpkg.com/@types/fs-extra/-/fs-extra-11.0.1.tgz#f542ec47810532a8a252127e6e105f487e0a6ea5"
@@ -10404,11 +10343,6 @@ async-validator@^4.1.0:
   resolved "https://registry.yarnpkg.com/async-validator/-/async-validator-4.2.5.tgz#c96ea3332a521699d0afaaceed510a54656c6339"
   integrity sha512-7HhHjtERjqlNbZtqNqy2rckN/SpOOlmDliet+lP7k+eKZEjPk3DgyeU9lIXLdeLz0uBbbVp+9Qdow9wJWgwwfg==
 
-async@>=0.2.9:
-  version "3.2.4"
-  resolved "https://registry.yarnpkg.com/async/-/async-3.2.4.tgz#2d22e00f8cddeb5fde5dd33522b56d1cf569a81c"
-  integrity sha512-iAB+JbDEGXhyIUavoDl9WP/Jj106Kz9DEn1DPgYw5ruDn0e3Wgi3sKFm55sASdGBNOQB8F59d9qQ7deqrHA8wQ==
-
 async@^2.6.2:
   version "2.6.4"
   resolved "https://registry.yarnpkg.com/async/-/async-2.6.4.tgz#706b7ff6084664cd7eae713f6f965433b5504221"
@@ -11679,6 +11613,7 @@ caniuse-lite@^1.0.30001109, caniuse-lite@^1.0.30001251, caniuse-lite@^1.0.300012
   version "1.0.30001600"
   resolved "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001600.tgz"
   integrity sha512-+2S9/2JFhYmYaDpZvo0lKkfvuKIglrx68MwOBqMGHhQsNkLjB5xtc/TGoEPs+MxjSyN/72qer2g97nzR641mOQ==
+
 capital-case@^1.0.4:
   version "1.0.4"
   resolved "https://registry.yarnpkg.com/capital-case/-/capital-case-1.0.4.tgz#9d130292353c9249f6b00fa5852bee38a717e669"
@@ -16064,14 +15999,6 @@ flatted@^3.1.0:
   resolved "https://registry.yarnpkg.com/flatted/-/flatted-3.2.2.tgz#64bfed5cb68fe3ca78b3eb214ad97b63bedce561"
   integrity sha512-JaTY/wtrcSyvXJl4IMFHPKyFur1sE9AUqc0QnhOaJ0CxHtAoIV8pYDzeEfAaNEtGkOfq4gr3LBFmdXW5mOQFnA==
 
-fluent-ffmpeg@^2.1.2:
-  version "2.1.2"
-  resolved "https://registry.yarnpkg.com/fluent-ffmpeg/-/fluent-ffmpeg-2.1.2.tgz#c952de2240f812ebda0aa8006d7776ee2acf7d74"
-  integrity sha512-IZTB4kq5GK0DPp7sGQ0q/BWurGHffRtQQwVkiqDgeO6wYJLLV5ZhgNOQ65loZxxuPMKZKZcICCUnaGtlxBiR0Q==
-  dependencies:
-    async ">=0.2.9"
-    which "^1.1.1"
-
 flush-write-stream@^1.0.0:
   version "1.1.1"
   resolved "https://registry.yarnpkg.com/flush-write-stream/-/flush-write-stream-1.1.1.tgz#8dd7d873a1babc207d94ead0c2e0e44276ebf2e8"
@@ -26575,6 +26502,11 @@ rc@^1.2.7, rc@^1.2.8:
     minimist "^1.2.0"
     strip-json-comments "~2.0.1"
 
+re-resizable@^6.9.11:
+  version "6.9.11"
+  resolved "https://registry.yarnpkg.com/re-resizable/-/re-resizable-6.9.11.tgz#f356e27877f12d926d076ab9ad9ff0b95912b475"
+  integrity sha512-a3hiLWck/NkmyLvGWUuvkAmN1VhwAz4yOhS6FdMTaxCUVN9joIWkT11wsO68coG/iEYuwn+p/7qAmfQzRhiPLQ==
+
 react-color@^2.19.3:
   version "2.19.3"
   resolved "https://registry.yarnpkg.com/react-color/-/react-color-2.19.3.tgz#ec6c6b4568312a3c6a18420ab0472e146aa5683d"
@@ -31764,7 +31696,7 @@ which@2.0.2, which@^2.0.1, which@^2.0.2:
   dependencies:
     isexe "^2.0.0"
 
-which@^1.1.1, which@^1.2.14, which@^1.2.9:
+which@^1.2.14, which@^1.2.9:
   version "1.3.1"
   resolved "https://registry.yarnpkg.com/which/-/which-1.3.1.tgz#a45043d54f5805316da8d62f9f50918d3da70b0a"
   integrity sha512-HxJdYWq1MTIQbJ3nw0cqssHoTNU267KlrDuGZ1WYlxDStUtKUhOaJmh112/TZmHxxUfuJqPXSOm7tDyas0OSIQ==

From 54e7c52b5ea80d27e2e2b322063274d5ac1270b7 Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Thu, 11 Apr 2024 18:07:53 +0800
Subject: [PATCH 28/74] always update the state of the pdf item to
 succeeded/failed, even if we fail to parse the content

---
 .../api/src/services/update_pdf_content.ts    |  21 ++--
 packages/api/src/services/upload_file.ts      |  26 ++---
 packages/pdf-handler/src/index.ts             | 105 +++++++++++-------
 packages/pdf-handler/src/job.ts               |   5 +-
 4 files changed, 91 insertions(+), 66 deletions(-)

diff --git a/packages/api/src/services/update_pdf_content.ts b/packages/api/src/services/update_pdf_content.ts
index 75d3e8438..4c337cfb4 100644
--- a/packages/api/src/services/update_pdf_content.ts
+++ b/packages/api/src/services/update_pdf_content.ts
@@ -8,16 +8,17 @@ import { findUploadFileById, setFileUploadComplete } from './upload_file'
 
 export interface UpdateContentMessage {
   fileId: string
-  content: string
+  content?: string
   title?: string
   author?: string
   description?: string
+  state?: LibraryItemState
 }
 
 export const isUpdateContentMessage = (
   data: any
 ): data is UpdateContentMessage => {
-  return 'fileId' in data && 'content' in data
+  return 'fileId' in data
 }
 
 export const updateContentForFileItem = async (msg: UpdateContentMessage) => {
@@ -51,16 +52,13 @@ export const updateContentForFileItem = async (msg: UpdateContentMessage) => {
   }
 
   const itemToUpdate: QueryDeepPartialEntity<LibraryItem> = {
+    ...msg,
     originalContent: msg.content,
+    // This event is fired after the file is fully uploaded,
+    // so along with updating content, we mark it as
+    // succeeded or failed based on the message state
+    state: msg.state || LibraryItemState.Succeeded,
   }
-  if (msg.title) itemToUpdate.title = msg.title
-  if (msg.author) itemToUpdate.author = msg.author
-  if (msg.description) itemToUpdate.description = msg.description
-
-  // This event is fired after the file is fully uploaded,
-  // so along with updating content, we mark it as
-  // succeeded.
-  itemToUpdate.state = LibraryItemState.Succeeded
 
   try {
     const uploadFileData = await setFileUploadComplete(
@@ -80,7 +78,8 @@ export const updateContentForFileItem = async (msg: UpdateContentMessage) => {
   logger.info('Updating library item text', {
     id: libraryItem.id,
     result,
-    content: msg.content.substring(0, 20),
+    content: msg.content?.substring(0, 20),
+    state: msg.state,
   })
 
   return true
diff --git a/packages/api/src/services/upload_file.ts b/packages/api/src/services/upload_file.ts
index 1d284916f..148614352 100644
--- a/packages/api/src/services/upload_file.ts
+++ b/packages/api/src/services/upload_file.ts
@@ -18,6 +18,7 @@ import {
 } from '../utils/uploads'
 import { validateUrl } from './create_page_save_request'
 import { createOrUpdateLibraryItem } from './library_item'
+import { v4 as uuid } from 'uuid'
 
 const isFileUrl = (url: string): boolean => {
   const parsedUrl = new URL(url)
@@ -90,8 +91,18 @@ export const uploadFile = async (
     }
   }
 
+  let url = input.url
+
+  const uploadFileId = uuid()
+  const uploadFilePathName = generateUploadFilePathName(uploadFileId, fileName)
+  // If this is a file URL, we swap in a special URL
+  if (isFileUrl(url)) {
+    url = `https://omnivore.app/attachments/${uploadFilePathName}`
+  }
+
   const uploadFileData = await authTrx((t) =>
     t.getRepository(UploadFile).save({
+      id: uploadFileId,
       url: input.url,
       user: { id: uid },
       fileName,
@@ -99,24 +110,11 @@ export const uploadFile = async (
       contentType: input.contentType,
     })
   )
-  const uploadFileId = uploadFileData.id
-  const uploadFilePathName = generateUploadFilePathName(uploadFileId, fileName)
   const uploadSignedUrl = await generateUploadSignedUrl(
     uploadFilePathName,
     input.contentType
   )
 
-  // If this is a file URL, we swap in a special URL
-  const attachmentUrl = `https://omnivore.app/attachments/${uploadFilePathName}`
-  if (isFileUrl(input.url)) {
-    await authTrx(async (tx) => {
-      await tx.getRepository(UploadFile).update(uploadFileId, {
-        url: attachmentUrl,
-        status: UploadFileStatus.Initialized,
-      })
-    })
-  }
-
   const itemType = itemTypeForContentType(input.contentType)
   if (input.createPageEntry) {
     // If we have a file:// URL, don't try to match it
@@ -125,7 +123,7 @@ export const uploadFile = async (
     const item = await createOrUpdateLibraryItem(
       {
         id: input.clientRequestId || undefined,
-        originalUrl: isFileUrl(input.url) ? attachmentUrl : input.url,
+        originalUrl: url,
         user: { id: uid },
         title,
         readableContent: '',
diff --git a/packages/pdf-handler/src/index.ts b/packages/pdf-handler/src/index.ts
index e1623372b..cf3f3359c 100644
--- a/packages/pdf-handler/src/index.ts
+++ b/packages/pdf-handler/src/index.ts
@@ -1,7 +1,7 @@
 import { GetSignedUrlConfig, Storage } from '@google-cloud/storage'
 import * as Sentry from '@sentry/serverless'
 import { parsePdf } from './pdf'
-import { queueUpdatePageJob } from './job'
+import { queueUpdatePageJob, State } from './job'
 
 Sentry.GCPFunction.init({
   dsn: process.env.SENTRY_DSN,
@@ -50,10 +50,11 @@ const getDocumentUrl = async (
 
 export const updatePageContent = async (
   fileId: string,
-  content: string,
+  content?: string,
   title?: string,
   author?: string,
-  description?: string
+  description?: string,
+  state?: State
 ): Promise<string | undefined> => {
   const job = await queueUpdatePageJob({
     fileId,
@@ -61,6 +62,7 @@ export const updatePageContent = async (
     title,
     author,
     description,
+    state,
   })
   return job.id
 }
@@ -86,45 +88,68 @@ export const pdfHandler = Sentry.GCPFunction.wrapHttpFunction(
     if ('message' in req.body && 'data' in req.body.message) {
       const pubSubMessage = req.body.message.data as string
       const data = getStorageEventData(pubSubMessage)
-      if (data) {
-        try {
-          if (shouldHandle(data)) {
-            console.log('handling pdf data', data)
-
-            const url = await getDocumentUrl(data)
-            console.log('PDF url: ', url)
-            if (!url) {
-              console.log('Could not fetch PDF', data.bucket, data.name)
-              return res.status(404).send('Could not fetch PDF')
-            }
-
-            const parsed = await parsePdf(url)
-            const result = await updatePageContent(
-              data.name,
-              parsed.content,
-              parsed.title,
-              parsed.author,
-              parsed.description
-            )
-            console.log(
-              'publish result',
-              result,
-              'title',
-              parsed.title,
-              'author',
-              parsed.author
-            )
-          } else {
-            console.log('not handling pdf data', data)
-          }
-        } catch (err) {
-          console.log('error handling event', { err, data })
-          return res.status(500).send('Error handling event')
-        }
+      if (!data) {
+        console.log('no data found in pubsub message')
+        return res.send('ok')
+      }
+
+      if (!shouldHandle(data)) {
+        console.log('not handling pdf data', data)
+        return res.send('ok')
+      }
+
+      console.log('handling pdf data', data)
+
+      let content,
+        title,
+        author,
+        description,
+        state: State = 'SUCCEEDED' // Default to succeeded even if we fail to parse
+
+      try {
+        const url = await getDocumentUrl(data)
+        console.log('PDF url: ', url)
+        if (!url) {
+          console.log('Could not fetch PDF', data.bucket, data.name)
+          // If we can't fetch the PDF, mark it as failed
+          state = 'FAILED'
+
+          return res.status(404).send('Could not fetch PDF')
+        }
+
+        // Parse the PDF to update the content and metadata
+        const parsed = await parsePdf(url)
+        content = parsed.content
+        title = parsed.title
+        author = parsed.author
+        description = parsed.description
+      } catch (err) {
+        console.log('error parsing pdf', { err, data })
+
+        return res.status(500).send('Error parsing pdf')
+      } finally {
+        // Always update the state, even if we fail to parse
+        const result = await updatePageContent(
+          data.name,
+          content,
+          title,
+          author,
+          description,
+          state
+        )
+        console.log(
+          'publish result',
+          result,
+          'title',
+          title,
+          'author',
+          author,
+          'state',
+          state
+        )
       }
-    } else {
-      console.log('no pubsub message')
     }
+
     res.send('ok')
   }
 )
diff --git a/packages/pdf-handler/src/job.ts b/packages/pdf-handler/src/job.ts
index d244079d3..ca404aa0f 100644
--- a/packages/pdf-handler/src/job.ts
+++ b/packages/pdf-handler/src/job.ts
@@ -8,12 +8,15 @@ const queue = new Queue(QUEUE_NAME, {
   connection: redisDataSource.queueRedisClient,
 })
 
+export type State = 'SUCCEEDED' | 'FAILED'
+
 type UpdatePageJobData = {
   fileId: string
-  content: string
+  content?: string
   title?: string
   author?: string
   description?: string
+  state?: State
 }
 
 export const queueUpdatePageJob = async (data: UpdatePageJobData) => {

From 448c78a87e4778cec22820de2cedac2e0b8b0252 Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Thu, 11 Apr 2024 18:25:24 +0800
Subject: [PATCH 29/74] add dotenv

---
 packages/api/src/services/update_pdf_content.ts | 1 +
 packages/pdf-handler/src/redis_data_source.ts   | 1 +
 2 files changed, 2 insertions(+)

diff --git a/packages/api/src/services/update_pdf_content.ts b/packages/api/src/services/update_pdf_content.ts
index 4c337cfb4..96ed85de2 100644
--- a/packages/api/src/services/update_pdf_content.ts
+++ b/packages/api/src/services/update_pdf_content.ts
@@ -53,6 +53,7 @@ export const updateContentForFileItem = async (msg: UpdateContentMessage) => {
 
   const itemToUpdate: QueryDeepPartialEntity<LibraryItem> = {
     ...msg,
+    // content may not be present if we failed to parse the file
     originalContent: msg.content,
     // This event is fired after the file is fully uploaded,
     // so along with updating content, we mark it as
diff --git a/packages/pdf-handler/src/redis_data_source.ts b/packages/pdf-handler/src/redis_data_source.ts
index 1a95f9d6e..1c02c2a47 100644
--- a/packages/pdf-handler/src/redis_data_source.ts
+++ b/packages/pdf-handler/src/redis_data_source.ts
@@ -1,4 +1,5 @@
 import Redis, { RedisOptions } from 'ioredis'
+import 'dotenv/config'
 
 export type RedisDataSourceOptions = {
   REDIS_URL?: string

From f95642be2b1b0986d0f986deeb5ff308ed64e87e Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Thu, 11 Apr 2024 18:47:19 +0800
Subject: [PATCH 30/74] remove redundant field

---
 packages/api/src/services/update_pdf_content.ts | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/packages/api/src/services/update_pdf_content.ts b/packages/api/src/services/update_pdf_content.ts
index 96ed85de2..7da584b74 100644
--- a/packages/api/src/services/update_pdf_content.ts
+++ b/packages/api/src/services/update_pdf_content.ts
@@ -52,7 +52,9 @@ export const updateContentForFileItem = async (msg: UpdateContentMessage) => {
   }
 
   const itemToUpdate: QueryDeepPartialEntity<LibraryItem> = {
-    ...msg,
+    title: msg.title,
+    description: msg.description,
+    author: msg.author,
     // content may not be present if we failed to parse the file
     originalContent: msg.content,
     // This event is fired after the file is fully uploaded,

From e23d3d5933329e0dc2ad9c39b792142c51fe0fc4 Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Fri, 12 Apr 2024 13:26:11 +0800
Subject: [PATCH 31/74] update readable content instead

---
 packages/api/src/services/update_pdf_content.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/api/src/services/update_pdf_content.ts b/packages/api/src/services/update_pdf_content.ts
index 7da584b74..c96b67dce 100644
--- a/packages/api/src/services/update_pdf_content.ts
+++ b/packages/api/src/services/update_pdf_content.ts
@@ -56,7 +56,7 @@ export const updateContentForFileItem = async (msg: UpdateContentMessage) => {
     description: msg.description,
     author: msg.author,
     // content may not be present if we failed to parse the file
-    originalContent: msg.content,
+    readableContent: msg.content,
     // This event is fired after the file is fully uploaded,
     // so along with updating content, we mark it as
     // succeeded or failed based on the message state

From c176078cf83780725cde3ebd374d235986f0d271 Mon Sep 17 00:00:00 2001
From: Jackson Harper <jacksonh@gmail.com>
Date: Sat, 13 Apr 2024 09:28:15 +0800
Subject: [PATCH 32/74] Upgrade youtubei library

Fixes an exception when fetching transcripts
---
 packages/api/package.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/api/package.json b/packages/api/package.json
index c3ec7bd1b..0da5f742b 100644
--- a/packages/api/package.json
+++ b/packages/api/package.json
@@ -113,7 +113,7 @@
     "voca": "^1.4.0",
     "winston": "^3.3.3",
     "word-counting": "^1.1.4",
-    "youtubei": "1.3.7"
+    "youtubei": "1.4.0"
   },
   "devDependencies": {
     "@istanbuljs/nyc-config-typescript": "^1.0.2",

From ed2c78165a4822fecee346b7e3341ac10a9d272a Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Mon, 15 Apr 2024 15:42:24 +0800
Subject: [PATCH 33/74] add missing dotenv dependency to pdf-handler

---
 packages/pdf-handler/package.json | 1 +
 yarn.lock                         | 8 ++++----
 2 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/packages/pdf-handler/package.json b/packages/pdf-handler/package.json
index 52cf4ac97..01ac1887f 100644
--- a/packages/pdf-handler/package.json
+++ b/packages/pdf-handler/package.json
@@ -34,6 +34,7 @@
     "axios": "^0.27.2",
     "bullmq": "^5.1.4",
     "concurrently": "^7.0.0",
+    "dotenv": "^8.2.0",
     "ioredis": "^5.3.2",
     "pdfjs-dist": "^2.9.359"
   },
diff --git a/yarn.lock b/yarn.lock
index b437e5cf1..db12191a9 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -32183,10 +32183,10 @@ yocto-queue@^1.0.0:
   resolved "https://registry.yarnpkg.com/yocto-queue/-/yocto-queue-1.0.0.tgz#7f816433fb2cbc511ec8bf7d263c3b58a1a3c251"
   integrity sha512-9bnSc/HEW2uRy67wc+T8UwauLuPJVn28jb+GtJY16iiKWyvmYJRXVT4UamsAEGQfPohgr2q4Tq0sQbQlxTfi1g==
 
-youtubei@1.3.7:
-  version "1.3.7"
-  resolved "https://registry.yarnpkg.com/youtubei/-/youtubei-1.3.7.tgz#9cb3cf67b506e09bcf6d6a9cfbdf31ad68948dff"
-  integrity sha512-DBtwjxqDccpDdSzihXltuqXp4NlG4PZoGfilUbbyrfXvRHhMLDEZO91dzTcJdLXwBBv+wovtGr2VA/OikPnw/Q==
+youtubei@1.4.0:
+  version "1.4.0"
+  resolved "https://registry.yarnpkg.com/youtubei/-/youtubei-1.4.0.tgz#a853080a292ab1a002c2658929cb8edd9e756fda"
+  integrity sha512-n3/f+46Q91p/Rfso73g9IHtmHhpW7z6ML5mELdeYY0BXsh757KFDvTT91e7RCzUblrSnLiKGMyO3UM4hIUJFsw==
   dependencies:
     node-fetch "2.6.7"
     protobufjs "7.2.4"

From 4346746e0ac86ff85af526f5a6c1e3b8c7f86748 Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Mon, 15 Apr 2024 17:36:42 +0800
Subject: [PATCH 34/74] save empty url for email attachment in upload_file
 table as it is required

---
 packages/api/src/generated/graphql.ts            |  2 --
 packages/api/src/generated/schema.graphql        |  1 -
 packages/api/src/jobs/email/inbound_emails.ts    |  1 +
 packages/api/src/resolvers/function_resolvers.ts | 11 -----------
 packages/api/src/schema.ts                       |  1 -
 5 files changed, 1 insertion(+), 15 deletions(-)

diff --git a/packages/api/src/generated/graphql.ts b/packages/api/src/generated/graphql.ts
index cd4b67b31..ee17f1843 100644
--- a/packages/api/src/generated/graphql.ts
+++ b/packages/api/src/generated/graphql.ts
@@ -3734,7 +3734,6 @@ export type User = {
   __typename?: 'User';
   email?: Maybe<Scalars['String']>;
   featureList?: Maybe<Array<Feature>>;
-  features?: Maybe<Array<Maybe<Scalars['String']>>>;
   followersCount?: Maybe<Scalars['Int']>;
   friendsCount?: Maybe<Scalars['Int']>;
   id: Scalars['ID'];
@@ -7203,7 +7202,6 @@ export type UploadImportFileSuccessResolvers<ContextType = ResolverContext, Pare
 export type UserResolvers<ContextType = ResolverContext, ParentType extends ResolversParentTypes['User'] = ResolversParentTypes['User']> = {
   email?: Resolver<Maybe<ResolversTypes['String']>, ParentType, ContextType>;
   featureList?: Resolver<Maybe<Array<ResolversTypes['Feature']>>, ParentType, ContextType>;
-  features?: Resolver<Maybe<Array<Maybe<ResolversTypes['String']>>>, ParentType, ContextType>;
   followersCount?: Resolver<Maybe<ResolversTypes['Int']>, ParentType, ContextType>;
   friendsCount?: Resolver<Maybe<ResolversTypes['Int']>, ParentType, ContextType>;
   id?: Resolver<ResolversTypes['ID'], ParentType, ContextType>;
diff --git a/packages/api/src/generated/schema.graphql b/packages/api/src/generated/schema.graphql
index e48fe3cff..2e6be1a96 100644
--- a/packages/api/src/generated/schema.graphql
+++ b/packages/api/src/generated/schema.graphql
@@ -3014,7 +3014,6 @@ enum UploadImportFileType {
 type User {
   email: String
   featureList: [Feature!]
-  features: [String]
   followersCount: Int
   friendsCount: Int
   id: ID!
diff --git a/packages/api/src/jobs/email/inbound_emails.ts b/packages/api/src/jobs/email/inbound_emails.ts
index db8fc02eb..938b388b4 100644
--- a/packages/api/src/jobs/email/inbound_emails.ts
+++ b/packages/api/src/jobs/email/inbound_emails.ts
@@ -250,6 +250,7 @@ export const saveAttachmentJob = async (data: EmailJobData) => {
     (tx) =>
       tx.getRepository(UploadFile).save({
         ...uploadFile,
+        url: '', // no url for email attachments
         status: UploadFileStatus.Completed,
         user: { id: user.id },
       }),
diff --git a/packages/api/src/resolvers/function_resolvers.ts b/packages/api/src/resolvers/function_resolvers.ts
index 95bd3e219..0756bf07e 100644
--- a/packages/api/src/resolvers/function_resolvers.ts
+++ b/packages/api/src/resolvers/function_resolvers.ts
@@ -384,17 +384,6 @@ export const functionResolvers = {
 
       return findUserFeatures(ctx.claims.uid)
     },
-    async features(
-      user: User,
-      __: Record<string, unknown>,
-      ctx: WithDataSourcesContext
-    ) {
-      if (!ctx.claims?.uid) {
-        return undefined
-      }
-
-      return (await findUserFeatures(ctx.claims.uid)).map((f) => f.name)
-    },
   },
   Article: {
     async url(article: Article, _: unknown, ctx: WithDataSourcesContext) {
diff --git a/packages/api/src/schema.ts b/packages/api/src/schema.ts
index acae6f520..71192074b 100755
--- a/packages/api/src/schema.ts
+++ b/packages/api/src/schema.ts
@@ -88,7 +88,6 @@ const schema = gql`
     email: String
     source: String
     intercomHash: String
-    features: [String]
     featureList: [Feature!]
   }
 

From 9ccd1e90fe3ad8ec0a7fab56926f438d0d558b6e Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Mon, 15 Apr 2024 18:54:15 +0800
Subject: [PATCH 35/74] save received email as non-article by default

---
 packages/api/src/jobs/email/inbound_emails.ts | 29 +++++++++----------
 .../api/src/resolvers/recent_emails/index.ts  |  3 --
 2 files changed, 13 insertions(+), 19 deletions(-)

diff --git a/packages/api/src/jobs/email/inbound_emails.ts b/packages/api/src/jobs/email/inbound_emails.ts
index 938b388b4..ab953e6af 100644
--- a/packages/api/src/jobs/email/inbound_emails.ts
+++ b/packages/api/src/jobs/email/inbound_emails.ts
@@ -94,7 +94,7 @@ export const forwardEmailJob = async (data: EmailJobData) => {
     )
   ) {
     logger.info('handling as article')
-    const savedNewsletter = await saveNewsletter(
+    return saveNewsletter(
       {
         title: getTitleFromEmailSubject(subject),
         author: parsedFrom.name || from,
@@ -105,15 +105,6 @@ export const forwardEmailJob = async (data: EmailJobData) => {
       },
       newsletterEmail
     )
-    if (!savedNewsletter) {
-      logger.error('Failed to save email', { from, to, subject })
-      return false
-    }
-
-    // update received email type
-    await updateReceivedEmail(receivedEmailId, 'article', user.id)
-
-    return true
   }
 
   analytics.capture({
@@ -166,7 +157,7 @@ export const saveNewsletterJob = async (data: EmailJobData) => {
     text,
     html,
     user.id,
-    'article',
+    'non-article', // default to non-article
     replyTo
   )
 
@@ -178,6 +169,11 @@ export const saveNewsletterJob = async (data: EmailJobData) => {
       newsletterEmail.user.id
     )
 
+    if (result) {
+      // update received email type
+      await updateReceivedEmail(receivedEmailId, 'article', user.id)
+    }
+
     return result
   }
 
@@ -205,7 +201,7 @@ export const saveNewsletterJob = async (data: EmailJobData) => {
   }
 
   // save newsletter instead
-  const result = await saveNewsletter(
+  return saveNewsletter(
     {
       email: newsletterEmail.address,
       content,
@@ -219,8 +215,6 @@ export const saveNewsletterJob = async (data: EmailJobData) => {
     },
     newsletterEmail
   )
-
-  return result
 }
 
 export const saveAttachmentJob = async (data: EmailJobData) => {
@@ -235,14 +229,14 @@ export const saveAttachmentJob = async (data: EmailJobData) => {
   }
 
   const user = newsletterEmail.user
-  await saveReceivedEmail(
+  const receivedEmail = await saveReceivedEmail(
     from,
     to,
     subject,
     text,
     html,
     user.id,
-    'article',
+    'non-article',
     replyTo
   )
 
@@ -293,6 +287,9 @@ export const saveAttachmentJob = async (data: EmailJobData) => {
 
   await createOrUpdateLibraryItem(itemToCreate, user.id)
 
+  // update received email type
+  await updateReceivedEmail(receivedEmail.id, 'article', user.id)
+
   return true
 }
 
diff --git a/packages/api/src/resolvers/recent_emails/index.ts b/packages/api/src/resolvers/recent_emails/index.ts
index b8bbd275c..cbe6b3329 100644
--- a/packages/api/src/resolvers/recent_emails/index.ts
+++ b/packages/api/src/resolvers/recent_emails/index.ts
@@ -102,9 +102,6 @@ export const markEmailAsItemResolver = authorized<
     }
   }
 
-  // update received email type
-  await updateReceivedEmail(recentEmail.id, 'article', uid)
-
   const text = `A recent email marked as a library item
                     by: ${uid}
                     from: ${recentEmail.from}

From a9779a51e81bee9182c8d4e893e338e6097ff706 Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Tue, 16 Apr 2024 09:56:57 +0800
Subject: [PATCH 36/74] fix: LABEL_ALREADY_EXIST bug occured during label
 creation * comparing both user_id and label name when looking for an existing
 matching label

---
 packages/api/src/repository/label.ts       |  7 ++++--
 packages/api/src/resolvers/labels/index.ts |  2 +-
 packages/api/test/resolvers/labels.test.ts | 28 +++++++++++++++++++++-
 3 files changed, 33 insertions(+), 4 deletions(-)

diff --git a/packages/api/src/repository/label.ts b/packages/api/src/repository/label.ts
index baaac24bb..0faebd3db 100644
--- a/packages/api/src/repository/label.ts
+++ b/packages/api/src/repository/label.ts
@@ -46,9 +46,12 @@ export const labelRepository = appDataSource.getRepository(Label).extend({
     return this.findOneBy({ id })
   },
 
-  findByName(name: string) {
+  findByName(name: string, userId: string) {
     return this.createQueryBuilder()
-      .where('LOWER(name) = LOWER(:name)', { name }) // case insensitive
+      .where('user_id = :userId AND LOWER(name) = LOWER(:name)', {
+        name,
+        userId,
+      }) // case insensitive
       .getOne()
   },
 
diff --git a/packages/api/src/resolvers/labels/index.ts b/packages/api/src/resolvers/labels/index.ts
index 1da2f3053..cd5ec824b 100644
--- a/packages/api/src/resolvers/labels/index.ts
+++ b/packages/api/src/resolvers/labels/index.ts
@@ -90,7 +90,7 @@ export const createLabelResolver = authorized<
   CreateLabelError,
   MutationCreateLabelArgs
 >(async (_, { input }, { authTrx, uid }) => {
-  const existingLabel = await labelRepository.findByName(input.name)
+  const existingLabel = await labelRepository.findByName(input.name, uid)
   if (existingLabel) {
     return {
       errorCodes: [CreateLabelErrorCode.LabelAlreadyExists],
diff --git a/packages/api/test/resolvers/labels.test.ts b/packages/api/test/resolvers/labels.test.ts
index 8d1175c78..0dccc54d1 100644
--- a/packages/api/test/resolvers/labels.test.ts
+++ b/packages/api/test/resolvers/labels.test.ts
@@ -145,7 +145,7 @@ describe('Labels API', () => {
       })
     })
 
-    context('when name exists', () => {
+    context('when name exists in the user library', () => {
       let existingLabel: Label
 
       before(async () => {
@@ -177,6 +177,32 @@ describe('Labels API', () => {
       })
     })
 
+    context('when name exists in the other user library', () => {
+      let existingLabel: Label
+      let otherUser: User
+
+      before(async () => {
+        otherUser = await createTestUser('otherUser')
+        existingLabel = await createLabel('label3', '#ffffff', otherUser.id)
+      })
+
+      after(async () => {
+        // delete other user will also delete the label
+        await deleteUser(otherUser.id)
+      })
+
+      it('creates the label', async () => {
+        const res = await graphqlRequest(query, authToken, {
+          input: { name: existingLabel.name },
+        }).expect(200)
+        const label = await findLabelById(
+          res.body.data.createLabel.label.id,
+          user.id
+        )
+        expect(label).to.exist
+      })
+    })
+
     it('responds status code 400 when invalid query', async () => {
       const invalidQuery = `
         mutation {

From d79cc1c39fedd5c405c17165bce6240b3034893a Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Tue, 16 Apr 2024 11:26:29 +0800
Subject: [PATCH 37/74] wrap select in a auth transaction

---
 packages/api/src/resolvers/labels/index.ts | 17 +++++++++++------
 1 file changed, 11 insertions(+), 6 deletions(-)

diff --git a/packages/api/src/resolvers/labels/index.ts b/packages/api/src/resolvers/labels/index.ts
index cd5ec824b..2e3b9c6d2 100644
--- a/packages/api/src/resolvers/labels/index.ts
+++ b/packages/api/src/resolvers/labels/index.ts
@@ -90,17 +90,22 @@ export const createLabelResolver = authorized<
   CreateLabelError,
   MutationCreateLabelArgs
 >(async (_, { input }, { authTrx, uid }) => {
-  const existingLabel = await labelRepository.findByName(input.name, uid)
-  if (existingLabel) {
+  const label = await authTrx(async (tx) => {
+    const repo = tx.withRepository(labelRepository)
+    const existingLabel = await repo.findByName(input.name, uid)
+    if (existingLabel) {
+      return null
+    }
+
+    return repo.createLabel(input, uid)
+  })
+
+  if (!label) {
     return {
       errorCodes: [CreateLabelErrorCode.LabelAlreadyExists],
     }
   }
 
-  const label = await authTrx(async (tx) =>
-    tx.withRepository(labelRepository).createLabel(input, uid)
-  )
-
   analytics.capture({
     distinctId: uid,
     event: 'label_created',

From 6ce95194ee20443c8d716b9d28a2f3e33a2718cb Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Tue, 16 Apr 2024 11:41:19 +0800
Subject: [PATCH 38/74] alter read_labels policy to check user_id

---
 .../migrations/0173.do.alter_labels_table_policy.sql  | 11 +++++++++++
 .../0173.undo.alter_labels_table_policy.sql           | 11 +++++++++++
 2 files changed, 22 insertions(+)
 create mode 100755 packages/db/migrations/0173.do.alter_labels_table_policy.sql
 create mode 100755 packages/db/migrations/0173.undo.alter_labels_table_policy.sql

diff --git a/packages/db/migrations/0173.do.alter_labels_table_policy.sql b/packages/db/migrations/0173.do.alter_labels_table_policy.sql
new file mode 100755
index 000000000..1e7f4afdb
--- /dev/null
+++ b/packages/db/migrations/0173.do.alter_labels_table_policy.sql
@@ -0,0 +1,11 @@
+-- Type: DO
+-- Name: alter_labels_table_policy
+-- Description: Alter labels table select policy to check user_id
+
+BEGIN;
+
+ALTER POLICY read_labels ON omnivore.labels
+    TO omnivore_user
+    USING (user_id = omnivore.get_current_user_id());
+
+COMMIT;
diff --git a/packages/db/migrations/0173.undo.alter_labels_table_policy.sql b/packages/db/migrations/0173.undo.alter_labels_table_policy.sql
new file mode 100755
index 000000000..552a9e2c7
--- /dev/null
+++ b/packages/db/migrations/0173.undo.alter_labels_table_policy.sql
@@ -0,0 +1,11 @@
+-- Type: UNDO
+-- Name: alter_labels_table_policy
+-- Description: Alter labels table select policy to check user_id
+
+BEGIN;
+
+ALTER POLICY read_labels ON omnivore.labels
+    TO omnivore_user
+    USING (true);
+
+COMMIT;

From 8e31ac4feab5b41a6ea78e6fceab339a2178b4f7 Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Tue, 16 Apr 2024 12:24:20 +0800
Subject: [PATCH 39/74] fix tests

---
 packages/api/src/resolvers/highlight/index.ts | 9 +++++----
 packages/api/test/resolvers/highlight.test.ts | 5 ++---
 2 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/packages/api/src/resolvers/highlight/index.ts b/packages/api/src/resolvers/highlight/index.ts
index 8e0f22c9e..a142b3299 100644
--- a/packages/api/src/resolvers/highlight/index.ts
+++ b/packages/api/src/resolvers/highlight/index.ts
@@ -81,7 +81,7 @@ export const mergeHighlightResolver = authorized<
   MergeHighlightSuccess,
   MergeHighlightError,
   MutationMergeHighlightArgs
->(async (_, { input }, { log, pubsub, uid }) => {
+>(async (_, { input }, { authTrx, log, pubsub, uid }) => {
   const { overlapHighlightIdList, ...newHighlightInput } = input
 
   /* Compute merged annotation form the order of highlights appearing on page */
@@ -90,9 +90,10 @@ export const mergeHighlightResolver = authorized<
   const mergedColors: string[] = []
 
   try {
-    const existingHighlights = await highlightRepository.findByLibraryItemId(
-      input.articleId,
-      uid
+    const existingHighlights = await authTrx((tx) =>
+      tx
+        .withRepository(highlightRepository)
+        .findByLibraryItemId(input.articleId, uid)
     )
 
     existingHighlights.forEach((highlight) => {
diff --git a/packages/api/test/resolvers/highlight.test.ts b/packages/api/test/resolvers/highlight.test.ts
index a81602cd1..aef7aecae 100644
--- a/packages/api/test/resolvers/highlight.test.ts
+++ b/packages/api/test/resolvers/highlight.test.ts
@@ -1,10 +1,9 @@
+/* eslint-disable @typescript-eslint/restrict-template-expressions */
 import * as chai from 'chai'
 import { expect } from 'chai'
 import chaiString from 'chai-string'
 import 'mocha'
-import { Highlight } from '../../src/entity/highlight'
 import { User } from '../../src/entity/user'
-import { getRepository } from '../../src/repository'
 import {
   createHighlight,
   deleteHighlightById,
@@ -157,7 +156,7 @@ describe('Highlights API', () => {
       .post('/local/debug/fake-user-login')
       .send({ fakeEmail: user.email })
 
-    authToken = res.body.authToken
+    authToken = res.body.authToken as string
     itemId = (await createTestLibraryItem(user.id)).id
   })
 

From be57f81bf16ea76dbe62023b78de036fecada47a Mon Sep 17 00:00:00 2001
From: Jackson Harper <jacksonh@gmail.com>
Date: Tue, 16 Apr 2024 13:59:04 +0800
Subject: [PATCH 40/74] Temp re-add features attribute to User

---
 packages/api/src/generated/graphql.ts            |  2 ++
 packages/api/src/generated/schema.graphql        |  1 +
 packages/api/src/resolvers/function_resolvers.ts | 11 +++++++++++
 packages/api/src/schema.ts                       |  1 +
 4 files changed, 15 insertions(+)

diff --git a/packages/api/src/generated/graphql.ts b/packages/api/src/generated/graphql.ts
index ee17f1843..cd4b67b31 100644
--- a/packages/api/src/generated/graphql.ts
+++ b/packages/api/src/generated/graphql.ts
@@ -3734,6 +3734,7 @@ export type User = {
   __typename?: 'User';
   email?: Maybe<Scalars['String']>;
   featureList?: Maybe<Array<Feature>>;
+  features?: Maybe<Array<Maybe<Scalars['String']>>>;
   followersCount?: Maybe<Scalars['Int']>;
   friendsCount?: Maybe<Scalars['Int']>;
   id: Scalars['ID'];
@@ -7202,6 +7203,7 @@ export type UploadImportFileSuccessResolvers<ContextType = ResolverContext, Pare
 export type UserResolvers<ContextType = ResolverContext, ParentType extends ResolversParentTypes['User'] = ResolversParentTypes['User']> = {
   email?: Resolver<Maybe<ResolversTypes['String']>, ParentType, ContextType>;
   featureList?: Resolver<Maybe<Array<ResolversTypes['Feature']>>, ParentType, ContextType>;
+  features?: Resolver<Maybe<Array<Maybe<ResolversTypes['String']>>>, ParentType, ContextType>;
   followersCount?: Resolver<Maybe<ResolversTypes['Int']>, ParentType, ContextType>;
   friendsCount?: Resolver<Maybe<ResolversTypes['Int']>, ParentType, ContextType>;
   id?: Resolver<ResolversTypes['ID'], ParentType, ContextType>;
diff --git a/packages/api/src/generated/schema.graphql b/packages/api/src/generated/schema.graphql
index 2e6be1a96..e48fe3cff 100644
--- a/packages/api/src/generated/schema.graphql
+++ b/packages/api/src/generated/schema.graphql
@@ -3014,6 +3014,7 @@ enum UploadImportFileType {
 type User {
   email: String
   featureList: [Feature!]
+  features: [String]
   followersCount: Int
   friendsCount: Int
   id: ID!
diff --git a/packages/api/src/resolvers/function_resolvers.ts b/packages/api/src/resolvers/function_resolvers.ts
index 0756bf07e..d65d054c8 100644
--- a/packages/api/src/resolvers/function_resolvers.ts
+++ b/packages/api/src/resolvers/function_resolvers.ts
@@ -373,6 +373,17 @@ export const functionResolvers = {
       }
       return undefined
     },
+    async features(
+      _: User,
+      __: Record<string, unknown>,
+      ctx: WithDataSourcesContext
+    ) {
+      if (!ctx.claims?.uid) {
+        return undefined
+      }
+
+      return []
+    },
     async featureList(
       _: User,
       __: Record<string, unknown>,
diff --git a/packages/api/src/schema.ts b/packages/api/src/schema.ts
index 71192074b..acae6f520 100755
--- a/packages/api/src/schema.ts
+++ b/packages/api/src/schema.ts
@@ -88,6 +88,7 @@ const schema = gql`
     email: String
     source: String
     intercomHash: String
+    features: [String]
     featureList: [Feature!]
   }
 

From b4911639c7c7650bc4691b5effe8329e378d685b Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Fri, 12 Apr 2024 14:00:20 +0800
Subject: [PATCH 41/74] stub create digest job

---
 packages/api/src/jobs/create_digest.ts | 18 ++++++++++++++++
 packages/api/src/queue-processor.ts    |  3 +++
 packages/api/src/utils/createTask.ts   | 30 ++++++++++++++++++++++++++
 3 files changed, 51 insertions(+)
 create mode 100644 packages/api/src/jobs/create_digest.ts

diff --git a/packages/api/src/jobs/create_digest.ts b/packages/api/src/jobs/create_digest.ts
new file mode 100644
index 000000000..1e06132c0
--- /dev/null
+++ b/packages/api/src/jobs/create_digest.ts
@@ -0,0 +1,18 @@
+import { logger } from '../utils/logger'
+
+export const CREATE_DIGEST_JOB = 'CREATE_DIGEST_JOB'
+
+export interface CreateDigestJobData {
+  userId: string
+}
+
+export interface CreateDigestJobResponse {
+  jobId: string
+}
+
+export const processCreateDigestJob = async (data: CreateDigestJobData) => {
+  logger.info('processing create digest job', data)
+
+  // do something
+  await Promise.resolve()
+}
diff --git a/packages/api/src/queue-processor.ts b/packages/api/src/queue-processor.ts
index 136a854f4..2670e2777 100644
--- a/packages/api/src/queue-processor.ts
+++ b/packages/api/src/queue-processor.ts
@@ -18,6 +18,7 @@ import { TaskState } from './generated/graphql'
 import { aiSummarize, AI_SUMMARIZE_JOB_NAME } from './jobs/ai-summarize'
 import { bulkAction, BULK_ACTION_JOB_NAME } from './jobs/bulk_action'
 import { callWebhook, CALL_WEBHOOK_JOB_NAME } from './jobs/call_webhook'
+import { CREATE_DIGEST_JOB, processCreateDigestJob } from './jobs/create_digest'
 import { findThumbnail, THUMBNAIL_JOB } from './jobs/find_thumbnail'
 import {
   exportAllItems,
@@ -178,6 +179,8 @@ export const createWorker = (connection: ConnectionOptions) =>
           return saveNewsletterJob(job.data)
         case FORWARD_EMAIL_JOB:
           return forwardEmailJob(job.data)
+        case CREATE_DIGEST_JOB:
+          return processCreateDigestJob(job.data)
         default:
           logger.warning(`[queue-processor] unhandled job: ${job.name}`)
       }
diff --git a/packages/api/src/utils/createTask.ts b/packages/api/src/utils/createTask.ts
index 001daee64..d8fd2bbf5 100644
--- a/packages/api/src/utils/createTask.ts
+++ b/packages/api/src/utils/createTask.ts
@@ -18,6 +18,11 @@ import {
 import { AISummarizeJobData, AI_SUMMARIZE_JOB_NAME } from '../jobs/ai-summarize'
 import { BulkActionData, BULK_ACTION_JOB_NAME } from '../jobs/bulk_action'
 import { CallWebhookJobData, CALL_WEBHOOK_JOB_NAME } from '../jobs/call_webhook'
+import {
+  CreateDigestJobData,
+  CreateDigestJobResponse,
+  CREATE_DIGEST_JOB,
+} from '../jobs/create_digest'
 import { THUMBNAIL_JOB } from '../jobs/find_thumbnail'
 import { EXPORT_ALL_ITEMS_JOB_NAME } from '../jobs/integration/export_all_items'
 import {
@@ -81,6 +86,7 @@ export const getJobPriority = (jobName: string): number => {
     case `${REFRESH_FEED_JOB_NAME}_high`:
       return 10
     case PROCESS_YOUTUBE_TRANSCRIPT_JOB_NAME:
+    case CREATE_DIGEST_JOB:
       return 20
     case `${REFRESH_FEED_JOB_NAME}_low`:
     case EXPORT_ITEM_JOB_NAME:
@@ -853,4 +859,28 @@ export const enqueueSendEmail = async (jobData: SendEmailJobData) => {
   })
 }
 
+export const enqueueCreateDigest = async (
+  data: CreateDigestJobData
+): Promise<CreateDigestJobResponse> => {
+  const queue = await getBackendQueue()
+  if (!queue) {
+    throw new Error('No queue found')
+  }
+
+  const jobId = `create-digest-${data.userId}`
+  const job = await queue.add(CREATE_DIGEST_JOB, data, {
+    jobId, // dedupe by userId
+    removeOnComplete: true,
+    removeOnFail: true,
+    attempts: 3,
+    priority: getJobPriority(CREATE_DIGEST_JOB),
+  })
+
+  logger.info('create digest job enqueued', { jobId: job.id })
+
+  return {
+    jobId,
+  }
+}
+
 export default createHttpTaskWithToken

From 12aa7d4447d1699d1704856cb61ab0f9329bfca0 Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Fri, 12 Apr 2024 15:59:37 +0800
Subject: [PATCH 42/74] create digest job api

---
 packages/api/package.json                 |  2 +-
 packages/api/src/jobs/create_digest.ts    |  2 +-
 packages/api/src/queue-processor.ts       |  3 +
 packages/api/src/routers/digest_router.ts | 69 +++++++++++++++++++++++
 packages/api/src/utils/auth.ts            |  3 +-
 packages/api/src/utils/createTask.ts      |  4 +-
 yarn.lock                                 |  8 +--
 7 files changed, 81 insertions(+), 10 deletions(-)
 create mode 100644 packages/api/src/routers/digest_router.ts

diff --git a/packages/api/package.json b/packages/api/package.json
index 0da5f742b..469586434 100644
--- a/packages/api/package.json
+++ b/packages/api/package.json
@@ -124,7 +124,7 @@
     "@types/chai-as-promised": "^7.1.5",
     "@types/chai-string": "^1.4.2",
     "@types/cookie": "^0.4.0",
-    "@types/cookie-parser": "^1.4.2",
+    "@types/cookie-parser": "^1.4.7",
     "@types/csv-stringify": "^3.1.0",
     "@types/diff-match-patch": "^1.0.32",
     "@types/dompurify": "^2.0.4",
diff --git a/packages/api/src/jobs/create_digest.ts b/packages/api/src/jobs/create_digest.ts
index 1e06132c0..bab980480 100644
--- a/packages/api/src/jobs/create_digest.ts
+++ b/packages/api/src/jobs/create_digest.ts
@@ -1,6 +1,6 @@
 import { logger } from '../utils/logger'
 
-export const CREATE_DIGEST_JOB = 'CREATE_DIGEST_JOB'
+export const CREATE_DIGEST_JOB = 'create-digest'
 
 export interface CreateDigestJobData {
   userId: string
diff --git a/packages/api/src/queue-processor.ts b/packages/api/src/queue-processor.ts
index 2670e2777..86712a67b 100644
--- a/packages/api/src/queue-processor.ts
+++ b/packages/api/src/queue-processor.ts
@@ -96,6 +96,9 @@ export const getBackendQueue = async (): Promise<Queue | undefined> => {
   return backendQueue
 }
 
+export const createJobId = (jobName: string, userId: string) =>
+  `${jobName}_${userId}_${JOB_VERSION}`
+
 export const getJob = async (jobId: string) => {
   const queue = await getBackendQueue()
   if (!queue) {
diff --git a/packages/api/src/routers/digest_router.ts b/packages/api/src/routers/digest_router.ts
new file mode 100644
index 000000000..b54c73ceb
--- /dev/null
+++ b/packages/api/src/routers/digest_router.ts
@@ -0,0 +1,69 @@
+import cors from 'cors'
+import express from 'express'
+import { CREATE_DIGEST_JOB } from '../jobs/create_digest'
+import { createJobId, getJob } from '../queue-processor'
+import { findActiveUser } from '../services/user'
+import { getClaimsByToken, getTokenByRequest } from '../utils/auth'
+import { corsConfig } from '../utils/corsConfig'
+import { enqueueCreateDigest } from '../utils/createTask'
+import { logger } from '../utils/logger'
+
+export function digestRouter() {
+  const router = express.Router()
+
+  // v1 version of create digest api
+  router.post(
+    '/v1',
+    cors<express.Request>(corsConfig),
+    async (req: express.Request, res) => {
+      const token = getTokenByRequest(req)
+
+      let userId: string
+      try {
+        // get claims from token
+        const claims = await getClaimsByToken(token)
+        if (!claims) {
+          logger.info('Token not found')
+          return res.sendStatus(401)
+        }
+
+        // get user by uid from claims
+        userId = claims.uid
+      } catch (error) {
+        logger.info('Error while getting claims from token', error)
+        return res.sendStatus(401)
+      }
+
+      const user = await findActiveUser(userId)
+      if (!user) {
+        logger.info(`User not found: ${userId}`)
+        return res.sendStatus(401)
+      }
+
+      try {
+        // check if job is already in queue
+        // if yes then return 409 conflict
+        // else enqueue job
+        const jobId = createJobId(userId, CREATE_DIGEST_JOB)
+        const existingJob = await getJob(jobId)
+        if (existingJob) {
+          logger.info(`Job already in queue: ${jobId}`)
+          return res.sendStatus(409)
+        }
+
+        // enqueue job and return job id
+        const result = await enqueueCreateDigest({
+          userId,
+        })
+
+        // return job id
+        return res.status(200).send(result)
+      } catch (error) {
+        logger.error('Error while enqueuing create digest task', error)
+        return res.sendStatus(500)
+      }
+    }
+  )
+
+  return router
+}
diff --git a/packages/api/src/utils/auth.ts b/packages/api/src/utils/auth.ts
index 49c5fe9ff..c3269aec8 100644
--- a/packages/api/src/utils/auth.ts
+++ b/packages/api/src/utils/auth.ts
@@ -124,8 +124,7 @@ export const getTokenByRequest = (req: express.Request): string | undefined => {
   return (
     req.header(OmnivoreAuthorizationHeader) ||
     req.headers.authorization ||
-    // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
-    (req.cookies?.auth as string)
+    (req.cookies.auth as string | undefined)
   )
 }
 
diff --git a/packages/api/src/utils/createTask.ts b/packages/api/src/utils/createTask.ts
index d8fd2bbf5..7a8a2d23b 100644
--- a/packages/api/src/utils/createTask.ts
+++ b/packages/api/src/utils/createTask.ts
@@ -49,7 +49,7 @@ import {
   UPDATE_HIGHLIGHT_JOB,
   UPDATE_LABELS_JOB,
 } from '../jobs/update_db'
-import { getBackendQueue, JOB_VERSION } from '../queue-processor'
+import { createJobId, getBackendQueue, JOB_VERSION } from '../queue-processor'
 import { redisDataSource } from '../redis_data_source'
 import { signFeatureToken } from '../services/features'
 import { OmnivoreAuthorizationHeader } from './auth'
@@ -867,7 +867,7 @@ export const enqueueCreateDigest = async (
     throw new Error('No queue found')
   }
 
-  const jobId = `create-digest-${data.userId}`
+  const jobId = createJobId(CREATE_DIGEST_JOB, data.userId)
   const job = await queue.add(CREATE_DIGEST_JOB, data, {
     jobId, // dedupe by userId
     removeOnComplete: true,
diff --git a/yarn.lock b/yarn.lock
index db12191a9..034c6ee5c 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -7767,10 +7767,10 @@
   dependencies:
     "@types/node" "*"
 
-"@types/cookie-parser@^1.4.2":
-  version "1.4.2"
-  resolved "https://registry.yarnpkg.com/@types/cookie-parser/-/cookie-parser-1.4.2.tgz#e4d5c5ffda82b80672a88a4281aaceefb1bd9df5"
-  integrity sha512-uwcY8m6SDQqciHsqcKDGbo10GdasYsPCYkH3hVegj9qAah6pX5HivOnOuI3WYmyQMnOATV39zv/Ybs0bC/6iVg==
+"@types/cookie-parser@^1.4.7":
+  version "1.4.7"
+  resolved "https://registry.yarnpkg.com/@types/cookie-parser/-/cookie-parser-1.4.7.tgz#c874471f888c72423d78d2b3c32d1e8579cf3c8f"
+  integrity sha512-Fvuyi354Z+uayxzIGCwYTayFKocfV7TuDYZClCdIP9ckhvAu/ixDtCB6qx2TT0FKjPLf1f3P/J1rgf6lPs64mw==
   dependencies:
     "@types/express" "*"
 

From 1edb3419f872b3eda83a4c0c359bdebb9688d244 Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Fri, 12 Apr 2024 16:22:56 +0800
Subject: [PATCH 43/74] get digest api

---
 packages/api/src/routers/digest_router.ts | 154 ++++++++++++++++------
 1 file changed, 113 insertions(+), 41 deletions(-)

diff --git a/packages/api/src/routers/digest_router.ts b/packages/api/src/routers/digest_router.ts
index b54c73ceb..151b57364 100644
--- a/packages/api/src/routers/digest_router.ts
+++ b/packages/api/src/routers/digest_router.ts
@@ -1,69 +1,141 @@
 import cors from 'cors'
 import express from 'express'
+import { TaskState } from '../generated/graphql'
 import { CREATE_DIGEST_JOB } from '../jobs/create_digest'
-import { createJobId, getJob } from '../queue-processor'
+import { createJobId, getJob, jobStateToTaskState } from '../queue-processor'
+import { redisDataSource } from '../redis_data_source'
 import { findActiveUser } from '../services/user'
 import { getClaimsByToken, getTokenByRequest } from '../utils/auth'
 import { corsConfig } from '../utils/corsConfig'
 import { enqueueCreateDigest } from '../utils/createTask'
 import { logger } from '../utils/logger'
 
+interface Digest {
+  url: string
+  title: string
+  jobState: string
+  content: string
+  chapters: Chapter[]
+  urlsToAudio: string[]
+}
+
+interface Chapter {
+  title: string
+}
+
 export function digestRouter() {
   const router = express.Router()
 
   // v1 version of create digest api
-  router.post(
-    '/v1',
-    cors<express.Request>(corsConfig),
-    async (req: express.Request, res) => {
-      const token = getTokenByRequest(req)
+  router.post('/v1', cors<express.Request>(corsConfig), async (req, res) => {
+    const token = getTokenByRequest(req)
 
-      let userId: string
-      try {
-        // get claims from token
-        const claims = await getClaimsByToken(token)
-        if (!claims) {
-          logger.info('Token not found')
-          return res.sendStatus(401)
-        }
-
-        // get user by uid from claims
-        userId = claims.uid
-      } catch (error) {
-        logger.info('Error while getting claims from token', error)
+    let userId: string
+    try {
+      // get claims from token
+      const claims = await getClaimsByToken(token)
+      if (!claims) {
+        logger.info('Token not found')
         return res.sendStatus(401)
       }
 
+      // get user by uid from claims
+      userId = claims.uid
+    } catch (error) {
+      logger.info('Error while getting claims from token', error)
+      return res.sendStatus(401)
+    }
+
+    try {
       const user = await findActiveUser(userId)
       if (!user) {
         logger.info(`User not found: ${userId}`)
         return res.sendStatus(401)
       }
 
-      try {
-        // check if job is already in queue
-        // if yes then return 409 conflict
-        // else enqueue job
-        const jobId = createJobId(userId, CREATE_DIGEST_JOB)
-        const existingJob = await getJob(jobId)
-        if (existingJob) {
-          logger.info(`Job already in queue: ${jobId}`)
-          return res.sendStatus(409)
-        }
-
-        // enqueue job and return job id
-        const result = await enqueueCreateDigest({
-          userId,
-        })
-
-        // return job id
-        return res.status(200).send(result)
-      } catch (error) {
-        logger.error('Error while enqueuing create digest task', error)
-        return res.sendStatus(500)
+      // check if job is already in queue
+      // if yes then return 409 conflict
+      // else enqueue job
+      const jobId = createJobId(userId, CREATE_DIGEST_JOB)
+      const existingJob = await getJob(jobId)
+      if (existingJob) {
+        logger.info(`Job already in queue: ${jobId}`)
+        return res.sendStatus(409)
       }
+
+      // enqueue job and return job id
+      const result = await enqueueCreateDigest({
+        userId,
+      })
+
+      // return job id
+      return res.status(200).send(result)
+    } catch (error) {
+      logger.error('Error while enqueuing create digest task', error)
+      return res.sendStatus(500)
     }
-  )
+  })
+
+  // v1 version of get digest api
+  router.get('/v1', cors<express.Request>(corsConfig), async (req, res) => {
+    const token = getTokenByRequest(req)
+
+    let userId: string
+    try {
+      // get claims from token
+      const claims = await getClaimsByToken(token)
+      if (!claims) {
+        logger.info('Token not found')
+        return res.sendStatus(401)
+      }
+
+      // get user by uid from claims
+      userId = claims.uid
+    } catch (error) {
+      logger.info('Error while getting claims from token', error)
+      return res.sendStatus(401)
+    }
+
+    try {
+      const user = await findActiveUser(userId)
+      if (!user) {
+        logger.info(`User not found: ${userId}`)
+        return res.sendStatus(401)
+      }
+
+      // get job by user id
+      const jobId = createJobId(userId, CREATE_DIGEST_JOB)
+      const job = await getJob(jobId)
+      if (job) {
+        // if job is in queue then return job state
+        const jobState = await job.getState()
+        return res.send({
+          jobId: job.id,
+          jobState: jobStateToTaskState(jobState),
+        })
+      }
+
+      // if job is done and removed then get the digest from redis
+      const key = `digest:${userId}`
+      const digest = await redisDataSource.redisClient?.get(key)
+      if (!digest) {
+        logger.info(`Digest not found: ${key}`)
+        return res.sendStatus(404)
+      }
+
+      const digestObject = JSON.parse(digest) as Digest
+
+      // return digest
+      return res.send({
+        ...digestObject,
+        jobId,
+        jobState: TaskState.Succeeded,
+      })
+    } catch (error) {
+      logger.error('Error while getting digest', error)
+      return res.sendStatus(500)
+    }
+  })
 
   return router
 }

From e37bcd7408bed28dba719c54dc687df9e6ff9c9b Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Fri, 12 Apr 2024 16:35:22 +0800
Subject: [PATCH 44/74] send feedback api

---
 packages/api/src/routers/digest_router.ts | 74 +++++++++++++++++++++++
 packages/api/src/server.ts                |  2 +
 2 files changed, 76 insertions(+)

diff --git a/packages/api/src/routers/digest_router.ts b/packages/api/src/routers/digest_router.ts
index 151b57364..ae478af26 100644
--- a/packages/api/src/routers/digest_router.ts
+++ b/packages/api/src/routers/digest_router.ts
@@ -5,6 +5,7 @@ import { CREATE_DIGEST_JOB } from '../jobs/create_digest'
 import { createJobId, getJob, jobStateToTaskState } from '../queue-processor'
 import { redisDataSource } from '../redis_data_source'
 import { findActiveUser } from '../services/user'
+import { analytics } from '../utils/analytics'
 import { getClaimsByToken, getTokenByRequest } from '../utils/auth'
 import { corsConfig } from '../utils/corsConfig'
 import { enqueueCreateDigest } from '../utils/createTask'
@@ -23,6 +24,24 @@ interface Chapter {
   title: string
 }
 
+interface Feedback {
+  digestRating: number
+  rankingRating: number
+  summaryRating: number
+  voiceRating: number
+  musicRating: number
+}
+
+const isFeedback = (data: any): data is Feedback => {
+  return (
+    'digestRating' in data &&
+    'rankingRating' in data &&
+    'summaryRating' in data &&
+    'voiceRating' in data &&
+    'musicRating' in data
+  )
+}
+
 export function digestRouter() {
   const router = express.Router()
 
@@ -137,5 +156,60 @@ export function digestRouter() {
     }
   })
 
+  // v1 version of sending feedback api
+  router.post(
+    '/v1/feedback',
+    cors<express.Request>(corsConfig),
+    async (req, res) => {
+      const token = getTokenByRequest(req)
+
+      let userId: string
+      try {
+        // get claims from token
+        const claims = await getClaimsByToken(token)
+        if (!claims) {
+          logger.info('Token not found')
+          return res.sendStatus(401)
+        }
+
+        // get user by uid from claims
+        userId = claims.uid
+      } catch (error) {
+        logger.info('Error while getting claims from token', error)
+        return res.sendStatus(401)
+      }
+
+      try {
+        const user = await findActiveUser(userId)
+        if (!user) {
+          logger.info(`User not found: ${userId}`)
+          return res.sendStatus(401)
+        }
+
+        // get feedback from request body
+        if (!isFeedback(req.body)) {
+          logger.info('Invalid feedback format')
+          return res.sendStatus(400)
+        }
+
+        const feedback = req.body
+        // send feedback to analytics
+        logger.info(`Sending feedback: ${JSON.stringify(feedback)}`)
+
+        analytics.capture({
+          distinctId: userId,
+          event: 'digest_feedback',
+          properties: feedback,
+        })
+
+        // return success
+        return res.sendStatus(200)
+      } catch (error) {
+        logger.error('Error while saving feedback', error)
+        return res.sendStatus(500)
+      }
+    }
+  )
+
   return router
 }
diff --git a/packages/api/src/server.ts b/packages/api/src/server.ts
index 2c945581c..196c38f7c 100755
--- a/packages/api/src/server.ts
+++ b/packages/api/src/server.ts
@@ -21,6 +21,7 @@ import { aiSummariesRouter } from './routers/ai_summary_router'
 import { articleRouter } from './routers/article_router'
 import { authRouter } from './routers/auth/auth_router'
 import { mobileAuthRouter } from './routers/auth/mobile/mobile_auth_router'
+import { digestRouter } from './routers/digest_router'
 import { integrationRouter } from './routers/integration_router'
 import { localDebugRouter } from './routers/local_debug_router'
 import { notificationRouter } from './routers/notification_router'
@@ -89,6 +90,7 @@ export const createApp = (): Express => {
   app.use('/api/notification', notificationRouter())
   app.use('/api/integration', integrationRouter())
   app.use('/api/tasks', taskRouter())
+  app.use('/api/digest', digestRouter)
   app.use('/svc/pubsub/content', contentServiceRouter())
   app.use('/svc/pubsub/links', linkServiceRouter())
   app.use('/svc/pubsub/newsletters', newsletterServiceRouter())

From a47a5a7d9c433877585ed73c135b932b289e30fa Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Fri, 12 Apr 2024 16:56:16 +0800
Subject: [PATCH 45/74] mock saving digest in redis

---
 packages/api/src/jobs/create_digest.ts    | 26 ++++++++++++++++--
 packages/api/src/routers/digest_router.ts | 24 +++--------------
 packages/api/src/server.ts                |  2 +-
 packages/api/src/services/digest.ts       | 32 +++++++++++++++++++++++
 4 files changed, 61 insertions(+), 23 deletions(-)
 create mode 100644 packages/api/src/services/digest.ts

diff --git a/packages/api/src/jobs/create_digest.ts b/packages/api/src/jobs/create_digest.ts
index bab980480..f57c7eab8 100644
--- a/packages/api/src/jobs/create_digest.ts
+++ b/packages/api/src/jobs/create_digest.ts
@@ -1,3 +1,4 @@
+import { setDigest } from '../services/digest'
 import { logger } from '../utils/logger'
 
 export const CREATE_DIGEST_JOB = 'create-digest'
@@ -13,6 +14,27 @@ export interface CreateDigestJobResponse {
 export const processCreateDigestJob = async (data: CreateDigestJobData) => {
   logger.info('processing create digest job', data)
 
-  // do something
-  await Promise.resolve()
+  // TODO: create digest
+  const digest = {
+    url: 'https://example.com',
+    title: 'Example Digest',
+    jobState: 'completed',
+    content: 'This is an example digest',
+    chapters: [
+      {
+        title: 'Chapter 1',
+      },
+      {
+        title: 'Chapter 2',
+      },
+    ],
+    urlsToAudio: ['https://example.com/audio'],
+  }
+
+  // save digest to redis
+  await setDigest(data.userId, digest)
+
+  logger.info('digest created', digest)
+
+  return true
 }
diff --git a/packages/api/src/routers/digest_router.ts b/packages/api/src/routers/digest_router.ts
index ae478af26..24102de3b 100644
--- a/packages/api/src/routers/digest_router.ts
+++ b/packages/api/src/routers/digest_router.ts
@@ -3,7 +3,7 @@ import express from 'express'
 import { TaskState } from '../generated/graphql'
 import { CREATE_DIGEST_JOB } from '../jobs/create_digest'
 import { createJobId, getJob, jobStateToTaskState } from '../queue-processor'
-import { redisDataSource } from '../redis_data_source'
+import { getDigest } from '../services/digest'
 import { findActiveUser } from '../services/user'
 import { analytics } from '../utils/analytics'
 import { getClaimsByToken, getTokenByRequest } from '../utils/auth'
@@ -11,19 +11,6 @@ import { corsConfig } from '../utils/corsConfig'
 import { enqueueCreateDigest } from '../utils/createTask'
 import { logger } from '../utils/logger'
 
-interface Digest {
-  url: string
-  title: string
-  jobState: string
-  content: string
-  chapters: Chapter[]
-  urlsToAudio: string[]
-}
-
-interface Chapter {
-  title: string
-}
-
 interface Feedback {
   digestRating: number
   rankingRating: number
@@ -135,18 +122,15 @@ export function digestRouter() {
       }
 
       // if job is done and removed then get the digest from redis
-      const key = `digest:${userId}`
-      const digest = await redisDataSource.redisClient?.get(key)
+      const digest = await getDigest(userId)
       if (!digest) {
-        logger.info(`Digest not found: ${key}`)
+        logger.info(`Digest not found: ${userId}`)
         return res.sendStatus(404)
       }
 
-      const digestObject = JSON.parse(digest) as Digest
-
       // return digest
       return res.send({
-        ...digestObject,
+        ...digest,
         jobId,
         jobState: TaskState.Succeeded,
       })
diff --git a/packages/api/src/server.ts b/packages/api/src/server.ts
index 196c38f7c..e1603a6cb 100755
--- a/packages/api/src/server.ts
+++ b/packages/api/src/server.ts
@@ -90,7 +90,7 @@ export const createApp = (): Express => {
   app.use('/api/notification', notificationRouter())
   app.use('/api/integration', integrationRouter())
   app.use('/api/tasks', taskRouter())
-  app.use('/api/digest', digestRouter)
+  app.use('/api/digest', digestRouter())
   app.use('/svc/pubsub/content', contentServiceRouter())
   app.use('/svc/pubsub/links', linkServiceRouter())
   app.use('/svc/pubsub/newsletters', newsletterServiceRouter())
diff --git a/packages/api/src/services/digest.ts b/packages/api/src/services/digest.ts
new file mode 100644
index 000000000..ceb274d70
--- /dev/null
+++ b/packages/api/src/services/digest.ts
@@ -0,0 +1,32 @@
+import { redisDataSource } from '../redis_data_source'
+
+export interface Digest {
+  url: string
+  title: string
+  jobState: string
+  content: string
+  chapters: Chapter[]
+  urlsToAudio: string[]
+}
+
+interface Chapter {
+  title: string
+}
+
+const digestKey = (userId: string) => `digest:${userId}`
+
+export const getDigest = async (userId: string): Promise<Digest | null> => {
+  const digest = await redisDataSource.redisClient?.get(digestKey(userId))
+  return digest ? (JSON.parse(digest) as Digest) : null
+}
+
+export const setDigest = async (userId: string, digest: Digest) => {
+  const result = await redisDataSource.redisClient?.set(
+    digestKey(userId),
+    JSON.stringify(digest)
+  )
+
+  if (result != 'OK') {
+    throw new Error('Failed to set digest')
+  }
+}

From 521698646dcbcc4b225bdaa0d37d756d9e18b448 Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Fri, 12 Apr 2024 16:58:56 +0800
Subject: [PATCH 46/74] update digest mock data

---
 packages/api/src/jobs/create_digest.ts | 20 +++++++++-----------
 packages/api/src/services/digest.ts    |  1 -
 2 files changed, 9 insertions(+), 12 deletions(-)

diff --git a/packages/api/src/jobs/create_digest.ts b/packages/api/src/jobs/create_digest.ts
index f57c7eab8..e17fd26b4 100644
--- a/packages/api/src/jobs/create_digest.ts
+++ b/packages/api/src/jobs/create_digest.ts
@@ -16,19 +16,17 @@ export const processCreateDigestJob = async (data: CreateDigestJobData) => {
 
   // TODO: create digest
   const digest = {
-    url: 'https://example.com',
-    title: 'Example Digest',
-    jobState: 'completed',
-    content: 'This is an example digest',
+    url: '',
+    title: '',
+    content: '<readable html>',
     chapters: [
-      {
-        title: 'Chapter 1',
-      },
-      {
-        title: 'Chapter 2',
-      },
+      { title: '00:30 - China buying' },
+      { title: '10:20 - Twitter for sale' },
+    ],
+    urlsToAudio: [
+      'https://storage.google.com/...',
+      'https://api.omnivore.app/api/digest/audio/<id>',
     ],
-    urlsToAudio: ['https://example.com/audio'],
   }
 
   // save digest to redis
diff --git a/packages/api/src/services/digest.ts b/packages/api/src/services/digest.ts
index ceb274d70..46343a8ce 100644
--- a/packages/api/src/services/digest.ts
+++ b/packages/api/src/services/digest.ts
@@ -3,7 +3,6 @@ import { redisDataSource } from '../redis_data_source'
 export interface Digest {
   url: string
   title: string
-  jobState: string
   content: string
   chapters: Chapter[]
   urlsToAudio: string[]

From 4342cdeb3eae323981fe17c67d1deaea50040255 Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Fri, 12 Apr 2024 17:39:02 +0800
Subject: [PATCH 47/74] expire digest in one day

---
 packages/api/src/jobs/create_digest.ts    | 3 +++
 packages/api/src/routers/digest_router.ts | 4 ++--
 packages/api/src/services/digest.ts       | 4 +++-
 3 files changed, 8 insertions(+), 3 deletions(-)

diff --git a/packages/api/src/jobs/create_digest.ts b/packages/api/src/jobs/create_digest.ts
index e17fd26b4..fd5b1d842 100644
--- a/packages/api/src/jobs/create_digest.ts
+++ b/packages/api/src/jobs/create_digest.ts
@@ -32,6 +32,9 @@ export const processCreateDigestJob = async (data: CreateDigestJobData) => {
   // save digest to redis
   await setDigest(data.userId, digest)
 
+  // simulate long running task
+  await new Promise((resolve) => setTimeout(resolve, 5000))
+
   logger.info('digest created', digest)
 
   return true
diff --git a/packages/api/src/routers/digest_router.ts b/packages/api/src/routers/digest_router.ts
index 24102de3b..f495dc87e 100644
--- a/packages/api/src/routers/digest_router.ts
+++ b/packages/api/src/routers/digest_router.ts
@@ -62,7 +62,7 @@ export function digestRouter() {
       // check if job is already in queue
       // if yes then return 409 conflict
       // else enqueue job
-      const jobId = createJobId(userId, CREATE_DIGEST_JOB)
+      const jobId = createJobId(CREATE_DIGEST_JOB, userId)
       const existingJob = await getJob(jobId)
       if (existingJob) {
         logger.info(`Job already in queue: ${jobId}`)
@@ -110,7 +110,7 @@ export function digestRouter() {
       }
 
       // get job by user id
-      const jobId = createJobId(userId, CREATE_DIGEST_JOB)
+      const jobId = createJobId(CREATE_DIGEST_JOB, userId)
       const job = await getJob(jobId)
       if (job) {
         // if job is in queue then return job state
diff --git a/packages/api/src/services/digest.ts b/packages/api/src/services/digest.ts
index 46343a8ce..8a6160ecc 100644
--- a/packages/api/src/services/digest.ts
+++ b/packages/api/src/services/digest.ts
@@ -22,7 +22,9 @@ export const getDigest = async (userId: string): Promise<Digest | null> => {
 export const setDigest = async (userId: string, digest: Digest) => {
   const result = await redisDataSource.redisClient?.set(
     digestKey(userId),
-    JSON.stringify(digest)
+    JSON.stringify(digest),
+    'EX',
+    60 * 60 * 24 // 1 day
   )
 
   if (result != 'OK') {

From 111dcd1562b5abb8c17f88e0e4e88f9732dd69a4 Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Fri, 12 Apr 2024 18:32:48 +0800
Subject: [PATCH 48/74] send ranking models and summary models to posthog and
 log comment

---
 packages/api/src/jobs/create_digest.ts    | 21 +--------------------
 packages/api/src/routers/digest_router.ts | 13 +++++++++----
 2 files changed, 10 insertions(+), 24 deletions(-)

diff --git a/packages/api/src/jobs/create_digest.ts b/packages/api/src/jobs/create_digest.ts
index fd5b1d842..43550ef32 100644
--- a/packages/api/src/jobs/create_digest.ts
+++ b/packages/api/src/jobs/create_digest.ts
@@ -1,4 +1,3 @@
-import { setDigest } from '../services/digest'
 import { logger } from '../utils/logger'
 
 export const CREATE_DIGEST_JOB = 'create-digest'
@@ -14,28 +13,10 @@ export interface CreateDigestJobResponse {
 export const processCreateDigestJob = async (data: CreateDigestJobData) => {
   logger.info('processing create digest job', data)
 
-  // TODO: create digest
-  const digest = {
-    url: '',
-    title: '',
-    content: '<readable html>',
-    chapters: [
-      { title: '00:30 - China buying' },
-      { title: '10:20 - Twitter for sale' },
-    ],
-    urlsToAudio: [
-      'https://storage.google.com/...',
-      'https://api.omnivore.app/api/digest/audio/<id>',
-    ],
-  }
-
-  // save digest to redis
-  await setDigest(data.userId, digest)
-
   // simulate long running task
   await new Promise((resolve) => setTimeout(resolve, 5000))
 
-  logger.info('digest created', digest)
+  logger.info('digest created')
 
   return true
 }
diff --git a/packages/api/src/routers/digest_router.ts b/packages/api/src/routers/digest_router.ts
index f495dc87e..0c7e47ceb 100644
--- a/packages/api/src/routers/digest_router.ts
+++ b/packages/api/src/routers/digest_router.ts
@@ -13,10 +13,13 @@ import { logger } from '../utils/logger'
 
 interface Feedback {
   digestRating: number
+  rankingModels: string[]
   rankingRating: number
   summaryRating: number
+  summaryModels: string[]
   voiceRating: number
   musicRating: number
+  comment?: string
 }
 
 const isFeedback = (data: any): data is Feedback => {
@@ -60,13 +63,13 @@ export function digestRouter() {
       }
 
       // check if job is already in queue
-      // if yes then return 409 conflict
+      // if yes then return 202 accepted
       // else enqueue job
       const jobId = createJobId(CREATE_DIGEST_JOB, userId)
       const existingJob = await getJob(jobId)
       if (existingJob) {
         logger.info(`Job already in queue: ${jobId}`)
-        return res.sendStatus(409)
+        return res.sendStatus(202)
       }
 
       // enqueue job and return job id
@@ -75,7 +78,7 @@ export function digestRouter() {
       })
 
       // return job id
-      return res.status(200).send(result)
+      return res.status(201).send(result)
     } catch (error) {
       logger.error('Error while enqueuing create digest task', error)
       return res.sendStatus(500)
@@ -177,9 +180,11 @@ export function digestRouter() {
         }
 
         const feedback = req.body
-        // send feedback to analytics
         logger.info(`Sending feedback: ${JSON.stringify(feedback)}`)
 
+        // remove comment from feedback before sending to analytics
+        delete feedback.comment
+        // send feedback to analytics
         analytics.capture({
           distinctId: userId,
           event: 'digest_feedback',

From bc955fbeeb5e8308e03b6ebc89d04f9a88d26c25 Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Fri, 12 Apr 2024 18:51:42 +0800
Subject: [PATCH 49/74] remove stub job

---
 packages/api/src/routers/digest_router.ts | 47 +++++++++--------------
 packages/api/src/services/digest.ts       | 19 ++++++---
 2 files changed, 32 insertions(+), 34 deletions(-)

diff --git a/packages/api/src/routers/digest_router.ts b/packages/api/src/routers/digest_router.ts
index 0c7e47ceb..519a52688 100644
--- a/packages/api/src/routers/digest_router.ts
+++ b/packages/api/src/routers/digest_router.ts
@@ -3,12 +3,11 @@ import express from 'express'
 import { TaskState } from '../generated/graphql'
 import { CREATE_DIGEST_JOB } from '../jobs/create_digest'
 import { createJobId, getJob, jobStateToTaskState } from '../queue-processor'
-import { getDigest } from '../services/digest'
+import { getDigest, setDigest } from '../services/digest'
 import { findActiveUser } from '../services/user'
 import { analytics } from '../utils/analytics'
 import { getClaimsByToken, getTokenByRequest } from '../utils/auth'
 import { corsConfig } from '../utils/corsConfig'
-import { enqueueCreateDigest } from '../utils/createTask'
 import { logger } from '../utils/logger'
 
 interface Feedback {
@@ -62,23 +61,24 @@ export function digestRouter() {
         return res.sendStatus(401)
       }
 
-      // check if job is already in queue
+      // check if digest job is running
       // if yes then return 202 accepted
       // else enqueue job
-      const jobId = createJobId(CREATE_DIGEST_JOB, userId)
-      const existingJob = await getJob(jobId)
-      if (existingJob) {
-        logger.info(`Job already in queue: ${jobId}`)
+      const digest = await getDigest(userId)
+      if (digest?.jobState === TaskState.Running) {
+        logger.info(`job is running: ${userId}`)
         return res.sendStatus(202)
       }
 
       // enqueue job and return job id
-      const result = await enqueueCreateDigest({
-        userId,
+      const jobId = await setDigest(userId, {
+        jobState: TaskState.Running,
       })
 
       // return job id
-      return res.status(201).send(result)
+      return res.status(201).send({
+        jobId,
+      })
     } catch (error) {
       logger.error('Error while enqueuing create digest task', error)
       return res.sendStatus(500)
@@ -112,31 +112,22 @@ export function digestRouter() {
         return res.sendStatus(401)
       }
 
-      // get job by user id
-      const jobId = createJobId(CREATE_DIGEST_JOB, userId)
-      const job = await getJob(jobId)
-      if (job) {
-        // if job is in queue then return job state
-        const jobState = await job.getState()
-        return res.send({
-          jobId: job.id,
-          jobState: jobStateToTaskState(jobState),
-        })
-      }
-
-      // if job is done and removed then get the digest from redis
+      // get digest from redis
       const digest = await getDigest(userId)
       if (!digest) {
         logger.info(`Digest not found: ${userId}`)
         return res.sendStatus(404)
       }
 
+      if (digest.jobState === TaskState.Running) {
+        // if job is in queue then return job state
+        return res.status(202).send({
+          jobState: TaskState.Running,
+        })
+      }
+
       // return digest
-      return res.send({
-        ...digest,
-        jobId,
-        jobState: TaskState.Succeeded,
-      })
+      return res.send(digest)
     } catch (error) {
       logger.error('Error while getting digest', error)
       return res.sendStatus(500)
diff --git a/packages/api/src/services/digest.ts b/packages/api/src/services/digest.ts
index 8a6160ecc..41dac920f 100644
--- a/packages/api/src/services/digest.ts
+++ b/packages/api/src/services/digest.ts
@@ -1,11 +1,12 @@
 import { redisDataSource } from '../redis_data_source'
 
 export interface Digest {
-  url: string
-  title: string
-  content: string
-  chapters: Chapter[]
-  urlsToAudio: string[]
+  url?: string
+  title?: string
+  content?: string
+  chapters?: Chapter[]
+  urlsToAudio?: string[]
+  jobState: string
 }
 
 interface Chapter {
@@ -19,7 +20,11 @@ export const getDigest = async (userId: string): Promise<Digest | null> => {
   return digest ? (JSON.parse(digest) as Digest) : null
 }
 
-export const setDigest = async (userId: string, digest: Digest) => {
+export const setDigest = async (
+  userId: string,
+  digest: Digest
+): Promise<string> => {
+  const key = digestKey(userId)
   const result = await redisDataSource.redisClient?.set(
     digestKey(userId),
     JSON.stringify(digest),
@@ -30,4 +35,6 @@ export const setDigest = async (userId: string, digest: Digest) => {
   if (result != 'OK') {
     throw new Error('Failed to set digest')
   }
+
+  return key
 }

From cdb0b7525e6bb481132012e9314786f0e251d440 Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Fri, 12 Apr 2024 18:53:50 +0800
Subject: [PATCH 50/74] cleanup

---
 packages/api/src/routers/digest_router.ts | 2 --
 1 file changed, 2 deletions(-)

diff --git a/packages/api/src/routers/digest_router.ts b/packages/api/src/routers/digest_router.ts
index 519a52688..7f9ab327b 100644
--- a/packages/api/src/routers/digest_router.ts
+++ b/packages/api/src/routers/digest_router.ts
@@ -1,8 +1,6 @@
 import cors from 'cors'
 import express from 'express'
 import { TaskState } from '../generated/graphql'
-import { CREATE_DIGEST_JOB } from '../jobs/create_digest'
-import { createJobId, getJob, jobStateToTaskState } from '../queue-processor'
 import { getDigest, setDigest } from '../services/digest'
 import { findActiveUser } from '../services/user'
 import { analytics } from '../utils/analytics'

From 03e497b4897b2ef838a10a5f6e07da80cadb8326 Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Sat, 13 Apr 2024 14:41:10 +0800
Subject: [PATCH 51/74] use a separate queue for digest job

---
 packages/api/src/jobs/create_digest.ts    | 22 ---------
 packages/api/src/queue-processor.ts       | 20 ++++-----
 packages/api/src/routers/digest_router.ts | 55 ++++++++++++++---------
 packages/api/src/services/digest.ts       | 29 +++++-------
 packages/api/src/utils/createTask.ts      | 22 ++++-----
 5 files changed, 65 insertions(+), 83 deletions(-)
 delete mode 100644 packages/api/src/jobs/create_digest.ts

diff --git a/packages/api/src/jobs/create_digest.ts b/packages/api/src/jobs/create_digest.ts
deleted file mode 100644
index 43550ef32..000000000
--- a/packages/api/src/jobs/create_digest.ts
+++ /dev/null
@@ -1,22 +0,0 @@
-import { logger } from '../utils/logger'
-
-export const CREATE_DIGEST_JOB = 'create-digest'
-
-export interface CreateDigestJobData {
-  userId: string
-}
-
-export interface CreateDigestJobResponse {
-  jobId: string
-}
-
-export const processCreateDigestJob = async (data: CreateDigestJobData) => {
-  logger.info('processing create digest job', data)
-
-  // simulate long running task
-  await new Promise((resolve) => setTimeout(resolve, 5000))
-
-  logger.info('digest created')
-
-  return true
-}
diff --git a/packages/api/src/queue-processor.ts b/packages/api/src/queue-processor.ts
index 86712a67b..dac1f74d9 100644
--- a/packages/api/src/queue-processor.ts
+++ b/packages/api/src/queue-processor.ts
@@ -18,7 +18,6 @@ import { TaskState } from './generated/graphql'
 import { aiSummarize, AI_SUMMARIZE_JOB_NAME } from './jobs/ai-summarize'
 import { bulkAction, BULK_ACTION_JOB_NAME } from './jobs/bulk_action'
 import { callWebhook, CALL_WEBHOOK_JOB_NAME } from './jobs/call_webhook'
-import { CREATE_DIGEST_JOB, processCreateDigestJob } from './jobs/create_digest'
 import { findThumbnail, THUMBNAIL_JOB } from './jobs/find_thumbnail'
 import {
   exportAllItems,
@@ -66,18 +65,17 @@ import {
 } from './jobs/email/inbound_emails'
 
 export const QUEUE_NAME = 'omnivore-backend-queue'
+export const LONG_RUNNING_QUEUE_NAME = 'omnivore-backend-long-running-queue'
 export const JOB_VERSION = 'v001'
 
-let backendQueue: Queue | undefined
-export const getBackendQueue = async (): Promise<Queue | undefined> => {
-  if (backendQueue) {
-    await backendQueue.waitUntilReady()
-    return backendQueue
-  }
+export const getBackendQueue = async (
+  name = QUEUE_NAME
+): Promise<Queue | undefined> => {
   if (!redisDataSource.workerRedisClient) {
     throw new Error('Can not create queues, redis is not initialized')
   }
-  backendQueue = new Queue(QUEUE_NAME, {
+
+  const backendQueue = new Queue(name, {
     connection: redisDataSource.workerRedisClient,
     defaultJobOptions: {
       backoff: {
@@ -99,8 +97,8 @@ export const getBackendQueue = async (): Promise<Queue | undefined> => {
 export const createJobId = (jobName: string, userId: string) =>
   `${jobName}_${userId}_${JOB_VERSION}`
 
-export const getJob = async (jobId: string) => {
-  const queue = await getBackendQueue()
+export const getJob = async (jobId: string, queueName?: string) => {
+  const queue = await getBackendQueue(queueName)
   if (!queue) {
     return
   }
@@ -182,8 +180,6 @@ export const createWorker = (connection: ConnectionOptions) =>
           return saveNewsletterJob(job.data)
         case FORWARD_EMAIL_JOB:
           return forwardEmailJob(job.data)
-        case CREATE_DIGEST_JOB:
-          return processCreateDigestJob(job.data)
         default:
           logger.warning(`[queue-processor] unhandled job: ${job.name}`)
       }
diff --git a/packages/api/src/routers/digest_router.ts b/packages/api/src/routers/digest_router.ts
index 7f9ab327b..122097c09 100644
--- a/packages/api/src/routers/digest_router.ts
+++ b/packages/api/src/routers/digest_router.ts
@@ -1,11 +1,18 @@
 import cors from 'cors'
 import express from 'express'
-import { TaskState } from '../generated/graphql'
-import { getDigest, setDigest } from '../services/digest'
+import { env } from '../env'
+import {
+  createJobId,
+  getJob,
+  jobStateToTaskState,
+  LONG_RUNNING_QUEUE_NAME,
+} from '../queue-processor'
+import { CREATE_DIGEST_JOB, getDigest } from '../services/digest'
 import { findActiveUser } from '../services/user'
 import { analytics } from '../utils/analytics'
 import { getClaimsByToken, getTokenByRequest } from '../utils/auth'
 import { corsConfig } from '../utils/corsConfig'
+import { enqueueCreateDigest } from '../utils/createTask'
 import { logger } from '../utils/logger'
 
 interface Feedback {
@@ -59,24 +66,23 @@ export function digestRouter() {
         return res.sendStatus(401)
       }
 
-      // check if digest job is running
+      // check if job is already in queue
       // if yes then return 202 accepted
       // else enqueue job
-      const digest = await getDigest(userId)
-      if (digest?.jobState === TaskState.Running) {
-        logger.info(`job is running: ${userId}`)
+      const jobId = createJobId(CREATE_DIGEST_JOB, userId)
+      const existingJob = await getJob(jobId, LONG_RUNNING_QUEUE_NAME)
+      if (existingJob) {
+        logger.info(`Job already in queue: ${jobId}`)
         return res.sendStatus(202)
       }
 
       // enqueue job and return job id
-      const jobId = await setDigest(userId, {
-        jobState: TaskState.Running,
+      const result = await enqueueCreateDigest({
+        userId,
       })
 
       // return job id
-      return res.status(201).send({
-        jobId,
-      })
+      return res.status(201).send(result)
     } catch (error) {
       logger.error('Error while enqueuing create digest task', error)
       return res.sendStatus(500)
@@ -110,20 +116,25 @@ export function digestRouter() {
         return res.sendStatus(401)
       }
 
-      // get digest from redis
+      // get job by user id
+      const jobId = createJobId(CREATE_DIGEST_JOB, userId)
+      const job = await getJob(jobId, LONG_RUNNING_QUEUE_NAME)
+      if (job) {
+        // if job is in queue then return job state
+        const jobState = await job.getState()
+        return res.send({
+          jobId: job.id,
+          jobState: jobStateToTaskState(jobState),
+        })
+      }
+
+      // if job is done and removed then get the digest from redis
       const digest = await getDigest(userId)
       if (!digest) {
         logger.info(`Digest not found: ${userId}`)
         return res.sendStatus(404)
       }
 
-      if (digest.jobState === TaskState.Running) {
-        // if job is in queue then return job state
-        return res.status(202).send({
-          jobState: TaskState.Running,
-        })
-      }
-
       // return digest
       return res.send(digest)
     } catch (error) {
@@ -177,7 +188,11 @@ export function digestRouter() {
         analytics.capture({
           distinctId: userId,
           event: 'digest_feedback',
-          properties: feedback,
+          properties: {
+            ...feedback,
+            env: env.server.apiEnv,
+            userId,
+          },
         })
 
         // return success
diff --git a/packages/api/src/services/digest.ts b/packages/api/src/services/digest.ts
index 41dac920f..de2ca2909 100644
--- a/packages/api/src/services/digest.ts
+++ b/packages/api/src/services/digest.ts
@@ -1,5 +1,15 @@
 import { redisDataSource } from '../redis_data_source'
 
+export const CREATE_DIGEST_JOB = 'create-digest'
+
+export interface CreateDigestJobData {
+  userId: string
+}
+
+export interface CreateDigestJobResponse {
+  jobId: string
+}
+
 export interface Digest {
   url?: string
   title?: string
@@ -19,22 +29,3 @@ export const getDigest = async (userId: string): Promise<Digest | null> => {
   const digest = await redisDataSource.redisClient?.get(digestKey(userId))
   return digest ? (JSON.parse(digest) as Digest) : null
 }
-
-export const setDigest = async (
-  userId: string,
-  digest: Digest
-): Promise<string> => {
-  const key = digestKey(userId)
-  const result = await redisDataSource.redisClient?.set(
-    digestKey(userId),
-    JSON.stringify(digest),
-    'EX',
-    60 * 60 * 24 // 1 day
-  )
-
-  if (result != 'OK') {
-    throw new Error('Failed to set digest')
-  }
-
-  return key
-}
diff --git a/packages/api/src/utils/createTask.ts b/packages/api/src/utils/createTask.ts
index 7a8a2d23b..d64471252 100644
--- a/packages/api/src/utils/createTask.ts
+++ b/packages/api/src/utils/createTask.ts
@@ -18,11 +18,6 @@ import {
 import { AISummarizeJobData, AI_SUMMARIZE_JOB_NAME } from '../jobs/ai-summarize'
 import { BulkActionData, BULK_ACTION_JOB_NAME } from '../jobs/bulk_action'
 import { CallWebhookJobData, CALL_WEBHOOK_JOB_NAME } from '../jobs/call_webhook'
-import {
-  CreateDigestJobData,
-  CreateDigestJobResponse,
-  CREATE_DIGEST_JOB,
-} from '../jobs/create_digest'
 import { THUMBNAIL_JOB } from '../jobs/find_thumbnail'
 import { EXPORT_ALL_ITEMS_JOB_NAME } from '../jobs/integration/export_all_items'
 import {
@@ -49,8 +44,18 @@ import {
   UPDATE_HIGHLIGHT_JOB,
   UPDATE_LABELS_JOB,
 } from '../jobs/update_db'
-import { createJobId, getBackendQueue, JOB_VERSION } from '../queue-processor'
+import {
+  createJobId,
+  getBackendQueue,
+  JOB_VERSION,
+  LONG_RUNNING_QUEUE_NAME,
+} from '../queue-processor'
 import { redisDataSource } from '../redis_data_source'
+import {
+  CreateDigestJobData,
+  CreateDigestJobResponse,
+  CREATE_DIGEST_JOB,
+} from '../services/digest'
 import { signFeatureToken } from '../services/features'
 import { OmnivoreAuthorizationHeader } from './auth'
 import { CreateTaskError } from './errors'
@@ -86,8 +91,6 @@ export const getJobPriority = (jobName: string): number => {
     case `${REFRESH_FEED_JOB_NAME}_high`:
       return 10
     case PROCESS_YOUTUBE_TRANSCRIPT_JOB_NAME:
-    case CREATE_DIGEST_JOB:
-      return 20
     case `${REFRESH_FEED_JOB_NAME}_low`:
     case EXPORT_ITEM_JOB_NAME:
       return 50
@@ -862,7 +865,7 @@ export const enqueueSendEmail = async (jobData: SendEmailJobData) => {
 export const enqueueCreateDigest = async (
   data: CreateDigestJobData
 ): Promise<CreateDigestJobResponse> => {
-  const queue = await getBackendQueue()
+  const queue = await getBackendQueue(LONG_RUNNING_QUEUE_NAME)
   if (!queue) {
     throw new Error('No queue found')
   }
@@ -873,7 +876,6 @@ export const enqueueCreateDigest = async (
     removeOnComplete: true,
     removeOnFail: true,
     attempts: 3,
-    priority: getJobPriority(CREATE_DIGEST_JOB),
   })
 
   logger.info('create digest job enqueued', { jobId: job.id })

From e30fa13a2e5ef4581827e0bb0623f75d1c8b81a0 Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Sat, 13 Apr 2024 14:43:22 +0800
Subject: [PATCH 52/74] add env to the posthog event

---
 packages/api/src/routers/digest_router.ts | 1 -
 1 file changed, 1 deletion(-)

diff --git a/packages/api/src/routers/digest_router.ts b/packages/api/src/routers/digest_router.ts
index 122097c09..f2c5ec65d 100644
--- a/packages/api/src/routers/digest_router.ts
+++ b/packages/api/src/routers/digest_router.ts
@@ -191,7 +191,6 @@ export function digestRouter() {
           properties: {
             ...feedback,
             env: env.server.apiEnv,
-            userId,
           },
         })
 

From 4e61b360d7316721e95bfc21868269867166fd92 Mon Sep 17 00:00:00 2001
From: Jackson Harper <jacksonh@gmail.com>
Date: Mon, 15 Apr 2024 17:20:22 +0800
Subject: [PATCH 53/74] Add iOS digest views

---
 .../App/Views/AI/LibraryDigestView.swift      | 267 ++++++++++++++++++
 .../App/Views/BottomBars/CustomTabBar.swift   |  34 ++-
 .../App/Views/Home/HomeFeedViewIOS.swift      |  10 +
 .../Sources/App/Views/LibraryTabView.swift    |  18 +-
 .../App/Views/WebReader/WebReader.swift       |   2 +
 .../Views/WebReader/WebReaderContainer.swift  |  32 ++-
 .../Views/WebReader/WebReaderViewModel.swift  |   2 +
 .../Services/Authentication/AuthModels.swift  |   2 +-
 .../DataService/Errors/ServerError.swift      |   3 +
 .../Views/Article/OmnivoreWebView.swift       |  17 +-
 .../Sources/Views/Images/Images.swift         |   6 +
 .../_tab_digest.imageset/Contents.json        |  24 ++
 .../_tab_digest.imageset/_tab_digest.svg      |  18 ++
 .../Contents.json                             |  21 ++
 .../_tab_digest_selected.svg                  |  18 ++
 .../Contents.json                             |  24 ++
 .../digest-archive-button.svg                 |  13 +
 .../digest-dots-button.imageset/Contents.json |  24 ++
 .../digest-dots-button.svg                    |  13 +
 .../digest-play-button.imageset/Contents.json |  24 ++
 .../digest-play-button.svg                    |  11 +
 .../Contents.json                             |  24 ++
 .../digest-trash-button.svg                   |  15 +
 .../double_chevron_up.imageset/Contents.json  |  21 ++
 .../double_chevron_up.svg                     |   4 +
 apple/OmnivoreKit/Sources/Views/Theme.swift   |  20 ++
 26 files changed, 650 insertions(+), 17 deletions(-)
 create mode 100644 apple/OmnivoreKit/Sources/App/Views/AI/LibraryDigestView.swift
 create mode 100644 apple/OmnivoreKit/Sources/Views/Images/Images.xcassets/_tab_digest.imageset/Contents.json
 create mode 100644 apple/OmnivoreKit/Sources/Views/Images/Images.xcassets/_tab_digest.imageset/_tab_digest.svg
 create mode 100644 apple/OmnivoreKit/Sources/Views/Images/Images.xcassets/_tab_digest_selected.imageset/Contents.json
 create mode 100644 apple/OmnivoreKit/Sources/Views/Images/Images.xcassets/_tab_digest_selected.imageset/_tab_digest_selected.svg
 create mode 100644 apple/OmnivoreKit/Sources/Views/Images/Images.xcassets/digest-archive-button.imageset/Contents.json
 create mode 100644 apple/OmnivoreKit/Sources/Views/Images/Images.xcassets/digest-archive-button.imageset/digest-archive-button.svg
 create mode 100644 apple/OmnivoreKit/Sources/Views/Images/Images.xcassets/digest-dots-button.imageset/Contents.json
 create mode 100644 apple/OmnivoreKit/Sources/Views/Images/Images.xcassets/digest-dots-button.imageset/digest-dots-button.svg
 create mode 100644 apple/OmnivoreKit/Sources/Views/Images/Images.xcassets/digest-play-button.imageset/Contents.json
 create mode 100644 apple/OmnivoreKit/Sources/Views/Images/Images.xcassets/digest-play-button.imageset/digest-play-button.svg
 create mode 100644 apple/OmnivoreKit/Sources/Views/Images/Images.xcassets/digest-trash-button.imageset/Contents.json
 create mode 100644 apple/OmnivoreKit/Sources/Views/Images/Images.xcassets/digest-trash-button.imageset/digest-trash-button.svg
 create mode 100644 apple/OmnivoreKit/Sources/Views/Images/Images.xcassets/double_chevron_up.imageset/Contents.json
 create mode 100644 apple/OmnivoreKit/Sources/Views/Images/Images.xcassets/double_chevron_up.imageset/double_chevron_up.svg

diff --git a/apple/OmnivoreKit/Sources/App/Views/AI/LibraryDigestView.swift b/apple/OmnivoreKit/Sources/App/Views/AI/LibraryDigestView.swift
new file mode 100644
index 000000000..e6f4f91d7
--- /dev/null
+++ b/apple/OmnivoreKit/Sources/App/Views/AI/LibraryDigestView.swift
@@ -0,0 +1,267 @@
+import SwiftUI
+import Models
+import Services
+
+
+struct DigestItem {
+  let id: String
+  let site: String
+  let siteIcon: URL?
+  let author: String
+  let title: String
+  let summaryText: String
+  let keyPointsText: String
+  let highlightsText: String
+}
+
+@available(iOS 17.0, *)
+@MainActor
+struct LibraryDigestView: View {
+  let dataService: DataService
+    // @State private var currentIndex = 0
+  @State private var items: [DigestItem]
+    // @State private var preloadedItems: [Int: String] = [:]
+  @Environment(\.dismiss) private var dismiss
+
+  //  let itemCount = 10 // Number of items to initially load
+  //  let prefetchCount = 2 // Number of items to prefetch
+
+  public init(dataService: DataService) {
+    self.dataService = dataService
+    self.items = [
+      DigestItem(
+        id: "1468AFAA-88sdfsdfC-4546-BE02-EACF385288FC",
+        site: "CNBC.com",
+        siteIcon: URL(string: "https://www.cnbc.com/favicon.ico"),
+        author: "Kif Leswing",
+        title: "Apple shares just had their best day since last May",
+        summaryText: "In a significant political turn, the SOTU response faces unexpected collapse, marking a stark contrast to Trump's latest" +
+        " downturn, alongside an unprecedented surge in Biden's fundraising efforts as of 3/11/24, according to the TDPS Podcast. " +
+        "The analysis provides insights into the shifting dynamics of political support and the potential implications for future " +
+        "electoral strategies. ",
+        keyPointsText: "Key points from the article:",
+        highlightsText: "Highlights from the article:"
+      ),
+      DigestItem(
+        id: "1468AFAA-8sdfsdffsdf-4546-BE02-EACF385288FC",
+        site: "CNBC.com",
+        siteIcon: URL(string: "https://www.cnbc.com/favicon.ico"),
+        author: "Kif Leswing",
+        title: "Apple shares just had their best day since last May",
+        summaryText: "In a significant political turn, the SOTU response faces unexpected collapse, marking a stark contrast to Trump's latest" +
+        " downturn, alongside an unprecedented surge in Biden's fundraising efforts as of 3/11/24, according to the TDPS Podcast. " +
+        "The analysis provides insights into the shifting dynamics of political support and the potential implications for future " +
+        "electoral strategies. ",
+        keyPointsText: "Key points from the article:",
+        highlightsText: "Highlights from the article:"
+      ),
+      DigestItem(
+        id: "1468AFAA-882C-asdadfsa85288FC",
+        site: "CNBC.com",
+        siteIcon: URL(string: "https://www.cnbc.com/favicon.ico"),
+        author: "Kif Leswing",
+        title: "Apple shares just had their best day since last May",
+        summaryText: "In a significant political turn, the SOTU response faces unexpected collapse, marking a stark contrast to Trump's latest" +
+        " downturn, alongside an unprecedented surge in Biden's fundraising efforts as of 3/11/24, according to the TDPS Podcast. " +
+        "The analysis provides insights into the shifting dynamics of political support and the potential implications for future " +
+        "electoral strategies. ",
+        keyPointsText: "Key points from the article:",
+        highlightsText: "Highlights from the article:"
+      )
+    ]
+    // currentIndex = 0
+   // _preloadedItems = [Int:String]
+  }
+
+  var body: some View {
+    itemBody
+  }
+
+  @available(iOS 17.0, *)
+  var itemBody: some View {
+      ScrollView(.vertical) {
+        LazyVStack(spacing: 0) {
+          ForEach(Array(self.items.enumerated()), id: \.1.id) { idx, item in
+            PreviewItemView(
+              viewModel: PreviewItemViewModel(dataService: dataService, item: item, showSwipeHint: idx == 0)
+            )
+            .containerRelativeFrame([.horizontal, .vertical])
+          }
+        }
+        .scrollTargetLayout()
+      }
+      .scrollTargetBehavior(.paging)
+      .ignoresSafeArea()
+
+      
+      //        ScrollView(.horizontal, showsIndicators: false) {
+      //            HStack(spacing: 0) {
+      //                ForEach(items.indices, id: \.self) { index in
+      //                    if let item = preloadedItems[index] {
+      //                        ItemView(content: item)
+      //                            .onAppear {
+      //                                if index == items.count - prefetchCount {
+      //                                    fetchItems()
+      //                                }
+      //                            }
+      //                    }
+      //                }
+      //            }
+      //        }
+      //        .frame(maxWidth: .infinity, maxHeight: .infinity)
+      .onAppear {
+        // fetchItems()
+      }
+      .onReceive(NotificationCenter.default.publisher(for: UIApplication.willResignActiveNotification)) { _ in
+        // Pause any background tasks if necessary
+      }
+    
+    }
+    
+//    private func fetchItems() {
+//        // Simulate fetching items from an API
+//        for idx in currentIndex..<currentIndex + itemCount {
+//            fetchItem(index: idx)
+//        }
+//        currentIndex += itemCount
+//    }
+//    
+//    private func fetchItem(index: Int) {
+//        // Simulate fetching item content from an API
+//        DispatchQueue.main.asyncAfter(deadline: .now() + 1) {
+//            items.append("Item \(index + 1)")
+//            preloadNextItemIfNeeded(index: index)
+//        }
+//    }
+//    
+//    private func preloadNextItemIfNeeded(index: Int) {
+//        let nextIndex = index + 1
+//        if nextIndex < currentIndex + prefetchCount {
+//            fetchItem(index: nextIndex)
+//        }
+//    }
+}
+
+@MainActor
+public class PreviewItemViewModel: ObservableObject {
+  let dataService: DataService
+  @Published var item: DigestItem
+  let showSwipeHint: Bool
+
+  @Published var isLoading = false
+  @Published var resultText: String?
+  @Published var promptDisplayText: String?
+
+  init(dataService: DataService, item: DigestItem, showSwipeHint: Bool) {
+    self.dataService = dataService
+    self.item = item
+    self.showSwipeHint = showSwipeHint
+  }
+
+  func loadResult() async {
+//    isLoading = true
+//    let taskId = try? await dataService.createAITask(
+//      extraText: extraText,
+//      libraryItemId: item?.id ?? "",
+//      promptName: "summarize-001"
+//    )
+//
+//    if let taskId = taskId {
+//      do {
+//        let fetchedText = try await dataService.pollAITask(jobId: taskId, timeoutInterval: 30)
+//        resultText = fetchedText
+//      } catch {
+//        print("ERROR WITH RESULT TEXT: ", error)
+//      }
+//    } else {
+//      print("NO TASK ID: ", taskId)
+//    }
+//    isLoading = false
+  }
+}
+
+@MainActor
+struct PreviewItemView: View {
+  @StateObject var viewModel: PreviewItemViewModel
+
+    var body: some View {
+      VStack(spacing: 10) {
+        HStack {
+          AsyncImage(url: viewModel.item.siteIcon) { phase in
+            if let image = phase.image {
+              image
+                .resizable()
+                .aspectRatio(contentMode: .fill)
+                .frame(width: 20, height: 20, alignment: .center)
+            } else {
+              Color.appButtonBackground
+                .frame(width: 20, height: 20, alignment: .center)
+            }
+          }
+          Text(viewModel.item.site)
+            .font(Font.system(size: 14))
+            .frame(maxWidth: .infinity, alignment: .topLeading)
+        }
+        .padding(.top, 10)
+        Text(viewModel.item.title)
+          // .font(.body)
+          // .fontWeight(.semibold)
+          .font(Font.system(size: 18, weight: .semibold))
+          .frame(maxWidth: .infinity, alignment: .topLeading)
+
+        Text(viewModel.item.author)
+          .font(Font.system(size: 14))
+          .foregroundColor(Color(hex: "898989"))
+          .frame(maxWidth: .infinity, alignment: .topLeading)
+
+        Color(hex: "2A2A2A")
+          .frame(height: 1)
+          .frame(maxWidth: .infinity, alignment: .center)
+          .padding(.vertical, 20)
+
+        if viewModel.isLoading {
+          ProgressView()
+            .task {
+              await viewModel.loadResult()
+            }
+            .frame(maxWidth: .infinity, maxHeight: .infinity)
+        } else {
+          Text(viewModel.item.summaryText)
+            .font(Font.system(size: 16))
+            // .font(.body)
+            .lineSpacing(12.0)
+            .frame(maxWidth: .infinity, alignment: .topLeading)
+          HStack {
+            Button(action: {}, label: {
+              HStack(alignment: .center) {
+                Text("Start listening")
+                  .font(Font.system(size: 14))
+                  .frame(height: 42, alignment: .center)
+                Image(systemName: "play.fill")
+                  .resizable()
+                  .frame(width: 10, height: 10)
+              }
+              .padding(.horizontal, 15)
+              .background(Color.blue)
+              .foregroundColor(.white)
+              .cornerRadius(18)
+            })
+            Spacer()
+          }
+          .padding(.top, 20)
+        }
+        Spacer()
+        if viewModel.showSwipeHint {
+          VStack {
+            Image.doubleChevronUp
+            Text("Swipe up for next article")
+              .foregroundColor(Color(hex: "898989"))
+          }
+          .padding(.bottom, 50)
+        }
+      }.frame(maxWidth: .infinity, maxHeight: .infinity)
+        .padding(.top, 100)
+        .padding(.horizontal, 15)
+
+    }
+}
diff --git a/apple/OmnivoreKit/Sources/App/Views/BottomBars/CustomTabBar.swift b/apple/OmnivoreKit/Sources/App/Views/BottomBars/CustomTabBar.swift
index 4b9691d12..eb0bf53d1 100644
--- a/apple/OmnivoreKit/Sources/App/Views/BottomBars/CustomTabBar.swift
+++ b/apple/OmnivoreKit/Sources/App/Views/BottomBars/CustomTabBar.swift
@@ -8,10 +8,19 @@ struct CustomTabBar: View {
   var body: some View {
     HStack(spacing: 0) {
       if !hideFollowingTab {
-        TabBarButton(key: "following", image: Image.tabFollowing, selectedTab: $selectedTab)
+        TabBarButton(key: "following",
+                     image: Image.tabFollowing,
+                     selectedTab: $selectedTab,
+                     selectionColor: Color(hex: "EE8232"))
       }
-      TabBarButton(key: "inbox", image: Image.tabLibrary, selectedTab: $selectedTab)
-      TabBarButton(key: "profile", image: Image.tabProfile, selectedTab: $selectedTab)
+      TabBarButton(key: "digest",
+                   image: Image.tabDigest,
+                   selectedTab: $selectedTab,
+                   selectedImage: Image.tabDigestSelected)
+      TabBarButton(key: "inbox",
+                   image: Image.tabLibrary,
+                   selectedTab: $selectedTab)
+      // TabBarButton(key: "profile", image: Image.tabProfile, selectedTab: $selectedTab)
     }
     .padding(.top, 10)
     .padding(.bottom, 10)
@@ -23,6 +32,8 @@ struct TabBarButton: View {
   let key: String
   let image: Image
   @Binding var selectedTab: String
+  var selectedImage: Image?
+  var selectionColor: Color?
 
   var body: some View {
     Button(action: {
@@ -31,13 +42,20 @@ struct TabBarButton: View {
       }
       selectedTab = key
     }, label: {
-      image
-        .resizable()
-        .renderingMode(.template)
-        .aspectRatio(contentMode: .fit)
+     tabImage
         .frame(width: 28, height: 28)
-        .foregroundColor(selectedTab == key ? Color.blue : Color.themeTabButtonColor)
         .frame(maxWidth: .infinity)
     }).buttonStyle(.plain)
   }
+  
+  var tabImage: some View {
+    if let selectedImage = selectedImage, selectedTab == key {
+      return AnyView(selectedImage
+        .resizable()
+        .aspectRatio(contentMode: .fit))
+    } else {
+      return AnyView(image
+        .foregroundColor(selectedTab == key ? selectionColor ?? Color.blue  : Color.themeTabButtonColor))
+    }
+  }
 }
diff --git a/apple/OmnivoreKit/Sources/App/Views/Home/HomeFeedViewIOS.swift b/apple/OmnivoreKit/Sources/App/Views/Home/HomeFeedViewIOS.swift
index 5611537e9..52ae844be 100644
--- a/apple/OmnivoreKit/Sources/App/Views/Home/HomeFeedViewIOS.swift
+++ b/apple/OmnivoreKit/Sources/App/Views/Home/HomeFeedViewIOS.swift
@@ -191,6 +191,7 @@ struct AnimatingCellHeight: AnimatableModifier {
     @State var isListScrolled = false
     @State var listTitle = ""
     @State var showExpandedAudioPlayer = false
+    @State var showLibraryDigest = false
 
     @Binding var isEditMode: EditMode
 
@@ -382,6 +383,15 @@ struct AnimatingCellHeight: AnimatableModifier {
               if isEditMode == .active {
                 Button(action: { isEditMode = .inactive }, label: { Text("Cancel") })
               } else {
+                
+//                if #available(iOS 17.0, *) {
+//                  Button(
+//                    action: { showLibraryDigest = true },
+//                    label: { Image(systemName: "sparkles") }
+//                  )
+//                  .buttonStyle(.plain)
+//                  .padding(.trailing, 4)
+//                }
                 if prefersListLayout {
                   Button(
                     action: { isEditMode = isEditMode == .active ? .inactive : .active },
diff --git a/apple/OmnivoreKit/Sources/App/Views/LibraryTabView.swift b/apple/OmnivoreKit/Sources/App/Views/LibraryTabView.swift
index f8a16d598..0a340ceed 100644
--- a/apple/OmnivoreKit/Sources/App/Views/LibraryTabView.swift
+++ b/apple/OmnivoreKit/Sources/App/Views/LibraryTabView.swift
@@ -116,11 +116,19 @@ struct LibraryTabView: View {
           }.tag("following")
         }
 
-        NavigationView {
-          HomeFeedContainerView(viewModel: inboxViewModel, isEditMode: $isEditMode)
-            .navigationBarTitleDisplayMode(.inline)
-            .navigationViewStyle(.stack)
-        }.tag("inbox")
+        if #available(iOS 17.0, *) {
+          NavigationView {
+            LibraryDigestView(dataService: dataService)
+              .navigationBarTitleDisplayMode(.inline)
+              .navigationViewStyle(.stack)
+          }.tag("digest")
+        } else {
+          NavigationView {
+            HomeFeedContainerView(viewModel: inboxViewModel, isEditMode: $isEditMode)
+              .navigationBarTitleDisplayMode(.inline)
+              .navigationViewStyle(.stack)
+          }.tag("inbox")
+        }
 
         NavigationView {
           ProfileView()
diff --git a/apple/OmnivoreKit/Sources/App/Views/WebReader/WebReader.swift b/apple/OmnivoreKit/Sources/App/Views/WebReader/WebReader.swift
index 8c5468f16..beaaa9cc7 100644
--- a/apple/OmnivoreKit/Sources/App/Views/WebReader/WebReader.swift
+++ b/apple/OmnivoreKit/Sources/App/Views/WebReader/WebReader.swift
@@ -11,6 +11,7 @@ struct WebReader: PlatformViewRepresentable {
   let articleContent: ArticleContent
   let openLinkAction: (URL) -> Void
   let tapHandler: () -> Void
+  let explainHandler: ((String) -> Void)?
   let scrollPercentHandler: (Int) -> Void
   let webViewActionHandler: (WKScriptMessage, WKScriptMessageReplyHandler?) -> Void
   let navBarVisibilityUpdater: (Bool) -> Void
@@ -51,6 +52,7 @@ struct WebReader: PlatformViewRepresentable {
     let contentController = WKUserContentController()
 
     webView.tapHandler = tapHandler
+    webView.explainHandler = explainHandler
     webView.navigationDelegate = context.coordinator
     webView.configuration.userContentController = contentController
     webView.configuration.userContentController.removeAllScriptMessageHandlers()
diff --git a/apple/OmnivoreKit/Sources/App/Views/WebReader/WebReaderContainer.swift b/apple/OmnivoreKit/Sources/App/Views/WebReader/WebReaderContainer.swift
index 1cb4aab3d..5eebee5ee 100644
--- a/apple/OmnivoreKit/Sources/App/Views/WebReader/WebReaderContainer.swift
+++ b/apple/OmnivoreKit/Sources/App/Views/WebReader/WebReaderContainer.swift
@@ -41,6 +41,8 @@ struct WebReaderContainerView: View {
   @State var displayLinkSheet = false
   @State var linkToOpen: URL?
 
+  @State var showExplainSheet = false
+
   @EnvironmentObject var dataService: DataService
   @EnvironmentObject var audioController: AudioController
   @Environment(\.openURL) var openURL
@@ -92,6 +94,11 @@ struct WebReaderContainerView: View {
     }
   }
 
+  private func explainHandler(text: String) {
+    viewModel.explainText = String(text)
+    showExplainSheet = true
+  }
+
   private func handleHighlightAction(message: WKScriptMessage) {
     guard let messageBody = message.body as? [String: String] else { return }
     guard let actionID = messageBody["actionID"] else { return }
@@ -271,6 +278,13 @@ struct WebReaderContainerView: View {
         Spacer()
       #endif
 
+//      Button(
+//        action: { showExplainSheet = true },
+//        label: { Image(systemName: "sparkles") }
+//      )
+//      .buttonStyle(.plain)
+//      .padding(.trailing, 4)
+
       Button(
         action: { showLabelsModal = true },
         label: {
@@ -378,6 +392,7 @@ struct WebReaderContainerView: View {
             #endif
           },
           tapHandler: tapHandler,
+          explainHandler: explainHandler,
           scrollPercentHandler: scrollPercentHandler,
           webViewActionHandler: webViewActionHandler,
           navBarVisibilityUpdater: { visible in
@@ -485,6 +500,20 @@ struct WebReaderContainerView: View {
           .formSheet(isPresented: $showOpenArchiveSheet) {
             OpenArchiveTodayView(item: item)
           }
+          .formSheet(isPresented: $showExplainSheet) {
+            ExplainView(
+              viewModel: ExplainViewModel(
+                dataService: dataService,
+                item: self.item,
+                promptName: "explain-text-001",
+                extraText: viewModel.explainText
+              ),
+              dismissAction: {
+                viewModel.explainText = nil
+                showExplainSheet = false
+              }
+            )
+          }
         #endif
         .sheet(isPresented: $showHighlightAnnotationModal) {
           NavigationView {
@@ -633,7 +662,6 @@ struct WebReaderContainerView: View {
       try? WebViewManager.shared().dispatchEvent(.saveReadPosition)
     }
     .onDisappear {
-      // WebViewManager.shared().loadHTMLString("<html></html>", baseURL: nil)
       WebViewManager.shared().loadHTMLString(WebReaderContent.emptyContent(isDark: Color.isDarkMode), baseURL: nil)
     }
     .onReceive(NotificationCenter.default.publisher(for: Notification.Name("PopToRoot"))) { _ in
@@ -674,7 +702,7 @@ struct WebReaderContainerView: View {
     shareActionID = UUID()
   }
   
-  func print() {
+  func printReader() {
     shareActionID = UUID()
   }
 
diff --git a/apple/OmnivoreKit/Sources/App/Views/WebReader/WebReaderViewModel.swift b/apple/OmnivoreKit/Sources/App/Views/WebReader/WebReaderViewModel.swift
index 1a70626cf..7d5071194 100644
--- a/apple/OmnivoreKit/Sources/App/Views/WebReader/WebReaderViewModel.swift
+++ b/apple/OmnivoreKit/Sources/App/Views/WebReader/WebReaderViewModel.swift
@@ -21,6 +21,8 @@ struct SafariWebLink: Identifiable {
   @Published var showOperationToast: Bool = false
   @Published var operationStatus: OperationStatus = .none
 
+  @Published var explainText: String?
+
   func hasOriginalUrl(_ item: Models.LibraryItem) -> Bool {
     if let pageURLString = item.pageURLString, let host = URL(string: pageURLString)?.host {
       if host == "omnivore.app" {
diff --git a/apple/OmnivoreKit/Sources/Services/Authentication/AuthModels.swift b/apple/OmnivoreKit/Sources/Services/Authentication/AuthModels.swift
index 550389c56..fb450b0d6 100644
--- a/apple/OmnivoreKit/Sources/Services/Authentication/AuthModels.swift
+++ b/apple/OmnivoreKit/Sources/Services/Authentication/AuthModels.swift
@@ -75,7 +75,7 @@ extension LoginError {
       return .unauthorized
     case .unknown:
       return .unknown
-    case .pendingEmailVerification:
+    case .pendingEmailVerification, .stillProcessing:
       return .pendingEmailVerification
     }
   }
diff --git a/apple/OmnivoreKit/Sources/Services/DataService/Errors/ServerError.swift b/apple/OmnivoreKit/Sources/Services/DataService/Errors/ServerError.swift
index 601b2e4cf..7e51b2a1a 100644
--- a/apple/OmnivoreKit/Sources/Services/DataService/Errors/ServerError.swift
+++ b/apple/OmnivoreKit/Sources/Services/DataService/Errors/ServerError.swift
@@ -10,6 +10,7 @@ public enum ServerError: String, Error {
   case unauthenticated
   case timeout
   case unknown
+  case stillProcessing
   case pendingEmailVerification
 }
 
@@ -22,6 +23,8 @@ extension ServerError {
     case 401?, 403?:
       self = .unauthenticated
       return
+    case 202:
+      self = .stillProcessing
     default:
       break
     }
diff --git a/apple/OmnivoreKit/Sources/Views/Article/OmnivoreWebView.swift b/apple/OmnivoreKit/Sources/Views/Article/OmnivoreWebView.swift
index af128dd9b..05dcfb44d 100644
--- a/apple/OmnivoreKit/Sources/Views/Article/OmnivoreWebView.swift
+++ b/apple/OmnivoreKit/Sources/Views/Article/OmnivoreWebView.swift
@@ -23,6 +23,7 @@ public final class OmnivoreWebView: WKWebView {
   #endif
 
   public var tapHandler: (() -> Void)?
+  public var explainHandler: ((String) -> Void)?
 
   private var currentMenu: ContextMenu = .defaultMenu
 
@@ -298,6 +299,7 @@ public final class OmnivoreWebView: WKWebView {
       case #selector(removeSelection): return true
       case #selector(copy(_:)): return true
       case #selector(setLabels(_:)): return true
+      case #selector(explainSelection): return true
 
       case Selector(("_lookup:")): return (currentMenu == .defaultMenu)
       case Selector(("_define:")): return (currentMenu == .defaultMenu)
@@ -334,6 +336,18 @@ public final class OmnivoreWebView: WKWebView {
       hideMenu()
     }
 
+    @objc private func explainSelection() {
+      Task {
+        let selection = try? await self.evaluateJavaScript("window.getSelection().toString()")
+        if let selection = selection as? String, let explainHandler = explainHandler {
+          print("Explaining \(selection)")
+          explainHandler(selection)
+        } else {
+          showInReaderSnackbar("Error getting text to explain")
+        }
+      }
+    }
+
     @objc private func shareSelection() {
       do {
         try dispatchEvent(.share)
@@ -386,7 +400,8 @@ public final class OmnivoreWebView: WKWebView {
             return
           }
           let highlight = UICommand(title: LocalText.genericHighlight, action: #selector(highlightSelection))
-          items = [highlight, annotate]
+          // let explain = UICommand(title: "Explain", action: #selector(explainSelection))
+          items = [highlight, /* explain, */ annotate]
         } else {
           let remove = UICommand(title: "Remove", action: #selector(removeSelection))
           let setLabels = UICommand(title: LocalText.labelsGeneric, action: #selector(setLabels))
diff --git a/apple/OmnivoreKit/Sources/Views/Images/Images.swift b/apple/OmnivoreKit/Sources/Views/Images/Images.swift
index f90d34ede..9b684783e 100644
--- a/apple/OmnivoreKit/Sources/Views/Images/Images.swift
+++ b/apple/OmnivoreKit/Sources/Views/Images/Images.swift
@@ -9,6 +9,9 @@ public extension Image {
 
   static var tabFollowing: Image { Image("_tab_following", bundle: .module).renderingMode(.template) }
   static var tabLibrary: Image { Image("_tab_library", bundle: .module).renderingMode(.template) }
+  static var tabDigest: Image { Image("_tab_digest", bundle: .module).renderingMode(.template) }
+  static var tabDigestSelected: Image { Image("_tab_digest_selected", bundle: .module) }
+
   static var tabSearch: Image { Image("_tab_search", bundle: .module).renderingMode(.template) }
   static var tabHighlights: Image { Image("_tab_highlights", bundle: .module).renderingMode(.template) }
   static var tabProfile: Image { Image("_tab_profile", bundle: .module).renderingMode(.template) }
@@ -49,4 +52,7 @@ public extension Image {
   static var flairNewsletter: Image { Image("flair-newsletter", bundle: .module) }
   static var flairPinned: Image { Image("flair-pinned", bundle: .module) }
   static var flairRecommended: Image { Image("flair-recommended", bundle: .module) }
+  
+  static var doubleChevronUp: Image { Image("double_chevron_up", bundle: .module) }
+
 }
diff --git a/apple/OmnivoreKit/Sources/Views/Images/Images.xcassets/_tab_digest.imageset/Contents.json b/apple/OmnivoreKit/Sources/Views/Images/Images.xcassets/_tab_digest.imageset/Contents.json
new file mode 100644
index 000000000..83aca2101
--- /dev/null
+++ b/apple/OmnivoreKit/Sources/Views/Images/Images.xcassets/_tab_digest.imageset/Contents.json
@@ -0,0 +1,24 @@
+{
+  "images" : [
+    {
+      "idiom" : "universal",
+      "scale" : "1x"
+    },
+    {
+      "filename" : "_tab_digest.svg",
+      "idiom" : "universal",
+      "scale" : "2x"
+    },
+    {
+      "idiom" : "universal",
+      "scale" : "3x"
+    }
+  ],
+  "info" : {
+    "author" : "xcode",
+    "version" : 1
+  },
+  "properties" : {
+    "template-rendering-intent" : "template"
+  }
+}
diff --git a/apple/OmnivoreKit/Sources/Views/Images/Images.xcassets/_tab_digest.imageset/_tab_digest.svg b/apple/OmnivoreKit/Sources/Views/Images/Images.xcassets/_tab_digest.imageset/_tab_digest.svg
new file mode 100644
index 000000000..ccf49775d
--- /dev/null
+++ b/apple/OmnivoreKit/Sources/Views/Images/Images.xcassets/_tab_digest.imageset/_tab_digest.svg
@@ -0,0 +1,18 @@
+<svg width="25" height="25" viewBox="0 0 25 25" fill="none" xmlns="http://www.w3.org/2000/svg">
+<g clip-path="url(#clip0_10231_7167)">
+<path d="M12.5 3.5V5.5" stroke="#8E8E93" stroke-width="1.5" stroke-linecap="round" stroke-linejoin="round"/>
+<path d="M12.5 19.5V21.5" stroke="#8E8E93" stroke-width="1.5" stroke-linecap="round" stroke-linejoin="round"/>
+<path d="M12.5 8.5V16.5" stroke="#8E8E93" stroke-width="1.5" stroke-linecap="round" stroke-linejoin="round"/>
+<path d="M8.5 16.5V19.5" stroke="#8E8E93" stroke-width="1.5" stroke-linecap="round" stroke-linejoin="round"/>
+<path d="M4.5 10.5V15.5" stroke="#8E8E93" stroke-width="1.5" stroke-linecap="round" stroke-linejoin="round"/>
+<path d="M20.5 10.5V16.5" stroke="#8E8E93" stroke-width="1.5" stroke-linecap="round" stroke-linejoin="round"/>
+<path d="M8.5 6.5V13.5" stroke="#8E8E93" stroke-width="1.5" stroke-linecap="round" stroke-linejoin="round"/>
+<path d="M16.5 9.5V6.5" stroke="#8E8E93" stroke-width="1.5" stroke-linecap="round" stroke-linejoin="round"/>
+<path d="M16.5 19.5V12.5" stroke="#8E8E93" stroke-width="1.5" stroke-linecap="round" stroke-linejoin="round"/>
+</g>
+<defs>
+<clipPath id="clip0_10231_7167">
+<rect width="24" height="24" fill="white" transform="translate(0.5 0.5)"/>
+</clipPath>
+</defs>
+</svg>
diff --git a/apple/OmnivoreKit/Sources/Views/Images/Images.xcassets/_tab_digest_selected.imageset/Contents.json b/apple/OmnivoreKit/Sources/Views/Images/Images.xcassets/_tab_digest_selected.imageset/Contents.json
new file mode 100644
index 000000000..e5fed4197
--- /dev/null
+++ b/apple/OmnivoreKit/Sources/Views/Images/Images.xcassets/_tab_digest_selected.imageset/Contents.json
@@ -0,0 +1,21 @@
+{
+  "images" : [
+    {
+      "idiom" : "universal",
+      "scale" : "1x"
+    },
+    {
+      "filename" : "_tab_digest_selected.svg",
+      "idiom" : "universal",
+      "scale" : "2x"
+    },
+    {
+      "idiom" : "universal",
+      "scale" : "3x"
+    }
+  ],
+  "info" : {
+    "author" : "xcode",
+    "version" : 1
+  }
+}
diff --git a/apple/OmnivoreKit/Sources/Views/Images/Images.xcassets/_tab_digest_selected.imageset/_tab_digest_selected.svg b/apple/OmnivoreKit/Sources/Views/Images/Images.xcassets/_tab_digest_selected.imageset/_tab_digest_selected.svg
new file mode 100644
index 000000000..dfc6d541d
--- /dev/null
+++ b/apple/OmnivoreKit/Sources/Views/Images/Images.xcassets/_tab_digest_selected.imageset/_tab_digest_selected.svg
@@ -0,0 +1,18 @@
+<svg width="25" height="25" viewBox="0 0 25 25" fill="none" xmlns="http://www.w3.org/2000/svg">
+<g clip-path="url(#clip0_10210_6950)">
+<path d="M12.5 3.5V5.5" stroke="#767AF8" stroke-width="1.5" stroke-linecap="round" stroke-linejoin="round"/>
+<path d="M12.5 19.5V21.5" stroke="#DE76F8" stroke-width="1.5" stroke-linecap="round" stroke-linejoin="round"/>
+<path d="M12.5 8.5V16.5" stroke="#767AF8" stroke-width="1.5" stroke-linecap="round" stroke-linejoin="round"/>
+<path d="M8.5 16.5V19.5" stroke="#76AAF8" stroke-width="1.5" stroke-linecap="round" stroke-linejoin="round"/>
+<path d="M4.5 10.5V15.5" stroke="#76F8D1" stroke-width="1.5" stroke-linecap="round" stroke-linejoin="round"/>
+<path d="M20.5 10.5V16.5" stroke="#76F8D1" stroke-width="1.5" stroke-linecap="round" stroke-linejoin="round"/>
+<path d="M8.5 6.5V13.5" stroke="#CF76F8" stroke-width="1.5" stroke-linecap="round" stroke-linejoin="round"/>
+<path d="M16.5 9.5V6.5" stroke="#CF76F8" stroke-width="1.5" stroke-linecap="round" stroke-linejoin="round"/>
+<path d="M16.5 19.5V12.5" stroke="#76AAF8" stroke-width="1.5" stroke-linecap="round" stroke-linejoin="round"/>
+</g>
+<defs>
+<clipPath id="clip0_10210_6950">
+<rect width="24" height="24" fill="white" transform="translate(0.5 0.5)"/>
+</clipPath>
+</defs>
+</svg>
diff --git a/apple/OmnivoreKit/Sources/Views/Images/Images.xcassets/digest-archive-button.imageset/Contents.json b/apple/OmnivoreKit/Sources/Views/Images/Images.xcassets/digest-archive-button.imageset/Contents.json
new file mode 100644
index 000000000..340a487c3
--- /dev/null
+++ b/apple/OmnivoreKit/Sources/Views/Images/Images.xcassets/digest-archive-button.imageset/Contents.json
@@ -0,0 +1,24 @@
+{
+  "images" : [
+    {
+      "filename" : "digest-archive-button.svg",
+      "idiom" : "universal",
+      "scale" : "1x"
+    },
+    {
+      "idiom" : "universal",
+      "scale" : "2x"
+    },
+    {
+      "idiom" : "universal",
+      "scale" : "3x"
+    }
+  ],
+  "info" : {
+    "author" : "xcode",
+    "version" : 1
+  },
+  "properties" : {
+    "template-rendering-intent" : "template"
+  }
+}
diff --git a/apple/OmnivoreKit/Sources/Views/Images/Images.xcassets/digest-archive-button.imageset/digest-archive-button.svg b/apple/OmnivoreKit/Sources/Views/Images/Images.xcassets/digest-archive-button.imageset/digest-archive-button.svg
new file mode 100644
index 000000000..ecdbd0d54
--- /dev/null
+++ b/apple/OmnivoreKit/Sources/Views/Images/Images.xcassets/digest-archive-button.imageset/digest-archive-button.svg
@@ -0,0 +1,13 @@
+<svg width="35" height="35" viewBox="0 0 35 35" fill="none" xmlns="http://www.w3.org/2000/svg">
+<rect x="0.5" y="0.5" width="34" height="34" rx="17" stroke="#3D3D3D"/>
+<g clip-path="url(#clip0_9916_1773)">
+<path d="M10 12.5002C10 12.0581 10.1756 11.6342 10.4882 11.3217C10.8007 11.0091 11.2246 10.8335 11.6667 10.8335H23.3333C23.7754 10.8335 24.1993 11.0091 24.5118 11.3217C24.8244 11.6342 25 12.0581 25 12.5002C25 12.9422 24.8244 13.3661 24.5118 13.6787C24.1993 13.9912 23.7754 14.1668 23.3333 14.1668H11.6667C11.2246 14.1668 10.8007 13.9912 10.4882 13.6787C10.1756 13.3661 10 12.9422 10 12.5002Z" stroke="#EDEDED" stroke-width="1.25" stroke-linecap="round" stroke-linejoin="round"/>
+<path d="M11.6667 14.1665V22.4998C11.6667 22.9419 11.8423 23.3658 12.1549 23.6783C12.4675 23.9909 12.8914 24.1665 13.3334 24.1665H21.6667C22.1088 24.1665 22.5327 23.9909 22.8453 23.6783C23.1578 23.3658 23.3334 22.9419 23.3334 22.4998V14.1665" stroke="#EDEDED" stroke-width="1.25" stroke-linecap="round" stroke-linejoin="round"/>
+<path d="M15.8333 17.5H19.1666" stroke="#EDEDED" stroke-width="1.25" stroke-linecap="round" stroke-linejoin="round"/>
+</g>
+<defs>
+<clipPath id="clip0_9916_1773">
+<rect width="20" height="20" fill="white" transform="translate(7.5 7.5)"/>
+</clipPath>
+</defs>
+</svg>
diff --git a/apple/OmnivoreKit/Sources/Views/Images/Images.xcassets/digest-dots-button.imageset/Contents.json b/apple/OmnivoreKit/Sources/Views/Images/Images.xcassets/digest-dots-button.imageset/Contents.json
new file mode 100644
index 000000000..d8d84c76c
--- /dev/null
+++ b/apple/OmnivoreKit/Sources/Views/Images/Images.xcassets/digest-dots-button.imageset/Contents.json
@@ -0,0 +1,24 @@
+{
+  "images" : [
+    {
+      "filename" : "digest-dots-button.svg",
+      "idiom" : "universal",
+      "scale" : "1x"
+    },
+    {
+      "idiom" : "universal",
+      "scale" : "2x"
+    },
+    {
+      "idiom" : "universal",
+      "scale" : "3x"
+    }
+  ],
+  "info" : {
+    "author" : "xcode",
+    "version" : 1
+  },
+  "properties" : {
+    "template-rendering-intent" : "template"
+  }
+}
diff --git a/apple/OmnivoreKit/Sources/Views/Images/Images.xcassets/digest-dots-button.imageset/digest-dots-button.svg b/apple/OmnivoreKit/Sources/Views/Images/Images.xcassets/digest-dots-button.imageset/digest-dots-button.svg
new file mode 100644
index 000000000..41032d9ee
--- /dev/null
+++ b/apple/OmnivoreKit/Sources/Views/Images/Images.xcassets/digest-dots-button.imageset/digest-dots-button.svg
@@ -0,0 +1,13 @@
+<svg width="35" height="35" viewBox="0 0 35 35" fill="none" xmlns="http://www.w3.org/2000/svg">
+<rect x="0.5" y="0.5" width="34" height="34" rx="17" stroke="#3D3D3D"/>
+<g clip-path="url(#clip0_9916_1805)">
+<path d="M10.8333 17.4998C10.8333 17.7209 10.921 17.9328 11.0773 18.0891C11.2336 18.2454 11.4456 18.3332 11.6666 18.3332C11.8876 18.3332 12.0996 18.2454 12.2558 18.0891C12.4121 17.9328 12.4999 17.7209 12.4999 17.4998C12.4999 17.2788 12.4121 17.0669 12.2558 16.9106C12.0996 16.7543 11.8876 16.6665 11.6666 16.6665C11.4456 16.6665 11.2336 16.7543 11.0773 16.9106C10.921 17.0669 10.8333 17.2788 10.8333 17.4998Z" fill="#D9D9D9" stroke="#D9D9D9" stroke-width="1.5" stroke-linecap="round" stroke-linejoin="round"/>
+<path d="M16.6667 17.4998C16.6667 17.7209 16.7545 17.9328 16.9108 18.0891C17.0671 18.2454 17.2791 18.3332 17.5001 18.3332C17.7211 18.3332 17.9331 18.2454 18.0893 18.0891C18.2456 17.9328 18.3334 17.7209 18.3334 17.4998C18.3334 17.2788 18.2456 17.0669 18.0893 16.9106C17.9331 16.7543 17.7211 16.6665 17.5001 16.6665C17.2791 16.6665 17.0671 16.7543 16.9108 16.9106C16.7545 17.0669 16.6667 17.2788 16.6667 17.4998Z" fill="#D9D9D9" stroke="#D9D9D9" stroke-width="1.5" stroke-linecap="round" stroke-linejoin="round"/>
+<path d="M22.5 17.4998C22.5 17.7209 22.5878 17.9328 22.7441 18.0891C22.9004 18.2454 23.1123 18.3332 23.3333 18.3332C23.5543 18.3332 23.7663 18.2454 23.9226 18.0891C24.0789 17.9328 24.1667 17.7209 24.1667 17.4998C24.1667 17.2788 24.0789 17.0669 23.9226 16.9106C23.7663 16.7543 23.5543 16.6665 23.3333 16.6665C23.1123 16.6665 22.9004 16.7543 22.7441 16.9106C22.5878 17.0669 22.5 17.2788 22.5 17.4998Z" fill="#D9D9D9" stroke="#D9D9D9" stroke-width="1.5" stroke-linecap="round" stroke-linejoin="round"/>
+</g>
+<defs>
+<clipPath id="clip0_9916_1805">
+<rect width="20" height="20" fill="white" transform="translate(7.5 7.5)"/>
+</clipPath>
+</defs>
+</svg>
diff --git a/apple/OmnivoreKit/Sources/Views/Images/Images.xcassets/digest-play-button.imageset/Contents.json b/apple/OmnivoreKit/Sources/Views/Images/Images.xcassets/digest-play-button.imageset/Contents.json
new file mode 100644
index 000000000..04686abac
--- /dev/null
+++ b/apple/OmnivoreKit/Sources/Views/Images/Images.xcassets/digest-play-button.imageset/Contents.json
@@ -0,0 +1,24 @@
+{
+  "images" : [
+    {
+      "filename" : "digest-play-button.svg",
+      "idiom" : "universal",
+      "scale" : "1x"
+    },
+    {
+      "idiom" : "universal",
+      "scale" : "2x"
+    },
+    {
+      "idiom" : "universal",
+      "scale" : "3x"
+    }
+  ],
+  "info" : {
+    "author" : "xcode",
+    "version" : 1
+  },
+  "properties" : {
+    "template-rendering-intent" : "template"
+  }
+}
diff --git a/apple/OmnivoreKit/Sources/Views/Images/Images.xcassets/digest-play-button.imageset/digest-play-button.svg b/apple/OmnivoreKit/Sources/Views/Images/Images.xcassets/digest-play-button.imageset/digest-play-button.svg
new file mode 100644
index 000000000..45875dbb5
--- /dev/null
+++ b/apple/OmnivoreKit/Sources/Views/Images/Images.xcassets/digest-play-button.imageset/digest-play-button.svg
@@ -0,0 +1,11 @@
+<svg width="35" height="35" viewBox="0 0 35 35" fill="none" xmlns="http://www.w3.org/2000/svg">
+<rect x="0.5" y="0.5" width="34" height="34" rx="17" stroke="#3D3D3D"/>
+<g clip-path="url(#clip0_9916_1819)">
+<path d="M13.75 12.5002V22.5002C13.75 22.6114 13.7796 22.7206 13.8359 22.8165C13.8921 22.9124 13.9729 22.9916 14.07 23.0459C14.1671 23.1002 14.2769 23.1275 14.3881 23.1252C14.4992 23.1229 14.6078 23.0909 14.7025 23.0327L22.8275 18.0327C22.9185 17.9768 22.9937 17.8985 23.0458 17.8052C23.0979 17.712 23.1253 17.607 23.1253 17.5002C23.1253 17.3934 23.0979 17.2883 23.0458 17.1951C22.9937 17.1019 22.9185 17.0236 22.8275 16.9677L14.7025 11.9677C14.6078 11.9094 14.4992 11.8775 14.3881 11.8751C14.2769 11.8728 14.1671 11.9002 14.07 11.9545C13.9729 12.0087 13.8921 12.0879 13.8359 12.1838C13.7796 12.2798 13.75 12.389 13.75 12.5002Z" fill="#EDEDED"/>
+</g>
+<defs>
+<clipPath id="clip0_9916_1819">
+<rect width="15" height="15" fill="white" transform="translate(10 10)"/>
+</clipPath>
+</defs>
+</svg>
diff --git a/apple/OmnivoreKit/Sources/Views/Images/Images.xcassets/digest-trash-button.imageset/Contents.json b/apple/OmnivoreKit/Sources/Views/Images/Images.xcassets/digest-trash-button.imageset/Contents.json
new file mode 100644
index 000000000..fa3f18ec0
--- /dev/null
+++ b/apple/OmnivoreKit/Sources/Views/Images/Images.xcassets/digest-trash-button.imageset/Contents.json
@@ -0,0 +1,24 @@
+{
+  "images" : [
+    {
+      "filename" : "digest-trash-button.svg",
+      "idiom" : "universal",
+      "scale" : "1x"
+    },
+    {
+      "idiom" : "universal",
+      "scale" : "2x"
+    },
+    {
+      "idiom" : "universal",
+      "scale" : "3x"
+    }
+  ],
+  "info" : {
+    "author" : "xcode",
+    "version" : 1
+  },
+  "properties" : {
+    "template-rendering-intent" : "template"
+  }
+}
diff --git a/apple/OmnivoreKit/Sources/Views/Images/Images.xcassets/digest-trash-button.imageset/digest-trash-button.svg b/apple/OmnivoreKit/Sources/Views/Images/Images.xcassets/digest-trash-button.imageset/digest-trash-button.svg
new file mode 100644
index 000000000..23cef56cb
--- /dev/null
+++ b/apple/OmnivoreKit/Sources/Views/Images/Images.xcassets/digest-trash-button.imageset/digest-trash-button.svg
@@ -0,0 +1,15 @@
+<svg width="35" height="35" viewBox="0 0 35 35" fill="none" xmlns="http://www.w3.org/2000/svg">
+<rect x="0.5" y="0.5" width="34" height="34" rx="17" stroke="#3D3D3D"/>
+<g clip-path="url(#clip0_9916_1792)">
+<path d="M10.8333 13.3335H24.1666" stroke="#D9D9D9" stroke-width="1.25" stroke-linecap="round" stroke-linejoin="round"/>
+<path d="M15.8333 16.6665V21.6665" stroke="#D9D9D9" stroke-width="1.25" stroke-linecap="round" stroke-linejoin="round"/>
+<path d="M19.1667 16.6665V21.6665" stroke="#D9D9D9" stroke-width="1.25" stroke-linecap="round" stroke-linejoin="round"/>
+<path d="M11.6667 13.3335L12.5001 23.3335C12.5001 23.7755 12.6757 24.1994 12.9882 24.512C13.3008 24.8246 13.7247 25.0002 14.1667 25.0002H20.8334C21.2754 25.0002 21.6994 24.8246 22.0119 24.512C22.3245 24.1994 22.5001 23.7755 22.5001 23.3335L23.3334 13.3335" stroke="#D9D9D9" stroke-width="1.25" stroke-linecap="round" stroke-linejoin="round"/>
+<path d="M15 13.3333V10.8333C15 10.6123 15.0878 10.4004 15.2441 10.2441C15.4004 10.0878 15.6123 10 15.8333 10H19.1667C19.3877 10 19.5996 10.0878 19.7559 10.2441C19.9122 10.4004 20 10.6123 20 10.8333V13.3333" stroke="#D9D9D9" stroke-width="1.25" stroke-linecap="round" stroke-linejoin="round"/>
+</g>
+<defs>
+<clipPath id="clip0_9916_1792">
+<rect width="20" height="20" fill="white" transform="translate(7.5 7.5)"/>
+</clipPath>
+</defs>
+</svg>
diff --git a/apple/OmnivoreKit/Sources/Views/Images/Images.xcassets/double_chevron_up.imageset/Contents.json b/apple/OmnivoreKit/Sources/Views/Images/Images.xcassets/double_chevron_up.imageset/Contents.json
new file mode 100644
index 000000000..6a11721b2
--- /dev/null
+++ b/apple/OmnivoreKit/Sources/Views/Images/Images.xcassets/double_chevron_up.imageset/Contents.json
@@ -0,0 +1,21 @@
+{
+  "images" : [
+    {
+      "filename" : "double_chevron_up.svg",
+      "idiom" : "universal",
+      "scale" : "1x"
+    },
+    {
+      "idiom" : "universal",
+      "scale" : "2x"
+    },
+    {
+      "idiom" : "universal",
+      "scale" : "3x"
+    }
+  ],
+  "info" : {
+    "author" : "xcode",
+    "version" : 1
+  }
+}
diff --git a/apple/OmnivoreKit/Sources/Views/Images/Images.xcassets/double_chevron_up.imageset/double_chevron_up.svg b/apple/OmnivoreKit/Sources/Views/Images/Images.xcassets/double_chevron_up.imageset/double_chevron_up.svg
new file mode 100644
index 000000000..7ac7be15d
--- /dev/null
+++ b/apple/OmnivoreKit/Sources/Views/Images/Images.xcassets/double_chevron_up.imageset/double_chevron_up.svg
@@ -0,0 +1,4 @@
+<svg width="25" height="24" viewBox="0 0 25 24" fill="none" xmlns="http://www.w3.org/2000/svg">
+<path d="M17.4614 17L12.4614 12L7.46143 17" stroke="#898989" stroke-width="1.5" stroke-linecap="round" stroke-linejoin="round"/>
+<path d="M17.4614 11L12.4614 6L7.46143 11" stroke="#898989" stroke-width="1.5" stroke-linecap="round" stroke-linejoin="round"/>
+</svg>
diff --git a/apple/OmnivoreKit/Sources/Views/Theme.swift b/apple/OmnivoreKit/Sources/Views/Theme.swift
index e4b9147a9..69ebecf70 100644
--- a/apple/OmnivoreKit/Sources/Views/Theme.swift
+++ b/apple/OmnivoreKit/Sources/Views/Theme.swift
@@ -31,6 +31,22 @@ public enum Theme: String, CaseIterable {
     }
   }
 
+  public var fgColor: Color {
+    let prefersHighContrastText = UserDefaults.standard.bool(forKey: UserDefaultKey.prefersHighContrastWebFont.rawValue)
+    switch self {
+    case .system:
+      return Color.isDarkMode ? .white : .black
+    case .light:
+      return .black
+    case .dark:
+      return Color.white
+    case .sepia:
+      return prefersHighContrastText ? Color.black : (Color(hex: "#5F4B32") ?? Color.black)
+    case .apollo:
+      return prefersHighContrastText ? Color.white : (Color(hex: "#F3F3F3") ?? Color.white)
+    }
+  }
+
   public var toolbarColor: Color {
     ThemeManager.currentTheme.isDark ? Color.themeDarkWhiteGray : Color.themeMiddleGray
   }
@@ -98,6 +114,10 @@ public enum ThemeManager {
     currentTheme.bgColor
   }
 
+  public static var currentFgColor: Color {
+    currentTheme.fgColor
+  }
+
   public static var currentHighlightColor: Color {
     currentTheme.highlightColor
   }

From 2b53d7791a785bbc8e0ec212715b29f9c1f221cb Mon Sep 17 00:00:00 2001
From: Jackson Harper <jacksonh@gmail.com>
Date: Mon, 15 Apr 2024 20:27:36 +0800
Subject: [PATCH 54/74] Update tab bar to allow opting into digest

---
 .../App/Views/BottomBars/CustomTabBar.swift   |   26 +-
 .../Sources/App/Views/LibraryTabView.swift    |   58 +-
 .../CoreDataModel.xcdatamodel/contents        |    3 +-
 .../Sources/Models/DataModels/Feature.swift   |   10 +
 .../Services/DataService/GQLSchema.swift      | 3429 +++++++++++++++++
 .../Mutations/SetRuleMutation.swift           |    6 +
 .../DataService/Queries/ViewerFetcher.swift   |    6 +-
 .../Selections/FeatureSelection.swift         |    6 +
 8 files changed, 3526 insertions(+), 18 deletions(-)
 create mode 100644 apple/OmnivoreKit/Sources/Models/DataModels/Feature.swift
 create mode 100644 apple/OmnivoreKit/Sources/Services/DataService/Selections/FeatureSelection.swift

diff --git a/apple/OmnivoreKit/Sources/App/Views/BottomBars/CustomTabBar.swift b/apple/OmnivoreKit/Sources/App/Views/BottomBars/CustomTabBar.swift
index eb0bf53d1..67efbb9f8 100644
--- a/apple/OmnivoreKit/Sources/App/Views/BottomBars/CustomTabBar.swift
+++ b/apple/OmnivoreKit/Sources/App/Views/BottomBars/CustomTabBar.swift
@@ -2,25 +2,31 @@ import Foundation
 import SwiftUI
 
 struct CustomTabBar: View {
+  let displayTabs: [String]
   @Binding var selectedTab: String
-  let hideFollowingTab: Bool
 
   var body: some View {
     HStack(spacing: 0) {
-      if !hideFollowingTab {
+      if displayTabs.contains("following") {
         TabBarButton(key: "following",
                      image: Image.tabFollowing,
                      selectedTab: $selectedTab,
                      selectionColor: Color(hex: "EE8232"))
       }
-      TabBarButton(key: "digest",
-                   image: Image.tabDigest,
-                   selectedTab: $selectedTab,
-                   selectedImage: Image.tabDigestSelected)
-      TabBarButton(key: "inbox",
-                   image: Image.tabLibrary,
-                   selectedTab: $selectedTab)
-      // TabBarButton(key: "profile", image: Image.tabProfile, selectedTab: $selectedTab)
+      if displayTabs.contains("digest") {
+        TabBarButton(key: "digest",
+                     image: Image.tabDigest,
+                     selectedTab: $selectedTab,
+                     selectedImage: Image.tabDigestSelected)
+      }
+      if displayTabs.contains("inbox") {
+        TabBarButton(key: "inbox",
+                     image: Image.tabLibrary,
+                     selectedTab: $selectedTab)
+      }
+      if displayTabs.contains("profile") {
+        TabBarButton(key: "profile", image: Image.tabProfile, selectedTab: $selectedTab)
+      }
     }
     .padding(.top, 10)
     .padding(.bottom, 10)
diff --git a/apple/OmnivoreKit/Sources/App/Views/LibraryTabView.swift b/apple/OmnivoreKit/Sources/App/Views/LibraryTabView.swift
index 0a340ceed..79f9137b7 100644
--- a/apple/OmnivoreKit/Sources/App/Views/LibraryTabView.swift
+++ b/apple/OmnivoreKit/Sources/App/Views/LibraryTabView.swift
@@ -21,6 +21,9 @@ struct LibraryTabView: View {
   @AppStorage("LibraryTabView::hideFollowingTab") var hideFollowingTab = false
   @AppStorage(UserDefaultKey.lastSelectedTabItem.rawValue) var selectedTab = "inbox"
 
+  @AppStorage("LibraryTabView::digestEnabled") var digestEnabled = false
+  @AppStorage("LibraryTabView::hasCheckedForDigestFeature") var hasCheckedForDigestFeature = false
+
   @State var isEditMode: EditMode = .inactive
   @State var showExpandedAudioPlayer = false
   @State var presentPushContainer = true
@@ -76,6 +79,28 @@ struct LibraryTabView: View {
   @State var operationStatus: OperationStatus = .none
   @State var operationMessage: String?
 
+  var showDigest: Bool {
+    if digestEnabled, #available(iOS 17.0, *) {
+      return true
+    }
+    return false
+  }
+
+  var displayTabs: [String] {
+    var res = [String]()
+    if !hideFollowingTab {
+      res.append("following")
+    }
+    if showDigest {
+      res.append("digest")
+    }
+    res.append("inbox")
+    if !showDigest {
+      res.append("profile")
+    }
+    return res
+  }
+
   var body: some View {
     VStack(spacing: 0) {
       WindowLink(level: .alert, transition: .move(edge: .bottom), isPresented: $showOperationToast) {
@@ -116,24 +141,29 @@ struct LibraryTabView: View {
           }.tag("following")
         }
 
-        if #available(iOS 17.0, *) {
+        if showDigest, #available(iOS 17.0, *) {
           NavigationView {
             LibraryDigestView(dataService: dataService)
               .navigationBarTitleDisplayMode(.inline)
               .navigationViewStyle(.stack)
           }.tag("digest")
+          NavigationView {
+            HomeFeedContainerView(viewModel: inboxViewModel, isEditMode: $isEditMode)
+              .navigationBarTitleDisplayMode(.inline)
+              .navigationViewStyle(.stack)
+          }.tag("inbox")
         } else {
           NavigationView {
             HomeFeedContainerView(viewModel: inboxViewModel, isEditMode: $isEditMode)
               .navigationBarTitleDisplayMode(.inline)
               .navigationViewStyle(.stack)
           }.tag("inbox")
+          NavigationView {
+            ProfileView()
+              .navigationViewStyle(.stack)
+          }.tag("profile")
         }
 
-        NavigationView {
-          ProfileView()
-            .navigationViewStyle(.stack)
-        }.tag("profile")
       }
       if let audioProperties = audioController.itemAudioProperties {
         MiniPlayerViewer(itemAudioProperties: audioProperties)
@@ -146,7 +176,9 @@ struct LibraryTabView: View {
           .frame(maxWidth: .infinity)
       }
       if isEditMode != .active {
-        CustomTabBar(selectedTab: $selectedTab, hideFollowingTab: hideFollowingTab)
+        CustomTabBar(
+          displayTabs: displayTabs,
+          selectedTab: $selectedTab)
           .padding(0)
       }
     }
@@ -204,5 +236,19 @@ struct LibraryTabView: View {
       }
       selectedTab = "inbox"
     }
+    .task {
+      do {
+        if let viewer = try await dataService.fetchViewer() {
+          digestEnabled = viewer.digestEnabled ?? false
+          if !hasCheckedForDigestFeature {
+            hasCheckedForDigestFeature = true
+            selectedTab = "digest"
+          }
+        }
+      } catch {
+        print("ERROR FETCHING VIEWER: ", error)
+        print("")
+      }
+    }
   }
 }
diff --git a/apple/OmnivoreKit/Sources/Models/CoreData/CoreDataModel.xcdatamodeld/CoreDataModel.xcdatamodel/contents b/apple/OmnivoreKit/Sources/Models/CoreData/CoreDataModel.xcdatamodeld/CoreDataModel.xcdatamodel/contents
index 07612b47e..57cf07b2e 100644
--- a/apple/OmnivoreKit/Sources/Models/CoreData/CoreDataModel.xcdatamodeld/CoreDataModel.xcdatamodel/contents
+++ b/apple/OmnivoreKit/Sources/Models/CoreData/CoreDataModel.xcdatamodeld/CoreDataModel.xcdatamodel/contents
@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8" standalone="yes"?>
-<model type="com.apple.IDECoreDataModeler.DataModel" documentVersion="1.0" lastSavedToolsVersion="22522" systemVersion="23B81" minimumToolsVersion="Automatic" sourceLanguage="Swift" userDefinedModelVersionIdentifier="">
+<model type="com.apple.IDECoreDataModeler.DataModel" documentVersion="1.0" lastSavedToolsVersion="22757" systemVersion="23B81" minimumToolsVersion="Automatic" sourceLanguage="Swift" userDefinedModelVersionIdentifier="">
     <entity name="Filter" representedClassName="Filter" syncable="YES" codeGenerationType="class">
         <attribute name="defaultFilter" optional="YES" attributeType="Boolean" usesScalarValueType="YES"/>
         <attribute name="filter" optional="YES" attributeType="String"/>
@@ -137,6 +137,7 @@
         <attribute name="username" optional="YES" attributeType="String"/>
     </entity>
     <entity name="Viewer" representedClassName="Viewer" syncable="YES" codeGenerationType="class">
+        <attribute name="digestEnabled" optional="YES" attributeType="Boolean" usesScalarValueType="YES"/>
         <attribute name="name" attributeType="String"/>
         <attribute name="profileImageURL" optional="YES" attributeType="String"/>
         <attribute name="userID" attributeType="String"/>
diff --git a/apple/OmnivoreKit/Sources/Models/DataModels/Feature.swift b/apple/OmnivoreKit/Sources/Models/DataModels/Feature.swift
new file mode 100644
index 000000000..2bccfc724
--- /dev/null
+++ b/apple/OmnivoreKit/Sources/Models/DataModels/Feature.swift
@@ -0,0 +1,10 @@
+
+public struct FeatureInternal {
+  public let name: String
+  public let enabled: Bool
+
+  public init(name: String, enabled: Bool) {
+    self.name = name
+    self.enabled = enabled
+  }
+}
diff --git a/apple/OmnivoreKit/Sources/Services/DataService/GQLSchema.swift b/apple/OmnivoreKit/Sources/Services/DataService/GQLSchema.swift
index 49979b626..cd3cf2b2c 100644
--- a/apple/OmnivoreKit/Sources/Services/DataService/GQLSchema.swift
+++ b/apple/OmnivoreKit/Sources/Services/DataService/GQLSchema.swift
@@ -20,6 +20,137 @@ extension Objects.Subscription: GraphQLWebSocketOperation {
 // MARK: - Objects
 
 enum Objects {}
+extension Objects {
+    struct AddDiscoverFeedError {
+        let __typename: TypeName = .addDiscoverFeedError
+        let errorCodes: [String: [Enums.AddDiscoverFeedErrorCode]]
+
+        enum TypeName: String, Codable {
+            case addDiscoverFeedError = "AddDiscoverFeedError"
+        }
+    }
+}
+
+extension Objects.AddDiscoverFeedError: Decodable {
+    init(from decoder: Decoder) throws {
+        let container = try decoder.container(keyedBy: DynamicCodingKeys.self)
+
+        var map = HashMap()
+        for codingKey in container.allKeys {
+            if codingKey.isTypenameKey { continue }
+
+            let alias = codingKey.stringValue
+            let field = GraphQLField.getFieldNameFromAlias(alias)
+
+            switch field {
+            case "errorCodes":
+                if let value = try container.decode([Enums.AddDiscoverFeedErrorCode]?.self, forKey: codingKey) {
+                    map.set(key: field, hash: alias, value: value as Any)
+                }
+            default:
+                throw DecodingError.dataCorrupted(
+                    DecodingError.Context(
+                        codingPath: decoder.codingPath,
+                        debugDescription: "Unknown key \(field)."
+                    )
+                )
+            }
+        }
+
+        errorCodes = map["errorCodes"]
+    }
+}
+
+extension Fields where TypeLock == Objects.AddDiscoverFeedError {
+    func errorCodes() throws -> [Enums.AddDiscoverFeedErrorCode] {
+        let field = GraphQLField.leaf(
+            name: "errorCodes",
+            arguments: []
+        )
+        select(field)
+
+        switch response {
+        case let .decoding(data):
+            if let data = data.errorCodes[field.alias!] {
+                return data
+            }
+            throw HttpError.badpayload
+        case .mocking:
+            return []
+        }
+    }
+}
+
+extension Selection where TypeLock == Never, Type == Never {
+    typealias AddDiscoverFeedError<T> = Selection<T, Objects.AddDiscoverFeedError>
+}
+
+extension Objects {
+    struct AddDiscoverFeedSuccess {
+        let __typename: TypeName = .addDiscoverFeedSuccess
+        let feed: [String: Objects.DiscoverFeed]
+
+        enum TypeName: String, Codable {
+            case addDiscoverFeedSuccess = "AddDiscoverFeedSuccess"
+        }
+    }
+}
+
+extension Objects.AddDiscoverFeedSuccess: Decodable {
+    init(from decoder: Decoder) throws {
+        let container = try decoder.container(keyedBy: DynamicCodingKeys.self)
+
+        var map = HashMap()
+        for codingKey in container.allKeys {
+            if codingKey.isTypenameKey { continue }
+
+            let alias = codingKey.stringValue
+            let field = GraphQLField.getFieldNameFromAlias(alias)
+
+            switch field {
+            case "feed":
+                if let value = try container.decode(Objects.DiscoverFeed?.self, forKey: codingKey) {
+                    map.set(key: field, hash: alias, value: value as Any)
+                }
+            default:
+                throw DecodingError.dataCorrupted(
+                    DecodingError.Context(
+                        codingPath: decoder.codingPath,
+                        debugDescription: "Unknown key \(field)."
+                    )
+                )
+            }
+        }
+
+        feed = map["feed"]
+    }
+}
+
+extension Fields where TypeLock == Objects.AddDiscoverFeedSuccess {
+    func feed<Type>(selection: Selection<Type, Objects.DiscoverFeed>) throws -> Type {
+        let field = GraphQLField.composite(
+            name: "feed",
+            arguments: [],
+            selection: selection.selection
+        )
+        select(field)
+
+        switch response {
+        case let .decoding(data):
+            if let data = data.feed[field.alias!] {
+                return try selection.decode(data: data)
+            }
+            throw HttpError.badpayload
+        case .mocking:
+            return selection.mock()
+        }
+    }
+}
+
+extension Selection where TypeLock == Never, Type == Never {
+    typealias AddDiscoverFeedSuccess<T> = Selection<T, Objects.AddDiscoverFeedSuccess>
+}
+
 extension Objects {
     struct AddPopularReadError {
         let __typename: TypeName = .addPopularReadError
@@ -3921,6 +4052,266 @@ extension Selection where TypeLock == Never, Type == Never {
     typealias DeleteAccountSuccess<T> = Selection<T, Objects.DeleteAccountSuccess>
 }
 
+extension Objects {
+    struct DeleteDiscoverArticleError {
+        let __typename: TypeName = .deleteDiscoverArticleError
+        let errorCodes: [String: [Enums.DeleteDiscoverArticleErrorCode]]
+
+        enum TypeName: String, Codable {
+            case deleteDiscoverArticleError = "DeleteDiscoverArticleError"
+        }
+    }
+}
+
+extension Objects.DeleteDiscoverArticleError: Decodable {
+    init(from decoder: Decoder) throws {
+        let container = try decoder.container(keyedBy: DynamicCodingKeys.self)
+
+        var map = HashMap()
+        for codingKey in container.allKeys {
+            if codingKey.isTypenameKey { continue }
+
+            let alias = codingKey.stringValue
+            let field = GraphQLField.getFieldNameFromAlias(alias)
+
+            switch field {
+            case "errorCodes":
+                if let value = try container.decode([Enums.DeleteDiscoverArticleErrorCode]?.self, forKey: codingKey) {
+                    map.set(key: field, hash: alias, value: value as Any)
+                }
+            default:
+                throw DecodingError.dataCorrupted(
+                    DecodingError.Context(
+                        codingPath: decoder.codingPath,
+                        debugDescription: "Unknown key \(field)."
+                    )
+                )
+            }
+        }
+
+        errorCodes = map["errorCodes"]
+    }
+}
+
+extension Fields where TypeLock == Objects.DeleteDiscoverArticleError {
+    func errorCodes() throws -> [Enums.DeleteDiscoverArticleErrorCode] {
+        let field = GraphQLField.leaf(
+            name: "errorCodes",
+            arguments: []
+        )
+        select(field)
+
+        switch response {
+        case let .decoding(data):
+            if let data = data.errorCodes[field.alias!] {
+                return data
+            }
+            throw HttpError.badpayload
+        case .mocking:
+            return []
+        }
+    }
+}
+
+extension Selection where TypeLock == Never, Type == Never {
+    typealias DeleteDiscoverArticleError<T> = Selection<T, Objects.DeleteDiscoverArticleError>
+}
+
+extension Objects {
+    struct DeleteDiscoverArticleSuccess {
+        let __typename: TypeName = .deleteDiscoverArticleSuccess
+        let id: [String: String]
+
+        enum TypeName: String, Codable {
+            case deleteDiscoverArticleSuccess = "DeleteDiscoverArticleSuccess"
+        }
+    }
+}
+
+extension Objects.DeleteDiscoverArticleSuccess: Decodable {
+    init(from decoder: Decoder) throws {
+        let container = try decoder.container(keyedBy: DynamicCodingKeys.self)
+
+        var map = HashMap()
+        for codingKey in container.allKeys {
+            if codingKey.isTypenameKey { continue }
+
+            let alias = codingKey.stringValue
+            let field = GraphQLField.getFieldNameFromAlias(alias)
+
+            switch field {
+            case "id":
+                if let value = try container.decode(String?.self, forKey: codingKey) {
+                    map.set(key: field, hash: alias, value: value as Any)
+                }
+            default:
+                throw DecodingError.dataCorrupted(
+                    DecodingError.Context(
+                        codingPath: decoder.codingPath,
+                        debugDescription: "Unknown key \(field)."
+                    )
+                )
+            }
+        }
+
+        id = map["id"]
+    }
+}
+
+extension Fields where TypeLock == Objects.DeleteDiscoverArticleSuccess {
+    func id() throws -> String {
+        let field = GraphQLField.leaf(
+            name: "id",
+            arguments: []
+        )
+        select(field)
+
+        switch response {
+        case let .decoding(data):
+            if let data = data.id[field.alias!] {
+                return data
+            }
+            throw HttpError.badpayload
+        case .mocking:
+            return String.mockValue
+        }
+    }
+}
+
+extension Selection where TypeLock == Never, Type == Never {
+    typealias DeleteDiscoverArticleSuccess<T> = Selection<T, Objects.DeleteDiscoverArticleSuccess>
+}
+
+extension Objects {
+    struct DeleteDiscoverFeedError {
+        let __typename: TypeName = .deleteDiscoverFeedError
+        let errorCodes: [String: [Enums.DeleteDiscoverFeedErrorCode]]
+
+        enum TypeName: String, Codable {
+            case deleteDiscoverFeedError = "DeleteDiscoverFeedError"
+        }
+    }
+}
+
+extension Objects.DeleteDiscoverFeedError: Decodable {
+    init(from decoder: Decoder) throws {
+        let container = try decoder.container(keyedBy: DynamicCodingKeys.self)
+
+        var map = HashMap()
+        for codingKey in container.allKeys {
+            if codingKey.isTypenameKey { continue }
+
+            let alias = codingKey.stringValue
+            let field = GraphQLField.getFieldNameFromAlias(alias)
+
+            switch field {
+            case "errorCodes":
+                if let value = try container.decode([Enums.DeleteDiscoverFeedErrorCode]?.self, forKey: codingKey) {
+                    map.set(key: field, hash: alias, value: value as Any)
+                }
+            default:
+                throw DecodingError.dataCorrupted(
+                    DecodingError.Context(
+                        codingPath: decoder.codingPath,
+                        debugDescription: "Unknown key \(field)."
+                    )
+                )
+            }
+        }
+
+        errorCodes = map["errorCodes"]
+    }
+}
+
+extension Fields where TypeLock == Objects.DeleteDiscoverFeedError {
+    func errorCodes() throws -> [Enums.DeleteDiscoverFeedErrorCode] {
+        let field = GraphQLField.leaf(
+            name: "errorCodes",
+            arguments: []
+        )
+        select(field)
+
+        switch response {
+        case let .decoding(data):
+            if let data = data.errorCodes[field.alias!] {
+                return data
+            }
+            throw HttpError.badpayload
+        case .mocking:
+            return []
+        }
+    }
+}
+
+extension Selection where TypeLock == Never, Type == Never {
+    typealias DeleteDiscoverFeedError<T> = Selection<T, Objects.DeleteDiscoverFeedError>
+}
+
+extension Objects {
+    struct DeleteDiscoverFeedSuccess {
+        let __typename: TypeName = .deleteDiscoverFeedSuccess
+        let id: [String: String]
+
+        enum TypeName: String, Codable {
+            case deleteDiscoverFeedSuccess = "DeleteDiscoverFeedSuccess"
+        }
+    }
+}
+
+extension Objects.DeleteDiscoverFeedSuccess: Decodable {
+    init(from decoder: Decoder) throws {
+        let container = try decoder.container(keyedBy: DynamicCodingKeys.self)
+
+        var map = HashMap()
+        for codingKey in container.allKeys {
+            if codingKey.isTypenameKey { continue }
+
+            let alias = codingKey.stringValue
+            let field = GraphQLField.getFieldNameFromAlias(alias)
+
+            switch field {
+            case "id":
+                if let value = try container.decode(String?.self, forKey: codingKey) {
+                    map.set(key: field, hash: alias, value: value as Any)
+                }
+            default:
+                throw DecodingError.dataCorrupted(
+                    DecodingError.Context(
+                        codingPath: decoder.codingPath,
+                        debugDescription: "Unknown key \(field)."
+                    )
+                )
+            }
+        }
+
+        id = map["id"]
+    }
+}
+
+extension Fields where TypeLock == Objects.DeleteDiscoverFeedSuccess {
+    func id() throws -> String {
+        let field = GraphQLField.leaf(
+            name: "id",
+            arguments: []
+        )
+        select(field)
+
+        switch response {
+        case let .decoding(data):
+            if let data = data.id[field.alias!] {
+                return data
+            }
+            throw HttpError.badpayload
+        case .mocking:
+            return String.mockValue
+        }
+    }
+}
+
+extension Selection where TypeLock == Never, Type == Never {
+    typealias DeleteDiscoverFeedSuccess<T> = Selection<T, Objects.DeleteDiscoverFeedSuccess>
+}
+
 extension Objects {
     struct DeleteFilterError {
         let __typename: TypeName = .deleteFilterError
@@ -5475,6 +5866,867 @@ extension Selection where TypeLock == Never, Type == Never {
     typealias DeviceTokensSuccess<T> = Selection<T, Objects.DeviceTokensSuccess>
 }
 
+extension Objects {
+    struct DiscoverFeed {
+        let __typename: TypeName = .discoverFeed
+        let description: [String: String]
+        let id: [String: String]
+        let image: [String: String]
+        let link: [String: String]
+        let title: [String: String]
+        let type: [String: String]
+        let visibleName: [String: String]
+
+        enum TypeName: String, Codable {
+            case discoverFeed = "DiscoverFeed"
+        }
+    }
+}
+
+extension Objects.DiscoverFeed: Decodable {
+    init(from decoder: Decoder) throws {
+        let container = try decoder.container(keyedBy: DynamicCodingKeys.self)
+
+        var map = HashMap()
+        for codingKey in container.allKeys {
+            if codingKey.isTypenameKey { continue }
+
+            let alias = codingKey.stringValue
+            let field = GraphQLField.getFieldNameFromAlias(alias)
+
+            switch field {
+            case "description":
+                if let value = try container.decode(String?.self, forKey: codingKey) {
+                    map.set(key: field, hash: alias, value: value as Any)
+                }
+            case "id":
+                if let value = try container.decode(String?.self, forKey: codingKey) {
+                    map.set(key: field, hash: alias, value: value as Any)
+                }
+            case "image":
+                if let value = try container.decode(String?.self, forKey: codingKey) {
+                    map.set(key: field, hash: alias, value: value as Any)
+                }
+            case "link":
+                if let value = try container.decode(String?.self, forKey: codingKey) {
+                    map.set(key: field, hash: alias, value: value as Any)
+                }
+            case "title":
+                if let value = try container.decode(String?.self, forKey: codingKey) {
+                    map.set(key: field, hash: alias, value: value as Any)
+                }
+            case "type":
+                if let value = try container.decode(String?.self, forKey: codingKey) {
+                    map.set(key: field, hash: alias, value: value as Any)
+                }
+            case "visibleName":
+                if let value = try container.decode(String?.self, forKey: codingKey) {
+                    map.set(key: field, hash: alias, value: value as Any)
+                }
+            default:
+                throw DecodingError.dataCorrupted(
+                    DecodingError.Context(
+                        codingPath: decoder.codingPath,
+                        debugDescription: "Unknown key \(field)."
+                    )
+                )
+            }
+        }
+
+        description = map["description"]
+        id = map["id"]
+        image = map["image"]
+        link = map["link"]
+        title = map["title"]
+        type = map["type"]
+        visibleName = map["visibleName"]
+    }
+}
+
+extension Fields where TypeLock == Objects.DiscoverFeed {
+    func description() throws -> String? {
+        let field = GraphQLField.leaf(
+            name: "description",
+            arguments: []
+        )
+        select(field)
+
+        switch response {
+        case let .decoding(data):
+            return data.description[field.alias!]
+        case .mocking:
+            return nil
+        }
+    }
+
+    func id() throws -> String {
+        let field = GraphQLField.leaf(
+            name: "id",
+            arguments: []
+        )
+        select(field)
+
+        switch response {
+        case let .decoding(data):
+            if let data = data.id[field.alias!] {
+                return data
+            }
+            throw HttpError.badpayload
+        case .mocking:
+            return String.mockValue
+        }
+    }
+
+    func image() throws -> String? {
+        let field = GraphQLField.leaf(
+            name: "image",
+            arguments: []
+        )
+        select(field)
+
+        switch response {
+        case let .decoding(data):
+            return data.image[field.alias!]
+        case .mocking:
+            return nil
+        }
+    }
+
+    func link() throws -> String {
+        let field = GraphQLField.leaf(
+            name: "link",
+            arguments: []
+        )
+        select(field)
+
+        switch response {
+        case let .decoding(data):
+            if let data = data.link[field.alias!] {
+                return data
+            }
+            throw HttpError.badpayload
+        case .mocking:
+            return String.mockValue
+        }
+    }
+
+    func title() throws -> String {
+        let field = GraphQLField.leaf(
+            name: "title",
+            arguments: []
+        )
+        select(field)
+
+        switch response {
+        case let .decoding(data):
+            if let data = data.title[field.alias!] {
+                return data
+            }
+            throw HttpError.badpayload
+        case .mocking:
+            return String.mockValue
+        }
+    }
+
+    func type() throws -> String {
+        let field = GraphQLField.leaf(
+            name: "type",
+            arguments: []
+        )
+        select(field)
+
+        switch response {
+        case let .decoding(data):
+            if let data = data.type[field.alias!] {
+                return data
+            }
+            throw HttpError.badpayload
+        case .mocking:
+            return String.mockValue
+        }
+    }
+
+    func visibleName() throws -> String? {
+        let field = GraphQLField.leaf(
+            name: "visibleName",
+            arguments: []
+        )
+        select(field)
+
+        switch response {
+        case let .decoding(data):
+            return data.visibleName[field.alias!]
+        case .mocking:
+            return nil
+        }
+    }
+}
+
+extension Selection where TypeLock == Never, Type == Never {
+    typealias DiscoverFeed<T> = Selection<T, Objects.DiscoverFeed>
+}
+
+extension Objects {
+    struct DiscoverFeedArticle {
+        let __typename: TypeName = .discoverFeedArticle
+        let author: [String: String]
+        let description: [String: String]
+        let feed: [String: String]
+        let id: [String: String]
+        let image: [String: String]
+        let publishedDate: [String: DateTime]
+        let savedId: [String: String]
+        let savedLinkUrl: [String: String]
+        let siteName: [String: String]
+        let slug: [String: String]
+        let title: [String: String]
+        let url: [String: String]
+
+        enum TypeName: String, Codable {
+            case discoverFeedArticle = "DiscoverFeedArticle"
+        }
+    }
+}
+
+extension Objects.DiscoverFeedArticle: Decodable {
+    init(from decoder: Decoder) throws {
+        let container = try decoder.container(keyedBy: DynamicCodingKeys.self)
+
+        var map = HashMap()
+        for codingKey in container.allKeys {
+            if codingKey.isTypenameKey { continue }
+
+            let alias = codingKey.stringValue
+            let field = GraphQLField.getFieldNameFromAlias(alias)
+
+            switch field {
+            case "author":
+                if let value = try container.decode(String?.self, forKey: codingKey) {
+                    map.set(key: field, hash: alias, value: value as Any)
+                }
+            case "description":
+                if let value = try container.decode(String?.self, forKey: codingKey) {
+                    map.set(key: field, hash: alias, value: value as Any)
+                }
+            case "feed":
+                if let value = try container.decode(String?.self, forKey: codingKey) {
+                    map.set(key: field, hash: alias, value: value as Any)
+                }
+            case "id":
+                if let value = try container.decode(String?.self, forKey: codingKey) {
+                    map.set(key: field, hash: alias, value: value as Any)
+                }
+            case "image":
+                if let value = try container.decode(String?.self, forKey: codingKey) {
+                    map.set(key: field, hash: alias, value: value as Any)
+                }
+            case "publishedDate":
+                if let value = try container.decode(DateTime?.self, forKey: codingKey) {
+                    map.set(key: field, hash: alias, value: value as Any)
+                }
+            case "savedId":
+                if let value = try container.decode(String?.self, forKey: codingKey) {
+                    map.set(key: field, hash: alias, value: value as Any)
+                }
+            case "savedLinkUrl":
+                if let value = try container.decode(String?.self, forKey: codingKey) {
+                    map.set(key: field, hash: alias, value: value as Any)
+                }
+            case "siteName":
+                if let value = try container.decode(String?.self, forKey: codingKey) {
+                    map.set(key: field, hash: alias, value: value as Any)
+                }
+            case "slug":
+                if let value = try container.decode(String?.self, forKey: codingKey) {
+                    map.set(key: field, hash: alias, value: value as Any)
+                }
+            case "title":
+                if let value = try container.decode(String?.self, forKey: codingKey) {
+                    map.set(key: field, hash: alias, value: value as Any)
+                }
+            case "url":
+                if let value = try container.decode(String?.self, forKey: codingKey) {
+                    map.set(key: field, hash: alias, value: value as Any)
+                }
+            default:
+                throw DecodingError.dataCorrupted(
+                    DecodingError.Context(
+                        codingPath: decoder.codingPath,
+                        debugDescription: "Unknown key \(field)."
+                    )
+                )
+            }
+        }
+
+        author = map["author"]
+        description = map["description"]
+        feed = map["feed"]
+        id = map["id"]
+        image = map["image"]
+        publishedDate = map["publishedDate"]
+        savedId = map["savedId"]
+        savedLinkUrl = map["savedLinkUrl"]
+        siteName = map["siteName"]
+        slug = map["slug"]
+        title = map["title"]
+        url = map["url"]
+    }
+}
+
+extension Fields where TypeLock == Objects.DiscoverFeedArticle {
+    func author() throws -> String? {
+        let field = GraphQLField.leaf(
+            name: "author",
+            arguments: []
+        )
+        select(field)
+
+        switch response {
+        case let .decoding(data):
+            return data.author[field.alias!]
+        case .mocking:
+            return nil
+        }
+    }
+
+    func description() throws -> String {
+        let field = GraphQLField.leaf(
+            name: "description",
+            arguments: []
+        )
+        select(field)
+
+        switch response {
+        case let .decoding(data):
+            if let data = data.description[field.alias!] {
+                return data
+            }
+            throw HttpError.badpayload
+        case .mocking:
+            return String.mockValue
+        }
+    }
+
+    func feed() throws -> String {
+        let field = GraphQLField.leaf(
+            name: "feed",
+            arguments: []
+        )
+        select(field)
+
+        switch response {
+        case let .decoding(data):
+            if let data = data.feed[field.alias!] {
+                return data
+            }
+            throw HttpError.badpayload
+        case .mocking:
+            return String.mockValue
+        }
+    }
+
+    func id() throws -> String {
+        let field = GraphQLField.leaf(
+            name: "id",
+            arguments: []
+        )
+        select(field)
+
+        switch response {
+        case let .decoding(data):
+            if let data = data.id[field.alias!] {
+                return data
+            }
+            throw HttpError.badpayload
+        case .mocking:
+            return String.mockValue
+        }
+    }
+
+    func image() throws -> String? {
+        let field = GraphQLField.leaf(
+            name: "image",
+            arguments: []
+        )
+        select(field)
+
+        switch response {
+        case let .decoding(data):
+            return data.image[field.alias!]
+        case .mocking:
+            return nil
+        }
+    }
+
+    func publishedDate() throws -> DateTime? {
+        let field = GraphQLField.leaf(
+            name: "publishedDate",
+            arguments: []
+        )
+        select(field)
+
+        switch response {
+        case let .decoding(data):
+            return data.publishedDate[field.alias!]
+        case .mocking:
+            return nil
+        }
+    }
+
+    func savedId() throws -> String? {
+        let field = GraphQLField.leaf(
+            name: "savedId",
+            arguments: []
+        )
+        select(field)
+
+        switch response {
+        case let .decoding(data):
+            return data.savedId[field.alias!]
+        case .mocking:
+            return nil
+        }
+    }
+
+    func savedLinkUrl() throws -> String? {
+        let field = GraphQLField.leaf(
+            name: "savedLinkUrl",
+            arguments: []
+        )
+        select(field)
+
+        switch response {
+        case let .decoding(data):
+            return data.savedLinkUrl[field.alias!]
+        case .mocking:
+            return nil
+        }
+    }
+
+    func siteName() throws -> String? {
+        let field = GraphQLField.leaf(
+            name: "siteName",
+            arguments: []
+        )
+        select(field)
+
+        switch response {
+        case let .decoding(data):
+            return data.siteName[field.alias!]
+        case .mocking:
+            return nil
+        }
+    }
+
+    func slug() throws -> String {
+        let field = GraphQLField.leaf(
+            name: "slug",
+            arguments: []
+        )
+        select(field)
+
+        switch response {
+        case let .decoding(data):
+            if let data = data.slug[field.alias!] {
+                return data
+            }
+            throw HttpError.badpayload
+        case .mocking:
+            return String.mockValue
+        }
+    }
+
+    func title() throws -> String {
+        let field = GraphQLField.leaf(
+            name: "title",
+            arguments: []
+        )
+        select(field)
+
+        switch response {
+        case let .decoding(data):
+            if let data = data.title[field.alias!] {
+                return data
+            }
+            throw HttpError.badpayload
+        case .mocking:
+            return String.mockValue
+        }
+    }
+
+    func url() throws -> String {
+        let field = GraphQLField.leaf(
+            name: "url",
+            arguments: []
+        )
+        select(field)
+
+        switch response {
+        case let .decoding(data):
+            if let data = data.url[field.alias!] {
+                return data
+            }
+            throw HttpError.badpayload
+        case .mocking:
+            return String.mockValue
+        }
+    }
+}
+
+extension Selection where TypeLock == Never, Type == Never {
+    typealias DiscoverFeedArticle<T> = Selection<T, Objects.DiscoverFeedArticle>
+}
+
+extension Objects {
+    struct DiscoverFeedError {
+        let __typename: TypeName = .discoverFeedError
+        let errorCodes: [String: [Enums.DiscoverFeedErrorCode]]
+
+        enum TypeName: String, Codable {
+            case discoverFeedError = "DiscoverFeedError"
+        }
+    }
+}
+
+extension Objects.DiscoverFeedError: Decodable {
+    init(from decoder: Decoder) throws {
+        let container = try decoder.container(keyedBy: DynamicCodingKeys.self)
+
+        var map = HashMap()
+        for codingKey in container.allKeys {
+            if codingKey.isTypenameKey { continue }
+
+            let alias = codingKey.stringValue
+            let field = GraphQLField.getFieldNameFromAlias(alias)
+
+            switch field {
+            case "errorCodes":
+                if let value = try container.decode([Enums.DiscoverFeedErrorCode]?.self, forKey: codingKey) {
+                    map.set(key: field, hash: alias, value: value as Any)
+                }
+            default:
+                throw DecodingError.dataCorrupted(
+                    DecodingError.Context(
+                        codingPath: decoder.codingPath,
+                        debugDescription: "Unknown key \(field)."
+                    )
+                )
+            }
+        }
+
+        errorCodes = map["errorCodes"]
+    }
+}
+
+extension Fields where TypeLock == Objects.DiscoverFeedError {
+    func errorCodes() throws -> [Enums.DiscoverFeedErrorCode] {
+        let field = GraphQLField.leaf(
+            name: "errorCodes",
+            arguments: []
+        )
+        select(field)
+
+        switch response {
+        case let .decoding(data):
+            if let data = data.errorCodes[field.alias!] {
+                return data
+            }
+            throw HttpError.badpayload
+        case .mocking:
+            return []
+        }
+    }
+}
+
+extension Selection where TypeLock == Never, Type == Never {
+    typealias DiscoverFeedError<T> = Selection<T, Objects.DiscoverFeedError>
+}
+
+extension Objects {
+    struct DiscoverFeedSuccess {
+        let __typename: TypeName = .discoverFeedSuccess
+        let feeds: [String: [Objects.DiscoverFeed?]]
+
+        enum TypeName: String, Codable {
+            case discoverFeedSuccess = "DiscoverFeedSuccess"
+        }
+    }
+}
+
+extension Objects.DiscoverFeedSuccess: Decodable {
+    init(from decoder: Decoder) throws {
+        let container = try decoder.container(keyedBy: DynamicCodingKeys.self)
+
+        var map = HashMap()
+        for codingKey in container.allKeys {
+            if codingKey.isTypenameKey { continue }
+
+            let alias = codingKey.stringValue
+            let field = GraphQLField.getFieldNameFromAlias(alias)
+
+            switch field {
+            case "feeds":
+                if let value = try container.decode([Objects.DiscoverFeed?]?.self, forKey: codingKey) {
+                    map.set(key: field, hash: alias, value: value as Any)
+                }
+            default:
+                throw DecodingError.dataCorrupted(
+                    DecodingError.Context(
+                        codingPath: decoder.codingPath,
+                        debugDescription: "Unknown key \(field)."
+                    )
+                )
+            }
+        }
+
+        feeds = map["feeds"]
+    }
+}
+
+extension Fields where TypeLock == Objects.DiscoverFeedSuccess {
+    func feeds<Type>(selection: Selection<Type, [Objects.DiscoverFeed?]>) throws -> Type {
+        let field = GraphQLField.composite(
+            name: "feeds",
+            arguments: [],
+            selection: selection.selection
+        )
+        select(field)
+
+        switch response {
+        case let .decoding(data):
+            if let data = data.feeds[field.alias!] {
+                return try selection.decode(data: data)
+            }
+            throw HttpError.badpayload
+        case .mocking:
+            return selection.mock()
+        }
+    }
+}
+
+extension Selection where TypeLock == Never, Type == Never {
+    typealias DiscoverFeedSuccess<T> = Selection<T, Objects.DiscoverFeedSuccess>
+}
+
+extension Objects {
+    struct DiscoverTopic {
+        let __typename: TypeName = .discoverTopic
+        let description: [String: String]
+        let name: [String: String]
+
+        enum TypeName: String, Codable {
+            case discoverTopic = "DiscoverTopic"
+        }
+    }
+}
+
+extension Objects.DiscoverTopic: Decodable {
+    init(from decoder: Decoder) throws {
+        let container = try decoder.container(keyedBy: DynamicCodingKeys.self)
+
+        var map = HashMap()
+        for codingKey in container.allKeys {
+            if codingKey.isTypenameKey { continue }
+
+            let alias = codingKey.stringValue
+            let field = GraphQLField.getFieldNameFromAlias(alias)
+
+            switch field {
+            case "description":
+                if let value = try container.decode(String?.self, forKey: codingKey) {
+                    map.set(key: field, hash: alias, value: value as Any)
+                }
+            case "name":
+                if let value = try container.decode(String?.self, forKey: codingKey) {
+                    map.set(key: field, hash: alias, value: value as Any)
+                }
+            default:
+                throw DecodingError.dataCorrupted(
+                    DecodingError.Context(
+                        codingPath: decoder.codingPath,
+                        debugDescription: "Unknown key \(field)."
+                    )
+                )
+            }
+        }
+
+        description = map["description"]
+        name = map["name"]
+    }
+}
+
+extension Fields where TypeLock == Objects.DiscoverTopic {
+    func description() throws -> String {
+        let field = GraphQLField.leaf(
+            name: "description",
+            arguments: []
+        )
+        select(field)
+
+        switch response {
+        case let .decoding(data):
+            if let data = data.description[field.alias!] {
+                return data
+            }
+            throw HttpError.badpayload
+        case .mocking:
+            return String.mockValue
+        }
+    }
+
+    func name() throws -> String {
+        let field = GraphQLField.leaf(
+            name: "name",
+            arguments: []
+        )
+        select(field)
+
+        switch response {
+        case let .decoding(data):
+            if let data = data.name[field.alias!] {
+                return data
+            }
+            throw HttpError.badpayload
+        case .mocking:
+            return String.mockValue
+        }
+    }
+}
+
+extension Selection where TypeLock == Never, Type == Never {
+    typealias DiscoverTopic<T> = Selection<T, Objects.DiscoverTopic>
+}
+
+extension Objects {
+    struct EditDiscoverFeedError {
+        let __typename: TypeName = .editDiscoverFeedError
+        let errorCodes: [String: [Enums.EditDiscoverFeedErrorCode]]
+
+        enum TypeName: String, Codable {
+            case editDiscoverFeedError = "EditDiscoverFeedError"
+        }
+    }
+}
+
+extension Objects.EditDiscoverFeedError: Decodable {
+    init(from decoder: Decoder) throws {
+        let container = try decoder.container(keyedBy: DynamicCodingKeys.self)
+
+        var map = HashMap()
+        for codingKey in container.allKeys {
+            if codingKey.isTypenameKey { continue }
+
+            let alias = codingKey.stringValue
+            let field = GraphQLField.getFieldNameFromAlias(alias)
+
+            switch field {
+            case "errorCodes":
+                if let value = try container.decode([Enums.EditDiscoverFeedErrorCode]?.self, forKey: codingKey) {
+                    map.set(key: field, hash: alias, value: value as Any)
+                }
+            default:
+                throw DecodingError.dataCorrupted(
+                    DecodingError.Context(
+                        codingPath: decoder.codingPath,
+                        debugDescription: "Unknown key \(field)."
+                    )
+                )
+            }
+        }
+
+        errorCodes = map["errorCodes"]
+    }
+}
+
+extension Fields where TypeLock == Objects.EditDiscoverFeedError {
+    func errorCodes() throws -> [Enums.EditDiscoverFeedErrorCode] {
+        let field = GraphQLField.leaf(
+            name: "errorCodes",
+            arguments: []
+        )
+        select(field)
+
+        switch response {
+        case let .decoding(data):
+            if let data = data.errorCodes[field.alias!] {
+                return data
+            }
+            throw HttpError.badpayload
+        case .mocking:
+            return []
+        }
+    }
+}
+
+extension Selection where TypeLock == Never, Type == Never {
+    typealias EditDiscoverFeedError<T> = Selection<T, Objects.EditDiscoverFeedError>
+}
+
+extension Objects {
+    struct EditDiscoverFeedSuccess {
+        let __typename: TypeName = .editDiscoverFeedSuccess
+        let id: [String: String]
+
+        enum TypeName: String, Codable {
+            case editDiscoverFeedSuccess = "EditDiscoverFeedSuccess"
+        }
+    }
+}
+
+extension Objects.EditDiscoverFeedSuccess: Decodable {
+    init(from decoder: Decoder) throws {
+        let container = try decoder.container(keyedBy: DynamicCodingKeys.self)
+
+        var map = HashMap()
+        for codingKey in container.allKeys {
+            if codingKey.isTypenameKey { continue }
+
+            let alias = codingKey.stringValue
+            let field = GraphQLField.getFieldNameFromAlias(alias)
+
+            switch field {
+            case "id":
+                if let value = try container.decode(String?.self, forKey: codingKey) {
+                    map.set(key: field, hash: alias, value: value as Any)
+                }
+            default:
+                throw DecodingError.dataCorrupted(
+                    DecodingError.Context(
+                        codingPath: decoder.codingPath,
+                        debugDescription: "Unknown key \(field)."
+                    )
+                )
+            }
+        }
+
+        id = map["id"]
+    }
+}
+
+extension Fields where TypeLock == Objects.EditDiscoverFeedSuccess {
+    func id() throws -> String {
+        let field = GraphQLField.leaf(
+            name: "id",
+            arguments: []
+        )
+        select(field)
+
+        switch response {
+        case let .decoding(data):
+            if let data = data.id[field.alias!] {
+                return data
+            }
+            throw HttpError.badpayload
+        case .mocking:
+            return String.mockValue
+        }
+    }
+}
+
+extension Selection where TypeLock == Never, Type == Never {
+    typealias EditDiscoverFeedSuccess<T> = Selection<T, Objects.EditDiscoverFeedSuccess>
+}
+
 extension Objects {
     struct EmptyTrashError {
         let __typename: TypeName = .emptyTrashError
@@ -5602,6 +6854,137 @@ extension Selection where TypeLock == Never, Type == Never {
     typealias EmptyTrashSuccess<T> = Selection<T, Objects.EmptyTrashSuccess>
 }
 
+extension Objects {
+    struct ExportToIntegrationError {
+        let __typename: TypeName = .exportToIntegrationError
+        let errorCodes: [String: [Enums.ExportToIntegrationErrorCode]]
+
+        enum TypeName: String, Codable {
+            case exportToIntegrationError = "ExportToIntegrationError"
+        }
+    }
+}
+
+extension Objects.ExportToIntegrationError: Decodable {
+    init(from decoder: Decoder) throws {
+        let container = try decoder.container(keyedBy: DynamicCodingKeys.self)
+
+        var map = HashMap()
+        for codingKey in container.allKeys {
+            if codingKey.isTypenameKey { continue }
+
+            let alias = codingKey.stringValue
+            let field = GraphQLField.getFieldNameFromAlias(alias)
+
+            switch field {
+            case "errorCodes":
+                if let value = try container.decode([Enums.ExportToIntegrationErrorCode]?.self, forKey: codingKey) {
+                    map.set(key: field, hash: alias, value: value as Any)
+                }
+            default:
+                throw DecodingError.dataCorrupted(
+                    DecodingError.Context(
+                        codingPath: decoder.codingPath,
+                        debugDescription: "Unknown key \(field)."
+                    )
+                )
+            }
+        }
+
+        errorCodes = map["errorCodes"]
+    }
+}
+
+extension Fields where TypeLock == Objects.ExportToIntegrationError {
+    func errorCodes() throws -> [Enums.ExportToIntegrationErrorCode] {
+        let field = GraphQLField.leaf(
+            name: "errorCodes",
+            arguments: []
+        )
+        select(field)
+
+        switch response {
+        case let .decoding(data):
+            if let data = data.errorCodes[field.alias!] {
+                return data
+            }
+            throw HttpError.badpayload
+        case .mocking:
+            return []
+        }
+    }
+}
+
+extension Selection where TypeLock == Never, Type == Never {
+    typealias ExportToIntegrationError<T> = Selection<T, Objects.ExportToIntegrationError>
+}
+
+extension Objects {
+    struct ExportToIntegrationSuccess {
+        let __typename: TypeName = .exportToIntegrationSuccess
+        let task: [String: Objects.Task]
+
+        enum TypeName: String, Codable {
+            case exportToIntegrationSuccess = "ExportToIntegrationSuccess"
+        }
+    }
+}
+
+extension Objects.ExportToIntegrationSuccess: Decodable {
+    init(from decoder: Decoder) throws {
+        let container = try decoder.container(keyedBy: DynamicCodingKeys.self)
+
+        var map = HashMap()
+        for codingKey in container.allKeys {
+            if codingKey.isTypenameKey { continue }
+
+            let alias = codingKey.stringValue
+            let field = GraphQLField.getFieldNameFromAlias(alias)
+
+            switch field {
+            case "task":
+                if let value = try container.decode(Objects.Task?.self, forKey: codingKey) {
+                    map.set(key: field, hash: alias, value: value as Any)
+                }
+            default:
+                throw DecodingError.dataCorrupted(
+                    DecodingError.Context(
+                        codingPath: decoder.codingPath,
+                        debugDescription: "Unknown key \(field)."
+                    )
+                )
+            }
+        }
+
+        task = map["task"]
+    }
+}
+
+extension Fields where TypeLock == Objects.ExportToIntegrationSuccess {
+    func task<Type>(selection: Selection<Type, Objects.Task>) throws -> Type {
+        let field = GraphQLField.composite(
+            name: "task",
+            arguments: [],
+            selection: selection.selection
+        )
+        select(field)
+
+        switch response {
+        case let .decoding(data):
+            if let data = data.task[field.alias!] {
+                return try selection.decode(data: data)
+            }
+            throw HttpError.badpayload
+        case .mocking:
+            return selection.mock()
+        }
+    }
+}
+
+extension Selection where TypeLock == Never, Type == Never {
+    typealias ExportToIntegrationSuccess<T> = Selection<T, Objects.ExportToIntegrationSuccess>
+}
+
 extension Objects {
     struct Feature {
         let __typename: TypeName = .feature
@@ -7500,6 +8883,287 @@ extension Selection where TypeLock == Never, Type == Never {
     typealias GenerateApiKeySuccess<T> = Selection<T, Objects.GenerateApiKeySuccess>
 }
 
+extension Objects {
+    struct GetDiscoverFeedArticleError {
+        let __typename: TypeName = .getDiscoverFeedArticleError
+        let errorCodes: [String: [Enums.GetDiscoverFeedArticleErrorCode]]
+
+        enum TypeName: String, Codable {
+            case getDiscoverFeedArticleError = "GetDiscoverFeedArticleError"
+        }
+    }
+}
+
+extension Objects.GetDiscoverFeedArticleError: Decodable {
+    init(from decoder: Decoder) throws {
+        let container = try decoder.container(keyedBy: DynamicCodingKeys.self)
+
+        var map = HashMap()
+        for codingKey in container.allKeys {
+            if codingKey.isTypenameKey { continue }
+
+            let alias = codingKey.stringValue
+            let field = GraphQLField.getFieldNameFromAlias(alias)
+
+            switch field {
+            case "errorCodes":
+                if let value = try container.decode([Enums.GetDiscoverFeedArticleErrorCode]?.self, forKey: codingKey) {
+                    map.set(key: field, hash: alias, value: value as Any)
+                }
+            default:
+                throw DecodingError.dataCorrupted(
+                    DecodingError.Context(
+                        codingPath: decoder.codingPath,
+                        debugDescription: "Unknown key \(field)."
+                    )
+                )
+            }
+        }
+
+        errorCodes = map["errorCodes"]
+    }
+}
+
+extension Fields where TypeLock == Objects.GetDiscoverFeedArticleError {
+    func errorCodes() throws -> [Enums.GetDiscoverFeedArticleErrorCode] {
+        let field = GraphQLField.leaf(
+            name: "errorCodes",
+            arguments: []
+        )
+        select(field)
+
+        switch response {
+        case let .decoding(data):
+            if let data = data.errorCodes[field.alias!] {
+                return data
+            }
+            throw HttpError.badpayload
+        case .mocking:
+            return []
+        }
+    }
+}
+
+extension Selection where TypeLock == Never, Type == Never {
+    typealias GetDiscoverFeedArticleError<T> = Selection<T, Objects.GetDiscoverFeedArticleError>
+}
+
+extension Objects {
+    struct GetDiscoverFeedArticleSuccess {
+        let __typename: TypeName = .getDiscoverFeedArticleSuccess
+        let discoverArticles: [String: [Objects.DiscoverFeedArticle?]]
+        let pageInfo: [String: Objects.PageInfo]
+
+        enum TypeName: String, Codable {
+            case getDiscoverFeedArticleSuccess = "GetDiscoverFeedArticleSuccess"
+        }
+    }
+}
+
+extension Objects.GetDiscoverFeedArticleSuccess: Decodable {
+    init(from decoder: Decoder) throws {
+        let container = try decoder.container(keyedBy: DynamicCodingKeys.self)
+
+        var map = HashMap()
+        for codingKey in container.allKeys {
+            if codingKey.isTypenameKey { continue }
+
+            let alias = codingKey.stringValue
+            let field = GraphQLField.getFieldNameFromAlias(alias)
+
+            switch field {
+            case "discoverArticles":
+                if let value = try container.decode([Objects.DiscoverFeedArticle?]?.self, forKey: codingKey) {
+                    map.set(key: field, hash: alias, value: value as Any)
+                }
+            case "pageInfo":
+                if let value = try container.decode(Objects.PageInfo?.self, forKey: codingKey) {
+                    map.set(key: field, hash: alias, value: value as Any)
+                }
+            default:
+                throw DecodingError.dataCorrupted(
+                    DecodingError.Context(
+                        codingPath: decoder.codingPath,
+                        debugDescription: "Unknown key \(field)."
+                    )
+                )
+            }
+        }
+
+        discoverArticles = map["discoverArticles"]
+        pageInfo = map["pageInfo"]
+    }
+}
+
+extension Fields where TypeLock == Objects.GetDiscoverFeedArticleSuccess {
+    func discoverArticles<Type>(selection: Selection<Type, [Objects.DiscoverFeedArticle?]?>) throws -> Type {
+        let field = GraphQLField.composite(
+            name: "discoverArticles",
+            arguments: [],
+            selection: selection.selection
+        )
+        select(field)
+
+        switch response {
+        case let .decoding(data):
+            return try selection.decode(data: data.discoverArticles[field.alias!])
+        case .mocking:
+            return selection.mock()
+        }
+    }
+
+    func pageInfo<Type>(selection: Selection<Type, Objects.PageInfo>) throws -> Type {
+        let field = GraphQLField.composite(
+            name: "pageInfo",
+            arguments: [],
+            selection: selection.selection
+        )
+        select(field)
+
+        switch response {
+        case let .decoding(data):
+            if let data = data.pageInfo[field.alias!] {
+                return try selection.decode(data: data)
+            }
+            throw HttpError.badpayload
+        case .mocking:
+            return selection.mock()
+        }
+    }
+}
+
+extension Selection where TypeLock == Never, Type == Never {
+    typealias GetDiscoverFeedArticleSuccess<T> = Selection<T, Objects.GetDiscoverFeedArticleSuccess>
+}
+
+extension Objects {
+    struct GetDiscoverTopicError {
+        let __typename: TypeName = .getDiscoverTopicError
+        let errorCodes: [String: [Enums.GetDiscoverTopicErrorCode]]
+
+        enum TypeName: String, Codable {
+            case getDiscoverTopicError = "GetDiscoverTopicError"
+        }
+    }
+}
+
+extension Objects.GetDiscoverTopicError: Decodable {
+    init(from decoder: Decoder) throws {
+        let container = try decoder.container(keyedBy: DynamicCodingKeys.self)
+
+        var map = HashMap()
+        for codingKey in container.allKeys {
+            if codingKey.isTypenameKey { continue }
+
+            let alias = codingKey.stringValue
+            let field = GraphQLField.getFieldNameFromAlias(alias)
+
+            switch field {
+            case "errorCodes":
+                if let value = try container.decode([Enums.GetDiscoverTopicErrorCode]?.self, forKey: codingKey) {
+                    map.set(key: field, hash: alias, value: value as Any)
+                }
+            default:
+                throw DecodingError.dataCorrupted(
+                    DecodingError.Context(
+                        codingPath: decoder.codingPath,
+                        debugDescription: "Unknown key \(field)."
+                    )
+                )
+            }
+        }
+
+        errorCodes = map["errorCodes"]
+    }
+}
+
+extension Fields where TypeLock == Objects.GetDiscoverTopicError {
+    func errorCodes() throws -> [Enums.GetDiscoverTopicErrorCode] {
+        let field = GraphQLField.leaf(
+            name: "errorCodes",
+            arguments: []
+        )
+        select(field)
+
+        switch response {
+        case let .decoding(data):
+            if let data = data.errorCodes[field.alias!] {
+                return data
+            }
+            throw HttpError.badpayload
+        case .mocking:
+            return []
+        }
+    }
+}
+
+extension Selection where TypeLock == Never, Type == Never {
+    typealias GetDiscoverTopicError<T> = Selection<T, Objects.GetDiscoverTopicError>
+}
+
+extension Objects {
+    struct GetDiscoverTopicSuccess {
+        let __typename: TypeName = .getDiscoverTopicSuccess
+        let discoverTopics: [String: [Objects.DiscoverTopic]]
+
+        enum TypeName: String, Codable {
+            case getDiscoverTopicSuccess = "GetDiscoverTopicSuccess"
+        }
+    }
+}
+
+extension Objects.GetDiscoverTopicSuccess: Decodable {
+    init(from decoder: Decoder) throws {
+        let container = try decoder.container(keyedBy: DynamicCodingKeys.self)
+
+        var map = HashMap()
+        for codingKey in container.allKeys {
+            if codingKey.isTypenameKey { continue }
+
+            let alias = codingKey.stringValue
+            let field = GraphQLField.getFieldNameFromAlias(alias)
+
+            switch field {
+            case "discoverTopics":
+                if let value = try container.decode([Objects.DiscoverTopic]?.self, forKey: codingKey) {
+                    map.set(key: field, hash: alias, value: value as Any)
+                }
+            default:
+                throw DecodingError.dataCorrupted(
+                    DecodingError.Context(
+                        codingPath: decoder.codingPath,
+                        debugDescription: "Unknown key \(field)."
+                    )
+                )
+            }
+        }
+
+        discoverTopics = map["discoverTopics"]
+    }
+}
+
+extension Fields where TypeLock == Objects.GetDiscoverTopicSuccess {
+    func discoverTopics<Type>(selection: Selection<Type, [Objects.DiscoverTopic]?>) throws -> Type {
+        let field = GraphQLField.composite(
+            name: "discoverTopics",
+            arguments: [],
+            selection: selection.selection
+        )
+        select(field)
+
+        switch response {
+        case let .decoding(data):
+            return try selection.decode(data: data.discoverTopics[field.alias!])
+        case .mocking:
+            return selection.mock()
+        }
+    }
+}
+
+extension Selection where TypeLock == Never, Type == Never {
+    typealias GetDiscoverTopicSuccess<T> = Selection<T, Objects.GetDiscoverTopicSuccess>
+}
+
 extension Objects {
     struct GetFollowersError {
         let __typename: TypeName = .getFollowersError
@@ -9292,6 +10956,137 @@ extension Selection where TypeLock == Never, Type == Never {
     typealias Integration<T> = Selection<T, Objects.Integration>
 }
 
+extension Objects {
+    struct IntegrationError {
+        let __typename: TypeName = .integrationError
+        let errorCodes: [String: [Enums.IntegrationErrorCode]]
+
+        enum TypeName: String, Codable {
+            case integrationError = "IntegrationError"
+        }
+    }
+}
+
+extension Objects.IntegrationError: Decodable {
+    init(from decoder: Decoder) throws {
+        let container = try decoder.container(keyedBy: DynamicCodingKeys.self)
+
+        var map = HashMap()
+        for codingKey in container.allKeys {
+            if codingKey.isTypenameKey { continue }
+
+            let alias = codingKey.stringValue
+            let field = GraphQLField.getFieldNameFromAlias(alias)
+
+            switch field {
+            case "errorCodes":
+                if let value = try container.decode([Enums.IntegrationErrorCode]?.self, forKey: codingKey) {
+                    map.set(key: field, hash: alias, value: value as Any)
+                }
+            default:
+                throw DecodingError.dataCorrupted(
+                    DecodingError.Context(
+                        codingPath: decoder.codingPath,
+                        debugDescription: "Unknown key \(field)."
+                    )
+                )
+            }
+        }
+
+        errorCodes = map["errorCodes"]
+    }
+}
+
+extension Fields where TypeLock == Objects.IntegrationError {
+    func errorCodes() throws -> [Enums.IntegrationErrorCode] {
+        let field = GraphQLField.leaf(
+            name: "errorCodes",
+            arguments: []
+        )
+        select(field)
+
+        switch response {
+        case let .decoding(data):
+            if let data = data.errorCodes[field.alias!] {
+                return data
+            }
+            throw HttpError.badpayload
+        case .mocking:
+            return []
+        }
+    }
+}
+
+extension Selection where TypeLock == Never, Type == Never {
+    typealias IntegrationError<T> = Selection<T, Objects.IntegrationError>
+}
+
+extension Objects {
+    struct IntegrationSuccess {
+        let __typename: TypeName = .integrationSuccess
+        let integration: [String: Objects.Integration]
+
+        enum TypeName: String, Codable {
+            case integrationSuccess = "IntegrationSuccess"
+        }
+    }
+}
+
+extension Objects.IntegrationSuccess: Decodable {
+    init(from decoder: Decoder) throws {
+        let container = try decoder.container(keyedBy: DynamicCodingKeys.self)
+
+        var map = HashMap()
+        for codingKey in container.allKeys {
+            if codingKey.isTypenameKey { continue }
+
+            let alias = codingKey.stringValue
+            let field = GraphQLField.getFieldNameFromAlias(alias)
+
+            switch field {
+            case "integration":
+                if let value = try container.decode(Objects.Integration?.self, forKey: codingKey) {
+                    map.set(key: field, hash: alias, value: value as Any)
+                }
+            default:
+                throw DecodingError.dataCorrupted(
+                    DecodingError.Context(
+                        codingPath: decoder.codingPath,
+                        debugDescription: "Unknown key \(field)."
+                    )
+                )
+            }
+        }
+
+        integration = map["integration"]
+    }
+}
+
+extension Fields where TypeLock == Objects.IntegrationSuccess {
+    func integration<Type>(selection: Selection<Type, Objects.Integration>) throws -> Type {
+        let field = GraphQLField.composite(
+            name: "integration",
+            arguments: [],
+            selection: selection.selection
+        )
+        select(field)
+
+        switch response {
+        case let .decoding(data):
+            if let data = data.integration[field.alias!] {
+                return try selection.decode(data: data)
+            }
+            throw HttpError.badpayload
+        case .mocking:
+            return selection.mock()
+        }
+    }
+}
+
+extension Selection where TypeLock == Never, Type == Never {
+    typealias IntegrationSuccess<T> = Selection<T, Objects.IntegrationSuccess>
+}
+
 extension Objects {
     struct IntegrationsError {
         let __typename: TypeName = .integrationsError
@@ -11440,6 +13235,7 @@ extension Selection where TypeLock == Never, Type == Never {
 extension Objects {
     struct Mutation {
         let __typename: TypeName = .mutation
+        let addDiscoverFeed: [String: Unions.AddDiscoverFeedResult]
         let addPopularRead: [String: Unions.AddPopularReadResult]
         let bulkAction: [String: Unions.BulkActionResult]
         let createArticle: [String: Unions.CreateArticleResult]
@@ -11449,6 +13245,8 @@ extension Objects {
         let createLabel: [String: Unions.CreateLabelResult]
         let createNewsletterEmail: [String: Unions.CreateNewsletterEmailResult]
         let deleteAccount: [String: Unions.DeleteAccountResult]
+        let deleteDiscoverArticle: [String: Unions.DeleteDiscoverArticleResult]
+        let deleteDiscoverFeed: [String: Unions.DeleteDiscoverFeedResult]
         let deleteFilter: [String: Unions.DeleteFilterResult]
         let deleteHighlight: [String: Unions.DeleteHighlightResult]
         let deleteIntegration: [String: Unions.DeleteIntegrationResult]
@@ -11456,7 +13254,9 @@ extension Objects {
         let deleteNewsletterEmail: [String: Unions.DeleteNewsletterEmailResult]
         let deleteRule: [String: Unions.DeleteRuleResult]
         let deleteWebhook: [String: Unions.DeleteWebhookResult]
+        let editDiscoverFeed: [String: Unions.EditDiscoverFeedResult]
         let emptyTrash: [String: Unions.EmptyTrashResult]
+        let exportToIntegration: [String: Unions.ExportToIntegrationResult]
         let fetchContent: [String: Unions.FetchContentResult]
         let generateApiKey: [String: Unions.GenerateApiKeyResult]
         let googleLogin: [String: Unions.LoginResult]
@@ -11476,6 +13276,7 @@ extension Objects {
         let reportItem: [String: Objects.ReportItemResult]
         let revokeApiKey: [String: Unions.RevokeApiKeyResult]
         let saveArticleReadingProgress: [String: Unions.SaveArticleReadingProgressResult]
+        let saveDiscoverArticle: [String: Unions.SaveDiscoverArticleResult]
         let saveFile: [String: Unions.SaveResult]
         let saveFilter: [String: Unions.SaveFilterResult]
         let savePage: [String: Unions.SaveResult]
@@ -11522,6 +13323,10 @@ extension Objects.Mutation: Decodable {
             let field = GraphQLField.getFieldNameFromAlias(alias)
 
             switch field {
+            case "addDiscoverFeed":
+                if let value = try container.decode(Unions.AddDiscoverFeedResult?.self, forKey: codingKey) {
+                    map.set(key: field, hash: alias, value: value as Any)
+                }
             case "addPopularRead":
                 if let value = try container.decode(Unions.AddPopularReadResult?.self, forKey: codingKey) {
                     map.set(key: field, hash: alias, value: value as Any)
@@ -11558,6 +13363,14 @@ extension Objects.Mutation: Decodable {
                 if let value = try container.decode(Unions.DeleteAccountResult?.self, forKey: codingKey) {
                     map.set(key: field, hash: alias, value: value as Any)
                 }
+            case "deleteDiscoverArticle":
+                if let value = try container.decode(Unions.DeleteDiscoverArticleResult?.self, forKey: codingKey) {
+                    map.set(key: field, hash: alias, value: value as Any)
+                }
+            case "deleteDiscoverFeed":
+                if let value = try container.decode(Unions.DeleteDiscoverFeedResult?.self, forKey: codingKey) {
+                    map.set(key: field, hash: alias, value: value as Any)
+                }
             case "deleteFilter":
                 if let value = try container.decode(Unions.DeleteFilterResult?.self, forKey: codingKey) {
                     map.set(key: field, hash: alias, value: value as Any)
@@ -11586,10 +13399,18 @@ extension Objects.Mutation: Decodable {
                 if let value = try container.decode(Unions.DeleteWebhookResult?.self, forKey: codingKey) {
                     map.set(key: field, hash: alias, value: value as Any)
                 }
+            case "editDiscoverFeed":
+                if let value = try container.decode(Unions.EditDiscoverFeedResult?.self, forKey: codingKey) {
+                    map.set(key: field, hash: alias, value: value as Any)
+                }
             case "emptyTrash":
                 if let value = try container.decode(Unions.EmptyTrashResult?.self, forKey: codingKey) {
                     map.set(key: field, hash: alias, value: value as Any)
                 }
+            case "exportToIntegration":
+                if let value = try container.decode(Unions.ExportToIntegrationResult?.self, forKey: codingKey) {
+                    map.set(key: field, hash: alias, value: value as Any)
+                }
             case "fetchContent":
                 if let value = try container.decode(Unions.FetchContentResult?.self, forKey: codingKey) {
                     map.set(key: field, hash: alias, value: value as Any)
@@ -11666,6 +13487,10 @@ extension Objects.Mutation: Decodable {
                 if let value = try container.decode(Unions.SaveArticleReadingProgressResult?.self, forKey: codingKey) {
                     map.set(key: field, hash: alias, value: value as Any)
                 }
+            case "saveDiscoverArticle":
+                if let value = try container.decode(Unions.SaveDiscoverArticleResult?.self, forKey: codingKey) {
+                    map.set(key: field, hash: alias, value: value as Any)
+                }
             case "saveFile":
                 if let value = try container.decode(Unions.SaveResult?.self, forKey: codingKey) {
                     map.set(key: field, hash: alias, value: value as Any)
@@ -11784,6 +13609,7 @@ extension Objects.Mutation: Decodable {
             }
         }
 
+        addDiscoverFeed = map["addDiscoverFeed"]
         addPopularRead = map["addPopularRead"]
         bulkAction = map["bulkAction"]
         createArticle = map["createArticle"]
@@ -11793,6 +13619,8 @@ extension Objects.Mutation: Decodable {
         createLabel = map["createLabel"]
         createNewsletterEmail = map["createNewsletterEmail"]
         deleteAccount = map["deleteAccount"]
+        deleteDiscoverArticle = map["deleteDiscoverArticle"]
+        deleteDiscoverFeed = map["deleteDiscoverFeed"]
         deleteFilter = map["deleteFilter"]
         deleteHighlight = map["deleteHighlight"]
         deleteIntegration = map["deleteIntegration"]
@@ -11800,7 +13628,9 @@ extension Objects.Mutation: Decodable {
         deleteNewsletterEmail = map["deleteNewsletterEmail"]
         deleteRule = map["deleteRule"]
         deleteWebhook = map["deleteWebhook"]
+        editDiscoverFeed = map["editDiscoverFeed"]
         emptyTrash = map["emptyTrash"]
+        exportToIntegration = map["exportToIntegration"]
         fetchContent = map["fetchContent"]
         generateApiKey = map["generateApiKey"]
         googleLogin = map["googleLogin"]
@@ -11820,6 +13650,7 @@ extension Objects.Mutation: Decodable {
         reportItem = map["reportItem"]
         revokeApiKey = map["revokeApiKey"]
         saveArticleReadingProgress = map["saveArticleReadingProgress"]
+        saveDiscoverArticle = map["saveDiscoverArticle"]
         saveFile = map["saveFile"]
         saveFilter = map["saveFilter"]
         savePage = map["savePage"]
@@ -11851,6 +13682,25 @@ extension Objects.Mutation: Decodable {
 }
 
 extension Fields where TypeLock == Objects.Mutation {
+    func addDiscoverFeed<Type>(input: InputObjects.AddDiscoverFeedInput, selection: Selection<Type, Unions.AddDiscoverFeedResult>) throws -> Type {
+        let field = GraphQLField.composite(
+            name: "addDiscoverFeed",
+            arguments: [Argument(name: "input", type: "AddDiscoverFeedInput!", value: input)],
+            selection: selection.selection
+        )
+        select(field)
+
+        switch response {
+        case let .decoding(data):
+            if let data = data.addDiscoverFeed[field.alias!] {
+                return try selection.decode(data: data)
+            }
+            throw HttpError.badpayload
+        case .mocking:
+            return selection.mock()
+        }
+    }
+
     func addPopularRead<Type>(name: String, selection: Selection<Type, Unions.AddPopularReadResult>) throws -> Type {
         let field = GraphQLField.composite(
             name: "addPopularRead",
@@ -12022,6 +13872,44 @@ extension Fields where TypeLock == Objects.Mutation {
         }
     }
 
+    func deleteDiscoverArticle<Type>(input: InputObjects.DeleteDiscoverArticleInput, selection: Selection<Type, Unions.DeleteDiscoverArticleResult>) throws -> Type {
+        let field = GraphQLField.composite(
+            name: "deleteDiscoverArticle",
+            arguments: [Argument(name: "input", type: "DeleteDiscoverArticleInput!", value: input)],
+            selection: selection.selection
+        )
+        select(field)
+
+        switch response {
+        case let .decoding(data):
+            if let data = data.deleteDiscoverArticle[field.alias!] {
+                return try selection.decode(data: data)
+            }
+            throw HttpError.badpayload
+        case .mocking:
+            return selection.mock()
+        }
+    }
+
+    func deleteDiscoverFeed<Type>(input: InputObjects.DeleteDiscoverFeedInput, selection: Selection<Type, Unions.DeleteDiscoverFeedResult>) throws -> Type {
+        let field = GraphQLField.composite(
+            name: "deleteDiscoverFeed",
+            arguments: [Argument(name: "input", type: "DeleteDiscoverFeedInput!", value: input)],
+            selection: selection.selection
+        )
+        select(field)
+
+        switch response {
+        case let .decoding(data):
+            if let data = data.deleteDiscoverFeed[field.alias!] {
+                return try selection.decode(data: data)
+            }
+            throw HttpError.badpayload
+        case .mocking:
+            return selection.mock()
+        }
+    }
+
     func deleteFilter<Type>(id: String, selection: Selection<Type, Unions.DeleteFilterResult>) throws -> Type {
         let field = GraphQLField.composite(
             name: "deleteFilter",
@@ -12155,6 +14043,25 @@ extension Fields where TypeLock == Objects.Mutation {
         }
     }
 
+    func editDiscoverFeed<Type>(input: InputObjects.EditDiscoverFeedInput, selection: Selection<Type, Unions.EditDiscoverFeedResult>) throws -> Type {
+        let field = GraphQLField.composite(
+            name: "editDiscoverFeed",
+            arguments: [Argument(name: "input", type: "EditDiscoverFeedInput!", value: input)],
+            selection: selection.selection
+        )
+        select(field)
+
+        switch response {
+        case let .decoding(data):
+            if let data = data.editDiscoverFeed[field.alias!] {
+                return try selection.decode(data: data)
+            }
+            throw HttpError.badpayload
+        case .mocking:
+            return selection.mock()
+        }
+    }
+
     func emptyTrash<Type>(selection: Selection<Type, Unions.EmptyTrashResult>) throws -> Type {
         let field = GraphQLField.composite(
             name: "emptyTrash",
@@ -12174,6 +14081,25 @@ extension Fields where TypeLock == Objects.Mutation {
         }
     }
 
+    func exportToIntegration<Type>(integrationId: String, selection: Selection<Type, Unions.ExportToIntegrationResult>) throws -> Type {
+        let field = GraphQLField.composite(
+            name: "exportToIntegration",
+            arguments: [Argument(name: "integrationId", type: "ID!", value: integrationId)],
+            selection: selection.selection
+        )
+        select(field)
+
+        switch response {
+        case let .decoding(data):
+            if let data = data.exportToIntegration[field.alias!] {
+                return try selection.decode(data: data)
+            }
+            throw HttpError.badpayload
+        case .mocking:
+            return selection.mock()
+        }
+    }
+
     func fetchContent<Type>(id: String, selection: Selection<Type, Unions.FetchContentResult>) throws -> Type {
         let field = GraphQLField.composite(
             name: "fetchContent",
@@ -12535,6 +14461,25 @@ extension Fields where TypeLock == Objects.Mutation {
         }
     }
 
+    func saveDiscoverArticle<Type>(input: InputObjects.SaveDiscoverArticleInput, selection: Selection<Type, Unions.SaveDiscoverArticleResult>) throws -> Type {
+        let field = GraphQLField.composite(
+            name: "saveDiscoverArticle",
+            arguments: [Argument(name: "input", type: "SaveDiscoverArticleInput!", value: input)],
+            selection: selection.selection
+        )
+        select(field)
+
+        switch response {
+        case let .decoding(data):
+            if let data = data.saveDiscoverArticle[field.alias!] {
+                return try selection.decode(data: data)
+            }
+            throw HttpError.badpayload
+        case .mocking:
+            return selection.mock()
+        }
+    }
+
     func saveFile<Type>(input: InputObjects.SaveFileInput, selection: Selection<Type, Unions.SaveResult>) throws -> Type {
         let field = GraphQLField.composite(
             name: "saveFile",
@@ -14218,11 +16163,15 @@ extension Objects {
         let article: [String: Unions.ArticleResult]
         let articleSavingRequest: [String: Unions.ArticleSavingRequestResult]
         let deviceTokens: [String: Unions.DeviceTokensResult]
+        let discoverFeeds: [String: Unions.DiscoverFeedResult]
+        let discoverTopics: [String: Unions.GetDiscoverTopicResults]
         let feeds: [String: Unions.FeedsResult]
         let filters: [String: Unions.FiltersResult]
+        let getDiscoverFeedArticles: [String: Unions.GetDiscoverFeedArticleResults]
         let getUserPersonalization: [String: Unions.GetUserPersonalizationResult]
         let groups: [String: Unions.GroupsResult]
         let hello: [String: String]
+        let integration: [String: Unions.IntegrationResult]
         let integrations: [String: Unions.IntegrationsResult]
         let labels: [String: Unions.LabelsResult]
         let me: [String: Objects.User]
@@ -14276,6 +16225,14 @@ extension Objects.Query: Decodable {
                 if let value = try container.decode(Unions.DeviceTokensResult?.self, forKey: codingKey) {
                     map.set(key: field, hash: alias, value: value as Any)
                 }
+            case "discoverFeeds":
+                if let value = try container.decode(Unions.DiscoverFeedResult?.self, forKey: codingKey) {
+                    map.set(key: field, hash: alias, value: value as Any)
+                }
+            case "discoverTopics":
+                if let value = try container.decode(Unions.GetDiscoverTopicResults?.self, forKey: codingKey) {
+                    map.set(key: field, hash: alias, value: value as Any)
+                }
             case "feeds":
                 if let value = try container.decode(Unions.FeedsResult?.self, forKey: codingKey) {
                     map.set(key: field, hash: alias, value: value as Any)
@@ -14284,6 +16241,10 @@ extension Objects.Query: Decodable {
                 if let value = try container.decode(Unions.FiltersResult?.self, forKey: codingKey) {
                     map.set(key: field, hash: alias, value: value as Any)
                 }
+            case "getDiscoverFeedArticles":
+                if let value = try container.decode(Unions.GetDiscoverFeedArticleResults?.self, forKey: codingKey) {
+                    map.set(key: field, hash: alias, value: value as Any)
+                }
             case "getUserPersonalization":
                 if let value = try container.decode(Unions.GetUserPersonalizationResult?.self, forKey: codingKey) {
                     map.set(key: field, hash: alias, value: value as Any)
@@ -14296,6 +16257,10 @@ extension Objects.Query: Decodable {
                 if let value = try container.decode(String?.self, forKey: codingKey) {
                     map.set(key: field, hash: alias, value: value as Any)
                 }
+            case "integration":
+                if let value = try container.decode(Unions.IntegrationResult?.self, forKey: codingKey) {
+                    map.set(key: field, hash: alias, value: value as Any)
+                }
             case "integrations":
                 if let value = try container.decode(Unions.IntegrationsResult?.self, forKey: codingKey) {
                     map.set(key: field, hash: alias, value: value as Any)
@@ -14382,11 +16347,15 @@ extension Objects.Query: Decodable {
         article = map["article"]
         articleSavingRequest = map["articleSavingRequest"]
         deviceTokens = map["deviceTokens"]
+        discoverFeeds = map["discoverFeeds"]
+        discoverTopics = map["discoverTopics"]
         feeds = map["feeds"]
         filters = map["filters"]
+        getDiscoverFeedArticles = map["getDiscoverFeedArticles"]
         getUserPersonalization = map["getUserPersonalization"]
         groups = map["groups"]
         hello = map["hello"]
+        integration = map["integration"]
         integrations = map["integrations"]
         labels = map["labels"]
         me = map["me"]
@@ -14485,6 +16454,44 @@ extension Fields where TypeLock == Objects.Query {
         }
     }
 
+    func discoverFeeds<Type>(selection: Selection<Type, Unions.DiscoverFeedResult>) throws -> Type {
+        let field = GraphQLField.composite(
+            name: "discoverFeeds",
+            arguments: [],
+            selection: selection.selection
+        )
+        select(field)
+
+        switch response {
+        case let .decoding(data):
+            if let data = data.discoverFeeds[field.alias!] {
+                return try selection.decode(data: data)
+            }
+            throw HttpError.badpayload
+        case .mocking:
+            return selection.mock()
+        }
+    }
+
+    func discoverTopics<Type>(selection: Selection<Type, Unions.GetDiscoverTopicResults>) throws -> Type {
+        let field = GraphQLField.composite(
+            name: "discoverTopics",
+            arguments: [],
+            selection: selection.selection
+        )
+        select(field)
+
+        switch response {
+        case let .decoding(data):
+            if let data = data.discoverTopics[field.alias!] {
+                return try selection.decode(data: data)
+            }
+            throw HttpError.badpayload
+        case .mocking:
+            return selection.mock()
+        }
+    }
+
     func feeds<Type>(input: InputObjects.FeedsInput, selection: Selection<Type, Unions.FeedsResult>) throws -> Type {
         let field = GraphQLField.composite(
             name: "feeds",
@@ -14523,6 +16530,25 @@ extension Fields where TypeLock == Objects.Query {
         }
     }
 
+    func getDiscoverFeedArticles<Type>(after: OptionalArgument<String> = .absent(), discoverTopicId: String, feedId: OptionalArgument<String> = .absent(), first: OptionalArgument<Int> = .absent(), selection: Selection<Type, Unions.GetDiscoverFeedArticleResults>) throws -> Type {
+        let field = GraphQLField.composite(
+            name: "getDiscoverFeedArticles",
+            arguments: [Argument(name: "after", type: "String", value: after), Argument(name: "discoverTopicId", type: "String!", value: discoverTopicId), Argument(name: "feedId", type: "ID", value: feedId), Argument(name: "first", type: "Int", value: first)],
+            selection: selection.selection
+        )
+        select(field)
+
+        switch response {
+        case let .decoding(data):
+            if let data = data.getDiscoverFeedArticles[field.alias!] {
+                return try selection.decode(data: data)
+            }
+            throw HttpError.badpayload
+        case .mocking:
+            return selection.mock()
+        }
+    }
+
     func getUserPersonalization<Type>(selection: Selection<Type, Unions.GetUserPersonalizationResult>) throws -> Type {
         let field = GraphQLField.composite(
             name: "getUserPersonalization",
@@ -14576,6 +16602,25 @@ extension Fields where TypeLock == Objects.Query {
         }
     }
 
+    func integration<Type>(name: String, selection: Selection<Type, Unions.IntegrationResult>) throws -> Type {
+        let field = GraphQLField.composite(
+            name: "integration",
+            arguments: [Argument(name: "name", type: "String!", value: name)],
+            selection: selection.selection
+        )
+        select(field)
+
+        switch response {
+        case let .decoding(data):
+            if let data = data.integration[field.alias!] {
+                return try selection.decode(data: data)
+            }
+            throw HttpError.badpayload
+        case .mocking:
+            return selection.mock()
+        }
+    }
+
     func integrations<Type>(selection: Selection<Type, Unions.IntegrationsResult>) throws -> Type {
         let field = GraphQLField.composite(
             name: "integrations",
@@ -17133,6 +19178,7 @@ extension Objects {
         let createdAt: [String: DateTime]
         let enabled: [String: Bool]
         let eventTypes: [String: [Enums.RuleEventType]]
+        let failedAt: [String: DateTime]
         let filter: [String: String]
         let id: [String: String]
         let name: [String: String]
@@ -17172,6 +19218,10 @@ extension Objects.Rule: Decodable {
                 if let value = try container.decode([Enums.RuleEventType]?.self, forKey: codingKey) {
                     map.set(key: field, hash: alias, value: value as Any)
                 }
+            case "failedAt":
+                if let value = try container.decode(DateTime?.self, forKey: codingKey) {
+                    map.set(key: field, hash: alias, value: value as Any)
+                }
             case "filter":
                 if let value = try container.decode(String?.self, forKey: codingKey) {
                     map.set(key: field, hash: alias, value: value as Any)
@@ -17202,6 +19252,7 @@ extension Objects.Rule: Decodable {
         createdAt = map["createdAt"]
         enabled = map["enabled"]
         eventTypes = map["eventTypes"]
+        failedAt = map["failedAt"]
         filter = map["filter"]
         id = map["id"]
         name = map["name"]
@@ -17283,6 +19334,21 @@ extension Fields where TypeLock == Objects.Rule {
         }
     }
 
+    func failedAt() throws -> DateTime? {
+        let field = GraphQLField.leaf(
+            name: "failedAt",
+            arguments: []
+        )
+        select(field)
+
+        switch response {
+        case let .decoding(data):
+            return data.failedAt[field.alias!]
+        case .mocking:
+            return nil
+        }
+    }
+
     func filter() throws -> String {
         let field = GraphQLField.leaf(
             name: "filter",
@@ -17708,6 +19774,160 @@ extension Selection where TypeLock == Never, Type == Never {
     typealias SaveArticleReadingProgressSuccess<T> = Selection<T, Objects.SaveArticleReadingProgressSuccess>
 }
 
+extension Objects {
+    struct SaveDiscoverArticleError {
+        let __typename: TypeName = .saveDiscoverArticleError
+        let errorCodes: [String: [Enums.SaveDiscoverArticleErrorCode]]
+
+        enum TypeName: String, Codable {
+            case saveDiscoverArticleError = "SaveDiscoverArticleError"
+        }
+    }
+}
+
+extension Objects.SaveDiscoverArticleError: Decodable {
+    init(from decoder: Decoder) throws {
+        let container = try decoder.container(keyedBy: DynamicCodingKeys.self)
+
+        var map = HashMap()
+        for codingKey in container.allKeys {
+            if codingKey.isTypenameKey { continue }
+
+            let alias = codingKey.stringValue
+            let field = GraphQLField.getFieldNameFromAlias(alias)
+
+            switch field {
+            case "errorCodes":
+                if let value = try container.decode([Enums.SaveDiscoverArticleErrorCode]?.self, forKey: codingKey) {
+                    map.set(key: field, hash: alias, value: value as Any)
+                }
+            default:
+                throw DecodingError.dataCorrupted(
+                    DecodingError.Context(
+                        codingPath: decoder.codingPath,
+                        debugDescription: "Unknown key \(field)."
+                    )
+                )
+            }
+        }
+
+        errorCodes = map["errorCodes"]
+    }
+}
+
+extension Fields where TypeLock == Objects.SaveDiscoverArticleError {
+    func errorCodes() throws -> [Enums.SaveDiscoverArticleErrorCode] {
+        let field = GraphQLField.leaf(
+            name: "errorCodes",
+            arguments: []
+        )
+        select(field)
+
+        switch response {
+        case let .decoding(data):
+            if let data = data.errorCodes[field.alias!] {
+                return data
+            }
+            throw HttpError.badpayload
+        case .mocking:
+            return []
+        }
+    }
+}
+
+extension Selection where TypeLock == Never, Type == Never {
+    typealias SaveDiscoverArticleError<T> = Selection<T, Objects.SaveDiscoverArticleError>
+}
+
+extension Objects {
+    struct SaveDiscoverArticleSuccess {
+        let __typename: TypeName = .saveDiscoverArticleSuccess
+        let saveId: [String: String]
+        let url: [String: String]
+
+        enum TypeName: String, Codable {
+            case saveDiscoverArticleSuccess = "SaveDiscoverArticleSuccess"
+        }
+    }
+}
+
+extension Objects.SaveDiscoverArticleSuccess: Decodable {
+    init(from decoder: Decoder) throws {
+        let container = try decoder.container(keyedBy: DynamicCodingKeys.self)
+
+        var map = HashMap()
+        for codingKey in container.allKeys {
+            if codingKey.isTypenameKey { continue }
+
+            let alias = codingKey.stringValue
+            let field = GraphQLField.getFieldNameFromAlias(alias)
+
+            switch field {
+            case "saveId":
+                if let value = try container.decode(String?.self, forKey: codingKey) {
+                    map.set(key: field, hash: alias, value: value as Any)
+                }
+            case "url":
+                if let value = try container.decode(String?.self, forKey: codingKey) {
+                    map.set(key: field, hash: alias, value: value as Any)
+                }
+            default:
+                throw DecodingError.dataCorrupted(
+                    DecodingError.Context(
+                        codingPath: decoder.codingPath,
+                        debugDescription: "Unknown key \(field)."
+                    )
+                )
+            }
+        }
+
+        saveId = map["saveId"]
+        url = map["url"]
+    }
+}
+
+extension Fields where TypeLock == Objects.SaveDiscoverArticleSuccess {
+    func saveId() throws -> String {
+        let field = GraphQLField.leaf(
+            name: "saveId",
+            arguments: []
+        )
+        select(field)
+
+        switch response {
+        case let .decoding(data):
+            if let data = data.saveId[field.alias!] {
+                return data
+            }
+            throw HttpError.badpayload
+        case .mocking:
+            return String.mockValue
+        }
+    }
+
+    func url() throws -> String {
+        let field = GraphQLField.leaf(
+            name: "url",
+            arguments: []
+        )
+        select(field)
+
+        switch response {
+        case let .decoding(data):
+            if let data = data.url[field.alias!] {
+                return data
+            }
+            throw HttpError.badpayload
+        case .mocking:
+            return String.mockValue
+        }
+    }
+}
+
+extension Selection where TypeLock == Never, Type == Never {
+    typealias SaveDiscoverArticleSuccess<T> = Selection<T, Objects.SaveDiscoverArticleSuccess>
+}
+
 extension Objects {
     struct SaveError {
         let __typename: TypeName = .saveError
@@ -22200,6 +24420,227 @@ extension Selection where TypeLock == Never, Type == Never {
     typealias SyncUpdatedItemEdge<T> = Selection<T, Objects.SyncUpdatedItemEdge>
 }
 
+extension Objects {
+    struct Task {
+        let __typename: TypeName = .task
+        let cancellable: [String: Bool]
+        let createdAt: [String: DateTime]
+        let failedReason: [String: String]
+        let id: [String: String]
+        let name: [String: String]
+        let progress: [String: Double]
+        let runningTime: [String: Int]
+        let state: [String: Enums.TaskState]
+
+        enum TypeName: String, Codable {
+            case task = "Task"
+        }
+    }
+}
+
+extension Objects.Task: Decodable {
+    init(from decoder: Decoder) throws {
+        let container = try decoder.container(keyedBy: DynamicCodingKeys.self)
+
+        var map = HashMap()
+        for codingKey in container.allKeys {
+            if codingKey.isTypenameKey { continue }
+
+            let alias = codingKey.stringValue
+            let field = GraphQLField.getFieldNameFromAlias(alias)
+
+            switch field {
+            case "cancellable":
+                if let value = try container.decode(Bool?.self, forKey: codingKey) {
+                    map.set(key: field, hash: alias, value: value as Any)
+                }
+            case "createdAt":
+                if let value = try container.decode(DateTime?.self, forKey: codingKey) {
+                    map.set(key: field, hash: alias, value: value as Any)
+                }
+            case "failedReason":
+                if let value = try container.decode(String?.self, forKey: codingKey) {
+                    map.set(key: field, hash: alias, value: value as Any)
+                }
+            case "id":
+                if let value = try container.decode(String?.self, forKey: codingKey) {
+                    map.set(key: field, hash: alias, value: value as Any)
+                }
+            case "name":
+                if let value = try container.decode(String?.self, forKey: codingKey) {
+                    map.set(key: field, hash: alias, value: value as Any)
+                }
+            case "progress":
+                if let value = try container.decode(Double?.self, forKey: codingKey) {
+                    map.set(key: field, hash: alias, value: value as Any)
+                }
+            case "runningTime":
+                if let value = try container.decode(Int?.self, forKey: codingKey) {
+                    map.set(key: field, hash: alias, value: value as Any)
+                }
+            case "state":
+                if let value = try container.decode(Enums.TaskState?.self, forKey: codingKey) {
+                    map.set(key: field, hash: alias, value: value as Any)
+                }
+            default:
+                throw DecodingError.dataCorrupted(
+                    DecodingError.Context(
+                        codingPath: decoder.codingPath,
+                        debugDescription: "Unknown key \(field)."
+                    )
+                )
+            }
+        }
+
+        cancellable = map["cancellable"]
+        createdAt = map["createdAt"]
+        failedReason = map["failedReason"]
+        id = map["id"]
+        name = map["name"]
+        progress = map["progress"]
+        runningTime = map["runningTime"]
+        state = map["state"]
+    }
+}
+
+extension Fields where TypeLock == Objects.Task {
+    func cancellable() throws -> Bool? {
+        let field = GraphQLField.leaf(
+            name: "cancellable",
+            arguments: []
+        )
+        select(field)
+
+        switch response {
+        case let .decoding(data):
+            return data.cancellable[field.alias!]
+        case .mocking:
+            return nil
+        }
+    }
+
+    func createdAt() throws -> DateTime {
+        let field = GraphQLField.leaf(
+            name: "createdAt",
+            arguments: []
+        )
+        select(field)
+
+        switch response {
+        case let .decoding(data):
+            if let data = data.createdAt[field.alias!] {
+                return data
+            }
+            throw HttpError.badpayload
+        case .mocking:
+            return DateTime.mockValue
+        }
+    }
+
+    func failedReason() throws -> String? {
+        let field = GraphQLField.leaf(
+            name: "failedReason",
+            arguments: []
+        )
+        select(field)
+
+        switch response {
+        case let .decoding(data):
+            return data.failedReason[field.alias!]
+        case .mocking:
+            return nil
+        }
+    }
+
+    func id() throws -> String {
+        let field = GraphQLField.leaf(
+            name: "id",
+            arguments: []
+        )
+        select(field)
+
+        switch response {
+        case let .decoding(data):
+            if let data = data.id[field.alias!] {
+                return data
+            }
+            throw HttpError.badpayload
+        case .mocking:
+            return String.mockValue
+        }
+    }
+
+    func name() throws -> String {
+        let field = GraphQLField.leaf(
+            name: "name",
+            arguments: []
+        )
+        select(field)
+
+        switch response {
+        case let .decoding(data):
+            if let data = data.name[field.alias!] {
+                return data
+            }
+            throw HttpError.badpayload
+        case .mocking:
+            return String.mockValue
+        }
+    }
+
+    func progress() throws -> Double? {
+        let field = GraphQLField.leaf(
+            name: "progress",
+            arguments: []
+        )
+        select(field)
+
+        switch response {
+        case let .decoding(data):
+            return data.progress[field.alias!]
+        case .mocking:
+            return nil
+        }
+    }
+
+    func runningTime() throws -> Int? {
+        let field = GraphQLField.leaf(
+            name: "runningTime",
+            arguments: []
+        )
+        select(field)
+
+        switch response {
+        case let .decoding(data):
+            return data.runningTime[field.alias!]
+        case .mocking:
+            return nil
+        }
+    }
+
+    func state() throws -> Enums.TaskState {
+        let field = GraphQLField.leaf(
+            name: "state",
+            arguments: []
+        )
+        select(field)
+
+        switch response {
+        case let .decoding(data):
+            if let data = data.state[field.alias!] {
+                return data
+            }
+            throw HttpError.badpayload
+        case .mocking:
+            return Enums.TaskState.allCases.first!
+        }
+    }
+}
+
+extension Selection where TypeLock == Never, Type == Never {
+    typealias Task<T> = Selection<T, Objects.Task>
+}
+
 extension Objects {
     struct TypeaheadSearchError {
         let __typename: TypeName = .typeaheadSearchError
@@ -24845,6 +27286,7 @@ extension Objects {
     struct User {
         let __typename: TypeName = .user
         let email: [String: String]
+        let featureList: [String: [Objects.Feature]]
         let features: [String: [String?]]
         let followersCount: [String: Int]
         let friendsCount: [String: Int]
@@ -24884,6 +27326,10 @@ extension Objects.User: Decodable {
                 if let value = try container.decode(String?.self, forKey: codingKey) {
                     map.set(key: field, hash: alias, value: value as Any)
                 }
+            case "featureList":
+                if let value = try container.decode([Objects.Feature]?.self, forKey: codingKey) {
+                    map.set(key: field, hash: alias, value: value as Any)
+                }
             case "features":
                 if let value = try container.decode([String?]?.self, forKey: codingKey) {
                     map.set(key: field, hash: alias, value: value as Any)
@@ -24959,6 +27405,7 @@ extension Objects.User: Decodable {
         }
 
         email = map["email"]
+        featureList = map["featureList"]
         features = map["features"]
         followersCount = map["followersCount"]
         friendsCount = map["friendsCount"]
@@ -24994,6 +27441,22 @@ extension Fields where TypeLock == Objects.User {
         }
     }
 
+    func featureList<Type>(selection: Selection<Type, [Objects.Feature]?>) throws -> Type {
+        let field = GraphQLField.composite(
+            name: "featureList",
+            arguments: [],
+            selection: selection.selection
+        )
+        select(field)
+
+        switch response {
+        case let .decoding(data):
+            return try selection.decode(data: data.featureList[field.alias!])
+        case .mocking:
+            return selection.mock()
+        }
+    }
+
     func features() throws -> [String?]? {
         let field = GraphQLField.leaf(
             name: "features",
@@ -26308,6 +28771,80 @@ enum Interfaces {}
 // MARK: - Unions
 
 enum Unions {}
+extension Unions {
+    struct AddDiscoverFeedResult {
+        let __typename: TypeName
+        let errorCodes: [String: [Enums.AddDiscoverFeedErrorCode]]
+        let feed: [String: Objects.DiscoverFeed]
+
+        enum TypeName: String, Codable {
+            case addDiscoverFeedError = "AddDiscoverFeedError"
+            case addDiscoverFeedSuccess = "AddDiscoverFeedSuccess"
+        }
+    }
+}
+
+extension Unions.AddDiscoverFeedResult: Decodable {
+    init(from decoder: Decoder) throws {
+        let container = try decoder.container(keyedBy: DynamicCodingKeys.self)
+
+        var map = HashMap()
+        for codingKey in container.allKeys {
+            if codingKey.isTypenameKey { continue }
+
+            let alias = codingKey.stringValue
+            let field = GraphQLField.getFieldNameFromAlias(alias)
+
+            switch field {
+            case "errorCodes":
+                if let value = try container.decode([Enums.AddDiscoverFeedErrorCode]?.self, forKey: codingKey) {
+                    map.set(key: field, hash: alias, value: value as Any)
+                }
+            case "feed":
+                if let value = try container.decode(Objects.DiscoverFeed?.self, forKey: codingKey) {
+                    map.set(key: field, hash: alias, value: value as Any)
+                }
+            default:
+                throw DecodingError.dataCorrupted(
+                    DecodingError.Context(
+                        codingPath: decoder.codingPath,
+                        debugDescription: "Unknown key \(field)."
+                    )
+                )
+            }
+        }
+
+        __typename = try container.decode(TypeName.self, forKey: DynamicCodingKeys(stringValue: "__typename")!)
+
+        errorCodes = map["errorCodes"]
+        feed = map["feed"]
+    }
+}
+
+extension Fields where TypeLock == Unions.AddDiscoverFeedResult {
+    func on<Type>(addDiscoverFeedError: Selection<Type, Objects.AddDiscoverFeedError>, addDiscoverFeedSuccess: Selection<Type, Objects.AddDiscoverFeedSuccess>) throws -> Type {
+        select([GraphQLField.fragment(type: "AddDiscoverFeedError", selection: addDiscoverFeedError.selection), GraphQLField.fragment(type: "AddDiscoverFeedSuccess", selection: addDiscoverFeedSuccess.selection)])
+
+        switch response {
+        case let .decoding(data):
+            switch data.__typename {
+            case .addDiscoverFeedError:
+                let data = Objects.AddDiscoverFeedError(errorCodes: data.errorCodes)
+                return try addDiscoverFeedError.decode(data: data)
+            case .addDiscoverFeedSuccess:
+                let data = Objects.AddDiscoverFeedSuccess(feed: data.feed)
+                return try addDiscoverFeedSuccess.decode(data: data)
+            }
+        case .mocking:
+            return addDiscoverFeedError.mock()
+        }
+    }
+}
+
+extension Selection where TypeLock == Never, Type == Never {
+    typealias AddDiscoverFeedResult<T> = Selection<T, Unions.AddDiscoverFeedResult>
+}
+
 extension Unions {
     struct AddPopularReadResult {
         let __typename: TypeName
@@ -27590,6 +30127,154 @@ extension Selection where TypeLock == Never, Type == Never {
     typealias DeleteAccountResult<T> = Selection<T, Unions.DeleteAccountResult>
 }
 
+extension Unions {
+    struct DeleteDiscoverArticleResult {
+        let __typename: TypeName
+        let errorCodes: [String: [Enums.DeleteDiscoverArticleErrorCode]]
+        let id: [String: String]
+
+        enum TypeName: String, Codable {
+            case deleteDiscoverArticleError = "DeleteDiscoverArticleError"
+            case deleteDiscoverArticleSuccess = "DeleteDiscoverArticleSuccess"
+        }
+    }
+}
+
+extension Unions.DeleteDiscoverArticleResult: Decodable {
+    init(from decoder: Decoder) throws {
+        let container = try decoder.container(keyedBy: DynamicCodingKeys.self)
+
+        var map = HashMap()
+        for codingKey in container.allKeys {
+            if codingKey.isTypenameKey { continue }
+
+            let alias = codingKey.stringValue
+            let field = GraphQLField.getFieldNameFromAlias(alias)
+
+            switch field {
+            case "errorCodes":
+                if let value = try container.decode([Enums.DeleteDiscoverArticleErrorCode]?.self, forKey: codingKey) {
+                    map.set(key: field, hash: alias, value: value as Any)
+                }
+            case "id":
+                if let value = try container.decode(String?.self, forKey: codingKey) {
+                    map.set(key: field, hash: alias, value: value as Any)
+                }
+            default:
+                throw DecodingError.dataCorrupted(
+                    DecodingError.Context(
+                        codingPath: decoder.codingPath,
+                        debugDescription: "Unknown key \(field)."
+                    )
+                )
+            }
+        }
+
+        __typename = try container.decode(TypeName.self, forKey: DynamicCodingKeys(stringValue: "__typename")!)
+
+        errorCodes = map["errorCodes"]
+        id = map["id"]
+    }
+}
+
+extension Fields where TypeLock == Unions.DeleteDiscoverArticleResult {
+    func on<Type>(deleteDiscoverArticleError: Selection<Type, Objects.DeleteDiscoverArticleError>, deleteDiscoverArticleSuccess: Selection<Type, Objects.DeleteDiscoverArticleSuccess>) throws -> Type {
+        select([GraphQLField.fragment(type: "DeleteDiscoverArticleError", selection: deleteDiscoverArticleError.selection), GraphQLField.fragment(type: "DeleteDiscoverArticleSuccess", selection: deleteDiscoverArticleSuccess.selection)])
+
+        switch response {
+        case let .decoding(data):
+            switch data.__typename {
+            case .deleteDiscoverArticleError:
+                let data = Objects.DeleteDiscoverArticleError(errorCodes: data.errorCodes)
+                return try deleteDiscoverArticleError.decode(data: data)
+            case .deleteDiscoverArticleSuccess:
+                let data = Objects.DeleteDiscoverArticleSuccess(id: data.id)
+                return try deleteDiscoverArticleSuccess.decode(data: data)
+            }
+        case .mocking:
+            return deleteDiscoverArticleError.mock()
+        }
+    }
+}
+
+extension Selection where TypeLock == Never, Type == Never {
+    typealias DeleteDiscoverArticleResult<T> = Selection<T, Unions.DeleteDiscoverArticleResult>
+}
+
+extension Unions {
+    struct DeleteDiscoverFeedResult {
+        let __typename: TypeName
+        let errorCodes: [String: [Enums.DeleteDiscoverFeedErrorCode]]
+        let id: [String: String]
+
+        enum TypeName: String, Codable {
+            case deleteDiscoverFeedError = "DeleteDiscoverFeedError"
+            case deleteDiscoverFeedSuccess = "DeleteDiscoverFeedSuccess"
+        }
+    }
+}
+
+extension Unions.DeleteDiscoverFeedResult: Decodable {
+    init(from decoder: Decoder) throws {
+        let container = try decoder.container(keyedBy: DynamicCodingKeys.self)
+
+        var map = HashMap()
+        for codingKey in container.allKeys {
+            if codingKey.isTypenameKey { continue }
+
+            let alias = codingKey.stringValue
+            let field = GraphQLField.getFieldNameFromAlias(alias)
+
+            switch field {
+            case "errorCodes":
+                if let value = try container.decode([Enums.DeleteDiscoverFeedErrorCode]?.self, forKey: codingKey) {
+                    map.set(key: field, hash: alias, value: value as Any)
+                }
+            case "id":
+                if let value = try container.decode(String?.self, forKey: codingKey) {
+                    map.set(key: field, hash: alias, value: value as Any)
+                }
+            default:
+                throw DecodingError.dataCorrupted(
+                    DecodingError.Context(
+                        codingPath: decoder.codingPath,
+                        debugDescription: "Unknown key \(field)."
+                    )
+                )
+            }
+        }
+
+        __typename = try container.decode(TypeName.self, forKey: DynamicCodingKeys(stringValue: "__typename")!)
+
+        errorCodes = map["errorCodes"]
+        id = map["id"]
+    }
+}
+
+extension Fields where TypeLock == Unions.DeleteDiscoverFeedResult {
+    func on<Type>(deleteDiscoverFeedError: Selection<Type, Objects.DeleteDiscoverFeedError>, deleteDiscoverFeedSuccess: Selection<Type, Objects.DeleteDiscoverFeedSuccess>) throws -> Type {
+        select([GraphQLField.fragment(type: "DeleteDiscoverFeedError", selection: deleteDiscoverFeedError.selection), GraphQLField.fragment(type: "DeleteDiscoverFeedSuccess", selection: deleteDiscoverFeedSuccess.selection)])
+
+        switch response {
+        case let .decoding(data):
+            switch data.__typename {
+            case .deleteDiscoverFeedError:
+                let data = Objects.DeleteDiscoverFeedError(errorCodes: data.errorCodes)
+                return try deleteDiscoverFeedError.decode(data: data)
+            case .deleteDiscoverFeedSuccess:
+                let data = Objects.DeleteDiscoverFeedSuccess(id: data.id)
+                return try deleteDiscoverFeedSuccess.decode(data: data)
+            }
+        case .mocking:
+            return deleteDiscoverFeedError.mock()
+        }
+    }
+}
+
+extension Selection where TypeLock == Never, Type == Never {
+    typealias DeleteDiscoverFeedResult<T> = Selection<T, Unions.DeleteDiscoverFeedResult>
+}
+
 extension Unions {
     struct DeleteFilterResult {
         let __typename: TypeName
@@ -28404,6 +31089,154 @@ extension Selection where TypeLock == Never, Type == Never {
     typealias DeviceTokensResult<T> = Selection<T, Unions.DeviceTokensResult>
 }
 
+extension Unions {
+    struct DiscoverFeedResult {
+        let __typename: TypeName
+        let errorCodes: [String: [Enums.DiscoverFeedErrorCode]]
+        let feeds: [String: [Objects.DiscoverFeed?]]
+
+        enum TypeName: String, Codable {
+            case discoverFeedError = "DiscoverFeedError"
+            case discoverFeedSuccess = "DiscoverFeedSuccess"
+        }
+    }
+}
+
+extension Unions.DiscoverFeedResult: Decodable {
+    init(from decoder: Decoder) throws {
+        let container = try decoder.container(keyedBy: DynamicCodingKeys.self)
+
+        var map = HashMap()
+        for codingKey in container.allKeys {
+            if codingKey.isTypenameKey { continue }
+
+            let alias = codingKey.stringValue
+            let field = GraphQLField.getFieldNameFromAlias(alias)
+
+            switch field {
+            case "errorCodes":
+                if let value = try container.decode([Enums.DiscoverFeedErrorCode]?.self, forKey: codingKey) {
+                    map.set(key: field, hash: alias, value: value as Any)
+                }
+            case "feeds":
+                if let value = try container.decode([Objects.DiscoverFeed?]?.self, forKey: codingKey) {
+                    map.set(key: field, hash: alias, value: value as Any)
+                }
+            default:
+                throw DecodingError.dataCorrupted(
+                    DecodingError.Context(
+                        codingPath: decoder.codingPath,
+                        debugDescription: "Unknown key \(field)."
+                    )
+                )
+            }
+        }
+
+        __typename = try container.decode(TypeName.self, forKey: DynamicCodingKeys(stringValue: "__typename")!)
+
+        errorCodes = map["errorCodes"]
+        feeds = map["feeds"]
+    }
+}
+
+extension Fields where TypeLock == Unions.DiscoverFeedResult {
+    func on<Type>(discoverFeedError: Selection<Type, Objects.DiscoverFeedError>, discoverFeedSuccess: Selection<Type, Objects.DiscoverFeedSuccess>) throws -> Type {
+        select([GraphQLField.fragment(type: "DiscoverFeedError", selection: discoverFeedError.selection), GraphQLField.fragment(type: "DiscoverFeedSuccess", selection: discoverFeedSuccess.selection)])
+
+        switch response {
+        case let .decoding(data):
+            switch data.__typename {
+            case .discoverFeedError:
+                let data = Objects.DiscoverFeedError(errorCodes: data.errorCodes)
+                return try discoverFeedError.decode(data: data)
+            case .discoverFeedSuccess:
+                let data = Objects.DiscoverFeedSuccess(feeds: data.feeds)
+                return try discoverFeedSuccess.decode(data: data)
+            }
+        case .mocking:
+            return discoverFeedError.mock()
+        }
+    }
+}
+
+extension Selection where TypeLock == Never, Type == Never {
+    typealias DiscoverFeedResult<T> = Selection<T, Unions.DiscoverFeedResult>
+}
+
+extension Unions {
+    struct EditDiscoverFeedResult {
+        let __typename: TypeName
+        let errorCodes: [String: [Enums.EditDiscoverFeedErrorCode]]
+        let id: [String: String]
+
+        enum TypeName: String, Codable {
+            case editDiscoverFeedError = "EditDiscoverFeedError"
+            case editDiscoverFeedSuccess = "EditDiscoverFeedSuccess"
+        }
+    }
+}
+
+extension Unions.EditDiscoverFeedResult: Decodable {
+    init(from decoder: Decoder) throws {
+        let container = try decoder.container(keyedBy: DynamicCodingKeys.self)
+
+        var map = HashMap()
+        for codingKey in container.allKeys {
+            if codingKey.isTypenameKey { continue }
+
+            let alias = codingKey.stringValue
+            let field = GraphQLField.getFieldNameFromAlias(alias)
+
+            switch field {
+            case "errorCodes":
+                if let value = try container.decode([Enums.EditDiscoverFeedErrorCode]?.self, forKey: codingKey) {
+                    map.set(key: field, hash: alias, value: value as Any)
+                }
+            case "id":
+                if let value = try container.decode(String?.self, forKey: codingKey) {
+                    map.set(key: field, hash: alias, value: value as Any)
+                }
+            default:
+                throw DecodingError.dataCorrupted(
+                    DecodingError.Context(
+                        codingPath: decoder.codingPath,
+                        debugDescription: "Unknown key \(field)."
+                    )
+                )
+            }
+        }
+
+        __typename = try container.decode(TypeName.self, forKey: DynamicCodingKeys(stringValue: "__typename")!)
+
+        errorCodes = map["errorCodes"]
+        id = map["id"]
+    }
+}
+
+extension Fields where TypeLock == Unions.EditDiscoverFeedResult {
+    func on<Type>(editDiscoverFeedError: Selection<Type, Objects.EditDiscoverFeedError>, editDiscoverFeedSuccess: Selection<Type, Objects.EditDiscoverFeedSuccess>) throws -> Type {
+        select([GraphQLField.fragment(type: "EditDiscoverFeedError", selection: editDiscoverFeedError.selection), GraphQLField.fragment(type: "EditDiscoverFeedSuccess", selection: editDiscoverFeedSuccess.selection)])
+
+        switch response {
+        case let .decoding(data):
+            switch data.__typename {
+            case .editDiscoverFeedError:
+                let data = Objects.EditDiscoverFeedError(errorCodes: data.errorCodes)
+                return try editDiscoverFeedError.decode(data: data)
+            case .editDiscoverFeedSuccess:
+                let data = Objects.EditDiscoverFeedSuccess(id: data.id)
+                return try editDiscoverFeedSuccess.decode(data: data)
+            }
+        case .mocking:
+            return editDiscoverFeedError.mock()
+        }
+    }
+}
+
+extension Selection where TypeLock == Never, Type == Never {
+    typealias EditDiscoverFeedResult<T> = Selection<T, Unions.EditDiscoverFeedResult>
+}
+
 extension Unions {
     struct EmptyTrashResult {
         let __typename: TypeName
@@ -28478,6 +31311,80 @@ extension Selection where TypeLock == Never, Type == Never {
     typealias EmptyTrashResult<T> = Selection<T, Unions.EmptyTrashResult>
 }
 
+extension Unions {
+    struct ExportToIntegrationResult {
+        let __typename: TypeName
+        let errorCodes: [String: [Enums.ExportToIntegrationErrorCode]]
+        let task: [String: Objects.Task]
+
+        enum TypeName: String, Codable {
+            case exportToIntegrationError = "ExportToIntegrationError"
+            case exportToIntegrationSuccess = "ExportToIntegrationSuccess"
+        }
+    }
+}
+
+extension Unions.ExportToIntegrationResult: Decodable {
+    init(from decoder: Decoder) throws {
+        let container = try decoder.container(keyedBy: DynamicCodingKeys.self)
+
+        var map = HashMap()
+        for codingKey in container.allKeys {
+            if codingKey.isTypenameKey { continue }
+
+            let alias = codingKey.stringValue
+            let field = GraphQLField.getFieldNameFromAlias(alias)
+
+            switch field {
+            case "errorCodes":
+                if let value = try container.decode([Enums.ExportToIntegrationErrorCode]?.self, forKey: codingKey) {
+                    map.set(key: field, hash: alias, value: value as Any)
+                }
+            case "task":
+                if let value = try container.decode(Objects.Task?.self, forKey: codingKey) {
+                    map.set(key: field, hash: alias, value: value as Any)
+                }
+            default:
+                throw DecodingError.dataCorrupted(
+                    DecodingError.Context(
+                        codingPath: decoder.codingPath,
+                        debugDescription: "Unknown key \(field)."
+                    )
+                )
+            }
+        }
+
+        __typename = try container.decode(TypeName.self, forKey: DynamicCodingKeys(stringValue: "__typename")!)
+
+        errorCodes = map["errorCodes"]
+        task = map["task"]
+    }
+}
+
+extension Fields where TypeLock == Unions.ExportToIntegrationResult {
+    func on<Type>(exportToIntegrationError: Selection<Type, Objects.ExportToIntegrationError>, exportToIntegrationSuccess: Selection<Type, Objects.ExportToIntegrationSuccess>) throws -> Type {
+        select([GraphQLField.fragment(type: "ExportToIntegrationError", selection: exportToIntegrationError.selection), GraphQLField.fragment(type: "ExportToIntegrationSuccess", selection: exportToIntegrationSuccess.selection)])
+
+        switch response {
+        case let .decoding(data):
+            switch data.__typename {
+            case .exportToIntegrationError:
+                let data = Objects.ExportToIntegrationError(errorCodes: data.errorCodes)
+                return try exportToIntegrationError.decode(data: data)
+            case .exportToIntegrationSuccess:
+                let data = Objects.ExportToIntegrationSuccess(task: data.task)
+                return try exportToIntegrationSuccess.decode(data: data)
+            }
+        case .mocking:
+            return exportToIntegrationError.mock()
+        }
+    }
+}
+
+extension Selection where TypeLock == Never, Type == Never {
+    typealias ExportToIntegrationResult<T> = Selection<T, Unions.ExportToIntegrationResult>
+}
+
 extension Unions {
     struct FeedArticlesResult {
         let __typename: TypeName
@@ -28860,6 +31767,160 @@ extension Selection where TypeLock == Never, Type == Never {
     typealias GenerateApiKeyResult<T> = Selection<T, Unions.GenerateApiKeyResult>
 }
 
+extension Unions {
+    struct GetDiscoverFeedArticleResults {
+        let __typename: TypeName
+        let discoverArticles: [String: [Objects.DiscoverFeedArticle?]]
+        let errorCodes: [String: [Enums.GetDiscoverFeedArticleErrorCode]]
+        let pageInfo: [String: Objects.PageInfo]
+
+        enum TypeName: String, Codable {
+            case getDiscoverFeedArticleError = "GetDiscoverFeedArticleError"
+            case getDiscoverFeedArticleSuccess = "GetDiscoverFeedArticleSuccess"
+        }
+    }
+}
+
+extension Unions.GetDiscoverFeedArticleResults: Decodable {
+    init(from decoder: Decoder) throws {
+        let container = try decoder.container(keyedBy: DynamicCodingKeys.self)
+
+        var map = HashMap()
+        for codingKey in container.allKeys {
+            if codingKey.isTypenameKey { continue }
+
+            let alias = codingKey.stringValue
+            let field = GraphQLField.getFieldNameFromAlias(alias)
+
+            switch field {
+            case "discoverArticles":
+                if let value = try container.decode([Objects.DiscoverFeedArticle?]?.self, forKey: codingKey) {
+                    map.set(key: field, hash: alias, value: value as Any)
+                }
+            case "errorCodes":
+                if let value = try container.decode([Enums.GetDiscoverFeedArticleErrorCode]?.self, forKey: codingKey) {
+                    map.set(key: field, hash: alias, value: value as Any)
+                }
+            case "pageInfo":
+                if let value = try container.decode(Objects.PageInfo?.self, forKey: codingKey) {
+                    map.set(key: field, hash: alias, value: value as Any)
+                }
+            default:
+                throw DecodingError.dataCorrupted(
+                    DecodingError.Context(
+                        codingPath: decoder.codingPath,
+                        debugDescription: "Unknown key \(field)."
+                    )
+                )
+            }
+        }
+
+        __typename = try container.decode(TypeName.self, forKey: DynamicCodingKeys(stringValue: "__typename")!)
+
+        discoverArticles = map["discoverArticles"]
+        errorCodes = map["errorCodes"]
+        pageInfo = map["pageInfo"]
+    }
+}
+
+extension Fields where TypeLock == Unions.GetDiscoverFeedArticleResults {
+    func on<Type>(getDiscoverFeedArticleError: Selection<Type, Objects.GetDiscoverFeedArticleError>, getDiscoverFeedArticleSuccess: Selection<Type, Objects.GetDiscoverFeedArticleSuccess>) throws -> Type {
+        select([GraphQLField.fragment(type: "GetDiscoverFeedArticleError", selection: getDiscoverFeedArticleError.selection), GraphQLField.fragment(type: "GetDiscoverFeedArticleSuccess", selection: getDiscoverFeedArticleSuccess.selection)])
+
+        switch response {
+        case let .decoding(data):
+            switch data.__typename {
+            case .getDiscoverFeedArticleError:
+                let data = Objects.GetDiscoverFeedArticleError(errorCodes: data.errorCodes)
+                return try getDiscoverFeedArticleError.decode(data: data)
+            case .getDiscoverFeedArticleSuccess:
+                let data = Objects.GetDiscoverFeedArticleSuccess(discoverArticles: data.discoverArticles, pageInfo: data.pageInfo)
+                return try getDiscoverFeedArticleSuccess.decode(data: data)
+            }
+        case .mocking:
+            return getDiscoverFeedArticleError.mock()
+        }
+    }
+}
+
+extension Selection where TypeLock == Never, Type == Never {
+    typealias GetDiscoverFeedArticleResults<T> = Selection<T, Unions.GetDiscoverFeedArticleResults>
+}
+
+extension Unions {
+    struct GetDiscoverTopicResults {
+        let __typename: TypeName
+        let discoverTopics: [String: [Objects.DiscoverTopic]]
+        let errorCodes: [String: [Enums.GetDiscoverTopicErrorCode]]
+
+        enum TypeName: String, Codable {
+            case getDiscoverTopicError = "GetDiscoverTopicError"
+            case getDiscoverTopicSuccess = "GetDiscoverTopicSuccess"
+        }
+    }
+}
+
+extension Unions.GetDiscoverTopicResults: Decodable {
+    init(from decoder: Decoder) throws {
+        let container = try decoder.container(keyedBy: DynamicCodingKeys.self)
+
+        var map = HashMap()
+        for codingKey in container.allKeys {
+            if codingKey.isTypenameKey { continue }
+
+            let alias = codingKey.stringValue
+            let field = GraphQLField.getFieldNameFromAlias(alias)
+
+            switch field {
+            case "discoverTopics":
+                if let value = try container.decode([Objects.DiscoverTopic]?.self, forKey: codingKey) {
+                    map.set(key: field, hash: alias, value: value as Any)
+                }
+            case "errorCodes":
+                if let value = try container.decode([Enums.GetDiscoverTopicErrorCode]?.self, forKey: codingKey) {
+                    map.set(key: field, hash: alias, value: value as Any)
+                }
+            default:
+                throw DecodingError.dataCorrupted(
+                    DecodingError.Context(
+                        codingPath: decoder.codingPath,
+                        debugDescription: "Unknown key \(field)."
+                    )
+                )
+            }
+        }
+
+        __typename = try container.decode(TypeName.self, forKey: DynamicCodingKeys(stringValue: "__typename")!)
+
+        discoverTopics = map["discoverTopics"]
+        errorCodes = map["errorCodes"]
+    }
+}
+
+extension Fields where TypeLock == Unions.GetDiscoverTopicResults {
+    func on<Type>(getDiscoverTopicError: Selection<Type, Objects.GetDiscoverTopicError>, getDiscoverTopicSuccess: Selection<Type, Objects.GetDiscoverTopicSuccess>) throws -> Type {
+        select([GraphQLField.fragment(type: "GetDiscoverTopicError", selection: getDiscoverTopicError.selection), GraphQLField.fragment(type: "GetDiscoverTopicSuccess", selection: getDiscoverTopicSuccess.selection)])
+
+        switch response {
+        case let .decoding(data):
+            switch data.__typename {
+            case .getDiscoverTopicError:
+                let data = Objects.GetDiscoverTopicError(errorCodes: data.errorCodes)
+                return try getDiscoverTopicError.decode(data: data)
+            case .getDiscoverTopicSuccess:
+                let data = Objects.GetDiscoverTopicSuccess(discoverTopics: data.discoverTopics)
+                return try getDiscoverTopicSuccess.decode(data: data)
+            }
+        case .mocking:
+            return getDiscoverTopicError.mock()
+        }
+    }
+}
+
+extension Selection where TypeLock == Never, Type == Never {
+    typealias GetDiscoverTopicResults<T> = Selection<T, Unions.GetDiscoverTopicResults>
+}
+
 extension Unions {
     struct GetFollowersResult {
         let __typename: TypeName
@@ -29304,6 +32365,80 @@ extension Selection where TypeLock == Never, Type == Never {
     typealias ImportFromIntegrationResult<T> = Selection<T, Unions.ImportFromIntegrationResult>
 }
 
+extension Unions {
+    struct IntegrationResult {
+        let __typename: TypeName
+        let errorCodes: [String: [Enums.IntegrationErrorCode]]
+        let integration: [String: Objects.Integration]
+
+        enum TypeName: String, Codable {
+            case integrationError = "IntegrationError"
+            case integrationSuccess = "IntegrationSuccess"
+        }
+    }
+}
+
+extension Unions.IntegrationResult: Decodable {
+    init(from decoder: Decoder) throws {
+        let container = try decoder.container(keyedBy: DynamicCodingKeys.self)
+
+        var map = HashMap()
+        for codingKey in container.allKeys {
+            if codingKey.isTypenameKey { continue }
+
+            let alias = codingKey.stringValue
+            let field = GraphQLField.getFieldNameFromAlias(alias)
+
+            switch field {
+            case "errorCodes":
+                if let value = try container.decode([Enums.IntegrationErrorCode]?.self, forKey: codingKey) {
+                    map.set(key: field, hash: alias, value: value as Any)
+                }
+            case "integration":
+                if let value = try container.decode(Objects.Integration?.self, forKey: codingKey) {
+                    map.set(key: field, hash: alias, value: value as Any)
+                }
+            default:
+                throw DecodingError.dataCorrupted(
+                    DecodingError.Context(
+                        codingPath: decoder.codingPath,
+                        debugDescription: "Unknown key \(field)."
+                    )
+                )
+            }
+        }
+
+        __typename = try container.decode(TypeName.self, forKey: DynamicCodingKeys(stringValue: "__typename")!)
+
+        errorCodes = map["errorCodes"]
+        integration = map["integration"]
+    }
+}
+
+extension Fields where TypeLock == Unions.IntegrationResult {
+    func on<Type>(integrationError: Selection<Type, Objects.IntegrationError>, integrationSuccess: Selection<Type, Objects.IntegrationSuccess>) throws -> Type {
+        select([GraphQLField.fragment(type: "IntegrationError", selection: integrationError.selection), GraphQLField.fragment(type: "IntegrationSuccess", selection: integrationSuccess.selection)])
+
+        switch response {
+        case let .decoding(data):
+            switch data.__typename {
+            case .integrationError:
+                let data = Objects.IntegrationError(errorCodes: data.errorCodes)
+                return try integrationError.decode(data: data)
+            case .integrationSuccess:
+                let data = Objects.IntegrationSuccess(integration: data.integration)
+                return try integrationSuccess.decode(data: data)
+            }
+        case .mocking:
+            return integrationError.mock()
+        }
+    }
+}
+
+extension Selection where TypeLock == Never, Type == Never {
+    typealias IntegrationResult<T> = Selection<T, Unions.IntegrationResult>
+}
+
 extension Unions {
     struct IntegrationsResult {
         let __typename: TypeName
@@ -30864,6 +33999,86 @@ extension Selection where TypeLock == Never, Type == Never {
     typealias SaveArticleReadingProgressResult<T> = Selection<T, Unions.SaveArticleReadingProgressResult>
 }
 
+extension Unions {
+    struct SaveDiscoverArticleResult {
+        let __typename: TypeName
+        let errorCodes: [String: [Enums.SaveDiscoverArticleErrorCode]]
+        let saveId: [String: String]
+        let url: [String: String]
+
+        enum TypeName: String, Codable {
+            case saveDiscoverArticleError = "SaveDiscoverArticleError"
+            case saveDiscoverArticleSuccess = "SaveDiscoverArticleSuccess"
+        }
+    }
+}
+
+extension Unions.SaveDiscoverArticleResult: Decodable {
+    init(from decoder: Decoder) throws {
+        let container = try decoder.container(keyedBy: DynamicCodingKeys.self)
+
+        var map = HashMap()
+        for codingKey in container.allKeys {
+            if codingKey.isTypenameKey { continue }
+
+            let alias = codingKey.stringValue
+            let field = GraphQLField.getFieldNameFromAlias(alias)
+
+            switch field {
+            case "errorCodes":
+                if let value = try container.decode([Enums.SaveDiscoverArticleErrorCode]?.self, forKey: codingKey) {
+                    map.set(key: field, hash: alias, value: value as Any)
+                }
+            case "saveId":
+                if let value = try container.decode(String?.self, forKey: codingKey) {
+                    map.set(key: field, hash: alias, value: value as Any)
+                }
+            case "url":
+                if let value = try container.decode(String?.self, forKey: codingKey) {
+                    map.set(key: field, hash: alias, value: value as Any)
+                }
+            default:
+                throw DecodingError.dataCorrupted(
+                    DecodingError.Context(
+                        codingPath: decoder.codingPath,
+                        debugDescription: "Unknown key \(field)."
+                    )
+                )
+            }
+        }
+
+        __typename = try container.decode(TypeName.self, forKey: DynamicCodingKeys(stringValue: "__typename")!)
+
+        errorCodes = map["errorCodes"]
+        saveId = map["saveId"]
+        url = map["url"]
+    }
+}
+
+extension Fields where TypeLock == Unions.SaveDiscoverArticleResult {
+    func on<Type>(saveDiscoverArticleError: Selection<Type, Objects.SaveDiscoverArticleError>, saveDiscoverArticleSuccess: Selection<Type, Objects.SaveDiscoverArticleSuccess>) throws -> Type {
+        select([GraphQLField.fragment(type: "SaveDiscoverArticleError", selection: saveDiscoverArticleError.selection), GraphQLField.fragment(type: "SaveDiscoverArticleSuccess", selection: saveDiscoverArticleSuccess.selection)])
+
+        switch response {
+        case let .decoding(data):
+            switch data.__typename {
+            case .saveDiscoverArticleError:
+                let data = Objects.SaveDiscoverArticleError(errorCodes: data.errorCodes)
+                return try saveDiscoverArticleError.decode(data: data)
+            case .saveDiscoverArticleSuccess:
+                let data = Objects.SaveDiscoverArticleSuccess(saveId: data.saveId, url: data.url)
+                return try saveDiscoverArticleSuccess.decode(data: data)
+            }
+        case .mocking:
+            return saveDiscoverArticleError.mock()
+        }
+    }
+}
+
+extension Selection where TypeLock == Never, Type == Never {
+    typealias SaveDiscoverArticleResult<T> = Selection<T, Unions.SaveDiscoverArticleResult>
+}
+
 extension Unions {
     struct SaveFilterResult {
         let __typename: TypeName
@@ -33967,6 +37182,19 @@ extension Selection where TypeLock == Never, Type == Never {
 // MARK: - Enums
 
 enum Enums {}
+extension Enums {
+    /// AddDiscoverFeedErrorCode
+    enum AddDiscoverFeedErrorCode: String, CaseIterable, Codable {
+        case badRequest = "BAD_REQUEST"
+
+        case conflict = "CONFLICT"
+
+        case notFound = "NOT_FOUND"
+
+        case unauthorized = "UNAUTHORIZED"
+    }
+}
+
 extension Enums {
     /// AddPopularReadErrorCode
     enum AddPopularReadErrorCode: String, CaseIterable, Codable {
@@ -34201,6 +37429,30 @@ extension Enums {
     }
 }
 
+extension Enums {
+    /// DeleteDiscoverArticleErrorCode
+    enum DeleteDiscoverArticleErrorCode: String, CaseIterable, Codable {
+        case badRequest = "BAD_REQUEST"
+
+        case notFound = "NOT_FOUND"
+
+        case unauthorized = "UNAUTHORIZED"
+    }
+}
+
+extension Enums {
+    /// DeleteDiscoverFeedErrorCode
+    enum DeleteDiscoverFeedErrorCode: String, CaseIterable, Codable {
+        case badRequest = "BAD_REQUEST"
+
+        case conflict = "CONFLICT"
+
+        case notFound = "NOT_FOUND"
+
+        case unauthorized = "UNAUTHORIZED"
+    }
+}
+
 extension Enums {
     /// DeleteFilterErrorCode
     enum DeleteFilterErrorCode: String, CaseIterable, Codable {
@@ -34331,6 +37583,26 @@ extension Enums {
     }
 }
 
+extension Enums {
+    /// DiscoverFeedErrorCode
+    enum DiscoverFeedErrorCode: String, CaseIterable, Codable {
+        case badRequest = "BAD_REQUEST"
+
+        case unauthorized = "UNAUTHORIZED"
+    }
+}
+
+extension Enums {
+    /// EditDiscoverFeedErrorCode
+    enum EditDiscoverFeedErrorCode: String, CaseIterable, Codable {
+        case badRequest = "BAD_REQUEST"
+
+        case notFound = "NOT_FOUND"
+
+        case unauthorized = "UNAUTHORIZED"
+    }
+}
+
 extension Enums {
     /// EmptyTrashErrorCode
     enum EmptyTrashErrorCode: String, CaseIterable, Codable {
@@ -34338,6 +37610,15 @@ extension Enums {
     }
 }
 
+extension Enums {
+    /// ExportToIntegrationErrorCode
+    enum ExportToIntegrationErrorCode: String, CaseIterable, Codable {
+        case failedToCreateTask = "FAILED_TO_CREATE_TASK"
+
+        case unauthorized = "UNAUTHORIZED"
+    }
+}
+
 extension Enums {
     /// FeedArticlesErrorCode
     enum FeedArticlesErrorCode: String, CaseIterable, Codable {
@@ -34394,6 +37675,24 @@ extension Enums {
     }
 }
 
+extension Enums {
+    /// GetDiscoverFeedArticleErrorCode
+    enum GetDiscoverFeedArticleErrorCode: String, CaseIterable, Codable {
+        case badRequest = "BAD_REQUEST"
+
+        case notFound = "NOT_FOUND"
+
+        case unauthorized = "UNAUTHORIZED"
+    }
+}
+
+extension Enums {
+    /// GetDiscoverTopicErrorCode
+    enum GetDiscoverTopicErrorCode: String, CaseIterable, Codable {
+        case unauthorized = "UNAUTHORIZED"
+    }
+}
+
 extension Enums {
     /// GetFollowersErrorCode
     enum GetFollowersErrorCode: String, CaseIterable, Codable {
@@ -34457,6 +37756,13 @@ extension Enums {
     }
 }
 
+extension Enums {
+    /// IntegrationErrorCode
+    enum IntegrationErrorCode: String, CaseIterable, Codable {
+        case notFound = "NOT_FOUND"
+    }
+}
+
 extension Enums {
     /// IntegrationType
     enum IntegrationType: String, CaseIterable, Codable {
@@ -34744,15 +38050,25 @@ extension Enums {
 
         case delete = "DELETE"
 
+        case export = "EXPORT"
+
         case markAsRead = "MARK_AS_READ"
 
         case sendNotification = "SEND_NOTIFICATION"
+
+        case webhook = "WEBHOOK"
     }
 }
 
 extension Enums {
     /// RuleEventType
     enum RuleEventType: String, CaseIterable, Codable {
+        case highlightCreated = "HIGHLIGHT_CREATED"
+
+        case highlightUpdated = "HIGHLIGHT_UPDATED"
+
+        case labelCreated = "LABEL_CREATED"
+
         case pageCreated = "PAGE_CREATED"
 
         case pageUpdated = "PAGE_UPDATED"
@@ -34779,6 +38095,17 @@ extension Enums {
     }
 }
 
+extension Enums {
+    /// SaveDiscoverArticleErrorCode
+    enum SaveDiscoverArticleErrorCode: String, CaseIterable, Codable {
+        case badRequest = "BAD_REQUEST"
+
+        case notFound = "NOT_FOUND"
+
+        case unauthorized = "UNAUTHORIZED"
+    }
+}
+
 extension Enums {
     /// SaveErrorCode
     enum SaveErrorCode: String, CaseIterable, Codable {
@@ -35045,6 +38372,21 @@ extension Enums {
     }
 }
 
+extension Enums {
+    /// TaskState
+    enum TaskState: String, CaseIterable, Codable {
+        case cancelled = "CANCELLED"
+
+        case failed = "FAILED"
+
+        case pending = "PENDING"
+
+        case running = "RUNNING"
+
+        case succeeded = "SUCCEEDED"
+    }
+}
+
 extension Enums {
     /// TypeaheadSearchErrorCode
     enum TypeaheadSearchErrorCode: String, CaseIterable, Codable {
@@ -35342,6 +38684,21 @@ extension Enums {
 // MARK: - Input Objects
 
 enum InputObjects {}
+extension InputObjects {
+    struct AddDiscoverFeedInput: Encodable, Hashable {
+        var url: String
+
+        func encode(to encoder: Encoder) throws {
+            var container = encoder.container(keyedBy: CodingKeys.self)
+            try container.encode(url, forKey: .url)
+        }
+
+        enum CodingKeys: String, CodingKey {
+            case url
+        }
+    }
+}
+
 extension InputObjects {
     struct ArchiveLinkInput: Encodable, Hashable {
         var archived: Bool
@@ -35679,6 +39036,55 @@ extension InputObjects {
     }
 }
 
+extension InputObjects {
+    struct DeleteDiscoverArticleInput: Encodable, Hashable {
+        var discoverArticleId: String
+
+        func encode(to encoder: Encoder) throws {
+            var container = encoder.container(keyedBy: CodingKeys.self)
+            try container.encode(discoverArticleId, forKey: .discoverArticleId)
+        }
+
+        enum CodingKeys: String, CodingKey {
+            case discoverArticleId
+        }
+    }
+}
+
+extension InputObjects {
+    struct DeleteDiscoverFeedInput: Encodable, Hashable {
+        var feedId: String
+
+        func encode(to encoder: Encoder) throws {
+            var container = encoder.container(keyedBy: CodingKeys.self)
+            try container.encode(feedId, forKey: .feedId)
+        }
+
+        enum CodingKeys: String, CodingKey {
+            case feedId
+        }
+    }
+}
+
+extension InputObjects {
+    struct EditDiscoverFeedInput: Encodable, Hashable {
+        var feedId: String
+
+        var name: String
+
+        func encode(to encoder: Encoder) throws {
+            var container = encoder.container(keyedBy: CodingKeys.self)
+            try container.encode(feedId, forKey: .feedId)
+            try container.encode(name, forKey: .name)
+        }
+
+        enum CodingKeys: String, CodingKey {
+            case feedId
+            case name
+        }
+    }
+}
+
 extension InputObjects {
     struct FeedsInput: Encodable, Hashable {
         var after: OptionalArgument<String> = .absent()
@@ -36159,6 +39565,29 @@ extension InputObjects {
     }
 }
 
+extension InputObjects {
+    struct SaveDiscoverArticleInput: Encodable, Hashable {
+        var discoverArticleId: String
+
+        var locale: OptionalArgument<String> = .absent()
+
+        var timezone: OptionalArgument<String> = .absent()
+
+        func encode(to encoder: Encoder) throws {
+            var container = encoder.container(keyedBy: CodingKeys.self)
+            try container.encode(discoverArticleId, forKey: .discoverArticleId)
+            if locale.hasValue { try container.encode(locale, forKey: .locale) }
+            if timezone.hasValue { try container.encode(timezone, forKey: .timezone) }
+        }
+
+        enum CodingKeys: String, CodingKey {
+            case discoverArticleId
+            case locale
+            case timezone
+        }
+    }
+}
+
 extension InputObjects {
     struct SaveFileInput: Encodable, Hashable {
         var clientRequestId: String
diff --git a/apple/OmnivoreKit/Sources/Services/DataService/Mutations/SetRuleMutation.swift b/apple/OmnivoreKit/Sources/Services/DataService/Mutations/SetRuleMutation.swift
index 9dea39b9a..161df3925 100644
--- a/apple/OmnivoreKit/Sources/Services/DataService/Mutations/SetRuleMutation.swift
+++ b/apple/OmnivoreKit/Sources/Services/DataService/Mutations/SetRuleMutation.swift
@@ -15,6 +15,8 @@ public enum RuleActionType {
   case delete
   case markAsRead
   case sendNotification
+  case export
+  case webhook
 
   static func from(_ other: Enums.RuleActionType) -> RuleActionType {
     switch other {
@@ -28,6 +30,10 @@ public enum RuleActionType {
       return .sendNotification
     case .delete:
       return .delete
+    case Enums.RuleActionType.export:
+      return .export
+    case Enums.RuleActionType.webhook:
+      return .webhook
     }
   }
 }
diff --git a/apple/OmnivoreKit/Sources/Services/DataService/Queries/ViewerFetcher.swift b/apple/OmnivoreKit/Sources/Services/DataService/Queries/ViewerFetcher.swift
index 78117f8a0..f1251fdc0 100644
--- a/apple/OmnivoreKit/Sources/Services/DataService/Queries/ViewerFetcher.swift
+++ b/apple/OmnivoreKit/Sources/Services/DataService/Queries/ViewerFetcher.swift
@@ -17,7 +17,9 @@ public extension DataService {
         profileImageURL: try $0.profile(
           selection: .init { try $0.pictureUrl() }
         ),
-        intercomHash: try $0.intercomHash()
+        intercomHash: try $0.intercomHash(),
+        digestEnabled: true // (try $0.featureList(selection: featureSelection.list.nullable)?
+              // .filter { $0.enabled && $0.name == "digest" } ?? []).count > 0
       )
     }
 
@@ -65,6 +67,7 @@ public struct ViewerInternal {
   public let name: String
   public let profileImageURL: String?
   public let intercomHash: String?
+  public let digestEnabled: Bool?
 
   func persist(context: NSManagedObjectContext) throws {
     try context.performAndWait {
@@ -73,6 +76,7 @@ public struct ViewerInternal {
       viewer.username = username
       viewer.name = name
       viewer.profileImageURL = profileImageURL
+      viewer.digestEnabled = digestEnabled ?? false
 
       do {
         try context.save()
diff --git a/apple/OmnivoreKit/Sources/Services/DataService/Selections/FeatureSelection.swift b/apple/OmnivoreKit/Sources/Services/DataService/Selections/FeatureSelection.swift
new file mode 100644
index 000000000..b72a783a1
--- /dev/null
+++ b/apple/OmnivoreKit/Sources/Services/DataService/Selections/FeatureSelection.swift
@@ -0,0 +1,6 @@
+import Models
+import SwiftGraphQL
+
+let featureSelection = Selection.Feature {
+  FeatureInternal(name: try $0.name(), enabled: try $0.grantedAt() != nil)
+}

From d2ea93a7dd1b0b4a3e54af821ec1ae41a9a47cf0 Mon Sep 17 00:00:00 2001
From: Jackson Harper <jacksonh@gmail.com>
Date: Tue, 16 Apr 2024 13:31:55 +0800
Subject: [PATCH 55/74] Stub out some changes to the digest response

---
 packages/api/src/services/digest.ts | 37 +++++++++++++++++++++++++++++
 1 file changed, 37 insertions(+)

diff --git a/packages/api/src/services/digest.ts b/packages/api/src/services/digest.ts
index de2ca2909..21b34a01c 100644
--- a/packages/api/src/services/digest.ts
+++ b/packages/api/src/services/digest.ts
@@ -1,4 +1,5 @@
 import { redisDataSource } from '../redis_data_source'
+import { SpeechFile } from '@omnivore/text-to-speech-handler'
 
 export const CREATE_DIGEST_JOB = 'create-digest'
 
@@ -17,6 +18,8 @@ export interface Digest {
   chapters?: Chapter[]
   urlsToAudio?: string[]
   jobState: string
+
+  speechFile: SpeechFile
 }
 
 interface Chapter {
@@ -26,6 +29,40 @@ interface Chapter {
 const digestKey = (userId: string) => `digest:${userId}`
 
 export const getDigest = async (userId: string): Promise<Digest | null> => {
+  await redisDataSource.redisClient?.set(
+    digestKey(userId),
+    JSON.stringify({
+      id: 'BB3D5D89-70A2-4AE1-ADDC-713232B1281D',
+      title:
+        'SOTU response collapses, Trump hits new low, Biden fundraising explodes 3/11/24 TDPS Podcast',
+      content: 'content',
+      urlsToAudio: [],
+      jobState: 'completed',
+      speechFile: {
+        pageId: '1234',
+        wordCount: 2124,
+        language: 'en-US',
+        defaultVoice: 'en-US-ChristopherNeural',
+        utterances: [
+          {
+            idx: '',
+            text: 'TOP prospect JOINS Canucks - Team SPEAKS OUT on Demko Injury | Canucks News',
+            wordOffset: 0,
+            wordCount: 14,
+            voice: 'en-US-ChristopherNeural',
+          },
+          {
+            idx: '4',
+            text: 'Intro',
+            wordOffset: 14,
+            wordCount: 1,
+            voice: 'en-US-ChristopherNeural',
+          },
+        ],
+      },
+    })
+  )
+
   const digest = await redisDataSource.redisClient?.get(digestKey(userId))
   return digest ? (JSON.parse(digest) as Digest) : null
 }

From 47150c29913ecece27a755509815eecc8642bd17 Mon Sep 17 00:00:00 2001
From: Jackson Harper <jacksonh@gmail.com>
Date: Tue, 16 Apr 2024 13:34:12 +0800
Subject: [PATCH 56/74] Refactoring the UX for digest view

---
 .../App/Views/AI/FullScreenDigestView.swift   | 303 ++++++++++++++++++
 .../App/Views/AI/LibraryDigestView.swift      | 267 ---------------
 .../AudioPlayer/ExpandedAudioPlayer.swift     |  22 +-
 .../App/Views/AudioPlayer/MiniPlayer.swift    |   4 +-
 .../Views/AudioPlayer/MiniPlayerViewer.swift  |  12 +-
 .../App/Views/Home/HomeFeedViewIOS.swift      |  73 +++--
 .../Sources/App/Views/LibraryTabView.swift    |  32 +-
 .../App/Views/PrimaryContentView.swift        |   2 +-
 .../Views/WebReader/WebReaderContainer.swift  |  20 +-
 .../Sources/Models/DataModels/FeedItem.swift  |  39 ++-
 .../AudioSession/AudioController.swift        |  78 ++++-
 .../AudioSession/SpeechSynthesizer.swift      |   4 +-
 .../Services/DataService/AI/AITasks.swift     | 118 +++++++
 13 files changed, 608 insertions(+), 366 deletions(-)
 create mode 100644 apple/OmnivoreKit/Sources/App/Views/AI/FullScreenDigestView.swift
 delete mode 100644 apple/OmnivoreKit/Sources/App/Views/AI/LibraryDigestView.swift
 create mode 100644 apple/OmnivoreKit/Sources/Services/DataService/AI/AITasks.swift

diff --git a/apple/OmnivoreKit/Sources/App/Views/AI/FullScreenDigestView.swift b/apple/OmnivoreKit/Sources/App/Views/AI/FullScreenDigestView.swift
new file mode 100644
index 000000000..d2472344f
--- /dev/null
+++ b/apple/OmnivoreKit/Sources/App/Views/AI/FullScreenDigestView.swift
@@ -0,0 +1,303 @@
+import SwiftUI
+import Models
+import Services
+
+public class FullScreenDigestViewModel: ObservableObject {
+  @Published var isLoading = false
+  @Published var digest: DigestResult?
+
+  func load(dataService: DataService) async {
+    isLoading = true
+    if digest == nil {
+      do {
+        digest = try await dataService.getLatestDigest(timeoutInterval: 10)
+      } catch {
+        print("ERROR WITH DIGEST: ", error)
+      }
+    }
+    isLoading = false
+  }
+}
+
+struct DigestAudioItem: AudioItemProperties {
+  let audioItemType = Models.AudioItemType.digest
+  
+  var itemID = ""
+  
+  var title = "TITLE"
+  
+  var byline: String? = "byline"
+  
+  var imageURL: URL? = nil
+  
+  var language: String?
+  
+  var startIndex: Int = 0
+  var startOffset: Double = 0.0
+}
+
+@available(iOS 17.0, *)
+@MainActor
+struct FullScreenDigestView: View {
+  let viewModel: DigestViewModel = DigestViewModel()
+  let dataService: DataService
+  let audioController: AudioController
+
+  @Environment(\.dismiss) private var dismiss
+
+  let textBody = "In a significant political turn, the SOTU response faces unexpected collapse, " +
+  "marking a stark contrast to Trump's latest downturn, alongside an unprecedented " +
+  "surge in Biden's fundraising efforts as of 3/11/24, according to the TDPS Podcast. " +
+  "The analysis provides insights into the shifting dynamics of political support and " +
+  "the potential implications for future electoral strategies. Based on the information " +
+  "you provided, the video seems to discuss a recent event where former President " +
+  "Donald Trump made a controversial statement that shocked even his own audience. " +
+  "The video likely covers Trump's response to the State of the Union (SOTU) address " +
+  "and how it received negative feedback, possibly leading to a decline in his support " +
+  "or approval ratings. Additionally, it appears that the video touches upon a surge " +
+  "in fundraising for President Joe Biden's administration around March 11, 2024."
+
+  public init(dataService: DataService, audioController: AudioController) {
+    self.dataService = dataService
+    self.audioController = audioController
+  }
+
+  var body: some View {
+    // ZStack(alignment: Alignment(horizontal: .trailing, vertical: .top)) {
+    Group {
+      if viewModel.isLoading {
+        ProgressView()
+      } else {
+        itemBody
+          .task {
+            await viewModel.load(dataService: dataService)
+          }.onAppear {
+            self.audioController.play(itemAudioProperties: DigestAudioItem())
+          }
+      }
+    }            .navigationTitle("Omnivore digest")
+      .navigationBarTitleDisplayMode(.inline)
+
+//      HStack(alignment: .top) {
+//        Spacer()
+//        closeButton
+//      }
+//      .padding(20)
+    // }
+  }
+
+  var closeButton: some View {
+    Button(action: {
+      dismiss()
+    }, label: {
+      ZStack {
+        Circle()
+          .foregroundColor(Color.appGrayText)
+          .frame(width: 36, height: 36)
+          .opacity(0.1)
+
+        Image(systemName: "xmark")
+          .font(.appCallout)
+          .frame(width: 36, height: 36)
+      }
+    })
+    .buttonStyle(.plain)
+  }
+
+  @available(iOS 17.0, *)
+  var itemBody: some View {
+    VStack {
+      ScrollView(.vertical) {
+        VStack(spacing: 20) {
+          Text("SOTU response collapses, Trump hits new low, Biden fundraising explodes 3/11/24 TDPS Podcast")
+            .font(.title)
+          Text(textBody)
+            .font(.body)
+        }
+      }
+      // .scrollTargetBehavior(.paging)
+      // .ignoresSafeArea()
+      MiniPlayerViewer()
+        .padding(.top, 10)
+        .padding(.bottom, 40)
+        .background(Color.themeTabBarColor)
+        .onTapGesture {
+          // showExpandedAudioPlayer = true
+        }
+    }
+  }
+}
+
+@MainActor
+public class PreviewItemViewModel: ObservableObject {
+  let dataService: DataService
+  @Published var item: DigestItem
+  let showSwipeHint: Bool
+
+  @Published var isLoading = false
+  @Published var resultText: String?
+  @Published var promptDisplayText: String?
+
+  init(dataService: DataService, item: DigestItem, showSwipeHint: Bool) {
+    self.dataService = dataService
+    self.item = item
+    self.showSwipeHint = showSwipeHint
+  }
+
+  func loadResult() async {
+//    isLoading = true
+//    let taskId = try? await dataService.createAITask(
+//      extraText: extraText,
+//      libraryItemId: item?.id ?? "",
+//      promptName: "summarize-001"
+//    )
+//
+//    if let taskId = taskId {
+//      do {
+//        let fetchedText = try await dataService.pollAITask(jobId: taskId, timeoutInterval: 30)
+//        resultText = fetchedText
+//      } catch {
+//        print("ERROR WITH RESULT TEXT: ", error)
+//      }
+//    } else {
+//      print("NO TASK ID: ", taskId)
+//    }
+//    isLoading = false
+  }
+}
+
+@MainActor
+struct PreviewItemView: View {
+  @StateObject var viewModel: PreviewItemViewModel
+
+    var body: some View {
+      VStack(spacing: 10) {
+        HStack {
+          AsyncImage(url: viewModel.item.siteIcon) { phase in
+            if let image = phase.image {
+              image
+                .resizable()
+                .aspectRatio(contentMode: .fill)
+                .frame(width: 20, height: 20, alignment: .center)
+            } else {
+              Color.appButtonBackground
+                .frame(width: 20, height: 20, alignment: .center)
+            }
+          }
+          Text(viewModel.item.site)
+            .font(Font.system(size: 14))
+            .frame(maxWidth: .infinity, alignment: .topLeading)
+        }
+        .padding(.top, 10)
+        Text(viewModel.item.title)
+          // .font(.body)
+          // .fontWeight(.semibold)
+          .font(Font.system(size: 18, weight: .semibold))
+          .frame(maxWidth: .infinity, alignment: .topLeading)
+
+        Text(viewModel.item.author)
+          .font(Font.system(size: 14))
+          .foregroundColor(Color(hex: "898989"))
+          .frame(maxWidth: .infinity, alignment: .topLeading)
+
+        Color(hex: "2A2A2A")
+          .frame(height: 1)
+          .frame(maxWidth: .infinity, alignment: .center)
+          .padding(.vertical, 20)
+
+        if viewModel.isLoading {
+          ProgressView()
+            .task {
+              await viewModel.loadResult()
+            }
+            .frame(maxWidth: .infinity, maxHeight: .infinity)
+        } else {
+          Text(viewModel.item.summaryText)
+            .font(Font.system(size: 16))
+            // .font(.body)
+            .lineSpacing(12.0)
+            .frame(maxWidth: .infinity, alignment: .topLeading)
+          HStack {
+            Button(action: {}, label: {
+              HStack(alignment: .center) {
+                Text("Start listening")
+                  .font(Font.system(size: 14))
+                  .frame(height: 42, alignment: .center)
+                Image(systemName: "play.fill")
+                  .resizable()
+                  .frame(width: 10, height: 10)
+              }
+              .padding(.horizontal, 15)
+              .background(Color.blue)
+              .foregroundColor(.white)
+              .cornerRadius(18)
+            })
+            Spacer()
+          }
+          .padding(.top, 20)
+        }
+        Spacer()
+        if viewModel.showSwipeHint {
+          VStack {
+            Image.doubleChevronUp
+            Text("Swipe up for next article")
+              .foregroundColor(Color(hex: "898989"))
+          }
+          .padding(.bottom, 50)
+        }
+      }.frame(maxWidth: .infinity, maxHeight: .infinity)
+        .padding(.top, 100)
+        .padding(.horizontal, 15)
+
+    }
+}
+
+struct RatingView: View {
+    @State private var rating: Int = 0
+
+    var body: some View {
+      VStack(spacing: 30) {
+        Text("Rate today's digest")
+          .font(.title)
+          .padding(.vertical, 40)
+        Text("I liked the stories picked for today's digest")
+        RatingWidget()
+
+        Text("The stories were interesting")
+        RatingWidget()
+
+        Text("The voices sounded good")
+        RatingWidget()
+
+        Text("I liked the music")
+        RatingWidget()
+        Spacer()
+      }.padding(.top, 60)
+    }
+}
+
+struct StarView: View {
+    var isFilled: Bool
+    var body: some View {
+        Image(systemName: isFilled ? "star.fill" : "star")
+            .foregroundColor(isFilled ? Color.yellow : Color.gray)
+    }
+}
+
+struct RatingWidget: View {
+    @State private var rating: Int = 0
+    var body: some View {
+        HStack {
+            ForEach(1...5, id: \.self) { index in
+                StarView(isFilled: index <= rating)
+                    .onTapGesture {
+                        rating = index
+                    }
+            }
+        }
+        .padding()
+        .background(Color(hex: "313131"))
+        .cornerRadius(8)
+        // .shadow(radius: 3)
+    }
+}
diff --git a/apple/OmnivoreKit/Sources/App/Views/AI/LibraryDigestView.swift b/apple/OmnivoreKit/Sources/App/Views/AI/LibraryDigestView.swift
deleted file mode 100644
index e6f4f91d7..000000000
--- a/apple/OmnivoreKit/Sources/App/Views/AI/LibraryDigestView.swift
+++ /dev/null
@@ -1,267 +0,0 @@
-import SwiftUI
-import Models
-import Services
-
-
-struct DigestItem {
-  let id: String
-  let site: String
-  let siteIcon: URL?
-  let author: String
-  let title: String
-  let summaryText: String
-  let keyPointsText: String
-  let highlightsText: String
-}
-
-@available(iOS 17.0, *)
-@MainActor
-struct LibraryDigestView: View {
-  let dataService: DataService
-    // @State private var currentIndex = 0
-  @State private var items: [DigestItem]
-    // @State private var preloadedItems: [Int: String] = [:]
-  @Environment(\.dismiss) private var dismiss
-
-  //  let itemCount = 10 // Number of items to initially load
-  //  let prefetchCount = 2 // Number of items to prefetch
-
-  public init(dataService: DataService) {
-    self.dataService = dataService
-    self.items = [
-      DigestItem(
-        id: "1468AFAA-88sdfsdfC-4546-BE02-EACF385288FC",
-        site: "CNBC.com",
-        siteIcon: URL(string: "https://www.cnbc.com/favicon.ico"),
-        author: "Kif Leswing",
-        title: "Apple shares just had their best day since last May",
-        summaryText: "In a significant political turn, the SOTU response faces unexpected collapse, marking a stark contrast to Trump's latest" +
-        " downturn, alongside an unprecedented surge in Biden's fundraising efforts as of 3/11/24, according to the TDPS Podcast. " +
-        "The analysis provides insights into the shifting dynamics of political support and the potential implications for future " +
-        "electoral strategies. ",
-        keyPointsText: "Key points from the article:",
-        highlightsText: "Highlights from the article:"
-      ),
-      DigestItem(
-        id: "1468AFAA-8sdfsdffsdf-4546-BE02-EACF385288FC",
-        site: "CNBC.com",
-        siteIcon: URL(string: "https://www.cnbc.com/favicon.ico"),
-        author: "Kif Leswing",
-        title: "Apple shares just had their best day since last May",
-        summaryText: "In a significant political turn, the SOTU response faces unexpected collapse, marking a stark contrast to Trump's latest" +
-        " downturn, alongside an unprecedented surge in Biden's fundraising efforts as of 3/11/24, according to the TDPS Podcast. " +
-        "The analysis provides insights into the shifting dynamics of political support and the potential implications for future " +
-        "electoral strategies. ",
-        keyPointsText: "Key points from the article:",
-        highlightsText: "Highlights from the article:"
-      ),
-      DigestItem(
-        id: "1468AFAA-882C-asdadfsa85288FC",
-        site: "CNBC.com",
-        siteIcon: URL(string: "https://www.cnbc.com/favicon.ico"),
-        author: "Kif Leswing",
-        title: "Apple shares just had their best day since last May",
-        summaryText: "In a significant political turn, the SOTU response faces unexpected collapse, marking a stark contrast to Trump's latest" +
-        " downturn, alongside an unprecedented surge in Biden's fundraising efforts as of 3/11/24, according to the TDPS Podcast. " +
-        "The analysis provides insights into the shifting dynamics of political support and the potential implications for future " +
-        "electoral strategies. ",
-        keyPointsText: "Key points from the article:",
-        highlightsText: "Highlights from the article:"
-      )
-    ]
-    // currentIndex = 0
-   // _preloadedItems = [Int:String]
-  }
-
-  var body: some View {
-    itemBody
-  }
-
-  @available(iOS 17.0, *)
-  var itemBody: some View {
-      ScrollView(.vertical) {
-        LazyVStack(spacing: 0) {
-          ForEach(Array(self.items.enumerated()), id: \.1.id) { idx, item in
-            PreviewItemView(
-              viewModel: PreviewItemViewModel(dataService: dataService, item: item, showSwipeHint: idx == 0)
-            )
-            .containerRelativeFrame([.horizontal, .vertical])
-          }
-        }
-        .scrollTargetLayout()
-      }
-      .scrollTargetBehavior(.paging)
-      .ignoresSafeArea()
-
-      
-      //        ScrollView(.horizontal, showsIndicators: false) {
-      //            HStack(spacing: 0) {
-      //                ForEach(items.indices, id: \.self) { index in
-      //                    if let item = preloadedItems[index] {
-      //                        ItemView(content: item)
-      //                            .onAppear {
-      //                                if index == items.count - prefetchCount {
-      //                                    fetchItems()
-      //                                }
-      //                            }
-      //                    }
-      //                }
-      //            }
-      //        }
-      //        .frame(maxWidth: .infinity, maxHeight: .infinity)
-      .onAppear {
-        // fetchItems()
-      }
-      .onReceive(NotificationCenter.default.publisher(for: UIApplication.willResignActiveNotification)) { _ in
-        // Pause any background tasks if necessary
-      }
-    
-    }
-    
-//    private func fetchItems() {
-//        // Simulate fetching items from an API
-//        for idx in currentIndex..<currentIndex + itemCount {
-//            fetchItem(index: idx)
-//        }
-//        currentIndex += itemCount
-//    }
-//    
-//    private func fetchItem(index: Int) {
-//        // Simulate fetching item content from an API
-//        DispatchQueue.main.asyncAfter(deadline: .now() + 1) {
-//            items.append("Item \(index + 1)")
-//            preloadNextItemIfNeeded(index: index)
-//        }
-//    }
-//    
-//    private func preloadNextItemIfNeeded(index: Int) {
-//        let nextIndex = index + 1
-//        if nextIndex < currentIndex + prefetchCount {
-//            fetchItem(index: nextIndex)
-//        }
-//    }
-}
-
-@MainActor
-public class PreviewItemViewModel: ObservableObject {
-  let dataService: DataService
-  @Published var item: DigestItem
-  let showSwipeHint: Bool
-
-  @Published var isLoading = false
-  @Published var resultText: String?
-  @Published var promptDisplayText: String?
-
-  init(dataService: DataService, item: DigestItem, showSwipeHint: Bool) {
-    self.dataService = dataService
-    self.item = item
-    self.showSwipeHint = showSwipeHint
-  }
-
-  func loadResult() async {
-//    isLoading = true
-//    let taskId = try? await dataService.createAITask(
-//      extraText: extraText,
-//      libraryItemId: item?.id ?? "",
-//      promptName: "summarize-001"
-//    )
-//
-//    if let taskId = taskId {
-//      do {
-//        let fetchedText = try await dataService.pollAITask(jobId: taskId, timeoutInterval: 30)
-//        resultText = fetchedText
-//      } catch {
-//        print("ERROR WITH RESULT TEXT: ", error)
-//      }
-//    } else {
-//      print("NO TASK ID: ", taskId)
-//    }
-//    isLoading = false
-  }
-}
-
-@MainActor
-struct PreviewItemView: View {
-  @StateObject var viewModel: PreviewItemViewModel
-
-    var body: some View {
-      VStack(spacing: 10) {
-        HStack {
-          AsyncImage(url: viewModel.item.siteIcon) { phase in
-            if let image = phase.image {
-              image
-                .resizable()
-                .aspectRatio(contentMode: .fill)
-                .frame(width: 20, height: 20, alignment: .center)
-            } else {
-              Color.appButtonBackground
-                .frame(width: 20, height: 20, alignment: .center)
-            }
-          }
-          Text(viewModel.item.site)
-            .font(Font.system(size: 14))
-            .frame(maxWidth: .infinity, alignment: .topLeading)
-        }
-        .padding(.top, 10)
-        Text(viewModel.item.title)
-          // .font(.body)
-          // .fontWeight(.semibold)
-          .font(Font.system(size: 18, weight: .semibold))
-          .frame(maxWidth: .infinity, alignment: .topLeading)
-
-        Text(viewModel.item.author)
-          .font(Font.system(size: 14))
-          .foregroundColor(Color(hex: "898989"))
-          .frame(maxWidth: .infinity, alignment: .topLeading)
-
-        Color(hex: "2A2A2A")
-          .frame(height: 1)
-          .frame(maxWidth: .infinity, alignment: .center)
-          .padding(.vertical, 20)
-
-        if viewModel.isLoading {
-          ProgressView()
-            .task {
-              await viewModel.loadResult()
-            }
-            .frame(maxWidth: .infinity, maxHeight: .infinity)
-        } else {
-          Text(viewModel.item.summaryText)
-            .font(Font.system(size: 16))
-            // .font(.body)
-            .lineSpacing(12.0)
-            .frame(maxWidth: .infinity, alignment: .topLeading)
-          HStack {
-            Button(action: {}, label: {
-              HStack(alignment: .center) {
-                Text("Start listening")
-                  .font(Font.system(size: 14))
-                  .frame(height: 42, alignment: .center)
-                Image(systemName: "play.fill")
-                  .resizable()
-                  .frame(width: 10, height: 10)
-              }
-              .padding(.horizontal, 15)
-              .background(Color.blue)
-              .foregroundColor(.white)
-              .cornerRadius(18)
-            })
-            Spacer()
-          }
-          .padding(.top, 20)
-        }
-        Spacer()
-        if viewModel.showSwipeHint {
-          VStack {
-            Image.doubleChevronUp
-            Text("Swipe up for next article")
-              .foregroundColor(Color(hex: "898989"))
-          }
-          .padding(.bottom, 50)
-        }
-      }.frame(maxWidth: .infinity, maxHeight: .infinity)
-        .padding(.top, 100)
-        .padding(.horizontal, 15)
-
-    }
-}
diff --git a/apple/OmnivoreKit/Sources/App/Views/AudioPlayer/ExpandedAudioPlayer.swift b/apple/OmnivoreKit/Sources/App/Views/AudioPlayer/ExpandedAudioPlayer.swift
index bf23ae8af..2a0a32049 100644
--- a/apple/OmnivoreKit/Sources/App/Views/AudioPlayer/ExpandedAudioPlayer.swift
+++ b/apple/OmnivoreKit/Sources/App/Views/AudioPlayer/ExpandedAudioPlayer.swift
@@ -45,7 +45,7 @@
       if audioController.playbackError {
         return AnyView(Color.clear)
       }
-      if let itemID = audioController.itemAudioProperties?.itemID, audioController.isLoadingItem(itemID: itemID) {
+      if audioController.isLoadingItem(audioController.itemAudioProperties) {
         return AnyView(ProgressView())
       } else {
         return AnyView(Button(
@@ -98,11 +98,11 @@
           }
         ).padding(.trailing, 5)
 
-        if !(audioController.itemAudioProperties?.isArchived ?? false) {
+        if !((audioController.itemAudioProperties as? LinkedItemAudioProperties)?.isArchived ?? false) {
           Button(
             action: { performArchive() },
             label: {
-              if audioController.itemAudioProperties?.isArchived ?? false {
+              if (audioController.itemAudioProperties as? LinkedItemAudioProperties)?.isArchived ?? false {
                 Image
                   .toolbarUnarchive
                   .foregroundColor(Color.toolbarItemForeground)
@@ -130,20 +130,20 @@
       }
     }
 
-    func performViewArticle() {
-      if let objectID = audioController.itemAudioProperties?.objectID {
-        viewArticle(objectID)
-      }
-    }
+//    func performViewArticle() {
+//      if let objectID = audioController.itemAudioProperties?.objectID {
+//        viewArticle(objectID)
+//      }
+//    }
 
     func performDelete() {
-      if let objectID = audioController.itemAudioProperties?.objectID {
+      if let objectID = (audioController.itemAudioProperties as? LinkedItemAudioProperties)?.objectID {
         delete(objectID)
       }
     }
 
     func performArchive() {
-      if let objectID = audioController.itemAudioProperties?.objectID {
+      if let objectID = (audioController.itemAudioProperties as? LinkedItemAudioProperties)?.objectID {
         archive(objectID)
       }
     }
@@ -467,7 +467,7 @@
     }
 
     public var innerBody: some View {
-      if let itemAudioProperties = self.audioController.itemAudioProperties {
+      if let itemAudioProperties = self.audioController.itemAudioProperties as? LinkedItemAudioProperties {
         return AnyView(
           playerContent(itemAudioProperties)
             .tint(.appGrayTextContrast)
diff --git a/apple/OmnivoreKit/Sources/App/Views/AudioPlayer/MiniPlayer.swift b/apple/OmnivoreKit/Sources/App/Views/AudioPlayer/MiniPlayer.swift
index e86a00351..8f194d5ae 100644
--- a/apple/OmnivoreKit/Sources/App/Views/AudioPlayer/MiniPlayer.swift
+++ b/apple/OmnivoreKit/Sources/App/Views/AudioPlayer/MiniPlayer.swift
@@ -23,14 +23,14 @@
     public var body: some View {
       ZStack(alignment: .center) {
         presentingView
-        if let itemAudioProperties = self.audioController.itemAudioProperties {
+        if self.audioController.itemAudioProperties != nil {
           ZStack(alignment: .bottom) {
             Color.systemBackground.edgesIgnoringSafeArea(.bottom)
               .frame(height: expanded ? 0 : 110, alignment: .bottom)
 
             VStack {
               Spacer(minLength: 0)
-              MiniPlayerViewer(itemAudioProperties: itemAudioProperties)
+              MiniPlayerViewer()
             }
           }
         }
diff --git a/apple/OmnivoreKit/Sources/App/Views/AudioPlayer/MiniPlayerViewer.swift b/apple/OmnivoreKit/Sources/App/Views/AudioPlayer/MiniPlayerViewer.swift
index 75f41b36a..fb0c31b20 100644
--- a/apple/OmnivoreKit/Sources/App/Views/AudioPlayer/MiniPlayerViewer.swift
+++ b/apple/OmnivoreKit/Sources/App/Views/AudioPlayer/MiniPlayerViewer.swift
@@ -13,8 +13,6 @@
 
     @State var expanded = true
 
-    let itemAudioProperties: LinkedItemAudioProperties
-
     var playPauseButtonImage: String {
       switch audioController.state {
       case .playing:
@@ -32,7 +30,7 @@
       if audioController.playbackError {
         return AnyView(Color.clear)
       }
-      if let itemID = audioController.itemAudioProperties?.itemID, audioController.isLoadingItem(itemID: itemID) {
+      if audioController.isLoadingItem(audioController.itemAudioProperties) {
         return AnyView(ProgressView())
       } else {
         return AnyView(Button(
@@ -84,8 +82,8 @@
       .buttonStyle(PlainButtonStyle())
     }
 
-    func artwork(_ itemAudioProperties: LinkedItemAudioProperties, forDimensions dim: Double) -> some View {
-      if let imageURL = itemAudioProperties.imageURL {
+    func artwork(_ itemAudioProperties: AudioItemProperties?, forDimensions dim: Double) -> some View {
+      if let imageURL = itemAudioProperties?.imageURL {
         return AnyView(AsyncImage(url: imageURL) { phase in
           if let image = phase.image {
             image
@@ -125,9 +123,9 @@
             Text("There was an error playing back your audio.").foregroundColor(Color.red).font(.footnote)
             Spacer(minLength: 0)
           } else {
-            artwork(itemAudioProperties, forDimensions: 50)
+            artwork(audioController.itemAudioProperties, forDimensions: 50)
 
-            Text(itemAudioProperties.title)
+            Text(audioController.itemAudioProperties?.title ?? "")
               .font(Font.system(size: 17, weight: .medium))
               .fixedSize(horizontal: false, vertical: true)
               .lineLimit(2)
diff --git a/apple/OmnivoreKit/Sources/App/Views/Home/HomeFeedViewIOS.swift b/apple/OmnivoreKit/Sources/App/Views/Home/HomeFeedViewIOS.swift
index 52ae844be..3b22ac24e 100644
--- a/apple/OmnivoreKit/Sources/App/Views/Home/HomeFeedViewIOS.swift
+++ b/apple/OmnivoreKit/Sources/App/Views/Home/HomeFeedViewIOS.swift
@@ -204,6 +204,9 @@ struct AnimatingCellHeight: AnimatableModifier {
     @ObservedObject var viewModel: HomeFeedViewModel
     @State private var selection = Set<String>()
 
+    @AppStorage("LibraryList::digestEnabled") var digestEnabled = false
+    @AppStorage("LibraryList::hasCheckedForDigestFeature") var hasCheckedForDigestFeature = false
+
     init(viewModel: HomeFeedViewModel, isEditMode: Binding<EditMode>) {
       _viewModel = ObservedObject(wrappedValue: viewModel)
       _isEditMode = isEditMode
@@ -225,7 +228,7 @@ struct AnimatingCellHeight: AnimatableModifier {
     }
 
     var body: some View {
-      ZStack {        
+      ZStack {
         HomeFeedView(
           listTitle: $listTitle,
           isListScrolled: $isListScrolled,
@@ -265,8 +268,8 @@ struct AnimatingCellHeight: AnimatableModifier {
           VStack(spacing: 0) {
             Spacer()
 
-            if let audioProperties = audioController.itemAudioProperties {
-              MiniPlayerViewer(itemAudioProperties: audioProperties)
+            if audioController.itemAudioProperties != nil {
+              MiniPlayerViewer()
                 .padding(.top, 10)
                 .padding(.bottom, 20)
                 .background(Color.themeTabBarColor)
@@ -317,6 +320,15 @@ struct AnimatingCellHeight: AnimatableModifier {
           }
         )
       }
+      .fullScreenCover(isPresented: $showLibraryDigest) {
+        if #available(iOS 17.0, *) {
+          NavigationView {
+            FullScreenDigestView(dataService: dataService, audioController: audioController)
+          }
+        } else {
+          Text("Sorry digest is only available on iOS 17 and above")
+        }
+      }
       .toolbar {
         toolbarItems
       }
@@ -339,6 +351,20 @@ struct AnimatingCellHeight: AnimatableModifier {
           viewModel.stopUsingFollowingPrimer = true
         }
       }
+      .task {
+        do {
+          if let viewer = try await dataService.fetchViewer() {
+            digestEnabled = viewer.digestEnabled ?? false
+            if !hasCheckedForDigestFeature {
+              hasCheckedForDigestFeature = true
+              // selectedTab = "digest"
+            }
+          }
+        } catch {
+          print("ERROR FETCHING VIEWER: ", error)
+          print("")
+        }
+      }
       .environment(\.editMode, self.$isEditMode)
       .navigationBarTitleDisplayMode(.inline)
     }
@@ -383,15 +409,14 @@ struct AnimatingCellHeight: AnimatableModifier {
               if isEditMode == .active {
                 Button(action: { isEditMode = .inactive }, label: { Text("Cancel") })
               } else {
-                
-//                if #available(iOS 17.0, *) {
-//                  Button(
-//                    action: { showLibraryDigest = true },
-//                    label: { Image(systemName: "sparkles") }
-//                  )
-//                  .buttonStyle(.plain)
-//                  .padding(.trailing, 4)
-//                }
+                if #available(iOS 17.0, *) {
+                  Button(
+                    action: { showLibraryDigest = true },
+                    label: { Image.tabDigestSelected }
+                  )
+                  .buttonStyle(.plain)
+                  .padding(.trailing, 4)
+                }
                 if prefersListLayout {
                   Button(
                     action: { isEditMode = isEditMode == .active ? .inactive : .active },
@@ -481,12 +506,12 @@ struct AnimatingCellHeight: AnimatableModifier {
     let showFeatureCards: Bool
     var slideTransition: PresentationLinkTransition {
       PresentationLinkTransition.slide(
-        options: PresentationLinkTransition.SlideTransitionOptions(edge: .trailing,
-                                                                   options:
-                                                                    PresentationLinkTransition.Options(
-                                                                      modalPresentationCapturesStatusBarAppearance: true
-                                                                    )
-                                                                  ))
+        options: PresentationLinkTransition.SlideTransitionOptions(
+          edge: .trailing,
+          options: PresentationLinkTransition.Options(
+            modalPresentationCapturesStatusBarAppearance: true
+          )
+        ))
     }
 
     var body: some View {
@@ -494,12 +519,12 @@ struct AnimatingCellHeight: AnimatableModifier {
         if let linkRequest = viewModel.linkRequest, viewModel.currentListConfig?.hasReadNowSection ?? false {
           PresentationLink(
             transition: PresentationLinkTransition.slide(
-              options: PresentationLinkTransition.SlideTransitionOptions(edge: .trailing,
-                                                                         options:
-                                                                         PresentationLinkTransition.Options(
-                                                                           modalPresentationCapturesStatusBarAppearance: true,
-                                                                           preferredPresentationBackgroundColor: ThemeManager.currentBgColor
-                                                                         ))),
+              options: PresentationLinkTransition.SlideTransitionOptions(
+                edge: .trailing,
+                options: PresentationLinkTransition.Options(
+                  modalPresentationCapturesStatusBarAppearance: true,
+                  preferredPresentationBackgroundColor: ThemeManager.currentBgColor
+                ))),
             isPresented: $viewModel.presentWebContainer,
             destination: {
               WebReaderLoadingContainer(requestID: linkRequest.serverID)
diff --git a/apple/OmnivoreKit/Sources/App/Views/LibraryTabView.swift b/apple/OmnivoreKit/Sources/App/Views/LibraryTabView.swift
index 79f9137b7..e98786b5a 100644
--- a/apple/OmnivoreKit/Sources/App/Views/LibraryTabView.swift
+++ b/apple/OmnivoreKit/Sources/App/Views/LibraryTabView.swift
@@ -1,10 +1,3 @@
-//
-//  File.swift
-//
-//
-//  Created by Jackson Harper on 6/29/23.
-//
-
 import Foundation
 import Models
 import Services
@@ -21,9 +14,6 @@ struct LibraryTabView: View {
   @AppStorage("LibraryTabView::hideFollowingTab") var hideFollowingTab = false
   @AppStorage(UserDefaultKey.lastSelectedTabItem.rawValue) var selectedTab = "inbox"
 
-  @AppStorage("LibraryTabView::digestEnabled") var digestEnabled = false
-  @AppStorage("LibraryTabView::hasCheckedForDigestFeature") var hasCheckedForDigestFeature = false
-
   @State var isEditMode: EditMode = .inactive
   @State var showExpandedAudioPlayer = false
   @State var presentPushContainer = true
@@ -79,6 +69,8 @@ struct LibraryTabView: View {
   @State var operationStatus: OperationStatus = .none
   @State var operationMessage: String?
 
+  @State var digestEnabled = false
+
   var showDigest: Bool {
     if digestEnabled, #available(iOS 17.0, *) {
       return true
@@ -143,7 +135,7 @@ struct LibraryTabView: View {
 
         if showDigest, #available(iOS 17.0, *) {
           NavigationView {
-            LibraryDigestView(dataService: dataService)
+            DigestView(dataService: dataService)
               .navigationBarTitleDisplayMode(.inline)
               .navigationViewStyle(.stack)
           }.tag("digest")
@@ -165,8 +157,8 @@ struct LibraryTabView: View {
         }
 
       }
-      if let audioProperties = audioController.itemAudioProperties {
-        MiniPlayerViewer(itemAudioProperties: audioProperties)
+      if audioController.itemAudioProperties != nil {
+        MiniPlayerViewer()
           .onTapGesture {
             showExpandedAudioPlayer = true
           }
@@ -236,19 +228,5 @@ struct LibraryTabView: View {
       }
       selectedTab = "inbox"
     }
-    .task {
-      do {
-        if let viewer = try await dataService.fetchViewer() {
-          digestEnabled = viewer.digestEnabled ?? false
-          if !hasCheckedForDigestFeature {
-            hasCheckedForDigestFeature = true
-            selectedTab = "digest"
-          }
-        }
-      } catch {
-        print("ERROR FETCHING VIEWER: ", error)
-        print("")
-      }
-    }
   }
 }
diff --git a/apple/OmnivoreKit/Sources/App/Views/PrimaryContentView.swift b/apple/OmnivoreKit/Sources/App/Views/PrimaryContentView.swift
index 634553453..34de3e874 100644
--- a/apple/OmnivoreKit/Sources/App/Views/PrimaryContentView.swift
+++ b/apple/OmnivoreKit/Sources/App/Views/PrimaryContentView.swift
@@ -52,7 +52,7 @@ import Transmission
       return AnyView(splitView)
     #endif
   }
-  
+
   func startTimer(amount: Int) {
     self.snackbarTimer = Timer.scheduledTimer(withTimeInterval: TimeInterval(amount / 1000), repeats: false) { _ in
       DispatchQueue.main.async {
diff --git a/apple/OmnivoreKit/Sources/App/Views/WebReader/WebReaderContainer.swift b/apple/OmnivoreKit/Sources/App/Views/WebReader/WebReaderContainer.swift
index 5eebee5ee..93491b275 100644
--- a/apple/OmnivoreKit/Sources/App/Views/WebReader/WebReaderContainer.swift
+++ b/apple/OmnivoreKit/Sources/App/Views/WebReader/WebReaderContainer.swift
@@ -133,7 +133,7 @@ struct WebReaderContainerView: View {
 
   #if os(iOS)
     var audioNavbarItem: some View {
-      if !audioController.playbackError, audioController.isLoadingItem(itemID: item.unwrappedID) {
+      if audioController.isLoadingItem(audioController.itemAudioProperties) {
         return AnyView(ProgressView()
           .padding(.horizontal))
       } else {
@@ -500,20 +500,6 @@ struct WebReaderContainerView: View {
           .formSheet(isPresented: $showOpenArchiveSheet) {
             OpenArchiveTodayView(item: item)
           }
-          .formSheet(isPresented: $showExplainSheet) {
-            ExplainView(
-              viewModel: ExplainViewModel(
-                dataService: dataService,
-                item: self.item,
-                promptName: "explain-text-001",
-                extraText: viewModel.explainText
-              ),
-              dismissAction: {
-                viewModel.explainText = nil
-                showExplainSheet = false
-              }
-            )
-          }
         #endif
         .sheet(isPresented: $showHighlightAnnotationModal) {
           NavigationView {
@@ -629,8 +615,8 @@ struct WebReaderContainerView: View {
             .offset(y: navBarVisible ? 0 : -150)
 
           Spacer()
-          if let audioProperties = audioController.itemAudioProperties {
-            MiniPlayerViewer(itemAudioProperties: audioProperties)
+          if audioController.itemAudioProperties != nil {
+            MiniPlayerViewer()
               .padding(.top, 10)
               .padding(.bottom, showBottomBar ? 10 : 40)
               .background(Color.themeTabBarColor)
diff --git a/apple/OmnivoreKit/Sources/Models/DataModels/FeedItem.swift b/apple/OmnivoreKit/Sources/Models/DataModels/FeedItem.swift
index 53aa7a504..cbf918195 100644
--- a/apple/OmnivoreKit/Sources/Models/DataModels/FeedItem.swift
+++ b/apple/OmnivoreKit/Sources/Models/DataModels/FeedItem.swift
@@ -27,8 +27,7 @@ public struct LinkedItemSyncResult {
               hasMore: Bool,
               mostRecentUpdatedAt: Date?,
               oldestUpdatedAt: Date?,
-              isEmpty: Bool)
-  {
+              isEmpty: Bool) {
     self.updatedItemIDs = updatedItemIDs
     self.cursor = cursor
     self.hasMore = hasMore
@@ -38,7 +37,41 @@ public struct LinkedItemSyncResult {
   }
 }
 
-public struct LinkedItemAudioProperties {
+public enum AudioItemType {
+  case digest
+  case libraryItem
+}
+
+public protocol AudioItemProperties {
+  var audioItemType: AudioItemType {
+    get
+  }
+  var itemID: String {
+    get
+  }
+  var title: String {
+    get
+  }
+  var byline: String? {
+    get
+  }
+  var imageURL: URL? {
+    get
+  }
+  var language: String? {
+    get
+  }
+  var startIndex: Int {
+    get
+  }
+  var startOffset: Double {
+    get
+  }
+}
+
+public struct LinkedItemAudioProperties: AudioItemProperties {
+  public let audioItemType = AudioItemType.libraryItem
+
   public let itemID: String
   public let objectID: NSManagedObjectID
   public let title: String
diff --git a/apple/OmnivoreKit/Sources/Services/AudioSession/AudioController.swift b/apple/OmnivoreKit/Sources/Services/AudioSession/AudioController.swift
index 318c3388a..313265e66 100644
--- a/apple/OmnivoreKit/Sources/Services/AudioSession/AudioController.swift
+++ b/apple/OmnivoreKit/Sources/Services/AudioSession/AudioController.swift
@@ -33,7 +33,7 @@
     @Published public var currentAudioIndex: Int = 0
     @Published public var readText: String = ""
     @Published public var unreadText: String = ""
-    @Published public var itemAudioProperties: LinkedItemAudioProperties?
+    @Published public var itemAudioProperties: AudioItemProperties?
 
     @Published public var timeElapsed: TimeInterval = 0
     @Published public var duration: TimeInterval = 0
@@ -72,7 +72,7 @@
       observer = nil
     }
 
-    public func play(itemAudioProperties: LinkedItemAudioProperties) {
+    public func play(itemAudioProperties: AudioItemProperties) {
       stop()
 
       playbackError = false
@@ -539,11 +539,14 @@
       state == .playing
     }
 
-    public func isLoadingItem(itemID: String) -> Bool {
+    public func isLoadingItem(_ audioItem: AudioItemProperties?) -> Bool {
       if state == .reachedEnd {
         return false
       }
-      return itemAudioProperties?.itemID == itemID && isLoading
+      if audioItem?.itemID == nil {
+        return false
+      }
+      return itemAudioProperties?.itemID == audioItem?.itemID && isLoading
     }
 
     public func isPlayingItem(itemID: String) -> Bool {
@@ -860,8 +863,19 @@
       }
       return ""
     }
-
+    
     func downloadSpeechFile(itemID: String, priority: DownloadPriority) async throws -> SpeechDocument? {
+      switch(self.itemAudioProperties?.audioItemType) {
+      case .digest:
+        return try await downloadDigestItemSpeechFile(itemID: itemID, priority: priority)
+      case .libraryItem:
+        return try await downloadLibraryItemSpeechFile(itemID: itemID, priority: priority)
+      case .none:
+        return nil
+      }
+    }
+
+    func downloadLibraryItemSpeechFile(itemID: String, priority: DownloadPriority) async throws -> SpeechDocument? {
       let decoder = JSONDecoder()
       let speechFileUrl = pathForSpeechFile(itemID: itemID)
 
@@ -910,6 +924,60 @@
       return nil
     }
 
+    func downloadDigestItemSpeechFile(itemID: String, priority: DownloadPriority) async throws -> SpeechDocument? {
+      let decoder = JSONDecoder()
+      let speechFileUrl = URL.om_documentsDirectory.appendingPathComponent("digest").appendingPathComponent("speech-\(currentVoice).json")
+
+      if FileManager.default.fileExists(atPath: speechFileUrl.path) {
+        let data = try Data(contentsOf: speechFileUrl)
+        document = try decoder.decode(SpeechDocument.self, from: data)
+        // If we can't load it from disk we make the API call
+        if let document = document {
+          return document
+        }
+      }
+
+      let path = "/api/digest/v1/"
+      guard let url = URL(string: path, relativeTo: dataService.appEnvironment.serverBaseURL) else {
+        throw BasicError.message(messageText: "Invalid audio URL")
+      }
+
+      var request = URLRequest(url: url)
+      request.httpMethod = "GET"
+      for (header, value) in dataService.networker.defaultHeaders {
+        request.setValue(value, forHTTPHeaderField: header)
+      }
+
+      let result: (Data, URLResponse)? = try? await URLSession.shared.data(for: request)
+      guard let httpResponse = result?.1 as? HTTPURLResponse, 200 ..< 300 ~= httpResponse.statusCode else {
+        throw BasicError.message(messageText: "audioFetch failed. no response or bad status code.")
+      }
+
+      guard let data = result?.0 else {
+        throw BasicError.message(messageText: "audioFetch failed. no data received.")
+      }
+
+      let str = String(decoding: data, as: UTF8.self)
+      print("result digest file: ", str)
+
+      do {
+        let digest = try JSONDecoder().decode(DigestResult.self, from: data)
+        let directory = URL.om_documentsDirectory.appendingPathComponent("digest")
+         // do {
+            try? FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true)
+            try data.write(to: speechFileUrl)
+            return digest.speechFile
+//          } catch {
+//            print("error writing file", error)
+//          }
+        // }
+      } catch {
+        print("error with digest file", error)
+      }
+
+      return nil
+    }
+
     func getSpeechFile(itemID: String, priority: DownloadPriority) async throws -> SpeechDocument? {
       document = try await downloadSpeechFile(itemID: itemID, priority: priority)
       return document
diff --git a/apple/OmnivoreKit/Sources/Services/AudioSession/SpeechSynthesizer.swift b/apple/OmnivoreKit/Sources/Services/AudioSession/SpeechSynthesizer.swift
index 2939c2cf4..2946c0fb0 100644
--- a/apple/OmnivoreKit/Sources/Services/AudioSession/SpeechSynthesizer.swift
+++ b/apple/OmnivoreKit/Sources/Services/AudioSession/SpeechSynthesizer.swift
@@ -20,7 +20,7 @@ struct UtteranceRequest: Codable {
   let isOpenAIVoice: Bool
 }
 
-struct Utterance: Decodable {
+public struct Utterance: Decodable {
   public let idx: String
   public let text: String
   public let voice: String?
@@ -39,7 +39,7 @@ struct Utterance: Decodable {
   }
 }
 
-struct SpeechDocument: Decodable {
+public struct SpeechDocument: Decodable {
   static let averageWPM: Double = 195
 
   public let pageId: String
diff --git a/apple/OmnivoreKit/Sources/Services/DataService/AI/AITasks.swift b/apple/OmnivoreKit/Sources/Services/DataService/AI/AITasks.swift
new file mode 100644
index 000000000..7ea58022c
--- /dev/null
+++ b/apple/OmnivoreKit/Sources/Services/DataService/AI/AITasks.swift
@@ -0,0 +1,118 @@
+import CoreData
+import Foundation
+import Models
+import Utils
+
+struct AITaskRequest: Decodable {
+  public let requestId: String
+}
+
+public struct DigestResult: Decodable {
+  public let id: String
+  public let title: String
+  public let content: String
+  public let urlsToAudio: [String]
+  public let speechFile: SpeechDocument
+
+  public let jobState: String
+}
+
+public struct DigestItem: Decodable {
+  public let id: String
+  public let site: String
+  public let siteIcon: URL?
+  public let author: String
+  public let title: String
+  public let summaryText: String
+  public let keyPointsText: String
+  public let highlightsText: String
+  public init(id: String, site: String, siteIcon: URL?, 
+              author: String, title: String, summaryText: String,
+              keyPointsText: String, highlightsText: String) {
+    self.id = id
+    self.site = site
+    self.siteIcon = siteIcon
+    self.author = author
+    self.title = title
+    self.summaryText = summaryText
+    self.keyPointsText = keyPointsText
+    self.highlightsText = highlightsText
+  }
+}
+
+extension DataService {
+//  public func createAITask(extraText: String?, libraryItemId: String, promptName: String) async throws -> String? {
+//    let jsonData = try JSONSerialization.data(withJSONObject: [
+//      "libraryItemId": libraryItemId,
+//      "promptName": promptName,
+//      "extraText": extraText
+//    ])
+//
+//    let urlRequest = URLRequest.create(
+//      baseURL: appEnvironment.serverBaseURL,
+//      urlPath: "/api/ai-task",
+//      requestMethod: .post(params: jsonData),
+//      includeAuthToken: true
+//    )
+//    let resource = ServerResource<AITaskRequest>(
+//      urlRequest: urlRequest,
+//      decode: AITaskRequest.decode
+//    )
+//
+//    do {
+//      let taskRequest = try await networker.urlSession.performRequest(resource: resource)
+//      return taskRequest.requestId
+//    } catch {
+//      return nil
+//    }
+//  }
+
+  // Function to poll the status of the AI task with timeout
+  public func getLatestDigest(timeoutInterval: TimeInterval) async throws -> DigestResult? {
+    var count = 0
+      let startTime = Date()
+      while true {
+        count += 1
+        if count > 3 {
+          return nil
+        }
+          do {
+              // Check if timeout has occurred
+              if -startTime.timeIntervalSinceNow >= timeoutInterval {
+                  throw NSError(domain: "Timeout Error", code: -1, userInfo: nil)
+              }
+
+              let urlRequest = URLRequest.create(
+                baseURL: appEnvironment.serverBaseURL,
+                urlPath: "/api/digest/v1/",
+                requestMethod: .get,
+                includeAuthToken: true
+              )
+
+            let resource = ServerResource<DigestResult>(
+              urlRequest: urlRequest,
+              decode: DigestResult.decode
+            )
+
+            do {
+              let digest = try await networker.urlSession.performRequest(resource: resource)
+              print("GOT RESPONSE: ", digest)
+              return digest
+            } catch {
+              print("ERROR FETCHING TASK: ", error)
+//              if let response = error as? ServerError {
+//                if response != .stillProcessing {
+//                  return nil
+//                }
+//              }
+            }
+              // Wait for some time before polling again
+              try? await Task.sleep(nanoseconds: 3_000_000_000)
+          } catch let error {
+              throw error
+          }
+      }
+  }
+}
+
+

From fbaa9ce66358017963cc2146e328d2ac84a8affc Mon Sep 17 00:00:00 2001
From: Jackson Harper <jacksonh@gmail.com>
Date: Tue, 16 Apr 2024 16:55:46 +0800
Subject: [PATCH 57/74] Digest job

---
 packages/api/src/jobs/ai/create_digest.ts | 167 ++++++++++++++++++++++
 1 file changed, 167 insertions(+)
 create mode 100644 packages/api/src/jobs/ai/create_digest.ts

diff --git a/packages/api/src/jobs/ai/create_digest.ts b/packages/api/src/jobs/ai/create_digest.ts
new file mode 100644
index 000000000..acf1d827e
--- /dev/null
+++ b/packages/api/src/jobs/ai/create_digest.ts
@@ -0,0 +1,167 @@
+import { logger } from '../../utils/logger'
+import { OpenAI } from '@langchain/openai'
+import { PromptTemplate } from '@langchain/core/prompts'
+import { LibraryItem } from '../../entity/library_item'
+import { CreateDigestJobData } from '../../services/digest'
+
+const USER_PROFILE_PROMPT =
+  'Create a user profile based on the supplied titles\n\ntitles:\n{titles}'
+
+const SUMMARIZE_PROMPT =
+  'Summarize the supplied article.\n\ntitle: {title}\nauthor: {author}\ncontent: {content}'
+
+// TODO: Makes multiple DB queries and combines the results
+const getPreferencesList = (userId: string): Promise<LibraryItem[]> => {
+  // use the queries from the digest definitions to lookup preferences
+  // There should be a list of multiple queries we use. For now we can
+  // hardcode these queries:
+  // - query: "in:all is:read OR has:highlights sort:updated-desc wordsCount:>=20"
+  //   count: 21
+  //   reason: "recently read or highlighted items that are not part of the digest"
+  // - query: "in:all is:read OR has:highlights sort:saved-asc wordsCount:>=20"
+  //   count: 4
+  //   reason: "some older items that were interacted with"
+  return Promise.resolve([])
+}
+
+// TODO: Makes multiple DB queries and combines the results
+const getCandidatesList = (userId: string): Promise<LibraryItem[]> => {
+  // use the queries from the digest definitions to lookup preferences
+  // There should be a list of multiple queries we use. For now we can
+  // hardcode these queries:
+  // - query: "in:all is:unread saved:last24hrs sort:saved-desc wordsCount:>=500"
+  //   count: 100
+  //   reason: "most recent 100 items saved over 500 words
+  return Promise.resolve([])
+}
+
+// TODO: Takes a list of library items, and uses a prompt to generate
+// a text representation of a user profile
+const createUserProfile = async (
+  preferences: LibraryItem[]
+): Promise<string> => {
+  const llm = new OpenAI({
+    modelName: 'gpt-4-0125-preview',
+    configuration: {
+      apiKey: process.env.OPENAI_API_KEY,
+    },
+  })
+
+  const contextualTemplate = PromptTemplate.fromTemplate(USER_PROFILE_PROMPT)
+
+  const chain = contextualTemplate.pipe(llm)
+  const result = await chain.invoke({
+    titles: preferences.map((item) => `* ${item}`).join('\n'),
+  })
+
+  return result
+}
+
+// TODO: Checks redis for a user profile, if not found creates one and writes
+// it to redis
+const findOrCreateUserProfile = async (userId: string): Promise<string> => {
+  // check redis for user profile, return if found
+  // if not found
+  const preferences = await getPreferencesList(userId)
+  const profile = await createUserProfile(preferences)
+  // TODO: write to redis here
+  return profile
+}
+
+type RankedItem = {
+  topic: string
+  summary?: string
+  libraryItem: LibraryItem
+}
+
+// TODO: Uses OpenAI to rank all the titles based on the user profiles
+const rankCandidates = async (
+  candidates: LibraryItem[],
+  userProfile: string
+): Promise<RankedItem[]> => {
+  return Promise.resolve([])
+}
+
+// Does some grouping by topic while trying to maintain ranking
+// adds some basic topic diversity
+const chooseRankedSelections = (rankedCandidates: RankedItem[]) => {
+  const selected = []
+  const rankedTopics = []
+  const topicCount = {} as Record<string, number>
+
+  for (const item of rankedCandidates) {
+    if (selected.length >= 5) {
+      break
+    }
+
+    topicCount[item.topic] = (topicCount[item.topic] || 0) + 1
+
+    if (topicCount[item.topic] <= 2) {
+      selected.push(item)
+      if (rankedTopics.indexOf(item.topic) === -1) {
+        rankedTopics.push(item.topic)
+      }
+    }
+  }
+
+  console.log('rankedTopics: ', rankedTopics)
+  console.log('finalSelections: ', selected)
+
+  const finalSelections = []
+
+  for (const topic of rankedTopics) {
+    const matches = selected.filter((item) => item.topic == topic)
+    finalSelections.push(...matches)
+  }
+
+  console.log('finalSelections: ', finalSelections)
+
+  return finalSelections
+}
+
+// TODO: we could paralleize this step sending all the ranked candidates to openAI at once
+const summarizeItems = async (
+  rankedCandidates: RankedItem[]
+): Promise<RankedItem[]> => {
+  const llm = new OpenAI({
+    modelName: 'gpt-4-0125-preview',
+    configuration: {
+      apiKey: process.env.OPENAI_API_KEY,
+    },
+  })
+
+  for (const item of rankedCandidates) {
+    const contextualTemplate = PromptTemplate.fromTemplate(SUMMARIZE_PROMPT)
+
+    const chain = contextualTemplate.pipe(llm)
+    const summary = await chain.invoke({
+      title: item.libraryItem.title,
+      author: item.libraryItem.author ?? '',
+      content: item.libraryItem.readableContent, // markdown content
+    })
+    item.summary = summary
+  }
+
+  return rankedCandidates
+}
+
+// TODO: write the digest to redis here
+const writeDigest = (userId: string, selections: RankedItem[]) => {}
+
+export const CreateDigestJob = async (jobData: CreateDigestJobData) => {
+  try {
+    const candidates = await getCandidatesList(jobData.userId)
+    const userProfile = await findOrCreateUserProfile(jobData.userId)
+    const rankedCandidates = await rankCandidates(candidates, userProfile)
+    const selections = chooseRankedSelections(rankedCandidates)
+
+    const summaries = await summarizeItems(selections)
+
+    // TODO: we should have a QA step here that does some
+    // basic checks to make sure the summaries are good.
+
+    writeDigest(jobData.userId, summaries)
+  } catch (err) {
+    console.log('error creating summary: ', err)
+  }
+}

From bcddf556d5f8a2687bcad31bafff129905ccf0fc Mon Sep 17 00:00:00 2001
From: Jackson Harper <jacksonh@gmail.com>
Date: Tue, 16 Apr 2024 17:15:22 +0800
Subject: [PATCH 58/74] Write speech files to redis

---
 packages/api/src/jobs/ai/create_digest.ts | 54 +++++++++++++++++++++--
 packages/api/src/services/digest.ts       |  7 +--
 2 files changed, 55 insertions(+), 6 deletions(-)

diff --git a/packages/api/src/jobs/ai/create_digest.ts b/packages/api/src/jobs/ai/create_digest.ts
index acf1d827e..2026e73ee 100644
--- a/packages/api/src/jobs/ai/create_digest.ts
+++ b/packages/api/src/jobs/ai/create_digest.ts
@@ -1,8 +1,11 @@
 import { logger } from '../../utils/logger'
+import { v4 as uuid } from 'uuid'
+
 import { OpenAI } from '@langchain/openai'
 import { PromptTemplate } from '@langchain/core/prompts'
 import { LibraryItem } from '../../entity/library_item'
 import { CreateDigestJobData } from '../../services/digest'
+import { htmlToSsmlItems } from '@omnivore/text-to-speech-handler'
 
 const USER_PROFILE_PROMPT =
   'Create a user profile based on the supplied titles\n\ntitles:\n{titles}'
@@ -145,8 +148,53 @@ const summarizeItems = async (
   return rankedCandidates
 }
 
+// TODO: we can use something more sophisticated to generate titles
+const generateTitle = (selections: RankedItem[]): Promise<string> => {
+  return Promise.resolve(
+    'Omnivore digest: ' +
+      selections.map((item) => item.libraryItem.title).join(',')
+  )
+}
+
 // TODO: write the digest to redis here
-const writeDigest = (userId: string, selections: RankedItem[]) => {}
+// export interface Digest {
+//   jobState: string
+
+//   url?: string
+//   title?: string
+//   content?: string
+//   chapters?: Chapter[]
+
+//   urlsToAudio?: string[]
+//   speechFiles?: SpeechFile[]
+// }
+// export interface SpeechFile {
+//  wordCount: number;
+//  language: string;
+//  defaultVoice: string;
+//  utterances: Utterance[];
+//}
+const writeDigest = async (userId: string, selections: RankedItem[]) => {
+  const title = await generateTitle(selections)
+  const speechFiles = selections.map((selection) => {
+    // convert the summary item to a SpeechFile here
+    return {
+      wordCount: 0,
+      language: 'en',
+      defaultVoice: 'Josh',
+      utterances: [],
+    }
+  })
+  const digest = {
+    id: uuid(),
+    title: title,
+    content: 'content',
+    urlsToAudio: [],
+    jobState: 'completed',
+    speechFiles: speechFiles,
+  }
+  // write to redis
+}
 
 export const CreateDigestJob = async (jobData: CreateDigestJobData) => {
   try {
@@ -160,8 +208,8 @@ export const CreateDigestJob = async (jobData: CreateDigestJobData) => {
     // TODO: we should have a QA step here that does some
     // basic checks to make sure the summaries are good.
 
-    writeDigest(jobData.userId, summaries)
+    await writeDigest(jobData.userId, summaries)
   } catch (err) {
-    console.log('error creating summary: ', err)
+    console.log('error creating digest: ', err)
   }
 }
diff --git a/packages/api/src/services/digest.ts b/packages/api/src/services/digest.ts
index 21b34a01c..cb0f88141 100644
--- a/packages/api/src/services/digest.ts
+++ b/packages/api/src/services/digest.ts
@@ -12,14 +12,15 @@ export interface CreateDigestJobResponse {
 }
 
 export interface Digest {
+  jobState: string
+
   url?: string
   title?: string
   content?: string
   chapters?: Chapter[]
-  urlsToAudio?: string[]
-  jobState: string
 
-  speechFile: SpeechFile
+  urlsToAudio?: string[]
+  speechFiles?: SpeechFile[]
 }
 
 interface Chapter {

From daaf493d4bc2e4cdbe51f82ce424abde4228b272 Mon Sep 17 00:00:00 2001
From: Jackson Harper <jacksonh@gmail.com>
Date: Tue, 16 Apr 2024 17:27:38 +0800
Subject: [PATCH 59/74] Update stubbed data

---
 packages/api/src/services/digest.ts | 46 +++++++++++++++--------------
 1 file changed, 24 insertions(+), 22 deletions(-)

diff --git a/packages/api/src/services/digest.ts b/packages/api/src/services/digest.ts
index cb0f88141..a41bb23f8 100644
--- a/packages/api/src/services/digest.ts
+++ b/packages/api/src/services/digest.ts
@@ -39,28 +39,30 @@ export const getDigest = async (userId: string): Promise<Digest | null> => {
       content: 'content',
       urlsToAudio: [],
       jobState: 'completed',
-      speechFile: {
-        pageId: '1234',
-        wordCount: 2124,
-        language: 'en-US',
-        defaultVoice: 'en-US-ChristopherNeural',
-        utterances: [
-          {
-            idx: '',
-            text: 'TOP prospect JOINS Canucks - Team SPEAKS OUT on Demko Injury | Canucks News',
-            wordOffset: 0,
-            wordCount: 14,
-            voice: 'en-US-ChristopherNeural',
-          },
-          {
-            idx: '4',
-            text: 'Intro',
-            wordOffset: 14,
-            wordCount: 1,
-            voice: 'en-US-ChristopherNeural',
-          },
-        ],
-      },
+      speechFiles: [
+        {
+          pageId: '1234',
+          wordCount: 2124,
+          language: 'en-US',
+          defaultVoice: 'en-US-ChristopherNeural',
+          utterances: [
+            {
+              idx: '',
+              text: 'TOP prospect JOINS Canucks - Team SPEAKS OUT on Demko Injury | Canucks News',
+              wordOffset: 0,
+              wordCount: 14,
+              voice: 'en-US-ChristopherNeural',
+            },
+            {
+              idx: '4',
+              text: 'Intro',
+              wordOffset: 14,
+              wordCount: 1,
+              voice: 'en-US-ChristopherNeural',
+            },
+          ],
+        },
+      ],
     })
   )
 

From 71477b1712529bc235ab7d7367534e6a42f5e781 Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Tue, 16 Apr 2024 19:30:40 +0800
Subject: [PATCH 60/74] implement create digest job

---
 packages/api/package.json                 |   1 +
 packages/api/src/jobs/ai/create_digest.ts | 297 +++++++++++++++-------
 packages/api/src/queue-processor.ts       |   4 +-
 packages/api/src/routers/digest_router.ts |  14 +-
 packages/api/src/services/digest.ts       |  62 ++---
 packages/api/src/utils/createTask.ts      |  21 +-
 packages/api/src/utils/parser.ts          |   4 +
 yarn.lock                                 |   5 +
 8 files changed, 254 insertions(+), 154 deletions(-)

diff --git a/packages/api/package.json b/packages/api/package.json
index 469586434..cea023808 100644
--- a/packages/api/package.json
+++ b/packages/api/package.json
@@ -113,6 +113,7 @@
     "voca": "^1.4.0",
     "winston": "^3.3.3",
     "word-counting": "^1.1.4",
+    "yaml": "^2.4.1",
     "youtubei": "1.4.0"
   },
   "devDependencies": {
diff --git a/packages/api/src/jobs/ai/create_digest.ts b/packages/api/src/jobs/ai/create_digest.ts
index 2026e73ee..feb12f78d 100644
--- a/packages/api/src/jobs/ai/create_digest.ts
+++ b/packages/api/src/jobs/ai/create_digest.ts
@@ -4,17 +4,68 @@ import { v4 as uuid } from 'uuid'
 import { OpenAI } from '@langchain/openai'
 import { PromptTemplate } from '@langchain/core/prompts'
 import { LibraryItem } from '../../entity/library_item'
-import { CreateDigestJobData } from '../../services/digest'
-import { htmlToSsmlItems } from '@omnivore/text-to-speech-handler'
+import { htmlToSpeechFile, SpeechFile } from '@omnivore/text-to-speech-handler'
+import axios from 'axios'
+import { searchLibraryItems } from '../../services/library_item'
+import { redisDataSource } from '../../redis_data_source'
+import { htmlToMarkdown } from '../../utils/parser'
+import yaml from 'yaml'
+import { JsonOutputParser } from '@langchain/core/output_parsers'
+import showdown from 'showdown'
+import { Digest, writeDigest } from '../../services/digest'
 
-const USER_PROFILE_PROMPT =
-  'Create a user profile based on the supplied titles\n\ntitles:\n{titles}'
+interface Selector {
+  query: string
+  count: number
+  reason: string
+}
 
-const SUMMARIZE_PROMPT =
-  'Summarize the supplied article.\n\ntitle: {title}\nauthor: {author}\ncontent: {content}'
+interface ZeroShotDefinition {
+  userPreferencesProfilePrompt: string
+  rankPrompt: string
+}
 
-// TODO: Makes multiple DB queries and combines the results
-const getPreferencesList = (userId: string): Promise<LibraryItem[]> => {
+interface DigestDefinition {
+  name: string
+  preferenceSelectors: Selector[]
+  candidateSelectors: Selector[]
+  contentFeaturesPrompt: string
+  contentRatingPrompt: string
+  summaryPrompt: string
+  assemblePrompt: string
+
+  zeroShot: ZeroShotDefinition
+}
+
+export interface CreateDigestJobData {
+  userId: string
+}
+
+export interface CreateDigestJobResponse {
+  jobId: string
+}
+
+export const CREATE_DIGEST_JOB = 'create-digest'
+
+let digestDefinition: DigestDefinition
+
+const fetchDigestDefinition = async (): Promise<DigestDefinition> => {
+  const promptFileUrl = process.env.PROMPT_FILE_URL
+  if (!promptFileUrl) {
+    const msg = 'PROMPT_FILE_URL not set'
+    logger.error(msg)
+    throw new Error(msg)
+  }
+
+  // fetch the yaml file
+  const response = await axios.get<string>(promptFileUrl)
+
+  // parse the yaml file
+  return yaml.parse(response.data) as DigestDefinition
+}
+
+// Makes multiple DB queries and combines the results
+const getPreferencesList = async (userId: string): Promise<LibraryItem[]> => {
   // use the queries from the digest definitions to lookup preferences
   // There should be a list of multiple queries we use. For now we can
   // hardcode these queries:
@@ -24,21 +75,72 @@ const getPreferencesList = (userId: string): Promise<LibraryItem[]> => {
   // - query: "in:all is:read OR has:highlights sort:saved-asc wordsCount:>=20"
   //   count: 4
   //   reason: "some older items that were interacted with"
-  return Promise.resolve([])
+
+  const preferences = await Promise.all(
+    digestDefinition.preferenceSelectors.map(async (selector) => {
+      // use the selector to fetch items
+      const results = await searchLibraryItems(
+        {
+          query: selector.query,
+          size: selector.count,
+        },
+        userId
+      )
+
+      return results.libraryItems
+    })
+  )
+
+  // deduplicate and flatten the items
+  const dedupedPreferences = preferences
+    .flat()
+    .filter(
+      (item, index, self) => index === self.findIndex((t) => t.id === item.id)
+    )
+
+  return dedupedPreferences
 }
 
-// TODO: Makes multiple DB queries and combines the results
-const getCandidatesList = (userId: string): Promise<LibraryItem[]> => {
+// Makes multiple DB queries and combines the results
+const getCandidatesList = async (userId: string): Promise<LibraryItem[]> => {
   // use the queries from the digest definitions to lookup preferences
   // There should be a list of multiple queries we use. For now we can
   // hardcode these queries:
   // - query: "in:all is:unread saved:last24hrs sort:saved-desc wordsCount:>=500"
   //   count: 100
   //   reason: "most recent 100 items saved over 500 words
-  return Promise.resolve([])
+
+  const candidates = await Promise.all(
+    digestDefinition.candidateSelectors.map(async (selector) => {
+      // use the selector to fetch items
+      const results = await searchLibraryItems(
+        {
+          includeContent: true,
+          query: selector.query,
+          size: selector.count,
+        },
+        userId
+      )
+
+      return results.libraryItems
+    })
+  )
+
+  // deduplicate and flatten the items
+  const dedupedCandidates = candidates
+    .flat()
+    .filter(
+      (item, index, self) => index === self.findIndex((t) => t.id === item.id)
+    )
+    .map((item) => ({
+      ...item,
+      readableContent: htmlToMarkdown(item.readableContent),
+    })) // convert the html content to markdown
+
+  return dedupedCandidates
 }
 
-// TODO: Takes a list of library items, and uses a prompt to generate
+// Takes a list of library items, and uses a prompt to generate
 // a text representation of a user profile
 const createUserProfile = async (
   preferences: LibraryItem[]
@@ -50,39 +152,68 @@ const createUserProfile = async (
     },
   })
 
-  const contextualTemplate = PromptTemplate.fromTemplate(USER_PROFILE_PROMPT)
+  const contextualTemplate = PromptTemplate.fromTemplate(
+    digestDefinition.zeroShot.userPreferencesProfilePrompt
+  )
 
   const chain = contextualTemplate.pipe(llm)
   const result = await chain.invoke({
-    titles: preferences.map((item) => `* ${item}`).join('\n'),
+    titles: preferences.map((item) => `* ${item.title}`).join('\n'),
   })
 
   return result
 }
 
-// TODO: Checks redis for a user profile, if not found creates one and writes
+// Checks redis for a user profile, if not found creates one and writes
 // it to redis
 const findOrCreateUserProfile = async (userId: string): Promise<string> => {
   // check redis for user profile, return if found
+  const key = `userProfile:${userId}`
+  const existingProfile = await redisDataSource.redisClient?.get(key)
+  if (existingProfile) {
+    return existingProfile
+  }
+
   // if not found
   const preferences = await getPreferencesList(userId)
   const profile = await createUserProfile(preferences)
-  // TODO: write to redis here
+
+  // write to redis here
+  await redisDataSource.redisClient?.set(key, profile)
+
   return profile
 }
 
 type RankedItem = {
   topic: string
-  summary?: string
+  summary: string
   libraryItem: LibraryItem
 }
 
-// TODO: Uses OpenAI to rank all the titles based on the user profiles
+// Uses OpenAI to rank all the titles based on the user profiles
 const rankCandidates = async (
   candidates: LibraryItem[],
   userProfile: string
 ): Promise<RankedItem[]> => {
-  return Promise.resolve([])
+  const llm = new OpenAI({
+    modelName: 'gpt-4-0125-preview',
+    configuration: {
+      apiKey: process.env.OPENAI_API_KEY,
+    },
+  })
+
+  const contextualTemplate = PromptTemplate.fromTemplate(
+    digestDefinition.zeroShot.rankPrompt
+  )
+
+  const outputParser = new JsonOutputParser()
+  const chain = contextualTemplate.pipe(llm).pipe(outputParser)
+  const contextStr = await chain.invoke({
+    userProfile,
+    titles: JSON.stringify(candidates.map((item) => item.title)),
+  })
+
+  return contextStr as RankedItem[]
 }
 
 // Does some grouping by topic while trying to maintain ranking
@@ -107,8 +238,8 @@ const chooseRankedSelections = (rankedCandidates: RankedItem[]) => {
     }
   }
 
-  console.log('rankedTopics: ', rankedTopics)
-  console.log('finalSelections: ', selected)
+  logger.info('rankedTopics: ', rankedTopics)
+  logger.info('finalSelections: ', selected)
 
   const finalSelections = []
 
@@ -117,12 +248,11 @@ const chooseRankedSelections = (rankedCandidates: RankedItem[]) => {
     finalSelections.push(...matches)
   }
 
-  console.log('finalSelections: ', finalSelections)
+  logger.info('finalSelections: ', finalSelections)
 
   return finalSelections
 }
 
-// TODO: we could paralleize this step sending all the ranked candidates to openAI at once
 const summarizeItems = async (
   rankedCandidates: RankedItem[]
 ): Promise<RankedItem[]> => {
@@ -133,83 +263,78 @@ const summarizeItems = async (
     },
   })
 
-  for (const item of rankedCandidates) {
-    const contextualTemplate = PromptTemplate.fromTemplate(SUMMARIZE_PROMPT)
+  const contextualTemplate = PromptTemplate.fromTemplate(
+    digestDefinition.summaryPrompt
+  )
+  const chain = contextualTemplate.pipe(llm)
 
-    const chain = contextualTemplate.pipe(llm)
-    const summary = await chain.invoke({
+  // send all the ranked candidates to openAI at once in a batch
+  const summaries = await chain.batch(
+    rankedCandidates.map((item) => ({
       title: item.libraryItem.title,
       author: item.libraryItem.author ?? '',
       content: item.libraryItem.readableContent, // markdown content
-    })
-    item.summary = summary
-  }
+    }))
+  )
+
+  summaries.forEach(
+    (summary, index) => (rankedCandidates[index].summary = summary)
+  )
 
   return rankedCandidates
 }
 
-// TODO: we can use something more sophisticated to generate titles
-const generateTitle = (selections: RankedItem[]): Promise<string> => {
-  return Promise.resolve(
-    'Omnivore digest: ' +
-      selections.map((item) => item.libraryItem.title).join(',')
-  )
+// generate speech files from the summaries
+const generateSpeechFiles = (rankedItems: RankedItem[]): SpeechFile[] => {
+  // convert the summaries from markdown to HTML
+  const converter = new showdown.Converter({
+    backslashEscapesHTMLTags: true,
+  })
+
+  const speechFiles = rankedItems.map((item) => {
+    const html = converter.makeHtml(item.summary)
+    return htmlToSpeechFile({
+      content: html,
+      options: {},
+    })
+  })
+
+  return speechFiles
 }
 
-// TODO: write the digest to redis here
-// export interface Digest {
-//   jobState: string
+// TODO: we should have a QA step here that does some
+// basic checks to make sure the summaries are good.
+const filterSummaries = (summaries: RankedItem[]): RankedItem[] => {
+  return summaries.filter((item) => item.summary.length > 100)
+}
 
-//   url?: string
-//   title?: string
-//   content?: string
-//   chapters?: Chapter[]
+// TODO: we can use something more sophisticated to generate titles
+const generateTitle = (selections: RankedItem[]): string =>
+  'Omnivore digest: ' +
+  selections.map((item) => item.libraryItem.title).join(',')
 
-//   urlsToAudio?: string[]
-//   speechFiles?: SpeechFile[]
-// }
-// export interface SpeechFile {
-//  wordCount: number;
-//  language: string;
-//  defaultVoice: string;
-//  utterances: Utterance[];
-//}
-const writeDigest = async (userId: string, selections: RankedItem[]) => {
-  const title = await generateTitle(selections)
-  const speechFiles = selections.map((selection) => {
-    // convert the summary item to a SpeechFile here
-    return {
-      wordCount: 0,
-      language: 'en',
-      defaultVoice: 'Josh',
-      utterances: [],
-    }
-  })
-  const digest = {
+export const createDigestJob = async (jobData: CreateDigestJobData) => {
+  digestDefinition = await fetchDigestDefinition()
+
+  const candidates = await getCandidatesList(jobData.userId)
+  const userProfile = await findOrCreateUserProfile(jobData.userId)
+  const rankedCandidates = await rankCandidates(candidates, userProfile)
+  const selections = chooseRankedSelections(rankedCandidates)
+
+  const summaries = await summarizeItems(selections)
+
+  const filteredSummaries = filterSummaries(summaries)
+
+  const speechFiles = generateSpeechFiles(filteredSummaries)
+  const title = generateTitle(summaries)
+  const digest: Digest = {
     id: uuid(),
-    title: title,
+    title,
     content: 'content',
     urlsToAudio: [],
     jobState: 'completed',
-    speechFiles: speechFiles,
-  }
-  // write to redis
-}
-
-export const CreateDigestJob = async (jobData: CreateDigestJobData) => {
-  try {
-    const candidates = await getCandidatesList(jobData.userId)
-    const userProfile = await findOrCreateUserProfile(jobData.userId)
-    const rankedCandidates = await rankCandidates(candidates, userProfile)
-    const selections = chooseRankedSelections(rankedCandidates)
-
-    const summaries = await summarizeItems(selections)
-
-    // TODO: we should have a QA step here that does some
-    // basic checks to make sure the summaries are good.
-
-    await writeDigest(jobData.userId, summaries)
-  } catch (err) {
-    console.log('error creating digest: ', err)
+    speechFiles,
   }
+
+  await writeDigest(jobData.userId, digest)
 }
diff --git a/packages/api/src/queue-processor.ts b/packages/api/src/queue-processor.ts
index dac1f74d9..5f0fc6a21 100644
--- a/packages/api/src/queue-processor.ts
+++ b/packages/api/src/queue-processor.ts
@@ -16,6 +16,7 @@ import { appDataSource } from './data_source'
 import { env } from './env'
 import { TaskState } from './generated/graphql'
 import { aiSummarize, AI_SUMMARIZE_JOB_NAME } from './jobs/ai-summarize'
+import { createDigestJob, CREATE_DIGEST_JOB } from './jobs/ai/create_digest'
 import { bulkAction, BULK_ACTION_JOB_NAME } from './jobs/bulk_action'
 import { callWebhook, CALL_WEBHOOK_JOB_NAME } from './jobs/call_webhook'
 import { findThumbnail, THUMBNAIL_JOB } from './jobs/find_thumbnail'
@@ -65,7 +66,6 @@ import {
 } from './jobs/email/inbound_emails'
 
 export const QUEUE_NAME = 'omnivore-backend-queue'
-export const LONG_RUNNING_QUEUE_NAME = 'omnivore-backend-long-running-queue'
 export const JOB_VERSION = 'v001'
 
 export const getBackendQueue = async (
@@ -180,6 +180,8 @@ export const createWorker = (connection: ConnectionOptions) =>
           return saveNewsletterJob(job.data)
         case FORWARD_EMAIL_JOB:
           return forwardEmailJob(job.data)
+        case CREATE_DIGEST_JOB:
+          return createDigestJob(job.data)
         default:
           logger.warning(`[queue-processor] unhandled job: ${job.name}`)
       }
diff --git a/packages/api/src/routers/digest_router.ts b/packages/api/src/routers/digest_router.ts
index f2c5ec65d..0a7de6cf9 100644
--- a/packages/api/src/routers/digest_router.ts
+++ b/packages/api/src/routers/digest_router.ts
@@ -1,13 +1,9 @@
 import cors from 'cors'
 import express from 'express'
 import { env } from '../env'
-import {
-  createJobId,
-  getJob,
-  jobStateToTaskState,
-  LONG_RUNNING_QUEUE_NAME,
-} from '../queue-processor'
-import { CREATE_DIGEST_JOB, getDigest } from '../services/digest'
+import { CREATE_DIGEST_JOB } from '../jobs/ai/create_digest'
+import { createJobId, getJob, jobStateToTaskState } from '../queue-processor'
+import { getDigest } from '../services/digest'
 import { findActiveUser } from '../services/user'
 import { analytics } from '../utils/analytics'
 import { getClaimsByToken, getTokenByRequest } from '../utils/auth'
@@ -70,7 +66,7 @@ export function digestRouter() {
       // if yes then return 202 accepted
       // else enqueue job
       const jobId = createJobId(CREATE_DIGEST_JOB, userId)
-      const existingJob = await getJob(jobId, LONG_RUNNING_QUEUE_NAME)
+      const existingJob = await getJob(jobId)
       if (existingJob) {
         logger.info(`Job already in queue: ${jobId}`)
         return res.sendStatus(202)
@@ -118,7 +114,7 @@ export function digestRouter() {
 
       // get job by user id
       const jobId = createJobId(CREATE_DIGEST_JOB, userId)
-      const job = await getJob(jobId, LONG_RUNNING_QUEUE_NAME)
+      const job = await getJob(jobId)
       if (job) {
         // if job is in queue then return job state
         const jobState = await job.getState()
diff --git a/packages/api/src/services/digest.ts b/packages/api/src/services/digest.ts
index a41bb23f8..271e79765 100644
--- a/packages/api/src/services/digest.ts
+++ b/packages/api/src/services/digest.ts
@@ -1,17 +1,9 @@
 import { redisDataSource } from '../redis_data_source'
 import { SpeechFile } from '@omnivore/text-to-speech-handler'
-
-export const CREATE_DIGEST_JOB = 'create-digest'
-
-export interface CreateDigestJobData {
-  userId: string
-}
-
-export interface CreateDigestJobResponse {
-  jobId: string
-}
+import { logger } from '../utils/logger'
 
 export interface Digest {
+  id: string
   jobState: string
 
   url?: string
@@ -30,42 +22,20 @@ interface Chapter {
 const digestKey = (userId: string) => `digest:${userId}`
 
 export const getDigest = async (userId: string): Promise<Digest | null> => {
-  await redisDataSource.redisClient?.set(
-    digestKey(userId),
-    JSON.stringify({
-      id: 'BB3D5D89-70A2-4AE1-ADDC-713232B1281D',
-      title:
-        'SOTU response collapses, Trump hits new low, Biden fundraising explodes 3/11/24 TDPS Podcast',
-      content: 'content',
-      urlsToAudio: [],
-      jobState: 'completed',
-      speechFiles: [
-        {
-          pageId: '1234',
-          wordCount: 2124,
-          language: 'en-US',
-          defaultVoice: 'en-US-ChristopherNeural',
-          utterances: [
-            {
-              idx: '',
-              text: 'TOP prospect JOINS Canucks - Team SPEAKS OUT on Demko Injury | Canucks News',
-              wordOffset: 0,
-              wordCount: 14,
-              voice: 'en-US-ChristopherNeural',
-            },
-            {
-              idx: '4',
-              text: 'Intro',
-              wordOffset: 14,
-              wordCount: 1,
-              voice: 'en-US-ChristopherNeural',
-            },
-          ],
-        },
-      ],
-    })
-  )
-
   const digest = await redisDataSource.redisClient?.get(digestKey(userId))
   return digest ? (JSON.parse(digest) as Digest) : null
 }
+
+export const writeDigest = async (userId: string, digest: Digest) => {
+  // write to redis
+  const result = await redisDataSource.redisClient?.set(
+    digestKey(userId),
+    JSON.stringify(digest)
+  )
+
+  if (!result) {
+    const msg = `Error while writing digest to redis: ${userId}`
+    logger.error(msg)
+    throw new Error(msg)
+  }
+}
diff --git a/packages/api/src/utils/createTask.ts b/packages/api/src/utils/createTask.ts
index d64471252..cb835da13 100644
--- a/packages/api/src/utils/createTask.ts
+++ b/packages/api/src/utils/createTask.ts
@@ -16,6 +16,11 @@ import {
   CreateLabelInput,
 } from '../generated/graphql'
 import { AISummarizeJobData, AI_SUMMARIZE_JOB_NAME } from '../jobs/ai-summarize'
+import {
+  CreateDigestJobData,
+  CreateDigestJobResponse,
+  CREATE_DIGEST_JOB,
+} from '../jobs/ai/create_digest'
 import { BulkActionData, BULK_ACTION_JOB_NAME } from '../jobs/bulk_action'
 import { CallWebhookJobData, CALL_WEBHOOK_JOB_NAME } from '../jobs/call_webhook'
 import { THUMBNAIL_JOB } from '../jobs/find_thumbnail'
@@ -44,18 +49,8 @@ import {
   UPDATE_HIGHLIGHT_JOB,
   UPDATE_LABELS_JOB,
 } from '../jobs/update_db'
-import {
-  createJobId,
-  getBackendQueue,
-  JOB_VERSION,
-  LONG_RUNNING_QUEUE_NAME,
-} from '../queue-processor'
+import { createJobId, getBackendQueue, JOB_VERSION } from '../queue-processor'
 import { redisDataSource } from '../redis_data_source'
-import {
-  CreateDigestJobData,
-  CreateDigestJobResponse,
-  CREATE_DIGEST_JOB,
-} from '../services/digest'
 import { signFeatureToken } from '../services/features'
 import { OmnivoreAuthorizationHeader } from './auth'
 import { CreateTaskError } from './errors'
@@ -93,6 +88,7 @@ export const getJobPriority = (jobName: string): number => {
     case PROCESS_YOUTUBE_TRANSCRIPT_JOB_NAME:
     case `${REFRESH_FEED_JOB_NAME}_low`:
     case EXPORT_ITEM_JOB_NAME:
+    case CREATE_DIGEST_JOB:
       return 50
     case EXPORT_ALL_ITEMS_JOB_NAME:
     case REFRESH_ALL_FEEDS_JOB_NAME:
@@ -865,7 +861,7 @@ export const enqueueSendEmail = async (jobData: SendEmailJobData) => {
 export const enqueueCreateDigest = async (
   data: CreateDigestJobData
 ): Promise<CreateDigestJobResponse> => {
-  const queue = await getBackendQueue(LONG_RUNNING_QUEUE_NAME)
+  const queue = await getBackendQueue()
   if (!queue) {
     throw new Error('No queue found')
   }
@@ -876,6 +872,7 @@ export const enqueueCreateDigest = async (
     removeOnComplete: true,
     removeOnFail: true,
     attempts: 3,
+    priority: getJobPriority(CREATE_DIGEST_JOB),
   })
 
   logger.info('create digest job enqueued', { jobId: job.id })
diff --git a/packages/api/src/utils/parser.ts b/packages/api/src/utils/parser.ts
index 395363634..3122a72bd 100644
--- a/packages/api/src/utils/parser.ts
+++ b/packages/api/src/utils/parser.ts
@@ -703,6 +703,10 @@ export const htmlToMarkdown = (html: string) => {
   return nhm.translate(/* html */ html)
 }
 
+export const markdownToHtml = (markdown: string) => {
+  return nhm.translate(/* markdown */ markdown)
+}
+
 export const getDistillerResult = async (
   uid: string,
   html: string
diff --git a/yarn.lock b/yarn.lock
index 034c6ee5c..9c651f55f 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -32048,6 +32048,11 @@ yaml@^2.2.1:
   resolved "https://registry.yarnpkg.com/yaml/-/yaml-2.4.0.tgz#2376db1083d157f4b3a452995803dbcf43b08140"
   integrity sha512-j9iR8g+/t0lArF4V6NE/QCfT+CO7iLqrXAHZbJdo+LfjqP1vR8Fg5bSiaq6Q2lOD1AUEVrEVIgABvBFYojJVYQ==
 
+yaml@^2.4.1:
+  version "2.4.1"
+  resolved "https://registry.yarnpkg.com/yaml/-/yaml-2.4.1.tgz#2e57e0b5e995292c25c75d2658f0664765210eed"
+  integrity sha512-pIXzoImaqmfOrL7teGUBt/T7ZDnyeGBWyXQBvOVhLkWLN37GXv8NMLK406UY6dS51JfcQHsmcW5cJ441bHg6Lg==
+
 yargs-parser@20.2.4:
   version "20.2.4"
   resolved "https://registry.yarnpkg.com/yargs-parser/-/yargs-parser-20.2.4.tgz#b42890f14566796f85ae8e3a25290d205f154a54"

From 4bc1246a474bac66e3e5945a72b262e0020a0495 Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Tue, 16 Apr 2024 20:23:29 +0800
Subject: [PATCH 61/74] fixing html

---
 packages/api/src/jobs/ai/create_digest.ts | 58 +++++++++++++++++------
 packages/api/src/services/digest.ts       |  4 +-
 2 files changed, 46 insertions(+), 16 deletions(-)

diff --git a/packages/api/src/jobs/ai/create_digest.ts b/packages/api/src/jobs/ai/create_digest.ts
index feb12f78d..85c6423fb 100644
--- a/packages/api/src/jobs/ai/create_digest.ts
+++ b/packages/api/src/jobs/ai/create_digest.ts
@@ -14,6 +14,13 @@ import { JsonOutputParser } from '@langchain/core/output_parsers'
 import showdown from 'showdown'
 import { Digest, writeDigest } from '../../services/digest'
 
+export interface CreateDigestJobData {
+  userId: string
+}
+
+export interface CreateDigestJobResponse {
+  jobId: string
+}
 interface Selector {
   query: string
   count: number
@@ -37,12 +44,16 @@ interface DigestDefinition {
   zeroShot: ZeroShotDefinition
 }
 
-export interface CreateDigestJobData {
-  userId: string
+interface RankedItem {
+  topic: string
+  summary: string
+  libraryItem: LibraryItem
 }
 
-export interface CreateDigestJobResponse {
-  jobId: string
+interface RankedTitle {
+  topic: string
+  id: string
+  title: string
 }
 
 export const CREATE_DIGEST_JOB = 'create-digest'
@@ -178,18 +189,12 @@ const findOrCreateUserProfile = async (userId: string): Promise<string> => {
   const preferences = await getPreferencesList(userId)
   const profile = await createUserProfile(preferences)
 
-  // write to redis here
-  await redisDataSource.redisClient?.set(key, profile)
+  // write to redis here and ttl is 1 week
+  await redisDataSource.redisClient?.set(key, profile, 'EX', 60 * 60 * 24 * 7)
 
   return profile
 }
 
-type RankedItem = {
-  topic: string
-  summary: string
-  libraryItem: LibraryItem
-}
-
 // Uses OpenAI to rank all the titles based on the user profiles
 const rankCandidates = async (
   candidates: LibraryItem[],
@@ -210,10 +215,31 @@ const rankCandidates = async (
   const chain = contextualTemplate.pipe(llm).pipe(outputParser)
   const contextStr = await chain.invoke({
     userProfile,
-    titles: JSON.stringify(candidates.map((item) => item.title)),
+    titles: JSON.stringify(
+      candidates.map((item) => ({
+        id: item.id,
+        title: item.title,
+      }))
+    ),
   })
 
-  return contextStr as RankedItem[]
+  logger.info('contextStr: ', contextStr)
+  // convert the json output to an array of ranked candidates
+  const rankedCandidate = contextStr as RankedTitle[]
+
+  // map the ranked titles to the library items based on id
+  const rankedItems = rankedCandidate
+    .map((item) => {
+      const libraryItem = candidates.find((t) => t.id === item.id)
+      return {
+        topic: item.topic,
+        libraryItem,
+        summary: '',
+      }
+    })
+    .filter((item) => item.libraryItem !== undefined) as RankedItem[]
+
+  return rankedItems
 }
 
 // Does some grouping by topic while trying to maintain ranking
@@ -292,7 +318,9 @@ const generateSpeechFiles = (rankedItems: RankedItem[]): SpeechFile[] => {
   })
 
   const speechFiles = rankedItems.map((item) => {
-    const html = converter.makeHtml(item.summary)
+    const html = `<div id="readability-page-1">${converter.makeHtml(
+      item.summary
+    )}</div>`
     return htmlToSpeechFile({
       content: html,
       options: {},
diff --git a/packages/api/src/services/digest.ts b/packages/api/src/services/digest.ts
index 271e79765..e3b38d8b7 100644
--- a/packages/api/src/services/digest.ts
+++ b/packages/api/src/services/digest.ts
@@ -30,7 +30,9 @@ export const writeDigest = async (userId: string, digest: Digest) => {
   // write to redis
   const result = await redisDataSource.redisClient?.set(
     digestKey(userId),
-    JSON.stringify(digest)
+    JSON.stringify(digest),
+    'EX',
+    60 * 60 * 24 * 7 // 1 week
   )
 
   if (!result) {

From 6bfb4842e2a5f26ec20a24c84f19d642170a172a Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Tue, 16 Apr 2024 21:27:20 +0800
Subject: [PATCH 62/74] fix idx

---
 packages/api/src/jobs/ai/create_digest.ts | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/packages/api/src/jobs/ai/create_digest.ts b/packages/api/src/jobs/ai/create_digest.ts
index 85c6423fb..1c8bf442e 100644
--- a/packages/api/src/jobs/ai/create_digest.ts
+++ b/packages/api/src/jobs/ai/create_digest.ts
@@ -318,9 +318,12 @@ const generateSpeechFiles = (rankedItems: RankedItem[]): SpeechFile[] => {
   })
 
   const speechFiles = rankedItems.map((item) => {
-    const html = `<div id="readability-page-1">${converter.makeHtml(
-      item.summary
-    )}</div>`
+    const html = `
+    <div id="readability-content">
+      <div id="readability-page-1">
+        ${converter.makeHtml(item.summary)}
+      </div>
+    </div>`
     return htmlToSpeechFile({
       content: html,
       options: {},

From 47fd4d9663473f993e4d5df7c1f97ad7e31dcbcb Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Wed, 17 Apr 2024 09:58:11 +0800
Subject: [PATCH 63/74] save id, url, thumbnail and createdAt in the digest

---
 packages/api/src/jobs/ai/create_digest.ts |  6 ++++++
 packages/api/src/services/digest.ts       | 16 ++++++++++++----
 2 files changed, 18 insertions(+), 4 deletions(-)

diff --git a/packages/api/src/jobs/ai/create_digest.ts b/packages/api/src/jobs/ai/create_digest.ts
index 1c8bf442e..d264f08f1 100644
--- a/packages/api/src/jobs/ai/create_digest.ts
+++ b/packages/api/src/jobs/ai/create_digest.ts
@@ -365,6 +365,12 @@ export const createDigestJob = async (jobData: CreateDigestJobData) => {
     urlsToAudio: [],
     jobState: 'completed',
     speechFiles,
+    libraryItems: filteredSummaries.map((item) => ({
+      id: item.libraryItem.id,
+      url: item.libraryItem.originalUrl,
+      thumbnail: item.libraryItem.thumbnail ?? undefined,
+    })),
+    createdAt: new Date(),
   }
 
   await writeDigest(jobData.userId, digest)
diff --git a/packages/api/src/services/digest.ts b/packages/api/src/services/digest.ts
index e3b38d8b7..1c9d20a05 100644
--- a/packages/api/src/services/digest.ts
+++ b/packages/api/src/services/digest.ts
@@ -2,9 +2,20 @@ import { redisDataSource } from '../redis_data_source'
 import { SpeechFile } from '@omnivore/text-to-speech-handler'
 import { logger } from '../utils/logger'
 
+interface Chapter {
+  title: string
+}
+
+interface LibraryItem {
+  id: string
+  url: string
+  thumbnail?: string
+}
+
 export interface Digest {
   id: string
   jobState: string
+  createdAt: Date
 
   url?: string
   title?: string
@@ -13,10 +24,7 @@ export interface Digest {
 
   urlsToAudio?: string[]
   speechFiles?: SpeechFile[]
-}
-
-interface Chapter {
-  title: string
+  libraryItems?: LibraryItem[]
 }
 
 const digestKey = (userId: string) => `digest:${userId}`

From 6a4faf6f3c43d6e83a41e61846fd966e3d2d2f61 Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Wed, 17 Apr 2024 10:04:52 +0800
Subject: [PATCH 64/74] add description to the digest

---
 packages/api/src/jobs/ai/create_digest.ts | 12 ++++++++++--
 packages/api/src/services/digest.ts       |  4 +++-
 2 files changed, 13 insertions(+), 3 deletions(-)

diff --git a/packages/api/src/jobs/ai/create_digest.ts b/packages/api/src/jobs/ai/create_digest.ts
index d264f08f1..a7c0e02c4 100644
--- a/packages/api/src/jobs/ai/create_digest.ts
+++ b/packages/api/src/jobs/ai/create_digest.ts
@@ -13,6 +13,7 @@ import yaml from 'yaml'
 import { JsonOutputParser } from '@langchain/core/output_parsers'
 import showdown from 'showdown'
 import { Digest, writeDigest } from '../../services/digest'
+import { TaskState } from '../../generated/graphql'
 
 export interface CreateDigestJobData {
   userId: string
@@ -344,6 +345,12 @@ const generateTitle = (selections: RankedItem[]): string =>
   'Omnivore digest: ' +
   selections.map((item) => item.libraryItem.title).join(',')
 
+// TODO: generate description based on the summaries
+const generateDescription = (selections: RankedItem[]): string => 'description'
+
+// TODO: generate content based on the summaries
+const generateContent = (selections: RankedItem[]): string => 'content'
+
 export const createDigestJob = async (jobData: CreateDigestJobData) => {
   digestDefinition = await fetchDigestDefinition()
 
@@ -361,9 +368,9 @@ export const createDigestJob = async (jobData: CreateDigestJobData) => {
   const digest: Digest = {
     id: uuid(),
     title,
-    content: 'content',
+    content: generateContent(summaries),
     urlsToAudio: [],
-    jobState: 'completed',
+    jobState: TaskState.Succeeded,
     speechFiles,
     libraryItems: filteredSummaries.map((item) => ({
       id: item.libraryItem.id,
@@ -371,6 +378,7 @@ export const createDigestJob = async (jobData: CreateDigestJobData) => {
       thumbnail: item.libraryItem.thumbnail ?? undefined,
     })),
     createdAt: new Date(),
+    description: generateDescription(summaries),
   }
 
   await writeDigest(jobData.userId, digest)
diff --git a/packages/api/src/services/digest.ts b/packages/api/src/services/digest.ts
index 1c9d20a05..2149fade9 100644
--- a/packages/api/src/services/digest.ts
+++ b/packages/api/src/services/digest.ts
@@ -1,6 +1,7 @@
 import { redisDataSource } from '../redis_data_source'
 import { SpeechFile } from '@omnivore/text-to-speech-handler'
 import { logger } from '../utils/logger'
+import { TaskState } from '../generated/graphql'
 
 interface Chapter {
   title: string
@@ -14,8 +15,9 @@ interface LibraryItem {
 
 export interface Digest {
   id: string
-  jobState: string
+  jobState: TaskState
   createdAt: Date
+  description: string
 
   url?: string
   title?: string

From 259b778aba181abdde06a2d92dc331b75017e54d Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Wed, 17 Apr 2024 12:16:15 +0800
Subject: [PATCH 65/74] add schedule and other parameters

---
 packages/api/src/jobs/ai/create_digest.ts | 28 +++++++++++++++++++----
 packages/api/src/routers/digest_router.ts | 24 +++++++++++++++----
 packages/api/src/services/digest.ts       |  5 +---
 packages/api/src/utils/createTask.ts      | 10 +++++++-
 4 files changed, 53 insertions(+), 14 deletions(-)

diff --git a/packages/api/src/jobs/ai/create_digest.ts b/packages/api/src/jobs/ai/create_digest.ts
index a7c0e02c4..fa7a975ff 100644
--- a/packages/api/src/jobs/ai/create_digest.ts
+++ b/packages/api/src/jobs/ai/create_digest.ts
@@ -4,7 +4,11 @@ import { v4 as uuid } from 'uuid'
 import { OpenAI } from '@langchain/openai'
 import { PromptTemplate } from '@langchain/core/prompts'
 import { LibraryItem } from '../../entity/library_item'
-import { htmlToSpeechFile, SpeechFile } from '@omnivore/text-to-speech-handler'
+import {
+  htmlToSpeechFile,
+  SpeechFile,
+  SSMLOptions,
+} from '@omnivore/text-to-speech-handler'
 import axios from 'axios'
 import { searchLibraryItems } from '../../services/library_item'
 import { redisDataSource } from '../../redis_data_source'
@@ -15,8 +19,13 @@ import showdown from 'showdown'
 import { Digest, writeDigest } from '../../services/digest'
 import { TaskState } from '../../generated/graphql'
 
+export type CreateDigestJobSchedule = 'daily' | 'weekly'
+
 export interface CreateDigestJobData {
   userId: string
+  voices?: string[]
+  language?: string
+  rate?: string
 }
 
 export interface CreateDigestJobResponse {
@@ -312,7 +321,10 @@ const summarizeItems = async (
 }
 
 // generate speech files from the summaries
-const generateSpeechFiles = (rankedItems: RankedItem[]): SpeechFile[] => {
+const generateSpeechFiles = (
+  rankedItems: RankedItem[],
+  options: SSMLOptions
+): SpeechFile[] => {
   // convert the summaries from markdown to HTML
   const converter = new showdown.Converter({
     backslashEscapesHTMLTags: true,
@@ -327,7 +339,7 @@ const generateSpeechFiles = (rankedItems: RankedItem[]): SpeechFile[] => {
     </div>`
     return htmlToSpeechFile({
       content: html,
-      options: {},
+      options,
     })
   })
 
@@ -363,7 +375,11 @@ export const createDigestJob = async (jobData: CreateDigestJobData) => {
 
   const filteredSummaries = filterSummaries(summaries)
 
-  const speechFiles = generateSpeechFiles(filteredSummaries)
+  const speechFiles = generateSpeechFiles(filteredSummaries, {
+    ...jobData,
+    primaryVoice: jobData.voices?.[0],
+    secondaryVoice: jobData.voices?.[1],
+  })
   const title = generateTitle(summaries)
   const digest: Digest = {
     id: uuid(),
@@ -372,10 +388,12 @@ export const createDigestJob = async (jobData: CreateDigestJobData) => {
     urlsToAudio: [],
     jobState: TaskState.Succeeded,
     speechFiles,
-    libraryItems: filteredSummaries.map((item) => ({
+    chapters: filteredSummaries.map((item, index) => ({
+      title: item.libraryItem.title,
       id: item.libraryItem.id,
       url: item.libraryItem.originalUrl,
       thumbnail: item.libraryItem.thumbnail ?? undefined,
+      wordCount: speechFiles[index].wordCount,
     })),
     createdAt: new Date(),
     description: generateDescription(summaries),
diff --git a/packages/api/src/routers/digest_router.ts b/packages/api/src/routers/digest_router.ts
index 0a7de6cf9..c415ea6c8 100644
--- a/packages/api/src/routers/digest_router.ts
+++ b/packages/api/src/routers/digest_router.ts
@@ -1,7 +1,10 @@
 import cors from 'cors'
 import express from 'express'
 import { env } from '../env'
-import { CREATE_DIGEST_JOB } from '../jobs/ai/create_digest'
+import {
+  CreateDigestJobSchedule,
+  CREATE_DIGEST_JOB,
+} from '../jobs/ai/create_digest'
 import { createJobId, getJob, jobStateToTaskState } from '../queue-processor'
 import { getDigest } from '../services/digest'
 import { findActiveUser } from '../services/user'
@@ -32,6 +35,13 @@ const isFeedback = (data: any): data is Feedback => {
   )
 }
 
+interface CreateDigestRequest {
+  voices?: string[]
+  language?: string
+  rate?: string
+  schedule?: CreateDigestJobSchedule
+}
+
 export function digestRouter() {
   const router = express.Router()
 
@@ -72,10 +82,16 @@ export function digestRouter() {
         return res.sendStatus(202)
       }
 
+      const data = req.body as CreateDigestRequest
+
       // enqueue job and return job id
-      const result = await enqueueCreateDigest({
-        userId,
-      })
+      const result = await enqueueCreateDigest(
+        {
+          userId,
+          ...data,
+        },
+        data.schedule
+      )
 
       // return job id
       return res.status(201).send(result)
diff --git a/packages/api/src/services/digest.ts b/packages/api/src/services/digest.ts
index 2149fade9..85f239f1f 100644
--- a/packages/api/src/services/digest.ts
+++ b/packages/api/src/services/digest.ts
@@ -5,11 +5,9 @@ import { TaskState } from '../generated/graphql'
 
 interface Chapter {
   title: string
-}
-
-interface LibraryItem {
   id: string
   url: string
+  wordCount: number
   thumbnail?: string
 }
 
@@ -26,7 +24,6 @@ export interface Digest {
 
   urlsToAudio?: string[]
   speechFiles?: SpeechFile[]
-  libraryItems?: LibraryItem[]
 }
 
 const digestKey = (userId: string) => `digest:${userId}`
diff --git a/packages/api/src/utils/createTask.ts b/packages/api/src/utils/createTask.ts
index cb835da13..3c6b7ccf9 100644
--- a/packages/api/src/utils/createTask.ts
+++ b/packages/api/src/utils/createTask.ts
@@ -19,6 +19,7 @@ import { AISummarizeJobData, AI_SUMMARIZE_JOB_NAME } from '../jobs/ai-summarize'
 import {
   CreateDigestJobData,
   CreateDigestJobResponse,
+  CreateDigestJobSchedule,
   CREATE_DIGEST_JOB,
 } from '../jobs/ai/create_digest'
 import { BulkActionData, BULK_ACTION_JOB_NAME } from '../jobs/bulk_action'
@@ -859,7 +860,8 @@ export const enqueueSendEmail = async (jobData: SendEmailJobData) => {
 }
 
 export const enqueueCreateDigest = async (
-  data: CreateDigestJobData
+  data: CreateDigestJobData,
+  schedule?: CreateDigestJobSchedule
 ): Promise<CreateDigestJobResponse> => {
   const queue = await getBackendQueue()
   if (!queue) {
@@ -873,6 +875,12 @@ export const enqueueCreateDigest = async (
     removeOnFail: true,
     attempts: 3,
     priority: getJobPriority(CREATE_DIGEST_JOB),
+    repeat: schedule
+      ? {
+          immediately: true, // run immediately
+          pattern: schedule === 'daily' ? '0 0 * * *' : '0 0 * * 7', // every day at midnight or every Sunday at midnight
+        }
+      : undefined,
   })
 
   logger.info('create digest job enqueued', { jobId: job.id })

From b2a55a78d3a5aeed7039a459c52491075cbe6275 Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Wed, 17 Apr 2024 12:53:29 +0800
Subject: [PATCH 66/74] check if feature is granted

---
 packages/api/src/routers/digest_router.ts | 28 +++++++++++++++++++++++
 packages/api/src/services/features.ts     |  4 ++++
 2 files changed, 32 insertions(+)

diff --git a/packages/api/src/routers/digest_router.ts b/packages/api/src/routers/digest_router.ts
index c415ea6c8..0bf492d0a 100644
--- a/packages/api/src/routers/digest_router.ts
+++ b/packages/api/src/routers/digest_router.ts
@@ -7,6 +7,7 @@ import {
 } from '../jobs/ai/create_digest'
 import { createJobId, getJob, jobStateToTaskState } from '../queue-processor'
 import { getDigest } from '../services/digest'
+import { FeatureName, findGrantedFeatureByName } from '../services/features'
 import { findActiveUser } from '../services/user'
 import { analytics } from '../utils/analytics'
 import { getClaimsByToken, getTokenByRequest } from '../utils/auth'
@@ -72,6 +73,15 @@ export function digestRouter() {
         return res.sendStatus(401)
       }
 
+      const feature = await findGrantedFeatureByName(
+        FeatureName.AIDigest,
+        userId
+      )
+      if (!feature) {
+        logger.info(`${FeatureName.AIDigest} not granted: ${userId}`)
+        return res.sendStatus(403)
+      }
+
       // check if job is already in queue
       // if yes then return 202 accepted
       // else enqueue job
@@ -128,6 +138,15 @@ export function digestRouter() {
         return res.sendStatus(401)
       }
 
+      const feature = await findGrantedFeatureByName(
+        FeatureName.AIDigest,
+        userId
+      )
+      if (!feature) {
+        logger.info(`${FeatureName.AIDigest} not granted: ${userId}`)
+        return res.sendStatus(403)
+      }
+
       // get job by user id
       const jobId = createJobId(CREATE_DIGEST_JOB, userId)
       const job = await getJob(jobId)
@@ -185,6 +204,15 @@ export function digestRouter() {
           return res.sendStatus(401)
         }
 
+        const feature = await findGrantedFeatureByName(
+          FeatureName.AIDigest,
+          userId
+        )
+        if (!feature) {
+          logger.info(`${FeatureName.AIDigest} not granted: ${userId}`)
+          return res.sendStatus(403)
+        }
+
         // get feedback from request body
         if (!isFeedback(req.body)) {
           logger.info('Invalid feedback format')
diff --git a/packages/api/src/services/features.ts b/packages/api/src/services/features.ts
index 664461a50..d22e65c74 100644
--- a/packages/api/src/services/features.ts
+++ b/packages/api/src/services/features.ts
@@ -9,12 +9,14 @@ import { logger } from '../utils/logger'
 const MAX_ULTRA_REALISTIC_USERS = 1500
 const MAX_YOUTUBE_TRANSCRIPT_USERS = 500
 const MAX_NOTION_USERS = 1000
+const MAX_AIDIGEST_USERS = 5
 
 export enum FeatureName {
   AISummaries = 'ai-summaries',
   YouTubeTranscripts = 'youtube-transcripts',
   UltraRealisticVoice = 'ultra-realistic-voice',
   Notion = 'notion',
+  AIDigest = 'ai-digest',
 }
 
 export const getFeatureName = (name: string): FeatureName | undefined => {
@@ -40,6 +42,8 @@ export const optInFeature = async (
       )
     case FeatureName.Notion:
       return optInLimitedFeature(FeatureName.Notion, uid, MAX_NOTION_USERS)
+    case FeatureName.AIDigest:
+      return optInLimitedFeature(FeatureName.AIDigest, uid, MAX_AIDIGEST_USERS)
     default:
       return undefined
   }

From 448e4328073e60b56fdbaeb99537e6fd392b31f8 Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Wed, 17 Apr 2024 13:03:48 +0800
Subject: [PATCH 67/74] runs at 1pm pst

---
 packages/api/src/utils/createTask.ts | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/packages/api/src/utils/createTask.ts b/packages/api/src/utils/createTask.ts
index 3c6b7ccf9..70306ff76 100644
--- a/packages/api/src/utils/createTask.ts
+++ b/packages/api/src/utils/createTask.ts
@@ -878,7 +878,8 @@ export const enqueueCreateDigest = async (
     repeat: schedule
       ? {
           immediately: true, // run immediately
-          pattern: schedule === 'daily' ? '0 0 * * *' : '0 0 * * 7', // every day at midnight or every Sunday at midnight
+          pattern: schedule === 'daily' ? '0 13 * * *' : '0 13 * * 7', // every day or every Sunday at 1PM
+          utc: true,
         }
       : undefined,
   })

From 6222d567a4444fe9e556f8dcb88e1e0d3f7e0daf Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Wed, 17 Apr 2024 14:32:07 +0800
Subject: [PATCH 68/74] add byline to the digest

---
 packages/api/src/jobs/ai/create_digest.ts | 12 ++++++++----
 packages/api/src/services/digest.ts       |  1 +
 2 files changed, 9 insertions(+), 4 deletions(-)

diff --git a/packages/api/src/jobs/ai/create_digest.ts b/packages/api/src/jobs/ai/create_digest.ts
index fa7a975ff..e424b679c 100644
--- a/packages/api/src/jobs/ai/create_digest.ts
+++ b/packages/api/src/jobs/ai/create_digest.ts
@@ -353,15 +353,18 @@ const filterSummaries = (summaries: RankedItem[]): RankedItem[] => {
 }
 
 // TODO: we can use something more sophisticated to generate titles
-const generateTitle = (selections: RankedItem[]): string =>
+const generateTitle = (summaries: RankedItem[]): string =>
   'Omnivore digest: ' +
-  selections.map((item) => item.libraryItem.title).join(',')
+  summaries.map((item) => item.libraryItem.title).join(',')
 
 // TODO: generate description based on the summaries
-const generateDescription = (selections: RankedItem[]): string => 'description'
+const generateDescription = (summaries: RankedItem[]): string => 'description'
 
 // TODO: generate content based on the summaries
-const generateContent = (selections: RankedItem[]): string => 'content'
+const generateContent = (summaries: RankedItem[]): string => 'content'
+
+const generateByline = (summaries: RankedItem[]): string =>
+  summaries.map((item) => item.libraryItem.author).join(',')
 
 export const createDigestJob = async (jobData: CreateDigestJobData) => {
   digestDefinition = await fetchDigestDefinition()
@@ -397,6 +400,7 @@ export const createDigestJob = async (jobData: CreateDigestJobData) => {
     })),
     createdAt: new Date(),
     description: generateDescription(summaries),
+    byline: generateByline(summaries),
   }
 
   await writeDigest(jobData.userId, digest)
diff --git a/packages/api/src/services/digest.ts b/packages/api/src/services/digest.ts
index 85f239f1f..24be606cc 100644
--- a/packages/api/src/services/digest.ts
+++ b/packages/api/src/services/digest.ts
@@ -16,6 +16,7 @@ export interface Digest {
   jobState: TaskState
   createdAt: Date
   description: string
+  byline: string
 
   url?: string
   title?: string

From af184ded22e80608416d437f65eee90a0baa1851 Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Wed, 17 Apr 2024 15:36:16 +0800
Subject: [PATCH 69/74] dedupe repeated jobs

---
 packages/api/src/jobs/ai/create_digest.ts |  4 +-
 packages/api/src/routers/digest_router.ts | 46 ++++++++++-------------
 packages/api/src/services/digest.ts       |  6 +--
 packages/api/src/utils/createTask.ts      | 18 +++++++--
 4 files changed, 40 insertions(+), 34 deletions(-)

diff --git a/packages/api/src/jobs/ai/create_digest.ts b/packages/api/src/jobs/ai/create_digest.ts
index e424b679c..f5d636709 100644
--- a/packages/api/src/jobs/ai/create_digest.ts
+++ b/packages/api/src/jobs/ai/create_digest.ts
@@ -22,6 +22,7 @@ import { TaskState } from '../../generated/graphql'
 export type CreateDigestJobSchedule = 'daily' | 'weekly'
 
 export interface CreateDigestJobData {
+  id: string
   userId: string
   voices?: string[]
   language?: string
@@ -30,6 +31,7 @@ export interface CreateDigestJobData {
 
 export interface CreateDigestJobResponse {
   jobId: string
+  jobState: TaskState
 }
 interface Selector {
   query: string
@@ -385,7 +387,7 @@ export const createDigestJob = async (jobData: CreateDigestJobData) => {
   })
   const title = generateTitle(summaries)
   const digest: Digest = {
-    id: uuid(),
+    id: jobData.id,
     title,
     content: generateContent(summaries),
     urlsToAudio: [],
diff --git a/packages/api/src/routers/digest_router.ts b/packages/api/src/routers/digest_router.ts
index 0bf492d0a..24f2bb103 100644
--- a/packages/api/src/routers/digest_router.ts
+++ b/packages/api/src/routers/digest_router.ts
@@ -1,11 +1,8 @@
 import cors from 'cors'
 import express from 'express'
 import { env } from '../env'
-import {
-  CreateDigestJobSchedule,
-  CREATE_DIGEST_JOB,
-} from '../jobs/ai/create_digest'
-import { createJobId, getJob, jobStateToTaskState } from '../queue-processor'
+import { TaskState } from '../generated/graphql'
+import { CreateDigestJobSchedule } from '../jobs/ai/create_digest'
 import { getDigest } from '../services/digest'
 import { FeatureName, findGrantedFeatureByName } from '../services/features'
 import { findActiveUser } from '../services/user'
@@ -14,6 +11,7 @@ import { getClaimsByToken, getTokenByRequest } from '../utils/auth'
 import { corsConfig } from '../utils/corsConfig'
 import { enqueueCreateDigest } from '../utils/createTask'
 import { logger } from '../utils/logger'
+import { v4 as uuid } from 'uuid'
 
 interface Feedback {
   digestRating: number
@@ -82,21 +80,21 @@ export function digestRouter() {
         return res.sendStatus(403)
       }
 
-      // check if job is already in queue
+      const data = req.body as CreateDigestRequest
+
+      // check if job is running
       // if yes then return 202 accepted
       // else enqueue job
-      const jobId = createJobId(CREATE_DIGEST_JOB, userId)
-      const existingJob = await getJob(jobId)
-      if (existingJob) {
-        logger.info(`Job already in queue: ${jobId}`)
+      const digest = await getDigest(userId)
+      if (digest?.jobState === TaskState.Running) {
+        logger.info(`Digest job is running: ${userId}`)
         return res.sendStatus(202)
       }
 
-      const data = req.body as CreateDigestRequest
-
       // enqueue job and return job id
       const result = await enqueueCreateDigest(
         {
+          id: uuid(), // generate job id
           userId,
           ...data,
         },
@@ -147,26 +145,22 @@ export function digestRouter() {
         return res.sendStatus(403)
       }
 
-      // get job by user id
-      const jobId = createJobId(CREATE_DIGEST_JOB, userId)
-      const job = await getJob(jobId)
-      if (job) {
-        // if job is in queue then return job state
-        const jobState = await job.getState()
-        return res.send({
-          jobId: job.id,
-          jobState: jobStateToTaskState(jobState),
-        })
-      }
-
-      // if job is done and removed then get the digest from redis
+      // get the digest from redis
       const digest = await getDigest(userId)
       if (!digest) {
         logger.info(`Digest not found: ${userId}`)
         return res.sendStatus(404)
       }
 
-      // return digest
+      if (digest.jobState === TaskState.Running) {
+        // if job is running then return job state
+        return res.send({
+          jobId: digest.id,
+          jobState: digest.jobState,
+        })
+      }
+
+      // if job is done then return the digest
       return res.send(digest)
     } catch (error) {
       logger.error('Error while getting digest', error)
diff --git a/packages/api/src/services/digest.ts b/packages/api/src/services/digest.ts
index 24be606cc..811202efd 100644
--- a/packages/api/src/services/digest.ts
+++ b/packages/api/src/services/digest.ts
@@ -14,10 +14,10 @@ interface Chapter {
 export interface Digest {
   id: string
   jobState: TaskState
-  createdAt: Date
-  description: string
-  byline: string
 
+  createdAt?: Date
+  description?: string
+  byline?: string
   url?: string
   title?: string
   content?: string
diff --git a/packages/api/src/utils/createTask.ts b/packages/api/src/utils/createTask.ts
index 70306ff76..eeefd3692 100644
--- a/packages/api/src/utils/createTask.ts
+++ b/packages/api/src/utils/createTask.ts
@@ -14,6 +14,7 @@ import { env } from '../env'
 import {
   ArticleSavingRequestStatus,
   CreateLabelInput,
+  TaskState,
 } from '../generated/graphql'
 import { AISummarizeJobData, AI_SUMMARIZE_JOB_NAME } from '../jobs/ai-summarize'
 import {
@@ -50,7 +51,7 @@ import {
   UPDATE_HIGHLIGHT_JOB,
   UPDATE_LABELS_JOB,
 } from '../jobs/update_db'
-import { createJobId, getBackendQueue, JOB_VERSION } from '../queue-processor'
+import { getBackendQueue, JOB_VERSION } from '../queue-processor'
 import { redisDataSource } from '../redis_data_source'
 import { signFeatureToken } from '../services/features'
 import { OmnivoreAuthorizationHeader } from './auth'
@@ -58,6 +59,7 @@ import { CreateTaskError } from './errors'
 import { stringToHash } from './helpers'
 import { logger } from './logger'
 import View = google.cloud.tasks.v2.Task.View
+import { writeDigest } from '../services/digest'
 
 // Instantiates a client.
 const client = new CloudTasksClient()
@@ -868,9 +870,8 @@ export const enqueueCreateDigest = async (
     throw new Error('No queue found')
   }
 
-  const jobId = createJobId(CREATE_DIGEST_JOB, data.userId)
   const job = await queue.add(CREATE_DIGEST_JOB, data, {
-    jobId, // dedupe by userId
+    jobId: data.id, // dedupe by job id
     removeOnComplete: true,
     removeOnFail: true,
     attempts: 3,
@@ -886,8 +887,17 @@ export const enqueueCreateDigest = async (
 
   logger.info('create digest job enqueued', { jobId: job.id })
 
+  const digest = {
+    id: data.id,
+    jobState: TaskState.Running,
+  }
+
+  // update digest job state in redis
+  await writeDigest(data.userId, digest)
+
   return {
-    jobId,
+    jobId: digest.id,
+    jobState: digest.jobState,
   }
 }
 

From 57cb8978390fe4f26e777e7a356fb784aac8a02f Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Wed, 17 Apr 2024 15:45:05 +0800
Subject: [PATCH 70/74] generate content

---
 packages/api/src/jobs/ai/create_digest.ts | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/packages/api/src/jobs/ai/create_digest.ts b/packages/api/src/jobs/ai/create_digest.ts
index f5d636709..8646cb405 100644
--- a/packages/api/src/jobs/ai/create_digest.ts
+++ b/packages/api/src/jobs/ai/create_digest.ts
@@ -362,11 +362,17 @@ const generateTitle = (summaries: RankedItem[]): string =>
 // TODO: generate description based on the summaries
 const generateDescription = (summaries: RankedItem[]): string => 'description'
 
-// TODO: generate content based on the summaries
-const generateContent = (summaries: RankedItem[]): string => 'content'
+// generate content based on the summaries
+const generateContent = (summaries: RankedItem[]): string =>
+  summaries
+    .map((summary) => `## ${summary.libraryItem.title}\n ${summary.summary}`)
+    .join('\n\n')
 
 const generateByline = (summaries: RankedItem[]): string =>
-  summaries.map((item) => item.libraryItem.author).join(',')
+  summaries
+    .filter((summary) => !!summary.libraryItem.author)
+    .map((item) => item.libraryItem.author)
+    .join(',')
 
 export const createDigestJob = async (jobData: CreateDigestJobData) => {
   digestDefinition = await fetchDigestDefinition()

From aeda4f5d5d6558ded75476910b1aabd68531a8eb Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Wed, 17 Apr 2024 16:20:00 +0800
Subject: [PATCH 71/74] add libraryItemIds to the payload

---
 packages/api/src/jobs/ai/create_digest.ts | 15 +++++++++++--
 packages/api/src/routers/digest_router.ts |  1 +
 packages/api/src/services/library_item.ts | 27 +++++++++++++----------
 3 files changed, 29 insertions(+), 14 deletions(-)

diff --git a/packages/api/src/jobs/ai/create_digest.ts b/packages/api/src/jobs/ai/create_digest.ts
index 8646cb405..41aa5b7ec 100644
--- a/packages/api/src/jobs/ai/create_digest.ts
+++ b/packages/api/src/jobs/ai/create_digest.ts
@@ -10,7 +10,10 @@ import {
   SSMLOptions,
 } from '@omnivore/text-to-speech-handler'
 import axios from 'axios'
-import { searchLibraryItems } from '../../services/library_item'
+import {
+  findLibraryItemsByIds,
+  searchLibraryItems,
+} from '../../services/library_item'
 import { redisDataSource } from '../../redis_data_source'
 import { htmlToMarkdown } from '../../utils/parser'
 import yaml from 'yaml'
@@ -27,6 +30,7 @@ export interface CreateDigestJobData {
   voices?: string[]
   language?: string
   rate?: string
+  libraryItemIds?: string[]
 }
 
 export interface CreateDigestJobResponse {
@@ -125,7 +129,10 @@ const getPreferencesList = async (userId: string): Promise<LibraryItem[]> => {
 }
 
 // Makes multiple DB queries and combines the results
-const getCandidatesList = async (userId: string): Promise<LibraryItem[]> => {
+const getCandidatesList = async (
+  userId: string,
+  libraryItemIds?: string[]
+): Promise<LibraryItem[]> => {
   // use the queries from the digest definitions to lookup preferences
   // There should be a list of multiple queries we use. For now we can
   // hardcode these queries:
@@ -133,6 +140,10 @@ const getCandidatesList = async (userId: string): Promise<LibraryItem[]> => {
   //   count: 100
   //   reason: "most recent 100 items saved over 500 words
 
+  if (libraryItemIds) {
+    return findLibraryItemsByIds(libraryItemIds, userId)
+  }
+
   const candidates = await Promise.all(
     digestDefinition.candidateSelectors.map(async (selector) => {
       // use the selector to fetch items
diff --git a/packages/api/src/routers/digest_router.ts b/packages/api/src/routers/digest_router.ts
index 24f2bb103..2cc8649c7 100644
--- a/packages/api/src/routers/digest_router.ts
+++ b/packages/api/src/routers/digest_router.ts
@@ -39,6 +39,7 @@ interface CreateDigestRequest {
   language?: string
   rate?: string
   schedule?: CreateDigestJobSchedule
+  libraryItemIds?: string[]
 }
 
 export function digestRouter() {
diff --git a/packages/api/src/services/library_item.ts b/packages/api/src/services/library_item.ts
index 9adcde7b0..647b248db 100644
--- a/packages/api/src/services/library_item.ts
+++ b/packages/api/src/services/library_item.ts
@@ -21,6 +21,7 @@ import { redisDataSource } from '../redis_data_source'
 import {
   authTrx,
   getColumns,
+  getColumnsDbName,
   getRepository,
   queryBuilderToRawSql,
 } from '../repository'
@@ -625,13 +626,11 @@ export const buildQuery = (
   userId: string
 ) => {
   // select all columns except content
-  const selects: Select[] = getColumns(libraryItemRepository)
-    .map((column) => ({ column: `library_item.${column}` }))
+  const selects: Select[] = getColumnsDbName(libraryItemRepository)
     .filter(
-      (select) =>
-        select.column !== 'library_item.readableContent' &&
-        select.column !== 'library_item.originalContent'
+      (select) => select !== 'readable_content' && select !== 'original_content'
     )
+    .map((column) => ({ column: `library_item.${column}` }))
 
   const parameters: ObjectLiteral[] = []
   const orders: Sort[] = []
@@ -652,10 +651,18 @@ export const buildQuery = (
   queryBuilder.where('library_item.user_id = :userId', { userId })
 
   // add select
-  selects.forEach((select) => {
+  selects.forEach((select, index) => {
+    if (index === 0) {
+      queryBuilder.select(select.column, select.alias)
+    }
+
     queryBuilder.addSelect(select.column, select.alias)
   })
 
+  if (args.includeContent) {
+    queryBuilder.addSelect('library_item.readable_content')
+  }
+
   if (!args.includePending) {
     queryBuilder.andWhere("library_item.state <> 'PROCESSING'")
   }
@@ -754,18 +761,14 @@ export const findRecentLibraryItems = async (
 }
 
 export const findLibraryItemsByIds = async (ids: string[], userId: string) => {
-  const selectColumns = getColumns(libraryItemRepository)
-    .filter(
-      (column) => column !== 'readableContent' && column !== 'originalContent'
-    )
+  const selectColumns = getColumnsDbName(libraryItemRepository)
+    .filter((column) => column !== 'original_content')
     .map((column) => `library_item.${column}`)
   return authTrx(
     async (tx) =>
       tx
         .createQueryBuilder(LibraryItem, 'library_item')
         .select(selectColumns)
-        .leftJoinAndSelect('library_item.labels', 'labels')
-        .leftJoinAndSelect('library_item.highlights', 'highlights')
         .where('library_item.id IN (:...ids)', { ids })
         .getMany(),
     undefined,

From 0eb6460becfc9bdf05b11ca218cecf719214b680 Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Wed, 17 Apr 2024 16:35:29 +0800
Subject: [PATCH 72/74] fix tests

---
 packages/api/src/jobs/ai/create_digest.ts | 17 +++++++++++------
 packages/api/src/services/library_item.ts | 11 +++++------
 2 files changed, 16 insertions(+), 12 deletions(-)

diff --git a/packages/api/src/jobs/ai/create_digest.ts b/packages/api/src/jobs/ai/create_digest.ts
index 41aa5b7ec..686efefd2 100644
--- a/packages/api/src/jobs/ai/create_digest.ts
+++ b/packages/api/src/jobs/ai/create_digest.ts
@@ -359,19 +359,24 @@ const generateSpeechFiles = (
   return speechFiles
 }
 
-// TODO: we should have a QA step here that does some
+// we should have a QA step here that does some
 // basic checks to make sure the summaries are good.
 const filterSummaries = (summaries: RankedItem[]): RankedItem[] => {
   return summaries.filter((item) => item.summary.length > 100)
 }
 
-// TODO: we can use something more sophisticated to generate titles
+// we can use something more sophisticated to generate titles
 const generateTitle = (summaries: RankedItem[]): string =>
   'Omnivore digest: ' +
-  summaries.map((item) => item.libraryItem.title).join(',')
+  summaries.map((item) => item.libraryItem.title).join(', ')
 
-// TODO: generate description based on the summaries
-const generateDescription = (summaries: RankedItem[]): string => 'description'
+// generate description based on the summaries
+const generateDescription = (summaries: RankedItem[]): string =>
+  `We selected ${
+    summaries.length
+  } articles from your last 24 hours of saved items, covering ${summaries
+    .map((summary) => summary.topic)
+    .join(', ')}.`
 
 // generate content based on the summaries
 const generateContent = (summaries: RankedItem[]): string =>
@@ -383,7 +388,7 @@ const generateByline = (summaries: RankedItem[]): string =>
   summaries
     .filter((summary) => !!summary.libraryItem.author)
     .map((item) => item.libraryItem.author)
-    .join(',')
+    .join(', ')
 
 export const createDigestJob = async (jobData: CreateDigestJobData) => {
   digestDefinition = await fetchDigestDefinition()
diff --git a/packages/api/src/services/library_item.ts b/packages/api/src/services/library_item.ts
index 647b248db..832aea3fa 100644
--- a/packages/api/src/services/library_item.ts
+++ b/packages/api/src/services/library_item.ts
@@ -21,7 +21,6 @@ import { redisDataSource } from '../redis_data_source'
 import {
   authTrx,
   getColumns,
-  getColumnsDbName,
   getRepository,
   queryBuilderToRawSql,
 } from '../repository'
@@ -626,9 +625,9 @@ export const buildQuery = (
   userId: string
 ) => {
   // select all columns except content
-  const selects: Select[] = getColumnsDbName(libraryItemRepository)
+  const selects: Select[] = getColumns(libraryItemRepository)
     .filter(
-      (select) => select !== 'readable_content' && select !== 'original_content'
+      (select) => select !== 'readableContent' && select !== 'originalContent'
     )
     .map((column) => ({ column: `library_item.${column}` }))
 
@@ -660,7 +659,7 @@ export const buildQuery = (
   })
 
   if (args.includeContent) {
-    queryBuilder.addSelect('library_item.readable_content')
+    queryBuilder.addSelect('library_item.readableContent')
   }
 
   if (!args.includePending) {
@@ -761,8 +760,8 @@ export const findRecentLibraryItems = async (
 }
 
 export const findLibraryItemsByIds = async (ids: string[], userId: string) => {
-  const selectColumns = getColumnsDbName(libraryItemRepository)
-    .filter((column) => column !== 'original_content')
+  const selectColumns = getColumns(libraryItemRepository)
+    .filter((column) => column !== 'originalContent')
     .map((column) => `library_item.${column}`)
   return authTrx(
     async (tx) =>

From 3e51aa7c03c6cd8368867b8dd311e35403a5fb1b Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Wed, 17 Apr 2024 16:49:19 +0800
Subject: [PATCH 73/74] generate description

---
 packages/api/src/jobs/ai/create_digest.ts | 26 +++++++++++++++--------
 1 file changed, 17 insertions(+), 9 deletions(-)

diff --git a/packages/api/src/jobs/ai/create_digest.ts b/packages/api/src/jobs/ai/create_digest.ts
index 686efefd2..33012507f 100644
--- a/packages/api/src/jobs/ai/create_digest.ts
+++ b/packages/api/src/jobs/ai/create_digest.ts
@@ -141,6 +141,7 @@ const getCandidatesList = async (
   //   reason: "most recent 100 items saved over 500 words
 
   if (libraryItemIds) {
+    logger.info('Using libraryItemIds')
     return findLibraryItemsByIds(libraryItemIds, userId)
   }
 
@@ -299,7 +300,7 @@ const chooseRankedSelections = (rankedCandidates: RankedItem[]) => {
 
   logger.info('finalSelections: ', finalSelections)
 
-  return finalSelections
+  return { finalSelections, rankedTopics }
 }
 
 const summarizeItems = async (
@@ -371,12 +372,15 @@ const generateTitle = (summaries: RankedItem[]): string =>
   summaries.map((item) => item.libraryItem.title).join(', ')
 
 // generate description based on the summaries
-const generateDescription = (summaries: RankedItem[]): string =>
+const generateDescription = (
+  summaries: RankedItem[],
+  rankedTopics: string[]
+): string =>
   `We selected ${
     summaries.length
-  } articles from your last 24 hours of saved items, covering ${summaries
-    .map((summary) => summary.topic)
-    .join(', ')}.`
+  } articles from your last 24 hours of saved items, covering ${rankedTopics.join(
+    ', '
+  )}.`
 
 // generate content based on the summaries
 const generateContent = (summaries: RankedItem[]): string =>
@@ -393,12 +397,16 @@ const generateByline = (summaries: RankedItem[]): string =>
 export const createDigestJob = async (jobData: CreateDigestJobData) => {
   digestDefinition = await fetchDigestDefinition()
 
-  const candidates = await getCandidatesList(jobData.userId)
+  const candidates = await getCandidatesList(
+    jobData.userId,
+    jobData.libraryItemIds
+  )
   const userProfile = await findOrCreateUserProfile(jobData.userId)
   const rankedCandidates = await rankCandidates(candidates, userProfile)
-  const selections = chooseRankedSelections(rankedCandidates)
+  const { finalSelections, rankedTopics } =
+    chooseRankedSelections(rankedCandidates)
 
-  const summaries = await summarizeItems(selections)
+  const summaries = await summarizeItems(finalSelections)
 
   const filteredSummaries = filterSummaries(summaries)
 
@@ -423,7 +431,7 @@ export const createDigestJob = async (jobData: CreateDigestJobData) => {
       wordCount: speechFiles[index].wordCount,
     })),
     createdAt: new Date(),
-    description: generateDescription(summaries),
+    description: generateDescription(summaries, rankedTopics),
     byline: generateByline(summaries),
   }
 

From 26579d9cbbd24e4e3d0248fbd4c7e716306ecca2 Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Wed, 17 Apr 2024 18:26:48 +0800
Subject: [PATCH 74/74] use ### for title in the digest content

---
 packages/api/src/jobs/ai/create_digest.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/api/src/jobs/ai/create_digest.ts b/packages/api/src/jobs/ai/create_digest.ts
index 33012507f..b6470d402 100644
--- a/packages/api/src/jobs/ai/create_digest.ts
+++ b/packages/api/src/jobs/ai/create_digest.ts
@@ -385,7 +385,7 @@ const generateDescription = (
 // generate content based on the summaries
 const generateContent = (summaries: RankedItem[]): string =>
   summaries
-    .map((summary) => `## ${summary.libraryItem.title}\n ${summary.summary}`)
+    .map((summary) => `### ${summary.libraryItem.title}\n ${summary.summary}`)
     .join('\n\n')
 
 const generateByline = (summaries: RankedItem[]): string =>