From ec748dd1e5905fd1caeb01e725b9f6e9fda41395 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 4 Mar 2024 04:09:16 +0000
Subject: [PATCH 01/67] Bump es5-ext from 0.10.62 to 0.10.64

Bumps [es5-ext](https://github.com/medikoo/es5-ext) from 0.10.62 to 0.10.64.
- [Release notes](https://github.com/medikoo/es5-ext/releases)
- [Changelog](https://github.com/medikoo/es5-ext/blob/main/CHANGELOG.md)
- [Commits](https://github.com/medikoo/es5-ext/compare/v0.10.62...v0.10.64)

---
updated-dependencies:
- dependency-name: es5-ext
  dependency-type: indirect
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 yarn.lock | 19 +++++++++++++++----
 1 file changed, 15 insertions(+), 4 deletions(-)

diff --git a/yarn.lock b/yarn.lock
index 8631bcdbc..4c9cce4f1 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -14205,13 +14205,14 @@ es-to-primitive@^1.2.1:
     is-date-object "^1.0.1"
     is-symbol "^1.0.2"
 
-es5-ext@^0.10.35, es5-ext@^0.10.50, es5-ext@~0.10.14:
-  version "0.10.62"
-  resolved "https://registry.yarnpkg.com/es5-ext/-/es5-ext-0.10.62.tgz#5e6adc19a6da524bf3d1e02bbc8960e5eb49a9a5"
-  integrity sha512-BHLqn0klhEpnOKSrzn/Xsz2UIW8j+cGmo9JLzr8BiUapV8hPL9+FliFqjwr9ngW7jWdnxv6eO+/LqyhJVqgrjA==
+es5-ext@^0.10.35, es5-ext@^0.10.50, es5-ext@^0.10.62, es5-ext@~0.10.14:
+  version "0.10.64"
+  resolved "https://registry.yarnpkg.com/es5-ext/-/es5-ext-0.10.64.tgz#12e4ffb48f1ba2ea777f1fcdd1918ef73ea21714"
+  integrity sha512-p2snDhiLaXe6dahss1LddxqEm+SkuDvV8dnIQG0MWjyHpcMNfXKPE+/Cc0y+PhxJX3A4xGNeFCj5oc0BUh6deg==
   dependencies:
     es6-iterator "^2.0.3"
     es6-symbol "^3.1.3"
+    esniff "^2.0.1"
     next-tick "^1.1.0"
 
 es5-shim@^4.5.13:
@@ -14652,6 +14653,16 @@ eslint@^8.6.0:
     text-table "^0.2.0"
     v8-compile-cache "^2.0.3"
 
+esniff@^2.0.1:
+  version "2.0.1"
+  resolved "https://registry.yarnpkg.com/esniff/-/esniff-2.0.1.tgz#a4d4b43a5c71c7ec51c51098c1d8a29081f9b308"
+  integrity sha512-kTUIGKQ/mDPFoJ0oVfcmyJn4iBDRptjNVIzwIFR7tqWXdVI9xfA2RMwY/gbSpJG3lkdWNEjLap/NqVHZiJsdfg==
+  dependencies:
+    d "^1.0.1"
+    es5-ext "^0.10.62"
+    event-emitter "^0.3.5"
+    type "^2.7.2"
+
 espree@^9.0.0:
   version "9.4.1"
   resolved "https://registry.yarnpkg.com/espree/-/espree-9.4.1.tgz#51d6092615567a2c2cff7833445e37c28c0065bd"

From 94db3e0de7cc9efab6669f66197ca3f713c5ffa1 Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Mon, 4 Mar 2024 17:30:05 +0800
Subject: [PATCH 02/67] Add authorization with notion

---
 .../api/src/routers/integration_router.ts     | 32 +++--------
 .../api/src/services/integrations/index.ts    |  6 ++-
 .../src/services/integrations/integration.ts  |  2 +
 .../api/src/services/integrations/notion.ts   | 54 +++++++++++++++++++
 .../api/src/services/integrations/pocket.ts   | 23 ++++++++
 .../api/src/services/integrations/readwise.ts | 14 +++--
 packages/api/src/util.ts                      | 14 +++++
 7 files changed, 115 insertions(+), 30 deletions(-)
 create mode 100644 packages/api/src/services/integrations/notion.ts

diff --git a/packages/api/src/routers/integration_router.ts b/packages/api/src/routers/integration_router.ts
index cc6a520a7..d9a89f43a 100644
--- a/packages/api/src/routers/integration_router.ts
+++ b/packages/api/src/routers/integration_router.ts
@@ -2,6 +2,7 @@ import axios from 'axios'
 import cors from 'cors'
 import express from 'express'
 import { env } from '../env'
+import { getIntegrationClient } from '../services/integrations'
 import { getClaimsByToken } from '../utils/auth'
 import { corsConfig } from '../utils/corsConfig'
 import { logger } from '../utils/logger'
@@ -10,10 +11,9 @@ export function integrationRouter() {
   const router = express.Router()
   // request token from pocket
   router.post(
-    '/pocket/auth',
+    '/:name/auth',
     cors<express.Request>(corsConfig),
     async (req: express.Request, res: express.Response) => {
-      logger.info('pocket/request-token')
       // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
       const token = (req.cookies.auth as string) || req.headers.authorization
       const claims = await getClaimsByToken(token)
@@ -21,37 +21,19 @@ export function integrationRouter() {
         return res.status(401).send('UNAUTHORIZED')
       }
 
-      const consumerKey = env.pocket.consumerKey
-      const redirectUri = `${env.client.url}/settings/integrations`
+      const integrationClient = getIntegrationClient(req.params.name)
+
       // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
       const state = req.body.state as string
       try {
-        // make a POST request to Pocket to get a request token
-        const response = await axios.post<{ code: string }>(
-          'https://getpocket.com/v3/oauth/request',
-          {
-            consumer_key: consumerKey,
-            redirect_uri: redirectUri,
-          },
-          {
-            headers: {
-              'Content-Type': 'application/json',
-              'X-Accept': 'application/json',
-            },
-          }
-        )
-        const { code } = response.data
+        const redirectUri = await integrationClient.auth(state)
         // redirect the user to Pocket to authorize the request token
-        res.redirect(
-          `https://getpocket.com/auth/authorize?request_token=${code}&redirect_uri=${redirectUri}${encodeURIComponent(
-            `?pocketToken=${code}&state=${state}`
-          )}`
-        )
+        res.redirect(redirectUri)
       } catch (error) {
         if (axios.isAxiosError(error)) {
           logger.error(error.response)
         } else {
-          logger.error('pocket/request-token exception:', error)
+          logger.error(error)
         }
 
         res.redirect(
diff --git a/packages/api/src/services/integrations/index.ts b/packages/api/src/services/integrations/index.ts
index 286ac59e7..435ca98e2 100644
--- a/packages/api/src/services/integrations/index.ts
+++ b/packages/api/src/services/integrations/index.ts
@@ -2,16 +2,20 @@ import { DeepPartial, FindOptionsWhere } from 'typeorm'
 import { Integration } from '../../entity/integration'
 import { authTrx } from '../../repository'
 import { IntegrationClient } from './integration'
+import { NotionClient } from './notion'
 import { PocketClient } from './pocket'
 import { ReadwiseClient } from './readwise'
 
 const integrations: IntegrationClient[] = [
   new ReadwiseClient(),
   new PocketClient(),
+  new NotionClient(),
 ]
 
 export const getIntegrationClient = (name: string): IntegrationClient => {
-  const service = integrations.find((s) => s.name === name)
+  const service = integrations.find(
+    (s) => s.name.toLowerCase() === name.toLowerCase()
+  )
   if (!service) {
     throw new Error(`Integration client not found: ${name}`)
   }
diff --git a/packages/api/src/services/integrations/integration.ts b/packages/api/src/services/integrations/integration.ts
index e3f1edbc8..f5183f097 100644
--- a/packages/api/src/services/integrations/integration.ts
+++ b/packages/api/src/services/integrations/integration.ts
@@ -24,5 +24,7 @@ export interface IntegrationClient {
 
   accessToken(token: string): Promise<string | null>
 
+  auth(state: string): Promise<string>
+
   export(token: string, items: LibraryItem[]): Promise<boolean>
 }
diff --git a/packages/api/src/services/integrations/notion.ts b/packages/api/src/services/integrations/notion.ts
new file mode 100644
index 000000000..8e8cedcad
--- /dev/null
+++ b/packages/api/src/services/integrations/notion.ts
@@ -0,0 +1,54 @@
+import axios from 'axios'
+import { env } from '../../env'
+import { logger } from '../../utils/logger'
+import { IntegrationClient } from './integration'
+
+export class NotionClient implements IntegrationClient {
+  name = 'notion'
+  apiUrl = 'https://api.notion.com/v1'
+  headers = {
+    'Content-Type': 'application/json',
+    Accept: 'application/json',
+  }
+
+  accessToken = async (code: string): Promise<string | null> => {
+    const authUrl = `${this.apiUrl}/oauth/token`
+    try {
+      // encode in base 64
+      const encoded = Buffer.from(
+        `${env.notion.clientId}:${env.notion.clientSecret}`
+      ).toString('base64')
+
+      const response = await axios.post<{ access_token: string }>(
+        authUrl,
+        {
+          grant_type: 'authorization_code',
+          code,
+        },
+        {
+          headers: {
+            authorization: `Basic ${encoded}`,
+            ...this.headers,
+          },
+          timeout: 5000, // 5 seconds
+        }
+      )
+      return response.data.access_token
+    } catch (error) {
+      if (axios.isAxiosError(error)) {
+        logger.error(error.response)
+      } else {
+        logger.error(error)
+      }
+      return null
+    }
+  }
+
+  async auth(state: string): Promise<string> {
+    return Promise.resolve(env.notion.authUrl)
+  }
+
+  export = () => {
+    throw new Error('Method not implemented.')
+  }
+}
diff --git a/packages/api/src/services/integrations/pocket.ts b/packages/api/src/services/integrations/pocket.ts
index 517d3befa..661d1ba0d 100644
--- a/packages/api/src/services/integrations/pocket.ts
+++ b/packages/api/src/services/integrations/pocket.ts
@@ -39,4 +39,27 @@ export class PocketClient implements IntegrationClient {
   export = async (): Promise<boolean> => {
     return Promise.resolve(false)
   }
+
+  async auth(state: string) {
+    const consumerKey = env.pocket.consumerKey
+    const redirectUri = `${env.client.url}/settings/integrations`
+
+    // make a POST request to Pocket to get a request token
+    const response = await axios.post<{ code: string }>(
+      `${this.apiUrl}/oauth/request`,
+      {
+        consumer_key: consumerKey,
+        redirect_uri: redirectUri,
+      },
+      {
+        headers: this.headers,
+        timeout: 5000, // 5 seconds
+      }
+    )
+    const { code } = response.data
+
+    return `https://getpocket.com/auth/authorize?request_token=${code}&redirect_uri=${redirectUri}${encodeURIComponent(
+      `?pocketToken=${code}&state=${state}`
+    )}`
+  }
 }
diff --git a/packages/api/src/services/integrations/readwise.ts b/packages/api/src/services/integrations/readwise.ts
index ae23810bd..e82597a9c 100644
--- a/packages/api/src/services/integrations/readwise.ts
+++ b/packages/api/src/services/integrations/readwise.ts
@@ -57,17 +57,23 @@ export class ReadwiseClient implements IntegrationClient {
   export = async (token: string, items: LibraryItem[]): Promise<boolean> => {
     let result = true
 
-    const highlights = items.flatMap(this.itemToReadwiseHighlight)
+    const highlights = items.flatMap(this._itemToReadwiseHighlight)
 
     // If there are no highlights, we will skip the sync
     if (highlights.length > 0) {
-      result = await this.syncWithReadwise(token, highlights)
+      result = await this._syncWithReadwise(token, highlights)
     }
 
     return result
   }
 
-  itemToReadwiseHighlight = (item: LibraryItem): ReadwiseHighlight[] => {
+  auth(state: string): Promise<string> {
+    throw new Error('Method not implemented.')
+  }
+
+  private _itemToReadwiseHighlight = (
+    item: LibraryItem
+  ): ReadwiseHighlight[] => {
     const category = item.siteName === 'Twitter' ? 'tweets' : 'articles'
     return item.highlights
       ?.map((highlight) => {
@@ -93,7 +99,7 @@ export class ReadwiseClient implements IntegrationClient {
       .filter((highlight) => highlight !== undefined) as ReadwiseHighlight[]
   }
 
-  syncWithReadwise = async (
+  private _syncWithReadwise = async (
     token: string,
     highlights: ReadwiseHighlight[]
   ): Promise<boolean> => {
diff --git a/packages/api/src/util.ts b/packages/api/src/util.ts
index b03a19c15..d42f84ce1 100755
--- a/packages/api/src/util.ts
+++ b/packages/api/src/util.ts
@@ -113,6 +113,11 @@ export interface BackendEnv {
     mq: redisConfig
     cache: redisConfig
   }
+  notion: {
+    clientId: string
+    clientSecret: string
+    authUrl: string
+  }
 }
 
 const nullableEnvVars = [
@@ -165,6 +170,9 @@ const nullableEnvVars = [
   'MQ_REDIS_CERT',
   'IMPORTER_METRICS_COLLECTOR_URL',
   'INTERNAL_API_URL',
+  'NOTION_CLIENT_ID',
+  'NOTION_CLIENT_SECRET',
+  'NOTION_AUTH_URL',
 ] // Allow some vars to be null/empty
 
 /* If not in GAE and Prod/QA/Demo env (f.e. on localhost/dev env), allow following env vars to be null */
@@ -311,6 +319,11 @@ export function getEnv(): BackendEnv {
       cert: parse('REDIS_CERT')?.replace(/\\n/g, '\n'), // replace \n with new line
     },
   }
+  const notion = {
+    clientId: parse('NOTION_CLIENT_ID'),
+    clientSecret: parse('NOTION_CLIENT_SECRET'),
+    authUrl: parse('NOTION_AUTH_URL'),
+  }
 
   return {
     pg,
@@ -333,6 +346,7 @@ export function getEnv(): BackendEnv {
     pocket,
     subscription,
     redis,
+    notion,
   }
 }
 

From d36c937765475baed56694aea26890231bb17217 Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Mon, 4 Mar 2024 17:49:59 +0800
Subject: [PATCH 03/67] fix redirect_uri error

---
 packages/api/src/services/integrations/notion.ts | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/packages/api/src/services/integrations/notion.ts b/packages/api/src/services/integrations/notion.ts
index 8e8cedcad..fd46dc1bb 100644
--- a/packages/api/src/services/integrations/notion.ts
+++ b/packages/api/src/services/integrations/notion.ts
@@ -4,7 +4,7 @@ import { logger } from '../../utils/logger'
 import { IntegrationClient } from './integration'
 
 export class NotionClient implements IntegrationClient {
-  name = 'notion'
+  name = 'NOTION'
   apiUrl = 'https://api.notion.com/v1'
   headers = {
     'Content-Type': 'application/json',
@@ -24,6 +24,7 @@ export class NotionClient implements IntegrationClient {
         {
           grant_type: 'authorization_code',
           code,
+          redirect_uri: `${env.client.url}/settings/integrations`,
         },
         {
           headers: {

From e20908a0c29c8a371433fcc3dc8c7dfbdd138f00 Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Mon, 4 Mar 2024 18:44:25 +0800
Subject: [PATCH 04/67] add export to notion method

---
 .../api/src/services/integrations/notion.ts   | 94 ++++++++++++++++++-
 .../api/src/services/integrations/pocket.ts   |  4 +-
 .../api/src/services/integrations/readwise.ts |  2 +-
 3 files changed, 93 insertions(+), 7 deletions(-)

diff --git a/packages/api/src/services/integrations/notion.ts b/packages/api/src/services/integrations/notion.ts
index fd46dc1bb..bcd6573e0 100644
--- a/packages/api/src/services/integrations/notion.ts
+++ b/packages/api/src/services/integrations/notion.ts
@@ -1,15 +1,46 @@
 import axios from 'axios'
+import { LibraryItem } from '../../entity/library_item'
 import { env } from '../../env'
 import { logger } from '../../utils/logger'
 import { IntegrationClient } from './integration'
 
+interface NotionPage {
+  parent: {
+    database_id: string
+  }
+  cover?: {
+    external: {
+      url: string
+    }
+  }
+  properties: {
+    Name: {
+      title: Array<{
+        text: {
+          content: string
+        }
+      }>
+    }
+    URL: {
+      url: string
+    }
+    Tags: {
+      multi_select: Array<{
+        name: string
+      }>
+    }
+  }
+}
+
 export class NotionClient implements IntegrationClient {
   name = 'NOTION'
   apiUrl = 'https://api.notion.com/v1'
   headers = {
     'Content-Type': 'application/json',
     Accept: 'application/json',
+    'Notion-Version': '2022-06-28',
   }
+  timeout = 5000 // 5 seconds
 
   accessToken = async (code: string): Promise<string | null> => {
     const authUrl = `${this.apiUrl}/oauth/token`
@@ -31,7 +62,7 @@ export class NotionClient implements IntegrationClient {
             authorization: `Basic ${encoded}`,
             ...this.headers,
           },
-          timeout: 5000, // 5 seconds
+          timeout: this.timeout,
         }
       )
       return response.data.access_token
@@ -45,11 +76,66 @@ export class NotionClient implements IntegrationClient {
     }
   }
 
-  async auth(state: string): Promise<string> {
+  async auth(): Promise<string> {
     return Promise.resolve(env.notion.authUrl)
   }
 
-  export = () => {
-    throw new Error('Method not implemented.')
+  private _itemToNotionPage = (item: LibraryItem): NotionPage => {
+    return {
+      parent: {
+        database_id: item.id,
+      },
+      cover: item.thumbnail
+        ? {
+            external: {
+              url: item.thumbnail,
+            },
+          }
+        : undefined,
+      properties: {
+        Name: {
+          title: [
+            {
+              text: {
+                content: item.title,
+              },
+            },
+          ],
+        },
+        URL: {
+          url: item.originalUrl,
+        },
+        Tags: {
+          multi_select:
+            item.labels?.map((label) => {
+              return {
+                name: label.name,
+              }
+            }) || [],
+        },
+      },
+    }
+  }
+
+  export = async (token: string, items: LibraryItem[]): Promise<boolean> => {
+    const url = `${this.apiUrl}/pages`
+    const page = this._itemToNotionPage(items[0])
+    try {
+      const response = await axios.post(url, page, {
+        headers: {
+          Authorization: `Bearer ${token}`,
+          ...this.headers,
+        },
+        timeout: this.timeout,
+      })
+      return response.status === 200
+    } catch (error) {
+      if (axios.isAxiosError(error)) {
+        logger.error(error.response)
+      } else {
+        logger.error(error)
+      }
+      return false
+    }
   }
 }
diff --git a/packages/api/src/services/integrations/pocket.ts b/packages/api/src/services/integrations/pocket.ts
index 661d1ba0d..8f9e8ed50 100644
--- a/packages/api/src/services/integrations/pocket.ts
+++ b/packages/api/src/services/integrations/pocket.ts
@@ -36,8 +36,8 @@ export class PocketClient implements IntegrationClient {
     }
   }
 
-  export = async (): Promise<boolean> => {
-    return Promise.resolve(false)
+  export = () => {
+    throw new Error('Method not implemented.')
   }
 
   async auth(state: string) {
diff --git a/packages/api/src/services/integrations/readwise.ts b/packages/api/src/services/integrations/readwise.ts
index e82597a9c..84b15a25a 100644
--- a/packages/api/src/services/integrations/readwise.ts
+++ b/packages/api/src/services/integrations/readwise.ts
@@ -67,7 +67,7 @@ export class ReadwiseClient implements IntegrationClient {
     return result
   }
 
-  auth(state: string): Promise<string> {
+  auth = () => {
     throw new Error('Method not implemented.')
   }
 

From f2a04f8cf385942ffc9be3aa037b96b9b964184c Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 4 Mar 2024 13:30:04 +0000
Subject: [PATCH 05/67] Bump pg from 8.7.3 to 8.11.3

Bumps [pg](https://github.com/brianc/node-postgres/tree/HEAD/packages/pg) from 8.7.3 to 8.11.3.
- [Changelog](https://github.com/brianc/node-postgres/blob/master/CHANGELOG.md)
- [Commits](https://github.com/brianc/node-postgres/commits/pg@8.11.3/packages/pg)

---
updated-dependencies:
- dependency-name: pg
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 yarn.lock | 43 +++++++++++++++++++++++++------------------
 1 file changed, 25 insertions(+), 18 deletions(-)

diff --git a/yarn.lock b/yarn.lock
index 8631bcdbc..fec1b64e0 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -24621,25 +24621,30 @@ performance-now@^2.1.0:
   resolved "https://registry.yarnpkg.com/performance-now/-/performance-now-2.1.0.tgz#6309f4e0e5fa913ec1c69307ae364b4b377c9e7b"
   integrity sha1-Ywn04OX6kT7BxpMHrjZLSzd8nns=
 
-pg-connection-string@^2.5.0:
-  version "2.5.0"
-  resolved "https://registry.yarnpkg.com/pg-connection-string/-/pg-connection-string-2.5.0.tgz#538cadd0f7e603fc09a12590f3b8a452c2c0cf34"
-  integrity sha512-r5o/V/ORTA6TmUnyWZR9nCj1klXCO2CEKNRlVuJptZe85QuhFayC7WeMic7ndayT5IRIR0S0xFxFi2ousartlQ==
+pg-cloudflare@^1.1.1:
+  version "1.1.1"
+  resolved "https://registry.yarnpkg.com/pg-cloudflare/-/pg-cloudflare-1.1.1.tgz#e6d5833015b170e23ae819e8c5d7eaedb472ca98"
+  integrity sha512-xWPagP/4B6BgFO+EKz3JONXv3YDgvkbVrGw2mTo3D6tVDQRh1e7cqVGvyR3BE+eQgAvx1XhW/iEASj4/jCWl3Q==
+
+pg-connection-string@^2.6.2:
+  version "2.6.2"
+  resolved "https://registry.yarnpkg.com/pg-connection-string/-/pg-connection-string-2.6.2.tgz#713d82053de4e2bd166fab70cd4f26ad36aab475"
+  integrity sha512-ch6OwaeaPYcova4kKZ15sbJ2hKb/VP48ZD2gE7i1J+L4MspCtBMAx8nMgz7bksc7IojCIIWuEhHibSMFH8m8oA==
 
 pg-int8@1.0.1:
   version "1.0.1"
   resolved "https://registry.yarnpkg.com/pg-int8/-/pg-int8-1.0.1.tgz#943bd463bf5b71b4170115f80f8efc9a0c0eb78c"
   integrity sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw==
 
-pg-pool@^3.5.1:
-  version "3.5.1"
-  resolved "https://registry.yarnpkg.com/pg-pool/-/pg-pool-3.5.1.tgz#f499ce76f9bf5097488b3b83b19861f28e4ed905"
-  integrity sha512-6iCR0wVrro6OOHFsyavV+i6KYL4lVNyYAB9RD18w66xSzN+d8b66HiwuP30Gp1SH5O9T82fckkzsRjlrhD0ioQ==
+pg-pool@^3.6.1:
+  version "3.6.1"
+  resolved "https://registry.yarnpkg.com/pg-pool/-/pg-pool-3.6.1.tgz#5a902eda79a8d7e3c928b77abf776b3cb7d351f7"
+  integrity sha512-jizsIzhkIitxCGfPRzJn1ZdcosIt3pz9Sh3V01fm1vZnbnCMgmGl5wvGGdNN2EL9Rmb0EcFoCkixH4Pu+sP9Og==
 
-pg-protocol@*, pg-protocol@^1.5.0:
-  version "1.5.0"
-  resolved "https://registry.yarnpkg.com/pg-protocol/-/pg-protocol-1.5.0.tgz#b5dd452257314565e2d54ab3c132adc46565a6a0"
-  integrity sha512-muRttij7H8TqRNu/DxrAJQITO4Ac7RmX3Klyr/9mJEOBeIpgnF8f9jAfRz5d3XwQZl5qBjF9gLsUtMPJE0vezQ==
+pg-protocol@*, pg-protocol@^1.6.0:
+  version "1.6.0"
+  resolved "https://registry.yarnpkg.com/pg-protocol/-/pg-protocol-1.6.0.tgz#4c91613c0315349363af2084608db843502f8833"
+  integrity sha512-M+PDm637OY5WM307051+bsDia5Xej6d9IR4GwJse1qA1DIhiKlksvrneZOYQq42OM+spubpcNYEo2FcKQrDk+Q==
 
 pg-types@^2.1.0, pg-types@^2.2.0:
   version "2.2.0"
@@ -24653,17 +24658,19 @@ pg-types@^2.1.0, pg-types@^2.2.0:
     postgres-interval "^1.1.0"
 
 pg@^8.3.0, pg@^8.3.3:
-  version "8.7.3"
-  resolved "https://registry.yarnpkg.com/pg/-/pg-8.7.3.tgz#8a5bdd664ca4fda4db7997ec634c6e5455b27c44"
-  integrity sha512-HPmH4GH4H3AOprDJOazoIcpI49XFsHCe8xlrjHkWiapdbHK+HLtbm/GQzXYAZwmPju/kzKhjaSfMACG+8cgJcw==
+  version "8.11.3"
+  resolved "https://registry.yarnpkg.com/pg/-/pg-8.11.3.tgz#d7db6e3fe268fcedd65b8e4599cda0b8b4bf76cb"
+  integrity sha512-+9iuvG8QfaaUrrph+kpF24cXkH1YOOUeArRNYIxq1viYHZagBxrTno7cecY1Fa44tJeZvaoG+Djpkc3JwehN5g==
   dependencies:
     buffer-writer "2.0.0"
     packet-reader "1.0.0"
-    pg-connection-string "^2.5.0"
-    pg-pool "^3.5.1"
-    pg-protocol "^1.5.0"
+    pg-connection-string "^2.6.2"
+    pg-pool "^3.6.1"
+    pg-protocol "^1.6.0"
     pg-types "^2.1.0"
     pgpass "1.x"
+  optionalDependencies:
+    pg-cloudflare "^1.1.1"
 
 pgpass@1.x:
   version "1.0.4"

From 741f3213d94d39133a58b503fe9f157aba04da0d Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Tue, 5 Mar 2024 12:43:18 +0800
Subject: [PATCH 06/67] integrate with notion api client

---
 packages/api/package.json                     |   1 +
 .../api/src/jobs/integration/export_item.ts   |   4 +-
 .../api/src/resolvers/integrations/index.ts   |   4 +-
 .../api/src/routers/integration_router.ts     |   2 +-
 .../api/src/services/integrations/index.ts    |  26 ++--
 .../src/services/integrations/integration.ts  |   6 +-
 .../api/src/services/integrations/notion.ts   | 115 ++++++++----------
 .../api/src/services/integrations/pocket.ts   |  37 +++---
 .../api/src/services/integrations/readwise.ts |  38 +++---
 yarn.lock                                     |  24 ++--
 10 files changed, 127 insertions(+), 130 deletions(-)

diff --git a/packages/api/package.json b/packages/api/package.json
index 111fc6502..b387616f4 100644
--- a/packages/api/package.json
+++ b/packages/api/package.json
@@ -24,6 +24,7 @@
     "@google-cloud/tasks": "^4.0.0",
     "@graphql-tools/utils": "^9.1.1",
     "@langchain/openai": "^0.0.14",
+    "@notionhq/client": "^2.2.14",
     "@omnivore/content-handler": "1.0.0",
     "@omnivore/liqe": "1.0.0",
     "@omnivore/readability": "1.0.0",
diff --git a/packages/api/src/jobs/integration/export_item.ts b/packages/api/src/jobs/integration/export_item.ts
index 327f16206..a2a0f1456 100644
--- a/packages/api/src/jobs/integration/export_item.ts
+++ b/packages/api/src/jobs/integration/export_item.ts
@@ -44,9 +44,9 @@ export const exportItem = async (jobData: ExportItemJobData) => {
   }
   logger.info('exporting item...', logObject)
 
-  const client = getIntegrationClient(integration.name)
+  const client = getIntegrationClient(integration.name, integration.token)
 
-  const synced = await client.export(integration.token, libraryItems)
+  const synced = await client.export(libraryItems)
   if (!synced) {
     logger.error('failed to export item', logObject)
     return false
diff --git a/packages/api/src/resolvers/integrations/index.ts b/packages/api/src/resolvers/integrations/index.ts
index 90c118129..5148d0893 100644
--- a/packages/api/src/resolvers/integrations/index.ts
+++ b/packages/api/src/resolvers/integrations/index.ts
@@ -69,9 +69,9 @@ export const setIntegrationResolver = authorized<
       integrationToSave.taskName = existingIntegration.taskName
     } else {
       // Create
-      const integrationService = getIntegrationClient(input.name)
+      const integrationService = getIntegrationClient(input.name, input.token)
       // authorize and get access token
-      const token = await integrationService.accessToken(input.token)
+      const token = await integrationService.accessToken()
       if (!token) {
         return {
           errorCodes: [SetIntegrationErrorCode.InvalidToken],
diff --git a/packages/api/src/routers/integration_router.ts b/packages/api/src/routers/integration_router.ts
index d9a89f43a..7bf5cbdc6 100644
--- a/packages/api/src/routers/integration_router.ts
+++ b/packages/api/src/routers/integration_router.ts
@@ -21,7 +21,7 @@ export function integrationRouter() {
         return res.status(401).send('UNAUTHORIZED')
       }
 
-      const integrationClient = getIntegrationClient(req.params.name)
+      const integrationClient = getIntegrationClient(req.params.name, '')
 
       // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
       const state = req.body.state as string
diff --git a/packages/api/src/services/integrations/index.ts b/packages/api/src/services/integrations/index.ts
index 435ca98e2..953199052 100644
--- a/packages/api/src/services/integrations/index.ts
+++ b/packages/api/src/services/integrations/index.ts
@@ -6,20 +6,20 @@ import { NotionClient } from './notion'
 import { PocketClient } from './pocket'
 import { ReadwiseClient } from './readwise'
 
-const integrations: IntegrationClient[] = [
-  new ReadwiseClient(),
-  new PocketClient(),
-  new NotionClient(),
-]
-
-export const getIntegrationClient = (name: string): IntegrationClient => {
-  const service = integrations.find(
-    (s) => s.name.toLowerCase() === name.toLowerCase()
-  )
-  if (!service) {
-    throw new Error(`Integration client not found: ${name}`)
+export const getIntegrationClient = (
+  name: string,
+  token: string
+): IntegrationClient => {
+  switch (name.toLowerCase()) {
+    case 'readwise':
+      return new ReadwiseClient(token)
+    case 'pocket':
+      return new PocketClient(token)
+    case 'notion':
+      return new NotionClient(token)
+    default:
+      throw new Error(`Integration client not found: ${name}`)
   }
-  return service
 }
 
 export const deleteIntegrations = async (
diff --git a/packages/api/src/services/integrations/integration.ts b/packages/api/src/services/integrations/integration.ts
index f5183f097..13b61e82a 100644
--- a/packages/api/src/services/integrations/integration.ts
+++ b/packages/api/src/services/integrations/integration.ts
@@ -20,11 +20,11 @@ export interface RetrieveRequest {
 
 export interface IntegrationClient {
   name: string
-  apiUrl: string
+  _token: string
 
-  accessToken(token: string): Promise<string | null>
+  accessToken(): Promise<string | null>
 
   auth(state: string): Promise<string>
 
-  export(token: string, items: LibraryItem[]): Promise<boolean>
+  export(items: LibraryItem[]): Promise<boolean>
 }
diff --git a/packages/api/src/services/integrations/notion.ts b/packages/api/src/services/integrations/notion.ts
index bcd6573e0..539c1a3ef 100644
--- a/packages/api/src/services/integrations/notion.ts
+++ b/packages/api/src/services/integrations/notion.ts
@@ -1,3 +1,4 @@
+import { Client } from '@notionhq/client'
 import axios from 'axios'
 import { LibraryItem } from '../../entity/library_item'
 import { env } from '../../env'
@@ -6,7 +7,7 @@ import { IntegrationClient } from './integration'
 
 interface NotionPage {
   parent: {
-    database_id: string
+    page_id: string
   }
   cover?: {
     external: {
@@ -14,55 +15,56 @@ interface NotionPage {
     }
   }
   properties: {
-    Name: {
-      title: Array<{
-        text: {
-          content: string
-        }
-      }>
-    }
-    URL: {
-      url: string
-    }
-    Tags: {
-      multi_select: Array<{
-        name: string
-      }>
-    }
+    title: Array<{
+      text: {
+        content: string
+      }
+    }>
   }
 }
 
 export class NotionClient implements IntegrationClient {
   name = 'NOTION'
-  apiUrl = 'https://api.notion.com/v1'
-  headers = {
+  _headers = {
     'Content-Type': 'application/json',
     Accept: 'application/json',
     'Notion-Version': '2022-06-28',
   }
-  timeout = 5000 // 5 seconds
+  _timeout = 5000 // 5 seconds
+  _axios = axios.create({
+    baseURL: 'https://api.notion.com/v1',
+    timeout: this._timeout,
+  })
+  _token: string
+  _client: Client
 
-  accessToken = async (code: string): Promise<string | null> => {
-    const authUrl = `${this.apiUrl}/oauth/token`
+  constructor(token: string) {
+    this._token = token
+    this._client = new Client({
+      auth: token,
+      timeoutMs: this._timeout,
+    })
+  }
+
+  accessToken = async (): Promise<string | null> => {
     try {
       // encode in base 64
       const encoded = Buffer.from(
         `${env.notion.clientId}:${env.notion.clientSecret}`
       ).toString('base64')
 
-      const response = await axios.post<{ access_token: string }>(
-        authUrl,
+      const response = await this._axios.post<{ access_token: string }>(
+        '/oauth/token',
         {
           grant_type: 'authorization_code',
-          code,
+          code: this._token,
           redirect_uri: `${env.client.url}/settings/integrations`,
         },
         {
           headers: {
-            authorization: `Basic ${encoded}`,
-            ...this.headers,
+            ...this._headers,
+            Authorization: `Basic ${encoded}`,
           },
-          timeout: this.timeout,
         }
       )
       return response.data.access_token
@@ -83,7 +85,7 @@ export class NotionClient implements IntegrationClient {
   private _itemToNotionPage = (item: LibraryItem): NotionPage => {
     return {
       parent: {
-        database_id: item.id,
+        page_id: '83a3f627ab9e44ac83fe657141aec615',
       },
       cover: item.thumbnail
         ? {
@@ -93,49 +95,28 @@ export class NotionClient implements IntegrationClient {
           }
         : undefined,
       properties: {
-        Name: {
-          title: [
-            {
-              text: {
-                content: item.title,
-              },
+        title: [
+          {
+            text: {
+              content: item.title,
             },
-          ],
-        },
-        URL: {
-          url: item.originalUrl,
-        },
-        Tags: {
-          multi_select:
-            item.labels?.map((label) => {
-              return {
-                name: label.name,
-              }
-            }) || [],
-        },
+          },
+        ],
       },
     }
   }
 
-  export = async (token: string, items: LibraryItem[]): Promise<boolean> => {
-    const url = `${this.apiUrl}/pages`
-    const page = this._itemToNotionPage(items[0])
-    try {
-      const response = await axios.post(url, page, {
-        headers: {
-          Authorization: `Bearer ${token}`,
-          ...this.headers,
-        },
-        timeout: this.timeout,
-      })
-      return response.status === 200
-    } catch (error) {
-      if (axios.isAxiosError(error)) {
-        logger.error(error.response)
-      } else {
-        logger.error(error)
-      }
-      return false
-    }
+  _createPage = async (page: NotionPage) => {
+    await this._client.pages.create(page)
+  }
+
+  export = async (items: LibraryItem[]): Promise<boolean> => {
+    // find/create a parent page for all the items
+    const parentPageName = 'Omnivore'
+
+    const pages = items.map(this._itemToNotionPage)
+    await Promise.all(pages.map((page) => this._createPage(page)))
+
+    return true
   }
 }
diff --git a/packages/api/src/services/integrations/pocket.ts b/packages/api/src/services/integrations/pocket.ts
index 8f9e8ed50..20f4c212c 100644
--- a/packages/api/src/services/integrations/pocket.ts
+++ b/packages/api/src/services/integrations/pocket.ts
@@ -5,24 +5,27 @@ import { IntegrationClient } from './integration'
 
 export class PocketClient implements IntegrationClient {
   name = 'POCKET'
-  apiUrl = 'https://getpocket.com/v3'
-  headers = {
-    'Content-Type': 'application/json',
-    'X-Accept': 'application/json',
+  _token: string
+  _axios = axios.create({
+    baseURL: 'https://getpocket.com/v3',
+    headers: {
+      'Content-Type': 'application/json',
+      'X-Accept': 'application/json',
+    },
+    timeout: 5000, // 5 seconds
+  })
+
+  constructor(token: string) {
+    this._token = token
   }
 
-  accessToken = async (token: string): Promise<string | null> => {
-    const url = `${this.apiUrl}/oauth/authorize`
+  accessToken = async (): Promise<string | null> => {
     try {
-      const response = await axios.post<{ access_token: string }>(
-        url,
+      const response = await this._axios.post<{ access_token: string }>(
+        '/oauth/authorize',
         {
           consumer_key: env.pocket.consumerKey,
-          code: token,
-        },
-        {
-          headers: this.headers,
-          timeout: 5000, // 5 seconds
+          code: this._token,
         }
       )
       return response.data.access_token
@@ -45,15 +48,11 @@ export class PocketClient implements IntegrationClient {
     const redirectUri = `${env.client.url}/settings/integrations`
 
     // make a POST request to Pocket to get a request token
-    const response = await axios.post<{ code: string }>(
-      `${this.apiUrl}/oauth/request`,
+    const response = await this._axios.post<{ code: string }>(
+      '/oauth/request',
       {
         consumer_key: consumerKey,
         redirect_uri: redirectUri,
-      },
-      {
-        headers: this.headers,
-        timeout: 5000, // 5 seconds
       }
     )
     const { code } = response.data
diff --git a/packages/api/src/services/integrations/readwise.ts b/packages/api/src/services/integrations/readwise.ts
index 84b15a25a..25a61f438 100644
--- a/packages/api/src/services/integrations/readwise.ts
+++ b/packages/api/src/services/integrations/readwise.ts
@@ -33,17 +33,28 @@ interface ReadwiseHighlight {
 
 export class ReadwiseClient implements IntegrationClient {
   name = 'READWISE'
-  apiUrl = 'https://readwise.io/api/v2'
+  _headers = {
+    'Content-Type': 'application/json',
+  }
+  _axios = axios.create({
+    baseURL: 'https://readwise.io/api/v2',
+    timeout: 5000, // 5 seconds
+  })
+  _token: string
 
-  accessToken = async (token: string): Promise<string | null> => {
-    const authUrl = `${this.apiUrl}/auth`
+  constructor(token: string) {
+    this._token = token
+  }
+
+  accessToken = async (): Promise<string | null> => {
     try {
-      const response = await axios.get(authUrl, {
+      const response = await this._axios.get('/auth', {
         headers: {
-          Authorization: `Token ${token}`,
+          ...this._headers,
+          Authorization: `Token ${this._token}`,
         },
       })
-      return response.status === 204 ? token : null
+      return response.status === 204 ? this._token : null
     } catch (error) {
       if (axios.isAxiosError(error)) {
         logger.error(error.response)
@@ -54,14 +65,14 @@ export class ReadwiseClient implements IntegrationClient {
     }
   }
 
-  export = async (token: string, items: LibraryItem[]): Promise<boolean> => {
+  export = async (items: LibraryItem[]): Promise<boolean> => {
     let result = true
 
     const highlights = items.flatMap(this._itemToReadwiseHighlight)
 
     // If there are no highlights, we will skip the sync
     if (highlights.length > 0) {
-      result = await this._syncWithReadwise(token, highlights)
+      result = await this._syncWithReadwise(highlights)
     }
 
     return result
@@ -100,21 +111,18 @@ export class ReadwiseClient implements IntegrationClient {
   }
 
   private _syncWithReadwise = async (
-    token: string,
     highlights: ReadwiseHighlight[]
   ): Promise<boolean> => {
-    const url = `${this.apiUrl}/highlights`
-    const response = await axios.post(
-      url,
+    const response = await this._axios.post(
+      '/highlights',
       {
         highlights,
       },
       {
         headers: {
-          Authorization: `Token ${token}`,
-          'Content-Type': 'application/json',
+          ...this._headers,
+          Authorization: `Token ${this._token}`,
         },
-        timeout: 5000, // 5 seconds
       }
     )
     return response.status === 200
diff --git a/yarn.lock b/yarn.lock
index 8631bcdbc..cb71dd339 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -3931,6 +3931,14 @@
     "@nodelib/fs.scandir" "2.1.3"
     fastq "^1.6.0"
 
+"@notionhq/client@^2.2.14":
+  version "2.2.14"
+  resolved "https://registry.yarnpkg.com/@notionhq/client/-/client-2.2.14.tgz#6807ec27ee89584529abfd28d058b2661f828b74"
+  integrity sha512-oqUefZtCiJPCX+74A1Os9OVTef3fSnVWe2eVQtU1HJSD+nsfxfhwvDKnzJTh2Tw1ZHKLxpieHB/nzGdY+Uo12A==
+  dependencies:
+    "@types/node-fetch" "^2.5.10"
+    node-fetch "^2.6.1"
+
 "@npmcli/arborist@^5.6.3":
   version "5.6.3"
   resolved "https://registry.yarnpkg.com/@npmcli/arborist/-/arborist-5.6.3.tgz#40810080272e097b4a7a4f56108f4a31638a9874"
@@ -7877,6 +7885,14 @@
   dependencies:
     "@types/node" "*"
 
+"@types/node-fetch@^2.5.10", "@types/node-fetch@^2.6.4":
+  version "2.6.11"
+  resolved "https://registry.yarnpkg.com/@types/node-fetch/-/node-fetch-2.6.11.tgz#9b39b78665dae0e82a08f02f4967d62c66f95d24"
+  integrity sha512-24xFj9R5+rfQJLRyM56qh+wnVSYhyXC2tkoBndtY0U+vubqNsYXGjufB2nn8Q6gt0LrARwL6UBtMCSVCwl4B1g==
+  dependencies:
+    "@types/node" "*"
+    form-data "^4.0.0"
+
 "@types/node-fetch@^2.5.7":
   version "2.6.1"
   resolved "https://registry.yarnpkg.com/@types/node-fetch/-/node-fetch-2.6.1.tgz#8f127c50481db65886800ef496f20bbf15518975"
@@ -7885,14 +7901,6 @@
     "@types/node" "*"
     form-data "^3.0.0"
 
-"@types/node-fetch@^2.6.4":
-  version "2.6.11"
-  resolved "https://registry.yarnpkg.com/@types/node-fetch/-/node-fetch-2.6.11.tgz#9b39b78665dae0e82a08f02f4967d62c66f95d24"
-  integrity sha512-24xFj9R5+rfQJLRyM56qh+wnVSYhyXC2tkoBndtY0U+vubqNsYXGjufB2nn8Q6gt0LrARwL6UBtMCSVCwl4B1g==
-  dependencies:
-    "@types/node" "*"
-    form-data "^4.0.0"
-
 "@types/node-fetch@^2.6.6":
   version "2.6.7"
   resolved "https://registry.yarnpkg.com/@types/node-fetch/-/node-fetch-2.6.7.tgz#a1abe2ce24228b58ad97f99480fdcf9bbc6ab16d"

From f1b1f2c4c10a7981b430c6db830e30ac20aeab7c Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Tue, 5 Mar 2024 15:39:09 +0800
Subject: [PATCH 07/67] send created event only

---
 .../api/src/jobs/integration/export_item.ts   | 69 +++++++++++--------
 .../api/src/services/integrations/notion.ts   | 11 +--
 packages/api/src/services/library_item.ts     |  2 +-
 3 files changed, 47 insertions(+), 35 deletions(-)

diff --git a/packages/api/src/jobs/integration/export_item.ts b/packages/api/src/jobs/integration/export_item.ts
index a2a0f1456..a1b8f7cc2 100644
--- a/packages/api/src/jobs/integration/export_item.ts
+++ b/packages/api/src/jobs/integration/export_item.ts
@@ -35,41 +35,50 @@ export const exportItem = async (jobData: ExportItemJobData) => {
     return
   }
 
-  // currently only readwise integration is supported
-  const integration = integrations[0]
+  await Promise.all(
+    integrations.map(async (integration) => {
+      try {
+        const logObject = {
+          userId,
+          integrationId: integration.id,
+        }
+        logger.info('exporting item...', logObject)
 
-  const logObject = {
-    userId,
-    integrationId: integration.id,
-  }
-  logger.info('exporting item...', logObject)
+        const client = getIntegrationClient(integration.name, integration.token)
 
-  const client = getIntegrationClient(integration.name, integration.token)
+        const synced = await client.export(libraryItems)
+        if (!synced) {
+          logger.error('failed to export item', logObject)
+          return false
+        }
 
-  const synced = await client.export(libraryItems)
-  if (!synced) {
-    logger.error('failed to export item', logObject)
-    return false
-  }
+        const syncedAt = new Date()
+        logger.info('updating integration...', {
+          ...logObject,
+          syncedAt,
+        })
 
-  const syncedAt = new Date()
-  logger.info('updating integration...', {
-    ...logObject,
-    syncedAt,
-  })
-
-  // update integration syncedAt if successful
-  const updated = await updateIntegration(
-    integration.id,
-    {
-      syncedAt,
-    },
-    userId
+        // update integration syncedAt if successful
+        const updated = await updateIntegration(
+          integration.id,
+          {
+            syncedAt,
+          },
+          userId
+        )
+        logger.info('integration updated', {
+          ...logObject,
+          updated,
+        })
+      } catch (error) {
+        logger.error('failed to export item', {
+          userId,
+          integrationId: integration.id,
+          error,
+        })
+      }
+    })
   )
-  logger.info('integration updated', {
-    ...logObject,
-    updated,
-  })
 
   return true
 }
diff --git a/packages/api/src/services/integrations/notion.ts b/packages/api/src/services/integrations/notion.ts
index 539c1a3ef..868a5ec57 100644
--- a/packages/api/src/services/integrations/notion.ts
+++ b/packages/api/src/services/integrations/notion.ts
@@ -35,6 +35,8 @@ export class NotionClient implements IntegrationClient {
     baseURL: 'https://api.notion.com/v1',
     timeout: this._timeout,
   })
+  _parentPageId = process.env.NOTION_PAGE_ID
+
   _token: string
   _client: Client
 
@@ -83,9 +85,13 @@ export class NotionClient implements IntegrationClient {
   }
 
   private _itemToNotionPage = (item: LibraryItem): NotionPage => {
+    if (!this._parentPageId) {
+      throw new Error('Notion parent page ID is not set')
+    }
+
     return {
       parent: {
-        page_id: '83a3f627ab9e44ac83fe657141aec615',
+        page_id: this._parentPageId,
       },
       cover: item.thumbnail
         ? {
@@ -111,9 +117,6 @@ export class NotionClient implements IntegrationClient {
   }
 
   export = async (items: LibraryItem[]): Promise<boolean> => {
-    // find/create a parent page for all the items
-    const parentPageName = 'Omnivore'
-
     const pages = items.map(this._itemToNotionPage)
     await Promise.all(pages.map((page) => this._createPage(page)))
 
diff --git a/packages/api/src/services/library_item.ts b/packages/api/src/services/library_item.ts
index 4a16c7c18..784b79c8e 100644
--- a/packages/api/src/services/library_item.ts
+++ b/packages/api/src/services/library_item.ts
@@ -989,7 +989,7 @@ export const createOrUpdateLibraryItem = async (
     )
   }
 
-  if (skipPubSub) {
+  if (skipPubSub || libraryItem.state === LibraryItemState.Processing) {
     return newLibraryItem
   }
 

From 1edf801b024ebe32bd7f18cfa420320e618e3221 Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Tue, 5 Mar 2024 16:13:50 +0800
Subject: [PATCH 08/67] add settings jsonb column to the integrations

---
 packages/api/src/entity/integration.ts                   | 3 +++
 packages/api/src/generated/graphql.ts                    | 7 +++++++
 packages/api/src/generated/schema.graphql                | 4 ++++
 packages/api/src/jobs/integration/export_item.ts         | 6 +++++-
 packages/api/src/resolvers/integrations/index.ts         | 8 +++++++-
 packages/api/src/routers/integration_router.ts           | 2 +-
 packages/api/src/schema.ts                               | 2 ++
 packages/api/src/services/integrations/index.ts          | 5 +++--
 packages/api/src/services/integrations/notion.ts         | 8 +++++++-
 .../0167.do.add_settings_column_to_integrations.sql      | 9 +++++++++
 .../0167.undo.add_settings_column_to_integrations.sql    | 9 +++++++++
 11 files changed, 57 insertions(+), 6 deletions(-)
 create mode 100755 packages/db/migrations/0167.do.add_settings_column_to_integrations.sql
 create mode 100755 packages/db/migrations/0167.undo.add_settings_column_to_integrations.sql

diff --git a/packages/api/src/entity/integration.ts b/packages/api/src/entity/integration.ts
index e446f10c3..dcb7f03c1 100644
--- a/packages/api/src/entity/integration.ts
+++ b/packages/api/src/entity/integration.ts
@@ -59,4 +59,7 @@ export class Integration {
 
   @Column('enum', { enum: ImportItemState, nullable: true })
   importItemState?: ImportItemState | null
+
+  @Column('jsonb', { nullable: true })
+  settings?: any
 }
diff --git a/packages/api/src/generated/graphql.ts b/packages/api/src/generated/graphql.ts
index 581880e4f..ec46d4c65 100644
--- a/packages/api/src/generated/graphql.ts
+++ b/packages/api/src/generated/graphql.ts
@@ -1085,6 +1085,7 @@ export type Integration = {
   enabled: Scalars['Boolean'];
   id: Scalars['ID'];
   name: Scalars['String'];
+  settings?: Maybe<Scalars['JSON']>;
   taskName?: Maybe<Scalars['String']>;
   token: Scalars['String'];
   type: IntegrationType;
@@ -2404,6 +2405,7 @@ export enum SearchErrorCode {
 
 export type SearchItem = {
   __typename?: 'SearchItem';
+  aiSummary?: Maybe<Scalars['String']>;
   annotation?: Maybe<Scalars['String']>;
   archivedAt?: Maybe<Scalars['Date']>;
   author?: Maybe<Scalars['String']>;
@@ -2586,6 +2588,7 @@ export type SetIntegrationInput = {
   id?: InputMaybe<Scalars['ID']>;
   importItemState?: InputMaybe<ImportItemState>;
   name: Scalars['String'];
+  settings?: InputMaybe<Scalars['JSON']>;
   syncedAt?: InputMaybe<Scalars['Date']>;
   taskName?: InputMaybe<Scalars['String']>;
   token: Scalars['String'];
@@ -3378,6 +3381,7 @@ export enum UploadImportFileType {
 export type User = {
   __typename?: 'User';
   email?: Maybe<Scalars['String']>;
+  features?: Maybe<Array<Maybe<Scalars['String']>>>;
   followersCount?: Maybe<Scalars['Int']>;
   friendsCount?: Maybe<Scalars['Int']>;
   id: Scalars['ID'];
@@ -5310,6 +5314,7 @@ export type IntegrationResolvers<ContextType = ResolverContext, ParentType exten
   enabled?: Resolver<ResolversTypes['Boolean'], ParentType, ContextType>;
   id?: Resolver<ResolversTypes['ID'], ParentType, ContextType>;
   name?: Resolver<ResolversTypes['String'], ParentType, ContextType>;
+  settings?: Resolver<Maybe<ResolversTypes['JSON']>, ParentType, ContextType>;
   taskName?: Resolver<Maybe<ResolversTypes['String']>, ParentType, ContextType>;
   token?: Resolver<ResolversTypes['String'], ParentType, ContextType>;
   type?: Resolver<ResolversTypes['IntegrationType'], ParentType, ContextType>;
@@ -5945,6 +5950,7 @@ export type SearchErrorResolvers<ContextType = ResolverContext, ParentType exten
 };
 
 export type SearchItemResolvers<ContextType = ResolverContext, ParentType extends ResolversParentTypes['SearchItem'] = ResolversParentTypes['SearchItem']> = {
+  aiSummary?: Resolver<Maybe<ResolversTypes['String']>, ParentType, ContextType>;
   annotation?: Resolver<Maybe<ResolversTypes['String']>, ParentType, ContextType>;
   archivedAt?: Resolver<Maybe<ResolversTypes['Date']>, ParentType, ContextType>;
   author?: Resolver<Maybe<ResolversTypes['String']>, ParentType, ContextType>;
@@ -6528,6 +6534,7 @@ export type UploadImportFileSuccessResolvers<ContextType = ResolverContext, Pare
 
 export type UserResolvers<ContextType = ResolverContext, ParentType extends ResolversParentTypes['User'] = ResolversParentTypes['User']> = {
   email?: Resolver<Maybe<ResolversTypes['String']>, ParentType, ContextType>;
+  features?: Resolver<Maybe<Array<Maybe<ResolversTypes['String']>>>, ParentType, ContextType>;
   followersCount?: Resolver<Maybe<ResolversTypes['Int']>, ParentType, ContextType>;
   friendsCount?: Resolver<Maybe<ResolversTypes['Int']>, ParentType, ContextType>;
   id?: Resolver<ResolversTypes['ID'], ParentType, ContextType>;
diff --git a/packages/api/src/generated/schema.graphql b/packages/api/src/generated/schema.graphql
index b344a250a..bf37f919d 100644
--- a/packages/api/src/generated/schema.graphql
+++ b/packages/api/src/generated/schema.graphql
@@ -968,6 +968,7 @@ type Integration {
   enabled: Boolean!
   id: ID!
   name: String!
+  settings: JSON
   taskName: String
   token: String!
   type: IntegrationType!
@@ -1832,6 +1833,7 @@ enum SearchErrorCode {
 }
 
 type SearchItem {
+  aiSummary: String
   annotation: String
   archivedAt: Date
   author: String
@@ -2001,6 +2003,7 @@ input SetIntegrationInput {
   id: ID
   importItemState: ImportItemState
   name: String!
+  settings: JSON
   syncedAt: Date
   taskName: String
   token: String!
@@ -2733,6 +2736,7 @@ enum UploadImportFileType {
 
 type User {
   email: String
+  features: [String]
   followersCount: Int
   friendsCount: Int
   id: ID!
diff --git a/packages/api/src/jobs/integration/export_item.ts b/packages/api/src/jobs/integration/export_item.ts
index a1b8f7cc2..277ff5a9a 100644
--- a/packages/api/src/jobs/integration/export_item.ts
+++ b/packages/api/src/jobs/integration/export_item.ts
@@ -44,7 +44,11 @@ export const exportItem = async (jobData: ExportItemJobData) => {
         }
         logger.info('exporting item...', logObject)
 
-        const client = getIntegrationClient(integration.name, integration.token)
+        const client = getIntegrationClient(
+          integration.name,
+          integration.token,
+          integration.settings
+        )
 
         const synced = await client.export(libraryItems)
         if (!synced) {
diff --git a/packages/api/src/resolvers/integrations/index.ts b/packages/api/src/resolvers/integrations/index.ts
index 5148d0893..5d2a01525 100644
--- a/packages/api/src/resolvers/integrations/index.ts
+++ b/packages/api/src/resolvers/integrations/index.ts
@@ -55,6 +55,8 @@ export const setIntegrationResolver = authorized<
         input.type === IntegrationType.Import
           ? input.importItemState || ImportItemState.Unarchived // default to unarchived
           : undefined,
+      // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment
+      settings: input.settings,
     }
     if (input.id) {
       // Update
@@ -69,7 +71,11 @@ export const setIntegrationResolver = authorized<
       integrationToSave.taskName = existingIntegration.taskName
     } else {
       // Create
-      const integrationService = getIntegrationClient(input.name, input.token)
+      const integrationService = getIntegrationClient(
+        input.name,
+        input.token,
+        input.settings
+      )
       // authorize and get access token
       const token = await integrationService.accessToken()
       if (!token) {
diff --git a/packages/api/src/routers/integration_router.ts b/packages/api/src/routers/integration_router.ts
index 7bf5cbdc6..82645b8de 100644
--- a/packages/api/src/routers/integration_router.ts
+++ b/packages/api/src/routers/integration_router.ts
@@ -21,7 +21,7 @@ export function integrationRouter() {
         return res.status(401).send('UNAUTHORIZED')
       }
 
-      const integrationClient = getIntegrationClient(req.params.name, '')
+      const integrationClient = getIntegrationClient(req.params.name, '', null)
 
       // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
       const state = req.body.state as string
diff --git a/packages/api/src/schema.ts b/packages/api/src/schema.ts
index 86b1f46e2..aec8756e9 100755
--- a/packages/api/src/schema.ts
+++ b/packages/api/src/schema.ts
@@ -2008,6 +2008,7 @@ const schema = gql`
     createdAt: Date!
     updatedAt: Date
     taskName: String
+    settings: JSON
   }
 
   enum IntegrationType {
@@ -2043,6 +2044,7 @@ const schema = gql`
     syncedAt: Date
     importItemState: ImportItemState
     taskName: String
+    settings: JSON
   }
 
   union IntegrationsResult = IntegrationsSuccess | IntegrationsError
diff --git a/packages/api/src/services/integrations/index.ts b/packages/api/src/services/integrations/index.ts
index 953199052..aa59f900b 100644
--- a/packages/api/src/services/integrations/index.ts
+++ b/packages/api/src/services/integrations/index.ts
@@ -8,7 +8,8 @@ import { ReadwiseClient } from './readwise'
 
 export const getIntegrationClient = (
   name: string,
-  token: string
+  token: string,
+  settings: any
 ): IntegrationClient => {
   switch (name.toLowerCase()) {
     case 'readwise':
@@ -16,7 +17,7 @@ export const getIntegrationClient = (
     case 'pocket':
       return new PocketClient(token)
     case 'notion':
-      return new NotionClient(token)
+      return new NotionClient(token, settings)
     default:
       throw new Error(`Integration client not found: ${name}`)
   }
diff --git a/packages/api/src/services/integrations/notion.ts b/packages/api/src/services/integrations/notion.ts
index 868a5ec57..085c20566 100644
--- a/packages/api/src/services/integrations/notion.ts
+++ b/packages/api/src/services/integrations/notion.ts
@@ -23,6 +23,10 @@ interface NotionPage {
   }
 }
 
+interface Settings {
+  parentPageId: string
+}
+
 export class NotionClient implements IntegrationClient {
   name = 'NOTION'
   _headers = {
@@ -39,13 +43,15 @@ export class NotionClient implements IntegrationClient {
 
   _token: string
   _client: Client
+  _settings: Settings
 
-  constructor(token: string) {
+  constructor(token: string, settings: Settings) {
     this._token = token
     this._client = new Client({
       auth: token,
       timeoutMs: this._timeout,
     })
+    this._settings = settings
   }
 
   accessToken = async (): Promise<string | null> => {
diff --git a/packages/db/migrations/0167.do.add_settings_column_to_integrations.sql b/packages/db/migrations/0167.do.add_settings_column_to_integrations.sql
new file mode 100755
index 000000000..d8c2521b8
--- /dev/null
+++ b/packages/db/migrations/0167.do.add_settings_column_to_integrations.sql
@@ -0,0 +1,9 @@
+-- Type: DO
+-- Name: add_settings_column_to_integrations
+-- Description: Add settings column to integrations table
+
+BEGIN;
+
+ALTER TABLE omnivore.integrations ADD COLUMN settings jsonb;
+
+COMMIT;
diff --git a/packages/db/migrations/0167.undo.add_settings_column_to_integrations.sql b/packages/db/migrations/0167.undo.add_settings_column_to_integrations.sql
new file mode 100755
index 000000000..34a03ae7e
--- /dev/null
+++ b/packages/db/migrations/0167.undo.add_settings_column_to_integrations.sql
@@ -0,0 +1,9 @@
+-- Type: UNDO
+-- Name: add_settings_column_to_integrations
+-- Description: Add settings column to integrations table
+
+BEGIN;
+
+ALTER TABLE omnivore.integrations DROP COLUMN settings;
+
+COMMIT;

From dfa551251218f980dc0d71304714b89af26c04bb Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Tue, 5 Mar 2024 16:18:15 +0800
Subject: [PATCH 09/67] get parentPageId from settings

---
 packages/api/src/services/integrations/notion.ts | 7 +------
 1 file changed, 1 insertion(+), 6 deletions(-)

diff --git a/packages/api/src/services/integrations/notion.ts b/packages/api/src/services/integrations/notion.ts
index 085c20566..85ace7508 100644
--- a/packages/api/src/services/integrations/notion.ts
+++ b/packages/api/src/services/integrations/notion.ts
@@ -39,7 +39,6 @@ export class NotionClient implements IntegrationClient {
     baseURL: 'https://api.notion.com/v1',
     timeout: this._timeout,
   })
-  _parentPageId = process.env.NOTION_PAGE_ID
 
   _token: string
   _client: Client
@@ -91,13 +90,9 @@ export class NotionClient implements IntegrationClient {
   }
 
   private _itemToNotionPage = (item: LibraryItem): NotionPage => {
-    if (!this._parentPageId) {
-      throw new Error('Notion parent page ID is not set')
-    }
-
     return {
       parent: {
-        page_id: this._parentPageId,
+        page_id: this._settings.parentPageId,
       },
       cover: item.thumbnail
         ? {

From ec32945ba2ea9e4e5209be2539cce9ba71579e62 Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Tue, 5 Mar 2024 21:00:39 +0800
Subject: [PATCH 10/67] update block

---
 .../api/src/services/integrations/notion.ts   | 47 +++++++++++++++----
 1 file changed, 39 insertions(+), 8 deletions(-)

diff --git a/packages/api/src/services/integrations/notion.ts b/packages/api/src/services/integrations/notion.ts
index 85ace7508..9fc80c21d 100644
--- a/packages/api/src/services/integrations/notion.ts
+++ b/packages/api/src/services/integrations/notion.ts
@@ -1,30 +1,45 @@
 import { Client } from '@notionhq/client'
 import axios from 'axios'
-import { LibraryItem } from '../../entity/library_item'
+import { LibraryItem, LibraryItemState } from '../../entity/library_item'
 import { env } from '../../env'
 import { logger } from '../../utils/logger'
 import { IntegrationClient } from './integration'
 
 interface NotionPage {
   parent: {
-    page_id: string
+    database_id: string
   }
+  id: string
+  created_time: string
+  last_edited_time: string
+  archived: boolean
+  public_url: string
   cover?: {
     external: {
       url: string
     }
   }
+  icon?: {
+    external: {
+      url: string
+    }
+  }
   properties: {
-    title: Array<{
-      text: {
-        content: string
+    title: [
+      {
+        text: {
+          content: string
+          link: {
+            url: string
+          }
+        }
       }
-    }>
+    ]
   }
 }
 
 interface Settings {
-  parentPageId: string
+  parentDatabaseId: string
 }
 
 export class NotionClient implements IntegrationClient {
@@ -92,8 +107,20 @@ export class NotionClient implements IntegrationClient {
   private _itemToNotionPage = (item: LibraryItem): NotionPage => {
     return {
       parent: {
-        page_id: this._settings.parentPageId,
+        database_id: this._settings.parentDatabaseId,
       },
+      id: item.id,
+      archived: item.state === LibraryItemState.Archived,
+      created_time: item.savedAt.toISOString(),
+      last_edited_time: item.updatedAt.toISOString(),
+      public_url: item.originalUrl,
+      icon: item.siteIcon
+        ? {
+            external: {
+              url: item.siteIcon,
+            },
+          }
+        : undefined,
       cover: item.thumbnail
         ? {
             external: {
@@ -106,6 +133,9 @@ export class NotionClient implements IntegrationClient {
           {
             text: {
               content: item.title,
+              link: {
+                url: item.originalUrl,
+              },
             },
           },
         ],
@@ -119,6 +149,7 @@ export class NotionClient implements IntegrationClient {
 
   export = async (items: LibraryItem[]): Promise<boolean> => {
     const pages = items.map(this._itemToNotionPage)
+    console.log('pages', JSON.stringify(pages, null, 2))
     await Promise.all(pages.map((page) => this._createPage(page)))
 
     return true

From 39778feb8841214a29315d88da61524718306243 Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Wed, 6 Mar 2024 20:23:42 +0800
Subject: [PATCH 11/67] export highlights and labels

---
 .../api/src/jobs/integration/export_item.ts   |   2 +-
 .../api/src/resolvers/integrations/index.ts   |   6 +-
 .../api/src/routers/integration_router.ts     |   2 +-
 .../api/src/services/integrations/index.ts    |   4 +-
 .../api/src/services/integrations/notion.ts   | 202 +++++++++++++++---
 5 files changed, 175 insertions(+), 41 deletions(-)

diff --git a/packages/api/src/jobs/integration/export_item.ts b/packages/api/src/jobs/integration/export_item.ts
index 277ff5a9a..318a94ec0 100644
--- a/packages/api/src/jobs/integration/export_item.ts
+++ b/packages/api/src/jobs/integration/export_item.ts
@@ -47,7 +47,7 @@ export const exportItem = async (jobData: ExportItemJobData) => {
         const client = getIntegrationClient(
           integration.name,
           integration.token,
-          integration.settings
+          integration
         )
 
         const synced = await client.export(libraryItems)
diff --git a/packages/api/src/resolvers/integrations/index.ts b/packages/api/src/resolvers/integrations/index.ts
index 5d2a01525..d50992b28 100644
--- a/packages/api/src/resolvers/integrations/index.ts
+++ b/packages/api/src/resolvers/integrations/index.ts
@@ -71,11 +71,7 @@ export const setIntegrationResolver = authorized<
       integrationToSave.taskName = existingIntegration.taskName
     } else {
       // Create
-      const integrationService = getIntegrationClient(
-        input.name,
-        input.token,
-        input.settings
-      )
+      const integrationService = getIntegrationClient(input.name, input.token)
       // authorize and get access token
       const token = await integrationService.accessToken()
       if (!token) {
diff --git a/packages/api/src/routers/integration_router.ts b/packages/api/src/routers/integration_router.ts
index 82645b8de..7bf5cbdc6 100644
--- a/packages/api/src/routers/integration_router.ts
+++ b/packages/api/src/routers/integration_router.ts
@@ -21,7 +21,7 @@ export function integrationRouter() {
         return res.status(401).send('UNAUTHORIZED')
       }
 
-      const integrationClient = getIntegrationClient(req.params.name, '', null)
+      const integrationClient = getIntegrationClient(req.params.name, '')
 
       // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
       const state = req.body.state as string
diff --git a/packages/api/src/services/integrations/index.ts b/packages/api/src/services/integrations/index.ts
index aa59f900b..a925a2765 100644
--- a/packages/api/src/services/integrations/index.ts
+++ b/packages/api/src/services/integrations/index.ts
@@ -9,7 +9,7 @@ import { ReadwiseClient } from './readwise'
 export const getIntegrationClient = (
   name: string,
   token: string,
-  settings: any
+  integrationData?: Integration
 ): IntegrationClient => {
   switch (name.toLowerCase()) {
     case 'readwise':
@@ -17,7 +17,7 @@ export const getIntegrationClient = (
     case 'pocket':
       return new PocketClient(token)
     case 'notion':
-      return new NotionClient(token, settings)
+      return new NotionClient(token, integrationData)
     default:
       throw new Error(`Integration client not found: ${name}`)
   }
diff --git a/packages/api/src/services/integrations/notion.ts b/packages/api/src/services/integrations/notion.ts
index 9fc80c21d..b695b3737 100644
--- a/packages/api/src/services/integrations/notion.ts
+++ b/packages/api/src/services/integrations/notion.ts
@@ -1,19 +1,38 @@
 import { Client } from '@notionhq/client'
 import axios from 'axios'
-import { LibraryItem, LibraryItemState } from '../../entity/library_item'
+import { updateIntegration } from '.'
+import { Integration } from '../../entity/integration'
+import { LibraryItem } from '../../entity/library_item'
 import { env } from '../../env'
+import { Merge } from '../../util'
 import { logger } from '../../utils/logger'
 import { IntegrationClient } from './integration'
 
+type AnnotationColor =
+  | 'default'
+  | 'gray'
+  | 'brown'
+  | 'orange'
+  | 'yellow'
+  | 'green'
+  | 'blue'
+  | 'purple'
+  | 'pink'
+  | 'red'
+  | 'gray_background'
+  | 'brown_background'
+  | 'orange_background'
+  | 'yellow_background'
+  | 'green_background'
+  | 'blue_background'
+  | 'purple_background'
+  | 'pink_background'
+  | 'red_background'
+
 interface NotionPage {
   parent: {
     database_id: string
   }
-  id: string
-  created_time: string
-  last_edited_time: string
-  archived: boolean
-  public_url: string
   cover?: {
     external: {
       url: string
@@ -25,20 +44,48 @@ interface NotionPage {
     }
   }
   properties: {
-    title: [
-      {
-        text: {
-          content: string
-          link: {
-            url: string
+    Title: {
+      title: [
+        {
+          text: {
+            content: string
           }
         }
-      }
-    ]
+      ]
+    }
+    'Original URL': {
+      url: string | null
+    }
+    'Omnivore URL': {
+      url: string | null
+    }
+    Tags?: {
+      multi_select: Array<{ name: string }>
+    }
   }
+  children?: Array<{
+    type: 'paragraph'
+    paragraph: {
+      rich_text: Array<{
+        text: {
+          content: string
+          link?: { url: string }
+        }
+        annotations?: {
+          bold?: boolean
+          italic?: boolean
+          strikethrough?: boolean
+          underline?: boolean
+          code?: boolean
+          color?: AnnotationColor
+        }
+      }>
+    }
+  }>
 }
 
 interface Settings {
+  parentPageId: string
   parentDatabaseId: string
 }
 
@@ -57,15 +104,15 @@ export class NotionClient implements IntegrationClient {
 
   _token: string
   _client: Client
-  _settings: Settings
+  _integrationData?: Merge<Integration, { settings?: Settings }>
 
-  constructor(token: string, settings: Settings) {
+  constructor(token: string, integration?: Integration) {
     this._token = token
     this._client = new Client({
       auth: token,
       timeoutMs: this._timeout,
     })
-    this._settings = settings
+    this._integrationData = integration
   }
 
   accessToken = async (): Promise<string | null> => {
@@ -105,15 +152,15 @@ export class NotionClient implements IntegrationClient {
   }
 
   private _itemToNotionPage = (item: LibraryItem): NotionPage => {
+    const databaseId = this._integrationData?.settings?.parentDatabaseId
+    if (!databaseId) {
+      throw new Error('Notion database id not found')
+    }
+
     return {
       parent: {
-        database_id: this._settings.parentDatabaseId,
+        database_id: databaseId,
       },
-      id: item.id,
-      archived: item.state === LibraryItemState.Archived,
-      created_time: item.savedAt.toISOString(),
-      last_edited_time: item.updatedAt.toISOString(),
-      public_url: item.originalUrl,
       icon: item.siteIcon
         ? {
             external: {
@@ -129,17 +176,50 @@ export class NotionClient implements IntegrationClient {
           }
         : undefined,
       properties: {
-        title: [
-          {
-            text: {
-              content: item.title,
-              link: {
-                url: item.originalUrl,
+        Title: {
+          title: [
+            {
+              text: {
+                content: item.title,
               },
             },
-          },
-        ],
+          ],
+        },
+        'Original URL': {
+          url: item.originalUrl,
+        },
+        'Omnivore URL': {
+          url: `${env.client.url}/me/${item.slug}`,
+        },
+        Tags: item.labels
+          ? { multi_select: item.labels.map((label) => ({ name: label.name })) }
+          : undefined,
       },
+      children: item.highlights
+        ? item.highlights.map((highlight) => ({
+            type: 'paragraph',
+            paragraph: {
+              rich_text: [
+                {
+                  text: {
+                    content: highlight.quote || '',
+                  },
+                  annotations: {
+                    color: highlight.color as AnnotationColor,
+                  },
+                },
+                {
+                  text: {
+                    content: highlight.annotation || '',
+                  },
+                  annotations: {
+                    italic: true,
+                  },
+                },
+              ],
+            },
+          }))
+        : undefined,
     }
   }
 
@@ -148,8 +228,66 @@ export class NotionClient implements IntegrationClient {
   }
 
   export = async (items: LibraryItem[]): Promise<boolean> => {
+    if (!this._integrationData || !this._integrationData.settings) {
+      logger.error('Notion integration data not found')
+      return false
+    }
+
+    const pageId = this._integrationData.settings.parentPageId
+    if (!pageId) {
+      logger.error('Notion parent page id not found')
+      return false
+    }
+
+    const databaseId = this._integrationData.settings.parentDatabaseId
+    if (!databaseId) {
+      // create a database for the items
+      const database = await this._client.databases.create({
+        parent: {
+          page_id: pageId,
+        },
+        title: [
+          {
+            text: {
+              content: 'Library',
+            },
+          },
+        ],
+        description: [
+          {
+            text: {
+              content: 'Library of saved items from Omnivore',
+            },
+          },
+        ],
+        properties: {
+          Title: {
+            title: {},
+          },
+          'Original URL': {
+            url: {},
+          },
+          'Omnivore URL': {
+            url: {},
+          },
+          Tags: {
+            multi_select: {},
+          },
+        },
+      })
+
+      // save the database id
+      this._integrationData.settings.parentDatabaseId = database.id
+      await updateIntegration(
+        this._integrationData.id,
+        {
+          settings: this._integrationData.settings,
+        },
+        this._integrationData.user.id
+      )
+    }
+
     const pages = items.map(this._itemToNotionPage)
-    console.log('pages', JSON.stringify(pages, null, 2))
     await Promise.all(pages.map((page) => this._createPage(page)))
 
     return true

From 44178c1c1ef6ba089979d556664d090b7549c384 Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Thu, 7 Mar 2024 13:24:02 +0800
Subject: [PATCH 12/67] Add author

---
 .../api/src/services/integrations/notion.ts   | 21 ++++++++++++++++++-
 1 file changed, 20 insertions(+), 1 deletion(-)

diff --git a/packages/api/src/services/integrations/notion.ts b/packages/api/src/services/integrations/notion.ts
index b695b3737..4a21d5606 100644
--- a/packages/api/src/services/integrations/notion.ts
+++ b/packages/api/src/services/integrations/notion.ts
@@ -53,6 +53,13 @@ interface NotionPage {
         }
       ]
     }
+    Author: {
+      rich_text: Array<{
+        text: {
+          content: string
+        }
+      }>
+    }
     'Original URL': {
       url: string | null
     }
@@ -185,6 +192,15 @@ export class NotionClient implements IntegrationClient {
             },
           ],
         },
+        Author: {
+          rich_text: [
+            {
+              text: {
+                content: item.author || 'unknown',
+              },
+            },
+          ],
+        },
         'Original URL': {
           url: item.originalUrl,
         },
@@ -210,7 +226,7 @@ export class NotionClient implements IntegrationClient {
                 },
                 {
                   text: {
-                    content: highlight.annotation || '',
+                    content: `\n${highlight.annotation || ''}`,
                   },
                   annotations: {
                     italic: true,
@@ -264,6 +280,9 @@ export class NotionClient implements IntegrationClient {
           Title: {
             title: {},
           },
+          Author: {
+            rich_text: {},
+          },
           'Original URL': {
             url: {},
           },

From e8e93c1e7d984bf62bf76efe040f1c2d6ed94639 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Thu, 7 Mar 2024 18:11:34 +0000
Subject: [PATCH 13/67] Bump jose from 2.0.6 to 2.0.7

Bumps [jose](https://github.com/panva/jose) from 2.0.6 to 2.0.7.
- [Release notes](https://github.com/panva/jose/releases)
- [Changelog](https://github.com/panva/jose/blob/v2.0.7/CHANGELOG.md)
- [Commits](https://github.com/panva/jose/compare/v2.0.6...v2.0.7)

---
updated-dependencies:
- dependency-name: jose
  dependency-type: indirect
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 yarn.lock | 84 ++++++-------------------------------------------------
 1 file changed, 9 insertions(+), 75 deletions(-)

diff --git a/yarn.lock b/yarn.lock
index cc11cf86b..d181c6fbb 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -6084,11 +6084,6 @@
   resolved "https://registry.yarnpkg.com/@sqltools/formatter/-/formatter-1.2.3.tgz#1185726610acc37317ddab11c3c7f9066966bd20"
   integrity sha512-O3uyB/JbkAEMZaP3YqyHH7TMnex7tWyCbCI4EfJdOCoN6HIhqdJBWTM6aCCiWQ/5f5wxjgU735QAIpJbjDvmzg==
 
-"@sqltools/formatter@^1.2.5":
-  version "1.2.5"
-  resolved "https://registry.yarnpkg.com/@sqltools/formatter/-/formatter-1.2.5.tgz#3abc203c79b8c3e90fd6c156a0c62d5403520e12"
-  integrity sha512-Uy0+khmZqUrUGm5dmMqVlnvufZRSK0FbYzVgp0UMstm+F5+W2/jnEEQyc9vo1ZR/E5ZI/B1WjjoTqBqwJL6Krw==
-
 "@stitches/react@^1.2.5":
   version "1.2.8"
   resolved "https://registry.yarnpkg.com/@stitches/react/-/react-1.2.8.tgz#954f8008be8d9c65c4e58efa0937f32388ce3a38"
@@ -7932,13 +7927,6 @@
   dependencies:
     undici-types "~5.26.4"
 
-"@types/node@^20.11.0":
-  version "20.11.24"
-  resolved "https://registry.yarnpkg.com/@types/node/-/node-20.11.24.tgz#cc207511104694e84e9fb17f9a0c4c42d4517792"
-  integrity sha512-Kza43ewS3xoLgCEpQrsT+xRo/EJej1y0kVYGiLFE1NEODXGzTfwiC6tXTLMQskn1X4/Rjlh0MQUvx9W+L9long==
-  dependencies:
-    undici-types "~5.26.4"
-
 "@types/nodemailer@^6.4.4":
   version "6.4.4"
   resolved "https://registry.yarnpkg.com/@types/nodemailer/-/nodemailer-6.4.4.tgz#c265f7e7a51df587597b3a49a023acaf0c741f4b"
@@ -9466,11 +9454,6 @@ app-root-path@^3.0.0:
   resolved "https://registry.yarnpkg.com/app-root-path/-/app-root-path-3.0.0.tgz#210b6f43873227e18a4b810a032283311555d5ad"
   integrity sha512-qMcx+Gy2UZynHjOHOIXPNvpf+9cjvk3cWrBBK7zg4gH9+clobJRb9NGzcT7mQTcV/6Gm/1WelUtqxVXnNlrwcw==
 
-app-root-path@^3.1.0:
-  version "3.1.0"
-  resolved "https://registry.yarnpkg.com/app-root-path/-/app-root-path-3.1.0.tgz#5971a2fc12ba170369a7a1ef018c71e6e47c2e86"
-  integrity sha512-biN3PwB2gUtjaYy/isrU3aNWI5w+fAfvHkSvCKeQGxhmYpwKFUxudR3Yya+KqVRHBmEDYh+/lTozYCFbmzX4nA==
-
 apparatus@^0.0.10:
   version "0.0.10"
   resolved "https://registry.yarnpkg.com/apparatus/-/apparatus-0.0.10.tgz#81ea756772ada77863db54ceee8202c109bdca3e"
@@ -12929,7 +12912,7 @@ dateformat@^3.0.0, dateformat@^3.0.3:
   resolved "https://registry.yarnpkg.com/dateformat/-/dateformat-3.0.3.tgz#a6e37499a4d9a9cf85ef5872044d62901c9889ae"
   integrity sha512-jyCETtSl3VMZMWeRo7iY1FL19ges1t55hMo5yaam4Jrsm5EPL89UQkoQRyiI+Yf4k8r2ZpdngkV8hr1lIdjb3Q==
 
-dayjs@1.x, dayjs@^1.10.4, dayjs@^1.11.7, dayjs@^1.11.9:
+dayjs@1.x, dayjs@^1.10.4, dayjs@^1.11.7:
   version "1.11.10"
   resolved "https://registry.yarnpkg.com/dayjs/-/dayjs-1.11.10.tgz#68acea85317a6e164457d6d6947564029a6a16a0"
   integrity sha512-vjAczensTgRcqDERK0SR2XMwsF/tSvnvlv6VcF2GIhg6Sx4yOIt/irsr1RDJsKiIyBzJDpCoXiWWq28MqH2cnQ==
@@ -13660,11 +13643,6 @@ dotenv@^16.0.1:
   resolved "https://registry.yarnpkg.com/dotenv/-/dotenv-16.0.1.tgz#8f8f9d94876c35dac989876a5d3a82a267fdce1d"
   integrity sha512-1K6hR6wtk2FviQ4kEiSjFiH5rpzEVi8WW0x96aztHVMhEspNpc4DVOUTEHtEva5VThQ8IaBX1Pe4gSzpVVUsKQ==
 
-dotenv@^16.0.3:
-  version "16.4.5"
-  resolved "https://registry.yarnpkg.com/dotenv/-/dotenv-16.4.5.tgz#cdd3b3b604cb327e286b4762e13502f717cb099f"
-  integrity sha512-ZmdL2rui+eB2YwhsWzjInR8LldtZHGDoQ1ugH85ppHKwpUHL7j7rN0Ti9NCnGiQbhaZ11FpR+7ao1dNsmduNUg==
-
 dotenv@^8.0.0, dotenv@^8.2.0:
   version "8.6.0"
   resolved "https://registry.yarnpkg.com/dotenv/-/dotenv-8.6.0.tgz#061af664d19f7f4d8fc6e4ff9b584ce237adcb8b"
@@ -16329,7 +16307,7 @@ glob@7.2.0, glob@^7.1.1, glob@^7.1.2, glob@^7.1.3, glob@^7.1.4, glob@^7.1.6, glo
     once "^1.3.0"
     path-is-absolute "^1.0.0"
 
-glob@^10.2.2, glob@^10.3.10:
+glob@^10.2.2:
   version "10.3.10"
   resolved "https://registry.yarnpkg.com/glob/-/glob-10.3.10.tgz#0351ebb809fd187fe421ab96af83d3a70715df4b"
   integrity sha512-fa46+tv1Ak0UPK1TOy/pZrIybNNt4HCv7SDzwyfiOZkvZLEbjsZkJBPtDHVshZjbecAoAGSC20MjLDG/qr679g==
@@ -19299,16 +19277,16 @@ jest@^27.4.5:
     jest-cli "^27.5.1"
 
 jose@^2.0.5:
-  version "2.0.6"
-  resolved "https://registry.yarnpkg.com/jose/-/jose-2.0.6.tgz#894ba19169af339d3911be933f913dd02fc57c7c"
-  integrity sha512-FVoPY7SflDodE4lknJmbAHSUjLCzE2H1F6MS0RYKMQ8SR+lNccpMf8R4eqkNYyyUjR5qZReOzZo5C5YiHOCjjg==
+  version "2.0.7"
+  resolved "https://registry.yarnpkg.com/jose/-/jose-2.0.7.tgz#3aabbaec70bff313c108b9406498a163737b16ba"
+  integrity sha512-5hFWIigKqC+e/lRyQhfnirrAqUdIPMB7SJRqflJaO29dW7q5DFvH1XCSTmv6PQ6pb++0k6MJlLRoS0Wv4s38Wg==
   dependencies:
     "@panva/asn1.js" "^1.0.0"
 
 jose@^4.10.4:
-  version "4.13.1"
-  resolved "https://registry.yarnpkg.com/jose/-/jose-4.13.1.tgz#449111bb5ab171db85c03f1bd2cb1647ca06db1c"
-  integrity sha512-MSJQC5vXco5Br38mzaQKiq9mwt7lwj2eXpgpRyQYNHYt2lq1PjkWa7DLXX0WVcQLE9HhMh3jPiufS7fhJf+CLQ==
+  version "4.15.5"
+  resolved "https://registry.yarnpkg.com/jose/-/jose-4.15.5.tgz#6475d0f467ecd3c630a1b5dadd2735a7288df706"
+  integrity sha512-jc7BFxgKPKi94uOvEmzlSWFFe2+vASyXaKUpdQKatWAESU2MWjDfFf0fdfc83CDKcA5QecabZeNLyfhe3yKNkg==
 
 js-beautify@^1.13.0:
   version "1.14.0"
@@ -22081,11 +22059,6 @@ mkdirp@^1.0.3, mkdirp@^1.0.4:
   resolved "https://registry.yarnpkg.com/mkdirp/-/mkdirp-1.0.4.tgz#3eb5ed62622756d79a5f0e2a221dfebad75c2f7e"
   integrity sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw==
 
-mkdirp@^2.1.3:
-  version "2.1.6"
-  resolved "https://registry.yarnpkg.com/mkdirp/-/mkdirp-2.1.6.tgz#964fbcb12b2d8c5d6fbc62a963ac95a273e2cc19"
-  integrity sha512-+hEnITedc8LAtIP9u3HJDFIdcLV2vXP33sqLLIzkv1Db1zO/1OxbvYf0Y1OC/S/Qo5dxHXepofhmxL02PsKe+A==
-
 mkdirp@~0.3.5:
   version "0.3.5"
   resolved "https://registry.yarnpkg.com/mkdirp/-/mkdirp-0.3.5.tgz#de3e5f8961c88c787ee1368df849ac4413eca8d7"
@@ -26383,14 +26356,6 @@ read-pkg@^7.1.0:
     parse-json "^5.2.0"
     type-fest "^2.0.0"
 
-read-yaml-file@^2.1.0:
-  version "2.1.0"
-  resolved "https://registry.yarnpkg.com/read-yaml-file/-/read-yaml-file-2.1.0.tgz#c5866712db9ef5343b4d02c2413bada53c41c4a9"
-  integrity sha512-UkRNRIwnhG+y7hpqnycCL/xbTk7+ia9VuVTC0S+zVbwd65DI9eUpRMfsWIGrCWxTU/mi+JW8cHQCrv+zfCbEPQ==
-  dependencies:
-    js-yaml "^4.0.0"
-    strip-bom "^4.0.0"
-
 read@1, read@^1.0.7, read@~1.0.7:
   version "1.0.7"
   resolved "https://registry.yarnpkg.com/read/-/read-1.0.7.tgz#b3da19bd052431a97671d44a42634adf710b40c4"
@@ -26533,11 +26498,6 @@ reflect-metadata@^0.1.13:
   resolved "https://registry.yarnpkg.com/reflect-metadata/-/reflect-metadata-0.1.13.tgz#67ae3ca57c972a2aa1642b10fe363fe32d49dc08"
   integrity sha512-Ts1Y/anZELhSsjMcU605fU9RE4Oi3p5ORujwbIKXfWa+0Zxs510Qrmrce5/Jowq3cHSZSJqBjypxmHarc+vEWg==
 
-reflect-metadata@^0.2.1:
-  version "0.2.1"
-  resolved "https://registry.yarnpkg.com/reflect-metadata/-/reflect-metadata-0.2.1.tgz#8d5513c0f5ef2b4b9c3865287f3c0940c1f67f74"
-  integrity sha512-i5lLI6iw9AU3Uu4szRNPPEkomnkjRTaVt9hy/bn5g/oSzekBSMeLZblcjP74AW0vBabqERLLIrz+gR8QYR54Tw==
-
 reflect.getprototypeof@^1.0.4:
   version "1.0.4"
   resolved "https://registry.yarnpkg.com/reflect.getprototypeof/-/reflect.getprototypeof-1.0.4.tgz#aaccbf41aca3821b87bb71d9dcbc7ad0ba50a3f3"
@@ -29562,7 +29522,7 @@ tslib@^1.0.0, tslib@^1.8.1, tslib@^1.9.0, tslib@^1.9.3:
   resolved "https://registry.yarnpkg.com/tslib/-/tslib-1.14.1.tgz#cf2d38bdc34a134bcaf1091c41f6619e2f672d00"
   integrity sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==
 
-tslib@^2, tslib@^2.0.0, tslib@^2.0.1, tslib@^2.0.3, tslib@^2.1.0, tslib@^2.3.0, tslib@^2.3.1, tslib@^2.4.0, tslib@^2.5.0:
+tslib@^2, tslib@^2.0.0, tslib@^2.0.1, tslib@^2.0.3, tslib@^2.1.0, tslib@^2.3.0, tslib@^2.3.1, tslib@^2.4.0:
   version "2.6.2"
   resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.6.2.tgz#703ac29425e7b37cd6fd456e92404d46d1f3e4ae"
   integrity sha512-AEYxH93jGFPn/a2iVAwW87VuUIkR1FVUKB77NwMF7nBTDkDrrT/Hpt/IrCJ0QXhW27jTBDcf5ZY7w6RiqTMw2Q==
@@ -29778,27 +29738,6 @@ typeorm-naming-strategies@^4.1.0:
   resolved "https://registry.yarnpkg.com/typeorm-naming-strategies/-/typeorm-naming-strategies-4.1.0.tgz#1ec6eb296c8d7b69bb06764d5b9083ff80e814a9"
   integrity sha512-vPekJXzZOTZrdDvTl1YoM+w+sUIfQHG4kZTpbFYoTsufyv9NIBRe4Q+PdzhEAFA2std3D9LZHEb1EjE9zhRpiQ==
 
-typeorm@^0.3.19:
-  version "0.3.20"
-  resolved "https://registry.yarnpkg.com/typeorm/-/typeorm-0.3.20.tgz#4b61d737c6fed4e9f63006f88d58a5e54816b7ab"
-  integrity sha512-sJ0T08dV5eoZroaq9uPKBoNcGslHBR4E4y+EBHs//SiGbblGe7IeduP/IH4ddCcj0qp3PHwDwGnuvqEAnKlq/Q==
-  dependencies:
-    "@sqltools/formatter" "^1.2.5"
-    app-root-path "^3.1.0"
-    buffer "^6.0.3"
-    chalk "^4.1.2"
-    cli-highlight "^2.1.11"
-    dayjs "^1.11.9"
-    debug "^4.3.4"
-    dotenv "^16.0.3"
-    glob "^10.3.10"
-    mkdirp "^2.1.3"
-    reflect-metadata "^0.2.1"
-    sha.js "^2.4.11"
-    tslib "^2.5.0"
-    uuid "^9.0.0"
-    yargs "^17.6.2"
-
 typeorm@^0.3.4:
   version "0.3.7"
   resolved "https://registry.yarnpkg.com/typeorm/-/typeorm-0.3.7.tgz#5776ed5058f0acb75d64723b39ff458d21de64c1"
@@ -29837,11 +29776,6 @@ typescript@^4.4.4:
   resolved "https://registry.yarnpkg.com/typescript/-/typescript-4.9.5.tgz#095979f9bcc0d09da324d58d03ce8f8374cbe65a"
   integrity sha512-1FXk9E2Hm+QzZQ7z+McJiHL4NW1F2EzMu9Nq9i3zAaGqibafqYwCVU6WyWAuyQRRzOlxou8xZSyXLEN8oKj24g==
 
-typescript@^5.3.3:
-  version "5.3.3"
-  resolved "https://registry.yarnpkg.com/typescript/-/typescript-5.3.3.tgz#b3ce6ba258e72e6305ba66f5c9b452aaee3ffe37"
-  integrity sha512-pXWcraxM0uxAS+tN0AG/BF2TyqmHO014Z070UsJ+pFvYuRSq8KH8DmWpnbXe0pEPDHXZV3FcAbJkijJ5oNEnWw==
-
 ua-parser-js@^0.7.30:
   version "0.7.33"
   resolved "https://registry.yarnpkg.com/ua-parser-js/-/ua-parser-js-0.7.33.tgz#1d04acb4ccef9293df6f70f2c3d22f3030d8b532"

From ddfbdab3a2b861a25248d56f81612ffb58b2613a Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 11 Mar 2024 22:22:23 +0000
Subject: [PATCH 14/67] Bump @google-cloud/storage from 7.0.1 to 7.8.0

Bumps [@google-cloud/storage](https://github.com/googleapis/nodejs-storage) from 7.0.1 to 7.8.0.
- [Release notes](https://github.com/googleapis/nodejs-storage/releases)
- [Changelog](https://github.com/googleapis/nodejs-storage/blob/main/CHANGELOG.md)
- [Commits](https://github.com/googleapis/nodejs-storage/compare/v7.0.1...v7.8.0)

---
updated-dependencies:
- dependency-name: "@google-cloud/storage"
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 yarn.lock | 209 ++++++++++++++++++------------------------------------
 1 file changed, 70 insertions(+), 139 deletions(-)

diff --git a/yarn.lock b/yarn.lock
index cc11cf86b..c793d6957 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -2768,12 +2768,7 @@
   resolved "https://registry.yarnpkg.com/@google-cloud/projectify/-/projectify-4.0.0.tgz#d600e0433daf51b88c1fa95ac7f02e38e80a07be"
   integrity sha512-MmaX6HeSvyPbWGwFq7mXdo0uQZLGBYCwziiLIGq5JVX+/bdI3SAq6bP98trV5eTWfLuvsMcIC1YJOF2vfteLFA==
 
-"@google-cloud/promisify@^3.0.0":
-  version "3.0.0"
-  resolved "https://registry.yarnpkg.com/@google-cloud/promisify/-/promisify-3.0.0.tgz#5cd6941fc30c4acac18051706aa5af96069bd3e3"
-  integrity sha512-91ArYvRgXWb73YvEOBMmOcJc0bDRs5yiVHnqkwoG0f3nm7nZuipllz6e7BvFESBvjkDTBC0zMD8QxedUwNLc1A==
-
-"@google-cloud/promisify@^3.0.1":
+"@google-cloud/promisify@^3.0.0", "@google-cloud/promisify@^3.0.1":
   version "3.0.1"
   resolved "https://registry.yarnpkg.com/@google-cloud/promisify/-/promisify-3.0.1.tgz#8d724fb280f47d1ff99953aee0c1669b25238c2e"
   integrity sha512-z1CjRjtQyBOYL+5Qr9DdYIfrdLBe746jRTYfaYU6MeXkqp7UfYs/jX16lFFVzZ7PGEJvqZNqYUEtb1mvDww4pA==
@@ -2830,25 +2825,25 @@
     uuid "^8.0.0"
 
 "@google-cloud/storage@^7.0.1":
-  version "7.0.1"
-  resolved "https://registry.yarnpkg.com/@google-cloud/storage/-/storage-7.0.1.tgz#38c267bb8377d442066d4eccb4f942f58d119476"
-  integrity sha512-YBJ8HaDZvbeVDgEGWuC6sCsfZNCooVfKg1J+CJ4iXwRejIWbKFjl8laWz8w+/+ucJHM9qOdGkB95Q/mhh2CX/A==
+  version "7.8.0"
+  resolved "https://registry.yarnpkg.com/@google-cloud/storage/-/storage-7.8.0.tgz#78b41d575c05b35a6dae2dea0e5c2b09bc5ec532"
+  integrity sha512-4q8rKdLp35z8msAtrhr0pbos7BeD8T0tr6rMbBINewp9cfrwj7ROIElVwBluU8fZ596OvwQcjb6QCyBzTmkMRQ==
   dependencies:
-    "@google-cloud/paginator" "^3.0.7"
-    "@google-cloud/projectify" "^3.0.0"
-    "@google-cloud/promisify" "^3.0.0"
+    "@google-cloud/paginator" "^5.0.0"
+    "@google-cloud/projectify" "^4.0.0"
+    "@google-cloud/promisify" "^4.0.0"
     abort-controller "^3.0.0"
     async-retry "^1.3.3"
     compressible "^2.0.12"
-    duplexify "^4.0.0"
+    duplexify "^4.1.3"
     ent "^2.2.0"
-    fast-xml-parser "^4.2.2"
+    fast-xml-parser "^4.3.0"
     gaxios "^6.0.2"
-    google-auth-library "^9.0.0"
+    google-auth-library "^9.6.3"
     mime "^3.0.0"
     mime-types "^2.0.8"
     p-limit "^3.0.1"
-    retry-request "^6.0.0"
+    retry-request "^7.0.0"
     teeny-request "^9.0.0"
     uuid "^8.0.0"
 
@@ -6084,11 +6079,6 @@
   resolved "https://registry.yarnpkg.com/@sqltools/formatter/-/formatter-1.2.3.tgz#1185726610acc37317ddab11c3c7f9066966bd20"
   integrity sha512-O3uyB/JbkAEMZaP3YqyHH7TMnex7tWyCbCI4EfJdOCoN6HIhqdJBWTM6aCCiWQ/5f5wxjgU735QAIpJbjDvmzg==
 
-"@sqltools/formatter@^1.2.5":
-  version "1.2.5"
-  resolved "https://registry.yarnpkg.com/@sqltools/formatter/-/formatter-1.2.5.tgz#3abc203c79b8c3e90fd6c156a0c62d5403520e12"
-  integrity sha512-Uy0+khmZqUrUGm5dmMqVlnvufZRSK0FbYzVgp0UMstm+F5+W2/jnEEQyc9vo1ZR/E5ZI/B1WjjoTqBqwJL6Krw==
-
 "@stitches/react@^1.2.5":
   version "1.2.8"
   resolved "https://registry.yarnpkg.com/@stitches/react/-/react-1.2.8.tgz#954f8008be8d9c65c4e58efa0937f32388ce3a38"
@@ -7932,13 +7922,6 @@
   dependencies:
     undici-types "~5.26.4"
 
-"@types/node@^20.11.0":
-  version "20.11.24"
-  resolved "https://registry.yarnpkg.com/@types/node/-/node-20.11.24.tgz#cc207511104694e84e9fb17f9a0c4c42d4517792"
-  integrity sha512-Kza43ewS3xoLgCEpQrsT+xRo/EJej1y0kVYGiLFE1NEODXGzTfwiC6tXTLMQskn1X4/Rjlh0MQUvx9W+L9long==
-  dependencies:
-    undici-types "~5.26.4"
-
 "@types/nodemailer@^6.4.4":
   version "6.4.4"
   resolved "https://registry.yarnpkg.com/@types/nodemailer/-/nodemailer-6.4.4.tgz#c265f7e7a51df587597b3a49a023acaf0c741f4b"
@@ -8105,6 +8088,16 @@
     "@types/tough-cookie" "*"
     form-data "^2.5.0"
 
+"@types/request@^2.48.8":
+  version "2.48.12"
+  resolved "https://registry.yarnpkg.com/@types/request/-/request-2.48.12.tgz#0f590f615a10f87da18e9790ac94c29ec4c5ef30"
+  integrity sha512-G3sY+NpsA9jnwm0ixhAFQSJ3Q9JkpLZpJbI3GMv0mIAT0y3mRabYeINzal5WOChIiaTEGQYlHOKgkaM9EisWHw==
+  dependencies:
+    "@types/caseless" "*"
+    "@types/node" "*"
+    "@types/tough-cookie" "*"
+    form-data "^2.5.0"
+
 "@types/retry@0.12.0":
   version "0.12.0"
   resolved "https://registry.yarnpkg.com/@types/retry/-/retry-0.12.0.tgz#2b35eccfcee7d38cd72ad99232fbd58bffb3c84d"
@@ -9466,11 +9459,6 @@ app-root-path@^3.0.0:
   resolved "https://registry.yarnpkg.com/app-root-path/-/app-root-path-3.0.0.tgz#210b6f43873227e18a4b810a032283311555d5ad"
   integrity sha512-qMcx+Gy2UZynHjOHOIXPNvpf+9cjvk3cWrBBK7zg4gH9+clobJRb9NGzcT7mQTcV/6Gm/1WelUtqxVXnNlrwcw==
 
-app-root-path@^3.1.0:
-  version "3.1.0"
-  resolved "https://registry.yarnpkg.com/app-root-path/-/app-root-path-3.1.0.tgz#5971a2fc12ba170369a7a1ef018c71e6e47c2e86"
-  integrity sha512-biN3PwB2gUtjaYy/isrU3aNWI5w+fAfvHkSvCKeQGxhmYpwKFUxudR3Yya+KqVRHBmEDYh+/lTozYCFbmzX4nA==
-
 apparatus@^0.0.10:
   version "0.0.10"
   resolved "https://registry.yarnpkg.com/apparatus/-/apparatus-0.0.10.tgz#81ea756772ada77863db54ceee8202c109bdca3e"
@@ -12929,7 +12917,7 @@ dateformat@^3.0.0, dateformat@^3.0.3:
   resolved "https://registry.yarnpkg.com/dateformat/-/dateformat-3.0.3.tgz#a6e37499a4d9a9cf85ef5872044d62901c9889ae"
   integrity sha512-jyCETtSl3VMZMWeRo7iY1FL19ges1t55hMo5yaam4Jrsm5EPL89UQkoQRyiI+Yf4k8r2ZpdngkV8hr1lIdjb3Q==
 
-dayjs@1.x, dayjs@^1.10.4, dayjs@^1.11.7, dayjs@^1.11.9:
+dayjs@1.x, dayjs@^1.10.4, dayjs@^1.11.7:
   version "1.11.10"
   resolved "https://registry.yarnpkg.com/dayjs/-/dayjs-1.11.10.tgz#68acea85317a6e164457d6d6947564029a6a16a0"
   integrity sha512-vjAczensTgRcqDERK0SR2XMwsF/tSvnvlv6VcF2GIhg6Sx4yOIt/irsr1RDJsKiIyBzJDpCoXiWWq28MqH2cnQ==
@@ -13660,11 +13648,6 @@ dotenv@^16.0.1:
   resolved "https://registry.yarnpkg.com/dotenv/-/dotenv-16.0.1.tgz#8f8f9d94876c35dac989876a5d3a82a267fdce1d"
   integrity sha512-1K6hR6wtk2FviQ4kEiSjFiH5rpzEVi8WW0x96aztHVMhEspNpc4DVOUTEHtEva5VThQ8IaBX1Pe4gSzpVVUsKQ==
 
-dotenv@^16.0.3:
-  version "16.4.5"
-  resolved "https://registry.yarnpkg.com/dotenv/-/dotenv-16.4.5.tgz#cdd3b3b604cb327e286b4762e13502f717cb099f"
-  integrity sha512-ZmdL2rui+eB2YwhsWzjInR8LldtZHGDoQ1ugH85ppHKwpUHL7j7rN0Ti9NCnGiQbhaZ11FpR+7ao1dNsmduNUg==
-
 dotenv@^8.0.0, dotenv@^8.2.0:
   version "8.6.0"
   resolved "https://registry.yarnpkg.com/dotenv/-/dotenv-8.6.0.tgz#061af664d19f7f4d8fc6e4ff9b584ce237adcb8b"
@@ -13718,15 +13701,15 @@ duplexify@^3.4.2, duplexify@^3.6.0:
     readable-stream "^2.0.0"
     stream-shift "^1.0.0"
 
-duplexify@^4.0.0, duplexify@^4.1.1:
-  version "4.1.2"
-  resolved "https://registry.yarnpkg.com/duplexify/-/duplexify-4.1.2.tgz#18b4f8d28289132fa0b9573c898d9f903f81c7b0"
-  integrity sha512-fz3OjcNCHmRP12MJoZMPglx8m4rrFP8rovnk4vT8Fs+aonZoCwGg10dSsQsfP/E62eZcPTMSMP6686fu9Qlqtw==
+duplexify@^4.0.0, duplexify@^4.1.1, duplexify@^4.1.3:
+  version "4.1.3"
+  resolved "https://registry.yarnpkg.com/duplexify/-/duplexify-4.1.3.tgz#a07e1c0d0a2c001158563d32592ba58bddb0236f"
+  integrity sha512-M3BmBhwJRZsSx38lZyhE53Csddgzl5R7xGJNk7CVddZD6CcmwMCH8J+7AprIrQKH7TonKxaCjcv27Qmf+sQ+oA==
   dependencies:
     end-of-stream "^1.4.1"
     inherits "^2.0.3"
     readable-stream "^3.1.1"
-    stream-shift "^1.0.0"
+    stream-shift "^1.0.2"
 
 dynamic-dedupe@^0.3.0:
   version "0.3.0"
@@ -15208,10 +15191,10 @@ fast-text-encoding@^1.0.0, fast-text-encoding@^1.0.3:
   resolved "https://registry.yarnpkg.com/fast-text-encoding/-/fast-text-encoding-1.0.3.tgz#ec02ac8e01ab8a319af182dae2681213cfe9ce53"
   integrity sha512-dtm4QZH9nZtcDt8qJiOH9fcQd1NAgi+K1O2DbE6GG1PPCK/BWfOH3idCTRQ4ImXRUOyopDEgDEnVEE7Y/2Wrig==
 
-fast-xml-parser@^4.2.2:
-  version "4.2.7"
-  resolved "https://registry.yarnpkg.com/fast-xml-parser/-/fast-xml-parser-4.2.7.tgz#871f2ca299dc4334b29f8da3658c164e68395167"
-  integrity sha512-J8r6BriSLO1uj2miOk1NW0YVm8AGOOu3Si2HQp/cSmo6EA4m3fcwu2WKjJ4RK9wMLBtg69y1kS8baDiQBR41Ig==
+fast-xml-parser@^4.2.2, fast-xml-parser@^4.3.0:
+  version "4.3.5"
+  resolved "https://registry.yarnpkg.com/fast-xml-parser/-/fast-xml-parser-4.3.5.tgz#e2f2a2ae8377e9c3dc321b151e58f420ca7e5ccc"
+  integrity sha512-sWvP1Pl8H03B8oFJpFR3HE31HUfwtX7Rlf9BNsvdpujD4n7WMhfmu8h9wOV2u+c1k0ZilTADhPqypzx2J690ZQ==
   dependencies:
     strnum "^1.0.5"
 
@@ -15973,20 +15956,10 @@ gaxios@^5.0.0, gaxios@^5.0.1:
     is-stream "^2.0.0"
     node-fetch "^2.6.7"
 
-gaxios@^6.0.0, gaxios@^6.0.2:
-  version "6.0.4"
-  resolved "https://registry.yarnpkg.com/gaxios/-/gaxios-6.0.4.tgz#e8a2145653b5bad7e3cf358e2a9819160e8e6fa7"
-  integrity sha512-mwKfHJn7f3pLRfahdEPNyvygXRwjwgsgDPaIIoBRIDkgP4SFyezkYGWQ2aLCfrAnzimSrP+mAg1aSUj5gidXvw==
-  dependencies:
-    extend "^3.0.2"
-    https-proxy-agent "^7.0.1"
-    is-stream "^2.0.0"
-    node-fetch "^2.6.9"
-
-gaxios@^6.0.3:
-  version "6.1.0"
-  resolved "https://registry.yarnpkg.com/gaxios/-/gaxios-6.1.0.tgz#8ab08adbf9cc600368a57545f58e004ccf831ccb"
-  integrity sha512-EIHuesZxNyIkUGcTQKQPMICyOpDD/bi+LJIJx+NLsSGmnS7N+xCLRX5bi4e9yAu9AlSZdVq+qlyWWVuTh/483w==
+gaxios@^6.0.0, gaxios@^6.0.2, gaxios@^6.0.3, gaxios@^6.1.1:
+  version "6.3.0"
+  resolved "https://registry.yarnpkg.com/gaxios/-/gaxios-6.3.0.tgz#5cd858de47c6560caaf0f99bb5d89c5bdfbe9034"
+  integrity sha512-p+ggrQw3fBwH2F5N/PAI4k/G/y1art5OxKpb2J2chwNNHM4hHuAOtivjPuirMF4KNKwTTUal/lPfL2+7h2mEcg==
   dependencies:
     extend "^3.0.2"
     https-proxy-agent "^7.0.1"
@@ -16025,6 +15998,14 @@ gcp-metadata@^6.0.0:
     gaxios "^6.0.0"
     json-bigint "^1.0.0"
 
+gcp-metadata@^6.1.0:
+  version "6.1.0"
+  resolved "https://registry.yarnpkg.com/gcp-metadata/-/gcp-metadata-6.1.0.tgz#9b0dd2b2445258e7597f2024332d20611cbd6b8c"
+  integrity sha512-Jh/AIwwgaxan+7ZUUmRLCjtchyDiqh4KjBJ5tW3plBZb5iL/BPcso8A5DlzeD9qlw0duCamnNdpFjxwaT0KyKg==
+  dependencies:
+    gaxios "^6.0.0"
+    json-bigint "^1.0.0"
+
 gensync@^1.0.0-beta.1, gensync@^1.0.0-beta.2:
   version "1.0.0-beta.2"
   resolved "https://registry.yarnpkg.com/gensync/-/gensync-1.0.0-beta.2.tgz#32a6ee76c3d7f52d46b2b1ae5d93fea8580a25e0"
@@ -16329,7 +16310,7 @@ glob@7.2.0, glob@^7.1.1, glob@^7.1.2, glob@^7.1.3, glob@^7.1.4, glob@^7.1.6, glo
     once "^1.3.0"
     path-is-absolute "^1.0.0"
 
-glob@^10.2.2, glob@^10.3.10:
+glob@^10.2.2:
   version "10.3.10"
   resolved "https://registry.yarnpkg.com/glob/-/glob-10.3.10.tgz#0351ebb809fd187fe421ab96af83d3a70715df4b"
   integrity sha512-fa46+tv1Ak0UPK1TOy/pZrIybNNt4HCv7SDzwyfiOZkvZLEbjsZkJBPtDHVshZjbecAoAGSC20MjLDG/qr679g==
@@ -16531,18 +16512,17 @@ google-auth-library@^8.0.1, google-auth-library@^8.0.2:
     jws "^4.0.0"
     lru-cache "^6.0.0"
 
-google-auth-library@^9.0.0:
-  version "9.0.0"
-  resolved "https://registry.yarnpkg.com/google-auth-library/-/google-auth-library-9.0.0.tgz#b159d22464c679a6a25cb46d48a4ac97f9f426a2"
-  integrity sha512-IQGjgQoVUAfOk6khqTVMLvWx26R+yPw9uLyb1MNyMQpdKiKt0Fd9sp4NWoINjyGHR8S3iw12hMTYK7O8J07c6Q==
+google-auth-library@^9.0.0, google-auth-library@^9.6.3:
+  version "9.6.3"
+  resolved "https://registry.yarnpkg.com/google-auth-library/-/google-auth-library-9.6.3.tgz#add8935bc5b842a8e80f84fef2b5ed9febb41d48"
+  integrity sha512-4CacM29MLC2eT9Cey5GDVK4Q8t+MMp8+OEdOaqD9MG6b0dOyLORaaeJMPQ7EESVgm/+z5EKYyFLxgzBJlJgyHQ==
   dependencies:
     base64-js "^1.3.0"
     ecdsa-sig-formatter "^1.0.11"
-    gaxios "^6.0.0"
-    gcp-metadata "^6.0.0"
+    gaxios "^6.1.1"
+    gcp-metadata "^6.1.0"
     gtoken "^7.0.0"
     jws "^4.0.0"
-    lru-cache "^6.0.0"
 
 google-gax@^3.5.7:
   version "3.5.8"
@@ -21726,36 +21706,17 @@ miller-rabin@^4.0.0:
     bn.js "^4.0.0"
     brorand "^1.0.1"
 
-mime-db@1.44.0:
-  version "1.44.0"
-  resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.44.0.tgz#fa11c5eb0aca1334b4233cb4d52f10c5a6272f92"
-  integrity sha512-/NOTfLrsPBVeH7YtFPgsVWveuL+4SjjYxaQ1xtM1KMFj7HdxlBlxeyNLzhyJVx7r4rZGJAZ/6lkKCitSc/Nmpg==
-
-mime-db@1.49.0, "mime-db@>= 1.43.0 < 2":
-  version "1.49.0"
-  resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.49.0.tgz#f3dfde60c99e9cf3bc9701d687778f537001cbed"
-  integrity sha512-CIc8j9URtOVApSFCQIF+VBkX1RwXp/oMMOrqdyXSBXq5RWNEsRfyj1kiRnQgmNXmHxPoFIxOroKA3zcU9P+nAA==
-
 mime-db@1.52.0:
   version "1.52.0"
   resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.52.0.tgz#bbabcdc02859f4987301c856e3387ce5ec43bf70"
   integrity sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==
 
-mime-types@^2.0.8, mime-types@~2.1.24:
-  version "2.1.32"
-  resolved "https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.32.tgz#1d00e89e7de7fe02008db61001d9e02852670fd5"
-  integrity sha512-hJGaVS4G4c9TSMYh2n6SQAGrC4RnfU+daP8G7cSCmaqNjiOoUY0VHCMS42pxnQmVF1GWwFhbHWn3RIxCqTmZ9A==
-  dependencies:
-    mime-db "1.49.0"
+"mime-db@>= 1.43.0 < 2":
+  version "1.49.0"
+  resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.49.0.tgz#f3dfde60c99e9cf3bc9701d687778f537001cbed"
+  integrity sha512-CIc8j9URtOVApSFCQIF+VBkX1RwXp/oMMOrqdyXSBXq5RWNEsRfyj1kiRnQgmNXmHxPoFIxOroKA3zcU9P+nAA==
 
-mime-types@^2.1.12, mime-types@~2.1.19:
-  version "2.1.27"
-  resolved "https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.27.tgz#47949f98e279ea53119f5722e0f34e529bec009f"
-  integrity sha512-JIhqnCasI9yD+SsmkquHBxTSEuZdQX5BuQnS2Vc7puQQQ+8yiP5AY5uWhpdv4YL4VM5c6iliiYWPgJ/nJQLp7w==
-  dependencies:
-    mime-db "1.44.0"
-
-mime-types@^2.1.27, mime-types@^2.1.30, mime-types@^2.1.31, mime-types@~2.1.17, mime-types@~2.1.34:
+mime-types@^2.0.8, mime-types@^2.1.12, mime-types@^2.1.27, mime-types@^2.1.30, mime-types@^2.1.31, mime-types@~2.1.17, mime-types@~2.1.19, mime-types@~2.1.24, mime-types@~2.1.34:
   version "2.1.35"
   resolved "https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.35.tgz#381a871b62a734450660ae3deee44813f70d959a"
   integrity sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==
@@ -22081,11 +22042,6 @@ mkdirp@^1.0.3, mkdirp@^1.0.4:
   resolved "https://registry.yarnpkg.com/mkdirp/-/mkdirp-1.0.4.tgz#3eb5ed62622756d79a5f0e2a221dfebad75c2f7e"
   integrity sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw==
 
-mkdirp@^2.1.3:
-  version "2.1.6"
-  resolved "https://registry.yarnpkg.com/mkdirp/-/mkdirp-2.1.6.tgz#964fbcb12b2d8c5d6fbc62a963ac95a273e2cc19"
-  integrity sha512-+hEnITedc8LAtIP9u3HJDFIdcLV2vXP33sqLLIzkv1Db1zO/1OxbvYf0Y1OC/S/Qo5dxHXepofhmxL02PsKe+A==
-
 mkdirp@~0.3.5:
   version "0.3.5"
   resolved "https://registry.yarnpkg.com/mkdirp/-/mkdirp-0.3.5.tgz#de3e5f8961c88c787ee1368df849ac4413eca8d7"
@@ -26383,14 +26339,6 @@ read-pkg@^7.1.0:
     parse-json "^5.2.0"
     type-fest "^2.0.0"
 
-read-yaml-file@^2.1.0:
-  version "2.1.0"
-  resolved "https://registry.yarnpkg.com/read-yaml-file/-/read-yaml-file-2.1.0.tgz#c5866712db9ef5343b4d02c2413bada53c41c4a9"
-  integrity sha512-UkRNRIwnhG+y7hpqnycCL/xbTk7+ia9VuVTC0S+zVbwd65DI9eUpRMfsWIGrCWxTU/mi+JW8cHQCrv+zfCbEPQ==
-  dependencies:
-    js-yaml "^4.0.0"
-    strip-bom "^4.0.0"
-
 read@1, read@^1.0.7, read@~1.0.7:
   version "1.0.7"
   resolved "https://registry.yarnpkg.com/read/-/read-1.0.7.tgz#b3da19bd052431a97671d44a42634adf710b40c4"
@@ -26533,11 +26481,6 @@ reflect-metadata@^0.1.13:
   resolved "https://registry.yarnpkg.com/reflect-metadata/-/reflect-metadata-0.1.13.tgz#67ae3ca57c972a2aa1642b10fe363fe32d49dc08"
   integrity sha512-Ts1Y/anZELhSsjMcU605fU9RE4Oi3p5ORujwbIKXfWa+0Zxs510Qrmrce5/Jowq3cHSZSJqBjypxmHarc+vEWg==
 
-reflect-metadata@^0.2.1:
-  version "0.2.1"
-  resolved "https://registry.yarnpkg.com/reflect-metadata/-/reflect-metadata-0.2.1.tgz#8d5513c0f5ef2b4b9c3865287f3c0940c1f67f74"
-  integrity sha512-i5lLI6iw9AU3Uu4szRNPPEkomnkjRTaVt9hy/bn5g/oSzekBSMeLZblcjP74AW0vBabqERLLIrz+gR8QYR54Tw==
-
 reflect.getprototypeof@^1.0.4:
   version "1.0.4"
   resolved "https://registry.yarnpkg.com/reflect.getprototypeof/-/reflect.getprototypeof-1.0.4.tgz#aaccbf41aca3821b87bb71d9dcbc7ad0ba50a3f3"
@@ -27068,6 +27011,15 @@ retry-request@^6.0.0:
     debug "^4.1.1"
     extend "^3.0.2"
 
+retry-request@^7.0.0:
+  version "7.0.2"
+  resolved "https://registry.yarnpkg.com/retry-request/-/retry-request-7.0.2.tgz#60bf48cfb424ec01b03fca6665dee91d06dd95f3"
+  integrity sha512-dUOvLMJ0/JJYEn8NrpOaGNE7X3vpI5XlZS/u0ANjqtcZVKnIxP7IgCFwrKTxENw29emmwug53awKtaMm4i9g5w==
+  dependencies:
+    "@types/request" "^2.48.8"
+    extend "^3.0.2"
+    teeny-request "^9.0.0"
+
 retry@0.13.1, retry@^0.13.1:
   version "0.13.1"
   resolved "https://registry.yarnpkg.com/retry/-/retry-0.13.1.tgz#185b1587acf67919d63b357349e03537b2484658"
@@ -28322,6 +28274,11 @@ stream-shift@^1.0.0:
   resolved "https://registry.yarnpkg.com/stream-shift/-/stream-shift-1.0.1.tgz#d7088281559ab2778424279b0877da3c392d5a3d"
   integrity sha512-AiisoFqQ0vbGcZgQPY1cdP2I76glaVA/RauYR4G4thNFgkTqr90yXTo4LYX60Jl+sIlPNHHdGSwo01AvbKUSVQ==
 
+stream-shift@^1.0.2:
+  version "1.0.3"
+  resolved "https://registry.yarnpkg.com/stream-shift/-/stream-shift-1.0.3.tgz#85b8fab4d71010fc3ba8772e8046cc49b8a3864b"
+  integrity sha512-76ORR0DO1o1hlKwTbi/DM3EXWGf3ZJYO8cXX5RJwnul2DEg2oyoZyjLNoQM8WsvZiFKCRfC1O0J7iCvie3RZmQ==
+
 streamsearch@^1.1.0:
   version "1.1.0"
   resolved "https://registry.yarnpkg.com/streamsearch/-/streamsearch-1.1.0.tgz#404dd1e2247ca94af554e841a8ef0eaa238da764"
@@ -29562,7 +29519,7 @@ tslib@^1.0.0, tslib@^1.8.1, tslib@^1.9.0, tslib@^1.9.3:
   resolved "https://registry.yarnpkg.com/tslib/-/tslib-1.14.1.tgz#cf2d38bdc34a134bcaf1091c41f6619e2f672d00"
   integrity sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==
 
-tslib@^2, tslib@^2.0.0, tslib@^2.0.1, tslib@^2.0.3, tslib@^2.1.0, tslib@^2.3.0, tslib@^2.3.1, tslib@^2.4.0, tslib@^2.5.0:
+tslib@^2, tslib@^2.0.0, tslib@^2.0.1, tslib@^2.0.3, tslib@^2.1.0, tslib@^2.3.0, tslib@^2.3.1, tslib@^2.4.0:
   version "2.6.2"
   resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.6.2.tgz#703ac29425e7b37cd6fd456e92404d46d1f3e4ae"
   integrity sha512-AEYxH93jGFPn/a2iVAwW87VuUIkR1FVUKB77NwMF7nBTDkDrrT/Hpt/IrCJ0QXhW27jTBDcf5ZY7w6RiqTMw2Q==
@@ -29778,27 +29735,6 @@ typeorm-naming-strategies@^4.1.0:
   resolved "https://registry.yarnpkg.com/typeorm-naming-strategies/-/typeorm-naming-strategies-4.1.0.tgz#1ec6eb296c8d7b69bb06764d5b9083ff80e814a9"
   integrity sha512-vPekJXzZOTZrdDvTl1YoM+w+sUIfQHG4kZTpbFYoTsufyv9NIBRe4Q+PdzhEAFA2std3D9LZHEb1EjE9zhRpiQ==
 
-typeorm@^0.3.19:
-  version "0.3.20"
-  resolved "https://registry.yarnpkg.com/typeorm/-/typeorm-0.3.20.tgz#4b61d737c6fed4e9f63006f88d58a5e54816b7ab"
-  integrity sha512-sJ0T08dV5eoZroaq9uPKBoNcGslHBR4E4y+EBHs//SiGbblGe7IeduP/IH4ddCcj0qp3PHwDwGnuvqEAnKlq/Q==
-  dependencies:
-    "@sqltools/formatter" "^1.2.5"
-    app-root-path "^3.1.0"
-    buffer "^6.0.3"
-    chalk "^4.1.2"
-    cli-highlight "^2.1.11"
-    dayjs "^1.11.9"
-    debug "^4.3.4"
-    dotenv "^16.0.3"
-    glob "^10.3.10"
-    mkdirp "^2.1.3"
-    reflect-metadata "^0.2.1"
-    sha.js "^2.4.11"
-    tslib "^2.5.0"
-    uuid "^9.0.0"
-    yargs "^17.6.2"
-
 typeorm@^0.3.4:
   version "0.3.7"
   resolved "https://registry.yarnpkg.com/typeorm/-/typeorm-0.3.7.tgz#5776ed5058f0acb75d64723b39ff458d21de64c1"
@@ -29837,11 +29773,6 @@ typescript@^4.4.4:
   resolved "https://registry.yarnpkg.com/typescript/-/typescript-4.9.5.tgz#095979f9bcc0d09da324d58d03ce8f8374cbe65a"
   integrity sha512-1FXk9E2Hm+QzZQ7z+McJiHL4NW1F2EzMu9Nq9i3zAaGqibafqYwCVU6WyWAuyQRRzOlxou8xZSyXLEN8oKj24g==
 
-typescript@^5.3.3:
-  version "5.3.3"
-  resolved "https://registry.yarnpkg.com/typescript/-/typescript-5.3.3.tgz#b3ce6ba258e72e6305ba66f5c9b452aaee3ffe37"
-  integrity sha512-pXWcraxM0uxAS+tN0AG/BF2TyqmHO014Z070UsJ+pFvYuRSq8KH8DmWpnbXe0pEPDHXZV3FcAbJkijJ5oNEnWw==
-
 ua-parser-js@^0.7.30:
   version "0.7.33"
   resolved "https://registry.yarnpkg.com/ua-parser-js/-/ua-parser-js-0.7.33.tgz#1d04acb4ccef9293df6f70f2c3d22f3030d8b532"

From 8eaf754538f2b3620ab940e7a37d4a3a47a5ec39 Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Tue, 12 Mar 2024 15:37:19 +0800
Subject: [PATCH 15/67] do not get content from db when exporting item to
 integrations

---
 packages/api/src/services/library_item.ts | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/packages/api/src/services/library_item.ts b/packages/api/src/services/library_item.ts
index 4a16c7c18..9127ca8f9 100644
--- a/packages/api/src/services/library_item.ts
+++ b/packages/api/src/services/library_item.ts
@@ -700,10 +700,16 @@ export const findRecentLibraryItems = async (
 }
 
 export const findLibraryItemsByIds = async (ids: string[], userId: string) => {
+  const selectColumns = getColumns(libraryItemRepository)
+    .filter(
+      (column) => column !== 'readableContent' && column !== 'originalContent'
+    )
+    .map((column) => `library_item.${column}`)
   return authTrx(
     async (tx) =>
       tx
         .createQueryBuilder(LibraryItem, 'library_item')
+        .select(selectColumns)
         .leftJoinAndSelect('library_item.labels', 'labels')
         .leftJoinAndSelect('library_item.highlights', 'highlights')
         .where('library_item.id IN (:...ids)', { ids })

From c5cbe373e7be96726298b4b942f7e2fb5380305f Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Tue, 12 Mar 2024 22:38:44 +0800
Subject: [PATCH 16/67] Web UI for notion integration

---
 .../queries/useGetIntegrationsQuery.tsx       |   2 +
 packages/web/pages/settings/integrations.tsx  |  57 +++++++++-
 .../pages/settings/integrations/notion.tsx    |  97 ++++++++++++++++++
 packages/web/public/static/icons/notion.png   | Bin 0 -> 11406 bytes
 4 files changed, 151 insertions(+), 5 deletions(-)
 create mode 100644 packages/web/pages/settings/integrations/notion.tsx
 create mode 100644 packages/web/public/static/icons/notion.png

diff --git a/packages/web/lib/networking/queries/useGetIntegrationsQuery.tsx b/packages/web/lib/networking/queries/useGetIntegrationsQuery.tsx
index 9c28e2505..9b2623414 100644
--- a/packages/web/lib/networking/queries/useGetIntegrationsQuery.tsx
+++ b/packages/web/lib/networking/queries/useGetIntegrationsQuery.tsx
@@ -11,6 +11,7 @@ export interface Integration {
   createdAt: Date
   updatedAt: Date
   taskName?: string
+  settings?: unknown
 }
 
 export type IntegrationType = 'EXPORT' | 'IMPORT'
@@ -43,6 +44,7 @@ export function useGetIntegrationsQuery(): IntegrationsQueryResponse {
             createdAt
             updatedAt
             taskName
+            settings
           }
         }
         ... on IntegrationsError {
diff --git a/packages/web/pages/settings/integrations.tsx b/packages/web/pages/settings/integrations.tsx
index 4d549b217..14c019ab7 100644
--- a/packages/web/pages/settings/integrations.tsx
+++ b/packages/web/pages/settings/integrations.tsx
@@ -110,11 +110,14 @@ export default function Integrations(): JSX.Element {
     }
   }
 
-  const redirectToPocket = (importItemState: ImportItemState) => {
+  const redirectToIntegration = (
+    name: string,
+    importItemState: ImportItemState
+  ) => {
     // create a form and submit it to the backend
     const form = document.createElement('form')
     form.method = 'POST'
-    form.action = `${fetchEndpoint}/integration/pocket/auth`
+    form.action = `${fetchEndpoint}/integration/${name.toLowerCase()}/auth`
     const input = document.createElement('input')
     input.type = 'hidden'
     input.name = 'state'
@@ -158,10 +161,39 @@ export default function Integrations(): JSX.Element {
         router.replace('/settings/integrations')
       }
     }
+
+    const connectWithNotion = async () => {
+      try {
+        // get the token from query string
+        const token = router.query.code as string
+        const result = await setIntegrationMutation({
+          token,
+          name: 'NOTION',
+          type: 'EXPORT',
+          enabled: true,
+        })
+        if (result) {
+          revalidate()
+          showSuccessToast('Connected with Notion.')
+        } else {
+          showErrorToast('There was an error connecting to Notion.')
+        }
+      } catch (err) {
+        showErrorToast(
+          'There was an error connecting to Notion. Please try again.',
+          { duration: 5000 }
+        )
+      } finally {
+        router.replace('/settings/integrations')
+      }
+    }
     if (!router.isReady) return
     if (router.query.pocketToken && router.query.state && !pocketConnected) {
       connectToPocket()
     }
+    if (router.query.code) {
+      connectWithNotion()
+    }
   }, [router])
 
   useEffect(() => {
@@ -210,7 +242,7 @@ export default function Integrations(): JSX.Element {
           action: () => {
             pocketConnected
               ? deleteIntegration(pocketConnected.id)
-              : redirectToPocket(ImportItemState.Unarchived)
+              : redirectToIntegration('pocket', ImportItemState.Unarchived)
           },
           disabled: isImporting(pocketConnected),
           isDropdown: !pocketConnected,
@@ -218,18 +250,33 @@ export default function Integrations(): JSX.Element {
             {
               text: 'Import All',
               action: () => {
-                redirectToPocket(ImportItemState.All)
+                redirectToIntegration('pocket', ImportItemState.All)
               },
             },
             {
               text: 'Import Unarchived',
               action: () => {
-                redirectToPocket(ImportItemState.Unarchived)
+                redirectToIntegration('pocket', ImportItemState.Unarchived)
               },
             },
           ],
         },
       },
+      {
+        icon: '/static/icons/notion.png',
+        title: 'Notion',
+        subText:
+          'Notion is an all-in-one workspace. Use our Notion integration to sync your Omnivore items to Notion.',
+        button: {
+          text: 'Settings',
+          icon: <Link size={16} weight={'bold'} />,
+          style: 'ctaWhite',
+          action: () =>
+            router.push(
+              '/settings/integrations/notion'
+            ),
+        },
+      },
       {
         icon: '/static/icons/webhooks.svg',
         title: 'Webhooks',
diff --git a/packages/web/pages/settings/integrations/notion.tsx b/packages/web/pages/settings/integrations/notion.tsx
new file mode 100644
index 000000000..4b956065e
--- /dev/null
+++ b/packages/web/pages/settings/integrations/notion.tsx
@@ -0,0 +1,97 @@
+import { styled } from '@stitches/react'
+import { Button, Checkbox, Form, Input, Switch } from 'antd'
+import 'antd/dist/antd.compact.css'
+import Image from 'next/image'
+import { useMemo } from 'react'
+import {
+  Box,
+  HStack,
+  VStack,
+} from '../../../components/elements/LayoutPrimitives'
+import { PageMetaData } from '../../../components/patterns/PageMetaData'
+import { SettingsLayout } from '../../../components/templates/SettingsLayout'
+import { useGetIntegrationsQuery } from '../../../lib/networking/queries/useGetIntegrationsQuery'
+
+// Styles
+const Header = styled(Box, {
+  color: '$utilityTextDefault',
+  fontSize: 'x-large',
+  margin: '20px',
+})
+
+export default function Notion(): JSX.Element {
+  const { integrations, revalidate } = useGetIntegrationsQuery()
+  const notion = useMemo(() => {
+    return integrations.find((i) => i.name == 'NOTION' && i.type == 'EXPORT')
+  }, [integrations])
+
+  return (
+    <>
+      <PageMetaData title="Notion" path="/integrations/notion" />
+      <SettingsLayout>
+        <VStack
+          css={{
+            margin: 'auto',
+            width: '80%',
+          }}
+        >
+          <HStack>
+            <Image
+              src="/static/icons/notion.png"
+              alt="Integration Image"
+              width={75}
+              height={75}
+            />
+            <Header>Notion integration settings</Header>
+          </HStack>
+
+          {notion && (
+            <Form>
+              <VStack
+                css={{
+                  padding: '20px',
+                }}
+              >
+                <HStack>
+                  <Form.Item
+                    label="Notion Page Id"
+                    name="parentPageId"
+                    style={{ marginRight: '20px' }}
+                    rules={[
+                      {
+                        required: true,
+                        message: 'Please input your Notion Page Id!',
+                      },
+                    ]}
+                  >
+                    <Input />
+                  </Form.Item>
+
+                  <Button>Save</Button>
+                </HStack>
+
+                <Form.Item label="Notion Database Id" name="parentDatabaseId">
+                  <Input disabled />
+                </Form.Item>
+
+                <Form.Item label="Automatic Sync" name="autoSync">
+                  <Switch />
+                </Form.Item>
+
+                <Form.Item label="Data to Export" name="dataToExport">
+                  <Checkbox.Group>
+                    <Checkbox value="highlights">Highlights</Checkbox>
+                    <Checkbox value="labels">Labels</Checkbox>
+                    <Checkbox value="notes">Notes</Checkbox>
+                  </Checkbox.Group>
+                </Form.Item>
+
+                <Button>Export Recent Items</Button>
+              </VStack>
+            </Form>
+          )}
+        </VStack>
+      </SettingsLayout>
+    </>
+  )
+}
diff --git a/packages/web/public/static/icons/notion.png b/packages/web/public/static/icons/notion.png
new file mode 100644
index 0000000000000000000000000000000000000000..391051679c8cc33e7e52891593147283bf93dcb0
GIT binary patch
literal 11406
zcmbVycRZDE`2YPJ=Nx-vC5h9pvdc=wQB;T|Ss5o~6S7CngNE`kib&QeDMdyo+wn=!
zFtQ~hA}f23bH2Aeuiy8N-ygp}elIVt^W67+J=cBR<9c7~xod26nw>?E1pvUVcSgqq
z01SM@046;ASqkjffIrM$XDodHVCC8QN5JDW0oaN7n4CTV@|%SwU<2!<ZKw@EQ5@^m
zWgGy~2laHcO#_iX{o5CL4c;*hE!eD*l`sU1d_?4~oL%TH!Y(HG7xniK-6emDa*Fy#
z<LK)VYN&#UP3N|{+p?f)bNOufy$iLkY)Yr}Fyrx+m6cAN2I_t9y8|=KS{5gJ1#ABF
z_FU-8m?zV^=a{RD$SYUYT87t`e|<_w#j*!K04x{T$NJvOy=xz=D-VmQ3-v7&*)Fx#
zVT$sa^H}pcCr`S71r%?e=Fg|e9-)o{V!N2~Z?~v~qkg*p`f$&lMv=!s-H{t7BQ+A1
z+X!r-FaM=FMDW2e#u%A`IL`5`;)QRkMYct1T>FUvBo_=Q`O%XXdfYE0sM;{trSOQC
z)va*b2plsojyf$}@Zzf30H3)F7dD^!s~$k{XDw{W&Ssh$nvb&0HN-D+ttFE12uLCY
zrJr4XK%lZ=02{m0Xi~RJf-N`p5fM=Qqzm3_R8IzFy#QHQ@J8u!`z4L|xaQCA5=Q+V
zHRxV<MfW3U%%iq9oeOI2T?LcJa!*iQp0;BF{~PMU#v01;3r-~W8t<D7TF<klNe&t%
zbw6opZlg%gNqv*V5Lqp38IwA4Gu{a@`$dEJ`%vc8hYY#C_MXH5Cg4U2+9zQl*Sois
z8AD~ekkozh>t~q)29K*slu<Q4aLRG4Z({H$loy$))Q_x2s&%GHl!0mFw(YWs)}{B|
zf3Va8!FtDknz`x%f#&HeDA`7%@_j+m0Alw0z{G)`!yoYIPYO96A6bKlkhX(AdDnGG
z90qKR+uM`J1ZyybLCQ>TiMtSz>RbbbE}JsJ@Qacxv|IW+-Q?q|aB^*MjhP)SFC*dV
z9Ao1L|K%TMncrr4bU$Cm5%9E0(XgBq9#CGXAIr)wp38??OcrEBPkwo{JSz_L3+*)Q
za$9C2brjw(gHBGPjy2DAl9Lv2=*@>j#akn17%v{_3S=hr6#hPWWQ5T<+V;A&d`JwG
zH;;a(_XneUYk^%YO}*N-fWEQ0I;%ApS1X?Jjja5s*r{bk<I<%|E>2G3e`aP@Y6tuJ
z4E7C|kv<ACgZEc%WF}Z@6+R25<})tb!7m2x+pf^_W9k_kJV)PNwWF7g?regzhdN+O
z-nue3Xar6!e*Ju!aow_b-b9BG#WlCFJhe30nM+%haZn;0-OCQ%?|bU>>48>B*BY?l
z9MB5eI-^CVZ5_>t<D6y_5*B_Q85#LDpUqmd0asj91myEpGes!^*)j$1^b(l|szbUt
z8D!cGuI>K%?<owm@Ir?kMo_^cs5;}}JMW-s_P6iewSW4gw>Ofid%cpB)g=trXoXgs
zO;zxHJ8XEIh}PTJime8*tjdnB5$X_ULQ`dxrY5D`qv-U|_U7s~vGss5lQPKzAvR^^
z2RgU+tu1Vfw^fe%CGm;l?8!uIIR6BcOvP0OAq95Ks%c-AF1YZ(0=KrYoZAV69YzsS
z4}+TQVy{tayl!%L(`QLbcW(DySL(&<)?R<Qc7fp&nSZcS^0O(|zfAesu$nXM>|d#s
z3&_FK%Q}ABL@-*|-dT40=?WKM?ImNz4_?*L3xn4mIakjw{z~fJFXwG4AItXEU5V;l
z$9G8&7_Z7f-EuBSQs#Q2?RCqH56Hes?katG_|wUwvjU9`<fa8RUxORgg}TXX%MV5(
zq>6lZ=IVCwsuYoG?cDnfUG}z)zms<W^eoeX%2T8qHqRU2(0unTpM!#}{IRSimy`kT
z@y$5xjj^$@bFY6asI`hZIXO*?k4uy`?ufZRjsu7`(XMdSR#i=Nt_{}Qn5xD2_D^*e
zwBO+r5!!uKL1DA@i^WB!+Fb6Lej1Htv-LYzT{!MH?E*V3VE69b0Wyo4MZ%q)Ge2#f
zU-R^wedX}F(y1kbZ*h0)$b8`AFb+=6O`aG0<<)zaaxOodIg+;BKqY&58F$#J#x;z5
znz-AEPEAc!CIw|$#7wIA&}EMnb$EW9oexyc4Kgw^nz0(2O7BEN_y@3o$>+Okdw#5r
zHKqFc`8@<l7d>xBcaNv-ZfSpUu6^-$3q4fhTFHpxje9v^GASbwf8H4b0kNFE#PBuN
zm4#6Qr<w4O$-3hi!SV%+3$I_lriGkhT;H(FW&&B_arRG5>?ORc3#<9r!5zuSEHa)v
zetRAX4!tcLxiL<tjSUZdcmS(I$cYn8{@1(C-%1gLzP4St5XnX}e*R56xQ(bubMxTO
zy!`F43Pzhb(@Cb<W-gBfX?U46#sNBqM2a??Tzi_EW9vi&6<-@^R#Xi<3=id#m6dIq
zm~h#Z>2c@Io$XHWjy1{iK4@KgO>Sst*be3`tO{aW@ay`tZZ4@!2$R($)6!N}R!pLO
zXZ-7QF;rbwuT^qKv;Bm~T6YP%r)bwFNpYr<IsdwzNAu66a*=2(+Ou7oE@(fI`8GA+
z{iWo|UpBiL+_B+lrE_H@q0YeWTO$!ZIP`fOT~{z{bHyXV<R6aKkw|C@_BN)NOYY6Q
zR3_+k-zv=7YXG!5Zx2pqKF>)=h~_q9SXo_dZ-^7=^S7QWV?mN%-Wp>EMT`0Ojkp}?
zc_%nxE|V<39Vb*|j;ZgzZe1v}h(|v*(;M27&1yTocM0J6`69__QTH5A9E%=_IM40e
z>q^XxWo_OIhPUV+wRA@qHInEq;xU(p=Jk6C0;1s(>?LR1AORaNR0eOh?C>#{DSM#4
zKj&+}{bmN6onN{;&~dHp(J#sMjy-eu(Yvg-?L`TzVlJ_)V@?^s>ycmukiciO6?C{q
ze3Wn1c+99Yc7QL~+6#`ERJ(7Tlk%^u37kLRm-11bdcb}z?gFNKP;tDBBs|6pcE1eg
zrxq=Vh@x*>1@JwICeT>SmyrbX@VrI78s~uXFDBB~+(*vSk_W!cKqQuHD6Ha8e4WT1
z0pUu#4;cT8rwa7M#m2T(2F$W85d9(lyB#mi1<hXYu`b+0PaV<ddx_MN=|;z*9di5n
z`e=5tky`Ot0v1<%NNoAX$VQ#HeC5e9{Jz}D<Lt9-pW4)=K+F%!BOf~5;?2EUV~{P8
zeQ2m`XzXewCVp|gHfi5wM2&#&uqgeyy_h=#`kr>87Qy8?V(lV#VwT@1w6IWZT!{~p
zO!5UcWDtwHWp<(!d1Ez#t!*O_z3yCTPB>KIF%Kdd9j=tr>!TI1TU+!qf4Qf<HPwH)
zz6O;}x{O6nt7>VT1ERSqOkrzt`#0Gmie_Kk<cr4D<Le3xJ-JLVfzP03(^AN2+rtv=
zLWc%iNax~P!P$HhhCHBd@+B!7<&biTBz=ixy=LCh{ahbo29i4djg~<<Dyyo_LIr>W
z1G<7g#0!fI9pM-+LXaJFKJURYUO4`*AcNaXKvBtD+0f->YtQR;C?Cq_a#;|kmVeXW
z)(OaS<<)OPC4Bz2{%I*7#qFY)FXw*(I7c&qA{D}FcqtU;T{}sX^v6jNhw|N{0dEE4
zFB_H&`}pC-48GU$c1J~qK2)eM9!B!?gm!IhtrtH4t%BcF2x6*Gu#WH7_)0K^Vk#;s
z+oCT6an6yCE;rb|_`%qfv~Rb%Ww$;G3Qw)zfX4;TK-r&bKGT;Q*ByItd7@Ye#QjW1
z%hORY;-$nkZTXUCP6kWNss>UAfO~va1hKCpHPfIcd=LV@oO!331v4pfvP9^#8Td|+
z<-XSxF3OfCb4+$E=weeu9#GU_y-e}YKg?Wz`{Np$tgjLU`d~>Od);xA{m^1nxk_gu
zCEAk7EVnF5B?5K*D}$40hHo*g@|ze``QE|S>>mjxbN8lAHW4h!ro%2X=~Ql4gA_(R
zP-fy++cT9ds*mV{JyErX?kARV;lDo{AOw09uDo3_kXrJu`%|JLNo65A+Vc@Ma1zYG
zV^>0&^F1^DlNp#e))~_)j&d@Ma9CDOF1lBjRX(obtK~g7qB*Hz4jSY6|0VX-<6K1d
zRNEAy?N*ze_S<1r@gQE(oRz`DWb&riiaf7LjYXgwssV6brSkES!CXS*?d{C~a@zt#
zdo4SyrNn%6B!cx1jtj8tiX`7e+mgPa>K&1O8Up78tEN_CIliwP+J3lx>QB4tw0g>F
z6NY%Xxw*OR+qd(WiYuY=!rL?mVPx8RiMg5h2(#su@$}ce-9#V}pL^?xP&C`XNb%xZ
z>GXFOpNoc<S2D1GTWu!<l&=hAFTU-nPLVZYslO>5tL8ZTR8VWMdGQoNAt|gi^nQ1?
zLVl=d26o3)sN^R0-kdg93e(K|9Rpq9GOI_*Cacd6rQAZxKLZTW;A$D0?i~%qgnYZs
zYM0!9PEf61z?0xFl0z>Bx%Y=G4c?Od{L5$UmEcLc=HLX-dLov^Kea+dQ3_9ox+P|Q
z_4jv10w^&}#M;}To#-3s3#)H^Io2D8%2vMSqZnee?#4gt5oQd^PHjVZGwl65xqw}n
zHTY6Y*6#KJ0`Mi~VRX1uRaMWNRrc(!3@ga?Aw9OZ>oF(zYs6JZBXtFjP)>dq!ip(f
zPEichT4E{GFJcJ^3F+&wMN`<ru_(tq0#{Sy0j6KAI%jfCDX)jse%=<dv3upcv|OUX
z1orLzr08l2BdcwBuEEnpi>r%62b^k7GX_SgwEDAO!&klh`0cMC@|`6F(u5GHqD)iu
z8izZvK=5?H^J;AXP^G(irqc}0)UURHZUCrOiWXyO<%_K_p3FOc^1bsvSr-^hqyT8u
zhGC^|JPZ~09Rh&J$o&84YNd;U9?C}2x3aL%rJ#)^mtfk~dS=1t!Q-^qj;GzUrY6G;
z+IEQA@#8)Q6PO=3K>hKNad=}>FLHGp)%X)Uf3du*?97W7FD!B4*0UHc0Mv$_ga@__
zX=ol5x#7ude{twf-@A#0g(IUxU;$;4m)GRdQk3`f@Yr4)xPzb14_X{E<lq0_U`bsG
z5(M9UInZQpFFN!gdUs4R0~UpG9kHGNb%55Aez84{V1{{o2uPm+AW6Z6*z8a&m<o^5
z+k_%njt31OJnw8@o^tMh#)fdtKIn)C%npds9B{xQrNQ0o!mMMzy&N%bLs6HR1h>h(
z>)DueR;?N5K=I>Z;x&JNf5ASs94yF-X>yf^E?!uz9v7Jzw92P;lD46@!A<7|L-*ay
zBCoG)`ey>7cyAuQ--Lc+Ob$VZ-CN4^q<ix4|17N1G+}&b<y-t%)CXPW0v2a9ocC5C
zpT&UuMYbVEM#k-fibc^~M2=Pt4B)(1M|*#Vb30rFUD*Z7#PMM_ijnVzU`RwnD{R^>
z4Ix%;%hefw-D0PKegbIyU2zt@whN4h&}la?l;c>iQA}>w1Qoh~P#wH{7-^NkfaVPb
zcmZ0y71k;bF;GIVXl!&W8TiJ*E=M}(KLmW!`v3d=(bZ*!ffN9Y6A*ZMa9dFhZU88e
z|KD9SIfNQzThLA3UY_dKYMC9Z-7J8?(b<*^4Fhp;@w?aIg_JSj92hCciVO&~p<8K`
z#nl7|V~FcmfU^C=S)VmXUz;Ck(Gs~adxdfKVidi9ZScpyfKj3z02Xa)sCvtg2JOaO
z{q0#tBJ64H?7X<K^5+tLd2H-?F~jAXkE*Myby6U+V;@j*aPW7s=ytCGai?RCeG)k|
z)O|=~acQDmr+f(kkA}Cl)T=|++lJ_<Dl@S>4-lhcgmEF`(gC5>wY8ZX930%MAFkJC
zT}xmF4m^P1v&c3)a{SBgW8D5hK`FG&S*?chj{;jSqU#3nR2bG4BAo2Q`(vYC+jH5S
z+VZTqv8jAxdqYWqt9BJnpRgZ|f;93k^nRDUU?0v-n~&Y_@2G&@5cf;FMW>4H0y}Yd
zOIr05h~4J|_SH|>k#k-eq<;8kwKLZ|T@Gh$ZS7u_YwI%XR9|y{ff@LoU>j0+|0j4P
z$t4kkAO3xFLNjdZkp#S%xgXBT;+0O9jkPbE5Vxvyh*#gN&JO0%-`&&%C1%sZ1HHWp
zYjP6$h>Z<qwfKnT^adST^``qW_BgcbZuYc_=fIjg`I&4oriM(<STp~+a^<~U$5#5L
z=WYZwyIVH}l~dVU=4Ea+Dl13t{=E?p5U^RGFgL~i?HUHW=5Bb$M$7$aN(|XE%?#VS
zl*oI_%w9xiqi=1m@kECDhA8$N)dv(l-)ooRByc*pL-~p~f@oVou32k~(fyn9J7a9L
zDX7^BZqxtb3oFVE`<Y&a-BRWT61Jgd$Qw(NMZ$iPgjlrH2Ae*AbS6uiV8mSS)_>Nu
zB=u0@DL@HSd-vKd5ved`5l{-A>VJk;_2Y^%=JLq)^z_^;IDt9F10*sQX{q0`O{h=p
zFZAcd_{|Pf^O$|QXN2;;wvBbz9$r;5vYJ!%=qH^i$AWu~bAx%s#Yd%NTe&enx4pt|
z@hLCKNj>o#pg7~7Hbt|0=p)9O2(Wl$0$?W))|CYCFpW!KfTeRZ5kql@ulc;xD~6BO
zg2?$2=d(Tm0o``@@+<Ri1;!Y;FL!6rtHR`RcV)vk(2YHUA`f?tEApIrbRN?`x5F!L
zxm7`QE~fjH)HuAY;GO&V{th<rB%l;K-nu1zF5}P`Pc~{^U~4%2kn>lAFd&Jhv7m35
z099X=4>;^3+U}a5n|mlQ-&JUGo&_w310cmG1@w`4Ehu$)LIO`VXiS6F<dzBeWe7mm
z0W;9t(!vuj4u?}Ea2+0x!mA*^?>biqkTL;MUi`tI!wc3S^U@0}=+@4lB#sz{1*C(C
z7=j#}NddCwL_m)KeB%xP?un2$G-k?1lHf5EVDAfF(8L4KQxZ60G0YJvoM(yZ1f<2C
zM}4GK2m#I=qoWRVMA-lU!QHt~+CobVXjRPyJ?aQ(JdubDUj>DGFd&{Cc0u~aQ@)Jq
z8XAVw4QMvZppTP?Au(V9;W&IV43h231{QF=jSoyfUNQhz4J1L=X#n12^Ma{^fcOz=
zd>c$BnIuc%sKHDiz9>CCeTlZc+2IEn1Y^L}nf}TE7cyDPF9mYQ006hPHlJmJ!bT!&
z^b8El6kvVRVKWwhTT#^jIteeoJmvKY3(G)9VTv-IQ$+Jw{_$hS^q|AcYyqiocbias
zB+>*FLit(XDfaNRIuv1sOo8-ifc{(Wf~xYh?o7Wk{ES!LD>O6>ZP*1U#^+hagFVr4
zR!FM8Gu^$q!I!8uR0~E-G#n?vP&(a{M<^y7UWE^EZdH$nhnOJ+ERe7kV)ng4F#*#0
z0Oj}kKTk=8`;8?5|9(I;sMpmlVjF<7%8MYt>eu*q&f?C?Bu$uojZ&^-fCH%z$g>RR
z;L^{E10rj=0({pIPJ0W25nM3DM|2YloL@@ds=~I_{A}_t^{!w}4zfF&O3DV0bl}v{
zb#|#k9dZm3IBG4VJ?0U?n1~s&!vJ5@gi7J{3J!jZA_!0;#9}Ae<J)?BE$LSQ0T7+)
zZP7A0$`m|ZA~advS4fiqjHWCAlIo@oA=!WnP<%?5g^gJ#Q&=!A@R#qn8XP_&Ng_(3
zvOIv2^^~`Dc>rijqWRpg1*J~Jj30(1($`o!#4IGyYp{FdKPGOFNtr75_(x(ksDSt7
zhk)ucUScv6AZEhNfDsyV1vEw*!%ry*2?-s3@M!fB-v|J<ynTJ28G4CE7-JFVqX<|#
z>50}3n}U)QICk={q~DDInDM?cL`-9e8VPSbg#li1CRCy#-h_qn8ViQBp-!09+3dc7
z>}X?23;<`J8O}j!xHb={YEK2WN}0`ADCzL|@ZWVh?}e64@9#R5p>C!?VJW=CRH&ZT
zPkFgkaXZ)3V?O{sK39g|>3I;~W4l-&+!kdnfZlTEQHSIcklp_8Sb8=u@e_Q!tdB69
zXHnJEG%`MbfWSlv+&Das9o-CXgN(Bza9ezPF;|p;{s92u_}}4Nd8SQ)12;gk|KsNZ
znMgu@km?~u;ub7jG{uB7cpsWuT2j<zhJ&K+OhT&1k58#ENBm%hu%#nO5FCLw+X2e-
z@R*ufo0zI3&KRcdE%sC+G!qs=zz&lACt)*ufAqg){(Vfu>=2kk0{*|dCxQEmVte8C
zz?<pqeV-r>$CMS;+<4SUIP?cxft3pnTM!iYf03+QfotI)geZu|(4)Zqg{?8@sxdc4
zLLmD`x$ZBpNq9u;=Y+h@f5)B_hEj|`cn^XI6}7WLpb2jO|6-f>C5m!gD;vbL#Q*Cz
zZ7;)Rh<OB7W$eF>5V!cDkk_w2K%3yw-JN;ZQlT7{=UM)?^LE<s@Ooi;8A9hYY)?Tb
zwY+CSEMdWSo!kL4L;1Wo2yRk%vPZ;a4anK;JN7+<GvW$Vm}S@wxyE+06TW>wN0Q{}
zDTm}LoC9kQ1u4u-@<yJ(BHf(<P|GZkRxtsygMHj69~gpn;k~N_0(8H@zAG7^)qovJ
zr}smE=}W3O*fL`8`TkuWB<M&IJwafVyE*WsVV*-7N3Yso;%CBLhTjqsaDiz-7^<Ob
z)N>K4G~_n|lQdjEa2#Ks8p^9Te+6@}qNIxNX*^Hu@9)RTN5gILUR1`oK4aq|Kau%^
zEEewP;y>yM)-nXI2L?U@j(1I{FjI~hOIT(h1};xt#5}sATqj=}#6Z31dL|L+SV4mz
zj)#u1fO8q@6WMrAoUb5wlniMyb+0jOJrKO^<wZJ5kXM0l<N?!-J({S%mS`}^77FJM
zbB56!l&q{QtbBn4PD140t3HhA<&%`$j`%Z6(1QNu(|bS){Ji99oHoClyyua5s4YAW
z&P$5GIvfAtxo`-A>i+GJ0HzLTZpY9lasLX&0tfh?@PbvD3JjNKHr}$0fq(QPv20cW
z8Io3#PFmUtHA?zhGPQVeAsO9D6)4^iq^#^1e2)NVM>4d?83R>8?evYw$;0!3nkqvD
z4SC?^Vu$=6&53(<wPD#`zI>szY<@|WJ9u-|Ze(-qceSmYLZ-#BrE-MfT-!moyC1$^
z&8l=Aj+}>{oF+(~Tb?ri@|35Zc<Ek_cf$}rISIz9w6ib9hku_EdNQ*i9JRIxokl&w
zoR42RZi*k(tf$Fo*ydprH+U|-+x(}O%hxir>;sP!ru3UU*H5D77LvJFey)d`7}bQv
z^mAE2_u$Tlu<#xgu1!Y!F`l?;EQ6*B=`J7I!)77lR)3%RF?Fe-Ey+satV)eO*U_IA
zdveT~hbP)B*L%1EAx>QpO)ve*LUi6bL-JbOEiLQ%Gb<PC{hi`A>~qrPX1>*7V44f6
zJD?@A`f4jQe|e9>@tFlu&xZj)g*8~2u}mhaxuh7Pr3sUth9B4DOx##s`0zm3=k6Xa
zX{6yn;MNb{>YE*%V^;I@fV08_H8)Zz21y*Ch(-PfYrHN~PQdDSPU^`6LNGJ@ntoV<
zmiy&2BPYz~XYPT)FLQ-G^rL=03%1R^y@|PL(dJWs$EsgP9KRhHGaJ;CYhGZI{K0aa
z3CYxSINy1*^pZahY0yiKIMR=}tuXjrVp~v1h<21a|0OQFKRTK<lFqBZV8(cEVtV?k
z#=*jzS8Q|A6$}FozlMeg-{pdytHT4Xc1xIK*vc1dyti?*@n<>2rz?aoxmOSCpVGFT
zYjoZmpV`>h$e$W+DSfFG`TW@)vo{#7+sRmmi^OQf*{E{?^?6##5sxS2#5SDQLKe)o
zY;~fi+sNnaMHOlEsD@Ygz2|Sm76k0M+u?Ldb=~C)3-Rlg!N!f2(U<t!$A!%%^@(5C
z`rf{I$6^L%F7>idqw>s<10VW0Tdx&)ODTI5J=$PX=aljgC9HUN91Tm>R)~AZh`!?W
zdU)hJm%y;x?V7<qe37*d^baKtDIxvW={N4hq@MqGeSOi1zLXH2shf8NM@W0dCd(qo
zmkTCIe|QZ;9}VtQVwRUn>&Qm)o2D`%PD*#9!Zpo2hij!Ug*-y)AFVJ{uahKykJ6GY
z_e_DUCfaWvtxa#Cw!z4TyLvha0gUw&yOxzJ7Y{_FX`N0g%I8Cw5bx0*htE-uoi@kE
zR4!GY6)OG!D|huHy2lj4L{#{&liFG|l`E|(eNGQEo+(vSRz_!b^d)NFp`M>lKJ%p|
zyiJ8ox8d~up&Eg-r^R+(xjS!li&q$QrHh$=3se7`__w;)$wn?6Zh@sHaiz<#?CWml
z4eMXT&gZ(~)F;SW=F9ZJM9iDV{-?njbTTtTzCGa`DQ;fV#NQ3HZ-t0D$+?D)ZHm%X
z%i!8=@PWXSDKl@y7$I>!Ch6sH%X6AfbobiPXsybP%(~}!O5mmh&U8P)2q%>NPd&Ho
z_HKUVC&E=zNF|Ln(f(pP>BD@>@YJ`}*xCmL0`*fV#}i3^5YqS|c3IZ%Mk!I8eLb99
z293)1ALa_ncfS-aeE%%`n4@>r?2Yk3r12B1RGyI>a{GMd*>Vrb2fCmV7p&Kg{t*~j
zL$_L~>Y9OZEgi^r=L=F`8?SwlDeAs*?%PoBx2rI(xE;bE(E?MAWbp1PN8zK=3{2n|
zK8!sdNz37oK7a;V1F^~SkZ8pjAwS-|)dUxGBq_;9OlBDif3OnG-PM6<__O3A)7&|F
zT)?KSo`rbhva$d1*~$D%pw&MHiyGfTd;F4Z#50_^pNVl0`;_>>?Kgd#Trfb}eGGVX
zIRk+K!L=g}{(fH7l3b{N>+*8Vb8|3h7~cDbF-lC^ds`fooF735ru&DU1|K_@a#u`y
z9vCZ2fmIkdoq(b~Ys%+2m&Q^k0x0g^<{y$_D&Y3OGqzO*op<Vn5#Y1mjT`A}=4J?~
zOk~LL^VvL5zg!0rKe146l)nZYh7m>=9EW2)HxcleXPGWLuzbY8|L<(ig3I+&?p{D1
zW0AyB2<F~)NeBss<<*HP61W~-Cga$1nVO<AUz7n`D%RnhWNR2euh%cs@}YM|t1w*q
z3HOwDmT&|W(+2C&n>K+KT?m0qj{Wl+z1F%qZ4BWYEKDZRUo>h0w)AI^2aINVIYImr
ziw!M}rfbMCDP1gTy~^Osjg5yoV^<UUa(k97I}cp#Khy~4Qm?<Q6-f`h#|-v&i)q7f
z0t%`qf!|t_R$gl<$N^l}uW|F5GcgqIsSVqnY*`zu3QB-!H21Yjt@P|8x{`!z{yj!}
z)?wJX#9WA<FBfD8D_M&94U6~N-D^(jM)whdGH#&1zjHrfuU`U%3CZ@-#upmF{70m(
z#<5p+WIC;%UYe;zUa8F||9L{b(<$YLqYNI(f%U}BWXnI)+*Is75s??%XIiL-Mi}PD
zSC@Z15vEMp?Ikyw7a;_l1j=Eo&QFcpSoNa)bA9hT|K=i%DsKA=iac^^su0--aDMlc
zfokktOP6zaD?-pm&S!qUf%@r+{c`yJAV^~>pK<n%W8aAzwu0{bDo5-1M8NG$<mtHZ
zCh}&Yk*6sfUZP{-6Tv(&j;9K%T)qq?Bz<kFuuNECM7azjUUAL6-B|GXd`5^h6z1!?
zHjzo2(xL#Q!aYUQr}&!Kr*fN6DGV{^1l55LGu#Gako!;vKWWXdH+HVCmjhx^Qg4p(
zhFc*Adw%|kWAziIzqiJQ596OJw|e1}Js;um#a~vQadc$|@L1l1BrH<pVVb3529%}r
z5xWbgQZZB8K+d+Rae+aKmO9)e9Uxd3rN>ZQ+&^a?iBMFg*w2};Ud#oXE?k&5CUz$G
zbIW>-B`Mo!e{MIq%=s+)_NbAcn&EdeOjY<%f8xroUx)gRzdHExBd1%lftihp3(5cO
z-UVK=_};?>!lw(;Ts)mU0MkXLvNI`4%q&GLT)D_GZcNw7<EG;oYeLXY8L|f*tI`g5
z`j!!mvgx%E<Q17L5+QVRE^inyjG~BBWq3o>)ZLNdlKpZ|9cU#gC2HX2VXXU$uaB{}
zRCmRPBv96V#_+I-7T}%luqciW4<Fbkq*%;k!%jQeu-e7ABkB9e>>>eWy(PTF&o&Co
zsjX(hw1`JuQue;X#uXMcPiKtZg&4Y=WA#g(o$VwjpS;?6NJ0lRmg}{aNw7}O%shu`
zQc_yl$%#z}qspW(*zhGhSEcpG%UuM48dHYp%-{(Na-OBfV9(t;S$#xy6mgCd*gA++
zygIgv+FA+<(~r1u^V+YZx+M?*fBW1FU49WuT(g^uFH@9YSHn4G5BPBs*Pjj=ydSeN
znK1=S?@bdSepwv87aa*C%+HQL{=TloyBGYnGMX`ECYfI3i0a=H!{-cd48*{8rLx6Z
zf8pIDO^Y9!UxR<5Oo(z0V){tM9Fi!WP$QYVSW{Fi1)f;pqIi%ZN%=}K?IMVKM}+q6
z8})j&X|A+M)|^ePtir_K_TVn=<1Bpl?zgKgdZOORUu@gDMrL8mh!K^ql9^C<5L3n~
zSKY^=dxm|~WR1VnpGd!>W`oILmt()DRX6<@I~wZ3MND>hGi`Utd0opl3`by#J6rWK
zBTsE!C$;}f5_`WWJC$M5t^KImi00~8?PDOX=+3<)<{%c<I?V_gPfjn@*3>l5S1g^o
zJ8C+LCRDOjhpf)X(iY*T6%EC`FRX4==O%5=?tD;j#4lC2P8el0txd7ZP2Nu21Cdu*
z+;M%!%#ttok7f6~cjXKD$p?@jBp}i-+}mf!|MMAl{6Acz3k2%q2ZG4yji*YR&>z!x
z`P`L@r>8G5VW`-ENHOhh$)iOOP665wU8jj(6PO$6?eG6n%U?FJalV*N0B6tNLdRdE
z^>bA91O-Fe2Ly7QrWDZb$dJJQswP3-p{*v{aW2TXl+(t0$y~rO=L9irQB^->r^HcF
zWxdwHNc5l7K3-2%r`^b5oZHx(M{2W#H8wfnXI=piSI;WVPFK>F=kJ`hbX^g_qQ^}4
zla8~mFkD?u5Zo#0F)@HR1s7uoNE=h~F8)Rb{cmr(2^)e(7x*x?FL0aEp8tOMpc@Ro
zUBFa0-OpZ}?=9~Q_|}|TN4K$h019_tJ}oWnS$q4mtP<AoQK_$OON~tW(WkIZdRt~X
z`2diX_z&YJ+QriemoG@Cx5sBE)7&uB2ncxxc66{l3{6c<#h!JVNIQ{k7D?B$+Jy}5
zdc{?kiQi9<D%vCe7|V>5^j}GCK25ig`0AjvR<QK18ZvbMx(D|mY<#6<@$7u|;vL&3
zSAt!1{pja0RPI+?UgxWYp0Vk^pnjcs^;}ep{c}?DWtrg4#p@UUqqttJm^0=JM#gN^
z3I^RKgarAH(soY?Y(;{?XV$6M&fx{hI1l9tX+6m$*>&#raQ%?TQF>{o4=iRwV7<J&
zpx|&APq$$AI$LO&)Es{7^^G~W<@bJlKjfT9m<xnm?7fXcaKrgNmqGx@#~}v4iTi_H
z4ENH64}d)5X2S?!?$h`Y{hxAnHA|~3zk*-Jr+!2^%3fu>OBLJ^!@gXZ`SIx`j;PLX
zCWgUU@+@P-T38B+Vsn^!R;v7m6I`d@&UAbHxE}H2PceEHd!i}B3`&mM$uAKM^)bYt
ze2%bmFjSLU@>7)V4ZD5~9AZLG)#uS|l;%`wSU1UcO9fWkW25&j{IF!CKG${YN*BVz
zi<$RrPUwYg73lZBk7uBNdL_4Hd7GPxPi%#+Kdj8jIM+!bKRj{>xSx@P+G3&qb*?4%
zxAco%+DR<R4ncv^ng2@Mbu+n~b4v*!Me0xSGEq&<J9(Vev>qNAkK+c)@Y9_TXdQe%
zArb~KQW+u};F^}icL6~H7Vthv#usm?ep}A|^ju?#0E_TUpz+^=pFELUaX;PEtJ|E`
zikSdAOqiR!QVD&{xV*7B_<okmQ9Hw)Y3CvG>jh*Zh6PKVk^bW`H|SS>We$dT^tEk0
zX99b?)PQsNz|HRYx{y%+feFjN1T1=%2_Z%Xzwfp*_`MuL|JvMC^F<U~z1kx^e&u?M
zyu6mdZ}of7xjD>~W-2$!(~7HRE?9;-HPCiU3evq|K{j0q1q;(r>*d=8wI78IeqTKY
z8hgC#sCvsnhe)0LmTD@S&;76{##DPqo?Rc)$D9Kxi<R?ZTAJAh;<2c`1v|wd%rH<o
z{rznwaBB}fob!_&^7zVxkQFSJcX%a`5^*a$($7^wOMN-^i2>eg0$7fOHlq1iD>Y7q
zO<0lsstcU&-q*m;MN@G;U8!@f6GA-_2<6r*d9F={;h=}sM@saoe@^9PFK$#j4ievD
z(WJeSyQa-rY$s*^m>{QMDfx&rj}BAMIWmK`q!bn|F{v35)`scH!@M2)37ER(IVJZO
loqt5f+3jS~8Lke~5$A(tkMB1tsKJ^r(9<>2$v;7g_#eU<SQP*O

literal 0
HcmV?d00001


From 41d70bcfb01c0c00c64fb5615a3b143a963b8056 Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Wed, 13 Mar 2024 11:07:08 +0800
Subject: [PATCH 17/67] Add beta alert

---
 packages/web/components/templates/BetaFeature.tsx   | 5 +++++
 packages/web/pages/settings/integrations/notion.tsx | 2 ++
 2 files changed, 7 insertions(+)
 create mode 100644 packages/web/components/templates/BetaFeature.tsx

diff --git a/packages/web/components/templates/BetaFeature.tsx b/packages/web/components/templates/BetaFeature.tsx
new file mode 100644
index 000000000..6ab1de379
--- /dev/null
+++ b/packages/web/components/templates/BetaFeature.tsx
@@ -0,0 +1,5 @@
+import { Alert } from 'antd'
+
+export function BetaFeature(): JSX.Element {
+  return <Alert message="Beta" type="warning" showIcon />
+}
diff --git a/packages/web/pages/settings/integrations/notion.tsx b/packages/web/pages/settings/integrations/notion.tsx
index 4b956065e..f3152e0e1 100644
--- a/packages/web/pages/settings/integrations/notion.tsx
+++ b/packages/web/pages/settings/integrations/notion.tsx
@@ -9,6 +9,7 @@ import {
   VStack,
 } from '../../../components/elements/LayoutPrimitives'
 import { PageMetaData } from '../../../components/patterns/PageMetaData'
+import { BetaFeature } from '../../../components/templates/BetaFeature'
 import { SettingsLayout } from '../../../components/templates/SettingsLayout'
 import { useGetIntegrationsQuery } from '../../../lib/networking/queries/useGetIntegrationsQuery'
 
@@ -43,6 +44,7 @@ export default function Notion(): JSX.Element {
               height={75}
             />
             <Header>Notion integration settings</Header>
+            <BetaFeature />
           </HStack>
 
           {notion && (

From bdd89113c59ffe5f033a63eead3f02504edf9655 Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Wed, 13 Mar 2024 13:27:00 +0800
Subject: [PATCH 18/67] get value from database

---
 .../templates/{BetaFeature.tsx => Beta.tsx}   |   2 +-
 .../queries/useGetIntegrationsQuery.tsx       |   2 +-
 packages/web/pages/settings/integrations.tsx  |  35 ++--
 .../pages/settings/integrations/notion.tsx    | 160 +++++++++++++-----
 4 files changed, 136 insertions(+), 63 deletions(-)
 rename packages/web/components/templates/{BetaFeature.tsx => Beta.tsx} (66%)

diff --git a/packages/web/components/templates/BetaFeature.tsx b/packages/web/components/templates/Beta.tsx
similarity index 66%
rename from packages/web/components/templates/BetaFeature.tsx
rename to packages/web/components/templates/Beta.tsx
index 6ab1de379..00b8b473d 100644
--- a/packages/web/components/templates/BetaFeature.tsx
+++ b/packages/web/components/templates/Beta.tsx
@@ -1,5 +1,5 @@
 import { Alert } from 'antd'
 
-export function BetaFeature(): JSX.Element {
+export function Beta(): JSX.Element {
   return <Alert message="Beta" type="warning" showIcon />
 }
diff --git a/packages/web/lib/networking/queries/useGetIntegrationsQuery.tsx b/packages/web/lib/networking/queries/useGetIntegrationsQuery.tsx
index 9b2623414..abd3e3bab 100644
--- a/packages/web/lib/networking/queries/useGetIntegrationsQuery.tsx
+++ b/packages/web/lib/networking/queries/useGetIntegrationsQuery.tsx
@@ -11,7 +11,7 @@ export interface Integration {
   createdAt: Date
   updatedAt: Date
   taskName?: string
-  settings?: unknown
+  settings?: any
 }
 
 export type IntegrationType = 'EXPORT' | 'IMPORT'
diff --git a/packages/web/pages/settings/integrations.tsx b/packages/web/pages/settings/integrations.tsx
index 14c019ab7..cd83bb68c 100644
--- a/packages/web/pages/settings/integrations.tsx
+++ b/packages/web/pages/settings/integrations.tsx
@@ -89,6 +89,9 @@ export default function Integrations(): JSX.Element {
   const pocketConnected = useMemo(() => {
     return integrations.find((i) => i.name == 'POCKET' && i.type == 'IMPORT')
   }, [integrations])
+  const isConnected = (name: string) => {
+    return integrations.find((i) => i.name == name)?.enabled
+  }
 
   const deleteIntegration = async (id: string) => {
     try {
@@ -112,18 +115,20 @@ export default function Integrations(): JSX.Element {
 
   const redirectToIntegration = (
     name: string,
-    importItemState: ImportItemState
+    importItemState?: ImportItemState
   ) => {
     // create a form and submit it to the backend
     const form = document.createElement('form')
     form.method = 'POST'
     form.action = `${fetchEndpoint}/integration/${name.toLowerCase()}/auth`
-    const input = document.createElement('input')
-    input.type = 'hidden'
-    input.name = 'state'
-    input.value = importItemState
-    form.appendChild(input)
-    document.body.appendChild(form)
+    if (importItemState) {
+      const input = document.createElement('input')
+      input.type = 'hidden'
+      input.name = 'state'
+      input.value = importItemState
+      form.appendChild(input)
+    }
+
     form.submit()
   }
 
@@ -184,9 +189,10 @@ export default function Integrations(): JSX.Element {
           { duration: 5000 }
         )
       } finally {
-        router.replace('/settings/integrations')
+        router.replace('/settings/integrations/notion')
       }
     }
+
     if (!router.isReady) return
     if (router.query.pocketToken && router.query.state && !pocketConnected) {
       connectToPocket()
@@ -268,13 +274,14 @@ export default function Integrations(): JSX.Element {
         subText:
           'Notion is an all-in-one workspace. Use our Notion integration to sync your Omnivore items to Notion.',
         button: {
-          text: 'Settings',
+          text: isConnected('NOTION') ? 'Settings' : 'Connect',
           icon: <Link size={16} weight={'bold'} />,
           style: 'ctaWhite',
-          action: () =>
-            router.push(
-              '/settings/integrations/notion'
-            ),
+          action: () => {
+            isConnected('NOTION')
+              ? router.push('/settings/integrations/notion')
+              : redirectToIntegration('NOTION')
+          },
         },
       },
       {
@@ -305,7 +312,7 @@ export default function Integrations(): JSX.Element {
         },
       },
     ])
-  }, [pocketConnected, readwiseConnected, webhooks])
+  }, [pocketConnected, readwiseConnected, webhooks, integrations])
 
   return (
     <SettingsLayout>
diff --git a/packages/web/pages/settings/integrations/notion.tsx b/packages/web/pages/settings/integrations/notion.tsx
index f3152e0e1..47a541114 100644
--- a/packages/web/pages/settings/integrations/notion.tsx
+++ b/packages/web/pages/settings/integrations/notion.tsx
@@ -1,6 +1,7 @@
 import { styled } from '@stitches/react'
-import { Button, Checkbox, Form, Input, Switch } from 'antd'
+import { Button, Checkbox, Form, FormProps, Input, Space, Switch } from 'antd'
 import 'antd/dist/antd.compact.css'
+import { CheckboxValueType } from 'antd/lib/checkbox/Group'
 import Image from 'next/image'
 import { useMemo } from 'react'
 import {
@@ -9,23 +10,75 @@ import {
   VStack,
 } from '../../../components/elements/LayoutPrimitives'
 import { PageMetaData } from '../../../components/patterns/PageMetaData'
-import { BetaFeature } from '../../../components/templates/BetaFeature'
+import { Beta } from '../../../components/templates/Beta'
 import { SettingsLayout } from '../../../components/templates/SettingsLayout'
 import { useGetIntegrationsQuery } from '../../../lib/networking/queries/useGetIntegrationsQuery'
 
+interface FieldData {
+  name: string | number | (string | number)[]
+  value?: any
+  checked?: boolean
+  validating?: boolean
+  errors?: string[]
+}
+
+type FieldType = {
+  parentPageId?: string
+  parentDatabaseId?: string
+  autoSync?: boolean
+  properties?: string[]
+}
+
 // Styles
 const Header = styled(Box, {
   color: '$utilityTextDefault',
   fontSize: 'x-large',
-  margin: '20px',
+  margin: '20px 20px 40px 40px',
 })
 
 export default function Notion(): JSX.Element {
   const { integrations, revalidate } = useGetIntegrationsQuery()
-  const notion = useMemo(() => {
-    return integrations.find((i) => i.name == 'NOTION' && i.type == 'EXPORT')
+  const fields = useMemo<FieldData[]>(() => {
+    const notion = integrations.find(
+      (i) => i.name == 'NOTION' && i.type == 'EXPORT'
+    )
+    return [
+      {
+        name: 'parentPageId',
+        value: notion?.settings?.parentPageId,
+      },
+      {
+        name: 'parentDatabaseId',
+        value: notion?.settings?.parentDatabaseId,
+      },
+      {
+        name: 'autoSync',
+        checked: notion?.settings?.autoSync,
+      },
+      {
+        name: 'properties',
+        value: notion?.settings?.properties,
+      },
+    ]
   }, [integrations])
 
+  const [form] = Form.useForm<FieldType>()
+
+  const onFinish: FormProps<FieldType>['onFinish'] = (values) => {
+    console.log('Success:', values)
+  }
+
+  const onFinishFailed: FormProps<FieldType>['onFinishFailed'] = (
+    errorInfo
+  ) => {
+    console.log('Failed:', errorInfo)
+  }
+
+  const onDataChange = (value: Array<CheckboxValueType>) => {
+    form.setFieldsValue({ properties: value.map((v) => v.toString()) })
+    form.submit()
+  }
+
   return (
     <>
       <PageMetaData title="Notion" path="/integrations/notion" />
@@ -44,54 +97,67 @@ export default function Notion(): JSX.Element {
               height={75}
             />
             <Header>Notion integration settings</Header>
-            <BetaFeature />
+            <Beta />
           </HStack>
 
-          {notion && (
-            <Form>
-              <VStack
-                css={{
-                  padding: '20px',
-                }}
-              >
-                <HStack>
-                  <Form.Item
-                    label="Notion Page Id"
-                    name="parentPageId"
-                    style={{ marginRight: '20px' }}
-                    rules={[
-                      {
-                        required: true,
-                        message: 'Please input your Notion Page Id!',
-                      },
-                    ]}
-                  >
-                    <Input />
-                  </Form.Item>
+          <Form
+            labelCol={{ span: 4 }}
+            wrapperCol={{ span: 6 }}
+            labelAlign="left"
+            style={{ width: '100%' }}
+            form={form}
+            fields={fields}
+            onFinish={onFinish}
+            onFinishFailed={onFinishFailed}
+          >
+            <Form.Item<FieldType>
+              label="Notion Page Id"
+              name="parentPageId"
+              rules={[
+                {
+                  required: true,
+                  message: 'Please input your Notion Page Id!',
+                },
+              ]}
+            >
+              <Space>
+                <Input />
+                <Button type="primary" htmlType="submit">
+                  Save
+                </Button>
+              </Space>
+            </Form.Item>
 
-                  <Button>Save</Button>
-                </HStack>
+            <Form.Item<FieldType>
+              label="Notion Database Id"
+              name="parentDatabaseId"
+              hidden
+            >
+              <Input disabled />
+            </Form.Item>
 
-                <Form.Item label="Notion Database Id" name="parentDatabaseId">
-                  <Input disabled />
-                </Form.Item>
+            <Form.Item<FieldType> label="Automatic Sync" name="autoSync">
+              <Switch />
+            </Form.Item>
 
-                <Form.Item label="Automatic Sync" name="autoSync">
-                  <Switch />
-                </Form.Item>
+            <Form.Item<FieldType>
+              label="Properties to Export"
+              name="properties"
+            >
+              <Checkbox.Group onChange={onDataChange}>
+                <Checkbox value="highlights">Highlights</Checkbox>
+                <Checkbox value="labels">Labels</Checkbox>
+                <Checkbox value="notes">Notes</Checkbox>
+              </Checkbox.Group>
+            </Form.Item>
+          </Form>
 
-                <Form.Item label="Data to Export" name="dataToExport">
-                  <Checkbox.Group>
-                    <Checkbox value="highlights">Highlights</Checkbox>
-                    <Checkbox value="labels">Labels</Checkbox>
-                    <Checkbox value="notes">Notes</Checkbox>
-                  </Checkbox.Group>
-                </Form.Item>
-
-                <Button>Export Recent Items</Button>
-              </VStack>
-            </Form>
-          )}
+          <Space>
+            <Button type="primary">Export Recent Items</Button>
+            <Button type="primary" danger>
+              Disconnect
+            </Button>
+          </Space>
         </VStack>
       </SettingsLayout>
     </>

From 6310e5f83ba3d1c01bad776eda70332a582810b5 Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Wed, 13 Mar 2024 15:55:34 +0800
Subject: [PATCH 19/67] update form

---
 .../api/src/resolvers/integrations/index.ts   | 35 ---------
 .../api/src/services/integrations/index.ts    |  6 +-
 .../mutations/setIntegrationMutation.ts       |  2 +
 .../pages/settings/integrations/notion.tsx    | 78 ++++++++++++++-----
 4 files changed, 66 insertions(+), 55 deletions(-)

diff --git a/packages/api/src/resolvers/integrations/index.ts b/packages/api/src/resolvers/integrations/index.ts
index d50992b28..d0bfc45d1 100644
--- a/packages/api/src/resolvers/integrations/index.ts
+++ b/packages/api/src/resolvers/integrations/index.ts
@@ -34,7 +34,6 @@ import {
 import { analytics } from '../../utils/analytics'
 import {
   deleteTask,
-  enqueueExportAllItems,
   enqueueImportFromIntegration,
 } from '../../utils/createTask'
 import { authorized } from '../../utils/gql-utils'
@@ -85,40 +84,6 @@ export const setIntegrationResolver = authorized<
     // save integration
     const integration = await saveIntegration(integrationToSave, uid)
 
-    if (integrationToSave.type === IntegrationType.Export && !input.id) {
-      const authToken = await createIntegrationToken({
-        uid,
-        token: integration.token,
-      })
-      if (!authToken) {
-        log.error('failed to create auth token', {
-          integrationId: integration.id,
-        })
-        return {
-          errorCodes: [SetIntegrationErrorCode.BadRequest],
-        }
-      }
-
-      // create a task to sync all the pages if new integration or enable integration (export type)
-      await enqueueExportAllItems(integration.id, uid)
-    } else if (integrationToSave.taskName) {
-      // delete the task if disable integration and task exists
-      const result = await deleteTask(integrationToSave.taskName)
-      if (result) {
-        log.info('task deleted', integrationToSave.taskName)
-      }
-
-      // update task name in integration
-      await updateIntegration(
-        integration.id,
-        {
-          taskName: null,
-        },
-        uid
-      )
-      integration.taskName = null
-    }
-
     analytics.capture({
       distinctId: uid,
       event: 'integration_set',
diff --git a/packages/api/src/services/integrations/index.ts b/packages/api/src/services/integrations/index.ts
index a925a2765..2f2b99f0d 100644
--- a/packages/api/src/services/integrations/index.ts
+++ b/packages/api/src/services/integrations/index.ts
@@ -80,7 +80,11 @@ export const saveIntegration = async (
   userId: string
 ) => {
   return authTrx(
-    async (t) => t.getRepository(Integration).save(integration),
+    async (t) => {
+      const repo = t.getRepository(Integration)
+      const newIntegration = await repo.save(integration)
+      return repo.findOneByOrFail({ id: newIntegration.id })
+    },
     undefined,
     userId
   )
diff --git a/packages/web/lib/networking/mutations/setIntegrationMutation.ts b/packages/web/lib/networking/mutations/setIntegrationMutation.ts
index 66434ae7c..0c0bc4b6d 100644
--- a/packages/web/lib/networking/mutations/setIntegrationMutation.ts
+++ b/packages/web/lib/networking/mutations/setIntegrationMutation.ts
@@ -16,6 +16,7 @@ export type SetIntegrationInput = {
   token: string
   enabled: boolean
   importItemState?: ImportItemState
+  settings?: any
 }
 
 type SetIntegrationResult = {
@@ -52,6 +53,7 @@ export async function setIntegrationMutation(
             enabled
             createdAt
             updatedAt
+            settings
           }
         }
         ... on SetIntegrationError {
diff --git a/packages/web/pages/settings/integrations/notion.tsx b/packages/web/pages/settings/integrations/notion.tsx
index 47a541114..0d1107993 100644
--- a/packages/web/pages/settings/integrations/notion.tsx
+++ b/packages/web/pages/settings/integrations/notion.tsx
@@ -1,5 +1,14 @@
 import { styled } from '@stitches/react'
-import { Button, Checkbox, Form, FormProps, Input, Space, Switch } from 'antd'
+import {
+  Button,
+  Checkbox,
+  Form,
+  FormProps,
+  Input,
+  message,
+  Space,
+  Switch,
+} from 'antd'
 import 'antd/dist/antd.compact.css'
 import { CheckboxValueType } from 'antd/lib/checkbox/Group'
 import Image from 'next/image'
@@ -12,6 +21,7 @@ import {
 import { PageMetaData } from '../../../components/patterns/PageMetaData'
 import { Beta } from '../../../components/templates/Beta'
 import { SettingsLayout } from '../../../components/templates/SettingsLayout'
+import { setIntegrationMutation } from '../../../lib/networking/mutations/setIntegrationMutation'
 import { useGetIntegrationsQuery } from '../../../lib/networking/queries/useGetIntegrationsQuery'
 
 interface FieldData {
@@ -38,11 +48,12 @@ const Header = styled(Box, {
 
 export default function Notion(): JSX.Element {
   const { integrations, revalidate } = useGetIntegrationsQuery()
-  const fields = useMemo<FieldData[]>(() => {
-    const notion = integrations.find(
-      (i) => i.name == 'NOTION' && i.type == 'EXPORT'
-    )
-    return [
+  const notion = useMemo(
+    () => integrations.find((i) => i.name == 'NOTION' && i.type == 'EXPORT'),
+    [integrations]
+  )
+  const fields = useMemo<FieldData[]>(
+    () => [
       {
         name: 'parentPageId',
         value: notion?.settings?.parentPageId,
@@ -53,19 +64,43 @@ export default function Notion(): JSX.Element {
       },
       {
         name: 'autoSync',
-        checked: notion?.settings?.autoSync,
+        value: notion?.settings?.autoSync,
       },
       {
         name: 'properties',
         value: notion?.settings?.properties,
       },
-    ]
-  }, [integrations])
+    ],
+    [notion]
+  )
 
   const [form] = Form.useForm<FieldType>()
+  const [messageApi, contextHolder] = message.useMessage()
 
-  const onFinish: FormProps<FieldType>['onFinish'] = (values) => {
-    console.log('Success:', values)
+  const updateNotion = async (values: FieldType) => {
+    if (!notion) {
+      throw new Error('Notion integration not found')
+    }
+
+    await setIntegrationMutation({
+      id: notion.id,
+      name: notion.name,
+      type: notion.type,
+      token: notion.token,
+      enabled: notion.enabled,
+      settings: values,
+    })
+  }
+
+  const onFinish: FormProps<FieldType>['onFinish'] = async (values) => {
+    try {
+      await updateNotion(values)
+
+      revalidate()
+      messageApi.success('Notion settings updated successfully.')
+    } catch (error) {
+      messageApi.error('There was an error updating Notion settings.')
+    }
   }
 
   const onFinishFailed: FormProps<FieldType>['onFinishFailed'] = (
@@ -76,11 +111,11 @@ export default function Notion(): JSX.Element {
 
   const onDataChange = (value: Array<CheckboxValueType>) => {
     form.setFieldsValue({ properties: value.map((v) => v.toString()) })
-    form.submit()
   }
 
   return (
     <>
+      {contextHolder}
       <PageMetaData title="Notion" path="/integrations/notion" />
       <SettingsLayout>
         <VStack
@@ -120,12 +155,7 @@ export default function Notion(): JSX.Element {
                 },
               ]}
             >
-              <Space>
-                <Input />
-                <Button type="primary" htmlType="submit">
-                  Save
-                </Button>
-              </Space>
+              <Input />
             </Form.Item>
 
             <Form.Item<FieldType>
@@ -136,7 +166,11 @@ export default function Notion(): JSX.Element {
               <Input disabled />
             </Form.Item>
 
-            <Form.Item<FieldType> label="Automatic Sync" name="autoSync">
+            <Form.Item<FieldType>
+              label="Automatic Sync"
+              name="autoSync"
+              valuePropName="checked"
+            >
               <Switch />
             </Form.Item>
 
@@ -150,6 +184,12 @@ export default function Notion(): JSX.Element {
                 <Checkbox value="notes">Notes</Checkbox>
               </Checkbox.Group>
             </Form.Item>
+
+            <Form.Item>
+              <Button type="primary" htmlType="submit">
+                Save
+              </Button>
+            </Form.Item>
           </Form>
 
           <Space>

From 16e76538d8dcc6ff0d961efd6281af3d3efe366f Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Wed, 13 Mar 2024 18:45:26 +0800
Subject: [PATCH 20/67] add delete notion button

---
 packages/web/next.config.js                   |  2 +-
 packages/web/pages/settings/integrations.tsx  |  7 ++++---
 .../pages/settings/integrations/notion.tsx    | 20 ++++++++++++++++++-
 3 files changed, 24 insertions(+), 5 deletions(-)

diff --git a/packages/web/next.config.js b/packages/web/next.config.js
index e4d6e85dc..07228c7f9 100644
--- a/packages/web/next.config.js
+++ b/packages/web/next.config.js
@@ -3,7 +3,7 @@ const ContentSecurityPolicy = `
   base-uri 'self';
   connect-src 'self' ${process.env.NEXT_PUBLIC_SERVER_BASE_URL} https://proxy-prod.omnivore-image-cache.app https://accounts.google.com https://proxy-demo.omnivore-image-cache.app https://storage.googleapis.com https://api.segment.io https://cdn.segment.com https://widget.intercom.io https://api-iam.intercom.io https://static.intercomassets.com https://downloads.intercomcdn.com https://platform.twitter.com wss://nexus-websocket-a.intercom.io wss://nexus-websocket-b.intercom.io wss://nexus-europe-websocket.intercom.io wss://nexus-australia-websocket.intercom.io https://uploads.intercomcdn.com https://tools.applemediaservices.com;
   font-src 'self' data: https://cdn.jsdelivr.net https://js.intercomcdn.com https://fonts.intercomcdn.com;
-  form-action 'self' ${process.env.NEXT_PUBLIC_SERVER_BASE_URL} https://getpocket.com/auth/authorize https://intercom.help https://api-iam.intercom.io https://api-iam.eu.intercom.io https://api-iam.au.intercom.io;
+  form-action 'self' ${process.env.NEXT_PUBLIC_SERVER_BASE_URL} https://getpocket.com/auth/authorize https://intercom.help https://api-iam.intercom.io https://api-iam.eu.intercom.io https://api-iam.au.intercom.io https://www.notion.so https://api.notion.com;
   frame-ancestors 'none';
   frame-src 'self' https://accounts.google.com https://platform.twitter.com https://www.youtube.com https://www.youtube-nocookie.com;
   manifest-src 'self';
diff --git a/packages/web/pages/settings/integrations.tsx b/packages/web/pages/settings/integrations.tsx
index cd83bb68c..2e1e17d9b 100644
--- a/packages/web/pages/settings/integrations.tsx
+++ b/packages/web/pages/settings/integrations.tsx
@@ -128,6 +128,7 @@ export default function Integrations(): JSX.Element {
       input.value = importItemState
       form.appendChild(input)
     }
+    document.body.appendChild(form)
 
     form.submit()
   }
@@ -163,7 +164,7 @@ export default function Integrations(): JSX.Element {
           { duration: 5000 }
         )
       } finally {
-        router.replace('/settings/integrations')
+        router.push('/settings/integrations')
       }
     }
 
@@ -189,7 +190,7 @@ export default function Integrations(): JSX.Element {
           { duration: 5000 }
         )
       } finally {
-        router.replace('/settings/integrations/notion')
+        router.push('/settings/integrations/notion')
       }
     }
 
@@ -276,7 +277,7 @@ export default function Integrations(): JSX.Element {
         button: {
           text: isConnected('NOTION') ? 'Settings' : 'Connect',
           icon: <Link size={16} weight={'bold'} />,
-          style: 'ctaWhite',
+          style: isConnected('NOTION') ? 'ctaWhite' : 'ctaDarkYellow',
           action: () => {
             isConnected('NOTION')
               ? router.push('/settings/integrations/notion')
diff --git a/packages/web/pages/settings/integrations/notion.tsx b/packages/web/pages/settings/integrations/notion.tsx
index 0d1107993..28fac42bb 100644
--- a/packages/web/pages/settings/integrations/notion.tsx
+++ b/packages/web/pages/settings/integrations/notion.tsx
@@ -12,6 +12,7 @@ import {
 import 'antd/dist/antd.compact.css'
 import { CheckboxValueType } from 'antd/lib/checkbox/Group'
 import Image from 'next/image'
+import { useRouter } from 'next/router'
 import { useMemo } from 'react'
 import {
   Box,
@@ -21,8 +22,11 @@ import {
 import { PageMetaData } from '../../../components/patterns/PageMetaData'
 import { Beta } from '../../../components/templates/Beta'
 import { SettingsLayout } from '../../../components/templates/SettingsLayout'
+import { deleteIntegrationMutation } from '../../../lib/networking/mutations/deleteIntegrationMutation'
 import { setIntegrationMutation } from '../../../lib/networking/mutations/setIntegrationMutation'
 import { useGetIntegrationsQuery } from '../../../lib/networking/queries/useGetIntegrationsQuery'
+import { applyStoredTheme } from '../../../lib/themeUpdater'
+import { showSuccessToast } from '../../../lib/toastHelpers'
 
 interface FieldData {
   name: string | number | (string | number)[]
@@ -47,6 +51,9 @@ const Header = styled(Box, {
 })
 
 export default function Notion(): JSX.Element {
+  applyStoredTheme()
+
+  const router = useRouter()
   const { integrations, revalidate } = useGetIntegrationsQuery()
   const notion = useMemo(
     () => integrations.find((i) => i.name == 'NOTION' && i.type == 'EXPORT'),
@@ -77,6 +84,17 @@ export default function Notion(): JSX.Element {
   const [form] = Form.useForm<FieldType>()
   const [messageApi, contextHolder] = message.useMessage()
 
+  const deleteNotion = async () => {
+    if (!notion) {
+      throw new Error('Notion integration not found')
+    }
+
+    await deleteIntegrationMutation(notion.id)
+    showSuccessToast('Notion integration disconnected successfully.')
+
+    router.push('/settings/integrations')
+  }
+
   const updateNotion = async (values: FieldType) => {
     if (!notion) {
       throw new Error('Notion integration not found')
@@ -194,7 +212,7 @@ export default function Notion(): JSX.Element {
 
           <Space>
             <Button type="primary">Export Recent Items</Button>
-            <Button type="primary" danger>
+            <Button type="primary" danger onClick={deleteNotion}>
               Disconnect
             </Button>
           </Space>

From 1fb81eaa2b40aadc3c436e29e3b7683c1a615d7a Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Wed, 13 Mar 2024 18:54:57 +0800
Subject: [PATCH 21/67] hide notion settings if not connected

---
 .../pages/settings/integrations/notion.tsx    | 120 +++++++++---------
 1 file changed, 62 insertions(+), 58 deletions(-)

diff --git a/packages/web/pages/settings/integrations/notion.tsx b/packages/web/pages/settings/integrations/notion.tsx
index 28fac42bb..e61303385 100644
--- a/packages/web/pages/settings/integrations/notion.tsx
+++ b/packages/web/pages/settings/integrations/notion.tsx
@@ -153,69 +153,73 @@ export default function Notion(): JSX.Element {
             <Beta />
           </HStack>
 
-          <Form
-            labelCol={{ span: 4 }}
-            wrapperCol={{ span: 6 }}
-            labelAlign="left"
-            style={{ width: '100%' }}
-            form={form}
-            fields={fields}
-            onFinish={onFinish}
-            onFinishFailed={onFinishFailed}
-          >
-            <Form.Item<FieldType>
-              label="Notion Page Id"
-              name="parentPageId"
-              rules={[
-                {
-                  required: true,
-                  message: 'Please input your Notion Page Id!',
-                },
-              ]}
-            >
-              <Input />
-            </Form.Item>
+          {notion && (
+            <div style={{ width: '100%' }}>
+              <Form
+                labelCol={{ span: 4 }}
+                wrapperCol={{ span: 6 }}
+                labelAlign="left"
+                style={{ width: '100%' }}
+                form={form}
+                fields={fields}
+                onFinish={onFinish}
+                onFinishFailed={onFinishFailed}
+              >
+                <Form.Item<FieldType>
+                  label="Notion Page Id"
+                  name="parentPageId"
+                  rules={[
+                    {
+                      required: true,
+                      message: 'Please input your Notion Page Id!',
+                    },
+                  ]}
+                >
+                  <Input />
+                </Form.Item>
 
-            <Form.Item<FieldType>
-              label="Notion Database Id"
-              name="parentDatabaseId"
-              hidden
-            >
-              <Input disabled />
-            </Form.Item>
+                <Form.Item<FieldType>
+                  label="Notion Database Id"
+                  name="parentDatabaseId"
+                  hidden
+                >
+                  <Input disabled />
+                </Form.Item>
 
-            <Form.Item<FieldType>
-              label="Automatic Sync"
-              name="autoSync"
-              valuePropName="checked"
-            >
-              <Switch />
-            </Form.Item>
+                <Form.Item<FieldType>
+                  label="Automatic Sync"
+                  name="autoSync"
+                  valuePropName="checked"
+                >
+                  <Switch />
+                </Form.Item>
 
-            <Form.Item<FieldType>
-              label="Properties to Export"
-              name="properties"
-            >
-              <Checkbox.Group onChange={onDataChange}>
-                <Checkbox value="highlights">Highlights</Checkbox>
-                <Checkbox value="labels">Labels</Checkbox>
-                <Checkbox value="notes">Notes</Checkbox>
-              </Checkbox.Group>
-            </Form.Item>
+                <Form.Item<FieldType>
+                  label="Properties to Export"
+                  name="properties"
+                >
+                  <Checkbox.Group onChange={onDataChange}>
+                    <Checkbox value="highlights">Highlights</Checkbox>
+                    <Checkbox value="labels">Labels</Checkbox>
+                    <Checkbox value="notes">Notes</Checkbox>
+                  </Checkbox.Group>
+                </Form.Item>
 
-            <Form.Item>
-              <Button type="primary" htmlType="submit">
-                Save
-              </Button>
-            </Form.Item>
-          </Form>
+                <Form.Item>
+                  <Button type="primary" htmlType="submit">
+                    Save
+                  </Button>
+                </Form.Item>
+              </Form>
 
-          <Space>
-            <Button type="primary">Export Recent Items</Button>
-            <Button type="primary" danger onClick={deleteNotion}>
-              Disconnect
-            </Button>
-          </Space>
+              <Space>
+                <Button type="primary">Export Recent Items</Button>
+                <Button type="primary" danger onClick={deleteNotion}>
+                  Disconnect
+                </Button>
+              </Space>
+            </div>
+          )}
         </VStack>
       </SettingsLayout>
     </>

From 53cc193bd157acc1b5cf8e151a3912b7e6bd8553 Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Wed, 13 Mar 2024 19:10:11 +0800
Subject: [PATCH 22/67] update header

---
 .../templates/integrations/Readwise.tsx       | 25 +++++---------
 .../templates/settings/SettingsTable.tsx      |  9 ++++-
 .../pages/settings/integrations/notion.tsx    | 33 ++++++++-----------
 3 files changed, 30 insertions(+), 37 deletions(-)

diff --git a/packages/web/components/templates/integrations/Readwise.tsx b/packages/web/components/templates/integrations/Readwise.tsx
index 4e0469896..353687e1b 100644
--- a/packages/web/components/templates/integrations/Readwise.tsx
+++ b/packages/web/components/templates/integrations/Readwise.tsx
@@ -1,27 +1,18 @@
-import { useCallback, useMemo, useState } from 'react'
-import { styled } from '@stitches/react'
 import Image from 'next/image'
-
-import { Box, HStack, SpanBox, VStack } from '../../elements/LayoutPrimitives'
-import { Button } from '../../elements/Button'
-import { StyledText } from '../../elements/StyledText'
-import { FormInput } from '../../elements/FormElements'
-
+import { useRouter } from 'next/router'
+import { useCallback, useMemo, useState } from 'react'
+import { deleteIntegrationMutation } from '../../../lib/networking/mutations/deleteIntegrationMutation'
 import { setIntegrationMutation } from '../../../lib/networking/mutations/setIntegrationMutation'
 import {
   Integration,
   useGetIntegrationsQuery,
 } from '../../../lib/networking/queries/useGetIntegrationsQuery'
-import { useRouter } from 'next/router'
 import { showErrorToast, showSuccessToast } from '../../../lib/toastHelpers'
-import { deleteIntegrationMutation } from '../../../lib/networking/mutations/deleteIntegrationMutation'
-
-// Styles
-const Header = styled(Box, {
-  color: '$utilityTextDefault',
-  fontSize: 'x-large',
-  margin: '20px',
-})
+import { Button } from '../../elements/Button'
+import { FormInput } from '../../elements/FormElements'
+import { HStack, SpanBox, VStack } from '../../elements/LayoutPrimitives'
+import { StyledText } from '../../elements/StyledText'
+import { Header } from '../settings/SettingsTable'
 
 export function Readwise(): JSX.Element {
   const { integrations, revalidate } = useGetIntegrationsQuery()
diff --git a/packages/web/components/templates/settings/SettingsTable.tsx b/packages/web/components/templates/settings/SettingsTable.tsx
index 7e87c9a7a..1db5872a6 100644
--- a/packages/web/components/templates/settings/SettingsTable.tsx
+++ b/packages/web/components/templates/settings/SettingsTable.tsx
@@ -6,11 +6,18 @@ import { MoreOptionsIcon } from '../../elements/images/MoreOptionsIcon'
 import { InfoLink } from '../../elements/InfoLink'
 import { Box, HStack, SpanBox, VStack } from '../../elements/LayoutPrimitives'
 import { StyledText } from '../../elements/StyledText'
-import { theme } from '../../tokens/stitches.config'
+import { styled, theme } from '../../tokens/stitches.config'
 import { SettingsLayout } from '../SettingsLayout'
 import { usePersistedState } from '../../../lib/hooks/usePersistedState'
 import { FeatureHelpBox } from '../../elements/FeatureHelpBox'
 
+// Styles
+export const Header = styled(Box, {
+  color: '$utilityTextDefault',
+  fontSize: 'x-large',
+  margin: '20px',
+})
+
 type SettingsTableProps = {
   pageId: string
   pageInfoLink?: string | undefined
diff --git a/packages/web/pages/settings/integrations/notion.tsx b/packages/web/pages/settings/integrations/notion.tsx
index e61303385..9f5e7f8f1 100644
--- a/packages/web/pages/settings/integrations/notion.tsx
+++ b/packages/web/pages/settings/integrations/notion.tsx
@@ -1,4 +1,3 @@
-import { styled } from '@stitches/react'
 import {
   Button,
   Checkbox,
@@ -14,13 +13,10 @@ import { CheckboxValueType } from 'antd/lib/checkbox/Group'
 import Image from 'next/image'
 import { useRouter } from 'next/router'
 import { useMemo } from 'react'
-import {
-  Box,
-  HStack,
-  VStack,
-} from '../../../components/elements/LayoutPrimitives'
+import { HStack, VStack } from '../../../components/elements/LayoutPrimitives'
 import { PageMetaData } from '../../../components/patterns/PageMetaData'
 import { Beta } from '../../../components/templates/Beta'
+import { Header } from '../../../components/templates/settings/SettingsTable'
 import { SettingsLayout } from '../../../components/templates/SettingsLayout'
 import { deleteIntegrationMutation } from '../../../lib/networking/mutations/deleteIntegrationMutation'
 import { setIntegrationMutation } from '../../../lib/networking/mutations/setIntegrationMutation'
@@ -43,13 +39,6 @@ type FieldType = {
   properties?: string[]
 }
 
-// Styles
-const Header = styled(Box, {
-  color: '$utilityTextDefault',
-  fontSize: 'x-large',
-  margin: '20px 20px 40px 40px',
-})
-
 export default function Notion(): JSX.Element {
   applyStoredTheme()
 
@@ -138,11 +127,18 @@ export default function Notion(): JSX.Element {
       <SettingsLayout>
         <VStack
           css={{
-            margin: 'auto',
+            margin: '0 auto',
             width: '80%',
           }}
         >
-          <HStack>
+          <HStack
+            css={{
+              width: '100%',
+              pb: '$2',
+              borderBottom: '1px solid $utilityTextDefault',
+              pr: '$1',
+            }}
+          >
             <Image
               src="/static/icons/notion.png"
               alt="Integration Image"
@@ -154,12 +150,11 @@ export default function Notion(): JSX.Element {
           </HStack>
 
           {notion && (
-            <div style={{ width: '100%' }}>
+            <div style={{ width: '100%', marginTop: '40px' }}>
               <Form
-                labelCol={{ span: 4 }}
-                wrapperCol={{ span: 6 }}
+                labelCol={{ span: 6 }}
+                wrapperCol={{ span: 8 }}
                 labelAlign="left"
-                style={{ width: '100%' }}
                 form={form}
                 fields={fields}
                 onFinish={onFinish}

From 6a7d2b0ec24f163b46a2f66bcd4974a55906defe Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Wed, 13 Mar 2024 19:11:38 +0800
Subject: [PATCH 23/67] fix tests

---
 packages/api/test/resolvers/integrations.test.ts | 11 -----------
 1 file changed, 11 deletions(-)

diff --git a/packages/api/test/resolvers/integrations.test.ts b/packages/api/test/resolvers/integrations.test.ts
index 992cfe9ca..538841a53 100644
--- a/packages/api/test/resolvers/integrations.test.ts
+++ b/packages/api/test/resolvers/integrations.test.ts
@@ -222,17 +222,6 @@ describe('Integrations resolvers', () => {
               expect(res.body.data.setIntegration.integration.enabled).to.be
                 .false
             })
-
-            it('deletes cloud task', async () => {
-              const res = await graphqlRequest(
-                query(integrationId, integrationName, token, enabled),
-                authToken
-              )
-              const integration = await findIntegration({
-                id: res.body.data.setIntegration.integration.id,
-              }, loginUser.id)
-              expect(integration?.taskName).to.be.null
-            })
           })
 
           context('when enable is true', () => {

From 9ed23611a16b201968e567a9f7a1d3f7fcb5ebe7 Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Wed, 13 Mar 2024 19:41:39 +0800
Subject: [PATCH 24/67] fix styles

---
 packages/web/pages/settings/integrations.tsx  | 16 ++--
 .../pages/settings/integrations/notion.tsx    | 77 ++++++++-----------
 2 files changed, 38 insertions(+), 55 deletions(-)

diff --git a/packages/web/pages/settings/integrations.tsx b/packages/web/pages/settings/integrations.tsx
index 2e1e17d9b..c9f2e3296 100644
--- a/packages/web/pages/settings/integrations.tsx
+++ b/packages/web/pages/settings/integrations.tsx
@@ -172,25 +172,23 @@ export default function Integrations(): JSX.Element {
       try {
         // get the token from query string
         const token = router.query.code as string
-        const result = await setIntegrationMutation({
+        await setIntegrationMutation({
           token,
           name: 'NOTION',
           type: 'EXPORT',
           enabled: true,
         })
-        if (result) {
-          revalidate()
-          showSuccessToast('Connected with Notion.')
-        } else {
-          showErrorToast('There was an error connecting to Notion.')
-        }
+
+        showSuccessToast('Connected with Notion.')
+
+        router.push('/settings/integrations/notion')
       } catch (err) {
         showErrorToast(
           'There was an error connecting to Notion. Please try again.',
           { duration: 5000 }
         )
-      } finally {
-        router.push('/settings/integrations/notion')
+
+        router.push('/settings/integrations')
       }
     }
 
diff --git a/packages/web/pages/settings/integrations/notion.tsx b/packages/web/pages/settings/integrations/notion.tsx
index 9f5e7f8f1..eb56fa892 100644
--- a/packages/web/pages/settings/integrations/notion.tsx
+++ b/packages/web/pages/settings/integrations/notion.tsx
@@ -12,7 +12,7 @@ import 'antd/dist/antd.compact.css'
 import { CheckboxValueType } from 'antd/lib/checkbox/Group'
 import Image from 'next/image'
 import { useRouter } from 'next/router'
-import { useMemo } from 'react'
+import { useEffect, useState } from 'react'
 import { HStack, VStack } from '../../../components/elements/LayoutPrimitives'
 import { PageMetaData } from '../../../components/patterns/PageMetaData'
 import { Beta } from '../../../components/templates/Beta'
@@ -20,18 +20,13 @@ import { Header } from '../../../components/templates/settings/SettingsTable'
 import { SettingsLayout } from '../../../components/templates/SettingsLayout'
 import { deleteIntegrationMutation } from '../../../lib/networking/mutations/deleteIntegrationMutation'
 import { setIntegrationMutation } from '../../../lib/networking/mutations/setIntegrationMutation'
-import { useGetIntegrationsQuery } from '../../../lib/networking/queries/useGetIntegrationsQuery'
+import {
+  Integration,
+  useGetIntegrationsQuery,
+} from '../../../lib/networking/queries/useGetIntegrationsQuery'
 import { applyStoredTheme } from '../../../lib/themeUpdater'
 import { showSuccessToast } from '../../../lib/toastHelpers'
 
-interface FieldData {
-  name: string | number | (string | number)[]
-  value?: any
-  checked?: boolean
-  validating?: boolean
-  errors?: string[]
-}
-
 type FieldType = {
   parentPageId?: string
   parentDatabaseId?: string
@@ -44,35 +39,28 @@ export default function Notion(): JSX.Element {
 
   const router = useRouter()
   const { integrations, revalidate } = useGetIntegrationsQuery()
-  const notion = useMemo(
-    () => integrations.find((i) => i.name == 'NOTION' && i.type == 'EXPORT'),
-    [integrations]
-  )
-  const fields = useMemo<FieldData[]>(
-    () => [
-      {
-        name: 'parentPageId',
-        value: notion?.settings?.parentPageId,
-      },
-      {
-        name: 'parentDatabaseId',
-        value: notion?.settings?.parentDatabaseId,
-      },
-      {
-        name: 'autoSync',
-        value: notion?.settings?.autoSync,
-      },
-      {
-        name: 'properties',
-        value: notion?.settings?.properties,
-      },
-    ],
-    [notion]
-  )
+  const [notion, setNotion] = useState<Integration>()
 
   const [form] = Form.useForm<FieldType>()
   const [messageApi, contextHolder] = message.useMessage()
 
+  useEffect(() => {
+    const notion = integrations.find(
+      (i) => i.name == 'NOTION' && i.type == 'EXPORT'
+    )
+
+    if (notion) {
+      setNotion(notion)
+
+      form.setFieldsValue({
+        parentPageId: notion.settings?.parentPageId,
+        parentDatabaseId: notion.settings?.parentDatabaseId,
+        autoSync: notion.settings?.autoSync,
+        properties: notion.settings?.properties,
+      })
+    }
+  }, [form, integrations])
+
   const deleteNotion = async () => {
     if (!notion) {
       throw new Error('Notion integration not found')
@@ -156,7 +144,6 @@ export default function Notion(): JSX.Element {
                 wrapperCol={{ span: 8 }}
                 labelAlign="left"
                 form={form}
-                fields={fields}
                 onFinish={onFinish}
                 onFinishFailed={onFinishFailed}
               >
@@ -201,18 +188,16 @@ export default function Notion(): JSX.Element {
                 </Form.Item>
 
                 <Form.Item>
-                  <Button type="primary" htmlType="submit">
-                    Save
-                  </Button>
+                  <Space>
+                    <Button type="primary" htmlType="submit">
+                      Save
+                    </Button>
+                    <Button type="primary" danger onClick={deleteNotion}>
+                      Disconnect
+                    </Button>
+                  </Space>
                 </Form.Item>
               </Form>
-
-              <Space>
-                <Button type="primary">Export Recent Items</Button>
-                <Button type="primary" danger onClick={deleteNotion}>
-                  Disconnect
-                </Button>
-              </Space>
             </div>
           )}
         </VStack>

From 433fa00fa391a2a0a626f56083612c385468d2bc Mon Sep 17 00:00:00 2001
From: Jackson Harper <jacksonh@gmail.com>
Date: Wed, 13 Mar 2024 21:13:32 +0800
Subject: [PATCH 25/67] Use older version of transmission for swipe back
 gesture

Also comment out findSelected as it crashes in iOS 17.4
---
 .../xcshareddata/swiftpm/Package.resolved                   | 4 ++--
 apple/OmnivoreKit/Package.swift                             | 6 +++---
 .../OmnivoreKit/Sources/Views/Article/OmnivoreWebView.swift | 2 +-
 3 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/apple/Omnivore.xcworkspace/xcshareddata/swiftpm/Package.resolved b/apple/Omnivore.xcworkspace/xcshareddata/swiftpm/Package.resolved
index d5a6b0de1..ed109f98e 100644
--- a/apple/Omnivore.xcworkspace/xcshareddata/swiftpm/Package.resolved
+++ b/apple/Omnivore.xcworkspace/xcshareddata/swiftpm/Package.resolved
@@ -239,8 +239,8 @@
       "kind" : "remoteSourceControl",
       "location" : "https://github.com/nathantannar4/Transmission",
       "state" : {
-        "revision" : "3dac53ae4bddc7ab99e6374622a9c5eefbe50eed",
-        "version" : "1.1.4"
+        "revision" : "9517912f8f528c777f86f7896b5c35d7e43fa916",
+        "version" : "1.0.1"
       }
     },
     {
diff --git a/apple/OmnivoreKit/Package.swift b/apple/OmnivoreKit/Package.swift
index 93c47d10a..4683efbfa 100644
--- a/apple/OmnivoreKit/Package.swift
+++ b/apple/OmnivoreKit/Package.swift
@@ -71,9 +71,9 @@ var dependencies: [Package.Dependency] {
     .package(url: "https://github.com/google/GoogleSignIn-iOS", from: "6.2.2"),
     .package(url: "https://github.com/gonzalezreal/swift-markdown-ui", from: "2.0.0"),
     .package(url: "https://github.com/PostHog/posthog-ios.git", from: "2.0.0"),
-    .package(url: "https://github.com/nathantannar4/Engine", exact: "1.5.1"),
-    .package(url: "https://github.com/nathantannar4/Turbocharger", exact: "1.1.4"),
-    .package(url: "https://github.com/nathantannar4/Transmission", from: "1.1.4")
+//    .package(url: "https://github.com/nathantannar4/Engine", exact: "1.0.1"),
+//    .package(url: "https://github.com/nathantannar4/Turbocharger", exact: "1.1.4"),
+    .package(url: "https://github.com/nathantannar4/Transmission", exact: "1.0.1")
   ]
   // Comment out following line for macOS build
   deps.append(.package(url: "https://github.com/PSPDFKit/PSPDFKit-SP", from: "13.1.0"))
diff --git a/apple/OmnivoreKit/Sources/Views/Article/OmnivoreWebView.swift b/apple/OmnivoreKit/Sources/Views/Article/OmnivoreWebView.swift
index 23c9bd1b3..465518fb1 100644
--- a/apple/OmnivoreKit/Sources/Views/Article/OmnivoreWebView.swift
+++ b/apple/OmnivoreKit/Sources/Views/Article/OmnivoreWebView.swift
@@ -301,7 +301,7 @@ public final class OmnivoreWebView: WKWebView {
       case Selector(("_lookup:")): return (currentMenu == .defaultMenu)
       case Selector(("_define:")): return (currentMenu == .defaultMenu)
       case Selector(("_translate:")): return (currentMenu == .defaultMenu)
-      case Selector(("_findSelected:")): return (currentMenu == .defaultMenu)
+      // case Selector(("_findSelected:")): return (currentMenu == .defaultMenu)
       default: return false
       }
     }

From 2be835f23eb9585e4042334f31fd77ac8e154ed3 Mon Sep 17 00:00:00 2001
From: Jackson Harper <jacksonh@gmail.com>
Date: Wed, 13 Mar 2024 22:04:17 +0800
Subject: [PATCH 26/67] Enable fullscreen mode

This is needed on iOS to allow the YouTube embed to go fullscreen.
---
 apple/OmnivoreKit/Sources/App/Views/WebReader/WebReader.swift | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/apple/OmnivoreKit/Sources/App/Views/WebReader/WebReader.swift b/apple/OmnivoreKit/Sources/App/Views/WebReader/WebReader.swift
index ec64eac22..8c5468f16 100644
--- a/apple/OmnivoreKit/Sources/App/Views/WebReader/WebReader.swift
+++ b/apple/OmnivoreKit/Sources/App/Views/WebReader/WebReader.swift
@@ -66,6 +66,10 @@ struct WebReader: PlatformViewRepresentable {
       webView.scrollView.verticalScrollIndicatorInsets.top = readerViewNavBarHeight
       webView.configuration.userContentController.add(webView, name: "viewerAction")
 
+      if #available(iOS 15.4, *) {
+        webView.configuration.preferences.isElementFullscreenEnabled = true
+      }
+
       webView.scrollView.indicatorStyle = ThemeManager.currentTheme.isDark ?
         UIScrollView.IndicatorStyle.white :
         UIScrollView.IndicatorStyle.black

From d83ee5c647b4f997863acef2f6a402990c2bd41c Mon Sep 17 00:00:00 2001
From: Jackson Harper <jacksonh@gmail.com>
Date: Thu, 14 Mar 2024 10:40:30 +0800
Subject: [PATCH 27/67] Add fetch content to iOS, fix Transmission version

---
 .../App/Views/Profile/SubscriptionsView.swift |  87 +++---
 .../Models/DataModels/Subscription.swift      |  12 +-
 .../Services/DataService/GQLSchema.swift      | 267 ++++++++++++++++--
 .../Mutations/SetRuleMutation.swift           |   3 +
 .../Mutations/UpdateSubscription.swift        |   4 +-
 .../Selections/SubsciptionSelection.swift     |  25 +-
 6 files changed, 333 insertions(+), 65 deletions(-)

diff --git a/apple/OmnivoreKit/Sources/App/Views/Profile/SubscriptionsView.swift b/apple/OmnivoreKit/Sources/App/Views/Profile/SubscriptionsView.swift
index 197cbee59..d60e00b74 100644
--- a/apple/OmnivoreKit/Sources/App/Views/Profile/SubscriptionsView.swift
+++ b/apple/OmnivoreKit/Sources/App/Views/Profile/SubscriptionsView.swift
@@ -98,11 +98,11 @@ typealias OperationStatusHandler = (_: OperationStatus) -> Void
     }
   }
 
-  func updateSubscription(dataService: DataService, subscription: Subscription, folder: String? = nil, fetchContent: Bool? = nil) async {
+  func updateSubscription(dataService: DataService, subscription: Subscription, folder: String? = nil, fetchContentType: FetchContentType? = nil) async {
     operationMessage = "Updating subscription..."
     operationStatus = .isPerforming
     do {
-      try await dataService.updateSubscription(subscription.subscriptionID, folder: folder, fetchContent: fetchContent)
+      try await dataService.updateSubscription(subscription.subscriptionID, folder: folder, fetchContentType: fetchContentType)
       operationMessage = "Subscription updated"
       operationStatus = .success
     } catch {
@@ -240,23 +240,27 @@ struct SubscriptionsView: View {
     #endif
   }
 
+  private var emptyView: some View {
+    VStack(alignment: .center, spacing: 20) {
+      Text("You don't have any Feed items.")
+        .font(Font.system(size: 18, weight: .bold))
+
+      Text("Add an RSS/Atom feed")
+        .foregroundColor(Color.blue)
+        .onTapGesture {
+          showAddFeedView = true
+        }
+    }
+    .frame(minHeight: 80)
+    .frame(maxWidth: .infinity)
+    .padding()
+  }
+
   private var innerBody: some View {
-    Group {
+    List {
       Section("Feeds") {
         if viewModel.feeds.count <= 0, !viewModel.isLoading {
-          VStack(alignment: .center, spacing: 20) {
-            Text("You don't have any Feed items.")
-              .font(Font.system(size: 18, weight: .bold))
-
-            Text("Add an RSS/Atom feed")
-              .foregroundColor(Color.blue)
-              .onTapGesture {
-                showAddFeedView = true
-              }
-          }
-          .frame(minHeight: 80)
-          .frame(maxWidth: .infinity)
-          .padding()
+          emptyView
         } else {
           ForEach(viewModel.feeds, id: \.subscriptionID) { subscription in
             PresentationLink(transition: UIDevice.isIPad ? .popover : .sheet(detents: [.medium])) {
@@ -264,7 +268,7 @@ struct SubscriptionsView: View {
                 subscription: subscription,
                 viewModel: viewModel,
                 dataService: dataService,
-                prefetchContent: subscription.fetchContent,
+                fetchContentType: subscription.fetchContentType,
                 folderSelection: subscription.folder,
                 unsubscribe: { _ in
                   viewModel.operationStatus = .isPerforming
@@ -296,7 +300,7 @@ struct SubscriptionsView: View {
                 subscription: subscription,
                 viewModel: viewModel,
                 dataService: dataService,
-                prefetchContent: subscription.fetchContent,
+                fetchContentType: subscription.fetchContentType,
                 folderSelection: subscription.folder,
                 unsubscribe: { _ in
                   viewModel.operationStatus = .isPerforming
@@ -389,7 +393,7 @@ struct SubscriptionSettingsView: View {
   let viewModel: SubscriptionsViewModel
   let dataService: DataService
 
-  @State var prefetchContent = false
+  @State var fetchContentType: FetchContentType
   @State var deleteConfirmationShown = false
   @State var showDeleteCompleted = false
   @State var folderSelection: String = ""
@@ -428,6 +432,28 @@ struct SubscriptionSettingsView: View {
     return nil
   }
 
+  var fetchContentRow: some View {
+    Picker(selection: $fetchContentType, content: {
+      Text("Always").tag(FetchContentType.always)
+      Text("Never").tag(FetchContentType.never)
+      Text("When empty").tag(FetchContentType.whenEmpty)
+    }, label: { Text("Fetch link") })
+    .pickerStyle(MenuPickerStyle())
+    .onChange(of: fetchContentType) { newValue in
+      Task {
+        viewModel.showOperationToast = true
+        await viewModel.updateSubscription(
+          dataService: dataService,
+          subscription: subscription,
+          fetchContentType: newValue
+        )
+        DispatchQueue.main.asyncAfter(deadline: .now() + .milliseconds(1500)) {
+          viewModel.showOperationToast = false
+        }
+      }
+    }
+  }
+
   var folderRow: some View {
     HStack {
       Picker("Destination Folder", selection: $folderSelection) {
@@ -444,19 +470,6 @@ struct SubscriptionSettingsView: View {
           }
         }
       }
-      .onChange(of: prefetchContent) { newValue in
-        Task {
-          viewModel.showOperationToast = true
-          await viewModel.updateSubscription(
-            dataService: dataService,
-            subscription: subscription,
-            fetchContent: newValue
-          )
-          DispatchQueue.main.asyncAfter(deadline: .now() + .milliseconds(1500)) {
-            viewModel.showOperationToast = false
-          }
-        }
-      }
     }
   }
 
@@ -570,12 +583,10 @@ struct SubscriptionSettingsView: View {
         .padding(.horizontal, 15)
 
       List {
-//        if subscription.type != .newsletter {
-//          Toggle(isOn: $prefetchContent, label: { Text("Prefetch Content:") })
-//            .onAppear {
-//              prefetchContent = subscription.fetchContent
-//            }
-//        }
+        if subscription.type != .newsletter {
+          fetchContentRow
+        }
+
         folderRow
         labelRuleRow
 
diff --git a/apple/OmnivoreKit/Sources/Models/DataModels/Subscription.swift b/apple/OmnivoreKit/Sources/Models/DataModels/Subscription.swift
index 2a8ff19df..2b7493ca1 100644
--- a/apple/OmnivoreKit/Sources/Models/DataModels/Subscription.swift
+++ b/apple/OmnivoreKit/Sources/Models/DataModels/Subscription.swift
@@ -8,7 +8,7 @@ public struct Subscription {
   public let name: String
   public let type: SubscriptionType
   public let folder: String
-  public let fetchContent: Bool
+  public let fetchContentType: FetchContentType
   public let newsletterEmailAddress: String?
   public let status: SubscriptionStatus
   public let unsubscribeHttpUrl: String?
@@ -24,7 +24,7 @@ public struct Subscription {
     name: String,
     type: SubscriptionType,
     folder: String,
-    fetchContent: Bool,
+    fetchContentType: FetchContentType,
     newsletterEmailAddress: String?,
     status: SubscriptionStatus,
     unsubscribeHttpUrl: String?,
@@ -39,7 +39,7 @@ public struct Subscription {
     self.name = name
     self.type = type
     self.folder = folder
-    self.fetchContent = fetchContent
+    self.fetchContentType = fetchContentType
     self.newsletterEmailAddress = newsletterEmailAddress
     self.status = status
     self.unsubscribeHttpUrl = unsubscribeHttpUrl
@@ -60,3 +60,9 @@ public enum SubscriptionType {
   case newsletter
   case feed
 }
+
+public enum FetchContentType {
+  case always
+  case never
+  case whenEmpty
+}
diff --git a/apple/OmnivoreKit/Sources/Services/DataService/GQLSchema.swift b/apple/OmnivoreKit/Sources/Services/DataService/GQLSchema.swift
index e051efd4c..49979b626 100644
--- a/apple/OmnivoreKit/Sources/Services/DataService/GQLSchema.swift
+++ b/apple/OmnivoreKit/Sources/Services/DataService/GQLSchema.swift
@@ -667,6 +667,8 @@ extension Objects {
         let contentReader: [String: Enums.ContentReader]
         let createdAt: [String: DateTime]
         let description: [String: String]
+        let directionality: [String: Enums.DirectionalityType]
+        let feedContent: [String: String]
         let folder: [String: String]
         let hasContent: [String: Bool]
         let hash: [String: String]
@@ -742,6 +744,14 @@ extension Objects.Article: Decodable {
                 if let value = try container.decode(String?.self, forKey: codingKey) {
                     map.set(key: field, hash: alias, value: value as Any)
                 }
+            case "directionality":
+                if let value = try container.decode(Enums.DirectionalityType?.self, forKey: codingKey) {
+                    map.set(key: field, hash: alias, value: value as Any)
+                }
+            case "feedContent":
+                if let value = try container.decode(String?.self, forKey: codingKey) {
+                    map.set(key: field, hash: alias, value: value as Any)
+                }
             case "folder":
                 if let value = try container.decode(String?.self, forKey: codingKey) {
                     map.set(key: field, hash: alias, value: value as Any)
@@ -901,6 +911,8 @@ extension Objects.Article: Decodable {
         contentReader = map["contentReader"]
         createdAt = map["createdAt"]
         description = map["description"]
+        directionality = map["directionality"]
+        feedContent = map["feedContent"]
         folder = map["folder"]
         hasContent = map["hasContent"]
         hash = map["hash"]
@@ -1025,6 +1037,36 @@ extension Fields where TypeLock == Objects.Article {
         }
     }
 
+    func directionality() throws -> Enums.DirectionalityType? {
+        let field = GraphQLField.leaf(
+            name: "directionality",
+            arguments: []
+        )
+        select(field)
+
+        switch response {
+        case let .decoding(data):
+            return data.directionality[field.alias!]
+        case .mocking:
+            return nil
+        }
+    }
+
+    func feedContent() throws -> String? {
+        let field = GraphQLField.leaf(
+            name: "feedContent",
+            arguments: []
+        )
+        select(field)
+
+        switch response {
+        case let .decoding(data):
+            return data.feedContent[field.alias!]
+        case .mocking:
+            return nil
+        }
+    }
+
     func folder() throws -> String {
         let field = GraphQLField.leaf(
             name: "folder",
@@ -8127,6 +8169,7 @@ extension Objects {
         let quote: [String: String]
         let reactions: [String: [Objects.Reaction]]
         let replies: [String: [Objects.HighlightReply]]
+        let representation: [String: Enums.RepresentationType]
         let sharedAt: [String: DateTime]
         let shortId: [String: String]
         let suffix: [String: String]
@@ -8208,6 +8251,10 @@ extension Objects.Highlight: Decodable {
                 if let value = try container.decode([Objects.HighlightReply]?.self, forKey: codingKey) {
                     map.set(key: field, hash: alias, value: value as Any)
                 }
+            case "representation":
+                if let value = try container.decode(Enums.RepresentationType?.self, forKey: codingKey) {
+                    map.set(key: field, hash: alias, value: value as Any)
+                }
             case "sharedAt":
                 if let value = try container.decode(DateTime?.self, forKey: codingKey) {
                     map.set(key: field, hash: alias, value: value as Any)
@@ -8256,6 +8303,7 @@ extension Objects.Highlight: Decodable {
         quote = map["quote"]
         reactions = map["reactions"]
         replies = map["replies"]
+        representation = map["representation"]
         sharedAt = map["sharedAt"]
         shortId = map["shortId"]
         suffix = map["suffix"]
@@ -8494,6 +8542,24 @@ extension Fields where TypeLock == Objects.Highlight {
         }
     }
 
+    func representation() throws -> Enums.RepresentationType {
+        let field = GraphQLField.leaf(
+            name: "representation",
+            arguments: []
+        )
+        select(field)
+
+        switch response {
+        case let .decoding(data):
+            if let data = data.representation[field.alias!] {
+                return data
+            }
+            throw HttpError.badpayload
+        case .mocking:
+            return Enums.RepresentationType.allCases.first!
+        }
+    }
+
     func sharedAt() throws -> DateTime? {
         let field = GraphQLField.leaf(
             name: "sharedAt",
@@ -8985,6 +9051,7 @@ extension Objects {
         let enabled: [String: Bool]
         let id: [String: String]
         let name: [String: String]
+        let settings: [String: String]
         let taskName: [String: String]
         let token: [String: String]
         let type: [String: Enums.IntegrationType]
@@ -9024,6 +9091,10 @@ extension Objects.Integration: Decodable {
                 if let value = try container.decode(String?.self, forKey: codingKey) {
                     map.set(key: field, hash: alias, value: value as Any)
                 }
+            case "settings":
+                if let value = try container.decode(String?.self, forKey: codingKey) {
+                    map.set(key: field, hash: alias, value: value as Any)
+                }
             case "taskName":
                 if let value = try container.decode(String?.self, forKey: codingKey) {
                     map.set(key: field, hash: alias, value: value as Any)
@@ -9054,6 +9125,7 @@ extension Objects.Integration: Decodable {
         enabled = map["enabled"]
         id = map["id"]
         name = map["name"]
+        settings = map["settings"]
         taskName = map["taskName"]
         token = map["token"]
         type = map["type"]
@@ -9134,6 +9206,21 @@ extension Fields where TypeLock == Objects.Integration {
         }
     }
 
+    func settings() throws -> String? {
+        let field = GraphQLField.leaf(
+            name: "settings",
+            arguments: []
+        )
+        select(field)
+
+        switch response {
+        case let .decoding(data):
+            return data.settings[field.alias!]
+        case .mocking:
+            return nil
+        }
+    }
+
     func taskName() throws -> String? {
         let field = GraphQLField.leaf(
             name: "taskName",
@@ -18126,6 +18213,7 @@ extension Selection where TypeLock == Never, Type == Never {
 extension Objects {
     struct SearchItem {
         let __typename: TypeName = .searchItem
+        let aiSummary: [String: String]
         let annotation: [String: String]
         let archivedAt: [String: DateTime]
         let author: [String: String]
@@ -18134,6 +18222,8 @@ extension Objects {
         let contentReader: [String: Enums.ContentReader]
         let createdAt: [String: DateTime]
         let description: [String: String]
+        let directionality: [String: Enums.DirectionalityType]
+        let feedContent: [String: String]
         let folder: [String: String]
         let highlights: [String: [Objects.Highlight]]
         let id: [String: String]
@@ -18146,7 +18236,6 @@ extension Objects {
         let ownedByViewer: [String: Bool]
         let pageId: [String: String]
         let pageType: [String: Enums.PageType]
-        let previewContent: [String: String]
         let previewContentType: [String: String]
         let publishedAt: [String: DateTime]
         let quote: [String: String]
@@ -18188,6 +18277,10 @@ extension Objects.SearchItem: Decodable {
             let field = GraphQLField.getFieldNameFromAlias(alias)
 
             switch field {
+            case "aiSummary":
+                if let value = try container.decode(String?.self, forKey: codingKey) {
+                    map.set(key: field, hash: alias, value: value as Any)
+                }
             case "annotation":
                 if let value = try container.decode(String?.self, forKey: codingKey) {
                     map.set(key: field, hash: alias, value: value as Any)
@@ -18220,6 +18313,14 @@ extension Objects.SearchItem: Decodable {
                 if let value = try container.decode(String?.self, forKey: codingKey) {
                     map.set(key: field, hash: alias, value: value as Any)
                 }
+            case "directionality":
+                if let value = try container.decode(Enums.DirectionalityType?.self, forKey: codingKey) {
+                    map.set(key: field, hash: alias, value: value as Any)
+                }
+            case "feedContent":
+                if let value = try container.decode(String?.self, forKey: codingKey) {
+                    map.set(key: field, hash: alias, value: value as Any)
+                }
             case "folder":
                 if let value = try container.decode(String?.self, forKey: codingKey) {
                     map.set(key: field, hash: alias, value: value as Any)
@@ -18268,10 +18369,6 @@ extension Objects.SearchItem: Decodable {
                 if let value = try container.decode(Enums.PageType?.self, forKey: codingKey) {
                     map.set(key: field, hash: alias, value: value as Any)
                 }
-            case "previewContent":
-                if let value = try container.decode(String?.self, forKey: codingKey) {
-                    map.set(key: field, hash: alias, value: value as Any)
-                }
             case "previewContentType":
                 if let value = try container.decode(String?.self, forKey: codingKey) {
                     map.set(key: field, hash: alias, value: value as Any)
@@ -18370,6 +18467,7 @@ extension Objects.SearchItem: Decodable {
             }
         }
 
+        aiSummary = map["aiSummary"]
         annotation = map["annotation"]
         archivedAt = map["archivedAt"]
         author = map["author"]
@@ -18378,6 +18476,8 @@ extension Objects.SearchItem: Decodable {
         contentReader = map["contentReader"]
         createdAt = map["createdAt"]
         description = map["description"]
+        directionality = map["directionality"]
+        feedContent = map["feedContent"]
         folder = map["folder"]
         highlights = map["highlights"]
         id = map["id"]
@@ -18390,7 +18490,6 @@ extension Objects.SearchItem: Decodable {
         ownedByViewer = map["ownedByViewer"]
         pageId = map["pageId"]
         pageType = map["pageType"]
-        previewContent = map["previewContent"]
         previewContentType = map["previewContentType"]
         publishedAt = map["publishedAt"]
         quote = map["quote"]
@@ -18417,6 +18516,21 @@ extension Objects.SearchItem: Decodable {
 }
 
 extension Fields where TypeLock == Objects.SearchItem {
+    func aiSummary() throws -> String? {
+        let field = GraphQLField.leaf(
+            name: "aiSummary",
+            arguments: []
+        )
+        select(field)
+
+        switch response {
+        case let .decoding(data):
+            return data.aiSummary[field.alias!]
+        case .mocking:
+            return nil
+        }
+    }
+
     func annotation() throws -> String? {
         let field = GraphQLField.leaf(
             name: "annotation",
@@ -18543,6 +18657,36 @@ extension Fields where TypeLock == Objects.SearchItem {
         }
     }
 
+    func directionality() throws -> Enums.DirectionalityType? {
+        let field = GraphQLField.leaf(
+            name: "directionality",
+            arguments: []
+        )
+        select(field)
+
+        switch response {
+        case let .decoding(data):
+            return data.directionality[field.alias!]
+        case .mocking:
+            return nil
+        }
+    }
+
+    func feedContent() throws -> String? {
+        let field = GraphQLField.leaf(
+            name: "feedContent",
+            arguments: []
+        )
+        select(field)
+
+        switch response {
+        case let .decoding(data):
+            return data.feedContent[field.alias!]
+        case .mocking:
+            return nil
+        }
+    }
+
     func folder() throws -> String {
         let field = GraphQLField.leaf(
             name: "folder",
@@ -18737,21 +18881,6 @@ extension Fields where TypeLock == Objects.SearchItem {
         }
     }
 
-    func previewContent() throws -> String? {
-        let field = GraphQLField.leaf(
-            name: "previewContent",
-            arguments: []
-        )
-        select(field)
-
-        switch response {
-        case let .decoding(data):
-            return data.previewContent[field.alias!]
-        case .mocking:
-            return nil
-        }
-    }
-
     func previewContentType() throws -> String? {
         let field = GraphQLField.leaf(
             name: "previewContentType",
@@ -21284,6 +21413,7 @@ extension Objects {
         let description: [String: String]
         let failedAt: [String: DateTime]
         let fetchContent: [String: Bool]
+        let fetchContentType: [String: Enums.FetchContentType]
         let folder: [String: String]
         let icon: [String: String]
         let id: [String: String]
@@ -21342,6 +21472,10 @@ extension Objects.Subscription: Decodable {
                 if let value = try container.decode(Bool?.self, forKey: codingKey) {
                     map.set(key: field, hash: alias, value: value as Any)
                 }
+            case "fetchContentType":
+                if let value = try container.decode(Enums.FetchContentType?.self, forKey: codingKey) {
+                    map.set(key: field, hash: alias, value: value as Any)
+                }
             case "folder":
                 if let value = try container.decode(String?.self, forKey: codingKey) {
                     map.set(key: field, hash: alias, value: value as Any)
@@ -21418,6 +21552,7 @@ extension Objects.Subscription: Decodable {
         description = map["description"]
         failedAt = map["failedAt"]
         fetchContent = map["fetchContent"]
+        fetchContentType = map["fetchContentType"]
         folder = map["folder"]
         icon = map["icon"]
         id = map["id"]
@@ -21536,6 +21671,24 @@ extension Fields where TypeLock == Objects.Subscription {
         }
     }
 
+    func fetchContentType() throws -> Enums.FetchContentType {
+        let field = GraphQLField.leaf(
+            name: "fetchContentType",
+            arguments: []
+        )
+        select(field)
+
+        switch response {
+        case let .decoding(data):
+            if let data = data.fetchContentType[field.alias!] {
+                return data
+            }
+            throw HttpError.badpayload
+        case .mocking:
+            return Enums.FetchContentType.allCases.first!
+        }
+    }
+
     func folder() throws -> String {
         let field = GraphQLField.leaf(
             name: "folder",
@@ -24692,6 +24845,7 @@ extension Objects {
     struct User {
         let __typename: TypeName = .user
         let email: [String: String]
+        let features: [String: [String?]]
         let followersCount: [String: Int]
         let friendsCount: [String: Int]
         let id: [String: String]
@@ -24730,6 +24884,10 @@ extension Objects.User: Decodable {
                 if let value = try container.decode(String?.self, forKey: codingKey) {
                     map.set(key: field, hash: alias, value: value as Any)
                 }
+            case "features":
+                if let value = try container.decode([String?]?.self, forKey: codingKey) {
+                    map.set(key: field, hash: alias, value: value as Any)
+                }
             case "followersCount":
                 if let value = try container.decode(Int?.self, forKey: codingKey) {
                     map.set(key: field, hash: alias, value: value as Any)
@@ -24801,6 +24959,7 @@ extension Objects.User: Decodable {
         }
 
         email = map["email"]
+        features = map["features"]
         followersCount = map["followersCount"]
         friendsCount = map["friendsCount"]
         id = map["id"]
@@ -24835,6 +24994,21 @@ extension Fields where TypeLock == Objects.User {
         }
     }
 
+    func features() throws -> [String?]? {
+        let field = GraphQLField.leaf(
+            name: "features",
+            arguments: []
+        )
+        select(field)
+
+        switch response {
+        case let .decoding(data):
+            return data.features[field.alias!]
+        case .mocking:
+            return nil
+        }
+    }
+
     func followersCount() throws -> Int? {
         let field = GraphQLField.leaf(
             name: "followersCount",
@@ -34148,6 +34322,15 @@ extension Enums {
     }
 }
 
+extension Enums {
+    /// DirectionalityType
+    enum DirectionalityType: String, CaseIterable, Codable {
+        case ltr = "LTR"
+
+        case rtl = "RTL"
+    }
+}
+
 extension Enums {
     /// EmptyTrashErrorCode
     enum EmptyTrashErrorCode: String, CaseIterable, Codable {
@@ -34180,6 +34363,17 @@ extension Enums {
     }
 }
 
+extension Enums {
+    /// FetchContentType
+    enum FetchContentType: String, CaseIterable, Codable {
+        case always = "ALWAYS"
+
+        case never = "NEVER"
+
+        case whenEmpty = "WHEN_EMPTY"
+    }
+}
+
 extension Enums {
     /// FiltersErrorCode
     enum FiltersErrorCode: String, CaseIterable, Codable {
@@ -34521,6 +34715,15 @@ extension Enums {
     }
 }
 
+extension Enums {
+    /// RepresentationType
+    enum RepresentationType: String, CaseIterable, Codable {
+        case content = "CONTENT"
+
+        case feedContent = "FEED_CONTENT"
+    }
+}
+
 extension Enums {
     /// RevokeApiKeyErrorCode
     enum RevokeApiKeyErrorCode: String, CaseIterable, Codable {
@@ -34539,6 +34742,8 @@ extension Enums {
 
         case archive = "ARCHIVE"
 
+        case delete = "DELETE"
+
         case markAsRead = "MARK_AS_READ"
 
         case sendNotification = "SEND_NOTIFICATION"
@@ -35306,6 +35511,8 @@ extension InputObjects {
 
         var quote: OptionalArgument<String> = .absent()
 
+        var representation: OptionalArgument<Enums.RepresentationType> = .absent()
+
         var sharedAt: OptionalArgument<DateTime> = .absent()
 
         var shortId: String
@@ -35326,6 +35533,7 @@ extension InputObjects {
             if patch.hasValue { try container.encode(patch, forKey: .patch) }
             if prefix.hasValue { try container.encode(prefix, forKey: .prefix) }
             if quote.hasValue { try container.encode(quote, forKey: .quote) }
+            if representation.hasValue { try container.encode(representation, forKey: .representation) }
             if sharedAt.hasValue { try container.encode(sharedAt, forKey: .sharedAt) }
             try container.encode(shortId, forKey: .shortId)
             if suffix.hasValue { try container.encode(suffix, forKey: .suffix) }
@@ -35343,6 +35551,7 @@ extension InputObjects {
             case patch
             case prefix
             case quote
+            case representation
             case sharedAt
             case shortId
             case suffix
@@ -35602,6 +35811,8 @@ extension InputObjects {
 
         var quote: String
 
+        var representation: OptionalArgument<Enums.RepresentationType> = .absent()
+
         var shortId: String
 
         var suffix: OptionalArgument<String> = .absent()
@@ -35619,6 +35830,7 @@ extension InputObjects {
             try container.encode(patch, forKey: .patch)
             if prefix.hasValue { try container.encode(prefix, forKey: .prefix) }
             try container.encode(quote, forKey: .quote)
+            if representation.hasValue { try container.encode(representation, forKey: .representation) }
             try container.encode(shortId, forKey: .shortId)
             if suffix.hasValue { try container.encode(suffix, forKey: .suffix) }
         }
@@ -35635,6 +35847,7 @@ extension InputObjects {
             case patch
             case prefix
             case quote
+            case representation
             case shortId
             case suffix
         }
@@ -36228,6 +36441,8 @@ extension InputObjects {
 
         var name: String
 
+        var settings: OptionalArgument<String> = .absent()
+
         var syncedAt: OptionalArgument<DateTime> = .absent()
 
         var taskName: OptionalArgument<String> = .absent()
@@ -36242,6 +36457,7 @@ extension InputObjects {
             if id.hasValue { try container.encode(id, forKey: .id) }
             if importItemState.hasValue { try container.encode(importItemState, forKey: .importItemState) }
             try container.encode(name, forKey: .name)
+            if settings.hasValue { try container.encode(settings, forKey: .settings) }
             if syncedAt.hasValue { try container.encode(syncedAt, forKey: .syncedAt) }
             if taskName.hasValue { try container.encode(taskName, forKey: .taskName) }
             try container.encode(token, forKey: .token)
@@ -36253,6 +36469,7 @@ extension InputObjects {
             case id
             case importItemState
             case name
+            case settings
             case syncedAt
             case taskName
             case token
@@ -36511,6 +36728,8 @@ extension InputObjects {
 
         var fetchContent: OptionalArgument<Bool> = .absent()
 
+        var fetchContentType: OptionalArgument<Enums.FetchContentType> = .absent()
+
         var folder: OptionalArgument<String> = .absent()
 
         var isPrivate: OptionalArgument<Bool> = .absent()
@@ -36523,6 +36742,7 @@ extension InputObjects {
             var container = encoder.container(keyedBy: CodingKeys.self)
             if autoAddToLibrary.hasValue { try container.encode(autoAddToLibrary, forKey: .autoAddToLibrary) }
             if fetchContent.hasValue { try container.encode(fetchContent, forKey: .fetchContent) }
+            if fetchContentType.hasValue { try container.encode(fetchContentType, forKey: .fetchContentType) }
             if folder.hasValue { try container.encode(folder, forKey: .folder) }
             if isPrivate.hasValue { try container.encode(isPrivate, forKey: .isPrivate) }
             if subscriptionType.hasValue { try container.encode(subscriptionType, forKey: .subscriptionType) }
@@ -36532,6 +36752,7 @@ extension InputObjects {
         enum CodingKeys: String, CodingKey {
             case autoAddToLibrary
             case fetchContent
+            case fetchContentType
             case folder
             case isPrivate
             case subscriptionType
@@ -36828,6 +37049,8 @@ extension InputObjects {
 
         var fetchContent: OptionalArgument<Bool> = .absent()
 
+        var fetchContentType: OptionalArgument<Enums.FetchContentType> = .absent()
+
         var folder: OptionalArgument<String> = .absent()
 
         var id: String
@@ -36852,6 +37075,7 @@ extension InputObjects {
             if description.hasValue { try container.encode(description, forKey: .description) }
             if failedAt.hasValue { try container.encode(failedAt, forKey: .failedAt) }
             if fetchContent.hasValue { try container.encode(fetchContent, forKey: .fetchContent) }
+            if fetchContentType.hasValue { try container.encode(fetchContentType, forKey: .fetchContentType) }
             if folder.hasValue { try container.encode(folder, forKey: .folder) }
             try container.encode(id, forKey: .id)
             if isPrivate.hasValue { try container.encode(isPrivate, forKey: .isPrivate) }
@@ -36868,6 +37092,7 @@ extension InputObjects {
             case description
             case failedAt
             case fetchContent
+            case fetchContentType
             case folder
             case id
             case isPrivate
diff --git a/apple/OmnivoreKit/Sources/Services/DataService/Mutations/SetRuleMutation.swift b/apple/OmnivoreKit/Sources/Services/DataService/Mutations/SetRuleMutation.swift
index a59fd43e8..9dea39b9a 100644
--- a/apple/OmnivoreKit/Sources/Services/DataService/Mutations/SetRuleMutation.swift
+++ b/apple/OmnivoreKit/Sources/Services/DataService/Mutations/SetRuleMutation.swift
@@ -12,6 +12,7 @@ public struct Rule {
 public enum RuleActionType {
   case addLabel
   case archive
+  case delete
   case markAsRead
   case sendNotification
 
@@ -25,6 +26,8 @@ public enum RuleActionType {
       return .markAsRead
     case Enums.RuleActionType.sendNotification:
       return .sendNotification
+    case .delete:
+      return .delete
     }
   }
 }
diff --git a/apple/OmnivoreKit/Sources/Services/DataService/Mutations/UpdateSubscription.swift b/apple/OmnivoreKit/Sources/Services/DataService/Mutations/UpdateSubscription.swift
index 1c16479b1..9cf26cb80 100644
--- a/apple/OmnivoreKit/Sources/Services/DataService/Mutations/UpdateSubscription.swift
+++ b/apple/OmnivoreKit/Sources/Services/DataService/Mutations/UpdateSubscription.swift
@@ -4,7 +4,7 @@ import Models
 import SwiftGraphQL
 
 public extension DataService {
-  func updateSubscription(_ subscriptionID: String, folder: String? = nil, fetchContent: Bool? = nil) async throws {
+  func updateSubscription(_ subscriptionID: String, folder: String? = nil, fetchContentType: FetchContentType? = nil) async throws {
     enum MutationResult {
       case success(subscriptionID: String)
       case error(errorMessage: String)
@@ -20,7 +20,7 @@ public extension DataService {
     let mutation = Selection.Mutation {
       try $0.updateSubscription(
         input: InputObjects.UpdateSubscriptionInput(
-          fetchContent: OptionalArgument(fetchContent),
+          fetchContentType: OptionalArgument(fetchContentType?.toGQLType()),
           folder: OptionalArgument(folder),
           id: subscriptionID
         ),
diff --git a/apple/OmnivoreKit/Sources/Services/DataService/Selections/SubsciptionSelection.swift b/apple/OmnivoreKit/Sources/Services/DataService/Selections/SubsciptionSelection.swift
index f8e883e97..5ff68747a 100644
--- a/apple/OmnivoreKit/Sources/Services/DataService/Selections/SubsciptionSelection.swift
+++ b/apple/OmnivoreKit/Sources/Services/DataService/Selections/SubsciptionSelection.swift
@@ -11,7 +11,7 @@ let subscriptionSelection = Selection.Subscription {
     name: try $0.name(),
     type: try SubscriptionType.from($0.type()),
     folder: try $0.folder(),
-    fetchContent: try $0.fetchContent(),
+    fetchContentType: try FetchContentType.from($0.fetchContentType()),
     newsletterEmailAddress: try $0.newsletterEmail(),
     status: try SubscriptionStatus.make(from: $0.status()),
     unsubscribeHttpUrl: try $0.unsubscribeHttpUrl(),
@@ -45,3 +45,26 @@ extension SubscriptionType {
     }
   }
 }
+
+extension FetchContentType {
+  static func from(_ other: Enums.FetchContentType) -> FetchContentType {
+    switch other {
+    case .always:
+      return .always
+    case .never:
+      return .never
+    case .whenEmpty:
+      return .whenEmpty
+    }
+  }
+  func toGQLType() -> Enums.FetchContentType {
+    switch self {
+    case .always:
+      return .always
+    case .never:
+      return .never
+    case .whenEmpty:
+      return .whenEmpty
+    }
+  }
+}

From 81e489021e86c04f38cf50bdbc1497ff4f9eced4 Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Thu, 14 Mar 2024 11:14:37 +0800
Subject: [PATCH 28/67] add popular reads to the continue reading section of
 new user library

---
 packages/api/src/repository/library_item.ts | 10 ++++++++--
 packages/api/test/routers/auth.test.ts      |  4 ++--
 2 files changed, 10 insertions(+), 4 deletions(-)

diff --git a/packages/api/src/repository/library_item.ts b/packages/api/src/repository/library_item.ts
index f4003b7bc..fd933faf2 100644
--- a/packages/api/src/repository/library_item.ts
+++ b/packages/api/src/repository/library_item.ts
@@ -58,6 +58,8 @@ export const libraryItemRepository = appDataSource
     },
 
     createByPopularRead(name: string, userId: string) {
+      // set read_at to now and reading_progress_bottom_percent to 2
+      // so the items show up in continue reading section
       return this.query(
         `
         INSERT INTO omnivore.library_item (
@@ -73,7 +75,9 @@ export const libraryItemRepository = appDataSource
           published_at,
           site_name,
           user_id,
-          word_count
+          word_count,
+          read_at,
+          reading_progress_bottom_percent
         ) 
         SELECT
           slug,
@@ -88,7 +92,9 @@ export const libraryItemRepository = appDataSource
           published_at,
           site_name,
           $2,
-          word_count
+          word_count,
+          NOW(),
+          2
         FROM
           omnivore.popular_read
         WHERE
diff --git a/packages/api/test/routers/auth.test.ts b/packages/api/test/routers/auth.test.ts
index 2bb3d5831..f62900e18 100644
--- a/packages/api/test/routers/auth.test.ts
+++ b/packages/api/test/routers/auth.test.ts
@@ -591,7 +591,7 @@ describe('auth router', () => {
         await deleteUser(user.id)
       })
 
-      it('adds popular reads to the library', async () => {
+      it('adds popular reads to the continue reading section', async () => {
         const pendingUserToken = await createPendingUserToken({
           sourceUserId,
           email,
@@ -608,7 +608,7 @@ describe('auth router', () => {
         ).expect(200)
         const user = await userRepository.findOneByOrFail({ name })
         const { count } = await searchLibraryItems(
-          { query: 'in:all' },
+          { query: 'in:inbox sort:read-desc is:reading' },
           user.id
         )
 

From eecfd1c402d18f7eb4dec344211041207f112f5b Mon Sep 17 00:00:00 2001
From: Jackson Harper <jacksonh@gmail.com>
Date: Thu, 14 Mar 2024 12:47:15 +0800
Subject: [PATCH 29/67] Add a button to iOS to leave a rating/review

---
 .../App/Views/Profile/ProfileView.swift       | 23 ++++++++++++++-----
 1 file changed, 17 insertions(+), 6 deletions(-)

diff --git a/apple/OmnivoreKit/Sources/App/Views/Profile/ProfileView.swift b/apple/OmnivoreKit/Sources/App/Views/Profile/ProfileView.swift
index b394dc0f7..1dda93273 100644
--- a/apple/OmnivoreKit/Sources/App/Views/Profile/ProfileView.swift
+++ b/apple/OmnivoreKit/Sources/App/Views/Profile/ProfileView.swift
@@ -155,12 +155,21 @@ struct ProfileView: View {
           label: { Text(LocalText.documentationGeneric) }
         )
 
-        #if os(iOS)
-          Button(
-            action: { DataService.showIntercomMessenger?() },
-            label: { Text(LocalText.feedbackGeneric) }
-          )
-        #endif
+#if os(iOS)
+        Button(
+          action: { DataService.showIntercomMessenger?() },
+          label: { Text(LocalText.feedbackGeneric) }
+        )
+#endif
+
+        Button(
+          action: {
+            if let url = URL(string: "https://apps.apple.com/app/id1564031042?action=write-review") {
+              openURL(url)
+            }
+          },
+          label: { Text("Review Omnivore") }
+        )
 
         Button(
           action: {
@@ -170,7 +179,9 @@ struct ProfileView: View {
           },
           label: { Text("Join community on Discord") }
         )
+      }
 
+      Section {
         Button(
           action: {
             if let url = URL(string: "https://omnivore.app/privacy") {

From f7225b298a46e90fcac8eeb69f5fdadfd72d9b56 Mon Sep 17 00:00:00 2001
From: Jackson Harper <jacksonh@gmail.com>
Date: Tue, 12 Mar 2024 21:36:50 +0800
Subject: [PATCH 30/67] Rebase

---
 packages/api/package.json                     |   6 +-
 packages/api/src/jobs/get-youtube-info.ts     | 153 ++++++++++++++++++
 packages/api/src/pubsub.ts                    |   8 +-
 packages/api/src/queue-processor.ts           |   6 +
 packages/api/src/utils/createTask.ts          |  20 +++
 .../src/websites/youtube-handler.ts           |   7 +-
 6 files changed, 195 insertions(+), 5 deletions(-)
 create mode 100644 packages/api/src/jobs/get-youtube-info.ts

diff --git a/packages/api/package.json b/packages/api/package.json
index b387616f4..45ad5c0c9 100644
--- a/packages/api/package.json
+++ b/packages/api/package.json
@@ -97,6 +97,7 @@
     "sanitize-html": "^2.3.2",
     "sax": "^1.3.0",
     "search-query-parser": "^1.6.0",
+    "showdown": "^2.1.0",
     "snake-case": "^3.0.3",
     "supertest": "^6.2.2",
     "ts-loader": "^9.3.0",
@@ -107,7 +108,9 @@
     "uuid": "^8.3.1",
     "voca": "^1.4.0",
     "winston": "^3.3.3",
-    "word-counting": "^1.1.4"
+    "word-counting": "^1.1.4",
+    "youtubei": "^1.3.4",
+    "youtubei.js": "^9.1.0"
   },
   "devDependencies": {
     "@babel/register": "^7.14.5",
@@ -136,6 +139,7 @@
     "@types/private-ip": "^1.0.0",
     "@types/sanitize-html": "^1.27.1",
     "@types/sax": "^1.2.7",
+    "@types/showdown": "^2.0.6",
     "@types/sinon": "^10.0.13",
     "@types/sinon-chai": "^3.2.8",
     "@types/supertest": "^2.0.11",
diff --git a/packages/api/src/jobs/get-youtube-info.ts b/packages/api/src/jobs/get-youtube-info.ts
new file mode 100644
index 000000000..9f7abbdf2
--- /dev/null
+++ b/packages/api/src/jobs/get-youtube-info.ts
@@ -0,0 +1,153 @@
+import { logger } from '../utils/logger'
+import { loadSummarizationChain } from 'langchain/chains'
+import { ChatOpenAI } from '@langchain/openai'
+import {
+  CharacterTextSplitter,
+  RecursiveCharacterTextSplitter,
+} from 'langchain/text_splitter'
+import { DocumentInterface } from '@langchain/core/documents'
+import { YoutubeLoader } from 'langchain/document_loaders/web/youtube'
+import { authTrx } from '../repository'
+import { libraryItemRepository } from '../repository/library_item'
+import { htmlToMarkdown, parsePreparedContent } from '../utils/parser'
+import { AISummary } from '../entity/AISummary'
+import { LibraryItem, LibraryItemState } from '../entity/library_item'
+import { getAISummary } from '../services/ai-summaries'
+import { YoutubeTranscript, TranscriptResponse } from 'youtube-transcript'
+import { Converter } from 'showdown'
+import { Video, Client as YouTubeClient } from 'youtubei'
+
+export interface ProcessYouTubeVideoJobData {
+  userId: string
+  libraryItemId: string
+}
+
+export const PROCESS_YOU_TUBE_VIDEO_JOB_NAME = 'process-you-tube-video'
+
+export const processYouTubeVideo = async (
+  jobData: ProcessYouTubeVideoJobData
+) => {
+  try {
+    console.log(
+      '******************************* processYouTubeVideo *************************'
+    )
+    const libraryItem = await authTrx(
+      async (tx) =>
+        tx
+          .withRepository(libraryItemRepository)
+          .findById(jobData.libraryItemId),
+      undefined,
+      jobData.userId
+    )
+    if (!libraryItem || libraryItem.state !== LibraryItemState.Succeeded) {
+      logger.info(
+        `Not ready to get YouTube metadata job state: ${
+          libraryItem?.state ?? 'null'
+        }`
+      )
+      return
+    }
+
+    // const doc = await YoutubeLoader.createFromUrl(libraryItem.originalUrl, {
+    //   language: 'en',
+    //   addVideoInfo: true,
+    // }).load()
+
+    // console.log('doc from youtube:', doc)
+
+    const youtube = new YouTubeClient()
+    const video = (await youtube.getVideo(
+      'Y0fqyJUrwe0' /* libraryItem.originalUrl */
+    )) as Video
+    console.log('GOT VIDEO: ', video)
+    const transcript = await video.getTranscript()
+
+    console.log('description: ', video?.description)
+    console.log('chapters: ', video?.chapters)
+
+    // const transcript = await YoutubeTranscript.fetchTranscript(
+    //   libraryItem.originalUrl
+    // )
+
+    if (transcript) {
+      console.log(
+        'original transcript:\n',
+        transcript.map((item) => item.text).join(' '),
+        '\n\n'
+      )
+    } else {
+      console.log('no transcript found')
+    }
+
+    //   const prompt = `Given the following transcript data, supplied as a list of text segments, turn it into readable
+    //                   text adding punctuation and paragraphs. Format the output as markdown.
+
+    //                   ${JSON.stringify(transcript).replace(/"/g, '\\"')}
+    //                   `
+
+    //   const llm = new ChatOpenAI({
+    //     configuration: {
+    //       apiKey: process.env.OPENAI_API_KEY,
+    //     },
+    //   })
+    //   const response = await llm.generate([[prompt]])
+    //   console.log('response: ', response.generations, response.llmOutput)
+
+    //   const text = response.generations[0][0].text
+    //   const converter = new Converter()
+    //   const transcriptHTML = converter.makeHtml(text)
+
+    //   const html = `<html>
+    //   <head><title>1 Billion Rows Challenge</title>
+    //   <meta property="og:image" content="https://i.ytimg.com/vi/OO6l1DkYA0k/hqdefault.jpg" />
+    //   <meta property="og:image:secure_url" content="https://i.ytimg.com/vi/OO6l1DkYA0k/hqdefault.jpg" />
+    //   <meta property="og:title" content="1 Billion Rows Challenge" />
+    //   <meta property="og:description" content="" />
+    //   <meta property="og:article:author" content="ThePrimeTime" />
+    //   <meta property="og:site_name" content="YouTube" />
+    //   <meta property="og:type" content="video" />
+    //   </head>
+    //   <body>
+    //   <article>
+    //   <p id="_omnivore_youtube_video" class="_omnivore_youtube_video">
+    //     <iframe class="_omnivore_youtube_embed" width="619.4690265486726" height="350" src="https://www.youtube.com/embed/OO6l1DkYA0k" title="1 Billion Rows Challenge" frameborder="0" allow="accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture" allowfullscreen></iframe>
+    //   <p>
+    //   <a href="https://www.youtube.com/watch?v=OO6l1DkYA0k" target="_blank">1 Billion Rows Challenge</a></p>
+    //   <p itemscope="" itemprop="author" itemtype="http://schema.org/Person">By <a href="https://www.youtube.com/@ThePrimeTimeagen" target="_blank">ThePrimeTime</a></p>
+    //   </p>
+    //   <div id="_omnivore_youtube_transcript" class="_omnivore_youtube_transcript"></div>
+    //   </article>
+    //   </body>
+    // </html>`.replace(
+    //     '<div id="_omnivore_youtube_transcript" class="_omnivore_youtube_transcript"></div>',
+    //     `<div id="_omnivore_youtube_transcript" class="_omnivore_youtube_transcript">${transcriptHTML}</div>`
+    //   )
+
+    //   console.log('input HTML: ', html)
+    //   if (html) {
+    //     const preparedDocument = {
+    //       document: html,
+    //       pageInfo: {},
+    //     }
+    //     const updatedContent = await parsePreparedContent(
+    //       libraryItem.originalUrl,
+    //       preparedDocument,
+    //       true
+    //     )
+    //     console.log('updated content: ', updatedContent.parsedContent?.content)
+    //     libraryItem.readableContent =
+    //       updatedContent.parsedContent?.content ?? libraryItem.readableContent
+    //     const _ = await authTrx(
+    //       async (t) => {
+    //         return t
+    //           .getRepository(LibraryItem)
+    //           .update(jobData.libraryItemId, libraryItem)
+    //       },
+    //       undefined,
+    //       jobData.userId
+    //     )
+    //   }
+  } catch (err) {
+    console.log('error creating summary: ', err)
+  }
+}
diff --git a/packages/api/src/pubsub.ts b/packages/api/src/pubsub.ts
index 7bf25d921..5d2b21817 100644
--- a/packages/api/src/pubsub.ts
+++ b/packages/api/src/pubsub.ts
@@ -7,6 +7,7 @@ import { Merge } from './util'
 import {
   enqueueAISummarizeJob,
   enqueueExportItem,
+  enqueueProcessYouTubeVideo,
   enqueueTriggerRuleJob,
   enqueueWebhookJob,
 } from './utils/createTask'
@@ -17,6 +18,7 @@ import {
   findFeatureByName,
   getFeatureName,
 } from './services/features'
+import { processYouTubeVideo } from './jobs/get-youtube-info'
 
 const logger = buildLogger('pubsub')
 
@@ -89,7 +91,11 @@ export const createPubSubClient = (): PubsubClient => {
       })
 
       if (await findFeatureByName(FeatureName.AISummaries, userId)) {
-        await enqueueAISummarizeJob({
+        // await enqueueAISummarizeJob({
+        //   userId,
+        //   libraryItemId,
+        // })
+        await enqueueProcessYouTubeVideo({
           userId,
           libraryItemId,
         })
diff --git a/packages/api/src/queue-processor.ts b/packages/api/src/queue-processor.ts
index 959c866fc..7fc18f518 100644
--- a/packages/api/src/queue-processor.ts
+++ b/packages/api/src/queue-processor.ts
@@ -44,6 +44,10 @@ import { redisDataSource } from './redis_data_source'
 import { CACHED_READING_POSITION_PREFIX } from './services/cached_reading_position'
 import { getJobPriority } from './utils/createTask'
 import { logger } from './utils/logger'
+import {
+  PROCESS_YOU_TUBE_VIDEO_JOB_NAME,
+  processYouTubeVideo,
+} from './jobs/get-youtube-info'
 
 export const QUEUE_NAME = 'omnivore-backend-queue'
 export const JOB_VERSION = 'v001'
@@ -116,6 +120,8 @@ export const createWorker = (connection: ConnectionOptions) =>
           return exportItem(job.data)
         case AI_SUMMARIZE_JOB_NAME:
           return aiSummarize(job.data)
+        case PROCESS_YOU_TUBE_VIDEO_JOB_NAME:
+          return processYouTubeVideo(job.data)
         case EXPORT_ALL_ITEMS_JOB_NAME:
           return exportAllItems(job.data)
       }
diff --git a/packages/api/src/utils/createTask.ts b/packages/api/src/utils/createTask.ts
index 56e209091..6c9345f8f 100644
--- a/packages/api/src/utils/createTask.ts
+++ b/packages/api/src/utils/createTask.ts
@@ -45,6 +45,10 @@ import { stringToHash } from './helpers'
 import { logger } from './logger'
 import View = google.cloud.tasks.v2.Task.View
 import { AISummarizeJobData, AI_SUMMARIZE_JOB_NAME } from '../jobs/ai-summarize'
+import {
+  PROCESS_YOU_TUBE_VIDEO_JOB_NAME,
+  ProcessYouTubeVideoJobData,
+} from '../jobs/get-youtube-info'
 
 // Instantiates a client.
 const client = new CloudTasksClient()
@@ -78,6 +82,8 @@ export const getJobPriority = (jobName: string): number => {
     case REFRESH_ALL_FEEDS_JOB_NAME:
     case THUMBNAIL_JOB:
       return 100
+    case PROCESS_YOU_TUBE_VIDEO_JOB_NAME:
+      return 20
     default:
       logger.error(`unknown job name: ${jobName}`)
       return 1
@@ -708,6 +714,20 @@ export const enqueueAISummarizeJob = async (data: AISummarizeJobData) => {
   })
 }
 
+export const enqueueProcessYouTubeVideo = async (
+  data: ProcessYouTubeVideoJobData
+) => {
+  const queue = await getBackendQueue()
+  if (!queue) {
+    return undefined
+  }
+
+  return queue.add(PROCESS_YOU_TUBE_VIDEO_JOB_NAME, data, {
+    priority: getJobPriority(PROCESS_YOU_TUBE_VIDEO_JOB_NAME),
+    attempts: 3,
+  })
+}
+
 export const bulkEnqueueUpdateLabels = async (data: UpdateLabelsData[]) => {
   const queue = await getBackendQueue()
   if (!queue) {
diff --git a/packages/content-handler/src/websites/youtube-handler.ts b/packages/content-handler/src/websites/youtube-handler.ts
index e86eda113..33d24cf05 100644
--- a/packages/content-handler/src/websites/youtube-handler.ts
+++ b/packages/content-handler/src/websites/youtube-handler.ts
@@ -86,9 +86,10 @@ export class YoutubeHandler extends ContentHandler {
       <meta property="og:type" content="video" />
       </head>
       <body>
-      <iframe width="${width}" height="${height}" src="${src}" title="${escapedTitle}" frameborder="0" allow="accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture" allowfullscreen></iframe>
-        <p><a href="${url}" target="_blank">${escapedTitle}</a></p>
-        <p itemscope="" itemprop="author" itemtype="http://schema.org/Person">By <a href="${oembed.author_url}" target="_blank">${authorName}</a></p>
+      <iframe id="_omnivore_youtube_video" width="${width}" height="${height}" src="${src}" title="${escapedTitle}" frameborder="0" allow="accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture" allowfullscreen></iframe>
+      <p><a href="${url}" target="_blank">${escapedTitle}</a></p>
+      <p itemscope="" itemprop="author" itemtype="http://schema.org/Person">By <a href="${oembed.author_url}" target="_blank">${authorName}</a></p>
+      <div id="_omnivore_youtube_transcript"></div>
       </body>
     </html>`
 

From 2dbd16a61eaa26523ced24c3d4e496c6e6068cdb Mon Sep 17 00:00:00 2001
From: Jackson Harper <jacksonh@gmail.com>
Date: Tue, 12 Mar 2024 17:48:16 +0800
Subject: [PATCH 31/67] Pull duration and description from YouTube metadata

---
 packages/api/src/jobs/get-youtube-info.ts | 153 ----------------------
 packages/api/src/pubsub.ts                |  20 ++-
 packages/api/src/queue-processor.ts       |   6 +-
 packages/api/src/utils/createTask.ts      |  10 +-
 yarn.lock                                 |  41 ++++++
 5 files changed, 68 insertions(+), 162 deletions(-)
 delete mode 100644 packages/api/src/jobs/get-youtube-info.ts

diff --git a/packages/api/src/jobs/get-youtube-info.ts b/packages/api/src/jobs/get-youtube-info.ts
deleted file mode 100644
index 9f7abbdf2..000000000
--- a/packages/api/src/jobs/get-youtube-info.ts
+++ /dev/null
@@ -1,153 +0,0 @@
-import { logger } from '../utils/logger'
-import { loadSummarizationChain } from 'langchain/chains'
-import { ChatOpenAI } from '@langchain/openai'
-import {
-  CharacterTextSplitter,
-  RecursiveCharacterTextSplitter,
-} from 'langchain/text_splitter'
-import { DocumentInterface } from '@langchain/core/documents'
-import { YoutubeLoader } from 'langchain/document_loaders/web/youtube'
-import { authTrx } from '../repository'
-import { libraryItemRepository } from '../repository/library_item'
-import { htmlToMarkdown, parsePreparedContent } from '../utils/parser'
-import { AISummary } from '../entity/AISummary'
-import { LibraryItem, LibraryItemState } from '../entity/library_item'
-import { getAISummary } from '../services/ai-summaries'
-import { YoutubeTranscript, TranscriptResponse } from 'youtube-transcript'
-import { Converter } from 'showdown'
-import { Video, Client as YouTubeClient } from 'youtubei'
-
-export interface ProcessYouTubeVideoJobData {
-  userId: string
-  libraryItemId: string
-}
-
-export const PROCESS_YOU_TUBE_VIDEO_JOB_NAME = 'process-you-tube-video'
-
-export const processYouTubeVideo = async (
-  jobData: ProcessYouTubeVideoJobData
-) => {
-  try {
-    console.log(
-      '******************************* processYouTubeVideo *************************'
-    )
-    const libraryItem = await authTrx(
-      async (tx) =>
-        tx
-          .withRepository(libraryItemRepository)
-          .findById(jobData.libraryItemId),
-      undefined,
-      jobData.userId
-    )
-    if (!libraryItem || libraryItem.state !== LibraryItemState.Succeeded) {
-      logger.info(
-        `Not ready to get YouTube metadata job state: ${
-          libraryItem?.state ?? 'null'
-        }`
-      )
-      return
-    }
-
-    // const doc = await YoutubeLoader.createFromUrl(libraryItem.originalUrl, {
-    //   language: 'en',
-    //   addVideoInfo: true,
-    // }).load()
-
-    // console.log('doc from youtube:', doc)
-
-    const youtube = new YouTubeClient()
-    const video = (await youtube.getVideo(
-      'Y0fqyJUrwe0' /* libraryItem.originalUrl */
-    )) as Video
-    console.log('GOT VIDEO: ', video)
-    const transcript = await video.getTranscript()
-
-    console.log('description: ', video?.description)
-    console.log('chapters: ', video?.chapters)
-
-    // const transcript = await YoutubeTranscript.fetchTranscript(
-    //   libraryItem.originalUrl
-    // )
-
-    if (transcript) {
-      console.log(
-        'original transcript:\n',
-        transcript.map((item) => item.text).join(' '),
-        '\n\n'
-      )
-    } else {
-      console.log('no transcript found')
-    }
-
-    //   const prompt = `Given the following transcript data, supplied as a list of text segments, turn it into readable
-    //                   text adding punctuation and paragraphs. Format the output as markdown.
-
-    //                   ${JSON.stringify(transcript).replace(/"/g, '\\"')}
-    //                   `
-
-    //   const llm = new ChatOpenAI({
-    //     configuration: {
-    //       apiKey: process.env.OPENAI_API_KEY,
-    //     },
-    //   })
-    //   const response = await llm.generate([[prompt]])
-    //   console.log('response: ', response.generations, response.llmOutput)
-
-    //   const text = response.generations[0][0].text
-    //   const converter = new Converter()
-    //   const transcriptHTML = converter.makeHtml(text)
-
-    //   const html = `<html>
-    //   <head><title>1 Billion Rows Challenge</title>
-    //   <meta property="og:image" content="https://i.ytimg.com/vi/OO6l1DkYA0k/hqdefault.jpg" />
-    //   <meta property="og:image:secure_url" content="https://i.ytimg.com/vi/OO6l1DkYA0k/hqdefault.jpg" />
-    //   <meta property="og:title" content="1 Billion Rows Challenge" />
-    //   <meta property="og:description" content="" />
-    //   <meta property="og:article:author" content="ThePrimeTime" />
-    //   <meta property="og:site_name" content="YouTube" />
-    //   <meta property="og:type" content="video" />
-    //   </head>
-    //   <body>
-    //   <article>
-    //   <p id="_omnivore_youtube_video" class="_omnivore_youtube_video">
-    //     <iframe class="_omnivore_youtube_embed" width="619.4690265486726" height="350" src="https://www.youtube.com/embed/OO6l1DkYA0k" title="1 Billion Rows Challenge" frameborder="0" allow="accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture" allowfullscreen></iframe>
-    //   <p>
-    //   <a href="https://www.youtube.com/watch?v=OO6l1DkYA0k" target="_blank">1 Billion Rows Challenge</a></p>
-    //   <p itemscope="" itemprop="author" itemtype="http://schema.org/Person">By <a href="https://www.youtube.com/@ThePrimeTimeagen" target="_blank">ThePrimeTime</a></p>
-    //   </p>
-    //   <div id="_omnivore_youtube_transcript" class="_omnivore_youtube_transcript"></div>
-    //   </article>
-    //   </body>
-    // </html>`.replace(
-    //     '<div id="_omnivore_youtube_transcript" class="_omnivore_youtube_transcript"></div>',
-    //     `<div id="_omnivore_youtube_transcript" class="_omnivore_youtube_transcript">${transcriptHTML}</div>`
-    //   )
-
-    //   console.log('input HTML: ', html)
-    //   if (html) {
-    //     const preparedDocument = {
-    //       document: html,
-    //       pageInfo: {},
-    //     }
-    //     const updatedContent = await parsePreparedContent(
-    //       libraryItem.originalUrl,
-    //       preparedDocument,
-    //       true
-    //     )
-    //     console.log('updated content: ', updatedContent.parsedContent?.content)
-    //     libraryItem.readableContent =
-    //       updatedContent.parsedContent?.content ?? libraryItem.readableContent
-    //     const _ = await authTrx(
-    //       async (t) => {
-    //         return t
-    //           .getRepository(LibraryItem)
-    //           .update(jobData.libraryItemId, libraryItem)
-    //       },
-    //       undefined,
-    //       jobData.userId
-    //     )
-    //   }
-  } catch (err) {
-    console.log('error creating summary: ', err)
-  }
-}
diff --git a/packages/api/src/pubsub.ts b/packages/api/src/pubsub.ts
index 5d2b21817..2d7f12ba0 100644
--- a/packages/api/src/pubsub.ts
+++ b/packages/api/src/pubsub.ts
@@ -18,7 +18,7 @@ import {
   findFeatureByName,
   getFeatureName,
 } from './services/features'
-import { processYouTubeVideo } from './jobs/get-youtube-info'
+import { processYouTubeVideo } from './jobs/process-youtube-video'
 
 const logger = buildLogger('pubsub')
 
@@ -26,6 +26,18 @@ const client = new PubSub()
 
 type EntityData<T> = Merge<T, { libraryItemId: string }>
 
+const isYouTubeVideoURL = (url: string | undefined): Boolean => {
+  if (!url) {
+    return false
+  }
+  const u = new URL(url)
+  if (!u.host.endsWith('youtube.com') && !u.host.endsWith('youtu.be')) {
+    return false
+  }
+  const videoId = u.searchParams.get('v')
+  return videoId != null
+}
+
 export const createPubSubClient = (): PubsubClient => {
   const fieldsToDelete = ['user'] as const
 
@@ -95,6 +107,12 @@ export const createPubSubClient = (): PubsubClient => {
         //   userId,
         //   libraryItemId,
         // })
+      }
+
+      if (
+        'originalUrl' in data &&
+        isYouTubeVideoURL(data['originalUrl'] as string | undefined)
+      ) {
         await enqueueProcessYouTubeVideo({
           userId,
           libraryItemId,
diff --git a/packages/api/src/queue-processor.ts b/packages/api/src/queue-processor.ts
index 7fc18f518..ed7940879 100644
--- a/packages/api/src/queue-processor.ts
+++ b/packages/api/src/queue-processor.ts
@@ -45,9 +45,9 @@ import { CACHED_READING_POSITION_PREFIX } from './services/cached_reading_positi
 import { getJobPriority } from './utils/createTask'
 import { logger } from './utils/logger'
 import {
-  PROCESS_YOU_TUBE_VIDEO_JOB_NAME,
+  PROCESS_YOUTUBE_VIDEO_JOB_NAME,
   processYouTubeVideo,
-} from './jobs/get-youtube-info'
+} from './jobs/process-youtube-video'
 
 export const QUEUE_NAME = 'omnivore-backend-queue'
 export const JOB_VERSION = 'v001'
@@ -120,7 +120,7 @@ export const createWorker = (connection: ConnectionOptions) =>
           return exportItem(job.data)
         case AI_SUMMARIZE_JOB_NAME:
           return aiSummarize(job.data)
-        case PROCESS_YOU_TUBE_VIDEO_JOB_NAME:
+        case PROCESS_YOUTUBE_VIDEO_JOB_NAME:
           return processYouTubeVideo(job.data)
         case EXPORT_ALL_ITEMS_JOB_NAME:
           return exportAllItems(job.data)
diff --git a/packages/api/src/utils/createTask.ts b/packages/api/src/utils/createTask.ts
index 6c9345f8f..fd99e1668 100644
--- a/packages/api/src/utils/createTask.ts
+++ b/packages/api/src/utils/createTask.ts
@@ -46,9 +46,9 @@ import { logger } from './logger'
 import View = google.cloud.tasks.v2.Task.View
 import { AISummarizeJobData, AI_SUMMARIZE_JOB_NAME } from '../jobs/ai-summarize'
 import {
-  PROCESS_YOU_TUBE_VIDEO_JOB_NAME,
+  PROCESS_YOUTUBE_VIDEO_JOB_NAME,
   ProcessYouTubeVideoJobData,
-} from '../jobs/get-youtube-info'
+} from '../jobs/process-youtube-video'
 
 // Instantiates a client.
 const client = new CloudTasksClient()
@@ -82,7 +82,7 @@ export const getJobPriority = (jobName: string): number => {
     case REFRESH_ALL_FEEDS_JOB_NAME:
     case THUMBNAIL_JOB:
       return 100
-    case PROCESS_YOU_TUBE_VIDEO_JOB_NAME:
+    case PROCESS_YOUTUBE_VIDEO_JOB_NAME:
       return 20
     default:
       logger.error(`unknown job name: ${jobName}`)
@@ -722,8 +722,8 @@ export const enqueueProcessYouTubeVideo = async (
     return undefined
   }
 
-  return queue.add(PROCESS_YOU_TUBE_VIDEO_JOB_NAME, data, {
-    priority: getJobPriority(PROCESS_YOU_TUBE_VIDEO_JOB_NAME),
+  return queue.add(PROCESS_YOUTUBE_VIDEO_JOB_NAME, data, {
+    priority: getJobPriority(PROCESS_YOUTUBE_VIDEO_JOB_NAME),
     attempts: 3,
   })
 }
diff --git a/yarn.lock b/yarn.lock
index cf307822f..192ec8f0f 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -2428,6 +2428,11 @@
   dependencies:
     text-decoding "^1.0.0"
 
+"@fastify/busboy@^2.0.0":
+  version "2.1.1"
+  resolved "https://registry.yarnpkg.com/@fastify/busboy/-/busboy-2.1.1.tgz#b9da6a878a371829a0502c9b6c1c143ef6663f4d"
+  integrity sha512-vBZP4NlzfOlerQTnba4aqZoMhE/a9HY7HRqoOPaETQcSQuWEIyZMHGfVu6w9wGtGK5fED5qRs2DteVCjOH60sA==
+
 "@ffmpeg-installer/darwin-arm64@4.1.5":
   version "4.1.5"
   resolved "https://registry.yarnpkg.com/@ffmpeg-installer/darwin-arm64/-/darwin-arm64-4.1.5.tgz#b7b5c262dd96d1aea4807514e1cdcf6e11f82743"
@@ -8173,6 +8178,11 @@
   resolved "https://registry.yarnpkg.com/@types/showdown/-/showdown-2.0.1.tgz#24134738ba3107237d6a783e054a54773e739f81"
   integrity sha512-xdnAw2nFqomkaL0QdtEk0t7yz26UkaVPl4v1pYJvtE1T0fmfQEH3JaxErEhGByEAl3zUZrkNBlneuJp0WJGqEA==
 
+"@types/showdown@^2.0.6":
+  version "2.0.6"
+  resolved "https://registry.yarnpkg.com/@types/showdown/-/showdown-2.0.6.tgz#3d7affd5f971b4a17783ec2b23b4ad3b97477b7e"
+  integrity sha512-pTvD/0CIeqe4x23+YJWlX2gArHa8G0J0Oh6GKaVXV7TAeickpkkZiNOgFcFcmLQ5lB/K0qBJL1FtRYltBfbGCQ==
+
 "@types/sinon-chai@^3.2.8":
   version "3.2.8"
   resolved "https://registry.yarnpkg.com/@types/sinon-chai/-/sinon-chai-3.2.8.tgz#5871d09ab50d671d8e6dd72e9073f8e738ac61dc"
@@ -19286,6 +19296,13 @@ jest@^27.4.5:
     import-local "^3.0.2"
     jest-cli "^27.5.1"
 
+jintr@^1.1.0:
+  version "1.1.0"
+  resolved "https://registry.yarnpkg.com/jintr/-/jintr-1.1.0.tgz#223a3b07f5e03d410cec6e715c537c8ad1e714c3"
+  integrity sha512-Tu9wk3BpN2v+kb8yT6YBtue+/nbjeLFv4vvVC4PJ7oCidHKbifWhvORrAbQfxVIQZG+67am/mDagpiGSVtvrZg==
+  dependencies:
+    acorn "^8.8.0"
+
 jose@^2.0.5:
   version "2.0.7"
   resolved "https://registry.yarnpkg.com/jose/-/jose-2.0.7.tgz#3aabbaec70bff313c108b9406498a163737b16ba"
@@ -29876,6 +29893,13 @@ undici@^4.9.3:
   resolved "https://registry.yarnpkg.com/undici/-/undici-4.14.1.tgz#7633b143a8a10d6d63335e00511d071e8d52a1d9"
   integrity sha512-WJ+g+XqiZcATcBaUeluCajqy4pEDcQfK1vy+Fo+bC4/mqXI9IIQD/XWHLS70fkGUT6P52Drm7IFslO651OdLPQ==
 
+undici@^5.19.1:
+  version "5.28.3"
+  resolved "https://registry.yarnpkg.com/undici/-/undici-5.28.3.tgz#a731e0eff2c3fcfd41c1169a869062be222d1e5b"
+  integrity sha512-3ItfzbrhDlINjaP0duwnNsKpDQk3acHI3gVJ1z4fmwMK31k5G9OVIAMLSIaP6w4FaGkaAkN6zaQO9LUvZ1t7VA==
+  dependencies:
+    "@fastify/busboy" "^2.0.0"
+
 unfetch@^4.2.0:
   version "4.2.0"
   resolved "https://registry.yarnpkg.com/unfetch/-/unfetch-4.2.0.tgz#7e21b0ef7d363d8d9af0fb929a5555f6ef97a3be"
@@ -31536,6 +31560,23 @@ yocto-queue@^1.0.0:
   resolved "https://registry.yarnpkg.com/yocto-queue/-/yocto-queue-1.0.0.tgz#7f816433fb2cbc511ec8bf7d263c3b58a1a3c251"
   integrity sha512-9bnSc/HEW2uRy67wc+T8UwauLuPJVn28jb+GtJY16iiKWyvmYJRXVT4UamsAEGQfPohgr2q4Tq0sQbQlxTfi1g==
 
+youtubei.js@^9.1.0:
+  version "9.1.0"
+  resolved "https://registry.yarnpkg.com/youtubei.js/-/youtubei.js-9.1.0.tgz#bcf154c9fa21d3c8c1d00a5e10360d0a065c660e"
+  integrity sha512-C5GBJ4LgnS6vGAUkdIdQNOFFb5EZ1p3xBvUELNXmIG3Idr6vxWrKNBNy8ClZT3SuDVXaAJqDgF9b5jvY8lNKcg==
+  dependencies:
+    jintr "^1.1.0"
+    tslib "^2.5.0"
+    undici "^5.19.1"
+
+youtubei@^1.3.4:
+  version "1.3.4"
+  resolved "https://registry.yarnpkg.com/youtubei/-/youtubei-1.3.4.tgz#b9761e33dcc6e0a9569e6628ba1fc48c729636f0"
+  integrity sha512-xN6p2oddcTpreF/ojU2mChwdiUlV+TwwUL6xgP6lXRuxeGS5MokM1tzRdXCgIpxkzYYNNAWpt7xvPuAUQM0PCg==
+  dependencies:
+    node-fetch "2.6.7"
+    protobufjs "7.2.4"
+
 yup@^0.31.0:
   version "0.31.1"
   resolved "https://registry.yarnpkg.com/yup/-/yup-0.31.1.tgz#0954cb181161f397b804346037a04f8a4b31599e"

From 7c3d15e31a219f255cc29669cc0ca0996d5f1a92 Mon Sep 17 00:00:00 2001
From: Jackson Harper <jacksonh@gmail.com>
Date: Tue, 12 Mar 2024 18:53:17 +0800
Subject: [PATCH 32/67] Add some scrolling on youtube videos

---
 .../api/src/jobs/process-youtube-video.ts     | 88 +++++++++++++++++++
 .../components/templates/article/Article.tsx  | 22 +++++
 packages/web/styles/articleInnerStyling.css   | 28 ++++++
 3 files changed, 138 insertions(+)
 create mode 100644 packages/api/src/jobs/process-youtube-video.ts

diff --git a/packages/api/src/jobs/process-youtube-video.ts b/packages/api/src/jobs/process-youtube-video.ts
new file mode 100644
index 000000000..82bf1f946
--- /dev/null
+++ b/packages/api/src/jobs/process-youtube-video.ts
@@ -0,0 +1,88 @@
+import { logger } from '../utils/logger'
+import { authTrx } from '../repository'
+import { libraryItemRepository } from '../repository/library_item'
+import { LibraryItem, LibraryItemState } from '../entity/library_item'
+
+import { Video, Client as YouTubeClient } from 'youtubei'
+
+export interface ProcessYouTubeVideoJobData {
+  userId: string
+  libraryItemId: string
+}
+
+export const PROCESS_YOUTUBE_VIDEO_JOB_NAME = 'process-youtube-video'
+
+const calculateWordCount = (durationInSeconds: number): number => {
+  // Calculate word count using the formula: word count = read time (in seconds) * words per second
+  // Assuming average reading speed is 235 words per minute (or about 3.92 words per second)
+  const wordsPerSecond = 3.92
+  const wordCount = Math.round(durationInSeconds * wordsPerSecond)
+  return wordCount
+}
+
+export const processYouTubeVideo = async (
+  jobData: ProcessYouTubeVideoJobData
+) => {
+  try {
+    const libraryItem = await authTrx(
+      async (tx) =>
+        tx
+          .withRepository(libraryItemRepository)
+          .findById(jobData.libraryItemId),
+      undefined,
+      jobData.userId
+    )
+    if (!libraryItem || libraryItem.state !== LibraryItemState.Succeeded) {
+      logger.info(
+        `Not ready to get YouTube metadata job state: ${
+          libraryItem?.state ?? 'null'
+        }`
+      )
+      return
+    }
+
+    const u = new URL(libraryItem.originalUrl)
+    const videoId = u.searchParams.get('v')
+
+    if (!videoId) {
+      console.warn('no video id for supplied youtube url', {
+        url: libraryItem.originalUrl,
+      })
+      return
+    }
+
+    let needsUpdate = false
+    const youtube = new YouTubeClient()
+    const video = await youtube.getVideo(videoId)
+    if (!video) {
+      console.warn('no video found for youtube url', {
+        url: libraryItem.originalUrl,
+      })
+      return
+    }
+
+    if (video.description && libraryItem.description !== video.description) {
+      needsUpdate = true
+      libraryItem.description = video.description
+    }
+
+    if ('duration' in video && (video as Video).duration > 0) {
+      needsUpdate = true
+      libraryItem.wordCount = calculateWordCount((video as Video).duration)
+    }
+
+    if (needsUpdate) {
+      const _ = await authTrx(
+        async (t) => {
+          return t
+            .getRepository(LibraryItem)
+            .update(jobData.libraryItemId, libraryItem)
+        },
+        undefined,
+        jobData.userId
+      )
+    }
+  } catch (err) {
+    console.log('error creating summary: ', err)
+  }
+}
diff --git a/packages/web/components/templates/article/Article.tsx b/packages/web/components/templates/article/Article.tsx
index 5f7d48ba4..91ef82161 100644
--- a/packages/web/components/templates/article/Article.tsx
+++ b/packages/web/components/templates/article/Article.tsx
@@ -115,6 +115,28 @@ export function Article(props: ArticleProps): JSX.Element {
     }
   }, 2500)
 
+  useEffect(() => {
+    const youtubePlayer = document.getElementById('_omnivore_youtube_video')
+
+    const updateScroll = () => {
+      console.log('scroll y: ', window.scrollY, youtubePlayer)
+
+      if (youtubePlayer) {
+        if (window.scrollY > 200) {
+          youtubePlayer.classList.add('is-sticky')
+        } else {
+          youtubePlayer.classList.remove('is-sticky')
+        }
+      }
+    }
+    if (youtubePlayer) {
+      window.addEventListener('scroll', updateScroll)
+    }
+    return () => {
+      window.removeEventListener('scroll', updateScroll) // clean up
+    }
+  }, [props])
+
   // Scroll to initial anchor position
   useEffect(() => {
     if (typeof window === 'undefined') {
diff --git a/packages/web/styles/articleInnerStyling.css b/packages/web/styles/articleInnerStyling.css
index c38f290a7..b0a46dc39 100644
--- a/packages/web/styles/articleInnerStyling.css
+++ b/packages/web/styles/articleInnerStyling.css
@@ -610,3 +610,31 @@
   white-space: pre-wrap;
   overflow-wrap: break-word;
 }
+
+.is-sticky {
+  position: fixed;
+  right: 5px;
+  bottom: 5px;
+  top: auto;
+  left: auto;
+  max-width: 400px;
+  max-height: 222px;
+  width: 400px;
+  height: 222px;
+  animation-name: fadeInUp;
+  animation-duration: 0.5s;
+  animation-fill-mode: both;
+}
+
+@keyframes fadeInUp {
+  0% {
+      opacity: 0;
+      -webkit-transform: translate3d(0, 100%, 0);
+      transform: translate3d(0, 100%, 0);
+  }
+  100% {
+      opacity: 1;
+      -webkit-transform: none;
+      transform: none;
+  }
+}
\ No newline at end of file

From 84f0d940d2b3682e82a7ddad15cdf872cb124008 Mon Sep 17 00:00:00 2001
From: Jackson Harper <jacksonh@gmail.com>
Date: Tue, 12 Mar 2024 20:37:05 +0800
Subject: [PATCH 33/67] Add some box shadow on the sticky player

---
 packages/web/styles/articleInnerStyling.css | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/packages/web/styles/articleInnerStyling.css b/packages/web/styles/articleInnerStyling.css
index b0a46dc39..d76fa822e 100644
--- a/packages/web/styles/articleInnerStyling.css
+++ b/packages/web/styles/articleInnerStyling.css
@@ -617,6 +617,7 @@
   bottom: 5px;
   top: auto;
   left: auto;
+  z-index: 10;
   max-width: 400px;
   max-height: 222px;
   width: 400px;
@@ -624,6 +625,8 @@
   animation-name: fadeInUp;
   animation-duration: 0.5s;
   animation-fill-mode: both;
+  overflow: hidden;
+  box-shadow: 0px 4px 4px rgba(33, 33, 33, 0.1) !important;
 }
 
 @keyframes fadeInUp {

From 6bcdbfa8f019797ed015fbb5ea94ed2dfcb382ff Mon Sep 17 00:00:00 2001
From: Jackson Harper <jacksonh@gmail.com>
Date: Wed, 13 Mar 2024 10:02:23 +0800
Subject: [PATCH 34/67] Linting

---
 packages/api/src/pubsub.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/api/src/pubsub.ts b/packages/api/src/pubsub.ts
index 2d7f12ba0..9bc4858b1 100644
--- a/packages/api/src/pubsub.ts
+++ b/packages/api/src/pubsub.ts
@@ -26,7 +26,7 @@ const client = new PubSub()
 
 type EntityData<T> = Merge<T, { libraryItemId: string }>
 
-const isYouTubeVideoURL = (url: string | undefined): Boolean => {
+const isYouTubeVideoURL = (url: string | undefined): boolean => {
   if (!url) {
     return false
   }

From 39dfa920b5e42cb3c7c9a60ea8cb1188c6862960 Mon Sep 17 00:00:00 2001
From: Jackson Harper <jacksonh@gmail.com>
Date: Wed, 13 Mar 2024 10:02:53 +0800
Subject: [PATCH 35/67] More fixes to youtube processor

---
 .../api/src/jobs/process-youtube-video.ts     | 69 +++++++++++++++++--
 1 file changed, 64 insertions(+), 5 deletions(-)

diff --git a/packages/api/src/jobs/process-youtube-video.ts b/packages/api/src/jobs/process-youtube-video.ts
index 82bf1f946..3962c8078 100644
--- a/packages/api/src/jobs/process-youtube-video.ts
+++ b/packages/api/src/jobs/process-youtube-video.ts
@@ -3,7 +3,7 @@ import { authTrx } from '../repository'
 import { libraryItemRepository } from '../repository/library_item'
 import { LibraryItem, LibraryItemState } from '../entity/library_item'
 
-import { Video, Client as YouTubeClient } from 'youtubei'
+import { Chapter, Client as YouTubeClient } from 'youtubei'
 
 export interface ProcessYouTubeVideoJobData {
   userId: string
@@ -20,6 +20,44 @@ const calculateWordCount = (durationInSeconds: number): number => {
   return wordCount
 }
 
+interface ChapterProperties {
+  title: string
+  start: number
+}
+
+interface TranscriptProperties {
+  text: string
+  start: number
+  duration: number
+}
+
+export const addTranscriptChapters = (
+  chapters: ChapterProperties[],
+  transcript: TranscriptProperties[]
+): TranscriptProperties[] => {
+  chapters.sort((a, b) => a.start - b.start)
+
+  for (const chapter of chapters) {
+    const startOffset = chapter.start
+    const title = '## ' + chapter.title + '\n\n'
+
+    const index = transcript.findIndex(
+      (textItem) => textItem.start > startOffset
+    )
+
+    if (index !== -1) {
+      transcript.splice(index, 0, {
+        text: title,
+        duration: 1,
+        start: startOffset,
+      })
+    } else {
+      transcript.push({ text: title, duration: 0, start: startOffset })
+    }
+  }
+  return transcript
+}
+
 export const processYouTubeVideo = async (
   jobData: ProcessYouTubeVideoJobData
 ) => {
@@ -66,13 +104,31 @@ export const processYouTubeVideo = async (
       libraryItem.description = video.description
     }
 
-    if ('duration' in video && (video as Video).duration > 0) {
+    if ('duration' in video && video.duration > 0) {
       needsUpdate = true
-      libraryItem.wordCount = calculateWordCount((video as Video).duration)
+      libraryItem.wordCount = calculateWordCount(video.duration)
+    }
+
+    let chapters: Chapter[] = []
+    if ('chapters' in video) {
+      chapters = video.chapters
+      console.log('video.chapters: ', video.chapters)
+    }
+
+    let transcript: TranscriptProperties[] | undefined = undefined
+    if ('getTranscript' in video) {
+      transcript = await video.getTranscript()
+      console.log('transcript: ', transcript)
+    }
+
+    if (transcript) {
+      if (chapters) {
+        transcript = addTranscriptChapters(chapters, transcript)
+      }
     }
 
     if (needsUpdate) {
-      const _ = await authTrx(
+      const updated = await authTrx(
         async (t) => {
           return t
             .getRepository(LibraryItem)
@@ -81,8 +137,11 @@ export const processYouTubeVideo = async (
         undefined,
         jobData.userId
       )
+      if (!updated) {
+        console.warn('could not updated library item')
+      }
     }
   } catch (err) {
-    console.log('error creating summary: ', err)
+    console.warn('error creating summary: ', err)
   }
 }

From a7ad67b3bb8dcbdbc877cf60dee2c59594360125 Mon Sep 17 00:00:00 2001
From: Jackson Harper <jacksonh@gmail.com>
Date: Wed, 13 Mar 2024 10:13:19 +0800
Subject: [PATCH 36/67] Start to add tests for youtube processor

---
 .../api/test/jobs/process-youtube-job.test.ts | 101 ++++++++++++++++++
 1 file changed, 101 insertions(+)
 create mode 100644 packages/api/test/jobs/process-youtube-job.test.ts

diff --git a/packages/api/test/jobs/process-youtube-job.test.ts b/packages/api/test/jobs/process-youtube-job.test.ts
new file mode 100644
index 000000000..70b8658a7
--- /dev/null
+++ b/packages/api/test/jobs/process-youtube-job.test.ts
@@ -0,0 +1,101 @@
+import { expect } from 'chai'
+import 'mocha'
+import { addTranscriptChapters } from '../../src/jobs/process-youtube-video'
+
+describe('create transcript', () => {
+  describe('build items', () => {
+    it('properly adds chapter headers to transcript', async () => {
+      const chapters = [
+        {
+          title: 'Intro',
+          start: 0,
+        },
+        {
+          title: "Joe Biden's re-election effort",
+          start: 22000,
+        },
+        {
+          title: 'Ad break',
+          start: 909000,
+        },
+        {
+          title: "Trump's crazy speech & Orbán relationship",
+          start: 1060000,
+        },
+      ]
+      const transcript = [
+        {
+          text: "welcome to pod save America I'm John",
+          duration: 3280,
+          start: 80,
+        },
+        {
+          text: "favro I'm John L I'm Tommy VOR on",
+          duration: 3480,
+          start: 1480,
+        },
+        {
+          text: "today's show Donald Trump kicks off the",
+          duration: 3320,
+          start: 3360,
+        },
+        {
+          text: 'general election by mocking Joe Biden',
+          duration: 3400,
+          start: 4960,
+        },
+        {
+          text: 'stutter hosting a concert for Victor',
+          duration: 3680,
+          start: 6680,
+        },
+        {
+          text: 'Orban and floating cuts to Medicare and',
+          duration: 4239,
+          start: 8360,
+        },
+        {
+          text: 'Social Security Alabama Senator Katie',
+          duration: 3840,
+          start: 10360,
+        },
+        {
+          text: 'Brit and Republicans are still dealing',
+          duration: 3401,
+          start: 12599,
+        },
+        {
+          text: 'with the Fallout from what may have been',
+          duration: 3320,
+          start: 14200,
+        },
+        {
+          text: 'the worst ever State of the Union',
+          duration: 4600,
+          start: 16000,
+        },
+        {
+          text: 'response and later take appreciator is',
+          duration: 6640,
+          start: 17520,
+        },
+        {
+          text: 'back so is Elijah uh but first the man',
+          duration: 6519,
+          start: 20600,
+        },
+        {
+          text: 'Sean Hannity now calls jacked up Joe has',
+          duration: 4680,
+          start: 24160,
+        },
+      ]
+
+      const res = addTranscriptChapters(chapters, transcript)
+      console.log('res: ', res)
+
+      expect(res.length).to.eq(17)
+      expect(res[13].text).to.eq("## Joe Biden's re-election effort\n\n")
+    })
+  })
+})

From 3ee6787e395121198e17a8f68ca6d39a288d5ff7 Mon Sep 17 00:00:00 2001
From: Jackson Harper <jacksonh@gmail.com>
Date: Wed, 13 Mar 2024 17:50:07 +0800
Subject: [PATCH 37/67] Improve transcript generation

---
 packages/api/package.json                     |   1 +
 .../api/src/jobs/process-youtube-video.ts     | 112 +++++++++++++++++-
 packages/api/src/utils/createTask.ts          |   1 +
 .../src/websites/youtube-handler.ts           |  12 +-
 4 files changed, 118 insertions(+), 8 deletions(-)

diff --git a/packages/api/package.json b/packages/api/package.json
index 45ad5c0c9..3de277a1c 100644
--- a/packages/api/package.json
+++ b/packages/api/package.json
@@ -46,6 +46,7 @@
     "@sentry/integrations": "^7.10.0",
     "@sentry/node": "^5.26.0",
     "@sentry/tracing": "^7.9.0",
+    "@types/showdown": "^2.0.6",
     "addressparser": "^1.0.1",
     "apollo-datasource": "^3.3.1",
     "apollo-server-express": "^3.6.3",
diff --git a/packages/api/src/jobs/process-youtube-video.ts b/packages/api/src/jobs/process-youtube-video.ts
index 3962c8078..44235298a 100644
--- a/packages/api/src/jobs/process-youtube-video.ts
+++ b/packages/api/src/jobs/process-youtube-video.ts
@@ -4,6 +4,11 @@ import { libraryItemRepository } from '../repository/library_item'
 import { LibraryItem, LibraryItemState } from '../entity/library_item'
 
 import { Chapter, Client as YouTubeClient } from 'youtubei'
+import showdown from 'showdown'
+import { parseHTML } from 'linkedom'
+import { parsePreparedContent } from '../utils/parser'
+import { OpenAI } from '@langchain/openai'
+import { PromptTemplate } from '@langchain/core/prompts'
 
 export interface ProcessYouTubeVideoJobData {
   userId: string
@@ -39,7 +44,7 @@ export const addTranscriptChapters = (
 
   for (const chapter of chapters) {
     const startOffset = chapter.start
-    const title = '## ' + chapter.title + '\n\n'
+    const title = '\n\n## ' + chapter.title + '\n\n'
 
     const index = transcript.findIndex(
       (textItem) => textItem.start > startOffset
@@ -58,6 +63,92 @@ export const addTranscriptChapters = (
   return transcript
 }
 
+export const createTranscriptHTML = async (
+  transcript: TranscriptProperties[]
+): Promise<string> => {
+  let transcriptMarkdown = ''
+  if (process.env.YOUTUBE_TRANSCRIPT_PROMPT && process.env.OPENAI_API_KEY) {
+    const llm = new OpenAI({
+      modelName: 'gpt-4',
+      configuration: {
+        apiKey: process.env.OPENAI_API_KEY,
+      },
+    })
+    const promptTemplate = PromptTemplate.fromTemplate(
+      `${process.env.YOUTUBE_TRANSCRIPT_PROMPT}
+
+       Data:
+       {transcriptData}`
+    )
+    const chain = promptTemplate.pipe(llm)
+
+    let transcriptChunkLength = 0
+    let transcriptChunk: TranscriptProperties[] = []
+    for (const item of transcript) {
+      if (transcriptChunkLength + item.text.length > 8000) {
+        const result = await chain.invoke({
+          transcriptData: transcriptChunk.map((item) => item.text).join(' '),
+        })
+
+        transcriptMarkdown += result
+
+        transcriptChunk = []
+        transcriptChunkLength = 0
+      }
+
+      transcriptChunk.push(item)
+      transcriptChunkLength += item.text.length
+    }
+
+    if (transcriptChunk.length > 0) {
+      const result = await chain.invoke({
+        transcriptData: transcriptChunk.map((item) => item.text).join(' '),
+      })
+
+      transcriptMarkdown += result
+    }
+  }
+
+  // If the LLM didn't give us enough data fallback to the raw template
+  if (transcriptMarkdown.length < 1) {
+    transcriptMarkdown = transcript.map((item) => item.text).join(' ')
+  }
+
+  var converter = new showdown.Converter()
+  return converter.makeHtml(transcriptMarkdown)
+}
+
+export const addTranscriptToReadableContent = async (
+  originalUrl: string,
+  originalHTML: string,
+  transcriptHTML: string
+): Promise<string | undefined> => {
+  const html = parseHTML(originalHTML)
+
+  const transcriptNode = html.document.querySelector(
+    '#_omnivore_youtube_transcript'
+  )
+
+  if (transcriptNode) {
+    transcriptNode.innerHTML = transcriptHTML
+  } else {
+    const div = html.document.createElement('div')
+    div.innerHTML = transcriptHTML
+    html.document.body.appendChild(div)
+  }
+
+  const preparedDocument = {
+    document: html.document.toString(),
+    pageInfo: {},
+  }
+  const updatedContent = await parsePreparedContent(
+    originalUrl,
+    preparedDocument,
+    true
+  )
+  return updatedContent.parsedContent?.content
+}
+
 export const processYouTubeVideo = async (
   jobData: ProcessYouTubeVideoJobData
 ) => {
@@ -70,7 +161,11 @@ export const processYouTubeVideo = async (
       undefined,
       jobData.userId
     )
-    if (!libraryItem || libraryItem.state !== LibraryItemState.Succeeded) {
+    if (
+      !libraryItem ||
+      libraryItem.state !== LibraryItemState.Succeeded ||
+      !libraryItem.originalContent
+    ) {
       logger.info(
         `Not ready to get YouTube metadata job state: ${
           libraryItem?.state ?? 'null'
@@ -112,19 +207,28 @@ export const processYouTubeVideo = async (
     let chapters: Chapter[] = []
     if ('chapters' in video) {
       chapters = video.chapters
-      console.log('video.chapters: ', video.chapters)
     }
 
     let transcript: TranscriptProperties[] | undefined = undefined
     if ('getTranscript' in video) {
       transcript = await video.getTranscript()
-      console.log('transcript: ', transcript)
     }
 
     if (transcript) {
       if (chapters) {
         transcript = addTranscriptChapters(chapters, transcript)
       }
+      const transcriptHTML = await createTranscriptHTML(transcript)
+      const updatedContent = await addTranscriptToReadableContent(
+        libraryItem.originalUrl,
+        libraryItem.originalContent,
+        transcriptHTML
+      )
+
+      if (updatedContent) {
+        needsUpdate = true
+        libraryItem.readableContent = updatedContent
+      }
     }
 
     if (needsUpdate) {
diff --git a/packages/api/src/utils/createTask.ts b/packages/api/src/utils/createTask.ts
index fd99e1668..2f5060f72 100644
--- a/packages/api/src/utils/createTask.ts
+++ b/packages/api/src/utils/createTask.ts
@@ -725,6 +725,7 @@ export const enqueueProcessYouTubeVideo = async (
   return queue.add(PROCESS_YOUTUBE_VIDEO_JOB_NAME, data, {
     priority: getJobPriority(PROCESS_YOUTUBE_VIDEO_JOB_NAME),
     attempts: 3,
+    delay: 2000,
   })
 }
 
diff --git a/packages/content-handler/src/websites/youtube-handler.ts b/packages/content-handler/src/websites/youtube-handler.ts
index 33d24cf05..cd7f87c95 100644
--- a/packages/content-handler/src/websites/youtube-handler.ts
+++ b/packages/content-handler/src/websites/youtube-handler.ts
@@ -86,10 +86,14 @@ export class YoutubeHandler extends ContentHandler {
       <meta property="og:type" content="video" />
       </head>
       <body>
-      <iframe id="_omnivore_youtube_video" width="${width}" height="${height}" src="${src}" title="${escapedTitle}" frameborder="0" allow="accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture" allowfullscreen></iframe>
-      <p><a href="${url}" target="_blank">${escapedTitle}</a></p>
-      <p itemscope="" itemprop="author" itemtype="http://schema.org/Person">By <a href="${oembed.author_url}" target="_blank">${authorName}</a></p>
-      <div id="_omnivore_youtube_transcript"></div>
+      <div>
+        <article id="_omnivore_youtube">
+          <iframe id="_omnivore_youtube_video" width="${width}" height="${height}" src="${src}" title="${escapedTitle}" frameborder="0" allow="accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture" allowfullscreen></iframe>
+          <p><a href="${url}" target="_blank">${escapedTitle}</a></p>
+          <p itemscope="" itemprop="author" itemtype="http://schema.org/Person">By <a href="${oembed.author_url}" target="_blank">${authorName}</a></p>
+          <div id="_omnivore_youtube_transcript"></div>
+        </article>
+      </div>
       </body>
     </html>`
 

From fef28d1c6a02669ad917b89d22db607d7a1a5d96 Mon Sep 17 00:00:00 2001
From: Jackson Harper <jacksonh@gmail.com>
Date: Wed, 13 Mar 2024 18:07:29 +0800
Subject: [PATCH 38/67] Linting fix

---
 packages/api/src/jobs/process-youtube-video.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/api/src/jobs/process-youtube-video.ts b/packages/api/src/jobs/process-youtube-video.ts
index 44235298a..6c9307dc2 100644
--- a/packages/api/src/jobs/process-youtube-video.ts
+++ b/packages/api/src/jobs/process-youtube-video.ts
@@ -114,7 +114,7 @@ export const createTranscriptHTML = async (
     transcriptMarkdown = transcript.map((item) => item.text).join(' ')
   }
 
-  var converter = new showdown.Converter()
+  const converter = new showdown.Converter()
   return converter.makeHtml(transcriptMarkdown)
 }
 

From e9b15ebb06de190964ada6205e2939bb58dfa3fa Mon Sep 17 00:00:00 2001
From: Jackson Harper <jacksonh@gmail.com>
Date: Wed, 13 Mar 2024 18:24:43 +0800
Subject: [PATCH 39/67] Dont process youtube videos in this test

---
 packages/api/test/resolvers/article.test.ts | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/packages/api/test/resolvers/article.test.ts b/packages/api/test/resolvers/article.test.ts
index b7b638d22..8d3891af6 100644
--- a/packages/api/test/resolvers/article.test.ts
+++ b/packages/api/test/resolvers/article.test.ts
@@ -49,6 +49,7 @@ import {
   saveLabelsInLibraryItem,
 } from '../db'
 import { generateFakeUuid, graphqlRequest, request } from '../util'
+import { processYouTubeVideo } from '../../src/jobs/process-youtube-video'
 
 chai.use(chaiString)
 
@@ -640,6 +641,7 @@ describe('Article API', () => {
     context('when the source is rss-feeder and url is from youtube.com', () => {
       const source = 'rss-feeder'
       const stub = sinon.stub(createTask, 'enqueueParseRequest')
+      const stub2 = sinon.stub(createTask, 'enqueueProcessYouTubeVideo')
 
       before(() => {
         url = 'https://www.youtube.com/watch?v=123'

From 05fe1cb87c05cbe1250b3dcca5fdb86a3ea4e1ab Mon Sep 17 00:00:00 2001
From: Jackson Harper <jacksonh@gmail.com>
Date: Wed, 13 Mar 2024 20:19:13 +0800
Subject: [PATCH 40/67] Webkit support for slide in frame

---
 packages/web/styles/articleInnerStyling.css | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/packages/web/styles/articleInnerStyling.css b/packages/web/styles/articleInnerStyling.css
index d76fa822e..83f6adb3d 100644
--- a/packages/web/styles/articleInnerStyling.css
+++ b/packages/web/styles/articleInnerStyling.css
@@ -625,6 +625,9 @@
   animation-name: fadeInUp;
   animation-duration: 0.5s;
   animation-fill-mode: both;
+  -webkit-animation-name: fadeInUp;
+  -webkit-animation-duration: 0.5s;
+  -webkit-animation-fill-mode: both;
   overflow: hidden;
   box-shadow: 0px 4px 4px rgba(33, 33, 33, 0.1) !important;
 }
@@ -640,4 +643,17 @@
       -webkit-transform: none;
       transform: none;
   }
+}
+
+@-webkit-keyframes fadeInUp {
+  0% {
+      opacity: 0;
+      -webkit-transform: translate3d(0, 100%, 0);
+      transform: translate3d(0, 100%, 0);
+  }
+  100% {
+      opacity: 1;
+      -webkit-transform: none;
+      transform: none;
+  }
 }
\ No newline at end of file

From 566ac33401d475bf8cc59019f9a6dda99b3f0999 Mon Sep 17 00:00:00 2001
From: Jackson Harper <jacksonh@gmail.com>
Date: Wed, 13 Mar 2024 20:22:12 +0800
Subject: [PATCH 41/67] Better small screen support

---
 packages/web/styles/articleInnerStyling.css | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/packages/web/styles/articleInnerStyling.css b/packages/web/styles/articleInnerStyling.css
index 83f6adb3d..251da4bb7 100644
--- a/packages/web/styles/articleInnerStyling.css
+++ b/packages/web/styles/articleInnerStyling.css
@@ -632,6 +632,14 @@
   box-shadow: 0px 4px 4px rgba(33, 33, 33, 0.1) !important;
 }
 
+@media (max-width: 600px) {
+  .is-sticky {
+    max-width: 200px;
+    max-height: 110px;
+  }
+}
+
+
 @keyframes fadeInUp {
   0% {
       opacity: 0;

From 308b02fbb03a6b9abcec1e7307a37448d0e979e9 Mon Sep 17 00:00:00 2001
From: Jackson Harper <jacksonh@gmail.com>
Date: Wed, 13 Mar 2024 22:00:33 +0800
Subject: [PATCH 42/67] Add breaks to headers in test

---
 packages/api/test/jobs/process-youtube-job.test.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/api/test/jobs/process-youtube-job.test.ts b/packages/api/test/jobs/process-youtube-job.test.ts
index 70b8658a7..849aca41c 100644
--- a/packages/api/test/jobs/process-youtube-job.test.ts
+++ b/packages/api/test/jobs/process-youtube-job.test.ts
@@ -95,7 +95,7 @@ describe('create transcript', () => {
       console.log('res: ', res)
 
       expect(res.length).to.eq(17)
-      expect(res[13].text).to.eq("## Joe Biden's re-election effort\n\n")
+      expect(res[13].text).to.eq("\n\n## Joe Biden's re-election effort\n\n")
     })
   })
 })

From d3d181c33e9dfc59f4c5781e94dff2ff96c63dfa Mon Sep 17 00:00:00 2001
From: Jackson Harper <jacksonh@gmail.com>
Date: Wed, 13 Mar 2024 22:00:51 +0800
Subject: [PATCH 43/67] Remove unneed import

---
 packages/api/test/resolvers/article.test.ts | 1 -
 1 file changed, 1 deletion(-)

diff --git a/packages/api/test/resolvers/article.test.ts b/packages/api/test/resolvers/article.test.ts
index 8d3891af6..e5d05e6ae 100644
--- a/packages/api/test/resolvers/article.test.ts
+++ b/packages/api/test/resolvers/article.test.ts
@@ -49,7 +49,6 @@ import {
   saveLabelsInLibraryItem,
 } from '../db'
 import { generateFakeUuid, graphqlRequest, request } from '../util'
-import { processYouTubeVideo } from '../../src/jobs/process-youtube-video'
 
 chai.use(chaiString)
 

From 1ea09e20bd81f6feed9636ef3415ad1066c84729 Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Thu, 14 Mar 2024 12:56:42 +0800
Subject: [PATCH 44/67] fix: subscription not updated correctly after rss feed
 refreshed

* sort the shallow copy of the userIds array so the original array not mutated
---
 packages/api/src/jobs/rss/refreshAllFeeds.ts | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/packages/api/src/jobs/rss/refreshAllFeeds.ts b/packages/api/src/jobs/rss/refreshAllFeeds.ts
index d68263400..8d49b6096 100644
--- a/packages/api/src/jobs/rss/refreshAllFeeds.ts
+++ b/packages/api/src/jobs/rss/refreshAllFeeds.ts
@@ -43,7 +43,7 @@ export const refreshAllFeeds = async (db: DataSource): Promise<boolean> => {
         AND (s.scheduled_at <= NOW() OR s.scheduled_at IS NULL)
         AND u.status = $4
       GROUP BY
-        s.url
+        url
       `,
     ['RSS', 'ACTIVE', 'following', 'ACTIVE']
   )) as RssSubscriptionGroup[]
@@ -76,7 +76,10 @@ const updateSubscriptionGroup = async (
   refreshContext: RSSRefreshContext
 ) => {
   let feedURL = group.url
-  const userList = JSON.stringify(group.userIds.sort())
+  const userIds = group.userIds
+  // sort the user ids so that the job id is consistent
+  // [...userIds] creates a shallow copy, so sort() does not mutate the original
+  const userList = JSON.stringify([...userIds].sort())
   if (!feedURL) {
     logger.error('no url for feed group', group)
     return
@@ -105,7 +108,7 @@ const updateSubscriptionGroup = async (
     scheduledTimestamps: group.scheduledDates.map((timestamp) =>
       timestamp.getTime()
     ), // unix timestamp in milliseconds
-    userIds: group.userIds,
+    userIds,
     fetchContentTypes: group.fetchContentTypes,
     folders: group.folders,
   }

From 629c0442730180d76586c362b6292be72af02d61 Mon Sep 17 00:00:00 2001
From: Jackson Harper <jacksonh@gmail.com>
Date: Thu, 14 Mar 2024 15:09:50 +0800
Subject: [PATCH 45/67] Queue the transcript processing as a separate job

Handle YouTube in two steps, first get metadata then
get the transcript.
---
 .../api/src/jobs/process-youtube-video.ts     | 121 +++++++++++++++++-
 packages/api/src/queue-processor.ts           |   6 +
 packages/api/src/utils/createTask.ts          |  23 +++-
 3 files changed, 146 insertions(+), 4 deletions(-)

diff --git a/packages/api/src/jobs/process-youtube-video.ts b/packages/api/src/jobs/process-youtube-video.ts
index 6c9307dc2..76bacf23f 100644
--- a/packages/api/src/jobs/process-youtube-video.ts
+++ b/packages/api/src/jobs/process-youtube-video.ts
@@ -9,6 +9,7 @@ import { parseHTML } from 'linkedom'
 import { parsePreparedContent } from '../utils/parser'
 import { OpenAI } from '@langchain/openai'
 import { PromptTemplate } from '@langchain/core/prompts'
+import { enqueueProcessYouTubeTranscript } from '../utils/createTask'
 
 export interface ProcessYouTubeVideoJobData {
   userId: string
@@ -16,6 +17,10 @@ export interface ProcessYouTubeVideoJobData {
 }
 
 export const PROCESS_YOUTUBE_VIDEO_JOB_NAME = 'process-youtube-video'
+export const PROCESS_YOUTUBE_TRANSCRIPT_JOB_NAME = 'process-youtube-transcript'
+
+const TRANSCRIPT_PLACEHOLDER_TEXT =
+  '* Omnivore is preparing a transcript for this video'
 
 const calculateWordCount = (durationInSeconds: number): number => {
   // Calculate word count using the formula: word count = read time (in seconds) * words per second
@@ -77,7 +82,6 @@ export const createTranscriptHTML = async (
     const promptTemplate = PromptTemplate.fromTemplate(
       `${process.env.YOUTUBE_TRANSCRIPT_PROMPT}
 
-       Data:
        {transcriptData}`
     )
     const chain = promptTemplate.pipe(llm)
@@ -114,7 +118,9 @@ export const createTranscriptHTML = async (
     transcriptMarkdown = transcript.map((item) => item.text).join(' ')
   }
 
-  const converter = new showdown.Converter()
+  const converter = new showdown.Converter({
+    backslashEscapesHTMLTags: true,
+  })
   return converter.makeHtml(transcriptMarkdown)
 }
 
@@ -149,6 +155,36 @@ export const addTranscriptToReadableContent = async (
   return updatedContent.parsedContent?.content
 }
 
+export const addTranscriptPlaceholdReadableContent = async (
+  originalUrl: string,
+  originalHTML: string
+): Promise<string | undefined> => {
+  const html = parseHTML(originalHTML)
+
+  const transcriptNode = html.document.querySelector(
+    '#_omnivore_youtube_transcript'
+  )
+
+  if (transcriptNode) {
+    transcriptNode.innerHTML = TRANSCRIPT_PLACEHOLDER_TEXT
+  } else {
+    const div = html.document.createElement('div')
+    div.innerHTML = TRANSCRIPT_PLACEHOLDER_TEXT
+    html.document.body.appendChild(div)
+  }
+
+  const preparedDocument = {
+    document: html.document.toString(),
+    pageInfo: {},
+  }
+  const updatedContent = await parsePreparedContent(
+    originalUrl,
+    preparedDocument,
+    true
+  )
+  return updatedContent.parsedContent?.content
+}
+
 export const processYouTubeVideo = async (
   jobData: ProcessYouTubeVideoJobData
 ) => {
@@ -199,9 +235,90 @@ export const processYouTubeVideo = async (
       libraryItem.description = video.description
     }
 
+    let duration = -1
     if ('duration' in video && video.duration > 0) {
       needsUpdate = true
       libraryItem.wordCount = calculateWordCount(video.duration)
+      duration = video.duration
+    }
+
+    if ('getTranscript' in video && duration > 0 && duration < 1801) {
+      // If the video has a transcript available, put a placehold in and
+      // enqueue a job to process the full transcript
+      const updatedContent = await addTranscriptPlaceholdReadableContent(
+        libraryItem.originalUrl,
+        libraryItem.originalContent
+      )
+
+      if (updatedContent) {
+        needsUpdate = true
+        libraryItem.readableContent = updatedContent
+      }
+
+      await enqueueProcessYouTubeTranscript({
+        videoId,
+        ...jobData,
+      })
+    }
+
+    if (needsUpdate) {
+      const updated = await authTrx(
+        async (t) => {
+          return t
+            .getRepository(LibraryItem)
+            .update(jobData.libraryItemId, libraryItem)
+        },
+        undefined,
+        jobData.userId
+      )
+      if (!updated) {
+        console.warn('could not updated library item')
+      }
+    }
+  } catch (err) {
+    console.warn('error creating summary: ', err)
+  }
+}
+
+export interface ProcessYouTubeTranscriptJobData {
+  userId: string
+  videoId: string
+  libraryItemId: string
+}
+
+export const processYouTubeTranscript = async (
+  jobData: ProcessYouTubeTranscriptJobData
+) => {
+  try {
+    const libraryItem = await authTrx(
+      async (tx) =>
+        tx
+          .withRepository(libraryItemRepository)
+          .findById(jobData.libraryItemId),
+      undefined,
+      jobData.userId
+    )
+    if (
+      !libraryItem ||
+      libraryItem.state !== LibraryItemState.Succeeded ||
+      !libraryItem.originalContent
+    ) {
+      logger.info(
+        `Not ready to get YouTube metadata job state: ${
+          libraryItem?.state ?? 'null'
+        }`
+      )
+      return
+    }
+
+    let needsUpdate = false
+    const youtube = new YouTubeClient()
+    const video = await youtube.getVideo(jobData.videoId)
+    if (!video) {
+      logger.warn('no video found for youtube url', {
+        url: libraryItem.originalUrl,
+      })
+      return
     }
 
     let chapters: Chapter[] = []
diff --git a/packages/api/src/queue-processor.ts b/packages/api/src/queue-processor.ts
index ed7940879..36bea979b 100644
--- a/packages/api/src/queue-processor.ts
+++ b/packages/api/src/queue-processor.ts
@@ -45,7 +45,9 @@ import { CACHED_READING_POSITION_PREFIX } from './services/cached_reading_positi
 import { getJobPriority } from './utils/createTask'
 import { logger } from './utils/logger'
 import {
+  PROCESS_YOUTUBE_TRANSCRIPT_JOB_NAME,
   PROCESS_YOUTUBE_VIDEO_JOB_NAME,
+  processYouTubeTranscript,
   processYouTubeVideo,
 } from './jobs/process-youtube-video'
 
@@ -122,8 +124,12 @@ export const createWorker = (connection: ConnectionOptions) =>
           return aiSummarize(job.data)
         case PROCESS_YOUTUBE_VIDEO_JOB_NAME:
           return processYouTubeVideo(job.data)
+        case PROCESS_YOUTUBE_TRANSCRIPT_JOB_NAME:
+          return processYouTubeTranscript(job.data)
         case EXPORT_ALL_ITEMS_JOB_NAME:
           return exportAllItems(job.data)
+        default:
+          logger.warn(`[queue-processor] unhandled job: ${job.name}`)
       }
     },
     {
diff --git a/packages/api/src/utils/createTask.ts b/packages/api/src/utils/createTask.ts
index 2f5060f72..6d6c30baf 100644
--- a/packages/api/src/utils/createTask.ts
+++ b/packages/api/src/utils/createTask.ts
@@ -46,7 +46,9 @@ import { logger } from './logger'
 import View = google.cloud.tasks.v2.Task.View
 import { AISummarizeJobData, AI_SUMMARIZE_JOB_NAME } from '../jobs/ai-summarize'
 import {
+  PROCESS_YOUTUBE_TRANSCRIPT_JOB_NAME,
   PROCESS_YOUTUBE_VIDEO_JOB_NAME,
+  ProcessYouTubeTranscriptJobData,
   ProcessYouTubeVideoJobData,
 } from '../jobs/process-youtube-video'
 
@@ -71,10 +73,13 @@ export const getJobPriority = (jobName: string): number => {
     case TRIGGER_RULE_JOB_NAME:
     case CALL_WEBHOOK_JOB_NAME:
     case AI_SUMMARIZE_JOB_NAME:
+    case PROCESS_YOUTUBE_VIDEO_JOB_NAME:
       return 5
     case BULK_ACTION_JOB_NAME:
     case `${REFRESH_FEED_JOB_NAME}_high`:
       return 10
+    case PROCESS_YOUTUBE_TRANSCRIPT_JOB_NAME:
+      return 20
     case `${REFRESH_FEED_JOB_NAME}_low`:
     case EXPORT_ITEM_JOB_NAME:
       return 50
@@ -82,8 +87,7 @@ export const getJobPriority = (jobName: string): number => {
     case REFRESH_ALL_FEEDS_JOB_NAME:
     case THUMBNAIL_JOB:
       return 100
-    case PROCESS_YOUTUBE_VIDEO_JOB_NAME:
-      return 20
+
     default:
       logger.error(`unknown job name: ${jobName}`)
       return 1
@@ -729,6 +733,21 @@ export const enqueueProcessYouTubeVideo = async (
   })
 }
 
+export const enqueueProcessYouTubeTranscript = async (
+  data: ProcessYouTubeTranscriptJobData
+) => {
+  const queue = await getBackendQueue()
+  if (!queue) {
+    return undefined
+  }
+
+  return queue.add(PROCESS_YOUTUBE_TRANSCRIPT_JOB_NAME, data, {
+    priority: getJobPriority(PROCESS_YOUTUBE_TRANSCRIPT_JOB_NAME),
+    attempts: 3,
+    delay: 2000,
+  })
+}
+
 export const bulkEnqueueUpdateLabels = async (data: UpdateLabelsData[]) => {
   const queue = await getBackendQueue()
   if (!queue) {

From dc6c047aec8b532ce371853064583aaec59aa12b Mon Sep 17 00:00:00 2001
From: Jackson Harper <jacksonh@gmail.com>
Date: Thu, 14 Mar 2024 16:12:06 +0800
Subject: [PATCH 46/67] Add GCS cache

---
 .../api/src/jobs/process-youtube-video.ts     | 152 +++++++++++++++++-
 1 file changed, 150 insertions(+), 2 deletions(-)

diff --git a/packages/api/src/jobs/process-youtube-video.ts b/packages/api/src/jobs/process-youtube-video.ts
index 76bacf23f..7c430fa86 100644
--- a/packages/api/src/jobs/process-youtube-video.ts
+++ b/packages/api/src/jobs/process-youtube-video.ts
@@ -10,6 +10,11 @@ import { parsePreparedContent } from '../utils/parser'
 import { OpenAI } from '@langchain/openai'
 import { PromptTemplate } from '@langchain/core/prompts'
 import { enqueueProcessYouTubeTranscript } from '../utils/createTask'
+import { env } from '../env'
+import * as stream from 'stream'
+
+import { Storage } from '@google-cloud/storage'
+import { stringToHash } from '../utils/helpers'
 
 export interface ProcessYouTubeVideoJobData {
   userId: string
@@ -68,11 +73,29 @@ export const addTranscriptChapters = (
   return transcript
 }
 
+const createTranscriptHash = (transcript: TranscriptProperties[]): string => {
+  const rawTranscript = transcript.map((item) => item.text).join(' ')
+  return stringToHash(rawTranscript)
+}
+
 export const createTranscriptHTML = async (
+  videoId: string,
   transcript: TranscriptProperties[]
 ): Promise<string> => {
   let transcriptMarkdown = ''
+  const transcriptHash = createTranscriptHash(transcript)
+  const promptHash = stringToHash(process.env.YOUTUBE_TRANSCRIPT_PROMPT ?? '')
+
   if (process.env.YOUTUBE_TRANSCRIPT_PROMPT && process.env.OPENAI_API_KEY) {
+    const cachedTranscriptHTML = await fetchCachedYouTubeTranscript(
+      videoId,
+      transcriptHash,
+      promptHash
+    )
+    if (cachedTranscriptHTML) {
+      return cachedTranscriptHTML
+    }
+
     const llm = new OpenAI({
       modelName: 'gpt-4',
       configuration: {
@@ -121,7 +144,18 @@ export const createTranscriptHTML = async (
   const converter = new showdown.Converter({
     backslashEscapesHTMLTags: true,
   })
-  return converter.makeHtml(transcriptMarkdown)
+  const transcriptHTML = converter.makeHtml(transcriptMarkdown)
+
+  if (process.env.YOUTUBE_TRANSCRIPT_PROMPT && process.env.OPENAI_API_KEY) {
+    await cacheYouTubeTranscript(
+      videoId,
+      transcriptHash,
+      promptHash,
+      transcriptHTML
+    )
+  }
+
+  return transcriptHTML
 }
 
 export const addTranscriptToReadableContent = async (
@@ -185,6 +219,117 @@ export const addTranscriptPlaceholdReadableContent = async (
   return updatedContent.parsedContent?.content
 }
 
+async function readStringFromStorage(
+  bucketName: string,
+  fileName: string
+): Promise<string> {
+  try {
+    const storage = env.fileUpload?.gcsUploadSAKeyFilePath
+      ? new Storage({ keyFilename: env.fileUpload.gcsUploadSAKeyFilePath })
+      : new Storage()
+
+    const existsResponse = await storage
+      .bucket(bucketName)
+      .file(fileName)
+      .exists()
+    const exists = existsResponse[0]
+
+    if (!exists) {
+      throw new Error(
+        `File '${fileName}' does not exist in bucket '${bucketName}'.`
+      )
+    }
+
+    // Download the file contents as a string
+    const fileContentResponse = await storage
+      .bucket(bucketName)
+      .file(fileName)
+      .download()
+    const fileContent = fileContentResponse[0].toString()
+
+    console.log(`File '${fileName}' downloaded successfully as string.`)
+    return fileContent
+  } catch (error) {
+    console.error('Error downloading file:', error)
+    throw error
+  }
+}
+
+const writeStringToStorage = async (
+  bucketName: string,
+  fileName: string,
+  content: string
+): Promise<void> => {
+  try {
+    const storage = env.fileUpload?.gcsUploadSAKeyFilePath
+      ? new Storage({ keyFilename: env.fileUpload.gcsUploadSAKeyFilePath })
+      : new Storage()
+
+    const writableStream = storage
+      .bucket(bucketName)
+      .file(fileName)
+      .createWriteStream()
+
+    // Convert the string content to a readable stream
+    const readableStream = new stream.Readable()
+    readableStream.push(content)
+    readableStream.push(null) // Signal the end of the stream
+
+    // Pipe the readable stream to the writable stream to upload the file content
+    await new Promise((resolve, reject) => {
+      readableStream
+        .pipe(writableStream)
+        .on('finish', resolve)
+        .on('error', reject)
+    })
+
+    console.log(
+      `File '${fileName}' uploaded successfully to bucket '${bucketName}'.`
+    )
+  } catch (error) {
+    console.error('Error uploading file:', error)
+    throw error
+  }
+}
+
+const fetchCachedYouTubeTranscript = async (
+  videoId: string,
+  transcriptHash: string,
+  promptHash: string
+): Promise<string | undefined> => {
+  const bucketName = env.fileUpload.gcsUploadBucket
+
+  try {
+    return await readStringFromStorage(
+      bucketName,
+      `youtube-transcripts/${videoId}/${transcriptHash}.${promptHash}.html`
+    )
+  } catch (err) {
+    logger.info(`unable to fetch cached transcript: ${err}`)
+  }
+
+  return undefined
+}
+
+const cacheYouTubeTranscript = async (
+  videoId: string,
+  transcriptHash: string,
+  promptHash: string,
+  transcript: string
+): Promise<void> => {
+  const bucketName = env.fileUpload.gcsUploadBucket
+
+  try {
+    await writeStringToStorage(
+      bucketName,
+      `youtube-transcripts/${videoId}/${transcriptHash}.${promptHash}.html`,
+      transcript
+    )
+  } catch (err) {
+    logger.info(`unable to cache transcript: ${err}`)
+  }
+}
+
 export const processYouTubeVideo = async (
   jobData: ProcessYouTubeVideoJobData
 ) => {
@@ -335,7 +480,10 @@ export const processYouTubeTranscript = async (
       if (chapters) {
         transcript = addTranscriptChapters(chapters, transcript)
       }
-      const transcriptHTML = await createTranscriptHTML(transcript)
+      const transcriptHTML = await createTranscriptHTML(
+        jobData.videoId,
+        transcript
+      )
       const updatedContent = await addTranscriptToReadableContent(
         libraryItem.originalUrl,
         libraryItem.originalContent,

From f2d23626a51f22f8b5981942e51106117efdc497 Mon Sep 17 00:00:00 2001
From: Jackson Harper <jacksonh@gmail.com>
Date: Thu, 14 Mar 2024 17:39:13 +0800
Subject: [PATCH 47/67] Improve error logging / fix linting

---
 packages/api/src/jobs/process-youtube-video.ts | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/packages/api/src/jobs/process-youtube-video.ts b/packages/api/src/jobs/process-youtube-video.ts
index 7c430fa86..b4e5f49e7 100644
--- a/packages/api/src/jobs/process-youtube-video.ts
+++ b/packages/api/src/jobs/process-youtube-video.ts
@@ -305,7 +305,7 @@ const fetchCachedYouTubeTranscript = async (
       `youtube-transcripts/${videoId}/${transcriptHash}.${promptHash}.html`
     )
   } catch (err) {
-    logger.info(`unable to fetch cached transcript: ${err}`)
+    logger.info(`unable to fetch cached transcript`, { error: err })
   }
 
   return undefined
@@ -326,7 +326,7 @@ const cacheYouTubeTranscript = async (
       transcript
     )
   } catch (err) {
-    logger.info(`unable to cache transcript: ${err}`)
+    logger.info(`unable to cache transcript`, { error: err })
   }
 }
 

From 9618e452bc6ca4a584a1e6dc3042ec641d2dd6c9 Mon Sep 17 00:00:00 2001
From: Jackson Harper <jacksonh@gmail.com>
Date: Thu, 14 Mar 2024 18:09:38 +0800
Subject: [PATCH 48/67] Fix some issues with unsubscribing on web

---
 packages/web/components/patterns/CardMenu.tsx |  4 +-
 .../components/patterns/ConfirmationModal.tsx |  3 +-
 .../mutations/unsubscribeMutation.ts          | 14 ++++---
 .../queries/useGetLibraryItemsQuery.tsx       | 42 +++++++++----------
 4 files changed, 34 insertions(+), 29 deletions(-)

diff --git a/packages/web/components/patterns/CardMenu.tsx b/packages/web/components/patterns/CardMenu.tsx
index 6a3c27dc0..23fb6e7bc 100644
--- a/packages/web/components/patterns/CardMenu.tsx
+++ b/packages/web/components/patterns/CardMenu.tsx
@@ -81,14 +81,14 @@ export function CardMenu(props: CardMenuProps): JSX.Element {
         }}
         title="Remove"
       />
-      {!!props.item.subscription && (
+      {/* {!!props.item.subscription && (
         <DropdownOption
           onSelect={() => {
             props.actionHandler('unsubscribe')
           }}
           title="Unsubscribe"
         />
-      )}
+      )} */}
     </Dropdown>
   )
 }
diff --git a/packages/web/components/patterns/ConfirmationModal.tsx b/packages/web/components/patterns/ConfirmationModal.tsx
index 25618fdbe..6e78b9566 100644
--- a/packages/web/components/patterns/ConfirmationModal.tsx
+++ b/packages/web/components/patterns/ConfirmationModal.tsx
@@ -21,10 +21,11 @@ type ConfirmationModalProps = {
 export function ConfirmationModal(props: ConfirmationModalProps): JSX.Element {
   const safeOnOpenChange = useCallback(
     (open: boolean) => {
-      props.onOpenChange(open)
       setTimeout(() => {
+        console.log('body style: ', document.body.style)
         document.body.style.removeProperty('pointer-events')
       }, 200)
+      props.onOpenChange(open)
     },
     [props]
   )
diff --git a/packages/web/lib/networking/mutations/unsubscribeMutation.ts b/packages/web/lib/networking/mutations/unsubscribeMutation.ts
index 24e2505e1..c1bcbd49f 100644
--- a/packages/web/lib/networking/mutations/unsubscribeMutation.ts
+++ b/packages/web/lib/networking/mutations/unsubscribeMutation.ts
@@ -12,12 +12,12 @@ type Unsubscribe = {
 }
 
 export async function unsubscribeMutation(
-  subscribeName: string,
-  id = ''
+  subscribtionName: string,
+  id: string
 ): Promise<any | undefined> {
   const mutation = gql`
-    mutation {
-      unsubscribe(name: "${subscribeName}", subscriptionId: "${id}") {
+    mutation Unsubscribe($subscribtionName: String!, $subscriptionId: ID!) {
+      unsubscribe(name: $subscribtionName, subscriptionId: $subscriptionId) {
         ... on UnsubscribeSuccess {
           subscription {
             id
@@ -31,7 +31,11 @@ export async function unsubscribeMutation(
   `
 
   try {
-    const data = (await gqlFetcher(mutation)) as UnsubscribeResult
+    const data = (await gqlFetcher(mutation, {
+      subscriptionId: id,
+      subscribtionName: subscribtionName,
+    })) as UnsubscribeResult
+
     return data.unsubscribe.errorCodes
       ? undefined
       : data.unsubscribe.subscription.id
diff --git a/packages/web/lib/networking/queries/useGetLibraryItemsQuery.tsx b/packages/web/lib/networking/queries/useGetLibraryItemsQuery.tsx
index f7fcad0d5..3434df3dc 100644
--- a/packages/web/lib/networking/queries/useGetLibraryItemsQuery.tsx
+++ b/packages/web/lib/networking/queries/useGetLibraryItemsQuery.tsx
@@ -413,27 +413,27 @@ export function useGetLibraryItemsQuery({
           readingProgressAnchorIndex: 0,
         })
         break
-      case 'unsubscribe':
-        if (!!item.node.subscription) {
-          updateData({
-            cursor: item.cursor,
-            node: {
-              ...item.node,
-              subscription: undefined,
-            },
-          })
-          unsubscribeMutation(item.node.subscription).then((res) => {
-            if (res) {
-              showSuccessToast('Unsubscribed successfully', {
-                position: 'bottom-right',
-              })
-            } else {
-              showErrorToast('Error unsubscribing', {
-                position: 'bottom-right',
-              })
-            }
-          })
-        }
+      // case 'unsubscribe':
+      //   if (!!item.node.subscription) {
+      //     updateData({
+      //       cursor: item.cursor,
+      //       node: {
+      //         ...item.node,
+      //         subscription: undefined,
+      //       },
+      //     })
+      //     unsubscribeMutation(item.node.subscription).then((res) => {
+      //       if (res) {
+      //         showSuccessToast('Unsubscribed successfully', {
+      //           position: 'bottom-right',
+      //         })
+      //       } else {
+      //         showErrorToast('Error unsubscribing', {
+      //           position: 'bottom-right',
+      //         })
+      //       }
+      //     })
+      //   }
       case 'update-item':
         updateData(item)
         break

From 69e74d432dde9492e34b7466ba76c6a763b23752 Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Thu, 14 Mar 2024 19:50:26 +0800
Subject: [PATCH 49/67] append highlights to the existing page

---
 .../api/src/services/integrations/notion.ts   | 220 +++++++++++-------
 1 file changed, 133 insertions(+), 87 deletions(-)

diff --git a/packages/api/src/services/integrations/notion.ts b/packages/api/src/services/integrations/notion.ts
index 4a21d5606..f60b4e34b 100644
--- a/packages/api/src/services/integrations/notion.ts
+++ b/packages/api/src/services/integrations/notion.ts
@@ -61,10 +61,10 @@ interface NotionPage {
       }>
     }
     'Original URL': {
-      url: string | null
+      url: string
     }
     'Omnivore URL': {
-      url: string | null
+      url: string
     }
     Tags?: {
       multi_select: Array<{ name: string }>
@@ -158,89 +158,106 @@ export class NotionClient implements IntegrationClient {
     return Promise.resolve(env.notion.authUrl)
   }
 
-  private _itemToNotionPage = (item: LibraryItem): NotionPage => {
-    const databaseId = this._integrationData?.settings?.parentDatabaseId
-    if (!databaseId) {
-      throw new Error('Notion database id not found')
-    }
+  private itemToNotionPage = (
+    item: LibraryItem,
+    databaseId: string
+  ): NotionPage => ({
+    parent: {
+      database_id: databaseId,
+    },
+    icon: item.siteIcon
+      ? {
+          external: {
+            url: item.siteIcon,
+          },
+        }
+      : undefined,
+    cover: item.thumbnail
+      ? {
+          external: {
+            url: item.thumbnail,
+          },
+        }
+      : undefined,
+    properties: {
+      Title: {
+        title: [
+          {
+            text: {
+              content: item.title,
+            },
+          },
+        ],
+      },
+      Author: {
+        rich_text: [
+          {
+            text: {
+              content: item.author || 'unknown',
+            },
+          },
+        ],
+      },
+      'Original URL': {
+        url: item.originalUrl,
+      },
+      'Omnivore URL': {
+        url: `${env.client.url}/me/${item.slug}`,
+      },
+      Tags: item.labels
+        ? { multi_select: item.labels.map((label) => ({ name: label.name })) }
+        : undefined,
+    },
+    children: item.highlights
+      ? item.highlights.map((highlight) => ({
+          type: 'paragraph',
+          paragraph: {
+            rich_text: [
+              {
+                text: {
+                  content: highlight.quote || '',
+                  link: {
+                    url: `${env.client.url}/me/${item.slug}#${highlight.id}`,
+                  },
+                },
+                annotations: {
+                  color: highlight.color as AnnotationColor,
+                },
+              },
+              {
+                text: {
+                  content: `\n${highlight.annotation || ''}`,
+                },
+                annotations: {
+                  italic: true,
+                },
+              },
+            ],
+          },
+        }))
+      : undefined,
+  })
 
-    return {
-      parent: {
-        database_id: databaseId,
-      },
-      icon: item.siteIcon
-        ? {
-            external: {
-              url: item.siteIcon,
-            },
-          }
-        : undefined,
-      cover: item.thumbnail
-        ? {
-            external: {
-              url: item.thumbnail,
-            },
-          }
-        : undefined,
-      properties: {
-        Title: {
-          title: [
-            {
-              text: {
-                content: item.title,
-              },
-            },
-          ],
-        },
-        Author: {
-          rich_text: [
-            {
-              text: {
-                content: item.author || 'unknown',
-              },
-            },
-          ],
-        },
-        'Original URL': {
-          url: item.originalUrl,
-        },
-        'Omnivore URL': {
-          url: `${env.client.url}/me/${item.slug}`,
-        },
-        Tags: item.labels
-          ? { multi_select: item.labels.map((label) => ({ name: label.name })) }
-          : undefined,
-      },
-      children: item.highlights
-        ? item.highlights.map((highlight) => ({
-            type: 'paragraph',
-            paragraph: {
-              rich_text: [
-                {
-                  text: {
-                    content: highlight.quote || '',
-                  },
-                  annotations: {
-                    color: highlight.color as AnnotationColor,
-                  },
-                },
-                {
-                  text: {
-                    content: `\n${highlight.annotation || ''}`,
-                  },
-                  annotations: {
-                    italic: true,
-                  },
-                },
-              ],
-            },
-          }))
-        : undefined,
-    }
+  private createPage = async (page: NotionPage) => {
+    await this._client.pages.create(page)
   }
 
-  _createPage = async (page: NotionPage) => {
-    await this._client.pages.create(page)
+  private findPage = async (url: string, databaseId: string) => {
+    const response = await this._client.databases.query({
+      database_id: databaseId,
+      page_size: 1,
+      filter: {
+        property: 'Omnivore URL',
+        url: {
+          equals: url,
+        },
+      },
+    })
+    if (response.results.length > 0) {
+      return response.results[0]
+    }
+
+    return null
   }
 
   export = async (items: LibraryItem[]): Promise<boolean> => {
@@ -255,7 +272,7 @@ export class NotionClient implements IntegrationClient {
       return false
     }
 
-    const databaseId = this._integrationData.settings.parentDatabaseId
+    let databaseId = this._integrationData.settings.parentDatabaseId
     if (!databaseId) {
       // create a database for the items
       const database = await this._client.databases.create({
@@ -296,18 +313,47 @@ export class NotionClient implements IntegrationClient {
       })
 
       // save the database id
-      this._integrationData.settings.parentDatabaseId = database.id
+      databaseId = database.id
       await updateIntegration(
         this._integrationData.id,
         {
-          settings: this._integrationData.settings,
+          settings: {
+            ...this._integrationData.settings,
+            parentDatabaseId: databaseId,
+          },
         },
         this._integrationData.user.id
       )
     }
 
-    const pages = items.map(this._itemToNotionPage)
-    await Promise.all(pages.map((page) => this._createPage(page)))
+    await Promise.all(
+      items.map(async (item) => {
+        const notionPage = this.itemToNotionPage(item, databaseId)
+        const url = notionPage.properties['Omnivore URL'].url
+
+        const existingPage = await this.findPage(url, databaseId)
+        if (existingPage) {
+          // update the page
+          await this._client.pages.update({
+            page_id: existingPage.id,
+            properties: notionPage.properties,
+          })
+
+          // append the children
+          if (notionPage.children) {
+            await this._client.blocks.children.append({
+              block_id: existingPage.id,
+              children: notionPage.children,
+            })
+          }
+
+          return
+        }
+
+        // create the page
+        return this.createPage(notionPage)
+      })
+    )
 
     return true
   }

From 1085dcc8241303fd6303014a097249aab5c583be Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Thu, 14 Mar 2024 21:36:47 +0800
Subject: [PATCH 50/67] read settings

---
 .../src/services/integrations/integration.ts  |   2 +-
 .../api/src/services/integrations/notion.ts   | 213 ++++++++++--------
 .../api/src/services/integrations/pocket.ts   |   6 +-
 .../api/src/services/integrations/readwise.ts |  11 +-
 4 files changed, 125 insertions(+), 107 deletions(-)

diff --git a/packages/api/src/services/integrations/integration.ts b/packages/api/src/services/integrations/integration.ts
index 13b61e82a..44c95e680 100644
--- a/packages/api/src/services/integrations/integration.ts
+++ b/packages/api/src/services/integrations/integration.ts
@@ -20,7 +20,7 @@ export interface RetrieveRequest {
 
 export interface IntegrationClient {
   name: string
-  _token: string
+  token: string
 
   accessToken(): Promise<string | null>
 
diff --git a/packages/api/src/services/integrations/notion.ts b/packages/api/src/services/integrations/notion.ts
index f60b4e34b..21770fdc4 100644
--- a/packages/api/src/services/integrations/notion.ts
+++ b/packages/api/src/services/integrations/notion.ts
@@ -5,6 +5,7 @@ import { Integration } from '../../entity/integration'
 import { LibraryItem } from '../../entity/library_item'
 import { env } from '../../env'
 import { Merge } from '../../util'
+import { highlightUrl } from '../../utils/helpers'
 import { logger } from '../../utils/logger'
 import { IntegrationClient } from './integration'
 
@@ -91,35 +92,39 @@ interface NotionPage {
   }>
 }
 
+type Property = 'highlights' | 'labels' | 'notes'
+
 interface Settings {
   parentPageId: string
   parentDatabaseId: string
+  properties: Property[]
 }
 
 export class NotionClient implements IntegrationClient {
   name = 'NOTION'
-  _headers = {
+  token: string
+
+  private headers = {
     'Content-Type': 'application/json',
     Accept: 'application/json',
     'Notion-Version': '2022-06-28',
   }
-  _timeout = 5000 // 5 seconds
-  _axios = axios.create({
+  private timeout = 5000 // 5 seconds
+  private axiosInstance = axios.create({
     baseURL: 'https://api.notion.com/v1',
-    timeout: this._timeout,
+    timeout: this.timeout,
   })
 
-  _token: string
-  _client: Client
-  _integrationData?: Merge<Integration, { settings?: Settings }>
+  private client: Client
+  private integrationData?: Merge<Integration, { settings?: Settings }>
 
   constructor(token: string, integration?: Integration) {
-    this._token = token
-    this._client = new Client({
+    this.token = token
+    this.client = new Client({
       auth: token,
-      timeoutMs: this._timeout,
+      timeoutMs: this.timeout,
     })
-    this._integrationData = integration
+    this.integrationData = integration
   }
 
   accessToken = async (): Promise<string | null> => {
@@ -129,16 +134,16 @@ export class NotionClient implements IntegrationClient {
         `${env.notion.clientId}:${env.notion.clientSecret}`
       ).toString('base64')
 
-      const response = await this._axios.post<{ access_token: string }>(
+      const response = await this.axiosInstance.post<{ access_token: string }>(
         '/oauth/token',
         {
           grant_type: 'authorization_code',
-          code: this._token,
+          code: this.token,
           redirect_uri: `${env.client.url}/settings/integrations`,
         },
         {
           headers: {
-            ...this._headers,
+            ...this.headers,
             Authorization: `Basic ${encoded}`,
           },
         }
@@ -160,90 +165,101 @@ export class NotionClient implements IntegrationClient {
 
   private itemToNotionPage = (
     item: LibraryItem,
-    databaseId: string
-  ): NotionPage => ({
-    parent: {
-      database_id: databaseId,
-    },
-    icon: item.siteIcon
-      ? {
-          external: {
-            url: item.siteIcon,
-          },
-        }
-      : undefined,
-    cover: item.thumbnail
-      ? {
-          external: {
-            url: item.thumbnail,
-          },
-        }
-      : undefined,
-    properties: {
-      Title: {
-        title: [
-          {
-            text: {
-              content: item.title,
+    settings: Settings
+  ): NotionPage => {
+    return {
+      parent: {
+        database_id: settings.parentDatabaseId,
+      },
+      icon: item.siteIcon
+        ? {
+            external: {
+              url: item.siteIcon,
             },
-          },
-        ],
-      },
-      Author: {
-        rich_text: [
-          {
-            text: {
-              content: item.author || 'unknown',
-            },
-          },
-        ],
-      },
-      'Original URL': {
-        url: item.originalUrl,
-      },
-      'Omnivore URL': {
-        url: `${env.client.url}/me/${item.slug}`,
-      },
-      Tags: item.labels
-        ? { multi_select: item.labels.map((label) => ({ name: label.name })) }
+          }
         : undefined,
-    },
-    children: item.highlights
-      ? item.highlights.map((highlight) => ({
-          type: 'paragraph',
-          paragraph: {
-            rich_text: [
-              {
-                text: {
-                  content: highlight.quote || '',
-                  link: {
-                    url: `${env.client.url}/me/${item.slug}#${highlight.id}`,
+      cover: item.thumbnail
+        ? {
+            external: {
+              url: item.thumbnail,
+            },
+          }
+        : undefined,
+      properties: {
+        Title: {
+          title: [
+            {
+              text: {
+                content: item.title,
+              },
+            },
+          ],
+        },
+        Author: {
+          rich_text: [
+            {
+              text: {
+                content: item.author || 'unknown',
+              },
+            },
+          ],
+        },
+        'Original URL': {
+          url: item.originalUrl,
+        },
+        'Omnivore URL': {
+          url: `${env.client.url}/me/${item.slug}`,
+        },
+        Tags:
+          item.labels && settings.properties.includes('labels')
+            ? {
+                multi_select: item.labels.map((label) => ({
+                  name: label.name,
+                })),
+              }
+            : undefined,
+      },
+      children: item.highlights
+        ? item.highlights.map((highlight) => ({
+            type: 'paragraph',
+            paragraph: {
+              rich_text: [
+                {
+                  text: {
+                    content: settings.properties.includes('highlights')
+                      ? highlight.quote || ''
+                      : '',
+                    link: {
+                      url: highlightUrl(item.slug, highlight.id),
+                    },
+                  },
+                  annotations: {
+                    color: highlight.color as AnnotationColor,
                   },
                 },
-                annotations: {
-                  color: highlight.color as AnnotationColor,
+                {
+                  text: {
+                    content: settings.properties.includes('notes')
+                      ? `\n${highlight.annotation || ''}`
+                      : '',
+                  },
+                  annotations: {
+                    italic: true,
+                  },
                 },
-              },
-              {
-                text: {
-                  content: `\n${highlight.annotation || ''}`,
-                },
-                annotations: {
-                  italic: true,
-                },
-              },
-            ],
-          },
-        }))
-      : undefined,
-  })
+              ],
+            },
+          }))
+        : undefined,
+    }
+  }
 
   private createPage = async (page: NotionPage) => {
-    await this._client.pages.create(page)
+    await this.client.pages.create(page)
   }
 
   private findPage = async (url: string, databaseId: string) => {
-    const response = await this._client.databases.query({
+    const response = await this.client.databases.query({
       database_id: databaseId,
       page_size: 1,
       filter: {
@@ -261,21 +277,22 @@ export class NotionClient implements IntegrationClient {
   }
 
   export = async (items: LibraryItem[]): Promise<boolean> => {
-    if (!this._integrationData || !this._integrationData.settings) {
+    const settings = this.integrationData?.settings
+    if (!this.integrationData || !settings) {
       logger.error('Notion integration data not found')
       return false
     }
 
-    const pageId = this._integrationData.settings.parentPageId
+    const pageId = settings.parentPageId
     if (!pageId) {
       logger.error('Notion parent page id not found')
       return false
     }
 
-    let databaseId = this._integrationData.settings.parentDatabaseId
+    let databaseId = settings.parentDatabaseId
     if (!databaseId) {
       // create a database for the items
-      const database = await this._client.databases.create({
+      const database = await this.client.databases.create({
         parent: {
           page_id: pageId,
         },
@@ -315,33 +332,33 @@ export class NotionClient implements IntegrationClient {
       // save the database id
       databaseId = database.id
       await updateIntegration(
-        this._integrationData.id,
+        this.integrationData.id,
         {
           settings: {
-            ...this._integrationData.settings,
+            ...this.integrationData.settings,
             parentDatabaseId: databaseId,
           },
         },
-        this._integrationData.user.id
+        this.integrationData.user.id
       )
     }
 
     await Promise.all(
       items.map(async (item) => {
-        const notionPage = this.itemToNotionPage(item, databaseId)
+        const notionPage = this.itemToNotionPage(item, settings)
         const url = notionPage.properties['Omnivore URL'].url
 
         const existingPage = await this.findPage(url, databaseId)
         if (existingPage) {
           // update the page
-          await this._client.pages.update({
+          await this.client.pages.update({
             page_id: existingPage.id,
             properties: notionPage.properties,
           })
 
           // append the children
           if (notionPage.children) {
-            await this._client.blocks.children.append({
+            await this.client.blocks.children.append({
               block_id: existingPage.id,
               children: notionPage.children,
             })
diff --git a/packages/api/src/services/integrations/pocket.ts b/packages/api/src/services/integrations/pocket.ts
index 20f4c212c..9945c7a6e 100644
--- a/packages/api/src/services/integrations/pocket.ts
+++ b/packages/api/src/services/integrations/pocket.ts
@@ -5,7 +5,7 @@ import { IntegrationClient } from './integration'
 
 export class PocketClient implements IntegrationClient {
   name = 'POCKET'
-  _token: string
+  token: string
   _axios = axios.create({
     baseURL: 'https://getpocket.com/v3',
     headers: {
@@ -16,7 +16,7 @@ export class PocketClient implements IntegrationClient {
   })
 
   constructor(token: string) {
-    this._token = token
+    this.token = token
   }
 
   accessToken = async (): Promise<string | null> => {
@@ -25,7 +25,7 @@ export class PocketClient implements IntegrationClient {
         '/oauth/authorize',
         {
           consumer_key: env.pocket.consumerKey,
-          code: this._token,
+          code: this.token,
         }
       )
       return response.data.access_token
diff --git a/packages/api/src/services/integrations/readwise.ts b/packages/api/src/services/integrations/readwise.ts
index 25a61f438..dfc5b43db 100644
--- a/packages/api/src/services/integrations/readwise.ts
+++ b/packages/api/src/services/integrations/readwise.ts
@@ -33,6 +33,8 @@ interface ReadwiseHighlight {
 
 export class ReadwiseClient implements IntegrationClient {
   name = 'READWISE'
+  token: string
+
   _headers = {
     'Content-Type': 'application/json',
   }
@@ -40,10 +42,9 @@ export class ReadwiseClient implements IntegrationClient {
     baseURL: 'https://readwise.io/api/v2',
     timeout: 5000, // 5 seconds
   })
-  _token: string
 
   constructor(token: string) {
-    this._token = token
+    this.token = token
   }
 
   accessToken = async (): Promise<string | null> => {
@@ -51,10 +52,10 @@ export class ReadwiseClient implements IntegrationClient {
       const response = await this._axios.get('/auth', {
         headers: {
           ...this._headers,
-          Authorization: `Token ${this._token}`,
+          Authorization: `Token ${this.token}`,
         },
       })
-      return response.status === 204 ? this._token : null
+      return response.status === 204 ? this.token : null
     } catch (error) {
       if (axios.isAxiosError(error)) {
         logger.error(error.response)
@@ -121,7 +122,7 @@ export class ReadwiseClient implements IntegrationClient {
       {
         headers: {
           ...this._headers,
-          Authorization: `Token ${this._token}`,
+          Authorization: `Token ${this.token}`,
         },
       }
     )

From 17e66aa01018b4f91acdbdfe9029e1a673d42904 Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Thu, 14 Mar 2024 22:13:27 +0800
Subject: [PATCH 51/67] append new highlights only

---
 .../api/src/services/integrations/notion.ts   | 28 ++++++++++++++++---
 1 file changed, 24 insertions(+), 4 deletions(-)

diff --git a/packages/api/src/services/integrations/notion.ts b/packages/api/src/services/integrations/notion.ts
index 21770fdc4..a4e8a84e5 100644
--- a/packages/api/src/services/integrations/notion.ts
+++ b/packages/api/src/services/integrations/notion.ts
@@ -356,12 +356,32 @@ export class NotionClient implements IntegrationClient {
             properties: notionPage.properties,
           })
 
-          // append the children
-          if (notionPage.children) {
-            await this.client.blocks.children.append({
+          const children = notionPage.children
+          if (children) {
+            // get the existing children
+            const response = await this.client.blocks.children.list({
               block_id: existingPage.id,
-              children: notionPage.children,
             })
+            if (response.results.length > 0) {
+              const existingChildren =
+                response.results as NotionPage['children']
+              // delete the existing children from children
+              notionPage.children = children.filter(
+                (child) =>
+                  !existingChildren?.some(
+                    (existingChild) =>
+                      existingChild.paragraph.rich_text[0].text.link?.url ===
+                      child.paragraph.rich_text[0].text.link?.url
+                  )
+              )
+            }
+            // append the children
+            if (notionPage.children && notionPage.children.length > 0) {
+              await this.client.blocks.children.append({
+                block_id: existingPage.id,
+                children: notionPage.children,
+              })
+            }
           }
 
           return

From c54cfee3f1c3186292b6061bc106c97e753a29d6 Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Thu, 14 Mar 2024 22:18:18 +0800
Subject: [PATCH 52/67] turn off auto sync will disable notion

---
 packages/web/pages/settings/integrations/notion.tsx | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/packages/web/pages/settings/integrations/notion.tsx b/packages/web/pages/settings/integrations/notion.tsx
index eb56fa892..df72c75be 100644
--- a/packages/web/pages/settings/integrations/notion.tsx
+++ b/packages/web/pages/settings/integrations/notion.tsx
@@ -30,7 +30,7 @@ import { showSuccessToast } from '../../../lib/toastHelpers'
 type FieldType = {
   parentPageId?: string
   parentDatabaseId?: string
-  autoSync?: boolean
+  enabled: boolean
   properties?: string[]
 }
 
@@ -55,7 +55,7 @@ export default function Notion(): JSX.Element {
       form.setFieldsValue({
         parentPageId: notion.settings?.parentPageId,
         parentDatabaseId: notion.settings?.parentDatabaseId,
-        autoSync: notion.settings?.autoSync,
+        enabled: notion.enabled,
         properties: notion.settings?.properties,
       })
     }
@@ -82,7 +82,7 @@ export default function Notion(): JSX.Element {
       name: notion.name,
       type: notion.type,
       token: notion.token,
-      enabled: notion.enabled,
+      enabled: values.enabled,
       settings: values,
     })
   }
@@ -170,7 +170,7 @@ export default function Notion(): JSX.Element {
 
                 <Form.Item<FieldType>
                   label="Automatic Sync"
-                  name="autoSync"
+                  name="enabled"
                   valuePropName="checked"
                 >
                   <Switch />

From 5cbed3f396cc216e83963f9b003bb0b4717e2bbd Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Thu, 14 Mar 2024 22:23:42 +0800
Subject: [PATCH 53/67] disable notion by default

---
 packages/web/pages/settings/integrations.tsx  |  2 +-
 .../pages/settings/integrations/notion.tsx    | 34 +++++++------------
 2 files changed, 14 insertions(+), 22 deletions(-)

diff --git a/packages/web/pages/settings/integrations.tsx b/packages/web/pages/settings/integrations.tsx
index c9f2e3296..39af9ec23 100644
--- a/packages/web/pages/settings/integrations.tsx
+++ b/packages/web/pages/settings/integrations.tsx
@@ -176,7 +176,7 @@ export default function Integrations(): JSX.Element {
           token,
           name: 'NOTION',
           type: 'EXPORT',
-          enabled: true,
+          enabled: false,
         })
 
         showSuccessToast('Connected with Notion.')
diff --git a/packages/web/pages/settings/integrations/notion.tsx b/packages/web/pages/settings/integrations/notion.tsx
index df72c75be..dce6e0a24 100644
--- a/packages/web/pages/settings/integrations/notion.tsx
+++ b/packages/web/pages/settings/integrations/notion.tsx
@@ -12,7 +12,7 @@ import 'antd/dist/antd.compact.css'
 import { CheckboxValueType } from 'antd/lib/checkbox/Group'
 import Image from 'next/image'
 import { useRouter } from 'next/router'
-import { useEffect, useState } from 'react'
+import { useEffect, useMemo } from 'react'
 import { HStack, VStack } from '../../../components/elements/LayoutPrimitives'
 import { PageMetaData } from '../../../components/patterns/PageMetaData'
 import { Beta } from '../../../components/templates/Beta'
@@ -20,10 +20,7 @@ import { Header } from '../../../components/templates/settings/SettingsTable'
 import { SettingsLayout } from '../../../components/templates/SettingsLayout'
 import { deleteIntegrationMutation } from '../../../lib/networking/mutations/deleteIntegrationMutation'
 import { setIntegrationMutation } from '../../../lib/networking/mutations/setIntegrationMutation'
-import {
-  Integration,
-  useGetIntegrationsQuery,
-} from '../../../lib/networking/queries/useGetIntegrationsQuery'
+import { useGetIntegrationsQuery } from '../../../lib/networking/queries/useGetIntegrationsQuery'
 import { applyStoredTheme } from '../../../lib/themeUpdater'
 import { showSuccessToast } from '../../../lib/toastHelpers'
 
@@ -39,27 +36,21 @@ export default function Notion(): JSX.Element {
 
   const router = useRouter()
   const { integrations, revalidate } = useGetIntegrationsQuery()
-  const [notion, setNotion] = useState<Integration>()
+  const notion = useMemo(() => {
+    return integrations.find((i) => i.name == 'NOTION' && i.type == 'EXPORT')
+  }, [integrations])
 
   const [form] = Form.useForm<FieldType>()
   const [messageApi, contextHolder] = message.useMessage()
 
   useEffect(() => {
-    const notion = integrations.find(
-      (i) => i.name == 'NOTION' && i.type == 'EXPORT'
-    )
-
-    if (notion) {
-      setNotion(notion)
-
-      form.setFieldsValue({
-        parentPageId: notion.settings?.parentPageId,
-        parentDatabaseId: notion.settings?.parentDatabaseId,
-        enabled: notion.enabled,
-        properties: notion.settings?.properties,
-      })
-    }
-  }, [form, integrations])
+    form.setFieldsValue({
+      parentPageId: notion?.settings?.parentPageId,
+      parentDatabaseId: notion?.settings?.parentDatabaseId,
+      enabled: notion?.enabled,
+      properties: notion?.settings?.properties,
+    })
+  }, [form, notion])
 
   const deleteNotion = async () => {
     if (!notion) {
@@ -69,6 +60,7 @@ export default function Notion(): JSX.Element {
     await deleteIntegrationMutation(notion.id)
     showSuccessToast('Notion integration disconnected successfully.')
 
+    revalidate()
     router.push('/settings/integrations')
   }
 

From 5df1c1cd3e2bf57ea2f4f4a337d1cdc02bf67d62 Mon Sep 17 00:00:00 2001
From: Jackson Harper <jacksonh@gmail.com>
Date: Fri, 15 Mar 2024 19:59:50 +0800
Subject: [PATCH 54/67] Simplify code, handle crazy people with 12hr time

---
 .../Sources/Views/FeedItem/GridCard.swift     |  6 +++
 .../Views/FeedItem/LibraryItemCard.swift      | 38 +++++++++++++++----
 2 files changed, 37 insertions(+), 7 deletions(-)

diff --git a/apple/OmnivoreKit/Sources/Views/FeedItem/GridCard.swift b/apple/OmnivoreKit/Sources/Views/FeedItem/GridCard.swift
index 15a79b8e5..cd96cbcac 100644
--- a/apple/OmnivoreKit/Sources/Views/FeedItem/GridCard.swift
+++ b/apple/OmnivoreKit/Sources/Views/FeedItem/GridCard.swift
@@ -12,11 +12,13 @@ public enum GridCardAction {
 
 public struct GridCard: View {
   @ObservedObject var item: Models.LibraryItem
+  let savedAtStr: String
 
   public init(
     item: Models.LibraryItem
   ) {
     self.item = item
+    self.savedAtStr = savedDateString(item.savedAt)
   }
 
   var imageBox: some View {
@@ -198,6 +200,10 @@ public struct GridCard: View {
         $0.icon
       }
 
+      Text(savedAtStr)
+        .font(.footnote)
+        .foregroundColor(Color.themeLibraryItemSubtle)
++
       Text("\(estimatedReadingTime)")
         .font(.caption2).fontWeight(.medium)
         .foregroundColor(Color.themeLibraryItemSubtle)
diff --git a/apple/OmnivoreKit/Sources/Views/FeedItem/LibraryItemCard.swift b/apple/OmnivoreKit/Sources/Views/FeedItem/LibraryItemCard.swift
index 261e45dc5..d301f13d6 100644
--- a/apple/OmnivoreKit/Sources/Views/FeedItem/LibraryItemCard.swift
+++ b/apple/OmnivoreKit/Sources/Views/FeedItem/LibraryItemCard.swift
@@ -44,14 +44,33 @@ public extension View {
   }
 }
 
+func savedDateString(_ savedAt: Date?) -> String {
+  if let savedAt = savedAt {
+    let locale = Locale.current
+    let dateFormatter = DateFormatter()
+    if Calendar.current.isDateInToday(savedAt) {
+      dateFormatter.dateStyle = .none
+      dateFormatter.timeStyle = .short
+    } else {
+      dateFormatter.dateFormat = "MMM dd"
+    }
+    dateFormatter.locale = locale
+    return dateFormatter.string(from: savedAt) + " • "
+  }
+  return ""
+}
+
 public struct LibraryItemCard: View {
   let viewer: Viewer?
   @ObservedObject var item: Models.LibraryItem
   @State var noteLineLimit: Int? = 3
 
+  let savedAtStr: String
+
   public init(item: Models.LibraryItem, viewer: Viewer?) {
     self.item = item
     self.viewer = viewer
+    self.savedAtStr = savedDateString(item.savedAt)
   }
 
   public var body: some View {
@@ -215,23 +234,28 @@ public struct LibraryItemCard: View {
         $0.icon
       }
 
+      Text(savedAtStr)
+        .font(.footnote)
+        .foregroundColor(Color.themeLibraryItemSubtle)
+
+      +
       Text("\(estimatedReadingTime)")
-        .font(.caption2).fontWeight(.medium)
+        .font(.footnote)
         .foregroundColor(Color.themeLibraryItemSubtle)
 
         +
         Text("\(readingProgress)")
-        .font(.caption2).fontWeight(.medium)
+        .font(.footnote)
         .foregroundColor(isPartiallyRead ? Color.appGreenSuccess : Color.themeLibraryItemSubtle)
 
         +
         Text("\(highlightsText)")
-        .font(.caption2).fontWeight(.medium)
+        .font(.footnote)
         .foregroundColor(Color.themeLibraryItemSubtle)
 
         +
         Text("\(notesText)")
-        .font(.caption2).fontWeight(.medium)
+        .font(.footnote)
         .foregroundColor(Color.themeLibraryItemSubtle)
     }
     .frame(maxWidth: .infinity, alignment: .leading)
@@ -281,13 +305,13 @@ public struct LibraryItemCard: View {
   var byLine: some View {
     if let origin = cardSiteName(item.pageURLString) {
       Text(bylineStr + " | " + origin)
-        .font(.caption2)
+        .font(.footnote)
         .foregroundColor(Color.themeLibraryItemSubtle)
         .frame(maxWidth: .infinity, alignment: .leading)
         .lineLimit(1)
     } else {
       Text(bylineStr)
-        .font(.caption2)
+        .font(.footnote)
         .foregroundColor(Color.themeLibraryItemSubtle)
         .frame(maxWidth: .infinity, alignment: .leading)
         .lineLimit(1)
@@ -295,7 +319,7 @@ public struct LibraryItemCard: View {
   }
 
   public var articleInfo: some View {
-    VStack(alignment: .leading, spacing: 5) {
+    VStack(alignment: .leading, spacing: 7) {
       readInfo
         .dynamicTypeSize(.xSmall ... .medium)
 

From b87073b1fb0018a084d6a98c0bdf3a565fd89539 Mon Sep 17 00:00:00 2001
From: Jackson Harper <jacksonh@gmail.com>
Date: Fri, 15 Mar 2024 20:25:44 +0800
Subject: [PATCH 55/67] Add youtube-transcripts beta feature

---
 packages/api/src/services/features.ts | 1 +
 1 file changed, 1 insertion(+)

diff --git a/packages/api/src/services/features.ts b/packages/api/src/services/features.ts
index 699c0f66e..f362d6b56 100644
--- a/packages/api/src/services/features.ts
+++ b/packages/api/src/services/features.ts
@@ -8,6 +8,7 @@ import { logger } from '../utils/logger'
 
 export enum FeatureName {
   AISummaries = 'ai-summaries',
+  YouTubeTranscripts = 'youtube-transcripts',
   UltraRealisticVoice = 'ultra-realistic-voice',
 }
 

From 596ab5a7aa0d7170bb420bb6035c1e0dfbb6abf5 Mon Sep 17 00:00:00 2001
From: Jackson Harper <jacksonh@gmail.com>
Date: Fri, 15 Mar 2024 20:28:02 +0800
Subject: [PATCH 56/67] Feature flag the transcripts

---
 .../api/src/jobs/process-youtube-video.ts     | 35 +++++++++++--------
 1 file changed, 20 insertions(+), 15 deletions(-)

diff --git a/packages/api/src/jobs/process-youtube-video.ts b/packages/api/src/jobs/process-youtube-video.ts
index b4e5f49e7..fa4edcc76 100644
--- a/packages/api/src/jobs/process-youtube-video.ts
+++ b/packages/api/src/jobs/process-youtube-video.ts
@@ -15,6 +15,7 @@ import * as stream from 'stream'
 
 import { Storage } from '@google-cloud/storage'
 import { stringToHash } from '../utils/helpers'
+import { FeatureName, findFeatureByName } from '../services/features'
 
 export interface ProcessYouTubeVideoJobData {
   userId: string
@@ -387,23 +388,27 @@ export const processYouTubeVideo = async (
       duration = video.duration
     }
 
-    if ('getTranscript' in video && duration > 0 && duration < 1801) {
-      // If the video has a transcript available, put a placehold in and
-      // enqueue a job to process the full transcript
-      const updatedContent = await addTranscriptPlaceholdReadableContent(
-        libraryItem.originalUrl,
-        libraryItem.originalContent
-      )
+    if (
+      await findFeatureByName(FeatureName.YouTubeTranscripts, jobData.userId)
+    ) {
+      if ('getTranscript' in video && duration > 0 && duration < 1801) {
+        // If the video has a transcript available, put a placehold in and
+        // enqueue a job to process the full transcript
+        const updatedContent = await addTranscriptPlaceholdReadableContent(
+          libraryItem.originalUrl,
+          libraryItem.originalContent
+        )
 
-      if (updatedContent) {
-        needsUpdate = true
-        libraryItem.readableContent = updatedContent
+        if (updatedContent) {
+          needsUpdate = true
+          libraryItem.readableContent = updatedContent
+        }
+
+        await enqueueProcessYouTubeTranscript({
+          videoId,
+          ...jobData,
+        })
       }
-
-      await enqueueProcessYouTubeTranscript({
-        videoId,
-        ...jobData,
-      })
     }
 
     if (needsUpdate) {

From 8386e54a29fe511e3eeeb7c17581830522986c85 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Sat, 16 Mar 2024 22:46:14 +0000
Subject: [PATCH 57/67] Bump follow-redirects from 1.15.4 to 1.15.6 in
 /pkg/admin

Bumps [follow-redirects](https://github.com/follow-redirects/follow-redirects) from 1.15.4 to 1.15.6.
- [Release notes](https://github.com/follow-redirects/follow-redirects/releases)
- [Commits](https://github.com/follow-redirects/follow-redirects/compare/v1.15.4...v1.15.6)

---
updated-dependencies:
- dependency-name: follow-redirects
  dependency-type: indirect
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 pkg/admin/yarn.lock | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/pkg/admin/yarn.lock b/pkg/admin/yarn.lock
index 40fe46870..11d22d07e 100644
--- a/pkg/admin/yarn.lock
+++ b/pkg/admin/yarn.lock
@@ -2830,9 +2830,9 @@ fn.name@1.x.x:
   integrity sha512-GRnmB5gPyJpAhTQdSZTSp9uaPSvl09KoYcMQtsB9rQoOmzs9dH6ffeccH+Z+cv6P68Hu5bC6JjRh4Ah/mHSNRw==
 
 follow-redirects@^1.14.0:
-  version "1.15.4"
-  resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.15.4.tgz#cdc7d308bf6493126b17ea2191ea0ccf3e535adf"
-  integrity sha512-Cr4D/5wlrb0z9dgERpUL3LrmPKVDsETIJhaCMeDfuFYcqa5bldGV6wBsAN6X/vxlXQtFBMrXdXxdL8CbDTGniw==
+  version "1.15.6"
+  resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.15.6.tgz#7f815c0cda4249c74ff09e95ef97c23b5fd0399b"
+  integrity sha512-wWN62YITEaOpSK584EZXJafH1AGpO8RVgElfkuXbTOrPX4fIfOyEpW/CsiNd8JdYrAoOvafRTOEnvsO++qCqFA==
 
 formidable@^1.0.17:
   version "1.2.2"

From 8298bf3b136cfa71e62f327741991f8c51b79dcd Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Sat, 16 Mar 2024 23:14:08 +0000
Subject: [PATCH 58/67] Bump follow-redirects from 1.15.4 to 1.15.6

Bumps [follow-redirects](https://github.com/follow-redirects/follow-redirects) from 1.15.4 to 1.15.6.
- [Release notes](https://github.com/follow-redirects/follow-redirects/releases)
- [Commits](https://github.com/follow-redirects/follow-redirects/compare/v1.15.4...v1.15.6)

---
updated-dependencies:
- dependency-name: follow-redirects
  dependency-type: indirect
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 yarn.lock | 13 ++++---------
 1 file changed, 4 insertions(+), 9 deletions(-)

diff --git a/yarn.lock b/yarn.lock
index cf307822f..8bd6f9e9f 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -15540,15 +15540,10 @@ fn.name@1.x.x:
   resolved "https://registry.yarnpkg.com/fn.name/-/fn.name-1.1.0.tgz#26cad8017967aea8731bc42961d04a3d5988accc"
   integrity sha512-GRnmB5gPyJpAhTQdSZTSp9uaPSvl09KoYcMQtsB9rQoOmzs9dH6ffeccH+Z+cv6P68Hu5bC6JjRh4Ah/mHSNRw==
 
-follow-redirects@^1.0.0, follow-redirects@^1.14.4, follow-redirects@^1.14.8, follow-redirects@^1.14.9, follow-redirects@^1.15.0:
-  version "1.15.4"
-  resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.15.4.tgz#cdc7d308bf6493126b17ea2191ea0ccf3e535adf"
-  integrity sha512-Cr4D/5wlrb0z9dgERpUL3LrmPKVDsETIJhaCMeDfuFYcqa5bldGV6wBsAN6X/vxlXQtFBMrXdXxdL8CbDTGniw==
-
-follow-redirects@^1.15.4:
-  version "1.15.5"
-  resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.15.5.tgz#54d4d6d062c0fa7d9d17feb008461550e3ba8020"
-  integrity sha512-vSFWUON1B+yAw1VN4xMfxgn5fTUiaOzAJCKBwIIgT/+7CuGy9+r+5gITvP62j3RmaD5Ph65UaERdOSRGUzZtgw==
+follow-redirects@^1.0.0, follow-redirects@^1.14.4, follow-redirects@^1.14.8, follow-redirects@^1.14.9, follow-redirects@^1.15.0, follow-redirects@^1.15.4:
+  version "1.15.6"
+  resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.15.6.tgz#7f815c0cda4249c74ff09e95ef97c23b5fd0399b"
+  integrity sha512-wWN62YITEaOpSK584EZXJafH1AGpO8RVgElfkuXbTOrPX4fIfOyEpW/CsiNd8JdYrAoOvafRTOEnvsO++qCqFA==
 
 for-each@^0.3.3:
   version "0.3.3"

From 7535dc4f805277fa6e804107e7ee7631d00d3c97 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Sun, 17 Mar 2024 01:43:40 +0000
Subject: [PATCH 59/67] Bump express and @types/express

Bumps [express](https://github.com/expressjs/express) and [@types/express](https://github.com/DefinitelyTyped/DefinitelyTyped/tree/HEAD/types/express). These dependencies needed to be updated together.

Updates `express` from 4.18.2 to 4.18.3
- [Release notes](https://github.com/expressjs/express/releases)
- [Changelog](https://github.com/expressjs/express/blob/master/History.md)
- [Commits](https://github.com/expressjs/express/compare/4.18.2...4.18.3)

Updates `@types/express` from 4.17.17 to 4.17.21
- [Release notes](https://github.com/DefinitelyTyped/DefinitelyTyped/releases)
- [Commits](https://github.com/DefinitelyTyped/DefinitelyTyped/commits/HEAD/types/express)

---
updated-dependencies:
- dependency-name: express
  dependency-type: direct:production
  update-type: version-update:semver-patch
- dependency-name: "@types/express"
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 yarn.lock | 56 +++++++++++++++++++++++--------------------------------
 1 file changed, 23 insertions(+), 33 deletions(-)

diff --git a/yarn.lock b/yarn.lock
index 192ec8f0f..c01b40e5c 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -7489,10 +7489,10 @@
   dependencies:
     "@types/express" "*"
 
-"@types/express@*", "@types/express@^4.17.13", "@types/express@^4.17.14", "@types/express@^4.17.7":
-  version "4.17.17"
-  resolved "https://registry.yarnpkg.com/@types/express/-/express-4.17.17.tgz#01d5437f6ef9cfa8668e616e13c2f2ac9a491ae4"
-  integrity sha512-Q4FmmuLGBG58btUnfS1c1r/NQdlp3DMfGDGig8WhfpA2YRUtEkxAjkZb0yvplJGYdF1fsQ81iMDcH24sSCNC/Q==
+"@types/express@*", "@types/express@^4.17.13", "@types/express@^4.17.14", "@types/express@^4.17.21", "@types/express@^4.17.7":
+  version "4.17.21"
+  resolved "https://registry.yarnpkg.com/@types/express/-/express-4.17.21.tgz#c26d4a151e60efe0084b23dc3369ebc631ed192d"
+  integrity sha512-ejlPM315qwLpaQlQDTjPdsUFSc6ZsP4AN6AlWnogPjQ7CVi7PYF3YVz+CY3jE2pwYf7E/7HlDAN0rV2GxTG0HQ==
   dependencies:
     "@types/body-parser" "*"
     "@types/express-serve-static-core" "^4.17.33"
@@ -7509,16 +7509,6 @@
     "@types/qs" "*"
     "@types/serve-static" "*"
 
-"@types/express@^4.17.21":
-  version "4.17.21"
-  resolved "https://registry.yarnpkg.com/@types/express/-/express-4.17.21.tgz#c26d4a151e60efe0084b23dc3369ebc631ed192d"
-  integrity sha512-ejlPM315qwLpaQlQDTjPdsUFSc6ZsP4AN6AlWnogPjQ7CVi7PYF3YVz+CY3jE2pwYf7E/7HlDAN0rV2GxTG0HQ==
-  dependencies:
-    "@types/body-parser" "*"
-    "@types/express-serve-static-core" "^4.17.33"
-    "@types/qs" "*"
-    "@types/serve-static" "*"
-
 "@types/filesystem@*":
   version "0.0.32"
   resolved "https://registry.yarnpkg.com/@types/filesystem/-/filesystem-0.0.32.tgz#307df7cc084a2293c3c1a31151b178063e0a8edf"
@@ -10570,13 +10560,13 @@ bn.js@^5.2.1:
   resolved "https://registry.yarnpkg.com/bn.js/-/bn.js-5.2.1.tgz#0bc527a6a0d18d0aa8d5b0538ce4a77dccfa7b70"
   integrity sha512-eXRvHzWyYPBuB4NBy0cmYQjGitUrtqwbvlzP3G6VFnNRbsZQIxQ10PbKKHt8gZ/HW/D/747aDl+QkDqg3KQLMQ==
 
-body-parser@1.20.1, body-parser@^1.18.3, body-parser@^1.19.0:
-  version "1.20.1"
-  resolved "https://registry.yarnpkg.com/body-parser/-/body-parser-1.20.1.tgz#b1812a8912c195cd371a3ee5e66faa2338a5c668"
-  integrity sha512-jWi7abTbYwajOytWCQc37VulmWiRae5RyTpaCyDcS5/lMdtwSz5lOpDE67srw/HYe35f1z3fDQw+3txg7gNtWw==
+body-parser@1.20.2, body-parser@^1.18.3, body-parser@^1.19.0:
+  version "1.20.2"
+  resolved "https://registry.yarnpkg.com/body-parser/-/body-parser-1.20.2.tgz#6feb0e21c4724d06de7ff38da36dad4f57a747fd"
+  integrity sha512-ml9pReCu3M61kGlqoTm2umSXTlRTuGTx0bfYj+uIUKKYycG5NtSbeetV3faSU6R7ajOPw0g/J1PvK4qNy7s5bA==
   dependencies:
     bytes "3.1.2"
-    content-type "~1.0.4"
+    content-type "~1.0.5"
     debug "2.6.9"
     depd "2.0.0"
     destroy "1.2.0"
@@ -10584,7 +10574,7 @@ body-parser@1.20.1, body-parser@^1.18.3, body-parser@^1.19.0:
     iconv-lite "0.4.24"
     on-finished "2.4.1"
     qs "6.11.0"
-    raw-body "2.5.1"
+    raw-body "2.5.2"
     type-is "~1.6.18"
     unpipe "1.0.0"
 
@@ -12180,10 +12170,10 @@ content-disposition@0.5.4:
   dependencies:
     safe-buffer "5.2.1"
 
-content-type@~1.0.4:
-  version "1.0.4"
-  resolved "https://registry.yarnpkg.com/content-type/-/content-type-1.0.4.tgz#e138cc75e040c727b1966fe5e5f8c9aee256fe3b"
-  integrity sha512-hIP3EEPs8tB9AT1L+NUqtwOAps4mk2Zob89MWXMHjHWg9milF/j4osnnQLXBCBFBk/tvIG/tUc9mOUJiPBhPXA==
+content-type@~1.0.4, content-type@~1.0.5:
+  version "1.0.5"
+  resolved "https://registry.yarnpkg.com/content-type/-/content-type-1.0.5.tgz#8b773162656d1d1086784c8f23a54ce6d73d7918"
+  integrity sha512-nTjqfcBFEipKdXCv4YDQWCfmcLZKm81ldF0pAopTvyrFGVbcR6P/VAAd5G7N+0tTr8QqiU0tFadD6FK4NtJwOA==
 
 conventional-changelog-angular@6.0.0:
   version "6.0.0"
@@ -14962,13 +14952,13 @@ express-rate-limit@^6.3.0:
   integrity sha512-8+UpWtQY25lJaa4+3WxDBGDcAu4atcTruSs3QSL5VPEplYy6kmk84wutG9rUkkK5LmMQQ7TFHWLZYITwVNbbEg==
 
 express@^4.16.4, express@^4.17.1, express@^4.18.2:
-  version "4.18.2"
-  resolved "https://registry.yarnpkg.com/express/-/express-4.18.2.tgz#3fabe08296e930c796c19e3c516979386ba9fd59"
-  integrity sha512-5/PsL6iGPdfQ/lKM1UuielYgv3BUoJfz1aUwU9vHZ+J7gyvwdQXFEBIEIaxeGf0GIcreATNyBExtalisDbuMqQ==
+  version "4.18.3"
+  resolved "https://registry.yarnpkg.com/express/-/express-4.18.3.tgz#6870746f3ff904dee1819b82e4b51509afffb0d4"
+  integrity sha512-6VyCijWQ+9O7WuVMTRBTl+cjNNIzD5cY5mQ1WM8r/LEkI2u8EYpOotESNwzNlyCn3g+dmjKYI6BmNneSr/FSRw==
   dependencies:
     accepts "~1.3.8"
     array-flatten "1.1.1"
-    body-parser "1.20.1"
+    body-parser "1.20.2"
     content-disposition "0.5.4"
     content-type "~1.0.4"
     cookie "0.5.0"
@@ -25515,10 +25505,10 @@ range-parser@^1.2.1, range-parser@~1.2.1:
   resolved "https://registry.yarnpkg.com/range-parser/-/range-parser-1.2.1.tgz#3cf37023d199e1c24d1a55b84800c2f3e6468031"
   integrity sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg==
 
-raw-body@2.5.1:
-  version "2.5.1"
-  resolved "https://registry.yarnpkg.com/raw-body/-/raw-body-2.5.1.tgz#fe1b1628b181b700215e5fd42389f98b71392857"
-  integrity sha512-qqJBtEyVgS0ZmPGdCFPWJ3FreoqvG4MVQln/kCgF7Olq95IbOp0/BWyMwbdtn4VTvkM8Y7khCQ2Xgk/tcrCXig==
+raw-body@2.5.2:
+  version "2.5.2"
+  resolved "https://registry.yarnpkg.com/raw-body/-/raw-body-2.5.2.tgz#99febd83b90e08975087e8f1f9419a149366b68a"
+  integrity sha512-8zGqypfENjCIqGhgXToC8aB2r7YrBX+AQAfIPs/Mlk+BtPTztOvTS01NRW/3Eh60J+a48lt8qsCzirQ6loCVfA==
   dependencies:
     bytes "3.1.2"
     http-errors "2.0.0"
@@ -29551,7 +29541,7 @@ tslib@^1.0.0, tslib@^1.8.1, tslib@^1.9.0, tslib@^1.9.3:
   resolved "https://registry.yarnpkg.com/tslib/-/tslib-1.14.1.tgz#cf2d38bdc34a134bcaf1091c41f6619e2f672d00"
   integrity sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==
 
-tslib@^2, tslib@^2.0.0, tslib@^2.0.1, tslib@^2.0.3, tslib@^2.1.0, tslib@^2.3.0, tslib@^2.3.1, tslib@^2.4.0:
+tslib@^2, tslib@^2.0.0, tslib@^2.0.1, tslib@^2.0.3, tslib@^2.1.0, tslib@^2.3.0, tslib@^2.3.1, tslib@^2.4.0, tslib@^2.5.0:
   version "2.6.2"
   resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.6.2.tgz#703ac29425e7b37cd6fd456e92404d46d1f3e4ae"
   integrity sha512-AEYxH93jGFPn/a2iVAwW87VuUIkR1FVUKB77NwMF7nBTDkDrrT/Hpt/IrCJ0QXhW27jTBDcf5ZY7w6RiqTMw2Q==

From a21a76a8f837d30e22e47a038144b9c08c09aaf0 Mon Sep 17 00:00:00 2001
From: Jackson Harper <jacksonh@gmail.com>
Date: Mon, 18 Mar 2024 10:05:00 +0800
Subject: [PATCH 60/67] Remove ad_id permission on Android

Need to do a bit more testing to see if this breaks anything,
but I don't see any packages we include that need it.
---
 android/Omnivore/app/src/main/AndroidManifest.xml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/android/Omnivore/app/src/main/AndroidManifest.xml b/android/Omnivore/app/src/main/AndroidManifest.xml
index b143c2f12..a989e07f2 100644
--- a/android/Omnivore/app/src/main/AndroidManifest.xml
+++ b/android/Omnivore/app/src/main/AndroidManifest.xml
@@ -4,7 +4,6 @@
 
     <uses-permission android:name="android.permission.INTERNET" />
     <uses-permission android:name="android.permission.ACCESS_NETWORK_STATE"/>
-    <uses-permission android:name="com.google.android.gms.permission.AD_ID"/>
     <uses-permission android:name="android.permission.WAKE_LOCK" />
 
     <application

From 21d4efc2b531c7c417f0ae259f12c6d661c43543 Mon Sep 17 00:00:00 2001
From: Jackson Harper <jacksonh@gmail.com>
Date: Mon, 18 Mar 2024 13:56:37 +0800
Subject: [PATCH 61/67] Allow opting into the YouTube transcript feature

---
 packages/api/src/services/features.ts | 32 ++++++++++++++++++++-------
 1 file changed, 24 insertions(+), 8 deletions(-)

diff --git a/packages/api/src/services/features.ts b/packages/api/src/services/features.ts
index f362d6b56..ebb87eaaa 100644
--- a/packages/api/src/services/features.ts
+++ b/packages/api/src/services/features.ts
@@ -6,6 +6,9 @@ import { env } from '../env'
 import { getRepository } from '../repository'
 import { logger } from '../utils/logger'
 
+const MAX_ULTRA_REALISTIC_USERS = 1500
+const MAX_YOUTUBE_TRANSCRIPT_USERS = 100
+
 export enum FeatureName {
   AISummaries = 'ai-summaries',
   YouTubeTranscripts = 'youtube-transcripts',
@@ -20,16 +23,31 @@ export const optInFeature = async (
   name: FeatureName,
   uid: string
 ): Promise<Feature | undefined> => {
-  if (name === FeatureName.UltraRealisticVoice) {
-    return optInUltraRealisticVoice(uid)
+  switch (name) {
+    case FeatureName.UltraRealisticVoice:
+      return optInLimitedFeature(
+        FeatureName.UltraRealisticVoice,
+        uid,
+        MAX_ULTRA_REALISTIC_USERS
+      )
+    case FeatureName.YouTubeTranscripts:
+      return optInLimitedFeature(
+        FeatureName.YouTubeTranscripts,
+        uid,
+        MAX_YOUTUBE_TRANSCRIPT_USERS
+      )
   }
   return undefined
 }
 
-const optInUltraRealisticVoice = async (uid: string): Promise<Feature> => {
+const optInLimitedFeature = async (
+  featureName: string,
+  uid: string,
+  maxUsers: number
+): Promise<Feature> => {
   const feature = await getRepository(Feature).findOne({
     where: {
-      name: FeatureName.UltraRealisticVoice,
+      name: featureName,
       grantedAt: Not(IsNull()),
       user: { id: uid },
     },
@@ -41,8 +59,6 @@ const optInUltraRealisticVoice = async (uid: string): Promise<Feature> => {
     return feature
   }
 
-  const MAX_USERS = 1500
-  // opt in to feature for the first 1500 users
   const optedInFeatures = (await appDataSource.query(
     `insert into omnivore.features (user_id, name, granted_at) 
     select $1, $2, $3 from omnivore.features 
@@ -51,7 +67,7 @@ const optInUltraRealisticVoice = async (uid: string): Promise<Feature> => {
     on conflict (user_id, name) 
     do update set granted_at = $3 
     returning *, granted_at as "grantedAt", created_at as "createdAt", updated_at as "updatedAt";`,
-    [uid, FeatureName.UltraRealisticVoice, new Date(), MAX_USERS]
+    [uid, featureName, new Date(), maxUsers]
   )) as Feature[]
 
   // if no new features were created then user has exceeded max users
@@ -61,7 +77,7 @@ const optInUltraRealisticVoice = async (uid: string): Promise<Feature> => {
     // create/update an opt-in record with null grantedAt
     const optInRecord = {
       user: { id: uid },
-      name: FeatureName.UltraRealisticVoice,
+      name: featureName,
       grantedAt: null,
     }
     const result = await getRepository(Feature).upsert(optInRecord, [

From 10bd05c6a89156b96b113b5ef389b1673ac9330d Mon Sep 17 00:00:00 2001
From: Jackson Harper <jacksonh@gmail.com>
Date: Mon, 18 Mar 2024 15:00:12 +0800
Subject: [PATCH 62/67] Allow opt into transcripts feature, create UI

---
 packages/web/lib/networking/queries/useGetViewerQuery.tsx | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/packages/web/lib/networking/queries/useGetViewerQuery.tsx b/packages/web/lib/networking/queries/useGetViewerQuery.tsx
index 88beccd8b..abf88c1ff 100644
--- a/packages/web/lib/networking/queries/useGetViewerQuery.tsx
+++ b/packages/web/lib/networking/queries/useGetViewerQuery.tsx
@@ -3,6 +3,7 @@ import useSWR from 'swr'
 import { publicGqlFetcher } from '../networkHelpers'
 
 type ViewerQueryResponse = {
+  mutate: () => void
   viewerData?: ViewerQueryResponseData
   viewerDataError?: unknown
   isLoading: boolean
@@ -51,9 +52,10 @@ export function useGetViewerQuery(): ViewerQueryResponse {
     }
   `
 
-  const { data, error } = useSWR(query, publicGqlFetcher)
+  const { data, error, mutate } = useSWR(query, publicGqlFetcher)
 
   return {
+    mutate,
     viewerData: data as ViewerQueryResponseData,
     viewerDataError: error, // TODO: figure out error possibilities
     isLoading: !error && !data,

From 36c312ca754b00ffdcf5afe9a603217f8ffc70f6 Mon Sep 17 00:00:00 2001
From: Jackson Harper <jacksonh@gmail.com>
Date: Tue, 19 Mar 2024 10:12:49 +0800
Subject: [PATCH 63/67] Add missing YT files

---
 .../mutations/optIntoFeatureMutation.ts       |  49 ++++
 packages/web/pages/settings/features/beta.tsx | 231 ++++++++++++++++++
 2 files changed, 280 insertions(+)
 create mode 100644 packages/web/lib/networking/mutations/optIntoFeatureMutation.ts
 create mode 100644 packages/web/pages/settings/features/beta.tsx

diff --git a/packages/web/lib/networking/mutations/optIntoFeatureMutation.ts b/packages/web/lib/networking/mutations/optIntoFeatureMutation.ts
new file mode 100644
index 000000000..26ac49d47
--- /dev/null
+++ b/packages/web/lib/networking/mutations/optIntoFeatureMutation.ts
@@ -0,0 +1,49 @@
+import { gql } from 'graphql-request'
+import { gqlFetcher } from '../networkHelpers'
+
+export interface OptInFeatureInput {
+  name: string
+}
+
+export interface OptInFeatureSuccess {
+  feature: { id: string }
+}
+
+interface Response {
+  optInFeature: OptInFeatureSuccess
+}
+
+export async function optInFeature(
+  input: OptInFeatureInput
+): Promise<boolean | undefined> {
+  const mutation = gql`
+    mutation OptInFeature($input: OptInFeatureInput!) {
+      optInFeature(input: $input) {
+        ... on OptInFeatureSuccess {
+          feature {
+            id
+          }
+        }
+        ... on OptInFeatureError {
+          errorCodes
+        }
+      }
+    }
+  `
+  try {
+    const data = await gqlFetcher(mutation, {
+      input,
+    })
+    const output = data as Response | undefined
+    if (
+      !output ||
+      !output.optInFeature ||
+      'errorCodes' in output?.optInFeature
+    ) {
+      return false
+    }
+    return true
+  } catch (err) {
+    return undefined
+  }
+}
diff --git a/packages/web/pages/settings/features/beta.tsx b/packages/web/pages/settings/features/beta.tsx
new file mode 100644
index 000000000..c48d274de
--- /dev/null
+++ b/packages/web/pages/settings/features/beta.tsx
@@ -0,0 +1,231 @@
+import { useCallback, useEffect, useMemo, useState } from 'react'
+import { Toaster } from 'react-hot-toast'
+import { Button } from '../../../components/elements/Button'
+import {
+  Box,
+  HStack,
+  SpanBox,
+  VStack,
+} from '../../../components/elements/LayoutPrimitives'
+import { StyledText } from '../../../components/elements/StyledText'
+import { SettingsLayout } from '../../../components/templates/SettingsLayout'
+import { styled, theme } from '../../../components/tokens/stitches.config'
+import { updateEmailMutation } from '../../../lib/networking/mutations/updateEmailMutation'
+import { updateUserMutation } from '../../../lib/networking/mutations/updateUserMutation'
+import { updateUserProfileMutation } from '../../../lib/networking/mutations/updateUserProfileMutation'
+import { useGetLibraryItemsQuery } from '../../../lib/networking/queries/useGetLibraryItemsQuery'
+import { useGetViewerQuery } from '../../../lib/networking/queries/useGetViewerQuery'
+import { useValidateUsernameQuery } from '../../../lib/networking/queries/useValidateUsernameQuery'
+import { applyStoredTheme } from '../../../lib/themeUpdater'
+import { showErrorToast, showSuccessToast } from '../../../lib/toastHelpers'
+import { ConfirmationModal } from '../../../components/patterns/ConfirmationModal'
+import { ProgressBar } from '../../../components/elements/ProgressBar'
+import { emptyTrashMutation } from '../../../lib/networking/mutations/emptyTrashMutation'
+import { ProgressIndicator } from '@radix-ui/react-progress'
+import { Spinner } from 'phosphor-react'
+import { optInFeature } from '../../../lib/networking/mutations/optIntoFeatureMutation'
+
+const ACCOUNT_LIMIT = 50_000
+
+const StyledLabel = styled('label', {
+  fontWeight: 600,
+  fontSize: '16px',
+  marginBottom: '5px',
+})
+
+export default function Account(): JSX.Element {
+  const { viewerData, isLoading, mutate } = useGetViewerQuery()
+  const [pageLoading, setPageLoading] = useState(false)
+
+  const showSpinner = useMemo(() => {
+    return isLoading || pageLoading
+  }, [isLoading, pageLoading])
+
+  const requestFeatureAccess = useCallback(
+    async (featureName: string) => {
+      setPageLoading(true)
+      const result = await optInFeature({ name: featureName })
+      if (!result) {
+        showErrorToast('Error opting into feature.')
+      } else {
+        showSuccessToast('Feature added')
+      }
+      mutate()
+      setPageLoading(false)
+    },
+    [setPageLoading, mutate]
+  )
+
+  const hasYouTube = useMemo(() => {
+    return (
+      (viewerData?.me?.features.indexOf('youtube-transcripts') ?? -1) !== -1
+    )
+  }, [viewerData])
+
+  applyStoredTheme()
+
+  return (
+    <SettingsLayout>
+      <Toaster
+        containerStyle={{
+          top: '5rem',
+        }}
+      />
+
+      <VStack
+        css={{ width: '100%', height: '100%' }}
+        distribution="start"
+        alignment="center"
+      >
+        <VStack
+          css={{
+            padding: '24px',
+            width: '100%',
+            height: '100%',
+            minWidth: '300px',
+            maxWidth: '865px',
+          }}
+        >
+          <VStack
+            css={{
+              padding: '24px',
+              width: '100%',
+              height: '100%',
+              bg: '$grayBg',
+              gap: '10px',
+              borderRadius: '5px',
+            }}
+          >
+            <StyledLabel>Enabled beta features</StyledLabel>
+            {!showSpinner ? (
+              <>
+                {viewerData?.me?.features.map((feature) => {
+                  return (
+                    <StyledText
+                      key={`feature-${feature}`}
+                      style="footnote"
+                      css={{
+                        display: 'flex',
+                        gap: '5px',
+                        marginBlockStart: '0px',
+                        marginBlockEnd: '0px',
+                      }}
+                    >
+                      <input
+                        type="checkbox"
+                        checked={true}
+                        disabled={true}
+                      ></input>
+                      {feature}
+                    </StyledText>
+                  )
+                })}
+
+                {!hasYouTube /* || !hasAISummaries || !hasDigest */ && (
+                  <StyledLabel css={{ mt: '25px' }}>
+                    Available beta features
+                  </StyledLabel>
+                )}
+
+                <VStack css={{ gap: '15px' }}>
+                  {!hasYouTube && (
+                    <VStack
+                      alignment="start"
+                      distribution="start"
+                      css={{ width: '100%' }}
+                    >
+                      <StyledText
+                        style="footnote"
+                        css={{ display: 'flex', gap: '5px' }}
+                      >
+                        - YouTube transcripts: nicely formatted documents
+                        generated from YouTube transcript data. Currently
+                        limited to videos under 30 minutes.
+                      </StyledText>
+                      <Button
+                        style="ctaDarkYellow"
+                        onClick={(event) => {
+                          requestFeatureAccess('youtube-transcripts')
+                          event.preventDefault()
+                        }}
+                      >
+                        Request feature
+                      </Button>
+                    </VStack>
+                  )}
+
+                  {/* <VStack
+                    alignment="start"
+                    distribution="start"
+                    css={{ width: '100%' }}
+                  >
+                    <StyledText
+                      style="footnote"
+                      css={{ display: 'flex', gap: '5px' }}
+                    >
+                      - AI Summaries: Short summaries of your newly saved
+                      articles
+                    </StyledText>
+                    <Button
+                      style="ctaDarkYellow"
+                      onClick={(event) => {
+                        requestFeatureAccess('ai-summaries')
+                        event.preventDefault()
+                      }}
+                    >
+                      Request feature
+                    </Button>
+                  </VStack>
+
+                  <VStack
+                    alignment="start"
+                    distribution="start"
+                    css={{ width: '100%' }}
+                  >
+                    <StyledText
+                      style="footnote"
+                      css={{ display: 'flex', gap: '5px' }}
+                    >
+                      - Daily digest: Every day we pick some of the items we
+                      think You will enjoy reading the most and create a daily
+                      digest of them.
+                    </StyledText>
+                    <Button
+                      style="ctaDarkYellow"
+                      onClick={(event) => {
+                        requestFeatureAccess('daily-digest')
+                        event.preventDefault()
+                      }}
+                    >
+                      Request feature
+                    </Button>
+                  </VStack> */}
+                </VStack>
+              </>
+            ) : (
+              <HStack
+                distribution="center"
+                css={{
+                  width: '100%',
+                  svg: {
+                    animation: 'spin 1s infinite linear',
+                    '@keyframes spin': {
+                      from: {
+                        transform: 'scale(1) rotate(0deg)',
+                      },
+                      to: {
+                        transform: 'scale(1) rotate(360deg)',
+                      },
+                    },
+                  },
+                }}
+              >
+                <Spinner size={16} />
+              </HStack>
+            )}
+          </VStack>
+        </VStack>
+      </VStack>
+    </SettingsLayout>
+  )
+}

From ad32aa82ce6c910258baa52aeb3ceb4eb9889429 Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Tue, 19 Mar 2024 11:17:42 +0800
Subject: [PATCH 64/67] Export highlight notes as children of highlights block
 in notion

---
 packages/api/src/entity/highlight.ts          |   2 +-
 .../api/src/services/integrations/notion.ts   | 174 ++++++++++--------
 .../pages/settings/integrations/notion.tsx    |   2 -
 3 files changed, 98 insertions(+), 80 deletions(-)

diff --git a/packages/api/src/entity/highlight.ts b/packages/api/src/entity/highlight.ts
index 602c9f959..60b8eed3a 100644
--- a/packages/api/src/entity/highlight.ts
+++ b/packages/api/src/entity/highlight.ts
@@ -62,7 +62,7 @@ export class Highlight {
   createdAt!: Date
 
   @UpdateDateColumn()
-  updatedAt?: Date | null
+  updatedAt!: Date
 
   @Column('timestamp')
   sharedAt?: Date
diff --git a/packages/api/src/services/integrations/notion.ts b/packages/api/src/services/integrations/notion.ts
index a4e8a84e5..4d823e031 100644
--- a/packages/api/src/services/integrations/notion.ts
+++ b/packages/api/src/services/integrations/notion.ts
@@ -67,32 +67,46 @@ interface NotionPage {
     'Omnivore URL': {
       url: string
     }
+    'Saved At': {
+      date: {
+        start: string
+      }
+    }
+    'Last Updated': {
+      date: {
+        start: string
+      }
+    }
     Tags?: {
       multi_select: Array<{ name: string }>
     }
   }
   children?: Array<{
-    type: 'paragraph'
     paragraph: {
       rich_text: Array<{
         text: {
           content: string
           link?: { url: string }
         }
-        annotations?: {
-          bold?: boolean
-          italic?: boolean
-          strikethrough?: boolean
-          underline?: boolean
-          code?: boolean
-          color?: AnnotationColor
+        annotations: {
+          code: boolean
+          color: AnnotationColor
+        }
+      }>
+      children?: Array<{
+        paragraph: {
+          rich_text: Array<{
+            text: {
+              content: string
+            }
+          }>
         }
       }>
     }
   }>
 }
 
-type Property = 'highlights' | 'labels' | 'notes'
+type Property = 'highlights'
 
 interface Settings {
   parentPageId: string
@@ -165,7 +179,8 @@ export class NotionClient implements IntegrationClient {
 
   private itemToNotionPage = (
     item: LibraryItem,
-    settings: Settings
+    settings: Settings,
+    lastSync?: Date | null
   ): NotionPage => {
     return {
       parent: {
@@ -210,47 +225,64 @@ export class NotionClient implements IntegrationClient {
         'Omnivore URL': {
           url: `${env.client.url}/me/${item.slug}`,
         },
-        Tags:
-          item.labels && settings.properties.includes('labels')
-            ? {
-                multi_select: item.labels.map((label) => ({
-                  name: label.name,
-                })),
-              }
-            : undefined,
+        'Saved At': {
+          date: {
+            start: item.createdAt.toISOString(),
+          },
+        },
+        'Last Updated': {
+          date: {
+            start: item.updatedAt.toISOString(),
+          },
+        },
+        Tags: item.labels
+          ? {
+              multi_select: item.labels.map((label) => ({
+                name: label.name,
+              })),
+            }
+          : undefined,
       },
-      children: item.highlights
-        ? item.highlights.map((highlight) => ({
-            type: 'paragraph',
-            paragraph: {
-              rich_text: [
-                {
-                  text: {
-                    content: settings.properties.includes('highlights')
-                      ? highlight.quote || ''
-                      : '',
-                    link: {
-                      url: highlightUrl(item.slug, highlight.id),
+      children:
+        settings.properties.includes('highlights') && item.highlights
+          ? item.highlights
+              .filter(
+                (highlight) => !lastSync || highlight.updatedAt > lastSync // only new highlights
+              )
+              .map((highlight) => ({
+                paragraph: {
+                  rich_text: [
+                    {
+                      text: {
+                        content: highlight.quote || '',
+                        link: {
+                          url: highlightUrl(item.slug, highlight.id),
+                        },
+                      },
+                      annotations: {
+                        code: true,
+                        color: highlight.color as AnnotationColor,
+                      },
                     },
-                  },
-                  annotations: {
-                    color: highlight.color as AnnotationColor,
-                  },
+                  ],
+                  children: highlight.annotation
+                    ? [
+                        {
+                          paragraph: {
+                            rich_text: [
+                              {
+                                text: {
+                                  content: highlight.annotation || '',
+                                },
+                              },
+                            ],
+                          },
+                        },
+                      ]
+                    : undefined,
                 },
-                {
-                  text: {
-                    content: settings.properties.includes('notes')
-                      ? `\n${highlight.annotation || ''}`
-                      : '',
-                  },
-                  annotations: {
-                    italic: true,
-                  },
-                },
-              ],
-            },
-          }))
-        : undefined,
+              }))
+          : undefined,
     }
   }
 
@@ -323,6 +355,12 @@ export class NotionClient implements IntegrationClient {
           'Omnivore URL': {
             url: {},
           },
+          'Saved At': {
+            date: {},
+          },
+          'Last Updated': {
+            date: {},
+          },
           Tags: {
             multi_select: {},
           },
@@ -331,13 +369,11 @@ export class NotionClient implements IntegrationClient {
 
       // save the database id
       databaseId = database.id
+      settings.parentDatabaseId = databaseId
       await updateIntegration(
         this.integrationData.id,
         {
-          settings: {
-            ...this.integrationData.settings,
-            parentDatabaseId: databaseId,
-          },
+          settings,
         },
         this.integrationData.user.id
       )
@@ -345,7 +381,11 @@ export class NotionClient implements IntegrationClient {
 
     await Promise.all(
       items.map(async (item) => {
-        const notionPage = this.itemToNotionPage(item, settings)
+        const notionPage = this.itemToNotionPage(
+          item,
+          settings,
+          this.integrationData?.syncedAt
+        )
         const url = notionPage.properties['Omnivore URL'].url
 
         const existingPage = await this.findPage(url, databaseId)
@@ -356,32 +396,12 @@ export class NotionClient implements IntegrationClient {
             properties: notionPage.properties,
           })
 
-          const children = notionPage.children
-          if (children) {
-            // get the existing children
-            const response = await this.client.blocks.children.list({
+          // append the children incrementally
+          if (notionPage.children && notionPage.children.length > 0) {
+            await this.client.blocks.children.append({
               block_id: existingPage.id,
+              children: notionPage.children,
             })
-            if (response.results.length > 0) {
-              const existingChildren =
-                response.results as NotionPage['children']
-              // delete the existing children from children
-              notionPage.children = children.filter(
-                (child) =>
-                  !existingChildren?.some(
-                    (existingChild) =>
-                      existingChild.paragraph.rich_text[0].text.link?.url ===
-                      child.paragraph.rich_text[0].text.link?.url
-                  )
-              )
-            }
-            // append the children
-            if (notionPage.children && notionPage.children.length > 0) {
-              await this.client.blocks.children.append({
-                block_id: existingPage.id,
-                children: notionPage.children,
-              })
-            }
           }
 
           return
diff --git a/packages/web/pages/settings/integrations/notion.tsx b/packages/web/pages/settings/integrations/notion.tsx
index dce6e0a24..fd92082e4 100644
--- a/packages/web/pages/settings/integrations/notion.tsx
+++ b/packages/web/pages/settings/integrations/notion.tsx
@@ -174,8 +174,6 @@ export default function Notion(): JSX.Element {
                 >
                   <Checkbox.Group onChange={onDataChange}>
                     <Checkbox value="highlights">Highlights</Checkbox>
-                    <Checkbox value="labels">Labels</Checkbox>
-                    <Checkbox value="notes">Notes</Checkbox>
                   </Checkbox.Group>
                 </Form.Item>
 

From 21335d3ed41e76c5ed5364af3e38ce6b2ee42932 Mon Sep 17 00:00:00 2001
From: Jackson Harper <jacksonh@gmail.com>
Date: Tue, 19 Mar 2024 12:33:40 +0800
Subject: [PATCH 65/67] Fix for pages that dont need auth like terms and
 conditions page

---
 packages/api/src/jobs/ai-summarize.ts         | 15 +++--
 packages/api/src/services/ai-summaries.ts     | 66 +++++++++++++++++++
 .../components/templates/SettingsLayout.tsx   |  1 -
 packages/web/pages/404.tsx                    |  6 +-
 packages/web/pages/500.tsx                    |  8 +--
 packages/web/pages/support.tsx                |  6 +-
 packages/web/pages/terms.tsx                  |  6 +-
 7 files changed, 89 insertions(+), 19 deletions(-)

diff --git a/packages/api/src/jobs/ai-summarize.ts b/packages/api/src/jobs/ai-summarize.ts
index 974a6b347..3658876b6 100644
--- a/packages/api/src/jobs/ai-summarize.ts
+++ b/packages/api/src/jobs/ai-summarize.ts
@@ -4,10 +4,13 @@ import { ChatOpenAI } from '@langchain/openai'
 import { RecursiveCharacterTextSplitter } from 'langchain/text_splitter'
 import { authTrx } from '../repository'
 import { libraryItemRepository } from '../repository/library_item'
-import { htmlToMarkdown } from '../utils/parser'
 import { AISummary } from '../entity/AISummary'
 import { LibraryItemState } from '../entity/library_item'
-import { getAISummary } from '../services/ai-summaries'
+import {
+  createSummarizableDocument,
+  getAISummary,
+} from '../services/ai-summaries'
+import { NodeHtmlMarkdown, TranslatorConfigObject } from 'node-html-markdown'
 
 export interface AISummarizeJobData {
   userId: string
@@ -55,19 +58,21 @@ export const aiSummarize = async (jobData: AISummarizeJobData) => {
       },
     })
     const textSplitter = new RecursiveCharacterTextSplitter({
-      chunkSize: 2000,
+      chunkSize: 12000,
     })
 
-    const document = htmlToMarkdown(libraryItem.readableContent)
+    const document = createSummarizableDocument(libraryItem.readableContent)
     const docs = await textSplitter.createDocuments([document])
     const chain = loadSummarizationChain(llm, {
       type: 'map_reduce', // you can choose from map_reduce, stuff or refine
       verbose: true, // to view the steps in the console
     })
-    const response = await chain.call({
+    const response = await chain.invoke({
       input_documents: docs,
     })
 
+    console.log('summary response: ', JSON.stringify(response))
+
     if (typeof response.text !== 'string') {
       logger.error(`AI summary did not return text`)
       return
diff --git a/packages/api/src/services/ai-summaries.ts b/packages/api/src/services/ai-summaries.ts
index 3c94ca4cd..9b9ebefd8 100644
--- a/packages/api/src/services/ai-summaries.ts
+++ b/packages/api/src/services/ai-summaries.ts
@@ -1,5 +1,71 @@
+import { ChatOpenAI } from '@langchain/openai'
 import { AISummary } from '../entity/AISummary'
 import { authTrx } from '../repository'
+import { RecursiveCharacterTextSplitter } from 'langchain/text_splitter'
+import { loadSummarizationChain } from 'langchain/chains'
+import { logger } from '../utils/logger'
+import { NodeHtmlMarkdown, TranslatorConfigObject } from 'node-html-markdown'
+
+// When creating markdown we remove external links in URLs
+// and images since these often contain per-user trackers
+// that can interfere with caching
+const removeLinksTransformer: TranslatorConfigObject = {
+  a: ({ node, options, visitor }) => {
+    return {
+      postprocess: ({ content }) => {
+        return `[${content}]()`
+      },
+    }
+  },
+  img: ({ node, options, visitor }) => {
+    const alt = node.getAttribute('alt')?.trim()
+    return {
+      content: `![${alt}]()`,
+    }
+  },
+}
+
+export const createSummarizableDocument = (readable: string): string => {
+  const nhm = new NodeHtmlMarkdown(
+    {
+      keepDataImages: false,
+    },
+    removeLinksTransformer
+  )
+  return nhm.translate(readable)
+}
+
+export const createAISummary = async (
+  readableContent: string
+): Promise<string | undefined> => {
+  const llm = new ChatOpenAI({
+    configuration: {
+      apiKey: process.env.OPENAI_API_KEY,
+    },
+  })
+  const textSplitter = new RecursiveCharacterTextSplitter({
+    chunkSize: 12000,
+  })
+
+  const document = createSummarizableDocument(readableContent)
+  const docs = await textSplitter.createDocuments([document])
+  const chain = loadSummarizationChain(llm, {
+    type: 'map_reduce', // you can choose from map_reduce, stuff or refine
+    verbose: true, // to view the steps in the console
+  })
+  const response = await chain.invoke({
+    input_documents: docs,
+  })
+
+  console.log('summary response: ', JSON.stringify(response))
+
+  if (typeof response.text !== 'string') {
+    logger.error(`AI summary did not return text`)
+    return
+  }
+
+  return response.text
+}
 
 export const getAISummary = async (data: {
   userId: string
diff --git a/packages/web/components/templates/SettingsLayout.tsx b/packages/web/components/templates/SettingsLayout.tsx
index 05c91ea8b..dd9be633a 100644
--- a/packages/web/components/templates/SettingsLayout.tsx
+++ b/packages/web/components/templates/SettingsLayout.tsx
@@ -1,5 +1,4 @@
 import { Box, HStack, VStack } from '../elements/LayoutPrimitives'
-import { useGetViewerQuery } from '../../lib/networking/queries/useGetViewerQuery'
 import { navigationCommands } from '../../lib/keyboardShortcuts/navigationShortcuts'
 import { useKeyboardShortcuts } from '../../lib/keyboardShortcuts/useKeyboardShortcuts'
 import { useRouter } from 'next/router'
diff --git a/packages/web/pages/404.tsx b/packages/web/pages/404.tsx
index 80e44ab9d..bdf8e40e4 100644
--- a/packages/web/pages/404.tsx
+++ b/packages/web/pages/404.tsx
@@ -1,6 +1,6 @@
 import Head from 'next/head'
 import { ErrorLayout } from '../components/templates/ErrorLayout'
-import { SettingsLayout } from '../components/templates/SettingsLayout'
+import { EmptyLayout } from '../components/templates/EmptyLayout'
 
 export default function Custom404(): JSX.Element {
   return (
@@ -8,9 +8,9 @@ export default function Custom404(): JSX.Element {
       <Head>
         <title>Page Not Found</title>
       </Head>
-      <SettingsLayout title="Page could not be found">
+      <EmptyLayout title="Page could not be found">
         <ErrorLayout statusCode={404} message="This page could not be found." />
-      </SettingsLayout>
+      </EmptyLayout>
     </>
   )
 }
diff --git a/packages/web/pages/500.tsx b/packages/web/pages/500.tsx
index 75d24835c..e4976faba 100644
--- a/packages/web/pages/500.tsx
+++ b/packages/web/pages/500.tsx
@@ -1,6 +1,6 @@
 import { ErrorLayout } from '../components/templates/ErrorLayout'
 import Head from 'next/head'
-import { SettingsLayout } from '../components/templates/SettingsLayout'
+import { EmptyLayout } from '../components/templates/EmptyLayout'
 
 export default function Custom500(): JSX.Element {
   return (
@@ -8,9 +8,9 @@ export default function Custom500(): JSX.Element {
       <Head>
         <title>An unknown error occurred.</title>
       </Head>
-      <SettingsLayout title="An unknown error occurred">
-       <ErrorLayout statusCode={404} message="An unknown error occurred." />
-      </SettingsLayout>
+      <EmptyLayout title="An unknown error occurred">
+        <ErrorLayout statusCode={404} message="An unknown error occurred." />
+      </EmptyLayout>
     </>
   )
 }
diff --git a/packages/web/pages/support.tsx b/packages/web/pages/support.tsx
index 98e25f366..edf0d7e09 100644
--- a/packages/web/pages/support.tsx
+++ b/packages/web/pages/support.tsx
@@ -1,7 +1,7 @@
 import { useEffect, useCallback } from 'react'
 import { Button } from '../components/elements/Button'
 import { HStack } from '../components/elements/LayoutPrimitives'
-import { SettingsLayout } from '../components/templates/SettingsLayout'
+import { EmptyLayout } from '../components/templates/EmptyLayout'
 import { setupAnalytics } from '../lib/analytics'
 
 export default function Support(): JSX.Element {
@@ -18,7 +18,7 @@ export default function Support(): JSX.Element {
   }, [initAnalytics])
 
   return (
-    <SettingsLayout title="Support">
+    <EmptyLayout title="Support">
       <HStack
         alignment="center"
         distribution="end"
@@ -46,6 +46,6 @@ export default function Support(): JSX.Element {
           {'Open Chat Window'}
         </Button>
       </HStack>
-    </SettingsLayout>
+    </EmptyLayout>
   )
 }
diff --git a/packages/web/pages/terms.tsx b/packages/web/pages/terms.tsx
index fd267cb75..b018d8d56 100644
--- a/packages/web/pages/terms.tsx
+++ b/packages/web/pages/terms.tsx
@@ -1,6 +1,6 @@
 import { useRouter } from 'next/router'
 import { TermsAndConditions } from '../components/templates/TermsAndConditions'
-import { SettingsLayout } from '../components/templates/SettingsLayout'
+import { EmptyLayout } from '../components/templates/DocsLayout'
 
 export default function Terms(): JSX.Element {
   const router = useRouter()
@@ -11,9 +11,9 @@ export default function Terms(): JSX.Element {
     return <TermsAndConditions />
   } else {
     return (
-      <SettingsLayout title="Terms and Conditions">
+      <EmptyLayout title="Terms and Conditions">
         <TermsAndConditions />
-      </SettingsLayout>
+      </EmptyLayout>
     )
   }
 }

From 55b6f8197cfda363aa71e485253049883bcaf24c Mon Sep 17 00:00:00 2001
From: Jackson Harper <jacksonh@gmail.com>
Date: Tue, 19 Mar 2024 12:39:10 +0800
Subject: [PATCH 66/67] Fixes for empty layout

---
 .../web/components/templates/EmptyLayout.tsx  | 47 +++++++++++++++++++
 packages/web/pages/terms.tsx                  |  2 +-
 2 files changed, 48 insertions(+), 1 deletion(-)
 create mode 100644 packages/web/components/templates/EmptyLayout.tsx

diff --git a/packages/web/components/templates/EmptyLayout.tsx b/packages/web/components/templates/EmptyLayout.tsx
new file mode 100644
index 000000000..53c655a1d
--- /dev/null
+++ b/packages/web/components/templates/EmptyLayout.tsx
@@ -0,0 +1,47 @@
+import { Box, HStack, VStack } from '../elements/LayoutPrimitives'
+import { PageMetaData } from '../patterns/PageMetaData'
+import { DEFAULT_HEADER_HEIGHT } from './homeFeed/HeaderSpacer'
+import { SettingsDropdown } from './navMenu/SettingsDropdown'
+
+type EmptyLayoutProps = {
+  title: string
+  children: React.ReactNode
+}
+
+export function EmptyLayout(props: EmptyLayoutProps): JSX.Element {
+  return (
+    <VStack
+      alignment="start"
+      distribution="start"
+      css={{ width: '100%', height: '100%', minHeight: '100vh' }}
+    >
+      <PageMetaData path="/" title={props.title} />
+      <VStack css={{ width: '100%', height: '100%' }}>
+        <Box
+          css={{
+            height: DEFAULT_HEADER_HEIGHT,
+            '@mdDown': {
+              display: 'none',
+            },
+          }}
+        ></Box>
+        <Box
+          css={{
+            p: '15px',
+            display: 'none',
+            height: DEFAULT_HEADER_HEIGHT,
+            '@mdDown': {
+              display: 'flex',
+            },
+          }}
+        >
+          <SettingsDropdown />
+        </Box>
+        <HStack css={{ width: '100%', height: '100%' }} distribution="start">
+          {props.children}
+        </HStack>
+        <Box css={{ height: '120px', width: '100%' }} />
+      </VStack>
+    </VStack>
+  )
+}
diff --git a/packages/web/pages/terms.tsx b/packages/web/pages/terms.tsx
index b018d8d56..bed50fe80 100644
--- a/packages/web/pages/terms.tsx
+++ b/packages/web/pages/terms.tsx
@@ -1,6 +1,6 @@
 import { useRouter } from 'next/router'
 import { TermsAndConditions } from '../components/templates/TermsAndConditions'
-import { EmptyLayout } from '../components/templates/DocsLayout'
+import { EmptyLayout } from '../components/templates/EmptyLayout'
 
 export default function Terms(): JSX.Element {
   const router = useRouter()

From 72e78cd465ade22762d5109828f3b23e5de87740 Mon Sep 17 00:00:00 2001
From: Jackson Harper <jacksonh@gmail.com>
Date: Tue, 19 Mar 2024 13:06:25 +0800
Subject: [PATCH 67/67] Revert WIP changes

---
 packages/api/src/jobs/ai-summarize.ts     | 15 ++----
 packages/api/src/services/ai-summaries.ts | 66 -----------------------
 2 files changed, 5 insertions(+), 76 deletions(-)

diff --git a/packages/api/src/jobs/ai-summarize.ts b/packages/api/src/jobs/ai-summarize.ts
index 3658876b6..974a6b347 100644
--- a/packages/api/src/jobs/ai-summarize.ts
+++ b/packages/api/src/jobs/ai-summarize.ts
@@ -4,13 +4,10 @@ import { ChatOpenAI } from '@langchain/openai'
 import { RecursiveCharacterTextSplitter } from 'langchain/text_splitter'
 import { authTrx } from '../repository'
 import { libraryItemRepository } from '../repository/library_item'
+import { htmlToMarkdown } from '../utils/parser'
 import { AISummary } from '../entity/AISummary'
 import { LibraryItemState } from '../entity/library_item'
-import {
-  createSummarizableDocument,
-  getAISummary,
-} from '../services/ai-summaries'
-import { NodeHtmlMarkdown, TranslatorConfigObject } from 'node-html-markdown'
+import { getAISummary } from '../services/ai-summaries'
 
 export interface AISummarizeJobData {
   userId: string
@@ -58,21 +55,19 @@ export const aiSummarize = async (jobData: AISummarizeJobData) => {
       },
     })
     const textSplitter = new RecursiveCharacterTextSplitter({
-      chunkSize: 12000,
+      chunkSize: 2000,
     })
 
-    const document = createSummarizableDocument(libraryItem.readableContent)
+    const document = htmlToMarkdown(libraryItem.readableContent)
     const docs = await textSplitter.createDocuments([document])
     const chain = loadSummarizationChain(llm, {
       type: 'map_reduce', // you can choose from map_reduce, stuff or refine
       verbose: true, // to view the steps in the console
     })
-    const response = await chain.invoke({
+    const response = await chain.call({
       input_documents: docs,
     })
 
-    console.log('summary response: ', JSON.stringify(response))
-
     if (typeof response.text !== 'string') {
       logger.error(`AI summary did not return text`)
       return
diff --git a/packages/api/src/services/ai-summaries.ts b/packages/api/src/services/ai-summaries.ts
index 9b9ebefd8..3c94ca4cd 100644
--- a/packages/api/src/services/ai-summaries.ts
+++ b/packages/api/src/services/ai-summaries.ts
@@ -1,71 +1,5 @@
-import { ChatOpenAI } from '@langchain/openai'
 import { AISummary } from '../entity/AISummary'
 import { authTrx } from '../repository'
-import { RecursiveCharacterTextSplitter } from 'langchain/text_splitter'
-import { loadSummarizationChain } from 'langchain/chains'
-import { logger } from '../utils/logger'
-import { NodeHtmlMarkdown, TranslatorConfigObject } from 'node-html-markdown'
-
-// When creating markdown we remove external links in URLs
-// and images since these often contain per-user trackers
-// that can interfere with caching
-const removeLinksTransformer: TranslatorConfigObject = {
-  a: ({ node, options, visitor }) => {
-    return {
-      postprocess: ({ content }) => {
-        return `[${content}]()`
-      },
-    }
-  },
-  img: ({ node, options, visitor }) => {
-    const alt = node.getAttribute('alt')?.trim()
-    return {
-      content: `![${alt}]()`,
-    }
-  },
-}
-
-export const createSummarizableDocument = (readable: string): string => {
-  const nhm = new NodeHtmlMarkdown(
-    {
-      keepDataImages: false,
-    },
-    removeLinksTransformer
-  )
-  return nhm.translate(readable)
-}
-
-export const createAISummary = async (
-  readableContent: string
-): Promise<string | undefined> => {
-  const llm = new ChatOpenAI({
-    configuration: {
-      apiKey: process.env.OPENAI_API_KEY,
-    },
-  })
-  const textSplitter = new RecursiveCharacterTextSplitter({
-    chunkSize: 12000,
-  })
-
-  const document = createSummarizableDocument(readableContent)
-  const docs = await textSplitter.createDocuments([document])
-  const chain = loadSummarizationChain(llm, {
-    type: 'map_reduce', // you can choose from map_reduce, stuff or refine
-    verbose: true, // to view the steps in the console
-  })
-  const response = await chain.invoke({
-    input_documents: docs,
-  })
-
-  console.log('summary response: ', JSON.stringify(response))
-
-  if (typeof response.text !== 'string') {
-    logger.error(`AI summary did not return text`)
-    return
-  }
-
-  return response.text
-}
 
 export const getAISummary = async (data: {
   userId: string