From 94db3e0de7cc9efab6669f66197ca3f713c5ffa1 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Mon, 4 Mar 2024 17:30:05 +0800 Subject: [PATCH 01/10] Add authorization with notion --- .../api/src/routers/integration_router.ts | 32 +++-------- .../api/src/services/integrations/index.ts | 6 ++- .../src/services/integrations/integration.ts | 2 + .../api/src/services/integrations/notion.ts | 54 +++++++++++++++++++ .../api/src/services/integrations/pocket.ts | 23 ++++++++ .../api/src/services/integrations/readwise.ts | 14 +++-- packages/api/src/util.ts | 14 +++++ 7 files changed, 115 insertions(+), 30 deletions(-) create mode 100644 packages/api/src/services/integrations/notion.ts diff --git a/packages/api/src/routers/integration_router.ts b/packages/api/src/routers/integration_router.ts index cc6a520a7..d9a89f43a 100644 --- a/packages/api/src/routers/integration_router.ts +++ b/packages/api/src/routers/integration_router.ts @@ -2,6 +2,7 @@ import axios from 'axios' import cors from 'cors' import express from 'express' import { env } from '../env' +import { getIntegrationClient } from '../services/integrations' import { getClaimsByToken } from '../utils/auth' import { corsConfig } from '../utils/corsConfig' import { logger } from '../utils/logger' @@ -10,10 +11,9 @@ export function integrationRouter() { const router = express.Router() // request token from pocket router.post( - '/pocket/auth', + '/:name/auth', cors(corsConfig), async (req: express.Request, res: express.Response) => { - logger.info('pocket/request-token') // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access const token = (req.cookies.auth as string) || req.headers.authorization const claims = await getClaimsByToken(token) @@ -21,37 +21,19 @@ export function integrationRouter() { return res.status(401).send('UNAUTHORIZED') } - const consumerKey = env.pocket.consumerKey - const redirectUri = `${env.client.url}/settings/integrations` + const integrationClient = getIntegrationClient(req.params.name) + // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access const state = req.body.state as string try { - // make a POST request to Pocket to get a request token - const response = await axios.post<{ code: string }>( - 'https://getpocket.com/v3/oauth/request', - { - consumer_key: consumerKey, - redirect_uri: redirectUri, - }, - { - headers: { - 'Content-Type': 'application/json', - 'X-Accept': 'application/json', - }, - } - ) - const { code } = response.data + const redirectUri = await integrationClient.auth(state) // redirect the user to Pocket to authorize the request token - res.redirect( - `https://getpocket.com/auth/authorize?request_token=${code}&redirect_uri=${redirectUri}${encodeURIComponent( - `?pocketToken=${code}&state=${state}` - )}` - ) + res.redirect(redirectUri) } catch (error) { if (axios.isAxiosError(error)) { logger.error(error.response) } else { - logger.error('pocket/request-token exception:', error) + logger.error(error) } res.redirect( diff --git a/packages/api/src/services/integrations/index.ts b/packages/api/src/services/integrations/index.ts index 286ac59e7..435ca98e2 100644 --- a/packages/api/src/services/integrations/index.ts +++ b/packages/api/src/services/integrations/index.ts @@ -2,16 +2,20 @@ import { DeepPartial, FindOptionsWhere } from 'typeorm' import { Integration } from '../../entity/integration' import { authTrx } from '../../repository' import { IntegrationClient } from './integration' +import { NotionClient } from './notion' import { PocketClient } from './pocket' import { ReadwiseClient } from './readwise' const integrations: IntegrationClient[] = [ new ReadwiseClient(), new PocketClient(), + new NotionClient(), ] export const getIntegrationClient = (name: string): IntegrationClient => { - const service = integrations.find((s) => s.name === name) + const service = integrations.find( + (s) => s.name.toLowerCase() === name.toLowerCase() + ) if (!service) { throw new Error(`Integration client not found: ${name}`) } diff --git a/packages/api/src/services/integrations/integration.ts b/packages/api/src/services/integrations/integration.ts index e3f1edbc8..f5183f097 100644 --- a/packages/api/src/services/integrations/integration.ts +++ b/packages/api/src/services/integrations/integration.ts @@ -24,5 +24,7 @@ export interface IntegrationClient { accessToken(token: string): Promise + auth(state: string): Promise + export(token: string, items: LibraryItem[]): Promise } diff --git a/packages/api/src/services/integrations/notion.ts b/packages/api/src/services/integrations/notion.ts new file mode 100644 index 000000000..8e8cedcad --- /dev/null +++ b/packages/api/src/services/integrations/notion.ts @@ -0,0 +1,54 @@ +import axios from 'axios' +import { env } from '../../env' +import { logger } from '../../utils/logger' +import { IntegrationClient } from './integration' + +export class NotionClient implements IntegrationClient { + name = 'notion' + apiUrl = 'https://api.notion.com/v1' + headers = { + 'Content-Type': 'application/json', + Accept: 'application/json', + } + + accessToken = async (code: string): Promise => { + const authUrl = `${this.apiUrl}/oauth/token` + try { + // encode in base 64 + const encoded = Buffer.from( + `${env.notion.clientId}:${env.notion.clientSecret}` + ).toString('base64') + + const response = await axios.post<{ access_token: string }>( + authUrl, + { + grant_type: 'authorization_code', + code, + }, + { + headers: { + authorization: `Basic ${encoded}`, + ...this.headers, + }, + timeout: 5000, // 5 seconds + } + ) + return response.data.access_token + } catch (error) { + if (axios.isAxiosError(error)) { + logger.error(error.response) + } else { + logger.error(error) + } + return null + } + } + + async auth(state: string): Promise { + return Promise.resolve(env.notion.authUrl) + } + + export = () => { + throw new Error('Method not implemented.') + } +} diff --git a/packages/api/src/services/integrations/pocket.ts b/packages/api/src/services/integrations/pocket.ts index 517d3befa..661d1ba0d 100644 --- a/packages/api/src/services/integrations/pocket.ts +++ b/packages/api/src/services/integrations/pocket.ts @@ -39,4 +39,27 @@ export class PocketClient implements IntegrationClient { export = async (): Promise => { return Promise.resolve(false) } + + async auth(state: string) { + const consumerKey = env.pocket.consumerKey + const redirectUri = `${env.client.url}/settings/integrations` + + // make a POST request to Pocket to get a request token + const response = await axios.post<{ code: string }>( + `${this.apiUrl}/oauth/request`, + { + consumer_key: consumerKey, + redirect_uri: redirectUri, + }, + { + headers: this.headers, + timeout: 5000, // 5 seconds + } + ) + const { code } = response.data + + return `https://getpocket.com/auth/authorize?request_token=${code}&redirect_uri=${redirectUri}${encodeURIComponent( + `?pocketToken=${code}&state=${state}` + )}` + } } diff --git a/packages/api/src/services/integrations/readwise.ts b/packages/api/src/services/integrations/readwise.ts index ae23810bd..e82597a9c 100644 --- a/packages/api/src/services/integrations/readwise.ts +++ b/packages/api/src/services/integrations/readwise.ts @@ -57,17 +57,23 @@ export class ReadwiseClient implements IntegrationClient { export = async (token: string, items: LibraryItem[]): Promise => { let result = true - const highlights = items.flatMap(this.itemToReadwiseHighlight) + const highlights = items.flatMap(this._itemToReadwiseHighlight) // If there are no highlights, we will skip the sync if (highlights.length > 0) { - result = await this.syncWithReadwise(token, highlights) + result = await this._syncWithReadwise(token, highlights) } return result } - itemToReadwiseHighlight = (item: LibraryItem): ReadwiseHighlight[] => { + auth(state: string): Promise { + throw new Error('Method not implemented.') + } + + private _itemToReadwiseHighlight = ( + item: LibraryItem + ): ReadwiseHighlight[] => { const category = item.siteName === 'Twitter' ? 'tweets' : 'articles' return item.highlights ?.map((highlight) => { @@ -93,7 +99,7 @@ export class ReadwiseClient implements IntegrationClient { .filter((highlight) => highlight !== undefined) as ReadwiseHighlight[] } - syncWithReadwise = async ( + private _syncWithReadwise = async ( token: string, highlights: ReadwiseHighlight[] ): Promise => { diff --git a/packages/api/src/util.ts b/packages/api/src/util.ts index b03a19c15..d42f84ce1 100755 --- a/packages/api/src/util.ts +++ b/packages/api/src/util.ts @@ -113,6 +113,11 @@ export interface BackendEnv { mq: redisConfig cache: redisConfig } + notion: { + clientId: string + clientSecret: string + authUrl: string + } } const nullableEnvVars = [ @@ -165,6 +170,9 @@ const nullableEnvVars = [ 'MQ_REDIS_CERT', 'IMPORTER_METRICS_COLLECTOR_URL', 'INTERNAL_API_URL', + 'NOTION_CLIENT_ID', + 'NOTION_CLIENT_SECRET', + 'NOTION_AUTH_URL', ] // Allow some vars to be null/empty /* If not in GAE and Prod/QA/Demo env (f.e. on localhost/dev env), allow following env vars to be null */ @@ -311,6 +319,11 @@ export function getEnv(): BackendEnv { cert: parse('REDIS_CERT')?.replace(/\\n/g, '\n'), // replace \n with new line }, } + const notion = { + clientId: parse('NOTION_CLIENT_ID'), + clientSecret: parse('NOTION_CLIENT_SECRET'), + authUrl: parse('NOTION_AUTH_URL'), + } return { pg, @@ -333,6 +346,7 @@ export function getEnv(): BackendEnv { pocket, subscription, redis, + notion, } } From d36c937765475baed56694aea26890231bb17217 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Mon, 4 Mar 2024 17:49:59 +0800 Subject: [PATCH 02/10] fix redirect_uri error --- packages/api/src/services/integrations/notion.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/api/src/services/integrations/notion.ts b/packages/api/src/services/integrations/notion.ts index 8e8cedcad..fd46dc1bb 100644 --- a/packages/api/src/services/integrations/notion.ts +++ b/packages/api/src/services/integrations/notion.ts @@ -4,7 +4,7 @@ import { logger } from '../../utils/logger' import { IntegrationClient } from './integration' export class NotionClient implements IntegrationClient { - name = 'notion' + name = 'NOTION' apiUrl = 'https://api.notion.com/v1' headers = { 'Content-Type': 'application/json', @@ -24,6 +24,7 @@ export class NotionClient implements IntegrationClient { { grant_type: 'authorization_code', code, + redirect_uri: `${env.client.url}/settings/integrations`, }, { headers: { From e20908a0c29c8a371433fcc3dc8c7dfbdd138f00 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Mon, 4 Mar 2024 18:44:25 +0800 Subject: [PATCH 03/10] add export to notion method --- .../api/src/services/integrations/notion.ts | 94 ++++++++++++++++++- .../api/src/services/integrations/pocket.ts | 4 +- .../api/src/services/integrations/readwise.ts | 2 +- 3 files changed, 93 insertions(+), 7 deletions(-) diff --git a/packages/api/src/services/integrations/notion.ts b/packages/api/src/services/integrations/notion.ts index fd46dc1bb..bcd6573e0 100644 --- a/packages/api/src/services/integrations/notion.ts +++ b/packages/api/src/services/integrations/notion.ts @@ -1,15 +1,46 @@ import axios from 'axios' +import { LibraryItem } from '../../entity/library_item' import { env } from '../../env' import { logger } from '../../utils/logger' import { IntegrationClient } from './integration' +interface NotionPage { + parent: { + database_id: string + } + cover?: { + external: { + url: string + } + } + properties: { + Name: { + title: Array<{ + text: { + content: string + } + }> + } + URL: { + url: string + } + Tags: { + multi_select: Array<{ + name: string + }> + } + } +} + export class NotionClient implements IntegrationClient { name = 'NOTION' apiUrl = 'https://api.notion.com/v1' headers = { 'Content-Type': 'application/json', Accept: 'application/json', + 'Notion-Version': '2022-06-28', } + timeout = 5000 // 5 seconds accessToken = async (code: string): Promise => { const authUrl = `${this.apiUrl}/oauth/token` @@ -31,7 +62,7 @@ export class NotionClient implements IntegrationClient { authorization: `Basic ${encoded}`, ...this.headers, }, - timeout: 5000, // 5 seconds + timeout: this.timeout, } ) return response.data.access_token @@ -45,11 +76,66 @@ export class NotionClient implements IntegrationClient { } } - async auth(state: string): Promise { + async auth(): Promise { return Promise.resolve(env.notion.authUrl) } - export = () => { - throw new Error('Method not implemented.') + private _itemToNotionPage = (item: LibraryItem): NotionPage => { + return { + parent: { + database_id: item.id, + }, + cover: item.thumbnail + ? { + external: { + url: item.thumbnail, + }, + } + : undefined, + properties: { + Name: { + title: [ + { + text: { + content: item.title, + }, + }, + ], + }, + URL: { + url: item.originalUrl, + }, + Tags: { + multi_select: + item.labels?.map((label) => { + return { + name: label.name, + } + }) || [], + }, + }, + } + } + + export = async (token: string, items: LibraryItem[]): Promise => { + const url = `${this.apiUrl}/pages` + const page = this._itemToNotionPage(items[0]) + try { + const response = await axios.post(url, page, { + headers: { + Authorization: `Bearer ${token}`, + ...this.headers, + }, + timeout: this.timeout, + }) + return response.status === 200 + } catch (error) { + if (axios.isAxiosError(error)) { + logger.error(error.response) + } else { + logger.error(error) + } + return false + } } } diff --git a/packages/api/src/services/integrations/pocket.ts b/packages/api/src/services/integrations/pocket.ts index 661d1ba0d..8f9e8ed50 100644 --- a/packages/api/src/services/integrations/pocket.ts +++ b/packages/api/src/services/integrations/pocket.ts @@ -36,8 +36,8 @@ export class PocketClient implements IntegrationClient { } } - export = async (): Promise => { - return Promise.resolve(false) + export = () => { + throw new Error('Method not implemented.') } async auth(state: string) { diff --git a/packages/api/src/services/integrations/readwise.ts b/packages/api/src/services/integrations/readwise.ts index e82597a9c..84b15a25a 100644 --- a/packages/api/src/services/integrations/readwise.ts +++ b/packages/api/src/services/integrations/readwise.ts @@ -67,7 +67,7 @@ export class ReadwiseClient implements IntegrationClient { return result } - auth(state: string): Promise { + auth = () => { throw new Error('Method not implemented.') } From 741f3213d94d39133a58b503fe9f157aba04da0d Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Tue, 5 Mar 2024 12:43:18 +0800 Subject: [PATCH 04/10] integrate with notion api client --- packages/api/package.json | 1 + .../api/src/jobs/integration/export_item.ts | 4 +- .../api/src/resolvers/integrations/index.ts | 4 +- .../api/src/routers/integration_router.ts | 2 +- .../api/src/services/integrations/index.ts | 26 ++-- .../src/services/integrations/integration.ts | 6 +- .../api/src/services/integrations/notion.ts | 115 ++++++++---------- .../api/src/services/integrations/pocket.ts | 37 +++--- .../api/src/services/integrations/readwise.ts | 38 +++--- yarn.lock | 24 ++-- 10 files changed, 127 insertions(+), 130 deletions(-) diff --git a/packages/api/package.json b/packages/api/package.json index 111fc6502..b387616f4 100644 --- a/packages/api/package.json +++ b/packages/api/package.json @@ -24,6 +24,7 @@ "@google-cloud/tasks": "^4.0.0", "@graphql-tools/utils": "^9.1.1", "@langchain/openai": "^0.0.14", + "@notionhq/client": "^2.2.14", "@omnivore/content-handler": "1.0.0", "@omnivore/liqe": "1.0.0", "@omnivore/readability": "1.0.0", diff --git a/packages/api/src/jobs/integration/export_item.ts b/packages/api/src/jobs/integration/export_item.ts index 327f16206..a2a0f1456 100644 --- a/packages/api/src/jobs/integration/export_item.ts +++ b/packages/api/src/jobs/integration/export_item.ts @@ -44,9 +44,9 @@ export const exportItem = async (jobData: ExportItemJobData) => { } logger.info('exporting item...', logObject) - const client = getIntegrationClient(integration.name) + const client = getIntegrationClient(integration.name, integration.token) - const synced = await client.export(integration.token, libraryItems) + const synced = await client.export(libraryItems) if (!synced) { logger.error('failed to export item', logObject) return false diff --git a/packages/api/src/resolvers/integrations/index.ts b/packages/api/src/resolvers/integrations/index.ts index 90c118129..5148d0893 100644 --- a/packages/api/src/resolvers/integrations/index.ts +++ b/packages/api/src/resolvers/integrations/index.ts @@ -69,9 +69,9 @@ export const setIntegrationResolver = authorized< integrationToSave.taskName = existingIntegration.taskName } else { // Create - const integrationService = getIntegrationClient(input.name) + const integrationService = getIntegrationClient(input.name, input.token) // authorize and get access token - const token = await integrationService.accessToken(input.token) + const token = await integrationService.accessToken() if (!token) { return { errorCodes: [SetIntegrationErrorCode.InvalidToken], diff --git a/packages/api/src/routers/integration_router.ts b/packages/api/src/routers/integration_router.ts index d9a89f43a..7bf5cbdc6 100644 --- a/packages/api/src/routers/integration_router.ts +++ b/packages/api/src/routers/integration_router.ts @@ -21,7 +21,7 @@ export function integrationRouter() { return res.status(401).send('UNAUTHORIZED') } - const integrationClient = getIntegrationClient(req.params.name) + const integrationClient = getIntegrationClient(req.params.name, '') // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access const state = req.body.state as string diff --git a/packages/api/src/services/integrations/index.ts b/packages/api/src/services/integrations/index.ts index 435ca98e2..953199052 100644 --- a/packages/api/src/services/integrations/index.ts +++ b/packages/api/src/services/integrations/index.ts @@ -6,20 +6,20 @@ import { NotionClient } from './notion' import { PocketClient } from './pocket' import { ReadwiseClient } from './readwise' -const integrations: IntegrationClient[] = [ - new ReadwiseClient(), - new PocketClient(), - new NotionClient(), -] - -export const getIntegrationClient = (name: string): IntegrationClient => { - const service = integrations.find( - (s) => s.name.toLowerCase() === name.toLowerCase() - ) - if (!service) { - throw new Error(`Integration client not found: ${name}`) +export const getIntegrationClient = ( + name: string, + token: string +): IntegrationClient => { + switch (name.toLowerCase()) { + case 'readwise': + return new ReadwiseClient(token) + case 'pocket': + return new PocketClient(token) + case 'notion': + return new NotionClient(token) + default: + throw new Error(`Integration client not found: ${name}`) } - return service } export const deleteIntegrations = async ( diff --git a/packages/api/src/services/integrations/integration.ts b/packages/api/src/services/integrations/integration.ts index f5183f097..13b61e82a 100644 --- a/packages/api/src/services/integrations/integration.ts +++ b/packages/api/src/services/integrations/integration.ts @@ -20,11 +20,11 @@ export interface RetrieveRequest { export interface IntegrationClient { name: string - apiUrl: string + _token: string - accessToken(token: string): Promise + accessToken(): Promise auth(state: string): Promise - export(token: string, items: LibraryItem[]): Promise + export(items: LibraryItem[]): Promise } diff --git a/packages/api/src/services/integrations/notion.ts b/packages/api/src/services/integrations/notion.ts index bcd6573e0..539c1a3ef 100644 --- a/packages/api/src/services/integrations/notion.ts +++ b/packages/api/src/services/integrations/notion.ts @@ -1,3 +1,4 @@ +import { Client } from '@notionhq/client' import axios from 'axios' import { LibraryItem } from '../../entity/library_item' import { env } from '../../env' @@ -6,7 +7,7 @@ import { IntegrationClient } from './integration' interface NotionPage { parent: { - database_id: string + page_id: string } cover?: { external: { @@ -14,55 +15,56 @@ interface NotionPage { } } properties: { - Name: { - title: Array<{ - text: { - content: string - } - }> - } - URL: { - url: string - } - Tags: { - multi_select: Array<{ - name: string - }> - } + title: Array<{ + text: { + content: string + } + }> } } export class NotionClient implements IntegrationClient { name = 'NOTION' - apiUrl = 'https://api.notion.com/v1' - headers = { + _headers = { 'Content-Type': 'application/json', Accept: 'application/json', 'Notion-Version': '2022-06-28', } - timeout = 5000 // 5 seconds + _timeout = 5000 // 5 seconds + _axios = axios.create({ + baseURL: 'https://api.notion.com/v1', + timeout: this._timeout, + }) + _token: string + _client: Client - accessToken = async (code: string): Promise => { - const authUrl = `${this.apiUrl}/oauth/token` + constructor(token: string) { + this._token = token + this._client = new Client({ + auth: token, + timeoutMs: this._timeout, + }) + } + + accessToken = async (): Promise => { try { // encode in base 64 const encoded = Buffer.from( `${env.notion.clientId}:${env.notion.clientSecret}` ).toString('base64') - const response = await axios.post<{ access_token: string }>( - authUrl, + const response = await this._axios.post<{ access_token: string }>( + '/oauth/token', { grant_type: 'authorization_code', - code, + code: this._token, redirect_uri: `${env.client.url}/settings/integrations`, }, { headers: { - authorization: `Basic ${encoded}`, - ...this.headers, + ...this._headers, + Authorization: `Basic ${encoded}`, }, - timeout: this.timeout, } ) return response.data.access_token @@ -83,7 +85,7 @@ export class NotionClient implements IntegrationClient { private _itemToNotionPage = (item: LibraryItem): NotionPage => { return { parent: { - database_id: item.id, + page_id: '83a3f627ab9e44ac83fe657141aec615', }, cover: item.thumbnail ? { @@ -93,49 +95,28 @@ export class NotionClient implements IntegrationClient { } : undefined, properties: { - Name: { - title: [ - { - text: { - content: item.title, - }, + title: [ + { + text: { + content: item.title, }, - ], - }, - URL: { - url: item.originalUrl, - }, - Tags: { - multi_select: - item.labels?.map((label) => { - return { - name: label.name, - } - }) || [], - }, + }, + ], }, } } - export = async (token: string, items: LibraryItem[]): Promise => { - const url = `${this.apiUrl}/pages` - const page = this._itemToNotionPage(items[0]) - try { - const response = await axios.post(url, page, { - headers: { - Authorization: `Bearer ${token}`, - ...this.headers, - }, - timeout: this.timeout, - }) - return response.status === 200 - } catch (error) { - if (axios.isAxiosError(error)) { - logger.error(error.response) - } else { - logger.error(error) - } - return false - } + _createPage = async (page: NotionPage) => { + await this._client.pages.create(page) + } + + export = async (items: LibraryItem[]): Promise => { + // find/create a parent page for all the items + const parentPageName = 'Omnivore' + + const pages = items.map(this._itemToNotionPage) + await Promise.all(pages.map((page) => this._createPage(page))) + + return true } } diff --git a/packages/api/src/services/integrations/pocket.ts b/packages/api/src/services/integrations/pocket.ts index 8f9e8ed50..20f4c212c 100644 --- a/packages/api/src/services/integrations/pocket.ts +++ b/packages/api/src/services/integrations/pocket.ts @@ -5,24 +5,27 @@ import { IntegrationClient } from './integration' export class PocketClient implements IntegrationClient { name = 'POCKET' - apiUrl = 'https://getpocket.com/v3' - headers = { - 'Content-Type': 'application/json', - 'X-Accept': 'application/json', + _token: string + _axios = axios.create({ + baseURL: 'https://getpocket.com/v3', + headers: { + 'Content-Type': 'application/json', + 'X-Accept': 'application/json', + }, + timeout: 5000, // 5 seconds + }) + + constructor(token: string) { + this._token = token } - accessToken = async (token: string): Promise => { - const url = `${this.apiUrl}/oauth/authorize` + accessToken = async (): Promise => { try { - const response = await axios.post<{ access_token: string }>( - url, + const response = await this._axios.post<{ access_token: string }>( + '/oauth/authorize', { consumer_key: env.pocket.consumerKey, - code: token, - }, - { - headers: this.headers, - timeout: 5000, // 5 seconds + code: this._token, } ) return response.data.access_token @@ -45,15 +48,11 @@ export class PocketClient implements IntegrationClient { const redirectUri = `${env.client.url}/settings/integrations` // make a POST request to Pocket to get a request token - const response = await axios.post<{ code: string }>( - `${this.apiUrl}/oauth/request`, + const response = await this._axios.post<{ code: string }>( + '/oauth/request', { consumer_key: consumerKey, redirect_uri: redirectUri, - }, - { - headers: this.headers, - timeout: 5000, // 5 seconds } ) const { code } = response.data diff --git a/packages/api/src/services/integrations/readwise.ts b/packages/api/src/services/integrations/readwise.ts index 84b15a25a..25a61f438 100644 --- a/packages/api/src/services/integrations/readwise.ts +++ b/packages/api/src/services/integrations/readwise.ts @@ -33,17 +33,28 @@ interface ReadwiseHighlight { export class ReadwiseClient implements IntegrationClient { name = 'READWISE' - apiUrl = 'https://readwise.io/api/v2' + _headers = { + 'Content-Type': 'application/json', + } + _axios = axios.create({ + baseURL: 'https://readwise.io/api/v2', + timeout: 5000, // 5 seconds + }) + _token: string - accessToken = async (token: string): Promise => { - const authUrl = `${this.apiUrl}/auth` + constructor(token: string) { + this._token = token + } + + accessToken = async (): Promise => { try { - const response = await axios.get(authUrl, { + const response = await this._axios.get('/auth', { headers: { - Authorization: `Token ${token}`, + ...this._headers, + Authorization: `Token ${this._token}`, }, }) - return response.status === 204 ? token : null + return response.status === 204 ? this._token : null } catch (error) { if (axios.isAxiosError(error)) { logger.error(error.response) @@ -54,14 +65,14 @@ export class ReadwiseClient implements IntegrationClient { } } - export = async (token: string, items: LibraryItem[]): Promise => { + export = async (items: LibraryItem[]): Promise => { let result = true const highlights = items.flatMap(this._itemToReadwiseHighlight) // If there are no highlights, we will skip the sync if (highlights.length > 0) { - result = await this._syncWithReadwise(token, highlights) + result = await this._syncWithReadwise(highlights) } return result @@ -100,21 +111,18 @@ export class ReadwiseClient implements IntegrationClient { } private _syncWithReadwise = async ( - token: string, highlights: ReadwiseHighlight[] ): Promise => { - const url = `${this.apiUrl}/highlights` - const response = await axios.post( - url, + const response = await this._axios.post( + '/highlights', { highlights, }, { headers: { - Authorization: `Token ${token}`, - 'Content-Type': 'application/json', + ...this._headers, + Authorization: `Token ${this._token}`, }, - timeout: 5000, // 5 seconds } ) return response.status === 200 diff --git a/yarn.lock b/yarn.lock index 8631bcdbc..cb71dd339 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3931,6 +3931,14 @@ "@nodelib/fs.scandir" "2.1.3" fastq "^1.6.0" +"@notionhq/client@^2.2.14": + version "2.2.14" + resolved "https://registry.yarnpkg.com/@notionhq/client/-/client-2.2.14.tgz#6807ec27ee89584529abfd28d058b2661f828b74" + integrity sha512-oqUefZtCiJPCX+74A1Os9OVTef3fSnVWe2eVQtU1HJSD+nsfxfhwvDKnzJTh2Tw1ZHKLxpieHB/nzGdY+Uo12A== + dependencies: + "@types/node-fetch" "^2.5.10" + node-fetch "^2.6.1" + "@npmcli/arborist@^5.6.3": version "5.6.3" resolved "https://registry.yarnpkg.com/@npmcli/arborist/-/arborist-5.6.3.tgz#40810080272e097b4a7a4f56108f4a31638a9874" @@ -7877,6 +7885,14 @@ dependencies: "@types/node" "*" +"@types/node-fetch@^2.5.10", "@types/node-fetch@^2.6.4": + version "2.6.11" + resolved "https://registry.yarnpkg.com/@types/node-fetch/-/node-fetch-2.6.11.tgz#9b39b78665dae0e82a08f02f4967d62c66f95d24" + integrity sha512-24xFj9R5+rfQJLRyM56qh+wnVSYhyXC2tkoBndtY0U+vubqNsYXGjufB2nn8Q6gt0LrARwL6UBtMCSVCwl4B1g== + dependencies: + "@types/node" "*" + form-data "^4.0.0" + "@types/node-fetch@^2.5.7": version "2.6.1" resolved "https://registry.yarnpkg.com/@types/node-fetch/-/node-fetch-2.6.1.tgz#8f127c50481db65886800ef496f20bbf15518975" @@ -7885,14 +7901,6 @@ "@types/node" "*" form-data "^3.0.0" -"@types/node-fetch@^2.6.4": - version "2.6.11" - resolved "https://registry.yarnpkg.com/@types/node-fetch/-/node-fetch-2.6.11.tgz#9b39b78665dae0e82a08f02f4967d62c66f95d24" - integrity sha512-24xFj9R5+rfQJLRyM56qh+wnVSYhyXC2tkoBndtY0U+vubqNsYXGjufB2nn8Q6gt0LrARwL6UBtMCSVCwl4B1g== - dependencies: - "@types/node" "*" - form-data "^4.0.0" - "@types/node-fetch@^2.6.6": version "2.6.7" resolved "https://registry.yarnpkg.com/@types/node-fetch/-/node-fetch-2.6.7.tgz#a1abe2ce24228b58ad97f99480fdcf9bbc6ab16d" From f1b1f2c4c10a7981b430c6db830e30ac20aeab7c Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Tue, 5 Mar 2024 15:39:09 +0800 Subject: [PATCH 05/10] send created event only --- .../api/src/jobs/integration/export_item.ts | 69 +++++++++++-------- .../api/src/services/integrations/notion.ts | 11 +-- packages/api/src/services/library_item.ts | 2 +- 3 files changed, 47 insertions(+), 35 deletions(-) diff --git a/packages/api/src/jobs/integration/export_item.ts b/packages/api/src/jobs/integration/export_item.ts index a2a0f1456..a1b8f7cc2 100644 --- a/packages/api/src/jobs/integration/export_item.ts +++ b/packages/api/src/jobs/integration/export_item.ts @@ -35,41 +35,50 @@ export const exportItem = async (jobData: ExportItemJobData) => { return } - // currently only readwise integration is supported - const integration = integrations[0] + await Promise.all( + integrations.map(async (integration) => { + try { + const logObject = { + userId, + integrationId: integration.id, + } + logger.info('exporting item...', logObject) - const logObject = { - userId, - integrationId: integration.id, - } - logger.info('exporting item...', logObject) + const client = getIntegrationClient(integration.name, integration.token) - const client = getIntegrationClient(integration.name, integration.token) + const synced = await client.export(libraryItems) + if (!synced) { + logger.error('failed to export item', logObject) + return false + } - const synced = await client.export(libraryItems) - if (!synced) { - logger.error('failed to export item', logObject) - return false - } + const syncedAt = new Date() + logger.info('updating integration...', { + ...logObject, + syncedAt, + }) - const syncedAt = new Date() - logger.info('updating integration...', { - ...logObject, - syncedAt, - }) - - // update integration syncedAt if successful - const updated = await updateIntegration( - integration.id, - { - syncedAt, - }, - userId + // update integration syncedAt if successful + const updated = await updateIntegration( + integration.id, + { + syncedAt, + }, + userId + ) + logger.info('integration updated', { + ...logObject, + updated, + }) + } catch (error) { + logger.error('failed to export item', { + userId, + integrationId: integration.id, + error, + }) + } + }) ) - logger.info('integration updated', { - ...logObject, - updated, - }) return true } diff --git a/packages/api/src/services/integrations/notion.ts b/packages/api/src/services/integrations/notion.ts index 539c1a3ef..868a5ec57 100644 --- a/packages/api/src/services/integrations/notion.ts +++ b/packages/api/src/services/integrations/notion.ts @@ -35,6 +35,8 @@ export class NotionClient implements IntegrationClient { baseURL: 'https://api.notion.com/v1', timeout: this._timeout, }) + _parentPageId = process.env.NOTION_PAGE_ID + _token: string _client: Client @@ -83,9 +85,13 @@ export class NotionClient implements IntegrationClient { } private _itemToNotionPage = (item: LibraryItem): NotionPage => { + if (!this._parentPageId) { + throw new Error('Notion parent page ID is not set') + } + return { parent: { - page_id: '83a3f627ab9e44ac83fe657141aec615', + page_id: this._parentPageId, }, cover: item.thumbnail ? { @@ -111,9 +117,6 @@ export class NotionClient implements IntegrationClient { } export = async (items: LibraryItem[]): Promise => { - // find/create a parent page for all the items - const parentPageName = 'Omnivore' - const pages = items.map(this._itemToNotionPage) await Promise.all(pages.map((page) => this._createPage(page))) diff --git a/packages/api/src/services/library_item.ts b/packages/api/src/services/library_item.ts index 4a16c7c18..784b79c8e 100644 --- a/packages/api/src/services/library_item.ts +++ b/packages/api/src/services/library_item.ts @@ -989,7 +989,7 @@ export const createOrUpdateLibraryItem = async ( ) } - if (skipPubSub) { + if (skipPubSub || libraryItem.state === LibraryItemState.Processing) { return newLibraryItem } From 1edf801b024ebe32bd7f18cfa420320e618e3221 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Tue, 5 Mar 2024 16:13:50 +0800 Subject: [PATCH 06/10] add settings jsonb column to the integrations --- packages/api/src/entity/integration.ts | 3 +++ packages/api/src/generated/graphql.ts | 7 +++++++ packages/api/src/generated/schema.graphql | 4 ++++ packages/api/src/jobs/integration/export_item.ts | 6 +++++- packages/api/src/resolvers/integrations/index.ts | 8 +++++++- packages/api/src/routers/integration_router.ts | 2 +- packages/api/src/schema.ts | 2 ++ packages/api/src/services/integrations/index.ts | 5 +++-- packages/api/src/services/integrations/notion.ts | 8 +++++++- .../0167.do.add_settings_column_to_integrations.sql | 9 +++++++++ .../0167.undo.add_settings_column_to_integrations.sql | 9 +++++++++ 11 files changed, 57 insertions(+), 6 deletions(-) create mode 100755 packages/db/migrations/0167.do.add_settings_column_to_integrations.sql create mode 100755 packages/db/migrations/0167.undo.add_settings_column_to_integrations.sql diff --git a/packages/api/src/entity/integration.ts b/packages/api/src/entity/integration.ts index e446f10c3..dcb7f03c1 100644 --- a/packages/api/src/entity/integration.ts +++ b/packages/api/src/entity/integration.ts @@ -59,4 +59,7 @@ export class Integration { @Column('enum', { enum: ImportItemState, nullable: true }) importItemState?: ImportItemState | null + + @Column('jsonb', { nullable: true }) + settings?: any } diff --git a/packages/api/src/generated/graphql.ts b/packages/api/src/generated/graphql.ts index 581880e4f..ec46d4c65 100644 --- a/packages/api/src/generated/graphql.ts +++ b/packages/api/src/generated/graphql.ts @@ -1085,6 +1085,7 @@ export type Integration = { enabled: Scalars['Boolean']; id: Scalars['ID']; name: Scalars['String']; + settings?: Maybe; taskName?: Maybe; token: Scalars['String']; type: IntegrationType; @@ -2404,6 +2405,7 @@ export enum SearchErrorCode { export type SearchItem = { __typename?: 'SearchItem'; + aiSummary?: Maybe; annotation?: Maybe; archivedAt?: Maybe; author?: Maybe; @@ -2586,6 +2588,7 @@ export type SetIntegrationInput = { id?: InputMaybe; importItemState?: InputMaybe; name: Scalars['String']; + settings?: InputMaybe; syncedAt?: InputMaybe; taskName?: InputMaybe; token: Scalars['String']; @@ -3378,6 +3381,7 @@ export enum UploadImportFileType { export type User = { __typename?: 'User'; email?: Maybe; + features?: Maybe>>; followersCount?: Maybe; friendsCount?: Maybe; id: Scalars['ID']; @@ -5310,6 +5314,7 @@ export type IntegrationResolvers; id?: Resolver; name?: Resolver; + settings?: Resolver, ParentType, ContextType>; taskName?: Resolver, ParentType, ContextType>; token?: Resolver; type?: Resolver; @@ -5945,6 +5950,7 @@ export type SearchErrorResolvers = { + aiSummary?: Resolver, ParentType, ContextType>; annotation?: Resolver, ParentType, ContextType>; archivedAt?: Resolver, ParentType, ContextType>; author?: Resolver, ParentType, ContextType>; @@ -6528,6 +6534,7 @@ export type UploadImportFileSuccessResolvers = { email?: Resolver, ParentType, ContextType>; + features?: Resolver>>, ParentType, ContextType>; followersCount?: Resolver, ParentType, ContextType>; friendsCount?: Resolver, ParentType, ContextType>; id?: Resolver; diff --git a/packages/api/src/generated/schema.graphql b/packages/api/src/generated/schema.graphql index b344a250a..bf37f919d 100644 --- a/packages/api/src/generated/schema.graphql +++ b/packages/api/src/generated/schema.graphql @@ -968,6 +968,7 @@ type Integration { enabled: Boolean! id: ID! name: String! + settings: JSON taskName: String token: String! type: IntegrationType! @@ -1832,6 +1833,7 @@ enum SearchErrorCode { } type SearchItem { + aiSummary: String annotation: String archivedAt: Date author: String @@ -2001,6 +2003,7 @@ input SetIntegrationInput { id: ID importItemState: ImportItemState name: String! + settings: JSON syncedAt: Date taskName: String token: String! @@ -2733,6 +2736,7 @@ enum UploadImportFileType { type User { email: String + features: [String] followersCount: Int friendsCount: Int id: ID! diff --git a/packages/api/src/jobs/integration/export_item.ts b/packages/api/src/jobs/integration/export_item.ts index a1b8f7cc2..277ff5a9a 100644 --- a/packages/api/src/jobs/integration/export_item.ts +++ b/packages/api/src/jobs/integration/export_item.ts @@ -44,7 +44,11 @@ export const exportItem = async (jobData: ExportItemJobData) => { } logger.info('exporting item...', logObject) - const client = getIntegrationClient(integration.name, integration.token) + const client = getIntegrationClient( + integration.name, + integration.token, + integration.settings + ) const synced = await client.export(libraryItems) if (!synced) { diff --git a/packages/api/src/resolvers/integrations/index.ts b/packages/api/src/resolvers/integrations/index.ts index 5148d0893..5d2a01525 100644 --- a/packages/api/src/resolvers/integrations/index.ts +++ b/packages/api/src/resolvers/integrations/index.ts @@ -55,6 +55,8 @@ export const setIntegrationResolver = authorized< input.type === IntegrationType.Import ? input.importItemState || ImportItemState.Unarchived // default to unarchived : undefined, + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment + settings: input.settings, } if (input.id) { // Update @@ -69,7 +71,11 @@ export const setIntegrationResolver = authorized< integrationToSave.taskName = existingIntegration.taskName } else { // Create - const integrationService = getIntegrationClient(input.name, input.token) + const integrationService = getIntegrationClient( + input.name, + input.token, + input.settings + ) // authorize and get access token const token = await integrationService.accessToken() if (!token) { diff --git a/packages/api/src/routers/integration_router.ts b/packages/api/src/routers/integration_router.ts index 7bf5cbdc6..82645b8de 100644 --- a/packages/api/src/routers/integration_router.ts +++ b/packages/api/src/routers/integration_router.ts @@ -21,7 +21,7 @@ export function integrationRouter() { return res.status(401).send('UNAUTHORIZED') } - const integrationClient = getIntegrationClient(req.params.name, '') + const integrationClient = getIntegrationClient(req.params.name, '', null) // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access const state = req.body.state as string diff --git a/packages/api/src/schema.ts b/packages/api/src/schema.ts index 86b1f46e2..aec8756e9 100755 --- a/packages/api/src/schema.ts +++ b/packages/api/src/schema.ts @@ -2008,6 +2008,7 @@ const schema = gql` createdAt: Date! updatedAt: Date taskName: String + settings: JSON } enum IntegrationType { @@ -2043,6 +2044,7 @@ const schema = gql` syncedAt: Date importItemState: ImportItemState taskName: String + settings: JSON } union IntegrationsResult = IntegrationsSuccess | IntegrationsError diff --git a/packages/api/src/services/integrations/index.ts b/packages/api/src/services/integrations/index.ts index 953199052..aa59f900b 100644 --- a/packages/api/src/services/integrations/index.ts +++ b/packages/api/src/services/integrations/index.ts @@ -8,7 +8,8 @@ import { ReadwiseClient } from './readwise' export const getIntegrationClient = ( name: string, - token: string + token: string, + settings: any ): IntegrationClient => { switch (name.toLowerCase()) { case 'readwise': @@ -16,7 +17,7 @@ export const getIntegrationClient = ( case 'pocket': return new PocketClient(token) case 'notion': - return new NotionClient(token) + return new NotionClient(token, settings) default: throw new Error(`Integration client not found: ${name}`) } diff --git a/packages/api/src/services/integrations/notion.ts b/packages/api/src/services/integrations/notion.ts index 868a5ec57..085c20566 100644 --- a/packages/api/src/services/integrations/notion.ts +++ b/packages/api/src/services/integrations/notion.ts @@ -23,6 +23,10 @@ interface NotionPage { } } +interface Settings { + parentPageId: string +} + export class NotionClient implements IntegrationClient { name = 'NOTION' _headers = { @@ -39,13 +43,15 @@ export class NotionClient implements IntegrationClient { _token: string _client: Client + _settings: Settings - constructor(token: string) { + constructor(token: string, settings: Settings) { this._token = token this._client = new Client({ auth: token, timeoutMs: this._timeout, }) + this._settings = settings } accessToken = async (): Promise => { diff --git a/packages/db/migrations/0167.do.add_settings_column_to_integrations.sql b/packages/db/migrations/0167.do.add_settings_column_to_integrations.sql new file mode 100755 index 000000000..d8c2521b8 --- /dev/null +++ b/packages/db/migrations/0167.do.add_settings_column_to_integrations.sql @@ -0,0 +1,9 @@ +-- Type: DO +-- Name: add_settings_column_to_integrations +-- Description: Add settings column to integrations table + +BEGIN; + +ALTER TABLE omnivore.integrations ADD COLUMN settings jsonb; + +COMMIT; diff --git a/packages/db/migrations/0167.undo.add_settings_column_to_integrations.sql b/packages/db/migrations/0167.undo.add_settings_column_to_integrations.sql new file mode 100755 index 000000000..34a03ae7e --- /dev/null +++ b/packages/db/migrations/0167.undo.add_settings_column_to_integrations.sql @@ -0,0 +1,9 @@ +-- Type: UNDO +-- Name: add_settings_column_to_integrations +-- Description: Add settings column to integrations table + +BEGIN; + +ALTER TABLE omnivore.integrations DROP COLUMN settings; + +COMMIT; From dfa551251218f980dc0d71304714b89af26c04bb Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Tue, 5 Mar 2024 16:18:15 +0800 Subject: [PATCH 07/10] get parentPageId from settings --- packages/api/src/services/integrations/notion.ts | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/packages/api/src/services/integrations/notion.ts b/packages/api/src/services/integrations/notion.ts index 085c20566..85ace7508 100644 --- a/packages/api/src/services/integrations/notion.ts +++ b/packages/api/src/services/integrations/notion.ts @@ -39,7 +39,6 @@ export class NotionClient implements IntegrationClient { baseURL: 'https://api.notion.com/v1', timeout: this._timeout, }) - _parentPageId = process.env.NOTION_PAGE_ID _token: string _client: Client @@ -91,13 +90,9 @@ export class NotionClient implements IntegrationClient { } private _itemToNotionPage = (item: LibraryItem): NotionPage => { - if (!this._parentPageId) { - throw new Error('Notion parent page ID is not set') - } - return { parent: { - page_id: this._parentPageId, + page_id: this._settings.parentPageId, }, cover: item.thumbnail ? { From ec32945ba2ea9e4e5209be2539cce9ba71579e62 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Tue, 5 Mar 2024 21:00:39 +0800 Subject: [PATCH 08/10] update block --- .../api/src/services/integrations/notion.ts | 47 +++++++++++++++---- 1 file changed, 39 insertions(+), 8 deletions(-) diff --git a/packages/api/src/services/integrations/notion.ts b/packages/api/src/services/integrations/notion.ts index 85ace7508..9fc80c21d 100644 --- a/packages/api/src/services/integrations/notion.ts +++ b/packages/api/src/services/integrations/notion.ts @@ -1,30 +1,45 @@ import { Client } from '@notionhq/client' import axios from 'axios' -import { LibraryItem } from '../../entity/library_item' +import { LibraryItem, LibraryItemState } from '../../entity/library_item' import { env } from '../../env' import { logger } from '../../utils/logger' import { IntegrationClient } from './integration' interface NotionPage { parent: { - page_id: string + database_id: string } + id: string + created_time: string + last_edited_time: string + archived: boolean + public_url: string cover?: { external: { url: string } } + icon?: { + external: { + url: string + } + } properties: { - title: Array<{ - text: { - content: string + title: [ + { + text: { + content: string + link: { + url: string + } + } } - }> + ] } } interface Settings { - parentPageId: string + parentDatabaseId: string } export class NotionClient implements IntegrationClient { @@ -92,8 +107,20 @@ export class NotionClient implements IntegrationClient { private _itemToNotionPage = (item: LibraryItem): NotionPage => { return { parent: { - page_id: this._settings.parentPageId, + database_id: this._settings.parentDatabaseId, }, + id: item.id, + archived: item.state === LibraryItemState.Archived, + created_time: item.savedAt.toISOString(), + last_edited_time: item.updatedAt.toISOString(), + public_url: item.originalUrl, + icon: item.siteIcon + ? { + external: { + url: item.siteIcon, + }, + } + : undefined, cover: item.thumbnail ? { external: { @@ -106,6 +133,9 @@ export class NotionClient implements IntegrationClient { { text: { content: item.title, + link: { + url: item.originalUrl, + }, }, }, ], @@ -119,6 +149,7 @@ export class NotionClient implements IntegrationClient { export = async (items: LibraryItem[]): Promise => { const pages = items.map(this._itemToNotionPage) + console.log('pages', JSON.stringify(pages, null, 2)) await Promise.all(pages.map((page) => this._createPage(page))) return true From 39778feb8841214a29315d88da61524718306243 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Wed, 6 Mar 2024 20:23:42 +0800 Subject: [PATCH 09/10] export highlights and labels --- .../api/src/jobs/integration/export_item.ts | 2 +- .../api/src/resolvers/integrations/index.ts | 6 +- .../api/src/routers/integration_router.ts | 2 +- .../api/src/services/integrations/index.ts | 4 +- .../api/src/services/integrations/notion.ts | 202 +++++++++++++++--- 5 files changed, 175 insertions(+), 41 deletions(-) diff --git a/packages/api/src/jobs/integration/export_item.ts b/packages/api/src/jobs/integration/export_item.ts index 277ff5a9a..318a94ec0 100644 --- a/packages/api/src/jobs/integration/export_item.ts +++ b/packages/api/src/jobs/integration/export_item.ts @@ -47,7 +47,7 @@ export const exportItem = async (jobData: ExportItemJobData) => { const client = getIntegrationClient( integration.name, integration.token, - integration.settings + integration ) const synced = await client.export(libraryItems) diff --git a/packages/api/src/resolvers/integrations/index.ts b/packages/api/src/resolvers/integrations/index.ts index 5d2a01525..d50992b28 100644 --- a/packages/api/src/resolvers/integrations/index.ts +++ b/packages/api/src/resolvers/integrations/index.ts @@ -71,11 +71,7 @@ export const setIntegrationResolver = authorized< integrationToSave.taskName = existingIntegration.taskName } else { // Create - const integrationService = getIntegrationClient( - input.name, - input.token, - input.settings - ) + const integrationService = getIntegrationClient(input.name, input.token) // authorize and get access token const token = await integrationService.accessToken() if (!token) { diff --git a/packages/api/src/routers/integration_router.ts b/packages/api/src/routers/integration_router.ts index 82645b8de..7bf5cbdc6 100644 --- a/packages/api/src/routers/integration_router.ts +++ b/packages/api/src/routers/integration_router.ts @@ -21,7 +21,7 @@ export function integrationRouter() { return res.status(401).send('UNAUTHORIZED') } - const integrationClient = getIntegrationClient(req.params.name, '', null) + const integrationClient = getIntegrationClient(req.params.name, '') // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access const state = req.body.state as string diff --git a/packages/api/src/services/integrations/index.ts b/packages/api/src/services/integrations/index.ts index aa59f900b..a925a2765 100644 --- a/packages/api/src/services/integrations/index.ts +++ b/packages/api/src/services/integrations/index.ts @@ -9,7 +9,7 @@ import { ReadwiseClient } from './readwise' export const getIntegrationClient = ( name: string, token: string, - settings: any + integrationData?: Integration ): IntegrationClient => { switch (name.toLowerCase()) { case 'readwise': @@ -17,7 +17,7 @@ export const getIntegrationClient = ( case 'pocket': return new PocketClient(token) case 'notion': - return new NotionClient(token, settings) + return new NotionClient(token, integrationData) default: throw new Error(`Integration client not found: ${name}`) } diff --git a/packages/api/src/services/integrations/notion.ts b/packages/api/src/services/integrations/notion.ts index 9fc80c21d..b695b3737 100644 --- a/packages/api/src/services/integrations/notion.ts +++ b/packages/api/src/services/integrations/notion.ts @@ -1,19 +1,38 @@ import { Client } from '@notionhq/client' import axios from 'axios' -import { LibraryItem, LibraryItemState } from '../../entity/library_item' +import { updateIntegration } from '.' +import { Integration } from '../../entity/integration' +import { LibraryItem } from '../../entity/library_item' import { env } from '../../env' +import { Merge } from '../../util' import { logger } from '../../utils/logger' import { IntegrationClient } from './integration' +type AnnotationColor = + | 'default' + | 'gray' + | 'brown' + | 'orange' + | 'yellow' + | 'green' + | 'blue' + | 'purple' + | 'pink' + | 'red' + | 'gray_background' + | 'brown_background' + | 'orange_background' + | 'yellow_background' + | 'green_background' + | 'blue_background' + | 'purple_background' + | 'pink_background' + | 'red_background' + interface NotionPage { parent: { database_id: string } - id: string - created_time: string - last_edited_time: string - archived: boolean - public_url: string cover?: { external: { url: string @@ -25,20 +44,48 @@ interface NotionPage { } } properties: { - title: [ - { - text: { - content: string - link: { - url: string + Title: { + title: [ + { + text: { + content: string } } - } - ] + ] + } + 'Original URL': { + url: string | null + } + 'Omnivore URL': { + url: string | null + } + Tags?: { + multi_select: Array<{ name: string }> + } } + children?: Array<{ + type: 'paragraph' + paragraph: { + rich_text: Array<{ + text: { + content: string + link?: { url: string } + } + annotations?: { + bold?: boolean + italic?: boolean + strikethrough?: boolean + underline?: boolean + code?: boolean + color?: AnnotationColor + } + }> + } + }> } interface Settings { + parentPageId: string parentDatabaseId: string } @@ -57,15 +104,15 @@ export class NotionClient implements IntegrationClient { _token: string _client: Client - _settings: Settings + _integrationData?: Merge - constructor(token: string, settings: Settings) { + constructor(token: string, integration?: Integration) { this._token = token this._client = new Client({ auth: token, timeoutMs: this._timeout, }) - this._settings = settings + this._integrationData = integration } accessToken = async (): Promise => { @@ -105,15 +152,15 @@ export class NotionClient implements IntegrationClient { } private _itemToNotionPage = (item: LibraryItem): NotionPage => { + const databaseId = this._integrationData?.settings?.parentDatabaseId + if (!databaseId) { + throw new Error('Notion database id not found') + } + return { parent: { - database_id: this._settings.parentDatabaseId, + database_id: databaseId, }, - id: item.id, - archived: item.state === LibraryItemState.Archived, - created_time: item.savedAt.toISOString(), - last_edited_time: item.updatedAt.toISOString(), - public_url: item.originalUrl, icon: item.siteIcon ? { external: { @@ -129,17 +176,50 @@ export class NotionClient implements IntegrationClient { } : undefined, properties: { - title: [ - { - text: { - content: item.title, - link: { - url: item.originalUrl, + Title: { + title: [ + { + text: { + content: item.title, }, }, - }, - ], + ], + }, + 'Original URL': { + url: item.originalUrl, + }, + 'Omnivore URL': { + url: `${env.client.url}/me/${item.slug}`, + }, + Tags: item.labels + ? { multi_select: item.labels.map((label) => ({ name: label.name })) } + : undefined, }, + children: item.highlights + ? item.highlights.map((highlight) => ({ + type: 'paragraph', + paragraph: { + rich_text: [ + { + text: { + content: highlight.quote || '', + }, + annotations: { + color: highlight.color as AnnotationColor, + }, + }, + { + text: { + content: highlight.annotation || '', + }, + annotations: { + italic: true, + }, + }, + ], + }, + })) + : undefined, } } @@ -148,8 +228,66 @@ export class NotionClient implements IntegrationClient { } export = async (items: LibraryItem[]): Promise => { + if (!this._integrationData || !this._integrationData.settings) { + logger.error('Notion integration data not found') + return false + } + + const pageId = this._integrationData.settings.parentPageId + if (!pageId) { + logger.error('Notion parent page id not found') + return false + } + + const databaseId = this._integrationData.settings.parentDatabaseId + if (!databaseId) { + // create a database for the items + const database = await this._client.databases.create({ + parent: { + page_id: pageId, + }, + title: [ + { + text: { + content: 'Library', + }, + }, + ], + description: [ + { + text: { + content: 'Library of saved items from Omnivore', + }, + }, + ], + properties: { + Title: { + title: {}, + }, + 'Original URL': { + url: {}, + }, + 'Omnivore URL': { + url: {}, + }, + Tags: { + multi_select: {}, + }, + }, + }) + + // save the database id + this._integrationData.settings.parentDatabaseId = database.id + await updateIntegration( + this._integrationData.id, + { + settings: this._integrationData.settings, + }, + this._integrationData.user.id + ) + } + const pages = items.map(this._itemToNotionPage) - console.log('pages', JSON.stringify(pages, null, 2)) await Promise.all(pages.map((page) => this._createPage(page))) return true From 44178c1c1ef6ba089979d556664d090b7549c384 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Thu, 7 Mar 2024 13:24:02 +0800 Subject: [PATCH 10/10] Add author --- .../api/src/services/integrations/notion.ts | 21 ++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/packages/api/src/services/integrations/notion.ts b/packages/api/src/services/integrations/notion.ts index b695b3737..4a21d5606 100644 --- a/packages/api/src/services/integrations/notion.ts +++ b/packages/api/src/services/integrations/notion.ts @@ -53,6 +53,13 @@ interface NotionPage { } ] } + Author: { + rich_text: Array<{ + text: { + content: string + } + }> + } 'Original URL': { url: string | null } @@ -185,6 +192,15 @@ export class NotionClient implements IntegrationClient { }, ], }, + Author: { + rich_text: [ + { + text: { + content: item.author || 'unknown', + }, + }, + ], + }, 'Original URL': { url: item.originalUrl, }, @@ -210,7 +226,7 @@ export class NotionClient implements IntegrationClient { }, { text: { - content: highlight.annotation || '', + content: `\n${highlight.annotation || ''}`, }, annotations: { italic: true, @@ -264,6 +280,9 @@ export class NotionClient implements IntegrationClient { Title: { title: {}, }, + Author: { + rich_text: {}, + }, 'Original URL': { url: {}, },