From 69e74d432dde9492e34b7466ba76c6a763b23752 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Thu, 14 Mar 2024 19:50:26 +0800 Subject: [PATCH 1/5] append highlights to the existing page --- .../api/src/services/integrations/notion.ts | 220 +++++++++++------- 1 file changed, 133 insertions(+), 87 deletions(-) diff --git a/packages/api/src/services/integrations/notion.ts b/packages/api/src/services/integrations/notion.ts index 4a21d5606..f60b4e34b 100644 --- a/packages/api/src/services/integrations/notion.ts +++ b/packages/api/src/services/integrations/notion.ts @@ -61,10 +61,10 @@ interface NotionPage { }> } 'Original URL': { - url: string | null + url: string } 'Omnivore URL': { - url: string | null + url: string } Tags?: { multi_select: Array<{ name: string }> @@ -158,89 +158,106 @@ export class NotionClient implements IntegrationClient { return Promise.resolve(env.notion.authUrl) } - private _itemToNotionPage = (item: LibraryItem): NotionPage => { - const databaseId = this._integrationData?.settings?.parentDatabaseId - if (!databaseId) { - throw new Error('Notion database id not found') - } + private itemToNotionPage = ( + item: LibraryItem, + databaseId: string + ): NotionPage => ({ + parent: { + database_id: databaseId, + }, + icon: item.siteIcon + ? { + external: { + url: item.siteIcon, + }, + } + : undefined, + cover: item.thumbnail + ? { + external: { + url: item.thumbnail, + }, + } + : undefined, + properties: { + Title: { + title: [ + { + text: { + content: item.title, + }, + }, + ], + }, + Author: { + rich_text: [ + { + text: { + content: item.author || 'unknown', + }, + }, + ], + }, + 'Original URL': { + url: item.originalUrl, + }, + 'Omnivore URL': { + url: `${env.client.url}/me/${item.slug}`, + }, + Tags: item.labels + ? { multi_select: item.labels.map((label) => ({ name: label.name })) } + : undefined, + }, + children: item.highlights + ? item.highlights.map((highlight) => ({ + type: 'paragraph', + paragraph: { + rich_text: [ + { + text: { + content: highlight.quote || '', + link: { + url: `${env.client.url}/me/${item.slug}#${highlight.id}`, + }, + }, + annotations: { + color: highlight.color as AnnotationColor, + }, + }, + { + text: { + content: `\n${highlight.annotation || ''}`, + }, + annotations: { + italic: true, + }, + }, + ], + }, + })) + : undefined, + }) - return { - parent: { - database_id: databaseId, - }, - icon: item.siteIcon - ? { - external: { - url: item.siteIcon, - }, - } - : undefined, - cover: item.thumbnail - ? { - external: { - url: item.thumbnail, - }, - } - : undefined, - properties: { - Title: { - title: [ - { - text: { - content: item.title, - }, - }, - ], - }, - Author: { - rich_text: [ - { - text: { - content: item.author || 'unknown', - }, - }, - ], - }, - 'Original URL': { - url: item.originalUrl, - }, - 'Omnivore URL': { - url: `${env.client.url}/me/${item.slug}`, - }, - Tags: item.labels - ? { multi_select: item.labels.map((label) => ({ name: label.name })) } - : undefined, - }, - children: item.highlights - ? item.highlights.map((highlight) => ({ - type: 'paragraph', - paragraph: { - rich_text: [ - { - text: { - content: highlight.quote || '', - }, - annotations: { - color: highlight.color as AnnotationColor, - }, - }, - { - text: { - content: `\n${highlight.annotation || ''}`, - }, - annotations: { - italic: true, - }, - }, - ], - }, - })) - : undefined, - } + private createPage = async (page: NotionPage) => { + await this._client.pages.create(page) } - _createPage = async (page: NotionPage) => { - await this._client.pages.create(page) + private findPage = async (url: string, databaseId: string) => { + const response = await this._client.databases.query({ + database_id: databaseId, + page_size: 1, + filter: { + property: 'Omnivore URL', + url: { + equals: url, + }, + }, + }) + if (response.results.length > 0) { + return response.results[0] + } + + return null } export = async (items: LibraryItem[]): Promise => { @@ -255,7 +272,7 @@ export class NotionClient implements IntegrationClient { return false } - const databaseId = this._integrationData.settings.parentDatabaseId + let databaseId = this._integrationData.settings.parentDatabaseId if (!databaseId) { // create a database for the items const database = await this._client.databases.create({ @@ -296,18 +313,47 @@ export class NotionClient implements IntegrationClient { }) // save the database id - this._integrationData.settings.parentDatabaseId = database.id + databaseId = database.id await updateIntegration( this._integrationData.id, { - settings: this._integrationData.settings, + settings: { + ...this._integrationData.settings, + parentDatabaseId: databaseId, + }, }, this._integrationData.user.id ) } - const pages = items.map(this._itemToNotionPage) - await Promise.all(pages.map((page) => this._createPage(page))) + await Promise.all( + items.map(async (item) => { + const notionPage = this.itemToNotionPage(item, databaseId) + const url = notionPage.properties['Omnivore URL'].url + + const existingPage = await this.findPage(url, databaseId) + if (existingPage) { + // update the page + await this._client.pages.update({ + page_id: existingPage.id, + properties: notionPage.properties, + }) + + // append the children + if (notionPage.children) { + await this._client.blocks.children.append({ + block_id: existingPage.id, + children: notionPage.children, + }) + } + + return + } + + // create the page + return this.createPage(notionPage) + }) + ) return true } From 1085dcc8241303fd6303014a097249aab5c583be Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Thu, 14 Mar 2024 21:36:47 +0800 Subject: [PATCH 2/5] read settings --- .../src/services/integrations/integration.ts | 2 +- .../api/src/services/integrations/notion.ts | 213 ++++++++++-------- .../api/src/services/integrations/pocket.ts | 6 +- .../api/src/services/integrations/readwise.ts | 11 +- 4 files changed, 125 insertions(+), 107 deletions(-) diff --git a/packages/api/src/services/integrations/integration.ts b/packages/api/src/services/integrations/integration.ts index 13b61e82a..44c95e680 100644 --- a/packages/api/src/services/integrations/integration.ts +++ b/packages/api/src/services/integrations/integration.ts @@ -20,7 +20,7 @@ export interface RetrieveRequest { export interface IntegrationClient { name: string - _token: string + token: string accessToken(): Promise diff --git a/packages/api/src/services/integrations/notion.ts b/packages/api/src/services/integrations/notion.ts index f60b4e34b..21770fdc4 100644 --- a/packages/api/src/services/integrations/notion.ts +++ b/packages/api/src/services/integrations/notion.ts @@ -5,6 +5,7 @@ import { Integration } from '../../entity/integration' import { LibraryItem } from '../../entity/library_item' import { env } from '../../env' import { Merge } from '../../util' +import { highlightUrl } from '../../utils/helpers' import { logger } from '../../utils/logger' import { IntegrationClient } from './integration' @@ -91,35 +92,39 @@ interface NotionPage { }> } +type Property = 'highlights' | 'labels' | 'notes' + interface Settings { parentPageId: string parentDatabaseId: string + properties: Property[] } export class NotionClient implements IntegrationClient { name = 'NOTION' - _headers = { + token: string + + private headers = { 'Content-Type': 'application/json', Accept: 'application/json', 'Notion-Version': '2022-06-28', } - _timeout = 5000 // 5 seconds - _axios = axios.create({ + private timeout = 5000 // 5 seconds + private axiosInstance = axios.create({ baseURL: 'https://api.notion.com/v1', - timeout: this._timeout, + timeout: this.timeout, }) - _token: string - _client: Client - _integrationData?: Merge + private client: Client + private integrationData?: Merge constructor(token: string, integration?: Integration) { - this._token = token - this._client = new Client({ + this.token = token + this.client = new Client({ auth: token, - timeoutMs: this._timeout, + timeoutMs: this.timeout, }) - this._integrationData = integration + this.integrationData = integration } accessToken = async (): Promise => { @@ -129,16 +134,16 @@ export class NotionClient implements IntegrationClient { `${env.notion.clientId}:${env.notion.clientSecret}` ).toString('base64') - const response = await this._axios.post<{ access_token: string }>( + const response = await this.axiosInstance.post<{ access_token: string }>( '/oauth/token', { grant_type: 'authorization_code', - code: this._token, + code: this.token, redirect_uri: `${env.client.url}/settings/integrations`, }, { headers: { - ...this._headers, + ...this.headers, Authorization: `Basic ${encoded}`, }, } @@ -160,90 +165,101 @@ export class NotionClient implements IntegrationClient { private itemToNotionPage = ( item: LibraryItem, - databaseId: string - ): NotionPage => ({ - parent: { - database_id: databaseId, - }, - icon: item.siteIcon - ? { - external: { - url: item.siteIcon, - }, - } - : undefined, - cover: item.thumbnail - ? { - external: { - url: item.thumbnail, - }, - } - : undefined, - properties: { - Title: { - title: [ - { - text: { - content: item.title, + settings: Settings + ): NotionPage => { + return { + parent: { + database_id: settings.parentDatabaseId, + }, + icon: item.siteIcon + ? { + external: { + url: item.siteIcon, }, - }, - ], - }, - Author: { - rich_text: [ - { - text: { - content: item.author || 'unknown', - }, - }, - ], - }, - 'Original URL': { - url: item.originalUrl, - }, - 'Omnivore URL': { - url: `${env.client.url}/me/${item.slug}`, - }, - Tags: item.labels - ? { multi_select: item.labels.map((label) => ({ name: label.name })) } + } : undefined, - }, - children: item.highlights - ? item.highlights.map((highlight) => ({ - type: 'paragraph', - paragraph: { - rich_text: [ - { - text: { - content: highlight.quote || '', - link: { - url: `${env.client.url}/me/${item.slug}#${highlight.id}`, + cover: item.thumbnail + ? { + external: { + url: item.thumbnail, + }, + } + : undefined, + properties: { + Title: { + title: [ + { + text: { + content: item.title, + }, + }, + ], + }, + Author: { + rich_text: [ + { + text: { + content: item.author || 'unknown', + }, + }, + ], + }, + 'Original URL': { + url: item.originalUrl, + }, + 'Omnivore URL': { + url: `${env.client.url}/me/${item.slug}`, + }, + Tags: + item.labels && settings.properties.includes('labels') + ? { + multi_select: item.labels.map((label) => ({ + name: label.name, + })), + } + : undefined, + }, + children: item.highlights + ? item.highlights.map((highlight) => ({ + type: 'paragraph', + paragraph: { + rich_text: [ + { + text: { + content: settings.properties.includes('highlights') + ? highlight.quote || '' + : '', + link: { + url: highlightUrl(item.slug, highlight.id), + }, + }, + annotations: { + color: highlight.color as AnnotationColor, }, }, - annotations: { - color: highlight.color as AnnotationColor, + { + text: { + content: settings.properties.includes('notes') + ? `\n${highlight.annotation || ''}` + : '', + }, + annotations: { + italic: true, + }, }, - }, - { - text: { - content: `\n${highlight.annotation || ''}`, - }, - annotations: { - italic: true, - }, - }, - ], - }, - })) - : undefined, - }) + ], + }, + })) + : undefined, + } + } private createPage = async (page: NotionPage) => { - await this._client.pages.create(page) + await this.client.pages.create(page) } private findPage = async (url: string, databaseId: string) => { - const response = await this._client.databases.query({ + const response = await this.client.databases.query({ database_id: databaseId, page_size: 1, filter: { @@ -261,21 +277,22 @@ export class NotionClient implements IntegrationClient { } export = async (items: LibraryItem[]): Promise => { - if (!this._integrationData || !this._integrationData.settings) { + const settings = this.integrationData?.settings + if (!this.integrationData || !settings) { logger.error('Notion integration data not found') return false } - const pageId = this._integrationData.settings.parentPageId + const pageId = settings.parentPageId if (!pageId) { logger.error('Notion parent page id not found') return false } - let databaseId = this._integrationData.settings.parentDatabaseId + let databaseId = settings.parentDatabaseId if (!databaseId) { // create a database for the items - const database = await this._client.databases.create({ + const database = await this.client.databases.create({ parent: { page_id: pageId, }, @@ -315,33 +332,33 @@ export class NotionClient implements IntegrationClient { // save the database id databaseId = database.id await updateIntegration( - this._integrationData.id, + this.integrationData.id, { settings: { - ...this._integrationData.settings, + ...this.integrationData.settings, parentDatabaseId: databaseId, }, }, - this._integrationData.user.id + this.integrationData.user.id ) } await Promise.all( items.map(async (item) => { - const notionPage = this.itemToNotionPage(item, databaseId) + const notionPage = this.itemToNotionPage(item, settings) const url = notionPage.properties['Omnivore URL'].url const existingPage = await this.findPage(url, databaseId) if (existingPage) { // update the page - await this._client.pages.update({ + await this.client.pages.update({ page_id: existingPage.id, properties: notionPage.properties, }) // append the children if (notionPage.children) { - await this._client.blocks.children.append({ + await this.client.blocks.children.append({ block_id: existingPage.id, children: notionPage.children, }) diff --git a/packages/api/src/services/integrations/pocket.ts b/packages/api/src/services/integrations/pocket.ts index 20f4c212c..9945c7a6e 100644 --- a/packages/api/src/services/integrations/pocket.ts +++ b/packages/api/src/services/integrations/pocket.ts @@ -5,7 +5,7 @@ import { IntegrationClient } from './integration' export class PocketClient implements IntegrationClient { name = 'POCKET' - _token: string + token: string _axios = axios.create({ baseURL: 'https://getpocket.com/v3', headers: { @@ -16,7 +16,7 @@ export class PocketClient implements IntegrationClient { }) constructor(token: string) { - this._token = token + this.token = token } accessToken = async (): Promise => { @@ -25,7 +25,7 @@ export class PocketClient implements IntegrationClient { '/oauth/authorize', { consumer_key: env.pocket.consumerKey, - code: this._token, + code: this.token, } ) return response.data.access_token diff --git a/packages/api/src/services/integrations/readwise.ts b/packages/api/src/services/integrations/readwise.ts index 25a61f438..dfc5b43db 100644 --- a/packages/api/src/services/integrations/readwise.ts +++ b/packages/api/src/services/integrations/readwise.ts @@ -33,6 +33,8 @@ interface ReadwiseHighlight { export class ReadwiseClient implements IntegrationClient { name = 'READWISE' + token: string + _headers = { 'Content-Type': 'application/json', } @@ -40,10 +42,9 @@ export class ReadwiseClient implements IntegrationClient { baseURL: 'https://readwise.io/api/v2', timeout: 5000, // 5 seconds }) - _token: string constructor(token: string) { - this._token = token + this.token = token } accessToken = async (): Promise => { @@ -51,10 +52,10 @@ export class ReadwiseClient implements IntegrationClient { const response = await this._axios.get('/auth', { headers: { ...this._headers, - Authorization: `Token ${this._token}`, + Authorization: `Token ${this.token}`, }, }) - return response.status === 204 ? this._token : null + return response.status === 204 ? this.token : null } catch (error) { if (axios.isAxiosError(error)) { logger.error(error.response) @@ -121,7 +122,7 @@ export class ReadwiseClient implements IntegrationClient { { headers: { ...this._headers, - Authorization: `Token ${this._token}`, + Authorization: `Token ${this.token}`, }, } ) From 17e66aa01018b4f91acdbdfe9029e1a673d42904 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Thu, 14 Mar 2024 22:13:27 +0800 Subject: [PATCH 3/5] append new highlights only --- .../api/src/services/integrations/notion.ts | 28 ++++++++++++++++--- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/packages/api/src/services/integrations/notion.ts b/packages/api/src/services/integrations/notion.ts index 21770fdc4..a4e8a84e5 100644 --- a/packages/api/src/services/integrations/notion.ts +++ b/packages/api/src/services/integrations/notion.ts @@ -356,12 +356,32 @@ export class NotionClient implements IntegrationClient { properties: notionPage.properties, }) - // append the children - if (notionPage.children) { - await this.client.blocks.children.append({ + const children = notionPage.children + if (children) { + // get the existing children + const response = await this.client.blocks.children.list({ block_id: existingPage.id, - children: notionPage.children, }) + if (response.results.length > 0) { + const existingChildren = + response.results as NotionPage['children'] + // delete the existing children from children + notionPage.children = children.filter( + (child) => + !existingChildren?.some( + (existingChild) => + existingChild.paragraph.rich_text[0].text.link?.url === + child.paragraph.rich_text[0].text.link?.url + ) + ) + } + // append the children + if (notionPage.children && notionPage.children.length > 0) { + await this.client.blocks.children.append({ + block_id: existingPage.id, + children: notionPage.children, + }) + } } return From c54cfee3f1c3186292b6061bc106c97e753a29d6 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Thu, 14 Mar 2024 22:18:18 +0800 Subject: [PATCH 4/5] turn off auto sync will disable notion --- packages/web/pages/settings/integrations/notion.tsx | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/web/pages/settings/integrations/notion.tsx b/packages/web/pages/settings/integrations/notion.tsx index eb56fa892..df72c75be 100644 --- a/packages/web/pages/settings/integrations/notion.tsx +++ b/packages/web/pages/settings/integrations/notion.tsx @@ -30,7 +30,7 @@ import { showSuccessToast } from '../../../lib/toastHelpers' type FieldType = { parentPageId?: string parentDatabaseId?: string - autoSync?: boolean + enabled: boolean properties?: string[] } @@ -55,7 +55,7 @@ export default function Notion(): JSX.Element { form.setFieldsValue({ parentPageId: notion.settings?.parentPageId, parentDatabaseId: notion.settings?.parentDatabaseId, - autoSync: notion.settings?.autoSync, + enabled: notion.enabled, properties: notion.settings?.properties, }) } @@ -82,7 +82,7 @@ export default function Notion(): JSX.Element { name: notion.name, type: notion.type, token: notion.token, - enabled: notion.enabled, + enabled: values.enabled, settings: values, }) } @@ -170,7 +170,7 @@ export default function Notion(): JSX.Element { label="Automatic Sync" - name="autoSync" + name="enabled" valuePropName="checked" > From 5cbed3f396cc216e83963f9b003bb0b4717e2bbd Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Thu, 14 Mar 2024 22:23:42 +0800 Subject: [PATCH 5/5] disable notion by default --- packages/web/pages/settings/integrations.tsx | 2 +- .../pages/settings/integrations/notion.tsx | 34 +++++++------------ 2 files changed, 14 insertions(+), 22 deletions(-) diff --git a/packages/web/pages/settings/integrations.tsx b/packages/web/pages/settings/integrations.tsx index c9f2e3296..39af9ec23 100644 --- a/packages/web/pages/settings/integrations.tsx +++ b/packages/web/pages/settings/integrations.tsx @@ -176,7 +176,7 @@ export default function Integrations(): JSX.Element { token, name: 'NOTION', type: 'EXPORT', - enabled: true, + enabled: false, }) showSuccessToast('Connected with Notion.') diff --git a/packages/web/pages/settings/integrations/notion.tsx b/packages/web/pages/settings/integrations/notion.tsx index df72c75be..dce6e0a24 100644 --- a/packages/web/pages/settings/integrations/notion.tsx +++ b/packages/web/pages/settings/integrations/notion.tsx @@ -12,7 +12,7 @@ import 'antd/dist/antd.compact.css' import { CheckboxValueType } from 'antd/lib/checkbox/Group' import Image from 'next/image' import { useRouter } from 'next/router' -import { useEffect, useState } from 'react' +import { useEffect, useMemo } from 'react' import { HStack, VStack } from '../../../components/elements/LayoutPrimitives' import { PageMetaData } from '../../../components/patterns/PageMetaData' import { Beta } from '../../../components/templates/Beta' @@ -20,10 +20,7 @@ import { Header } from '../../../components/templates/settings/SettingsTable' import { SettingsLayout } from '../../../components/templates/SettingsLayout' import { deleteIntegrationMutation } from '../../../lib/networking/mutations/deleteIntegrationMutation' import { setIntegrationMutation } from '../../../lib/networking/mutations/setIntegrationMutation' -import { - Integration, - useGetIntegrationsQuery, -} from '../../../lib/networking/queries/useGetIntegrationsQuery' +import { useGetIntegrationsQuery } from '../../../lib/networking/queries/useGetIntegrationsQuery' import { applyStoredTheme } from '../../../lib/themeUpdater' import { showSuccessToast } from '../../../lib/toastHelpers' @@ -39,27 +36,21 @@ export default function Notion(): JSX.Element { const router = useRouter() const { integrations, revalidate } = useGetIntegrationsQuery() - const [notion, setNotion] = useState() + const notion = useMemo(() => { + return integrations.find((i) => i.name == 'NOTION' && i.type == 'EXPORT') + }, [integrations]) const [form] = Form.useForm() const [messageApi, contextHolder] = message.useMessage() useEffect(() => { - const notion = integrations.find( - (i) => i.name == 'NOTION' && i.type == 'EXPORT' - ) - - if (notion) { - setNotion(notion) - - form.setFieldsValue({ - parentPageId: notion.settings?.parentPageId, - parentDatabaseId: notion.settings?.parentDatabaseId, - enabled: notion.enabled, - properties: notion.settings?.properties, - }) - } - }, [form, integrations]) + form.setFieldsValue({ + parentPageId: notion?.settings?.parentPageId, + parentDatabaseId: notion?.settings?.parentDatabaseId, + enabled: notion?.enabled, + properties: notion?.settings?.properties, + }) + }, [form, notion]) const deleteNotion = async () => { if (!notion) { @@ -69,6 +60,7 @@ export default function Notion(): JSX.Element { await deleteIntegrationMutation(notion.id) showSuccessToast('Notion integration disconnected successfully.') + revalidate() router.push('/settings/integrations') }