Merge pull request #3620 from omnivore-app/feature/notion-integration

Feature: Notion Integration
This commit is contained in:
Hongbo Wu 2024-03-12 22:37:51 +08:00 committed by GitHub
commit ff157bd453
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
18 changed files with 517 additions and 112 deletions

View file

@ -24,6 +24,7 @@
"@google-cloud/tasks": "^4.0.0",
"@graphql-tools/utils": "^9.1.1",
"@langchain/openai": "^0.0.14",
"@notionhq/client": "^2.2.14",
"@omnivore/content-handler": "1.0.0",
"@omnivore/liqe": "1.0.0",
"@omnivore/readability": "1.0.0",

View file

@ -59,4 +59,7 @@ export class Integration {
@Column('enum', { enum: ImportItemState, nullable: true })
importItemState?: ImportItemState | null
@Column('jsonb', { nullable: true })
settings?: any
}

View file

@ -1091,6 +1091,7 @@ export type Integration = {
enabled: Scalars['Boolean'];
id: Scalars['ID'];
name: Scalars['String'];
settings?: Maybe<Scalars['JSON']>;
taskName?: Maybe<Scalars['String']>;
token: Scalars['String'];
type: IntegrationType;
@ -2594,6 +2595,7 @@ export type SetIntegrationInput = {
id?: InputMaybe<Scalars['ID']>;
importItemState?: InputMaybe<ImportItemState>;
name: Scalars['String'];
settings?: InputMaybe<Scalars['JSON']>;
syncedAt?: InputMaybe<Scalars['Date']>;
taskName?: InputMaybe<Scalars['String']>;
token: Scalars['String'];
@ -5321,6 +5323,7 @@ export type IntegrationResolvers<ContextType = ResolverContext, ParentType exten
enabled?: Resolver<ResolversTypes['Boolean'], ParentType, ContextType>;
id?: Resolver<ResolversTypes['ID'], ParentType, ContextType>;
name?: Resolver<ResolversTypes['String'], ParentType, ContextType>;
settings?: Resolver<Maybe<ResolversTypes['JSON']>, ParentType, ContextType>;
taskName?: Resolver<Maybe<ResolversTypes['String']>, ParentType, ContextType>;
token?: Resolver<ResolversTypes['String'], ParentType, ContextType>;
type?: Resolver<ResolversTypes['IntegrationType'], ParentType, ContextType>;

View file

@ -974,6 +974,7 @@ type Integration {
enabled: Boolean!
id: ID!
name: String!
settings: JSON
taskName: String
token: String!
type: IntegrationType!
@ -2009,6 +2010,7 @@ input SetIntegrationInput {
id: ID
importItemState: ImportItemState
name: String!
settings: JSON
syncedAt: Date
taskName: String
token: String!

View file

@ -35,41 +35,54 @@ export const exportItem = async (jobData: ExportItemJobData) => {
return
}
// currently only readwise integration is supported
const integration = integrations[0]
await Promise.all(
integrations.map(async (integration) => {
try {
const logObject = {
userId,
integrationId: integration.id,
}
logger.info('exporting item...', logObject)
const logObject = {
userId,
integrationId: integration.id,
}
logger.info('exporting item...', logObject)
const client = getIntegrationClient(
integration.name,
integration.token,
integration
)
const client = getIntegrationClient(integration.name)
const synced = await client.export(libraryItems)
if (!synced) {
logger.error('failed to export item', logObject)
return false
}
const synced = await client.export(integration.token, libraryItems)
if (!synced) {
logger.error('failed to export item', logObject)
return false
}
const syncedAt = new Date()
logger.info('updating integration...', {
...logObject,
syncedAt,
})
const syncedAt = new Date()
logger.info('updating integration...', {
...logObject,
syncedAt,
})
// update integration syncedAt if successful
const updated = await updateIntegration(
integration.id,
{
syncedAt,
},
userId
// update integration syncedAt if successful
const updated = await updateIntegration(
integration.id,
{
syncedAt,
},
userId
)
logger.info('integration updated', {
...logObject,
updated,
})
} catch (error) {
logger.error('failed to export item', {
userId,
integrationId: integration.id,
error,
})
}
})
)
logger.info('integration updated', {
...logObject,
updated,
})
return true
}

View file

@ -55,6 +55,8 @@ export const setIntegrationResolver = authorized<
input.type === IntegrationType.Import
? input.importItemState || ImportItemState.Unarchived // default to unarchived
: undefined,
// eslint-disable-next-line @typescript-eslint/no-unsafe-assignment
settings: input.settings,
}
if (input.id) {
// Update
@ -69,9 +71,9 @@ export const setIntegrationResolver = authorized<
integrationToSave.taskName = existingIntegration.taskName
} else {
// Create
const integrationService = getIntegrationClient(input.name)
const integrationService = getIntegrationClient(input.name, input.token)
// authorize and get access token
const token = await integrationService.accessToken(input.token)
const token = await integrationService.accessToken()
if (!token) {
return {
errorCodes: [SetIntegrationErrorCode.InvalidToken],

View file

@ -2,6 +2,7 @@ import axios from 'axios'
import cors from 'cors'
import express from 'express'
import { env } from '../env'
import { getIntegrationClient } from '../services/integrations'
import { getClaimsByToken } from '../utils/auth'
import { corsConfig } from '../utils/corsConfig'
import { logger } from '../utils/logger'
@ -10,10 +11,9 @@ export function integrationRouter() {
const router = express.Router()
// request token from pocket
router.post(
'/pocket/auth',
'/:name/auth',
cors<express.Request>(corsConfig),
async (req: express.Request, res: express.Response) => {
logger.info('pocket/request-token')
// eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
const token = (req.cookies.auth as string) || req.headers.authorization
const claims = await getClaimsByToken(token)
@ -21,37 +21,19 @@ export function integrationRouter() {
return res.status(401).send('UNAUTHORIZED')
}
const consumerKey = env.pocket.consumerKey
const redirectUri = `${env.client.url}/settings/integrations`
const integrationClient = getIntegrationClient(req.params.name, '')
// eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
const state = req.body.state as string
try {
// make a POST request to Pocket to get a request token
const response = await axios.post<{ code: string }>(
'https://getpocket.com/v3/oauth/request',
{
consumer_key: consumerKey,
redirect_uri: redirectUri,
},
{
headers: {
'Content-Type': 'application/json',
'X-Accept': 'application/json',
},
}
)
const { code } = response.data
const redirectUri = await integrationClient.auth(state)
// redirect the user to Pocket to authorize the request token
res.redirect(
`https://getpocket.com/auth/authorize?request_token=${code}&redirect_uri=${redirectUri}${encodeURIComponent(
`?pocketToken=${code}&state=${state}`
)}`
)
res.redirect(redirectUri)
} catch (error) {
if (axios.isAxiosError(error)) {
logger.error(error.response)
} else {
logger.error('pocket/request-token exception:', error)
logger.error(error)
}
res.redirect(

View file

@ -2015,6 +2015,7 @@ const schema = gql`
createdAt: Date!
updatedAt: Date
taskName: String
settings: JSON
}
enum IntegrationType {
@ -2050,6 +2051,7 @@ const schema = gql`
syncedAt: Date
importItemState: ImportItemState
taskName: String
settings: JSON
}
union IntegrationsResult = IntegrationsSuccess | IntegrationsError

View file

@ -2,20 +2,25 @@ import { DeepPartial, FindOptionsWhere } from 'typeorm'
import { Integration } from '../../entity/integration'
import { authTrx } from '../../repository'
import { IntegrationClient } from './integration'
import { NotionClient } from './notion'
import { PocketClient } from './pocket'
import { ReadwiseClient } from './readwise'
const integrations: IntegrationClient[] = [
new ReadwiseClient(),
new PocketClient(),
]
export const getIntegrationClient = (name: string): IntegrationClient => {
const service = integrations.find((s) => s.name === name)
if (!service) {
throw new Error(`Integration client not found: ${name}`)
export const getIntegrationClient = (
name: string,
token: string,
integrationData?: Integration
): IntegrationClient => {
switch (name.toLowerCase()) {
case 'readwise':
return new ReadwiseClient(token)
case 'pocket':
return new PocketClient(token)
case 'notion':
return new NotionClient(token, integrationData)
default:
throw new Error(`Integration client not found: ${name}`)
}
return service
}
export const deleteIntegrations = async (

View file

@ -20,9 +20,11 @@ export interface RetrieveRequest {
export interface IntegrationClient {
name: string
apiUrl: string
_token: string
accessToken(token: string): Promise<string | null>
accessToken(): Promise<string | null>
export(token: string, items: LibraryItem[]): Promise<boolean>
auth(state: string): Promise<string>
export(items: LibraryItem[]): Promise<boolean>
}

View file

@ -0,0 +1,314 @@
import { Client } from '@notionhq/client'
import axios from 'axios'
import { updateIntegration } from '.'
import { Integration } from '../../entity/integration'
import { LibraryItem } from '../../entity/library_item'
import { env } from '../../env'
import { Merge } from '../../util'
import { logger } from '../../utils/logger'
import { IntegrationClient } from './integration'
type AnnotationColor =
| 'default'
| 'gray'
| 'brown'
| 'orange'
| 'yellow'
| 'green'
| 'blue'
| 'purple'
| 'pink'
| 'red'
| 'gray_background'
| 'brown_background'
| 'orange_background'
| 'yellow_background'
| 'green_background'
| 'blue_background'
| 'purple_background'
| 'pink_background'
| 'red_background'
interface NotionPage {
parent: {
database_id: string
}
cover?: {
external: {
url: string
}
}
icon?: {
external: {
url: string
}
}
properties: {
Title: {
title: [
{
text: {
content: string
}
}
]
}
Author: {
rich_text: Array<{
text: {
content: string
}
}>
}
'Original URL': {
url: string | null
}
'Omnivore URL': {
url: string | null
}
Tags?: {
multi_select: Array<{ name: string }>
}
}
children?: Array<{
type: 'paragraph'
paragraph: {
rich_text: Array<{
text: {
content: string
link?: { url: string }
}
annotations?: {
bold?: boolean
italic?: boolean
strikethrough?: boolean
underline?: boolean
code?: boolean
color?: AnnotationColor
}
}>
}
}>
}
interface Settings {
parentPageId: string
parentDatabaseId: string
}
export class NotionClient implements IntegrationClient {
name = 'NOTION'
_headers = {
'Content-Type': 'application/json',
Accept: 'application/json',
'Notion-Version': '2022-06-28',
}
_timeout = 5000 // 5 seconds
_axios = axios.create({
baseURL: 'https://api.notion.com/v1',
timeout: this._timeout,
})
_token: string
_client: Client
_integrationData?: Merge<Integration, { settings?: Settings }>
constructor(token: string, integration?: Integration) {
this._token = token
this._client = new Client({
auth: token,
timeoutMs: this._timeout,
})
this._integrationData = integration
}
accessToken = async (): Promise<string | null> => {
try {
// encode in base 64
const encoded = Buffer.from(
`${env.notion.clientId}:${env.notion.clientSecret}`
).toString('base64')
const response = await this._axios.post<{ access_token: string }>(
'/oauth/token',
{
grant_type: 'authorization_code',
code: this._token,
redirect_uri: `${env.client.url}/settings/integrations`,
},
{
headers: {
...this._headers,
Authorization: `Basic ${encoded}`,
},
}
)
return response.data.access_token
} catch (error) {
if (axios.isAxiosError(error)) {
logger.error(error.response)
} else {
logger.error(error)
}
return null
}
}
async auth(): Promise<string> {
return Promise.resolve(env.notion.authUrl)
}
private _itemToNotionPage = (item: LibraryItem): NotionPage => {
const databaseId = this._integrationData?.settings?.parentDatabaseId
if (!databaseId) {
throw new Error('Notion database id not found')
}
return {
parent: {
database_id: databaseId,
},
icon: item.siteIcon
? {
external: {
url: item.siteIcon,
},
}
: undefined,
cover: item.thumbnail
? {
external: {
url: item.thumbnail,
},
}
: undefined,
properties: {
Title: {
title: [
{
text: {
content: item.title,
},
},
],
},
Author: {
rich_text: [
{
text: {
content: item.author || 'unknown',
},
},
],
},
'Original URL': {
url: item.originalUrl,
},
'Omnivore URL': {
url: `${env.client.url}/me/${item.slug}`,
},
Tags: item.labels
? { multi_select: item.labels.map((label) => ({ name: label.name })) }
: undefined,
},
children: item.highlights
? item.highlights.map((highlight) => ({
type: 'paragraph',
paragraph: {
rich_text: [
{
text: {
content: highlight.quote || '',
},
annotations: {
color: highlight.color as AnnotationColor,
},
},
{
text: {
content: `\n${highlight.annotation || ''}`,
},
annotations: {
italic: true,
},
},
],
},
}))
: undefined,
}
}
_createPage = async (page: NotionPage) => {
await this._client.pages.create(page)
}
export = async (items: LibraryItem[]): Promise<boolean> => {
if (!this._integrationData || !this._integrationData.settings) {
logger.error('Notion integration data not found')
return false
}
const pageId = this._integrationData.settings.parentPageId
if (!pageId) {
logger.error('Notion parent page id not found')
return false
}
const databaseId = this._integrationData.settings.parentDatabaseId
if (!databaseId) {
// create a database for the items
const database = await this._client.databases.create({
parent: {
page_id: pageId,
},
title: [
{
text: {
content: 'Library',
},
},
],
description: [
{
text: {
content: 'Library of saved items from Omnivore',
},
},
],
properties: {
Title: {
title: {},
},
Author: {
rich_text: {},
},
'Original URL': {
url: {},
},
'Omnivore URL': {
url: {},
},
Tags: {
multi_select: {},
},
},
})
// save the database id
this._integrationData.settings.parentDatabaseId = database.id
await updateIntegration(
this._integrationData.id,
{
settings: this._integrationData.settings,
},
this._integrationData.user.id
)
}
const pages = items.map(this._itemToNotionPage)
await Promise.all(pages.map((page) => this._createPage(page)))
return true
}
}

View file

@ -5,24 +5,27 @@ import { IntegrationClient } from './integration'
export class PocketClient implements IntegrationClient {
name = 'POCKET'
apiUrl = 'https://getpocket.com/v3'
headers = {
'Content-Type': 'application/json',
'X-Accept': 'application/json',
_token: string
_axios = axios.create({
baseURL: 'https://getpocket.com/v3',
headers: {
'Content-Type': 'application/json',
'X-Accept': 'application/json',
},
timeout: 5000, // 5 seconds
})
constructor(token: string) {
this._token = token
}
accessToken = async (token: string): Promise<string | null> => {
const url = `${this.apiUrl}/oauth/authorize`
accessToken = async (): Promise<string | null> => {
try {
const response = await axios.post<{ access_token: string }>(
url,
const response = await this._axios.post<{ access_token: string }>(
'/oauth/authorize',
{
consumer_key: env.pocket.consumerKey,
code: token,
},
{
headers: this.headers,
timeout: 5000, // 5 seconds
code: this._token,
}
)
return response.data.access_token
@ -36,7 +39,26 @@ export class PocketClient implements IntegrationClient {
}
}
export = async (): Promise<boolean> => {
return Promise.resolve(false)
export = () => {
throw new Error('Method not implemented.')
}
async auth(state: string) {
const consumerKey = env.pocket.consumerKey
const redirectUri = `${env.client.url}/settings/integrations`
// make a POST request to Pocket to get a request token
const response = await this._axios.post<{ code: string }>(
'/oauth/request',
{
consumer_key: consumerKey,
redirect_uri: redirectUri,
}
)
const { code } = response.data
return `https://getpocket.com/auth/authorize?request_token=${code}&redirect_uri=${redirectUri}${encodeURIComponent(
`?pocketToken=${code}&state=${state}`
)}`
}
}

View file

@ -33,17 +33,28 @@ interface ReadwiseHighlight {
export class ReadwiseClient implements IntegrationClient {
name = 'READWISE'
apiUrl = 'https://readwise.io/api/v2'
_headers = {
'Content-Type': 'application/json',
}
_axios = axios.create({
baseURL: 'https://readwise.io/api/v2',
timeout: 5000, // 5 seconds
})
_token: string
accessToken = async (token: string): Promise<string | null> => {
const authUrl = `${this.apiUrl}/auth`
constructor(token: string) {
this._token = token
}
accessToken = async (): Promise<string | null> => {
try {
const response = await axios.get(authUrl, {
const response = await this._axios.get('/auth', {
headers: {
Authorization: `Token ${token}`,
...this._headers,
Authorization: `Token ${this._token}`,
},
})
return response.status === 204 ? token : null
return response.status === 204 ? this._token : null
} catch (error) {
if (axios.isAxiosError(error)) {
logger.error(error.response)
@ -54,20 +65,26 @@ export class ReadwiseClient implements IntegrationClient {
}
}
export = async (token: string, items: LibraryItem[]): Promise<boolean> => {
export = async (items: LibraryItem[]): Promise<boolean> => {
let result = true
const highlights = items.flatMap(this.itemToReadwiseHighlight)
const highlights = items.flatMap(this._itemToReadwiseHighlight)
// If there are no highlights, we will skip the sync
if (highlights.length > 0) {
result = await this.syncWithReadwise(token, highlights)
result = await this._syncWithReadwise(highlights)
}
return result
}
itemToReadwiseHighlight = (item: LibraryItem): ReadwiseHighlight[] => {
auth = () => {
throw new Error('Method not implemented.')
}
private _itemToReadwiseHighlight = (
item: LibraryItem
): ReadwiseHighlight[] => {
const category = item.siteName === 'Twitter' ? 'tweets' : 'articles'
return item.highlights
?.map((highlight) => {
@ -93,22 +110,19 @@ export class ReadwiseClient implements IntegrationClient {
.filter((highlight) => highlight !== undefined) as ReadwiseHighlight[]
}
syncWithReadwise = async (
token: string,
private _syncWithReadwise = async (
highlights: ReadwiseHighlight[]
): Promise<boolean> => {
const url = `${this.apiUrl}/highlights`
const response = await axios.post(
url,
const response = await this._axios.post(
'/highlights',
{
highlights,
},
{
headers: {
Authorization: `Token ${token}`,
'Content-Type': 'application/json',
...this._headers,
Authorization: `Token ${this._token}`,
},
timeout: 5000, // 5 seconds
}
)
return response.status === 200

View file

@ -995,7 +995,7 @@ export const createOrUpdateLibraryItem = async (
)
}
if (skipPubSub) {
if (skipPubSub || libraryItem.state === LibraryItemState.Processing) {
return newLibraryItem
}

View file

@ -113,6 +113,11 @@ export interface BackendEnv {
mq: redisConfig
cache: redisConfig
}
notion: {
clientId: string
clientSecret: string
authUrl: string
}
}
const nullableEnvVars = [
@ -165,6 +170,9 @@ const nullableEnvVars = [
'MQ_REDIS_CERT',
'IMPORTER_METRICS_COLLECTOR_URL',
'INTERNAL_API_URL',
'NOTION_CLIENT_ID',
'NOTION_CLIENT_SECRET',
'NOTION_AUTH_URL',
] // Allow some vars to be null/empty
/* If not in GAE and Prod/QA/Demo env (f.e. on localhost/dev env), allow following env vars to be null */
@ -311,6 +319,11 @@ export function getEnv(): BackendEnv {
cert: parse('REDIS_CERT')?.replace(/\\n/g, '\n'), // replace \n with new line
},
}
const notion = {
clientId: parse('NOTION_CLIENT_ID'),
clientSecret: parse('NOTION_CLIENT_SECRET'),
authUrl: parse('NOTION_AUTH_URL'),
}
return {
pg,
@ -333,6 +346,7 @@ export function getEnv(): BackendEnv {
pocket,
subscription,
redis,
notion,
}
}

View file

@ -0,0 +1,9 @@
-- Type: DO
-- Name: add_settings_column_to_integrations
-- Description: Add settings column to integrations table
BEGIN;
ALTER TABLE omnivore.integrations ADD COLUMN settings jsonb;
COMMIT;

View file

@ -0,0 +1,9 @@
-- Type: UNDO
-- Name: add_settings_column_to_integrations
-- Description: Add settings column to integrations table
BEGIN;
ALTER TABLE omnivore.integrations DROP COLUMN settings;
COMMIT;

View file

@ -3973,6 +3973,14 @@
"@nodelib/fs.scandir" "2.1.3"
fastq "^1.6.0"
"@notionhq/client@^2.2.14":
version "2.2.14"
resolved "https://registry.yarnpkg.com/@notionhq/client/-/client-2.2.14.tgz#6807ec27ee89584529abfd28d058b2661f828b74"
integrity sha512-oqUefZtCiJPCX+74A1Os9OVTef3fSnVWe2eVQtU1HJSD+nsfxfhwvDKnzJTh2Tw1ZHKLxpieHB/nzGdY+Uo12A==
dependencies:
"@types/node-fetch" "^2.5.10"
node-fetch "^2.6.1"
"@npmcli/arborist@^5.6.3":
version "5.6.3"
resolved "https://registry.yarnpkg.com/@npmcli/arborist/-/arborist-5.6.3.tgz#40810080272e097b4a7a4f56108f4a31638a9874"
@ -7871,6 +7879,14 @@
dependencies:
"@types/node" "*"
"@types/node-fetch@^2.5.10", "@types/node-fetch@^2.6.4":
version "2.6.11"
resolved "https://registry.yarnpkg.com/@types/node-fetch/-/node-fetch-2.6.11.tgz#9b39b78665dae0e82a08f02f4967d62c66f95d24"
integrity sha512-24xFj9R5+rfQJLRyM56qh+wnVSYhyXC2tkoBndtY0U+vubqNsYXGjufB2nn8Q6gt0LrARwL6UBtMCSVCwl4B1g==
dependencies:
"@types/node" "*"
form-data "^4.0.0"
"@types/node-fetch@^2.5.7":
version "2.6.1"
resolved "https://registry.yarnpkg.com/@types/node-fetch/-/node-fetch-2.6.1.tgz#8f127c50481db65886800ef496f20bbf15518975"
@ -7879,14 +7895,6 @@
"@types/node" "*"
form-data "^3.0.0"
"@types/node-fetch@^2.6.4":
version "2.6.11"
resolved "https://registry.yarnpkg.com/@types/node-fetch/-/node-fetch-2.6.11.tgz#9b39b78665dae0e82a08f02f4967d62c66f95d24"
integrity sha512-24xFj9R5+rfQJLRyM56qh+wnVSYhyXC2tkoBndtY0U+vubqNsYXGjufB2nn8Q6gt0LrARwL6UBtMCSVCwl4B1g==
dependencies:
"@types/node" "*"
form-data "^4.0.0"
"@types/node-fetch@^2.6.6":
version "2.6.7"
resolved "https://registry.yarnpkg.com/@types/node-fetch/-/node-fetch-2.6.7.tgz#a1abe2ce24228b58ad97f99480fdcf9bbc6ab16d"