prehistoric-systems/omnivore
1
0
Fork 0
mirror of https://github.com/omnivore-app/omnivore.git synced 2026-03-11 08:54:26 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge pull request #1896 from omnivore-app/debug-readwise-api-integration

Browse source 
Fix readwise integration issue
This commit is contained in:
Hongbo Wu 2023-03-10 16:51:07 +08:00 committed by GitHub
commit a3ab0fc35b
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
7 changed files with 94 additions and 60 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
packages/api/src/directives.ts
View file

@ -15,6 +15,7 @@ export const sanitizeDirectiveTransformer = (schema: GraphQLSchema) => {
}
const maxLength = sanitizeDirective.maxLength as number | undefined
const minLength = sanitizeDirective.minLength as number | undefined
const allowedTags = sanitizeDirective.allowedTags as string[] | undefined
const pattern = sanitizeDirective.pattern as string | undefined
@ -27,6 +28,7 @@ export const sanitizeDirectiveTransformer = (schema: GraphQLSchema) => {
fieldConfig.type.ofType,
allowedTags,
maxLength,
minLength,
pattern
)
)
@ -35,6 +37,7 @@ export const sanitizeDirectiveTransformer = (schema: GraphQLSchema) => {
fieldConfig.type,
allowedTags,
maxLength,
minLength,
pattern
)
} else {

1
packages/api/src/generated/graphql.ts
View file

@ -4062,6 +4062,7 @@ export type ResolversParentTypes = {
export type SanitizeDirectiveArgs = {
allowedTags?: Maybe<Array<Maybe<Scalars['String']>>>;
maxLength?: Maybe<Scalars['Int']>;
minLength?: Maybe<Scalars['Int']>;
pattern?: Maybe<Scalars['String']>;
};

2
packages/api/src/generated/schema.graphql
View file

@ -1,4 +1,4 @@
directive @sanitize(allowedTags: [String], maxLength: Int, pattern: String) on INPUT_FIELD_DEFINITION
directive @sanitize(allowedTags: [String], maxLength: Int, minLength: Int, pattern: String) on INPUT_FIELD_DEFINITION
type AddPopularReadError {
errorCodes: [AddPopularReadErrorCode!]!

26
packages/api/src/scalars.ts
View file

@ -9,6 +9,7 @@ export class SanitizedString extends GraphQLScalarType {
type: GraphQLScalarType,
allowedTags?: string[],
maxLength?: number,
minLength?: number,
pattern?: string
) {
super({
@ -25,11 +26,7 @@ export class SanitizedString extends GraphQLScalarType {
// invoked when a query is passed as a JSON object (for example, when Apollo Client makes a request
parseValue(value) {
if (maxLength && maxLength < value.length) {
throw new Error(
`Specified value cannot be longer than ${maxLength} characters`
)
}
checkLength(value)
if (pattern && !new RegExp(pattern).test(value)) {
throw new Error(`Specified value does not match pattern`)
}
@ -39,17 +36,26 @@ export class SanitizedString extends GraphQLScalarType {
// invoked when a query is passed as a string
parseLiteral(ast) {
const value = type.parseLiteral(ast, {})
if (maxLength && maxLength < value.length) {
throw new Error(
`Specified value cannot be longer than ${maxLength} characters`
)
}
checkLength(value)
if (pattern && !new RegExp(pattern).test(value)) {
throw new Error(`Specified value does not match pattern`)
}
return sanitize(value, { allowedTags: allowedTags || [] })
},
})
function checkLength(value: any) {
if (maxLength && maxLength < value.length) {
throw new Error(
`Specified value cannot be longer than ${maxLength} characters`
)
}
if (minLength && minLength > value.length) {
throw new Error(
`Specified value cannot be shorter than ${minLength} characters`
)
}
}
}
}

7
packages/api/src/schema.ts
View file

@ -8,6 +8,7 @@ const schema = gql`
directive @sanitize(
allowedTags: [String]
maxLength: Int
minLength: Int
pattern: String
) on INPUT_FIELD_DEFINITION
@ -688,7 +689,7 @@ const schema = gql`
shortId: String!
articleId: ID!
patch: String!
quote: String! @sanitize(maxLength: 6000)
quote: String! @sanitize(maxLength: 6000, minLength: 1)
prefix: String @sanitize
suffix: String @sanitize
annotation: String @sanitize(maxLength: 4000)
@ -720,7 +721,7 @@ const schema = gql`
shortId: ID!
articleId: ID!
patch: String!
quote: String! @sanitize(maxLength: 6000)
quote: String! @sanitize(maxLength: 6000, minLength: 1)
prefix: String @sanitize
suffix: String @sanitize
annotation: String @sanitize(maxLength: 8000)
@ -752,7 +753,7 @@ const schema = gql`
highlightId: ID!
annotation: String @sanitize(maxLength: 4000)
sharedAt: Date
quote: String @sanitize(maxLength: 6000)
quote: String @sanitize(maxLength: 6000, minLength: 1)
}
type UpdateHighlightSuccess {

74
packages/api/src/services/integrations.ts
View file

@ -65,22 +65,28 @@ const validateReadwiseToken = async (token: string): Promise<boolean> => {
const pageToReadwiseHighlight = (page: Page): ReadwiseHighlight[] => {
if (!page.highlights) return []
return page.highlights.map((highlight) => {
return {
text: highlight.quote,
title: page.title,
author: page.author || undefined,
highlight_url: getHighlightUrl(page.slug, highlight.id),
highlighted_at: new Date(highlight.createdAt).toISOString(),
category: 'articles',
image_url: page.image || undefined,
location: highlight.highlightPositionPercent || undefined,
location_type: 'order',
note: highlight.annotation || undefined,
source_type: 'omnivore',
source_url: page.url,
}
})
const category = page.siteName === 'Twitter' ? 'tweets' : 'articles'
return (
page.highlights
// filter out highlights with no quote
.filter((highlight) => highlight.quote.length === 0)
.map((highlight) => {
return {
text: highlight.quote,
title: page.title,
author: page.author || undefined,
highlight_url: getHighlightUrl(page.slug, highlight.id),
highlighted_at: new Date(highlight.createdAt).toISOString(),
category,
image_url: page.image || undefined,
// location: highlight.highlightPositionAnchorIndex || undefined,
location_type: 'order',
note: highlight.annotation || undefined,
source_type: 'omnivore',
source_url: page.url,
}
})
)
}
export const syncWithIntegration = async (
@ -131,19 +137,31 @@ export const syncWithReadwise = async (
)
return response.status === 200
} catch (error) {
if (
axios.isAxiosError(error) &&
error.response?.status === 429 &&
retryCount < 3
) {
console.log('Readwise API rate limit exceeded, retrying...')
// wait for Retry-After seconds in the header if rate limited
// max retry count is 3
const retryAfter = error.response?.headers['retry-after'] || '10' // default to 10 seconds
await wait(parseInt(retryAfter, 10) * 1000)
return syncWithReadwise(token, highlights, retryCount + 1)
if (axios.isAxiosError(error)) {
if (error.response) {
if (error.response.status === 429 && retryCount < 3) {
console.log('Readwise API rate limit exceeded, retrying...')
// wait for Retry-After seconds in the header if rate limited
// max retry count is 3
const retryAfter = error.response?.headers['retry-after'] || '10' // default to 10 seconds
await wait(parseInt(retryAfter, 10) * 1000)
return syncWithReadwise(token, highlights, retryCount + 1)
}
// The request was made and the server responded with a status code
// that falls out of the range of 2xx
console.log('Readwise error, response data', error.response.data)
} else if (error.request) {
// The request was made but no response was received
// `error.request` is an instance of XMLHttpRequest in the browser and an instance of
// http.ClientRequest in node.js
console.log('Readwise error, request', error.request)
} else {
// Something happened in setting up the request that triggered an Error
console.log('Error', error.message)
}
} else {
console.log('Error syncing with readwise', error)
}
console.log('Error creating highlights in Readwise', error)
return false
}
}

41
packages/content-handler/src/websites/twitter-handler.ts
View file

@ -228,23 +228,8 @@ const getTweetIds = async (
const ids: Set<string> = new Set()
// Find the first Show thread button and click it
const showRepliesButton = Array.from(
document.querySelectorAll('div[dir="auto"]')
)
.filter(
(node) => node.children[0] && node.children[0].tagName === 'SPAN'
)
.find((node) => node.children[0].innerHTML === 'Show replies')
if (showRepliesButton) {
;(showRepliesButton as HTMLElement).click()
await waitFor(2000)
}
const distance = 1080
const scrollHeight = document.body.scrollHeight
let scrollHeight = document.body.scrollHeight
let currentHeight = 0
// keep scrolling until there are no more elements
while (currentHeight < scrollHeight) {
@ -269,13 +254,31 @@ const getTweetIds = async (
const id = match[2]
const username = match[1]
// skip non-author replies
username === author && ids.add(id)
// stop at non-author replies
if (username !== author) return Array.from(ids)
ids.add(id)
}
window.scrollBy(0, distance)
await waitFor(500)
currentHeight += distance
// Find the show replies button and click it
if (currentHeight >= scrollHeight) {
const showRepliesButton = Array.from(
document.querySelectorAll('div[dir]')
)
.filter(
(node) => node.children[0] && node.children[0].tagName === 'SPAN'
)
.find((node) => node.children[0].innerHTML === 'Show replies')
if (showRepliesButton) {
;(showRepliesButton as HTMLElement).click()
await waitFor(1000)
scrollHeight = document.body.scrollHeight
}
}
}
return Array.from(ids)
@ -371,6 +374,8 @@ export class TwitterHandler extends ContentHandler {
<meta property="og:image:secure_url" content="${authorImage}" />
<meta property="og:title" content="${escapedTitle}" />
<meta property="og:description" content="${description}" />
<meta property="article:published_time" content="${tweetData.created_at}" />
<meta property="og:site_name" content="Twitter" />
</head>
<body>
<div>