diff --git a/packages/api/src/directives.ts b/packages/api/src/directives.ts index 435166539..292a546b8 100644 --- a/packages/api/src/directives.ts +++ b/packages/api/src/directives.ts @@ -15,6 +15,7 @@ export const sanitizeDirectiveTransformer = (schema: GraphQLSchema) => { } const maxLength = sanitizeDirective.maxLength as number | undefined + const minLength = sanitizeDirective.minLength as number | undefined const allowedTags = sanitizeDirective.allowedTags as string[] | undefined const pattern = sanitizeDirective.pattern as string | undefined @@ -27,6 +28,7 @@ export const sanitizeDirectiveTransformer = (schema: GraphQLSchema) => { fieldConfig.type.ofType, allowedTags, maxLength, + minLength, pattern ) ) @@ -35,6 +37,7 @@ export const sanitizeDirectiveTransformer = (schema: GraphQLSchema) => { fieldConfig.type, allowedTags, maxLength, + minLength, pattern ) } else { diff --git a/packages/api/src/generated/graphql.ts b/packages/api/src/generated/graphql.ts index d3bfcc675..3334b1968 100644 --- a/packages/api/src/generated/graphql.ts +++ b/packages/api/src/generated/graphql.ts @@ -4062,6 +4062,7 @@ export type ResolversParentTypes = { export type SanitizeDirectiveArgs = { allowedTags?: Maybe>>; maxLength?: Maybe; + minLength?: Maybe; pattern?: Maybe; }; diff --git a/packages/api/src/generated/schema.graphql b/packages/api/src/generated/schema.graphql index 96d6b560c..629d64e48 100644 --- a/packages/api/src/generated/schema.graphql +++ b/packages/api/src/generated/schema.graphql @@ -1,4 +1,4 @@ -directive @sanitize(allowedTags: [String], maxLength: Int, pattern: String) on INPUT_FIELD_DEFINITION +directive @sanitize(allowedTags: [String], maxLength: Int, minLength: Int, pattern: String) on INPUT_FIELD_DEFINITION type AddPopularReadError { errorCodes: [AddPopularReadErrorCode!]! diff --git a/packages/api/src/scalars.ts b/packages/api/src/scalars.ts index 22404534f..aa74cddb7 100644 --- a/packages/api/src/scalars.ts +++ b/packages/api/src/scalars.ts @@ -9,6 +9,7 @@ export class SanitizedString extends GraphQLScalarType { type: GraphQLScalarType, allowedTags?: string[], maxLength?: number, + minLength?: number, pattern?: string ) { super({ @@ -25,11 +26,7 @@ export class SanitizedString extends GraphQLScalarType { // invoked when a query is passed as a JSON object (for example, when Apollo Client makes a request parseValue(value) { - if (maxLength && maxLength < value.length) { - throw new Error( - `Specified value cannot be longer than ${maxLength} characters` - ) - } + checkLength(value) if (pattern && !new RegExp(pattern).test(value)) { throw new Error(`Specified value does not match pattern`) } @@ -39,17 +36,26 @@ export class SanitizedString extends GraphQLScalarType { // invoked when a query is passed as a string parseLiteral(ast) { const value = type.parseLiteral(ast, {}) - if (maxLength && maxLength < value.length) { - throw new Error( - `Specified value cannot be longer than ${maxLength} characters` - ) - } + checkLength(value) if (pattern && !new RegExp(pattern).test(value)) { throw new Error(`Specified value does not match pattern`) } return sanitize(value, { allowedTags: allowedTags || [] }) }, }) + + function checkLength(value: any) { + if (maxLength && maxLength < value.length) { + throw new Error( + `Specified value cannot be longer than ${maxLength} characters` + ) + } + if (minLength && minLength > value.length) { + throw new Error( + `Specified value cannot be shorter than ${minLength} characters` + ) + } + } } } diff --git a/packages/api/src/schema.ts b/packages/api/src/schema.ts index 1c43b4e22..c298c856a 100755 --- a/packages/api/src/schema.ts +++ b/packages/api/src/schema.ts @@ -8,6 +8,7 @@ const schema = gql` directive @sanitize( allowedTags: [String] maxLength: Int + minLength: Int pattern: String ) on INPUT_FIELD_DEFINITION @@ -688,7 +689,7 @@ const schema = gql` shortId: String! articleId: ID! patch: String! - quote: String! @sanitize(maxLength: 6000) + quote: String! @sanitize(maxLength: 6000, minLength: 1) prefix: String @sanitize suffix: String @sanitize annotation: String @sanitize(maxLength: 4000) @@ -720,7 +721,7 @@ const schema = gql` shortId: ID! articleId: ID! patch: String! - quote: String! @sanitize(maxLength: 6000) + quote: String! @sanitize(maxLength: 6000, minLength: 1) prefix: String @sanitize suffix: String @sanitize annotation: String @sanitize(maxLength: 8000) @@ -752,7 +753,7 @@ const schema = gql` highlightId: ID! annotation: String @sanitize(maxLength: 4000) sharedAt: Date - quote: String @sanitize(maxLength: 6000) + quote: String @sanitize(maxLength: 6000, minLength: 1) } type UpdateHighlightSuccess { diff --git a/packages/api/src/services/integrations.ts b/packages/api/src/services/integrations.ts index cde294253..9cff4567b 100644 --- a/packages/api/src/services/integrations.ts +++ b/packages/api/src/services/integrations.ts @@ -65,22 +65,28 @@ const validateReadwiseToken = async (token: string): Promise => { const pageToReadwiseHighlight = (page: Page): ReadwiseHighlight[] => { if (!page.highlights) return [] - return page.highlights.map((highlight) => { - return { - text: highlight.quote, - title: page.title, - author: page.author || undefined, - highlight_url: getHighlightUrl(page.slug, highlight.id), - highlighted_at: new Date(highlight.createdAt).toISOString(), - category: 'articles', - image_url: page.image || undefined, - location: highlight.highlightPositionPercent || undefined, - location_type: 'order', - note: highlight.annotation || undefined, - source_type: 'omnivore', - source_url: page.url, - } - }) + const category = page.siteName === 'Twitter' ? 'tweets' : 'articles' + return ( + page.highlights + // filter out highlights with no quote + .filter((highlight) => highlight.quote.length === 0) + .map((highlight) => { + return { + text: highlight.quote, + title: page.title, + author: page.author || undefined, + highlight_url: getHighlightUrl(page.slug, highlight.id), + highlighted_at: new Date(highlight.createdAt).toISOString(), + category, + image_url: page.image || undefined, + // location: highlight.highlightPositionAnchorIndex || undefined, + location_type: 'order', + note: highlight.annotation || undefined, + source_type: 'omnivore', + source_url: page.url, + } + }) + ) } export const syncWithIntegration = async ( @@ -131,19 +137,31 @@ export const syncWithReadwise = async ( ) return response.status === 200 } catch (error) { - if ( - axios.isAxiosError(error) && - error.response?.status === 429 && - retryCount < 3 - ) { - console.log('Readwise API rate limit exceeded, retrying...') - // wait for Retry-After seconds in the header if rate limited - // max retry count is 3 - const retryAfter = error.response?.headers['retry-after'] || '10' // default to 10 seconds - await wait(parseInt(retryAfter, 10) * 1000) - return syncWithReadwise(token, highlights, retryCount + 1) + if (axios.isAxiosError(error)) { + if (error.response) { + if (error.response.status === 429 && retryCount < 3) { + console.log('Readwise API rate limit exceeded, retrying...') + // wait for Retry-After seconds in the header if rate limited + // max retry count is 3 + const retryAfter = error.response?.headers['retry-after'] || '10' // default to 10 seconds + await wait(parseInt(retryAfter, 10) * 1000) + return syncWithReadwise(token, highlights, retryCount + 1) + } + // The request was made and the server responded with a status code + // that falls out of the range of 2xx + console.log('Readwise error, response data', error.response.data) + } else if (error.request) { + // The request was made but no response was received + // `error.request` is an instance of XMLHttpRequest in the browser and an instance of + // http.ClientRequest in node.js + console.log('Readwise error, request', error.request) + } else { + // Something happened in setting up the request that triggered an Error + console.log('Error', error.message) + } + } else { + console.log('Error syncing with readwise', error) } - console.log('Error creating highlights in Readwise', error) return false } } diff --git a/packages/content-handler/src/websites/twitter-handler.ts b/packages/content-handler/src/websites/twitter-handler.ts index 121e380a4..1116d5b7f 100644 --- a/packages/content-handler/src/websites/twitter-handler.ts +++ b/packages/content-handler/src/websites/twitter-handler.ts @@ -228,23 +228,8 @@ const getTweetIds = async ( const ids: Set = new Set() - // Find the first Show thread button and click it - const showRepliesButton = Array.from( - document.querySelectorAll('div[dir="auto"]') - ) - .filter( - (node) => node.children[0] && node.children[0].tagName === 'SPAN' - ) - .find((node) => node.children[0].innerHTML === 'Show replies') - - if (showRepliesButton) { - ;(showRepliesButton as HTMLElement).click() - - await waitFor(2000) - } - const distance = 1080 - const scrollHeight = document.body.scrollHeight + let scrollHeight = document.body.scrollHeight let currentHeight = 0 // keep scrolling until there are no more elements while (currentHeight < scrollHeight) { @@ -269,13 +254,31 @@ const getTweetIds = async ( const id = match[2] const username = match[1] - // skip non-author replies - username === author && ids.add(id) + // stop at non-author replies + if (username !== author) return Array.from(ids) + ids.add(id) } window.scrollBy(0, distance) await waitFor(500) currentHeight += distance + + // Find the show replies button and click it + if (currentHeight >= scrollHeight) { + const showRepliesButton = Array.from( + document.querySelectorAll('div[dir]') + ) + .filter( + (node) => node.children[0] && node.children[0].tagName === 'SPAN' + ) + .find((node) => node.children[0].innerHTML === 'Show replies') + + if (showRepliesButton) { + ;(showRepliesButton as HTMLElement).click() + await waitFor(1000) + scrollHeight = document.body.scrollHeight + } + } } return Array.from(ids) @@ -371,6 +374,8 @@ export class TwitterHandler extends ContentHandler { + +