diff --git a/packages/api/src/resolvers/article/index.ts b/packages/api/src/resolvers/article/index.ts index 417d83146..1c1bd5888 100644 --- a/packages/api/src/resolvers/article/index.ts +++ b/packages/api/src/resolvers/article/index.ts @@ -82,7 +82,7 @@ import { createLabels, getLabelsByIds, } from '../../services/labels' -import { cleanUrl, parsedContentToPage } from '../../services/save_page' +import { parsedContentToPage } from '../../services/save_page' import { traceAs } from '../../tracing' import { Merge } from '../../util' import { analytics } from '../../utils/analytics' @@ -90,6 +90,7 @@ import { isSiteBlockedForParse } from '../../utils/blocked' import { ContentParseError } from '../../utils/errors' import { authorized, + cleanUrl, generateSlug, isBase64Image, isParsingTimeout, @@ -195,6 +196,7 @@ export const createArticleResolver = authorized< ) } + url = cleanUrl(url) const { pathname } = new URL(url) const croppedPathname = decodeURIComponent( @@ -227,7 +229,7 @@ export const createArticleResolver = authorized< pageType: PageType.Unknown, contentReader: ContentReader.Web, author: '', - url: cleanUrl(canonicalUrl || url), + url, hash: '', isArchived: false, }, diff --git a/packages/api/src/resolvers/article_saving_request/index.ts b/packages/api/src/resolvers/article_saving_request/index.ts index 197f970dc..381a4a440 100644 --- a/packages/api/src/resolvers/article_saving_request/index.ts +++ b/packages/api/src/resolvers/article_saving_request/index.ts @@ -15,10 +15,10 @@ import { QueryArticleSavingRequestArgs, } from '../../generated/graphql' import { createPageSaveRequest } from '../../services/create_page_save_request' -import { cleanUrl } from '../../services/save_page' import { analytics } from '../../utils/analytics' import { authorized, + cleanUrl, isParsingTimeout, pageToArticleSavingRequest, } from '../../utils/helpers' diff --git a/packages/api/src/services/create_page_save_request.ts b/packages/api/src/services/create_page_save_request.ts index 20f067f52..724389618 100644 --- a/packages/api/src/services/create_page_save_request.ts +++ b/packages/api/src/services/create_page_save_request.ts @@ -15,8 +15,11 @@ import { CreateArticleSavingRequestErrorCode, } from '../generated/graphql' import { enqueueParseRequest } from '../utils/createTask' -import { generateSlug, pageToArticleSavingRequest } from '../utils/helpers' -import { cleanUrl } from './save_page' +import { + cleanUrl, + generateSlug, + pageToArticleSavingRequest, +} from '../utils/helpers' interface PageSaveRequest { userId: string @@ -104,7 +107,7 @@ export const createPageSaveRequest = async ({ priority = priority || (await getPriorityByRateLimit(userId)) // look for existing page - const normalizedUrl = cleanUrl(url) + url = cleanUrl(url) const ctx = { pubsub, @@ -113,10 +116,10 @@ export const createPageSaveRequest = async ({ } let page = await getPageByParam({ userId, - url: normalizedUrl, + url, }) if (!page) { - console.log('Page not exists', normalizedUrl) + console.log('Page not exists', url) page = { id: articleSavingRequestId, userId, @@ -127,7 +130,7 @@ export const createPageSaveRequest = async ({ readingProgressPercent: 0, slug: generateSlug(url), title: url, - url: normalizedUrl, + url, state: ArticleSavingRequestStatus.Processing, createdAt: new Date(), savedAt: new Date(), @@ -161,7 +164,7 @@ export const createPageSaveRequest = async ({ })) // enqueue task to parse page await enqueueParseRequest({ - url: normalizedUrl, + url, userId, saveRequestId: page.id, priority, diff --git a/packages/api/src/services/save_email.ts b/packages/api/src/services/save_email.ts index 04999329a..fa258ee44 100644 --- a/packages/api/src/services/save_email.ts +++ b/packages/api/src/services/save_email.ts @@ -3,6 +3,7 @@ import { createPage, getPageByParam, updatePage } from '../elastic/pages' import { ArticleSavingRequestStatus, Page } from '../elastic/types' import { enqueueThumbnailTask } from '../utils/createTask' import { + cleanUrl, generateSlug, stringToHash, validatedDate, @@ -13,7 +14,6 @@ import { parsePreparedContent, parseUrlMetadata, } from '../utils/parser' -import { cleanUrl } from './save_page' export type SaveContext = { pubsub: PubsubClient diff --git a/packages/api/src/services/save_page.ts b/packages/api/src/services/save_page.ts index a65a48d5e..560b6d3f0 100644 --- a/packages/api/src/services/save_page.ts +++ b/packages/api/src/services/save_page.ts @@ -1,5 +1,4 @@ import { Readability } from '@omnivore/readability' -import normalizeUrl from 'normalize-url' import { PubsubClient } from '../datalayer/pubsub' import { addHighlightToPage } from '../elastic/highlights' import { createPage, getPageByParam, updatePage } from '../elastic/pages' @@ -16,8 +15,10 @@ import { import { DataModels } from '../resolvers/types' import { enqueueThumbnailTask } from '../utils/createTask' import { + cleanUrl, generateSlug, stringToHash, + TWEET_URL_REGEX, validatedDate, wordsCount, } from '../utils/helpers' @@ -36,30 +37,12 @@ type SaverUserData = { username: string } -const TWEET_URL_REGEX = - /twitter\.com\/(?:#!\/)?(\w+)\/status(?:es)?\/(\d+)(?:\/.*)?/ - // where we can use APIs to fetch their underlying content. const FORCE_PUPPETEER_URLS = [ TWEET_URL_REGEX, /^((?:https?:)?\/\/)?((?:www|m)\.)?((?:youtube\.com|youtu.be))(\/(?:[\w-]+\?v=|embed\/|v\/)?)([\w-]+)(\S+)?$/, ] -export const cleanUrl = (url: string) => { - const trackingParams: (RegExp | string)[] = [/^utm_\w+/i] // remove utm tracking parameters - if (TWEET_URL_REGEX.test(url)) { - console.debug('cleaning tweet url', url) - // remove tracking parameters from tweet links: - // https://twitter.com/omnivore/status/1673218959624093698?s=12&t=R91quPajs0E53Yds-fhv2g - trackingParams.push('s', 't') - } - return normalizeUrl(url, { - stripHash: true, - stripWWW: false, - removeQueryParameters: trackingParams, - }) -} - const createSlug = (url: string, title?: Maybe | undefined) => { const { pathname } = new URL(url) const croppedPathname = decodeURIComponent( diff --git a/packages/api/src/utils/helpers.ts b/packages/api/src/utils/helpers.ts index 315a46804..daf305881 100644 --- a/packages/api/src/utils/helpers.ts +++ b/packages/api/src/utils/helpers.ts @@ -26,6 +26,9 @@ interface InputObject { [key: string]: any } +export const TWEET_URL_REGEX = + /twitter\.com\/(?:#!\/)?(\w+)\/status(?:es)?\/(\d+)(?:\/.*)?/ + export const keysToCamelCase = (object: InputObject): InputObject => { Object.keys(object).forEach((key) => { const parts = key.split('_') @@ -300,3 +303,17 @@ export const isUrl = (str: string): boolean => { return false } } + +export const cleanUrl = (url: string) => { + const trackingParams: (RegExp | string)[] = [/^utm_\w+/i] // remove utm tracking parameters + if (TWEET_URL_REGEX.test(url)) { + // remove tracking parameters from tweet links: + // https://twitter.com/omnivore/status/1673218959624093698?s=12&t=R91quPajs0E53Yds-fhv2g + trackingParams.push('s', 't') + } + return normalizeUrl(url, { + stripHash: true, + stripWWW: false, + removeQueryParameters: trackingParams, + }) +}