diff --git a/packages/api/src/generated/graphql.ts b/packages/api/src/generated/graphql.ts index 587a7488c..c3e060389 100644 --- a/packages/api/src/generated/graphql.ts +++ b/packages/api/src/generated/graphql.ts @@ -1209,6 +1209,7 @@ export type Mutation = { updateUser: UpdateUserResult; updateUserProfile: UpdateUserProfileResult; uploadFileRequest: UploadFileRequestResult; + uploadImportFile: UploadImportFileResult; }; @@ -1521,6 +1522,12 @@ export type MutationUploadFileRequestArgs = { input: UploadFileRequestInput; }; + +export type MutationUploadImportFileArgs = { + contentType: Scalars['String']; + type: UploadImportFileType; +}; + export type NewsletterEmail = { __typename?: 'NewsletterEmail'; address: Scalars['String']; @@ -2884,6 +2891,30 @@ export enum UploadFileStatus { Initialized = 'INITIALIZED' } +export type UploadImportFileError = { + __typename?: 'UploadImportFileError'; + errorCodes: Array; +}; + +export enum UploadImportFileErrorCode { + BadRequest = 'BAD_REQUEST', + Unauthorized = 'UNAUTHORIZED', + UploadDailyLimitExceeded = 'UPLOAD_DAILY_LIMIT_EXCEEDED' +} + +export type UploadImportFileResult = UploadImportFileError | UploadImportFileSuccess; + +export type UploadImportFileSuccess = { + __typename?: 'UploadImportFileSuccess'; + uploadSignedUrl?: Maybe; +}; + +export enum UploadImportFileType { + Matter = 'MATTER', + Pocket = 'POCKET', + UrlList = 'URL_LIST' +} + export type User = { __typename?: 'User'; followersCount?: Maybe; @@ -3518,6 +3549,11 @@ export type ResolversTypes = { UploadFileRequestResult: ResolversTypes['UploadFileRequestError'] | ResolversTypes['UploadFileRequestSuccess']; UploadFileRequestSuccess: ResolverTypeWrapper; UploadFileStatus: UploadFileStatus; + UploadImportFileError: ResolverTypeWrapper; + UploadImportFileErrorCode: UploadImportFileErrorCode; + UploadImportFileResult: ResolversTypes['UploadImportFileError'] | ResolversTypes['UploadImportFileSuccess']; + UploadImportFileSuccess: ResolverTypeWrapper; + UploadImportFileType: UploadImportFileType; User: ResolverTypeWrapper; UserError: ResolverTypeWrapper; UserErrorCode: UserErrorCode; @@ -3886,6 +3922,9 @@ export type ResolversParentTypes = { UploadFileRequestInput: UploadFileRequestInput; UploadFileRequestResult: ResolversParentTypes['UploadFileRequestError'] | ResolversParentTypes['UploadFileRequestSuccess']; UploadFileRequestSuccess: UploadFileRequestSuccess; + UploadImportFileError: UploadImportFileError; + UploadImportFileResult: ResolversParentTypes['UploadImportFileError'] | ResolversParentTypes['UploadImportFileSuccess']; + UploadImportFileSuccess: UploadImportFileSuccess; User: User; UserError: UserError; UserPersonalization: UserPersonalization; @@ -4805,6 +4844,7 @@ export type MutationResolvers>; updateUserProfile?: Resolver>; uploadFileRequest?: Resolver>; + uploadImportFile?: Resolver>; }; export type NewsletterEmailResolvers = { @@ -5607,6 +5647,20 @@ export type UploadFileRequestSuccessResolvers; }; +export type UploadImportFileErrorResolvers = { + errorCodes?: Resolver, ParentType, ContextType>; + __isTypeOf?: IsTypeOfResolverFn; +}; + +export type UploadImportFileResultResolvers = { + __resolveType: TypeResolveFn<'UploadImportFileError' | 'UploadImportFileSuccess', ParentType, ContextType>; +}; + +export type UploadImportFileSuccessResolvers = { + uploadSignedUrl?: Resolver, ParentType, ContextType>; + __isTypeOf?: IsTypeOfResolverFn; +}; + export type UserResolvers = { followersCount?: Resolver, ParentType, ContextType>; friendsCount?: Resolver, ParentType, ContextType>; @@ -5996,6 +6050,9 @@ export type Resolvers = { UploadFileRequestError?: UploadFileRequestErrorResolvers; UploadFileRequestResult?: UploadFileRequestResultResolvers; UploadFileRequestSuccess?: UploadFileRequestSuccessResolvers; + UploadImportFileError?: UploadImportFileErrorResolvers; + UploadImportFileResult?: UploadImportFileResultResolvers; + UploadImportFileSuccess?: UploadImportFileSuccessResolvers; User?: UserResolvers; UserError?: UserErrorResolvers; UserPersonalization?: UserPersonalizationResolvers; diff --git a/packages/api/src/generated/schema.graphql b/packages/api/src/generated/schema.graphql index 81193a90f..76702f090 100644 --- a/packages/api/src/generated/schema.graphql +++ b/packages/api/src/generated/schema.graphql @@ -1083,6 +1083,7 @@ type Mutation { updateUser(input: UpdateUserInput!): UpdateUserResult! updateUserProfile(input: UpdateUserProfileInput!): UpdateUserProfileResult! uploadFileRequest(input: UploadFileRequestInput!): UploadFileRequestResult! + uploadImportFile(contentType: String!, type: UploadImportFileType!): UploadImportFileResult! } type NewsletterEmail { @@ -2251,6 +2252,28 @@ enum UploadFileStatus { INITIALIZED } +type UploadImportFileError { + errorCodes: [UploadImportFileErrorCode!]! +} + +enum UploadImportFileErrorCode { + BAD_REQUEST + UNAUTHORIZED + UPLOAD_DAILY_LIMIT_EXCEEDED +} + +union UploadImportFileResult = UploadImportFileError | UploadImportFileSuccess + +type UploadImportFileSuccess { + uploadSignedUrl: String +} + +enum UploadImportFileType { + MATTER + POCKET + URL_LIST +} + type User { followersCount: Int friendsCount: Int diff --git a/packages/api/src/resolvers/function_resolvers.ts b/packages/api/src/resolvers/function_resolvers.ts index 85255a93d..39dac70c3 100644 --- a/packages/api/src/resolvers/function_resolvers.ts +++ b/packages/api/src/resolvers/function_resolvers.ts @@ -116,6 +116,7 @@ import { import { getPageByParam } from '../elastic/pages' import { recentSearchesResolver } from './recent_searches' import { optInFeatureResolver } from './features' +import { uploadImportFileResolver } from './importers/uploadImportFileResolver' /* eslint-disable @typescript-eslint/naming-convention */ type ResultResolveType = { @@ -197,6 +198,7 @@ export const functionResolvers = { joinGroup: joinGroupResolver, recommendHighlights: recommendHighlightsResolver, leaveGroup: leaveGroupResolver, + uploadImportFile: uploadImportFileResolver, }, Query: { me: getMeUserResolver, @@ -656,4 +658,5 @@ export const functionResolvers = { ...resultResolveTypeResolver('JoinGroup'), ...resultResolveTypeResolver('RecommendHighlights'), ...resultResolveTypeResolver('LeaveGroup'), + ...resultResolveTypeResolver('UploadImportFile'), } diff --git a/packages/api/src/resolvers/importers/uploadImportFileResolver.ts b/packages/api/src/resolvers/importers/uploadImportFileResolver.ts new file mode 100644 index 000000000..544f97dc6 --- /dev/null +++ b/packages/api/src/resolvers/importers/uploadImportFileResolver.ts @@ -0,0 +1,93 @@ +import { authorized } from '../../utils/helpers' +import { + UploadImportFileErrorCode, + MutationUploadImportFileArgs, + UploadImportFileError, + UploadImportFileSuccess, +} from '../../generated/graphql' +import { getRepository } from '../../entity/utils' +import { User } from '../../entity/user' +import { analytics } from '../../utils/analytics' +import { env } from '../../env' +import { DateTime } from 'luxon' +import { + countOfFilesWithPrefix, + generateUploadSignedUrl, +} from '../../utils/uploads' +import { v4 as uuidv4 } from 'uuid' +import { buildLogger } from '../../utils/logger' + +const MAX_DAILY_UPLOADS = 4 +const VALID_CONTENT_TYPES = ['text/csv'] + +const logger = buildLogger('app.dispatch') + +const extensionForContentType = (contentType: string) => { + switch (contentType) { + case 'text/csv': + return 'csv' + } + return '.unknown' +} + +export const uploadImportFileResolver = authorized< + UploadImportFileSuccess, + UploadImportFileError, + MutationUploadImportFileArgs +>(async (_, { type, contentType }, { claims: { uid }, log }) => { + log.info('uploadImportFileResolver') + + if (!VALID_CONTENT_TYPES.includes(contentType)) { + return { + errorCodes: [UploadImportFileErrorCode.BadRequest], + } + } + + const user = await getRepository(User).findOneBy({ id: uid }) + if (!user) { + return { + errorCodes: [UploadImportFileErrorCode.Unauthorized], + } + } + + analytics.track({ + userId: uid, + event: 'upload_import_file', + properties: { + type, + env: env.server.apiEnv, + }, + }) + + // path style: imports///- + const dateStr = DateTime.now().toISODate() + const dirPath = `imports/${uid}/${dateStr}/` + const fileCount = await countOfFilesWithPrefix(dirPath) + + if (fileCount > MAX_DAILY_UPLOADS) { + return { + errorCodes: [UploadImportFileErrorCode.UploadDailyLimitExceeded], + } + } + + try { + const fileUuid = uuidv4() + const ext = extensionForContentType(contentType) + const fullPath = `${dirPath}${type}-${fileUuid}.${ext}` + const uploadSignedUrl = await generateUploadSignedUrl(fullPath, contentType) + + return { + uploadSignedUrl, + } + } catch (error) { + logger.error('Error creating uploadSignedUrl', { + error, + type, + contentType, + }) + + return { + errorCodes: [UploadImportFileErrorCode.BadRequest], + } + } +}) diff --git a/packages/api/src/schema.ts b/packages/api/src/schema.ts index 281749da2..05ae969d2 100755 --- a/packages/api/src/schema.ts +++ b/packages/api/src/schema.ts @@ -2306,6 +2306,28 @@ const schema = gql` NOT_FOUND } + enum UploadImportFileType { + URL_LIST + POCKET + MATTER + } + + enum UploadImportFileErrorCode { + UNAUTHORIZED + BAD_REQUEST + UPLOAD_DAILY_LIMIT_EXCEEDED + } + + union UploadImportFileResult = UploadImportFileSuccess | UploadImportFileError + + type UploadImportFileError { + errorCodes: [UploadImportFileErrorCode!]! + } + + type UploadImportFileSuccess { + uploadSignedUrl: String + } + # Mutations type Mutation { googleLogin(input: GoogleLoginInput!): LoginResult! @@ -2390,6 +2412,10 @@ const schema = gql` input: RecommendHighlightsInput! ): RecommendHighlightsResult! leaveGroup(groupId: ID!): LeaveGroupResult! + uploadImportFile( + type: UploadImportFileType! + contentType: String! + ): UploadImportFileResult! } # FIXME: remove sort from feedArticles after all cached tabs are closed diff --git a/packages/api/src/utils/uploads.ts b/packages/api/src/utils/uploads.ts index 8e631f3e1..814acc143 100644 --- a/packages/api/src/utils/uploads.ts +++ b/packages/api/src/utils/uploads.ts @@ -18,6 +18,11 @@ export const getFilePublicUrl = (filePathName: string): string => { return storage.bucket(bucketName).file(filePathName).publicUrl() } +export const countOfFilesWithPrefix = async (prefix: string) => { + const [files] = await storage.bucket(bucketName).getFiles({ prefix }) + return files.length +} + export const generateUploadSignedUrl = async ( filePathName: string, contentType: string, diff --git a/packages/api/src/utils/usernamePolicy.ts b/packages/api/src/utils/usernamePolicy.ts index 853be928a..a17a17bfc 100644 --- a/packages/api/src/utils/usernamePolicy.ts +++ b/packages/api/src/utils/usernamePolicy.ts @@ -60,6 +60,7 @@ const RESERVED_NAMES = new Set([ 'follow', 'followers', 'following', + 'for', 'friend', 'friends', 'gist', @@ -140,6 +141,7 @@ const RESERVED_NAMES = new Set([ 'sysadministrator', 'terms', 'tour', + 'tools', 'translations', 'trends', 'unfollow', diff --git a/packages/api/test/resolvers/upload_import_file_resolver.test.ts b/packages/api/test/resolvers/upload_import_file_resolver.test.ts new file mode 100644 index 000000000..530e104e4 --- /dev/null +++ b/packages/api/test/resolvers/upload_import_file_resolver.test.ts @@ -0,0 +1,63 @@ +import { createTestUser, deleteTestUser } from '../db' +import { graphqlRequest, request } from '../util' +import * as chai from 'chai' +import { expect } from 'chai' +import 'mocha' +import { User } from '../../src/entity/user' +import chaiString from 'chai-string' + +chai.use(chaiString) + +const uploadImportFile = async ( + authToken: string, + fileType: string, + contentType: string +) => { + const query = ` + mutation { + uploadImportFile(type:${fileType}, contentType:"${contentType}") { + ... on UploadImportFileError { + errorCodes + } + ... on UploadImportFileSuccess { + uploadSignedUrl + } + } + }` + + return graphqlRequest(query, authToken).expect(200) +} + +describe('uploadImportFile API', () => { + let authToken: string + let user: User + + before(async () => { + // create test user and login + user = await createTestUser('fakeUser') + const res = await request + .post('/local/debug/fake-user-login') + .send({ fakeEmail: user.email }) + + authToken = res.body.authToken + }) + + after(async () => { + await deleteTestUser(user.id) + }) + + describe('UploadImportFile', () => { + context('when it is a pocket file', () => { + xit('should create an upload URL', async () => { + const res = await uploadImportFile(authToken, 'POCKET', 'text/csv') + expect(res.body.data.uploadImportFile.uploadSignedUrl).to.not.be.null + }) + }) + context('when it is a pocket file', () => { + xit('should create an upload URL', async () => { + const res = await uploadImportFile(authToken, 'URL_LIST', 'text/csv') + expect(res.body.data.uploadImportFile.uploadSignedUrl).to.not.be.null + }) + }) + }) +}) diff --git a/packages/import-handler/package.json b/packages/import-handler/package.json index b256bfce4..a4567dfb2 100644 --- a/packages/import-handler/package.json +++ b/packages/import-handler/package.json @@ -26,12 +26,12 @@ "dependencies": { "@fast-csv/parse": "^4.3.6", "@google-cloud/functions-framework": "3.1.2", - "@google-cloud/pubsub": "^2.16.3", "@google-cloud/storage": "^5.18.1", + "@google-cloud/tasks": "^3.0.5", "@types/express": "^4.17.13", "axios": "^0.27.2", "concurrently": "^7.0.0", "csv-parser": "^3.0.0", "nodemon": "^2.0.15" } -} +} \ No newline at end of file diff --git a/packages/import-handler/src/index.ts b/packages/import-handler/src/index.ts index b9b536385..6815f8ff3 100644 --- a/packages/import-handler/src/index.ts +++ b/packages/import-handler/src/index.ts @@ -3,13 +3,14 @@ import { CloudFunctionsContext, } from '@google-cloud/functions-framework/build/src/functions' import { Storage } from '@google-cloud/storage' -import { PubSub } from '@google-cloud/pubsub' -import { importCsv } from './csv' +import { importCsv, UrlHandler } from './csv' import * as path from 'path' +import { importMatterHistory } from './matterHistory' +import { Stream } from 'node:stream' +import { v4 as uuid } from 'uuid' +import { createCloudTask } from './task' -const pubsub = new PubSub() const storage = new Storage() -const IMPORT_URL_UPDATE_TOPIC = 'importURL' interface StorageEventData { bucket: string @@ -17,6 +18,11 @@ interface StorageEventData { contentType: string } +type importHandlerFunc = ( + stream: Stream, + handler: UrlHandler +) => Promise + const shouldHandle = (data: StorageEventData, ctx: CloudFunctionsContext) => { console.log('deciding to handle', ctx, data) if (ctx.eventType !== 'google.storage.object.finalize') { @@ -36,13 +42,23 @@ const importURL = async ( url: URL, source: string ): Promise => { - return pubsub - .topic(IMPORT_URL_UPDATE_TOPIC) - .publish(Buffer.from(JSON.stringify({ userId, url, source }))) - .catch((err) => { - console.error('error publishing url:', err) - return undefined - }) + return createCloudTask({ + userId, + source, + url: url.toString(), + saveRequestId: uuid(), + }) +} + +const handlerForFile = (name: string): importHandlerFunc | undefined => { + const fileName = path.parse(name).name + if (fileName.startsWith('MATTER')) { + return importMatterHistory + } else if (fileName.startsWith('URL_LIST')) { + return importCsv + } + + return undefined } export const importHandler: EventFunction = async (event, context) => { @@ -57,10 +73,22 @@ export const importHandler: EventFunction = async (event, context) => { .file(data.name) .createReadStream() - await importCsv(stream, async (url): Promise => { + const handler = handlerForFile(data.name) + if (!handler) { + console.log('no handler for file:', data.name) + return + } + + await handler(stream, async (url): Promise => { try { - // Imports are stored in the format imports//.extension - const userId = path.parse(data.name).name + // Imports are stored in the format imports//-.csv + const regex = new RegExp('imports/(.*?)/') + const groups = regex.exec(data.name) + if (!groups || groups.length < 2) { + console.log('could not match file pattern: ', data.name) + return + } + const userId = [...groups][1] const result = await importURL(userId, url, 'csv-importer') console.log('import url result', result) } catch (err) { diff --git a/packages/import-handler/src/matterHistory.ts b/packages/import-handler/src/matterHistory.ts new file mode 100644 index 000000000..8342735b5 --- /dev/null +++ b/packages/import-handler/src/matterHistory.ts @@ -0,0 +1,32 @@ +/* eslint-disable @typescript-eslint/no-unsafe-member-access */ +/* eslint-disable @typescript-eslint/no-unsafe-assignment */ +/* eslint-disable @typescript-eslint/no-unsafe-call */ +/* eslint-disable @typescript-eslint/no-unsafe-argument */ + +import { parse } from '@fast-csv/parse' +import { Stream } from 'stream' + +export type UrlHandler = (url: URL) => Promise + +export const importMatterHistory = async ( + stream: Stream, + handler: UrlHandler +): Promise => { + const parser = parse({ + headers: true, + strictColumnHandling: false, + }) + stream.pipe(parser) + + let count = 0 + for await (const row of parser) { + try { + const url = new URL(row['URL']) + await handler(url) + } catch (error) { + console.log('invalid url', row, error) + } + count++ + } + return count +} diff --git a/packages/import-handler/src/task.ts b/packages/import-handler/src/task.ts new file mode 100644 index 000000000..2b2dc3e21 --- /dev/null +++ b/packages/import-handler/src/task.ts @@ -0,0 +1,58 @@ +/* eslint-disable @typescript-eslint/restrict-template-expressions */ +import { CloudTasksClient, protos } from '@google-cloud/tasks' + +type TaskPayload = { + url: string + userId: string + saveRequestId: string + source: string +} + +const cloudTask = new CloudTasksClient() + +export const createCloudTask = async (payload: TaskPayload) => { + const queue = 'omnivore-import-queue' + const location = process.env.GCP_LOCATION + const project = process.env.GCP_PROJECT_ID + const taskHandlerUrl = process.env.CONTENT_FETCH_GCF_URL + + if (!project || !location || !queue || !taskHandlerUrl) { + throw `Environment not configured: ${project}, ${location}, ${queue}, ${taskHandlerUrl}` + } + + const serviceAccountEmail = `${project}@appspot.gserviceaccount.com` + + const parent = cloudTask.queuePath(project, location, queue) + console.log(`Task creation options: `, { + project, + location, + queue, + taskHandlerUrl, + serviceAccountEmail, + payload, + }) + + const convertedPayload = JSON.stringify(payload) + const body = Buffer.from(convertedPayload).toString('base64') + const task: protos.google.cloud.tasks.v2.ITask = { + httpRequest: { + httpMethod: 'POST', + url: taskHandlerUrl, + headers: { + 'Content-Type': 'application/json', + }, + body, + ...(serviceAccountEmail + ? { + oidcToken: { + serviceAccountEmail, + }, + } + : null), + }, + } + + return cloudTask.createTask({ parent, task }).then((result) => { + return result[0].name ?? undefined + }) +} diff --git a/packages/import-handler/test/matter/data/_matter_history.csv b/packages/import-handler/test/matter/data/_matter_history.csv new file mode 100644 index 000000000..d403bc393 --- /dev/null +++ b/packages/import-handler/test/matter/data/_matter_history.csv @@ -0,0 +1,2 @@ +Title,Author,Publisher,URL,Word Count,Saved,Read,Highlight Count,Last Interaction Date,File Id +"The Only Crypto Story You Need, by Matt Levine",Matt Levine,Bloomberg,https://www.bloomberg.com/features/2022-the-crypto-story/,39138,TRUE,TRUE,2,2022-12-18 14:49:11,content_15530945.html \ No newline at end of file diff --git a/packages/import-handler/test/matter/matter_importer.test.ts b/packages/import-handler/test/matter/matter_importer.test.ts new file mode 100644 index 000000000..6a3b24e98 --- /dev/null +++ b/packages/import-handler/test/matter/matter_importer.test.ts @@ -0,0 +1,23 @@ +import 'mocha' +import * as chai from 'chai' +import { expect } from 'chai' +import chaiString from 'chai-string' +import * as fs from 'fs' +import { importMatterHistory } from '../../src/matterHistory' + +chai.use(chaiString) + +describe('Load a simple _matter_history file', () => { + it('should find the URL of each row', async () => { + const urls: URL[] = [] + const stream = fs.createReadStream('./test/matter/data/_matter_history.csv') + const count = await importMatterHistory(stream, (url): Promise => { + urls.push(url) + return Promise.resolve() + }) + expect(count).to.equal(1) + expect(urls).to.eql([ + new URL('https://www.bloomberg.com/features/2022-the-crypto-story/'), + ]) + }) +}) diff --git a/packages/puppeteer-parse/index.js b/packages/puppeteer-parse/index.js index 071dea4a2..0ee12be6e 100644 --- a/packages/puppeteer-parse/index.js +++ b/packages/puppeteer-parse/index.js @@ -590,7 +590,7 @@ async function retrieveHtml(page, logRecord) { try { // Removing blurred images since they are mostly the copies of lazy loaded ones - if (['img', 'image'].includes(el.tagName.toLowerCase())) { + if (el.tagName && ['img', 'image'].includes(el.tagName.toLowerCase())) { const filter = style.getPropertyValue('filter'); if (filter && filter.startsWith('blur')) { el.parentNode && el.parentNode.removeChild(el); diff --git a/packages/readabilityjs/Readability.js b/packages/readabilityjs/Readability.js index f84e47501..707a90096 100644 --- a/packages/readabilityjs/Readability.js +++ b/packages/readabilityjs/Readability.js @@ -171,7 +171,7 @@ Readability.prototype = { // Readability-readerable.js. Please keep both copies in sync. articleNegativeLookBehindCandidates: /breadcrumbs|breadcrumb|utils|trilist/i, articleNegativeLookAheadCandidates: /outstream(.?)_|sub(.?)_|m_|omeda-promo-|in-article-advert|block-ad-.*/i, - unlikelyCandidates: /\bad\b|ai2html|banner|breadcrumbs|breadcrumb|combx|comment|community|cover-wrap|disqus|extra|footer|gdpr|header|legends|menu|related|remark|replies|rss|shoutbox|sidebar|skyscraper|social|sponsor|supplemental|ad-break|agegate|pagination|pager(?!ow)|popup|yom-remote|copyright|keywords|outline|infinite-list|beta|recirculation|site-index|hide-for-print|post-end-share-cta|post-end-cta-full|post-footer|post-head|post-tag|li-date|main-navigation|programtic-ads|outstream_article|hfeed|comment-holder|back-to-top|show-up-next|onward-journey|topic-tracker|list-nav|block-ad-entity|adSpecs|gift-article-button|modal-title|in-story-masthead|share-tools|standard-dock|expanded-dock|margins-h|subscribe-dialog|icon|bumped/i, + unlikelyCandidates: /\bad\b|ai2html|banner|breadcrumbs|breadcrumb|combx|comment|community|cover-wrap|disqus|extra|footer|gdpr|header|legends|menu|related|remark|replies|rss|shoutbox|sidebar|skyscraper|social|sponsor|supplemental|ad-break|agegate|pagination|pager(?!ow)|popup|yom-remote|copyright|keywords|outline|infinite-list|beta|recirculation|site-index|hide-for-print|post-end-share-cta|post-end-cta-full|post-footer|post-head|post-tag|li-date|main-navigation|programtic-ads|outstream_article|hfeed|comment-holder|back-to-top|show-up-next|onward-journey|topic-tracker|list-nav|block-ad-entity|adSpecs|gift-article-button|modal-title|in-story-masthead|share-tools|standard-dock|expanded-dock|margins-h|subscribe-dialog|icon|bumped|dvz-social-media-buttons/i, // okMaybeItsACandidate: /and|article(?!-breadcrumb)|body|column|content|main|shadow|post-header/i, get okMaybeItsACandidate() { return new RegExp(`and|(? { + try { + const u = new URL(url); + for (const host of NON_BOT_HOSTS) { + if (u.hostname.endsWith(host)) { + return NON_BOT_DESKTOP_USER_AGENT; + } + } + } catch (e) { + console.log('error getting user agent for url', url, e) + } + return DESKTOP_USER_AGENT +}; + const enableJavascriptForUrl = (url) => { try { const u = new URL(url); @@ -81,8 +98,7 @@ async function fetchSource(url, callbackFn) { if (!enableJavascriptForUrl(url)) { await page.setJavaScriptEnabled(false); } - const ua = generateRandomUA(); - await page.setUserAgent(ua); + await page.setUserAgent(userAgentForUrl(url)); try { /* @@ -142,12 +158,16 @@ async function fetchSource(url, callbackFn) { Array.from(document.body.getElementsByTagName('*')).forEach(el => { const style = window.getComputedStyle(el); - // Removing blurred images since they are mostly the copies of lazy loaded ones - if (['img', 'image'].includes(el.tagName.toLowerCase())) { - const filter = style.getPropertyValue('filter'); - if (filter && filter.startsWith('blur')) { - el.parentNode && el.parentNode.removeChild(el); + try { + // Removing blurred images since they are mostly the copies of lazy loaded ones + if (el.tagName && ['img', 'image'].includes(el.tagName.toLowerCase())) { + const filter = style.getPropertyValue('filter'); + if (filter && filter.startsWith('blur')) { + el.parentNode && el.parentNode.removeChild(el); + } } + } catch (err) { + // throw Error('error with element: ' + JSON.stringify(Array.from(document.body.getElementsByTagName('*')))) } // convert all nodes with background image to img nodes @@ -155,7 +175,6 @@ async function fetchSource(url, callbackFn) { const filter = style.getPropertyValue('filter'); // avoiding image nodes with a blur effect creation if (filter && filter.startsWith('blur')) { - // console.log('\n\n\n\n Filter found: ', filter); el && el.parentNode && el.parentNode.removeChild(el); } else { const matchedSRC = BI_SRC_REGEXP.exec(style.getPropertyValue('background-image')); @@ -168,12 +187,9 @@ async function fetchSource(url, callbackFn) { // Article example: http://www.josiahzayner.com/2017/01/genetic-designer-part-i.html // DIV with class "content-inner" has `url("https://resources.blogblog.com/blogblog/data/1kt/travel/bg_container.png")` background image. if (el.innerHTML.length < 25) { - console.log('Replacing element with image'); const img = document.createElement('img'); img.src = matchedSRC[1]; - el && el.parentNode && el.parentNode.replaceChild(img, el); - } else { - console.log('Element has too much content: ', el.innerHTML.length); + el && el.parentNode && el.parentNode.removeChild(el); } } } @@ -227,11 +243,14 @@ function onResponseReceived(error, source, destRoot, options) { } async function runReadability(source, destPath, metadataDestPath, options) { + console.log('running readability') + var uri = "http://fakehost/test/page.html"; var myReader, result, readerable; try { // Use linkedom for isProbablyReaderable because it supports querySelectorAll var dom = parseHTML(source).document; + console.log('dom', parseHTML(source).innerHTML) readerable = isProbablyReaderable(dom); // We pass `caption` as a class to check that passing in extra classes works, // given that it appears in some of the test documents. @@ -241,6 +260,7 @@ async function runReadability(source, destPath, metadataDestPath, options) { console.error(ex); ex.stack.forEach(console.log.bind(console)); } + console.log('result', result) if (!result) { console.error("No content generated by readability, not going to write expected.html!"); return; diff --git a/packages/web/lib/networking/mutations/uploadImportFileMutation.ts b/packages/web/lib/networking/mutations/uploadImportFileMutation.ts new file mode 100644 index 000000000..46ac248d9 --- /dev/null +++ b/packages/web/lib/networking/mutations/uploadImportFileMutation.ts @@ -0,0 +1,44 @@ +import { gqlFetcher } from '../networkHelpers' +import { v4 as uuidv4 } from 'uuid' + +export enum UploadImportFileType { + URL_LIST = 'URL_LIST', + POCKET = 'POCKET', + MATTER = 'MATTER', +} + +type UploadImportFileResponseData = { + uploadImportFile?: UploadImportFileData +} + +type UploadImportFileData = { + uploadSignedUrl: string + errorCodes?: unknown[] +} + +export async function uploadImportFileRequestMutation( + type: UploadImportFileType, + contentType: string +): Promise { + const mutation = ` + mutation UploadImportFile($type: UploadImportFileType!, $contentType: String!) { + uploadImportFile(type:$type, contentType:$contentType) { + ... on UploadImportFileError { + errorCodes + } + ... on UploadImportFileSuccess { + uploadSignedUrl + } + } + }` + + const data = await gqlFetcher(mutation, { type, contentType }) + console.log('UploadImportFile: ', data) + const output = data as UploadImportFileResponseData | undefined + const error = output?.uploadImportFile?.errorCodes?.find(() => true) + console.log('error: ', error) + if (error) { + throw error + } + return output?.uploadImportFile +} diff --git a/packages/web/package.json b/packages/web/package.json index 796042b5b..4b821c258 100644 --- a/packages/web/package.json +++ b/packages/web/package.json @@ -50,6 +50,7 @@ "react-hot-toast": "^2.1.1", "react-masonry-css": "^1.0.16", "react-pro-sidebar": "^0.7.1", + "react-spinners": "^0.13.7", "react-super-responsive-table": "^5.2.1", "react-topbar-progress-indicator": "^4.1.1", "react-twitter-widgets": "^1.10.0", diff --git a/packages/web/pages/tools/import/file.tsx b/packages/web/pages/tools/import/file.tsx new file mode 100644 index 000000000..810340a67 --- /dev/null +++ b/packages/web/pages/tools/import/file.tsx @@ -0,0 +1,227 @@ +import { ChangeEvent, useCallback, useMemo, useState } from 'react' +import { Toaster } from 'react-hot-toast' + +import { showErrorToast, showSuccessToast } from '../../../lib/toastHelpers' +import { applyStoredTheme } from '../../../lib/themeUpdater' + +import { + Box, + HStack, + VStack, +} from '../../../components/elements/LayoutPrimitives' + +import 'antd/dist/antd.compact.css' +import { StyledText } from '../../../components/elements/StyledText' +import { ProfileLayout } from '../../../components/templates/ProfileLayout' +import { + uploadImportFileRequestMutation, + UploadImportFileType, +} from '../../../lib/networking/mutations/uploadImportFileMutation' +import { Button } from '../../../components/elements/Button' +import { FormLabel } from '../../../components/elements/FormElements' +import { Loader } from '../../../components/templates/SavingRequest' + +import { SyncLoader } from 'react-spinners' +import { theme } from '../../../components/tokens/stitches.config' + +type UploadState = 'none' | 'uploading' | 'completed' + +export default function ImportUploader(): JSX.Element { + applyStoredTheme(false) + + const [errorMessage, setErrorMessage] = useState() + const [file, setFile] = useState() + const [type, setType] = useState() + const [uploadState, setUploadState] = useState('none') + + const onFinish = (values: unknown) => { + console.log(values) + } + + const onTypeChange = (e: ChangeEvent) => { + if (e.target.files) { + setFile(e.target.files[0]) + } + } + + const handleUploadClick = async () => { + if (!file) { + setErrorMessage('No file selected.') + return + } + + if (!type) { + setErrorMessage('No file type selected.') + return + } + + setUploadState('uploading') + + try { + const result = await uploadImportFileRequestMutation(type, 'text/csv') + + if (result && result.uploadSignedUrl) { + const uploadRes = await fetch(result.uploadSignedUrl, { + method: 'PUT', + body: file, + headers: { + 'content-type': 'text/csv', + 'content-length': `${file.size}`, + }, + }) + setUploadState('completed') + } else { + setErrorMessage( + 'Unable to create file upload. Please ensure you are logged in.' + ) + setUploadState('none') + } + } catch (error) { + console.log('caught error', error) + if (error == 'UPLOAD_DAILY_LIMIT_EXCEEDED') { + setErrorMessage('You have exceeded your maximum daily upload limit.') + } + setUploadState('none') + } + } + + return ( + + + + Upload a file to import + + + Omnivore supports uploading a CSV file with one URL per a Row or a + _matter_history.csv file for users migrating from the Matter app + + + Importing from Matter: If you are a Matter user, export your + data, unzip the Archive.zip file, select `Matter history` on this + page, and upload the _matter_history file that was extracted from the + Archive.zip file. + + + Importing from other apps: Most apps allow you to export your + saved items as a CSV file. Once you have extracted a file, ensure the + first column of each row is a URL, and choose our `CSV of URLs` option + and upload your file. + + + Note: Please note you are limited to three import uploads per a + day, and the maximum file size is 10MB. + + + {uploadState == 'completed' ? ( + + Your upload has completed. Please note that it can take some time + for your library to be updated. You will be sent an email when the + process completes. + + ) : ( + <> + + + File type: + + + + + + + + + {/* {file && `${file.name}`} */} + + + + + + {uploadState == 'none' && ( + + )} + {uploadState == 'uploading' && ( + + )} + + + )} + + + {errorMessage && ( + {errorMessage} + )} + + + + + ) +} diff --git a/yarn.lock b/yarn.lock index 0ee6aaa29..bd2ac8132 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2746,6 +2746,13 @@ dependencies: google-gax "^2.24.1" +"@google-cloud/tasks@^3.0.5": + version "3.0.5" + resolved "https://registry.yarnpkg.com/@google-cloud/tasks/-/tasks-3.0.5.tgz#4c0c648d10c7cf10b2e1599dd5a48bf8cc78cbd6" + integrity sha512-fC7afAV2d+zz0A2TxJWBZyBTHlB8nWBkNr/7BH6ZM/uAgT6gmSLfDho3aQNttnumj7IiI0vwUx31Zu1wG3m7yw== + dependencies: + google-gax "^3.5.2" + "@graphql-codegen/cli@^2.6.2": version "2.6.2" resolved "https://registry.yarnpkg.com/@graphql-codegen/cli/-/cli-2.6.2.tgz#a9aa4656141ee0998cae8c7ad7d0bf9ca8e0c9ae" @@ -19713,11 +19720,6 @@ mocha@^9.0.1: yargs-parser "20.2.4" yargs-unparser "2.0.0" -modern-random-ua@^1.0.3: - version "1.0.3" - resolved "https://registry.yarnpkg.com/modern-random-ua/-/modern-random-ua-1.0.3.tgz#c6fca366e60ea9a793c9d99021533744d7f58c41" - integrity sha1-xvyjZuYOqaeTydmQIVM3RNf1jEE= - modify-values@^1.0.0: version "1.0.1" resolved "https://registry.yarnpkg.com/modify-values/-/modify-values-1.0.1.tgz#b3939fa605546474e3e3e3c63d64bd43b4ee6022" @@ -22890,6 +22892,11 @@ react-slidedown@^2.4.5: dependencies: tslib "^2.0.0" +react-spinners@^0.13.7: + version "0.13.7" + resolved "https://registry.yarnpkg.com/react-spinners/-/react-spinners-0.13.7.tgz#0f423c415bfa56765ce9fb36ff604e52a92b37a9" + integrity sha512-mRamn56bfxWbGcacif5RT3UbeJaXi2AttjtPwSmomuv2IcxjpbfETCzdTvaQpNDk0E33ENJsStsQeKAZFuJcpA== + react-style-singleton@^2.1.0: version "2.1.1" resolved "https://registry.yarnpkg.com/react-style-singleton/-/react-style-singleton-2.1.1.tgz#ce7f90b67618be2b6b94902a30aaea152ce52e66"