From 9314c3d8f164f013e839111da592e32cebc4827e Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Mon, 19 Dec 2022 12:42:32 +0800 Subject: [PATCH 01/16] Add uploadImportFile API method Add uploadImportFile API method Fix prefix, counting max files uploaded Add resolver types Basic web ui for the uploader interface Allow selecting type when uploading import files --- packages/api/src/generated/graphql.ts | 56 +++++++++ packages/api/src/generated/schema.graphql | 22 ++++ .../api/src/resolvers/function_resolvers.ts | 3 + .../importers/uploadImportFileResolver.ts | 93 ++++++++++++++ packages/api/src/schema.ts | 25 ++++ packages/api/src/utils/uploads.ts | 5 + .../mutations/uploadImportFileMutation.ts | 40 ++++++ packages/web/pages/tools/import/upload.tsx | 119 ++++++++++++++++++ 8 files changed, 363 insertions(+) create mode 100644 packages/api/src/resolvers/importers/uploadImportFileResolver.ts create mode 100644 packages/web/lib/networking/mutations/uploadImportFileMutation.ts create mode 100644 packages/web/pages/tools/import/upload.tsx diff --git a/packages/api/src/generated/graphql.ts b/packages/api/src/generated/graphql.ts index 587a7488c..aa60707c9 100644 --- a/packages/api/src/generated/graphql.ts +++ b/packages/api/src/generated/graphql.ts @@ -1209,6 +1209,7 @@ export type Mutation = { updateUser: UpdateUserResult; updateUserProfile: UpdateUserProfileResult; uploadFileRequest: UploadFileRequestResult; + uploadImportFile: UploadImportFileResult; }; @@ -1521,6 +1522,12 @@ export type MutationUploadFileRequestArgs = { input: UploadFileRequestInput; }; + +export type MutationUploadImportFileArgs = { + contentType: Scalars['String']; + type: UploadImportFileType; +}; + export type NewsletterEmail = { __typename?: 'NewsletterEmail'; address: Scalars['String']; @@ -2884,6 +2891,29 @@ export enum UploadFileStatus { Initialized = 'INITIALIZED' } +export type UploadImportFileError = { + __typename?: 'UploadImportFileError'; + errorCodes: Array; +}; + +export enum UploadImportFileErrorCode { + BadRequest = 'BAD_REQUEST', + Unauthorized = 'UNAUTHORIZED', + UploadDailyLimitExceeded = 'UPLOAD_DAILY_LIMIT_EXCEEDED' +} + +export type UploadImportFileResult = UploadImportFileError | UploadImportFileSuccess; + +export type UploadImportFileSuccess = { + __typename?: 'UploadImportFileSuccess'; + uploadSignedUrl?: Maybe; +}; + +export enum UploadImportFileType { + Pocket = 'POCKET', + UrlList = 'URL_LIST' +} + export type User = { __typename?: 'User'; followersCount?: Maybe; @@ -3518,6 +3548,11 @@ export type ResolversTypes = { UploadFileRequestResult: ResolversTypes['UploadFileRequestError'] | ResolversTypes['UploadFileRequestSuccess']; UploadFileRequestSuccess: ResolverTypeWrapper; UploadFileStatus: UploadFileStatus; + UploadImportFileError: ResolverTypeWrapper; + UploadImportFileErrorCode: UploadImportFileErrorCode; + UploadImportFileResult: ResolversTypes['UploadImportFileError'] | ResolversTypes['UploadImportFileSuccess']; + UploadImportFileSuccess: ResolverTypeWrapper; + UploadImportFileType: UploadImportFileType; User: ResolverTypeWrapper; UserError: ResolverTypeWrapper; UserErrorCode: UserErrorCode; @@ -3886,6 +3921,9 @@ export type ResolversParentTypes = { UploadFileRequestInput: UploadFileRequestInput; UploadFileRequestResult: ResolversParentTypes['UploadFileRequestError'] | ResolversParentTypes['UploadFileRequestSuccess']; UploadFileRequestSuccess: UploadFileRequestSuccess; + UploadImportFileError: UploadImportFileError; + UploadImportFileResult: ResolversParentTypes['UploadImportFileError'] | ResolversParentTypes['UploadImportFileSuccess']; + UploadImportFileSuccess: UploadImportFileSuccess; User: User; UserError: UserError; UserPersonalization: UserPersonalization; @@ -4805,6 +4843,7 @@ export type MutationResolvers>; updateUserProfile?: Resolver>; uploadFileRequest?: Resolver>; + uploadImportFile?: Resolver>; }; export type NewsletterEmailResolvers = { @@ -5607,6 +5646,20 @@ export type UploadFileRequestSuccessResolvers; }; +export type UploadImportFileErrorResolvers = { + errorCodes?: Resolver, ParentType, ContextType>; + __isTypeOf?: IsTypeOfResolverFn; +}; + +export type UploadImportFileResultResolvers = { + __resolveType: TypeResolveFn<'UploadImportFileError' | 'UploadImportFileSuccess', ParentType, ContextType>; +}; + +export type UploadImportFileSuccessResolvers = { + uploadSignedUrl?: Resolver, ParentType, ContextType>; + __isTypeOf?: IsTypeOfResolverFn; +}; + export type UserResolvers = { followersCount?: Resolver, ParentType, ContextType>; friendsCount?: Resolver, ParentType, ContextType>; @@ -5996,6 +6049,9 @@ export type Resolvers = { UploadFileRequestError?: UploadFileRequestErrorResolvers; UploadFileRequestResult?: UploadFileRequestResultResolvers; UploadFileRequestSuccess?: UploadFileRequestSuccessResolvers; + UploadImportFileError?: UploadImportFileErrorResolvers; + UploadImportFileResult?: UploadImportFileResultResolvers; + UploadImportFileSuccess?: UploadImportFileSuccessResolvers; User?: UserResolvers; UserError?: UserErrorResolvers; UserPersonalization?: UserPersonalizationResolvers; diff --git a/packages/api/src/generated/schema.graphql b/packages/api/src/generated/schema.graphql index 81193a90f..6f81fc062 100644 --- a/packages/api/src/generated/schema.graphql +++ b/packages/api/src/generated/schema.graphql @@ -1083,6 +1083,7 @@ type Mutation { updateUser(input: UpdateUserInput!): UpdateUserResult! updateUserProfile(input: UpdateUserProfileInput!): UpdateUserProfileResult! uploadFileRequest(input: UploadFileRequestInput!): UploadFileRequestResult! + uploadImportFile(contentType: String!, type: UploadImportFileType!): UploadImportFileResult! } type NewsletterEmail { @@ -2251,6 +2252,27 @@ enum UploadFileStatus { INITIALIZED } +type UploadImportFileError { + errorCodes: [UploadImportFileErrorCode!]! +} + +enum UploadImportFileErrorCode { + BAD_REQUEST + UNAUTHORIZED + UPLOAD_DAILY_LIMIT_EXCEEDED +} + +union UploadImportFileResult = UploadImportFileError | UploadImportFileSuccess + +type UploadImportFileSuccess { + uploadSignedUrl: String +} + +enum UploadImportFileType { + POCKET + URL_LIST +} + type User { followersCount: Int friendsCount: Int diff --git a/packages/api/src/resolvers/function_resolvers.ts b/packages/api/src/resolvers/function_resolvers.ts index 85255a93d..39dac70c3 100644 --- a/packages/api/src/resolvers/function_resolvers.ts +++ b/packages/api/src/resolvers/function_resolvers.ts @@ -116,6 +116,7 @@ import { import { getPageByParam } from '../elastic/pages' import { recentSearchesResolver } from './recent_searches' import { optInFeatureResolver } from './features' +import { uploadImportFileResolver } from './importers/uploadImportFileResolver' /* eslint-disable @typescript-eslint/naming-convention */ type ResultResolveType = { @@ -197,6 +198,7 @@ export const functionResolvers = { joinGroup: joinGroupResolver, recommendHighlights: recommendHighlightsResolver, leaveGroup: leaveGroupResolver, + uploadImportFile: uploadImportFileResolver, }, Query: { me: getMeUserResolver, @@ -656,4 +658,5 @@ export const functionResolvers = { ...resultResolveTypeResolver('JoinGroup'), ...resultResolveTypeResolver('RecommendHighlights'), ...resultResolveTypeResolver('LeaveGroup'), + ...resultResolveTypeResolver('UploadImportFile'), } diff --git a/packages/api/src/resolvers/importers/uploadImportFileResolver.ts b/packages/api/src/resolvers/importers/uploadImportFileResolver.ts new file mode 100644 index 000000000..544f97dc6 --- /dev/null +++ b/packages/api/src/resolvers/importers/uploadImportFileResolver.ts @@ -0,0 +1,93 @@ +import { authorized } from '../../utils/helpers' +import { + UploadImportFileErrorCode, + MutationUploadImportFileArgs, + UploadImportFileError, + UploadImportFileSuccess, +} from '../../generated/graphql' +import { getRepository } from '../../entity/utils' +import { User } from '../../entity/user' +import { analytics } from '../../utils/analytics' +import { env } from '../../env' +import { DateTime } from 'luxon' +import { + countOfFilesWithPrefix, + generateUploadSignedUrl, +} from '../../utils/uploads' +import { v4 as uuidv4 } from 'uuid' +import { buildLogger } from '../../utils/logger' + +const MAX_DAILY_UPLOADS = 4 +const VALID_CONTENT_TYPES = ['text/csv'] + +const logger = buildLogger('app.dispatch') + +const extensionForContentType = (contentType: string) => { + switch (contentType) { + case 'text/csv': + return 'csv' + } + return '.unknown' +} + +export const uploadImportFileResolver = authorized< + UploadImportFileSuccess, + UploadImportFileError, + MutationUploadImportFileArgs +>(async (_, { type, contentType }, { claims: { uid }, log }) => { + log.info('uploadImportFileResolver') + + if (!VALID_CONTENT_TYPES.includes(contentType)) { + return { + errorCodes: [UploadImportFileErrorCode.BadRequest], + } + } + + const user = await getRepository(User).findOneBy({ id: uid }) + if (!user) { + return { + errorCodes: [UploadImportFileErrorCode.Unauthorized], + } + } + + analytics.track({ + userId: uid, + event: 'upload_import_file', + properties: { + type, + env: env.server.apiEnv, + }, + }) + + // path style: imports///- + const dateStr = DateTime.now().toISODate() + const dirPath = `imports/${uid}/${dateStr}/` + const fileCount = await countOfFilesWithPrefix(dirPath) + + if (fileCount > MAX_DAILY_UPLOADS) { + return { + errorCodes: [UploadImportFileErrorCode.UploadDailyLimitExceeded], + } + } + + try { + const fileUuid = uuidv4() + const ext = extensionForContentType(contentType) + const fullPath = `${dirPath}${type}-${fileUuid}.${ext}` + const uploadSignedUrl = await generateUploadSignedUrl(fullPath, contentType) + + return { + uploadSignedUrl, + } + } catch (error) { + logger.error('Error creating uploadSignedUrl', { + error, + type, + contentType, + }) + + return { + errorCodes: [UploadImportFileErrorCode.BadRequest], + } + } +}) diff --git a/packages/api/src/schema.ts b/packages/api/src/schema.ts index 281749da2..ba9423d61 100755 --- a/packages/api/src/schema.ts +++ b/packages/api/src/schema.ts @@ -2306,6 +2306,27 @@ const schema = gql` NOT_FOUND } + enum UploadImportFileType { + URL_LIST + POCKET + } + + enum UploadImportFileErrorCode { + UNAUTHORIZED + BAD_REQUEST + UPLOAD_DAILY_LIMIT_EXCEEDED + } + + union UploadImportFileResult = UploadImportFileSuccess | UploadImportFileError + + type UploadImportFileError { + errorCodes: [UploadImportFileErrorCode!]! + } + + type UploadImportFileSuccess { + uploadSignedUrl: String + } + # Mutations type Mutation { googleLogin(input: GoogleLoginInput!): LoginResult! @@ -2390,6 +2411,10 @@ const schema = gql` input: RecommendHighlightsInput! ): RecommendHighlightsResult! leaveGroup(groupId: ID!): LeaveGroupResult! + uploadImportFile( + type: UploadImportFileType! + contentType: String! + ): UploadImportFileResult! } # FIXME: remove sort from feedArticles after all cached tabs are closed diff --git a/packages/api/src/utils/uploads.ts b/packages/api/src/utils/uploads.ts index 8e631f3e1..814acc143 100644 --- a/packages/api/src/utils/uploads.ts +++ b/packages/api/src/utils/uploads.ts @@ -18,6 +18,11 @@ export const getFilePublicUrl = (filePathName: string): string => { return storage.bucket(bucketName).file(filePathName).publicUrl() } +export const countOfFilesWithPrefix = async (prefix: string) => { + const [files] = await storage.bucket(bucketName).getFiles({ prefix }) + return files.length +} + export const generateUploadSignedUrl = async ( filePathName: string, contentType: string, diff --git a/packages/web/lib/networking/mutations/uploadImportFileMutation.ts b/packages/web/lib/networking/mutations/uploadImportFileMutation.ts new file mode 100644 index 000000000..e272906b0 --- /dev/null +++ b/packages/web/lib/networking/mutations/uploadImportFileMutation.ts @@ -0,0 +1,40 @@ +import { gqlFetcher } from '../networkHelpers' +import { v4 as uuidv4 } from 'uuid' + +export type UploadImportFileType = 'URL_LIST' | 'POCKET' + +type UploadImportFileResponseData = { + uploadImportFile?: UploadImportFileData +} + +type UploadImportFileData = { + uploadSignedUrl: string + errorCodes?: unknown[] +} + +export async function uploadImportFileRequestMutation( + type: UploadImportFileType, + contentType: string +): Promise { + const mutation = ` + mutation UploadImportFile($type: UploadImportFileType!, $contentType: String!) { + uploadImportFile(type:$type, contentType:$contentType) { + ... on UploadImportFileError { + errorCodes + } + ... on UploadImportFileSuccess { + uploadSignedUrl + } + } + }` + + const data = await gqlFetcher(mutation, { type, contentType }) + console.log('UploadImportFile: ', data) + const output = data as UploadImportFileResponseData | undefined + const error = output?.uploadImportFile?.errorCodes?.find(() => true) + console.log('error: ', error) + if (error) { + throw error + } + return output?.uploadImportFile +} diff --git a/packages/web/pages/tools/import/upload.tsx b/packages/web/pages/tools/import/upload.tsx new file mode 100644 index 000000000..23a9bc991 --- /dev/null +++ b/packages/web/pages/tools/import/upload.tsx @@ -0,0 +1,119 @@ +import { useCallback, useMemo, useState } from 'react' +import { Toaster } from 'react-hot-toast' + +import { showErrorToast, showSuccessToast } from '../../../lib/toastHelpers' +import { applyStoredTheme } from '../../../lib/themeUpdater' + +import { + Box, + HStack, + SpanBox, + VStack, +} from '../../../components/elements/LayoutPrimitives' +import { PrimaryLayout } from '../../../components/templates/PrimaryLayout' + +import 'antd/dist/antd.compact.css' +import { StyledText } from '../../../components/elements/StyledText' +import { Button } from '../../../components/elements/Button' +import { ProfileLayout } from '../../../components/templates/ProfileLayout' +import { FormLabel } from '../../../components/elements/FormElements' +import { uploadImportFileRequestMutation } from '../../../lib/networking/mutations/uploadImportFileMutation' + +export default function ImportUploader(): JSX.Element { + applyStoredTheme(false) + + const [errorMessage, setErrorMessage] = useState() + const [file, setFile] = useState() + const [type, setType] = useState() + + const handleFileChange = (e: ChangeEvent) => { + if (e.target.files) { + setFile(e.target.files[0]) + } + } + + const handleUploadClick = async () => { + if (!file) { + return + } + + console.log('file type: ', file.type) + + try { + const result = await uploadImportFileRequestMutation( + 'URL_LIST', + 'text/csv' + ) + + if (result && result.uploadSignedUrl) { + const uploadRes = await fetch(result.uploadSignedUrl, { + method: 'PUT', + body: file, + headers: { + 'content-type': 'text/csv', + 'content-length': `${file.size}`, + }, + }) + console.log('upload result: ', uploadRes) + } + } catch (error) { + console.log('caught error', error) + if (error == 'UPLOAD_DAILY_LIMIT_EXCEEDED') { + setErrorMessage('You have exceeded your maximum daily upload limit.') + } + } + } + + return ( + + + + + Type + + + + + File + + +
{file && `${file.name} - ${file.type}`}
+ + + + {errorMessage && {errorMessage}} + + + + ) +} From 57563ee958b87f1d2e01ac4ba73e19d70a3cffb2 Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Wed, 28 Dec 2022 17:31:37 +0800 Subject: [PATCH 02/16] Import ChangeEvent --- packages/web/pages/tools/import/upload.tsx | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/packages/web/pages/tools/import/upload.tsx b/packages/web/pages/tools/import/upload.tsx index 23a9bc991..508182edf 100644 --- a/packages/web/pages/tools/import/upload.tsx +++ b/packages/web/pages/tools/import/upload.tsx @@ -1,16 +1,14 @@ -import { useCallback, useMemo, useState } from 'react' +import { ChangeEvent, useCallback, useMemo, useState } from 'react' import { Toaster } from 'react-hot-toast' import { showErrorToast, showSuccessToast } from '../../../lib/toastHelpers' import { applyStoredTheme } from '../../../lib/themeUpdater' import { - Box, HStack, SpanBox, VStack, } from '../../../components/elements/LayoutPrimitives' -import { PrimaryLayout } from '../../../components/templates/PrimaryLayout' import 'antd/dist/antd.compact.css' import { StyledText } from '../../../components/elements/StyledText' From f83ddbb308bac7ff5108270e17025a6b01a5cfe0 Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Wed, 28 Dec 2022 17:41:00 +0800 Subject: [PATCH 03/16] Update file importer path --- packages/web/pages/tools/import/{upload.tsx => file.tsx} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename packages/web/pages/tools/import/{upload.tsx => file.tsx} (100%) diff --git a/packages/web/pages/tools/import/upload.tsx b/packages/web/pages/tools/import/file.tsx similarity index 100% rename from packages/web/pages/tools/import/upload.tsx rename to packages/web/pages/tools/import/file.tsx From 1b357f8c21399eef283243693688288f35baf2e9 Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Tue, 3 Jan 2023 11:07:21 +0800 Subject: [PATCH 04/16] Add a test (have a feeling we will need to disable on CI due to GCS dependency) --- .../upload_import_file_resolver.test.ts | 63 +++++++++++++++++++ 1 file changed, 63 insertions(+) create mode 100644 packages/api/test/resolvers/upload_import_file_resolver.test.ts diff --git a/packages/api/test/resolvers/upload_import_file_resolver.test.ts b/packages/api/test/resolvers/upload_import_file_resolver.test.ts new file mode 100644 index 000000000..72c10a774 --- /dev/null +++ b/packages/api/test/resolvers/upload_import_file_resolver.test.ts @@ -0,0 +1,63 @@ +import { createTestUser, deleteTestUser } from '../db' +import { graphqlRequest, request } from '../util' +import * as chai from 'chai' +import { expect } from 'chai' +import 'mocha' +import { User } from '../../src/entity/user' +import chaiString from 'chai-string' + +chai.use(chaiString) + +const uploadImportFile = async ( + authToken: string, + fileType: string, + contentType: string +) => { + const query = ` + mutation { + uploadImportFile(type:${fileType}, contentType:"${contentType}") { + ... on UploadImportFileError { + errorCodes + } + ... on UploadImportFileSuccess { + uploadSignedUrl + } + } + }` + + return graphqlRequest(query, authToken).expect(200) +} + +describe('uploadImportFile API', () => { + let authToken: string + let user: User + + before(async () => { + // create test user and login + user = await createTestUser('fakeUser') + const res = await request + .post('/local/debug/fake-user-login') + .send({ fakeEmail: user.email }) + + authToken = res.body.authToken + }) + + after(async () => { + await deleteTestUser(user.id) + }) + + describe('UploadImportFile', () => { + context('when it is a pocket file', () => { + it('should create an upload URL', async () => { + const res = await uploadImportFile(authToken, 'POCKET', 'text/csv') + expect(res.body.data.uploadImportFile.uploadSignedUrl).to.not.be.null + }) + }) + context('when it is a pocket file', () => { + it('should create an upload URL', async () => { + const res = await uploadImportFile(authToken, 'URL_LIST', 'text/csv') + expect(res.body.data.uploadImportFile.uploadSignedUrl).to.not.be.null + }) + }) + }) +}) From 51544dfa50a180710c6bffb431c290ed7d9ab531 Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Tue, 3 Jan 2023 11:48:07 +0800 Subject: [PATCH 05/16] Use same user agents in generate-testcase as in puppeteer --- packages/readabilityjs/package.json | 1 - .../readabilityjs/test/generate-testcase.js | 46 +++++++++++++------ 2 files changed, 33 insertions(+), 14 deletions(-) diff --git a/packages/readabilityjs/package.json b/packages/readabilityjs/package.json index 7e248d421..c42d58c8a 100644 --- a/packages/readabilityjs/package.json +++ b/packages/readabilityjs/package.json @@ -36,7 +36,6 @@ }, "dependencies": { "html-entities": "^2.3.2", - "modern-random-ua": "^1.0.3", "parse-srcset": "^1.0.2" } } diff --git a/packages/readabilityjs/test/generate-testcase.js b/packages/readabilityjs/test/generate-testcase.js index b48d55198..3ec128f1a 100644 --- a/packages/readabilityjs/test/generate-testcase.js +++ b/packages/readabilityjs/test/generate-testcase.js @@ -6,7 +6,6 @@ var prettyPrint = require("./utils").prettyPrint; var htmltidy = require("htmltidy2").tidy; var { Readability, isProbablyReaderable } = require("../index"); -const { generate: generateRandomUA } = require("modern-random-ua/random_ua"); const puppeteer = require('puppeteer'); const { parseHTML } = require("linkedom"); @@ -14,7 +13,29 @@ var testcaseRoot = path.join(__dirname, "test-pages"); var argURL = process.argv[3]; // Could be undefined, we'll warn if it is if that is an issue. +const MOBILE_USER_AGENT = 'Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/93.0.4577.62 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)' +const DESKTOP_USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 11_6_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4372.0 Safari/537.36' +const BOT_DESKTOP_USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 11_6_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4372.0 Safari/537.36' +const NON_BOT_DESKTOP_USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 11_6_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4372.0 Safari/537.36' +const NON_BOT_HOSTS = ['bloomberg.com', 'forbes.com'] const NON_SCRIPT_HOSTS= ['medium.com', 'fastcompany.com']; + +const ALLOWED_CONTENT_TYPES = ['text/html', 'application/octet-stream', 'text/plain', 'application/pdf']; + +const userAgentForUrl = (url) => { + try { + const u = new URL(url); + for (const host of NON_BOT_HOSTS) { + if (u.hostname.endsWith(host)) { + return NON_BOT_DESKTOP_USER_AGENT; + } + } + } catch (e) { + console.log('error getting user agent for url', url, e) + } + return DESKTOP_USER_AGENT +}; + const enableJavascriptForUrl = (url) => { try { const u = new URL(url); @@ -81,8 +102,7 @@ async function fetchSource(url, callbackFn) { if (!enableJavascriptForUrl(url)) { await page.setJavaScriptEnabled(false); } - const ua = generateRandomUA(); - await page.setUserAgent(ua); + await page.setUserAgent(userAgentForUrl(url)); try { /* @@ -142,12 +162,16 @@ async function fetchSource(url, callbackFn) { Array.from(document.body.getElementsByTagName('*')).forEach(el => { const style = window.getComputedStyle(el); - // Removing blurred images since they are mostly the copies of lazy loaded ones - if (['img', 'image'].includes(el.tagName.toLowerCase())) { - const filter = style.getPropertyValue('filter'); - if (filter && filter.startsWith('blur')) { - el.parentNode && el.parentNode.removeChild(el); + try { + // Removing blurred images since they are mostly the copies of lazy loaded ones + if (el.tagName && ['img', 'image'].includes(el.tagName.toLowerCase())) { + const filter = style.getPropertyValue('filter'); + if (filter && filter.startsWith('blur')) { + el.parentNode && el.parentNode.removeChild(el); + } } + } catch (err) { + // throw Error('error with element: ' + JSON.stringify(Array.from(document.body.getElementsByTagName('*')))) } // convert all nodes with background image to img nodes @@ -155,7 +179,6 @@ async function fetchSource(url, callbackFn) { const filter = style.getPropertyValue('filter'); // avoiding image nodes with a blur effect creation if (filter && filter.startsWith('blur')) { - // console.log('\n\n\n\n Filter found: ', filter); el && el.parentNode && el.parentNode.removeChild(el); } else { const matchedSRC = BI_SRC_REGEXP.exec(style.getPropertyValue('background-image')); @@ -168,12 +191,9 @@ async function fetchSource(url, callbackFn) { // Article example: http://www.josiahzayner.com/2017/01/genetic-designer-part-i.html // DIV with class "content-inner" has `url("https://resources.blogblog.com/blogblog/data/1kt/travel/bg_container.png")` background image. if (el.innerHTML.length < 25) { - console.log('Replacing element with image'); const img = document.createElement('img'); img.src = matchedSRC[1]; - el && el.parentNode && el.parentNode.replaceChild(img, el); - } else { - console.log('Element has too much content: ', el.innerHTML.length); + el && el.parentNode && el.parentNode.removeChild(el); } } } From a0f51c94ee56dc5751f1f182c96708a0c1f6e2f6 Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Tue, 3 Jan 2023 12:33:12 +0800 Subject: [PATCH 06/16] Add another social media footer class to unlikely candidates --- packages/readabilityjs/Readability.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/readabilityjs/Readability.js b/packages/readabilityjs/Readability.js index f84e47501..707a90096 100644 --- a/packages/readabilityjs/Readability.js +++ b/packages/readabilityjs/Readability.js @@ -171,7 +171,7 @@ Readability.prototype = { // Readability-readerable.js. Please keep both copies in sync. articleNegativeLookBehindCandidates: /breadcrumbs|breadcrumb|utils|trilist/i, articleNegativeLookAheadCandidates: /outstream(.?)_|sub(.?)_|m_|omeda-promo-|in-article-advert|block-ad-.*/i, - unlikelyCandidates: /\bad\b|ai2html|banner|breadcrumbs|breadcrumb|combx|comment|community|cover-wrap|disqus|extra|footer|gdpr|header|legends|menu|related|remark|replies|rss|shoutbox|sidebar|skyscraper|social|sponsor|supplemental|ad-break|agegate|pagination|pager(?!ow)|popup|yom-remote|copyright|keywords|outline|infinite-list|beta|recirculation|site-index|hide-for-print|post-end-share-cta|post-end-cta-full|post-footer|post-head|post-tag|li-date|main-navigation|programtic-ads|outstream_article|hfeed|comment-holder|back-to-top|show-up-next|onward-journey|topic-tracker|list-nav|block-ad-entity|adSpecs|gift-article-button|modal-title|in-story-masthead|share-tools|standard-dock|expanded-dock|margins-h|subscribe-dialog|icon|bumped/i, + unlikelyCandidates: /\bad\b|ai2html|banner|breadcrumbs|breadcrumb|combx|comment|community|cover-wrap|disqus|extra|footer|gdpr|header|legends|menu|related|remark|replies|rss|shoutbox|sidebar|skyscraper|social|sponsor|supplemental|ad-break|agegate|pagination|pager(?!ow)|popup|yom-remote|copyright|keywords|outline|infinite-list|beta|recirculation|site-index|hide-for-print|post-end-share-cta|post-end-cta-full|post-footer|post-head|post-tag|li-date|main-navigation|programtic-ads|outstream_article|hfeed|comment-holder|back-to-top|show-up-next|onward-journey|topic-tracker|list-nav|block-ad-entity|adSpecs|gift-article-button|modal-title|in-story-masthead|share-tools|standard-dock|expanded-dock|margins-h|subscribe-dialog|icon|bumped|dvz-social-media-buttons/i, // okMaybeItsACandidate: /and|article(?!-breadcrumb)|body|column|content|main|shadow|post-header/i, get okMaybeItsACandidate() { return new RegExp(`and|(? Date: Tue, 3 Jan 2023 12:33:50 +0800 Subject: [PATCH 07/16] Update test case user agent --- packages/readabilityjs/test/generate-testcase.js | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/packages/readabilityjs/test/generate-testcase.js b/packages/readabilityjs/test/generate-testcase.js index 3ec128f1a..3784fd7cd 100644 --- a/packages/readabilityjs/test/generate-testcase.js +++ b/packages/readabilityjs/test/generate-testcase.js @@ -13,15 +13,11 @@ var testcaseRoot = path.join(__dirname, "test-pages"); var argURL = process.argv[3]; // Could be undefined, we'll warn if it is if that is an issue. -const MOBILE_USER_AGENT = 'Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/93.0.4577.62 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)' const DESKTOP_USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 11_6_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4372.0 Safari/537.36' -const BOT_DESKTOP_USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 11_6_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4372.0 Safari/537.36' -const NON_BOT_DESKTOP_USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 11_6_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4372.0 Safari/537.36' +const NON_BOT_DESKTOP_USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0 Safari/537.36' const NON_BOT_HOSTS = ['bloomberg.com', 'forbes.com'] const NON_SCRIPT_HOSTS= ['medium.com', 'fastcompany.com']; -const ALLOWED_CONTENT_TYPES = ['text/html', 'application/octet-stream', 'text/plain', 'application/pdf']; - const userAgentForUrl = (url) => { try { const u = new URL(url); From dc444a471de36664b5a4ced5c22d0304e729c550 Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Tue, 3 Jan 2023 12:42:06 +0800 Subject: [PATCH 08/16] Dont allow puppeteer to fail if empty tag is found --- packages/puppeteer-parse/index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/puppeteer-parse/index.js b/packages/puppeteer-parse/index.js index f2ec56f42..f83cebf1c 100644 --- a/packages/puppeteer-parse/index.js +++ b/packages/puppeteer-parse/index.js @@ -590,7 +590,7 @@ async function retrieveHtml(page, logRecord) { try { // Removing blurred images since they are mostly the copies of lazy loaded ones - if (['img', 'image'].includes(el.tagName.toLowerCase())) { + if (el.tagName && ['img', 'image'].includes(el.tagName.toLowerCase())) { const filter = style.getPropertyValue('filter'); if (filter && filter.startsWith('blur')) { el.parentNode && el.parentNode.removeChild(el); From 0edd91057ec85f41352996111bf14c3d63efc243 Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Tue, 3 Jan 2023 17:49:07 +0800 Subject: [PATCH 09/16] Update user interface for file import tool --- packages/api/src/generated/graphql.ts | 1 + packages/api/src/generated/schema.graphql | 1 + packages/api/src/schema.ts | 1 + .../readabilityjs/test/generate-testcase.js | 4 + .../mutations/uploadImportFileMutation.ts | 6 +- packages/web/package.json | 1 + packages/web/pages/tools/import/file.tsx | 202 ++++++++++++++---- yarn.lock | 10 +- 8 files changed, 174 insertions(+), 52 deletions(-) diff --git a/packages/api/src/generated/graphql.ts b/packages/api/src/generated/graphql.ts index aa60707c9..c3e060389 100644 --- a/packages/api/src/generated/graphql.ts +++ b/packages/api/src/generated/graphql.ts @@ -2910,6 +2910,7 @@ export type UploadImportFileSuccess = { }; export enum UploadImportFileType { + Matter = 'MATTER', Pocket = 'POCKET', UrlList = 'URL_LIST' } diff --git a/packages/api/src/generated/schema.graphql b/packages/api/src/generated/schema.graphql index 6f81fc062..76702f090 100644 --- a/packages/api/src/generated/schema.graphql +++ b/packages/api/src/generated/schema.graphql @@ -2269,6 +2269,7 @@ type UploadImportFileSuccess { } enum UploadImportFileType { + MATTER POCKET URL_LIST } diff --git a/packages/api/src/schema.ts b/packages/api/src/schema.ts index ba9423d61..05ae969d2 100755 --- a/packages/api/src/schema.ts +++ b/packages/api/src/schema.ts @@ -2309,6 +2309,7 @@ const schema = gql` enum UploadImportFileType { URL_LIST POCKET + MATTER } enum UploadImportFileErrorCode { diff --git a/packages/readabilityjs/test/generate-testcase.js b/packages/readabilityjs/test/generate-testcase.js index 3784fd7cd..90a5525fa 100644 --- a/packages/readabilityjs/test/generate-testcase.js +++ b/packages/readabilityjs/test/generate-testcase.js @@ -243,11 +243,14 @@ function onResponseReceived(error, source, destRoot, options) { } async function runReadability(source, destPath, metadataDestPath, options) { + console.log('running readability') + var uri = "http://fakehost/test/page.html"; var myReader, result, readerable; try { // Use linkedom for isProbablyReaderable because it supports querySelectorAll var dom = parseHTML(source).document; + console.log('dom', parseHTML(source).innerHTML) readerable = isProbablyReaderable(dom); // We pass `caption` as a class to check that passing in extra classes works, // given that it appears in some of the test documents. @@ -257,6 +260,7 @@ async function runReadability(source, destPath, metadataDestPath, options) { console.error(ex); ex.stack.forEach(console.log.bind(console)); } + console.log('result', result) if (!result) { console.error("No content generated by readability, not going to write expected.html!"); return; diff --git a/packages/web/lib/networking/mutations/uploadImportFileMutation.ts b/packages/web/lib/networking/mutations/uploadImportFileMutation.ts index e272906b0..46ac248d9 100644 --- a/packages/web/lib/networking/mutations/uploadImportFileMutation.ts +++ b/packages/web/lib/networking/mutations/uploadImportFileMutation.ts @@ -1,7 +1,11 @@ import { gqlFetcher } from '../networkHelpers' import { v4 as uuidv4 } from 'uuid' -export type UploadImportFileType = 'URL_LIST' | 'POCKET' +export enum UploadImportFileType { + URL_LIST = 'URL_LIST', + POCKET = 'POCKET', + MATTER = 'MATTER', +} type UploadImportFileResponseData = { uploadImportFile?: UploadImportFileData diff --git a/packages/web/package.json b/packages/web/package.json index 796042b5b..4b821c258 100644 --- a/packages/web/package.json +++ b/packages/web/package.json @@ -50,6 +50,7 @@ "react-hot-toast": "^2.1.1", "react-masonry-css": "^1.0.16", "react-pro-sidebar": "^0.7.1", + "react-spinners": "^0.13.7", "react-super-responsive-table": "^5.2.1", "react-topbar-progress-indicator": "^4.1.1", "react-twitter-widgets": "^1.10.0", diff --git a/packages/web/pages/tools/import/file.tsx b/packages/web/pages/tools/import/file.tsx index 508182edf..f45b5d260 100644 --- a/packages/web/pages/tools/import/file.tsx +++ b/packages/web/pages/tools/import/file.tsx @@ -5,26 +5,40 @@ import { showErrorToast, showSuccessToast } from '../../../lib/toastHelpers' import { applyStoredTheme } from '../../../lib/themeUpdater' import { + Box, HStack, - SpanBox, VStack, } from '../../../components/elements/LayoutPrimitives' import 'antd/dist/antd.compact.css' import { StyledText } from '../../../components/elements/StyledText' -import { Button } from '../../../components/elements/Button' import { ProfileLayout } from '../../../components/templates/ProfileLayout' +import { + uploadImportFileRequestMutation, + UploadImportFileType, +} from '../../../lib/networking/mutations/uploadImportFileMutation' +import { Button } from '../../../components/elements/Button' import { FormLabel } from '../../../components/elements/FormElements' -import { uploadImportFileRequestMutation } from '../../../lib/networking/mutations/uploadImportFileMutation' +import { Loader } from '../../../components/templates/SavingRequest' + +import { SyncLoader } from 'react-spinners' +import { theme } from '../../../components/tokens/stitches.config' + +type UploadState = 'none' | 'uploading' | 'completed' export default function ImportUploader(): JSX.Element { applyStoredTheme(false) const [errorMessage, setErrorMessage] = useState() const [file, setFile] = useState() - const [type, setType] = useState() + const [type, setType] = useState() + const [uploadState, setUploadState] = useState('none') - const handleFileChange = (e: ChangeEvent) => { + const onFinish = (values: unknown) => { + console.log(values) + } + + const onTypeChange = (e: ChangeEvent) => { if (e.target.files) { setFile(e.target.files[0]) } @@ -32,16 +46,19 @@ export default function ImportUploader(): JSX.Element { const handleUploadClick = async () => { if (!file) { + setErrorMessage('No file selected.') return } - console.log('file type: ', file.type) + if (!type) { + setErrorMessage('No file type selected.') + return + } + + setUploadState('uploading') try { - const result = await uploadImportFileRequestMutation( - 'URL_LIST', - 'text/csv' - ) + const result = await uploadImportFileRequestMutation(type, 'text/csv') if (result && result.uploadSignedUrl) { const uploadRes = await fetch(result.uploadSignedUrl, { @@ -52,20 +69,26 @@ export default function ImportUploader(): JSX.Element { 'content-length': `${file.size}`, }, }) - console.log('upload result: ', uploadRes) + setUploadState('completed') + } else { + setErrorMessage( + 'Unable to create file upload. Please ensure you are logged in.' + ) + setUploadState('none') } } catch (error) { console.log('caught error', error) if (error == 'UPLOAD_DAILY_LIMIT_EXCEEDED') { setErrorMessage('You have exceeded your maximum daily upload limit.') } + setUploadState('none') } } return ( - - - Type - - - - - File - - -
{file && `${file.name} - ${file.type}`}
- - - - {errorMessage && {errorMessage}} - + Upload a file to import + + + Omnivore supports uploading a CSV file with one URL per a Row or a + _matter_history.csv file for users migrating from the Matter app + + + Importing from Matter: If you are a Matter user, export your + data, unzip the Archive.zip file, select `Matter history` on this + page, and upload the _matter_history file that was extracted from the + Archive.zip file. + + + Importing from other apps: Most apps allow you to export your + saved items as a CSV file. Once you have extracted a file, ensure the + first column of each row is a URL, and choose our `CSV of URLs` option + and upload your file. + + + Note: Please note you are limited to three import uploads per a + day, and the maximum file size is 10MB. + + + {uploadState == 'completed' ? ( + + Your upload has completed. Please note that it can take some time + for your library to be updated. You will be sent an email when the + process completes. + + ) : ( + <> + + + File type: + + + + + + + + + {/* {file && `${file.name}`} */} + + + + + + {uploadState == 'none' && ( + + )} + {uploadState == 'uploading' && ( + + )} + + + )} + + + {errorMessage && ( + {errorMessage} + )} + + ) diff --git a/yarn.lock b/yarn.lock index c4990abaa..592b91ca7 100644 --- a/yarn.lock +++ b/yarn.lock @@ -19744,11 +19744,6 @@ mocha@^9.0.1: yargs-parser "20.2.4" yargs-unparser "2.0.0" -modern-random-ua@^1.0.3: - version "1.0.3" - resolved "https://registry.yarnpkg.com/modern-random-ua/-/modern-random-ua-1.0.3.tgz#c6fca366e60ea9a793c9d99021533744d7f58c41" - integrity sha1-xvyjZuYOqaeTydmQIVM3RNf1jEE= - modify-values@^1.0.0: version "1.0.1" resolved "https://registry.yarnpkg.com/modify-values/-/modify-values-1.0.1.tgz#b3939fa605546474e3e3e3c63d64bd43b4ee6022" @@ -22941,6 +22936,11 @@ react-slidedown@^2.4.5: dependencies: tslib "^2.0.0" +react-spinners@^0.13.7: + version "0.13.7" + resolved "https://registry.yarnpkg.com/react-spinners/-/react-spinners-0.13.7.tgz#0f423c415bfa56765ce9fb36ff604e52a92b37a9" + integrity sha512-mRamn56bfxWbGcacif5RT3UbeJaXi2AttjtPwSmomuv2IcxjpbfETCzdTvaQpNDk0E33ENJsStsQeKAZFuJcpA== + react-style-singleton@^2.1.0: version "2.1.1" resolved "https://registry.yarnpkg.com/react-style-singleton/-/react-style-singleton-2.1.1.tgz#ce7f90b67618be2b6b94902a30aaea152ce52e66" From 9213d222c4f2d04a9d9f46574c7f91d0d013f276 Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Tue, 3 Jan 2023 18:13:59 +0800 Subject: [PATCH 10/16] Add the _matter_history importer --- packages/import-handler/src/matterHistory.ts | 32 +++++++++++++++++++ .../test/matter/data/_matter_history.csv | 2 ++ .../test/matter/matter_importer.test.ts | 23 +++++++++++++ 3 files changed, 57 insertions(+) create mode 100644 packages/import-handler/src/matterHistory.ts create mode 100644 packages/import-handler/test/matter/data/_matter_history.csv create mode 100644 packages/import-handler/test/matter/matter_importer.test.ts diff --git a/packages/import-handler/src/matterHistory.ts b/packages/import-handler/src/matterHistory.ts new file mode 100644 index 000000000..8342735b5 --- /dev/null +++ b/packages/import-handler/src/matterHistory.ts @@ -0,0 +1,32 @@ +/* eslint-disable @typescript-eslint/no-unsafe-member-access */ +/* eslint-disable @typescript-eslint/no-unsafe-assignment */ +/* eslint-disable @typescript-eslint/no-unsafe-call */ +/* eslint-disable @typescript-eslint/no-unsafe-argument */ + +import { parse } from '@fast-csv/parse' +import { Stream } from 'stream' + +export type UrlHandler = (url: URL) => Promise + +export const importMatterHistory = async ( + stream: Stream, + handler: UrlHandler +): Promise => { + const parser = parse({ + headers: true, + strictColumnHandling: false, + }) + stream.pipe(parser) + + let count = 0 + for await (const row of parser) { + try { + const url = new URL(row['URL']) + await handler(url) + } catch (error) { + console.log('invalid url', row, error) + } + count++ + } + return count +} diff --git a/packages/import-handler/test/matter/data/_matter_history.csv b/packages/import-handler/test/matter/data/_matter_history.csv new file mode 100644 index 000000000..d403bc393 --- /dev/null +++ b/packages/import-handler/test/matter/data/_matter_history.csv @@ -0,0 +1,2 @@ +Title,Author,Publisher,URL,Word Count,Saved,Read,Highlight Count,Last Interaction Date,File Id +"The Only Crypto Story You Need, by Matt Levine",Matt Levine,Bloomberg,https://www.bloomberg.com/features/2022-the-crypto-story/,39138,TRUE,TRUE,2,2022-12-18 14:49:11,content_15530945.html \ No newline at end of file diff --git a/packages/import-handler/test/matter/matter_importer.test.ts b/packages/import-handler/test/matter/matter_importer.test.ts new file mode 100644 index 000000000..6a3b24e98 --- /dev/null +++ b/packages/import-handler/test/matter/matter_importer.test.ts @@ -0,0 +1,23 @@ +import 'mocha' +import * as chai from 'chai' +import { expect } from 'chai' +import chaiString from 'chai-string' +import * as fs from 'fs' +import { importMatterHistory } from '../../src/matterHistory' + +chai.use(chaiString) + +describe('Load a simple _matter_history file', () => { + it('should find the URL of each row', async () => { + const urls: URL[] = [] + const stream = fs.createReadStream('./test/matter/data/_matter_history.csv') + const count = await importMatterHistory(stream, (url): Promise => { + urls.push(url) + return Promise.resolve() + }) + expect(count).to.equal(1) + expect(urls).to.eql([ + new URL('https://www.bloomberg.com/features/2022-the-crypto-story/'), + ]) + }) +}) From f939786c2a063044a32831bfe2453dbbe10e8eb1 Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Tue, 3 Jan 2023 20:55:30 +0800 Subject: [PATCH 11/16] Reserve some names for tools and landing pages --- packages/api/src/utils/usernamePolicy.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/api/src/utils/usernamePolicy.ts b/packages/api/src/utils/usernamePolicy.ts index 853be928a..a17a17bfc 100644 --- a/packages/api/src/utils/usernamePolicy.ts +++ b/packages/api/src/utils/usernamePolicy.ts @@ -60,6 +60,7 @@ const RESERVED_NAMES = new Set([ 'follow', 'followers', 'following', + 'for', 'friend', 'friends', 'gist', @@ -140,6 +141,7 @@ const RESERVED_NAMES = new Set([ 'sysadministrator', 'terms', 'tour', + 'tools', 'translations', 'trends', 'unfollow', From 97981f9a7b03dac7730b68b13daa1844f0f8f751 Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Tue, 3 Jan 2023 20:55:53 +0800 Subject: [PATCH 12/16] Updates to handle new import file format --- packages/import-handler/src/index.ts | 37 +++++++++++++++++++++++++--- 1 file changed, 33 insertions(+), 4 deletions(-) diff --git a/packages/import-handler/src/index.ts b/packages/import-handler/src/index.ts index b9b536385..65ff113ed 100644 --- a/packages/import-handler/src/index.ts +++ b/packages/import-handler/src/index.ts @@ -4,8 +4,10 @@ import { } from '@google-cloud/functions-framework/build/src/functions' import { Storage } from '@google-cloud/storage' import { PubSub } from '@google-cloud/pubsub' -import { importCsv } from './csv' +import { importCsv, UrlHandler } from './csv' import * as path from 'path' +import { importMatterHistory } from './matterHistory' +import { Stream } from 'node:stream' const pubsub = new PubSub() const storage = new Storage() @@ -17,6 +19,11 @@ interface StorageEventData { contentType: string } +type importHandlerFunc = ( + stream: Stream, + handler: UrlHandler +) => Promise + const shouldHandle = (data: StorageEventData, ctx: CloudFunctionsContext) => { console.log('deciding to handle', ctx, data) if (ctx.eventType !== 'google.storage.object.finalize') { @@ -45,6 +52,17 @@ const importURL = async ( }) } +const handlerForFile = (name: string): importHandlerFunc | undefined => { + const fileName = path.parse(name).name + if (fileName.startsWith('MATTER')) { + return importMatterHistory + } else if (fileName.startsWith('URL_LIST')) { + return importCsv + } + + return undefined +} + export const importHandler: EventFunction = async (event, context) => { const data = event as StorageEventData const ctx = context as CloudFunctionsContext @@ -57,10 +75,21 @@ export const importHandler: EventFunction = async (event, context) => { .file(data.name) .createReadStream() - await importCsv(stream, async (url): Promise => { + const handler = handlerForFile(data.name) + if (!handler) { + console.log('no handler for file:', data.name) + return + } + + await handler(stream, async (url): Promise => { try { - // Imports are stored in the format imports//.extension - const userId = path.parse(data.name).name + // Imports are stored in the format imports//-.csv + const group = path.parse(data.name).name.match(/(?<=-).*/gi) + if (!group || group.length < 1) { + console.log('could not match file pattern: ', data.name) + return + } + const userId = [...group][0] const result = await importURL(userId, url, 'csv-importer') console.log('import url result', result) } catch (err) { From 87c73adebbae198d80ec6d06af900ed3a00b0b8a Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Wed, 4 Jan 2023 10:43:55 +0800 Subject: [PATCH 13/16] Use const instead of var --- packages/web/pages/tools/import/file.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/web/pages/tools/import/file.tsx b/packages/web/pages/tools/import/file.tsx index f45b5d260..810340a67 100644 --- a/packages/web/pages/tools/import/file.tsx +++ b/packages/web/pages/tools/import/file.tsx @@ -163,7 +163,7 @@ export default function ImportUploader(): JSX.Element {