Merge pull request #1559 from omnivore-app/feat/importer

New upload file importer

packages/api/src/generated/graphql.ts

@@ -1209,6 +1209,7 @@ export type Mutation = {
   updateUser: UpdateUserResult;
   updateUserProfile: UpdateUserProfileResult;
   uploadFileRequest: UploadFileRequestResult;
+  uploadImportFile: UploadImportFileResult;
 };
 
 
@@ -1521,6 +1522,12 @@ export type MutationUploadFileRequestArgs = {
   input: UploadFileRequestInput;
 };
 
+
+export type MutationUploadImportFileArgs = {
+  contentType: Scalars['String'];
+  type: UploadImportFileType;
+};
+
 export type NewsletterEmail = {
   __typename?: 'NewsletterEmail';
   address: Scalars['String'];
@@ -2884,6 +2891,30 @@ export enum UploadFileStatus {
   Initialized = 'INITIALIZED'
 }
 
+export type UploadImportFileError = {
+  __typename?: 'UploadImportFileError';
+  errorCodes: Array<UploadImportFileErrorCode>;
+};
+
+export enum UploadImportFileErrorCode {
+  BadRequest = 'BAD_REQUEST',
+  Unauthorized = 'UNAUTHORIZED',
+  UploadDailyLimitExceeded = 'UPLOAD_DAILY_LIMIT_EXCEEDED'
+}
+
+export type UploadImportFileResult = UploadImportFileError | UploadImportFileSuccess;
+
+export type UploadImportFileSuccess = {
+  __typename?: 'UploadImportFileSuccess';
+  uploadSignedUrl?: Maybe<Scalars['String']>;
+};
+
+export enum UploadImportFileType {
+  Matter = 'MATTER',
+  Pocket = 'POCKET',
+  UrlList = 'URL_LIST'
+}
+
 export type User = {
   __typename?: 'User';
   followersCount?: Maybe<Scalars['Int']>;
@@ -3518,6 +3549,11 @@ export type ResolversTypes = {
   UploadFileRequestResult: ResolversTypes['UploadFileRequestError'] | ResolversTypes['UploadFileRequestSuccess'];
   UploadFileRequestSuccess: ResolverTypeWrapper<UploadFileRequestSuccess>;
   UploadFileStatus: UploadFileStatus;
+  UploadImportFileError: ResolverTypeWrapper<UploadImportFileError>;
+  UploadImportFileErrorCode: UploadImportFileErrorCode;
+  UploadImportFileResult: ResolversTypes['UploadImportFileError'] | ResolversTypes['UploadImportFileSuccess'];
+  UploadImportFileSuccess: ResolverTypeWrapper<UploadImportFileSuccess>;
+  UploadImportFileType: UploadImportFileType;
   User: ResolverTypeWrapper<User>;
   UserError: ResolverTypeWrapper<UserError>;
   UserErrorCode: UserErrorCode;
@@ -3886,6 +3922,9 @@ export type ResolversParentTypes = {
   UploadFileRequestInput: UploadFileRequestInput;
   UploadFileRequestResult: ResolversParentTypes['UploadFileRequestError'] | ResolversParentTypes['UploadFileRequestSuccess'];
   UploadFileRequestSuccess: UploadFileRequestSuccess;
+  UploadImportFileError: UploadImportFileError;
+  UploadImportFileResult: ResolversParentTypes['UploadImportFileError'] | ResolversParentTypes['UploadImportFileSuccess'];
+  UploadImportFileSuccess: UploadImportFileSuccess;
   User: User;
   UserError: UserError;
   UserPersonalization: UserPersonalization;
@@ -4805,6 +4844,7 @@ export type MutationResolvers<ContextType = ResolverContext, ParentType extends
   updateUser?: Resolver<ResolversTypes['UpdateUserResult'], ParentType, ContextType, RequireFields<MutationUpdateUserArgs, 'input'>>;
   updateUserProfile?: Resolver<ResolversTypes['UpdateUserProfileResult'], ParentType, ContextType, RequireFields<MutationUpdateUserProfileArgs, 'input'>>;
   uploadFileRequest?: Resolver<ResolversTypes['UploadFileRequestResult'], ParentType, ContextType, RequireFields<MutationUploadFileRequestArgs, 'input'>>;
+  uploadImportFile?: Resolver<ResolversTypes['UploadImportFileResult'], ParentType, ContextType, RequireFields<MutationUploadImportFileArgs, 'contentType' | 'type'>>;
 };
 
 export type NewsletterEmailResolvers<ContextType = ResolverContext, ParentType extends ResolversParentTypes['NewsletterEmail'] = ResolversParentTypes['NewsletterEmail']> = {
@@ -5607,6 +5647,20 @@ export type UploadFileRequestSuccessResolvers<ContextType = ResolverContext, Par
   __isTypeOf?: IsTypeOfResolverFn<ParentType, ContextType>;
 };
 
+export type UploadImportFileErrorResolvers<ContextType = ResolverContext, ParentType extends ResolversParentTypes['UploadImportFileError'] = ResolversParentTypes['UploadImportFileError']> = {
+  errorCodes?: Resolver<Array<ResolversTypes['UploadImportFileErrorCode']>, ParentType, ContextType>;
+  __isTypeOf?: IsTypeOfResolverFn<ParentType, ContextType>;
+};
+
+export type UploadImportFileResultResolvers<ContextType = ResolverContext, ParentType extends ResolversParentTypes['UploadImportFileResult'] = ResolversParentTypes['UploadImportFileResult']> = {
+  __resolveType: TypeResolveFn<'UploadImportFileError' | 'UploadImportFileSuccess', ParentType, ContextType>;
+};
+
+export type UploadImportFileSuccessResolvers<ContextType = ResolverContext, ParentType extends ResolversParentTypes['UploadImportFileSuccess'] = ResolversParentTypes['UploadImportFileSuccess']> = {
+  uploadSignedUrl?: Resolver<Maybe<ResolversTypes['String']>, ParentType, ContextType>;
+  __isTypeOf?: IsTypeOfResolverFn<ParentType, ContextType>;
+};
+
 export type UserResolvers<ContextType = ResolverContext, ParentType extends ResolversParentTypes['User'] = ResolversParentTypes['User']> = {
   followersCount?: Resolver<Maybe<ResolversTypes['Int']>, ParentType, ContextType>;
   friendsCount?: Resolver<Maybe<ResolversTypes['Int']>, ParentType, ContextType>;
@@ -5996,6 +6050,9 @@ export type Resolvers<ContextType = ResolverContext> = {
   UploadFileRequestError?: UploadFileRequestErrorResolvers<ContextType>;
   UploadFileRequestResult?: UploadFileRequestResultResolvers<ContextType>;
   UploadFileRequestSuccess?: UploadFileRequestSuccessResolvers<ContextType>;
+  UploadImportFileError?: UploadImportFileErrorResolvers<ContextType>;
+  UploadImportFileResult?: UploadImportFileResultResolvers<ContextType>;
+  UploadImportFileSuccess?: UploadImportFileSuccessResolvers<ContextType>;
   User?: UserResolvers<ContextType>;
   UserError?: UserErrorResolvers<ContextType>;
   UserPersonalization?: UserPersonalizationResolvers<ContextType>;

packages/api/src/generated/schema.graphql

@@ -1083,6 +1083,7 @@ type Mutation {
   updateUser(input: UpdateUserInput!): UpdateUserResult!
   updateUserProfile(input: UpdateUserProfileInput!): UpdateUserProfileResult!
   uploadFileRequest(input: UploadFileRequestInput!): UploadFileRequestResult!
+  uploadImportFile(contentType: String!, type: UploadImportFileType!): UploadImportFileResult!
 }
 
 type NewsletterEmail {
@@ -2251,6 +2252,28 @@ enum UploadFileStatus {
   INITIALIZED
 }
 
+type UploadImportFileError {
+  errorCodes: [UploadImportFileErrorCode!]!
+}
+
+enum UploadImportFileErrorCode {
+  BAD_REQUEST
+  UNAUTHORIZED
+  UPLOAD_DAILY_LIMIT_EXCEEDED
+}
+
+union UploadImportFileResult = UploadImportFileError | UploadImportFileSuccess
+
+type UploadImportFileSuccess {
+  uploadSignedUrl: String
+}
+
+enum UploadImportFileType {
+  MATTER
+  POCKET
+  URL_LIST
+}
+
 type User {
   followersCount: Int
   friendsCount: Int

packages/api/src/resolvers/function_resolvers.ts

@@ -116,6 +116,7 @@ import {
 import { getPageByParam } from '../elastic/pages'
 import { recentSearchesResolver } from './recent_searches'
 import { optInFeatureResolver } from './features'
+import { uploadImportFileResolver } from './importers/uploadImportFileResolver'
 
 /* eslint-disable @typescript-eslint/naming-convention */
 type ResultResolveType = {
@@ -197,6 +198,7 @@ export const functionResolvers = {
     joinGroup: joinGroupResolver,
     recommendHighlights: recommendHighlightsResolver,
     leaveGroup: leaveGroupResolver,
+    uploadImportFile: uploadImportFileResolver,
   },
   Query: {
     me: getMeUserResolver,
@@ -656,4 +658,5 @@ export const functionResolvers = {
   ...resultResolveTypeResolver('JoinGroup'),
   ...resultResolveTypeResolver('RecommendHighlights'),
   ...resultResolveTypeResolver('LeaveGroup'),
+  ...resultResolveTypeResolver('UploadImportFile'),
 }

packages/api/src/resolvers/importers/uploadImportFileResolver.ts

@@ -0,0 +1,93 @@
+import { authorized } from '../../utils/helpers'
+import {
+  UploadImportFileErrorCode,
+  MutationUploadImportFileArgs,
+  UploadImportFileError,
+  UploadImportFileSuccess,
+} from '../../generated/graphql'
+import { getRepository } from '../../entity/utils'
+import { User } from '../../entity/user'
+import { analytics } from '../../utils/analytics'
+import { env } from '../../env'
+import { DateTime } from 'luxon'
+import {
+  countOfFilesWithPrefix,
+  generateUploadSignedUrl,
+} from '../../utils/uploads'
+import { v4 as uuidv4 } from 'uuid'
+import { buildLogger } from '../../utils/logger'
+
+const MAX_DAILY_UPLOADS = 4
+const VALID_CONTENT_TYPES = ['text/csv']
+
+const logger = buildLogger('app.dispatch')
+
+const extensionForContentType = (contentType: string) => {
+  switch (contentType) {
+    case 'text/csv':
+      return 'csv'
+  }
+  return '.unknown'
+}
+
+export const uploadImportFileResolver = authorized<
+  UploadImportFileSuccess,
+  UploadImportFileError,
+  MutationUploadImportFileArgs
+>(async (_, { type, contentType }, { claims: { uid }, log }) => {
+  log.info('uploadImportFileResolver')
+
+  if (!VALID_CONTENT_TYPES.includes(contentType)) {
+    return {
+      errorCodes: [UploadImportFileErrorCode.BadRequest],
+    }
+  }
+
+  const user = await getRepository(User).findOneBy({ id: uid })
+  if (!user) {
+    return {
+      errorCodes: [UploadImportFileErrorCode.Unauthorized],
+    }
+  }
+
+  analytics.track({
+    userId: uid,
+    event: 'upload_import_file',
+    properties: {
+      type,
+      env: env.server.apiEnv,
+    },
+  })
+
+  // path style: imports/<uid>/<date>/<type>-<uuid>
+  const dateStr = DateTime.now().toISODate()
+  const dirPath = `imports/${uid}/${dateStr}/`
+  const fileCount = await countOfFilesWithPrefix(dirPath)
+
+  if (fileCount > MAX_DAILY_UPLOADS) {
+    return {
+      errorCodes: [UploadImportFileErrorCode.UploadDailyLimitExceeded],
+    }
+  }
+
+  try {
+    const fileUuid = uuidv4()
+    const ext = extensionForContentType(contentType)
+    const fullPath = `${dirPath}${type}-${fileUuid}.${ext}`
+    const uploadSignedUrl = await generateUploadSignedUrl(fullPath, contentType)
+
+    return {
+      uploadSignedUrl,
+    }
+  } catch (error) {
+    logger.error('Error creating uploadSignedUrl', {
+      error,
+      type,
+      contentType,
+    })
+
+    return {
+      errorCodes: [UploadImportFileErrorCode.BadRequest],
+    }
+  }
+})

packages/api/src/schema.ts

@@ -2306,6 +2306,28 @@ const schema = gql`
     NOT_FOUND
   }
 
+  enum UploadImportFileType {
+    URL_LIST
+    POCKET
+    MATTER
+  }
+
+  enum UploadImportFileErrorCode {
+    UNAUTHORIZED
+    BAD_REQUEST
+    UPLOAD_DAILY_LIMIT_EXCEEDED
+  }
+
+  union UploadImportFileResult = UploadImportFileSuccess | UploadImportFileError
+
+  type UploadImportFileError {
+    errorCodes: [UploadImportFileErrorCode!]!
+  }
+
+  type UploadImportFileSuccess {
+    uploadSignedUrl: String
+  }
+
   # Mutations
   type Mutation {
     googleLogin(input: GoogleLoginInput!): LoginResult!
@@ -2390,6 +2412,10 @@ const schema = gql`
       input: RecommendHighlightsInput!
     ): RecommendHighlightsResult!
     leaveGroup(groupId: ID!): LeaveGroupResult!
+    uploadImportFile(
+      type: UploadImportFileType!
+      contentType: String!
+    ): UploadImportFileResult!
   }
 
   # FIXME: remove sort from feedArticles after all cached tabs are closed

packages/api/src/utils/uploads.ts

@@ -18,6 +18,11 @@ export const getFilePublicUrl = (filePathName: string): string => {
   return storage.bucket(bucketName).file(filePathName).publicUrl()
 }
 
+export const countOfFilesWithPrefix = async (prefix: string) => {
+  const [files] = await storage.bucket(bucketName).getFiles({ prefix })
+  return files.length
+}
+
 export const generateUploadSignedUrl = async (
   filePathName: string,
   contentType: string,

packages/api/src/utils/usernamePolicy.ts

@@ -60,6 +60,7 @@ const RESERVED_NAMES = new Set([
   'follow',
   'followers',
   'following',
+  'for',
   'friend',
   'friends',
   'gist',
@@ -140,6 +141,7 @@ const RESERVED_NAMES = new Set([
   'sysadministrator',
   'terms',
   'tour',
+  'tools',
   'translations',
   'trends',
   'unfollow',

packages/api/test/resolvers/upload_import_file_resolver.test.ts

@@ -0,0 +1,63 @@
+import { createTestUser, deleteTestUser } from '../db'
+import { graphqlRequest, request } from '../util'
+import * as chai from 'chai'
+import { expect } from 'chai'
+import 'mocha'
+import { User } from '../../src/entity/user'
+import chaiString from 'chai-string'
+
+chai.use(chaiString)
+
+const uploadImportFile = async (
+  authToken: string,
+  fileType: string,
+  contentType: string
+) => {
+  const query = `
+  mutation {
+    uploadImportFile(type:${fileType}, contentType:"${contentType}") {
+      ... on UploadImportFileError {
+        errorCodes
+      }
+      ... on UploadImportFileSuccess {
+        uploadSignedUrl
+      }
+    }
+  }`
+
+  return graphqlRequest(query, authToken).expect(200)
+}
+
+describe('uploadImportFile API', () => {
+  let authToken: string
+  let user: User
+
+  before(async () => {
+    // create test user and login
+    user = await createTestUser('fakeUser')
+    const res = await request
+      .post('/local/debug/fake-user-login')
+      .send({ fakeEmail: user.email })
+
+    authToken = res.body.authToken
+  })
+
+  after(async () => {
+    await deleteTestUser(user.id)
+  })
+
+  describe('UploadImportFile', () => {
+    context('when it is a pocket file', () => {
+      xit('should create an upload URL', async () => {
+        const res = await uploadImportFile(authToken, 'POCKET', 'text/csv')
+        expect(res.body.data.uploadImportFile.uploadSignedUrl).to.not.be.null
+      })
+    })
+    context('when it is a pocket file', () => {
+      xit('should create an upload URL', async () => {
+        const res = await uploadImportFile(authToken, 'URL_LIST', 'text/csv')
+        expect(res.body.data.uploadImportFile.uploadSignedUrl).to.not.be.null
+      })
+    })
+  })
+})

packages/import-handler/package.json

@@ -26,12 +26,12 @@
   "dependencies": {
     "@fast-csv/parse": "^4.3.6",
     "@google-cloud/functions-framework": "3.1.2",
-    "@google-cloud/pubsub": "^2.16.3",
     "@google-cloud/storage": "^5.18.1",
+    "@google-cloud/tasks": "^3.0.5",
     "@types/express": "^4.17.13",
     "axios": "^0.27.2",
     "concurrently": "^7.0.0",
     "csv-parser": "^3.0.0",
     "nodemon": "^2.0.15"
   }
-}
+}

packages/import-handler/src/index.ts

@@ -3,13 +3,14 @@ import {
   CloudFunctionsContext,
 } from '@google-cloud/functions-framework/build/src/functions'
 import { Storage } from '@google-cloud/storage'
-import { PubSub } from '@google-cloud/pubsub'
-import { importCsv } from './csv'
+import { importCsv, UrlHandler } from './csv'
 import * as path from 'path'
+import { importMatterHistory } from './matterHistory'
+import { Stream } from 'node:stream'
+import { v4 as uuid } from 'uuid'
+import { createCloudTask } from './task'
 
-const pubsub = new PubSub()
 const storage = new Storage()
-const IMPORT_URL_UPDATE_TOPIC = 'importURL'
 
 interface StorageEventData {
   bucket: string
@@ -17,6 +18,11 @@ interface StorageEventData {
   contentType: string
 }
 
+type importHandlerFunc = (
+  stream: Stream,
+  handler: UrlHandler
+) => Promise<number>
+
 const shouldHandle = (data: StorageEventData, ctx: CloudFunctionsContext) => {
   console.log('deciding to handle', ctx, data)
   if (ctx.eventType !== 'google.storage.object.finalize') {
@@ -36,13 +42,23 @@ const importURL = async (
   url: URL,
   source: string
 ): Promise<string | undefined> => {
-  return pubsub
-    .topic(IMPORT_URL_UPDATE_TOPIC)
-    .publish(Buffer.from(JSON.stringify({ userId, url, source })))
-    .catch((err) => {
-      console.error('error publishing url:', err)
-      return undefined
-    })
+  return createCloudTask({
+    userId,
+    source,
+    url: url.toString(),
+    saveRequestId: uuid(),
+  })
+}
+
+const handlerForFile = (name: string): importHandlerFunc | undefined => {
+  const fileName = path.parse(name).name
+  if (fileName.startsWith('MATTER')) {
+    return importMatterHistory
+  } else if (fileName.startsWith('URL_LIST')) {
+    return importCsv
+  }
+
+  return undefined
 }
 
 export const importHandler: EventFunction = async (event, context) => {
@@ -57,10 +73,22 @@ export const importHandler: EventFunction = async (event, context) => {
       .file(data.name)
       .createReadStream()
 
-    await importCsv(stream, async (url): Promise<void> => {
+    const handler = handlerForFile(data.name)
+    if (!handler) {
+      console.log('no handler for file:', data.name)
+      return
+    }
+
+    await handler(stream, async (url): Promise<void> => {
       try {
-        // Imports are stored in the format imports/<import id>/<user id>.extension
-        const userId = path.parse(data.name).name
+        // Imports are stored in the format imports/<user id>/<type>-<uuid>.csv
+        const regex = new RegExp('imports/(.*?)/')
+        const groups = regex.exec(data.name)
+        if (!groups || groups.length < 2) {
+          console.log('could not match file pattern: ', data.name)
+          return
+        }
+        const userId = [...groups][1]
         const result = await importURL(userId, url, 'csv-importer')
         console.log('import url result', result)
       } catch (err) {

packages/import-handler/src/matterHistory.ts

@@ -0,0 +1,32 @@
+/* eslint-disable @typescript-eslint/no-unsafe-member-access */
+/* eslint-disable @typescript-eslint/no-unsafe-assignment */
+/* eslint-disable @typescript-eslint/no-unsafe-call */
+/* eslint-disable @typescript-eslint/no-unsafe-argument */
+
+import { parse } from '@fast-csv/parse'
+import { Stream } from 'stream'
+
+export type UrlHandler = (url: URL) => Promise<void>
+
+export const importMatterHistory = async (
+  stream: Stream,
+  handler: UrlHandler
+): Promise<number> => {
+  const parser = parse({
+    headers: true,
+    strictColumnHandling: false,
+  })
+  stream.pipe(parser)
+
+  let count = 0
+  for await (const row of parser) {
+    try {
+      const url = new URL(row['URL'])
+      await handler(url)
+    } catch (error) {
+      console.log('invalid url', row, error)
+    }
+    count++
+  }
+  return count
+}

packages/import-handler/src/task.ts

@@ -0,0 +1,58 @@
+/* eslint-disable @typescript-eslint/restrict-template-expressions */
+import { CloudTasksClient, protos } from '@google-cloud/tasks'
+
+type TaskPayload = {
+  url: string
+  userId: string
+  saveRequestId: string
+  source: string
+}
+
+const cloudTask = new CloudTasksClient()
+
+export const createCloudTask = async (payload: TaskPayload) => {
+  const queue = 'omnivore-import-queue'
+  const location = process.env.GCP_LOCATION
+  const project = process.env.GCP_PROJECT_ID
+  const taskHandlerUrl = process.env.CONTENT_FETCH_GCF_URL
+
+  if (!project || !location || !queue || !taskHandlerUrl) {
+    throw `Environment not configured: ${project}, ${location}, ${queue}, ${taskHandlerUrl}`
+  }
+
+  const serviceAccountEmail = `${project}@appspot.gserviceaccount.com`
+
+  const parent = cloudTask.queuePath(project, location, queue)
+  console.log(`Task creation options: `, {
+    project,
+    location,
+    queue,
+    taskHandlerUrl,
+    serviceAccountEmail,
+    payload,
+  })
+
+  const convertedPayload = JSON.stringify(payload)
+  const body = Buffer.from(convertedPayload).toString('base64')
+  const task: protos.google.cloud.tasks.v2.ITask = {
+    httpRequest: {
+      httpMethod: 'POST',
+      url: taskHandlerUrl,
+      headers: {
+        'Content-Type': 'application/json',
+      },
+      body,
+      ...(serviceAccountEmail
+        ? {
+            oidcToken: {
+              serviceAccountEmail,
+            },
+          }
+        : null),
+    },
+  }
+
+  return cloudTask.createTask({ parent, task }).then((result) => {
+    return result[0].name ?? undefined
+  })
+}

packages/import-handler/test/matter/data/_matter_history.csv

@@ -0,0 +1,2 @@
+Title,Author,Publisher,URL,Word Count,Saved,Read,Highlight Count,Last Interaction Date,File Id
+"The Only Crypto Story You Need, by Matt Levine",Matt Levine,Bloomberg,https://www.bloomberg.com/features/2022-the-crypto-story/,39138,TRUE,TRUE,2,2022-12-18 14:49:11,content_15530945.html

packages/import-handler/test/matter/matter_importer.test.ts

@@ -0,0 +1,23 @@
+import 'mocha'
+import * as chai from 'chai'
+import { expect } from 'chai'
+import chaiString from 'chai-string'
+import * as fs from 'fs'
+import { importMatterHistory } from '../../src/matterHistory'
+
+chai.use(chaiString)
+
+describe('Load a simple _matter_history file', () => {
+  it('should find the URL of each row', async () => {
+    const urls: URL[] = []
+    const stream = fs.createReadStream('./test/matter/data/_matter_history.csv')
+    const count = await importMatterHistory(stream, (url): Promise<void> => {
+      urls.push(url)
+      return Promise.resolve()
+    })
+    expect(count).to.equal(1)
+    expect(urls).to.eql([
+      new URL('https://www.bloomberg.com/features/2022-the-crypto-story/'),
+    ])
+  })
+})

packages/puppeteer-parse/index.js

@@ -590,7 +590,7 @@ async function retrieveHtml(page, logRecord) {
 
         try {
           // Removing blurred images since they are mostly the copies of lazy loaded ones
-          if (['img', 'image'].includes(el.tagName.toLowerCase())) {
+          if (el.tagName && ['img', 'image'].includes(el.tagName.toLowerCase())) {
             const filter = style.getPropertyValue('filter');
             if (filter && filter.startsWith('blur')) {
               el.parentNode && el.parentNode.removeChild(el);

packages/readabilityjs/Readability.js

@@ -171,7 +171,7 @@ Readability.prototype = {
     // Readability-readerable.js. Please keep both copies in sync.
     articleNegativeLookBehindCandidates: /breadcrumbs|breadcrumb|utils|trilist/i,
     articleNegativeLookAheadCandidates: /outstream(.?)_|sub(.?)_|m_|omeda-promo-|in-article-advert|block-ad-.*/i,
-    unlikelyCandidates: /\bad\b|ai2html|banner|breadcrumbs|breadcrumb|combx|comment|community|cover-wrap|disqus|extra|footer|gdpr|header|legends|menu|related|remark|replies|rss|shoutbox|sidebar|skyscraper|social|sponsor|supplemental|ad-break|agegate|pagination|pager(?!ow)|popup|yom-remote|copyright|keywords|outline|infinite-list|beta|recirculation|site-index|hide-for-print|post-end-share-cta|post-end-cta-full|post-footer|post-head|post-tag|li-date|main-navigation|programtic-ads|outstream_article|hfeed|comment-holder|back-to-top|show-up-next|onward-journey|topic-tracker|list-nav|block-ad-entity|adSpecs|gift-article-button|modal-title|in-story-masthead|share-tools|standard-dock|expanded-dock|margins-h|subscribe-dialog|icon|bumped/i,
+    unlikelyCandidates: /\bad\b|ai2html|banner|breadcrumbs|breadcrumb|combx|comment|community|cover-wrap|disqus|extra|footer|gdpr|header|legends|menu|related|remark|replies|rss|shoutbox|sidebar|skyscraper|social|sponsor|supplemental|ad-break|agegate|pagination|pager(?!ow)|popup|yom-remote|copyright|keywords|outline|infinite-list|beta|recirculation|site-index|hide-for-print|post-end-share-cta|post-end-cta-full|post-footer|post-head|post-tag|li-date|main-navigation|programtic-ads|outstream_article|hfeed|comment-holder|back-to-top|show-up-next|onward-journey|topic-tracker|list-nav|block-ad-entity|adSpecs|gift-article-button|modal-title|in-story-masthead|share-tools|standard-dock|expanded-dock|margins-h|subscribe-dialog|icon|bumped|dvz-social-media-buttons/i,
     // okMaybeItsACandidate: /and|article(?!-breadcrumb)|body|column|content|main|shadow|post-header/i,
     get okMaybeItsACandidate() {
       return new RegExp(`and|(?<!${this.articleNegativeLookAheadCandidates.source})article(?!-(${this.articleNegativeLookBehindCandidates.source}))|body|column|content|^(?!main-navigation|main-header)main|shadow|post-header|hfeed site|blog-posts hfeed|container-banners|menu-opacity|header-with-anchor-widget`, 'i')

packages/readabilityjs/package.json

@@ -36,7 +36,6 @@
   },
   "dependencies": {
     "html-entities": "^2.3.2",
-    "modern-random-ua": "^1.0.3",
     "parse-srcset": "^1.0.2"
   }
 }

packages/readabilityjs/test/generate-testcase.js

@@ -6,7 +6,6 @@ var prettyPrint = require("./utils").prettyPrint;
 var htmltidy = require("htmltidy2").tidy;
 
 var { Readability, isProbablyReaderable } = require("../index");
-const { generate: generateRandomUA } = require("modern-random-ua/random_ua");
 const puppeteer = require('puppeteer');
 const { parseHTML } = require("linkedom");
 
@@ -14,7 +13,25 @@ var testcaseRoot = path.join(__dirname, "test-pages");
 
 var argURL = process.argv[3]; // Could be undefined, we'll warn if it is if that is an issue.
 
+const DESKTOP_USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 11_6_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4372.0 Safari/537.36'
+const NON_BOT_DESKTOP_USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0 Safari/537.36'
+const NON_BOT_HOSTS = ['bloomberg.com', 'forbes.com']
 const NON_SCRIPT_HOSTS= ['medium.com', 'fastcompany.com'];
+
+const userAgentForUrl = (url) => {
+  try {
+    const u = new URL(url);
+    for (const host of NON_BOT_HOSTS) {
+      if (u.hostname.endsWith(host)) {
+        return NON_BOT_DESKTOP_USER_AGENT;
+      }
+    }
+  } catch (e) {
+    console.log('error getting user agent for url', url, e)
+  }
+  return DESKTOP_USER_AGENT
+};
+
 const enableJavascriptForUrl = (url) => {
   try {
     const u = new URL(url);
@@ -81,8 +98,7 @@ async function fetchSource(url, callbackFn) {
   if (!enableJavascriptForUrl(url)) {
     await page.setJavaScriptEnabled(false);
   }
-  const ua = generateRandomUA();
-  await page.setUserAgent(ua);
+  await page.setUserAgent(userAgentForUrl(url));
 
   try {
     /*
@@ -142,12 +158,16 @@ async function fetchSource(url, callbackFn) {
       Array.from(document.body.getElementsByTagName('*')).forEach(el => {
         const style = window.getComputedStyle(el);
 
-        // Removing blurred images since they are mostly the copies of lazy loaded ones
-        if (['img', 'image'].includes(el.tagName.toLowerCase())) {
-          const filter = style.getPropertyValue('filter');
-          if (filter && filter.startsWith('blur')) {
-            el.parentNode && el.parentNode.removeChild(el);
+        try {
+          // Removing blurred images since they are mostly the copies of lazy loaded ones
+          if (el.tagName && ['img', 'image'].includes(el.tagName.toLowerCase())) {
+            const filter = style.getPropertyValue('filter');
+            if (filter && filter.startsWith('blur')) {
+              el.parentNode && el.parentNode.removeChild(el);
+            }
           }
+        } catch (err) {
+          // throw Error('error with element: ' + JSON.stringify(Array.from(document.body.getElementsByTagName('*'))))
         }
 
         // convert all nodes with background image to img nodes
@@ -155,7 +175,6 @@ async function fetchSource(url, callbackFn) {
           const filter = style.getPropertyValue('filter');
           // avoiding image nodes with a blur effect creation
           if (filter && filter.startsWith('blur')) {
-            // console.log('\n\n\n\n Filter found: ', filter);
             el && el.parentNode && el.parentNode.removeChild(el);
           } else {
             const matchedSRC = BI_SRC_REGEXP.exec(style.getPropertyValue('background-image'));
@@ -168,12 +187,9 @@ async function fetchSource(url, callbackFn) {
               // Article example: http://www.josiahzayner.com/2017/01/genetic-designer-part-i.html
               // DIV with class "content-inner" has `url("https://resources.blogblog.com/blogblog/data/1kt/travel/bg_container.png")` background image.
               if (el.innerHTML.length < 25) {
-                console.log('Replacing element with image');
                 const img = document.createElement('img');
                 img.src = matchedSRC[1];
-                el && el.parentNode && el.parentNode.replaceChild(img, el);
-              } else {
-                console.log('Element has too much content: ', el.innerHTML.length);
+                el && el.parentNode && el.parentNode.removeChild(el);
               }
             }
           }
@@ -227,11 +243,14 @@ function onResponseReceived(error, source, destRoot, options) {
 }
 
 async function runReadability(source, destPath, metadataDestPath, options) {
+  console.log('running readability')
+
   var uri = "http://fakehost/test/page.html";
   var myReader, result, readerable;
   try {
     // Use linkedom for isProbablyReaderable because it supports querySelectorAll
     var dom = parseHTML(source).document;
+    console.log('dom', parseHTML(source).innerHTML)
     readerable = isProbablyReaderable(dom);
     // We pass `caption` as a class to check that passing in extra classes works,
     // given that it appears in some of the test documents.
@@ -241,6 +260,7 @@ async function runReadability(source, destPath, metadataDestPath, options) {
     console.error(ex);
     ex.stack.forEach(console.log.bind(console));
   }
+  console.log('result', result)
   if (!result) {
     console.error("No content generated by readability, not going to write expected.html!");
     return;

packages/web/lib/networking/mutations/uploadImportFileMutation.ts

@@ -0,0 +1,44 @@
+import { gqlFetcher } from '../networkHelpers'
+import { v4 as uuidv4 } from 'uuid'
+
+export enum UploadImportFileType {
+  URL_LIST = 'URL_LIST',
+  POCKET = 'POCKET',
+  MATTER = 'MATTER',
+}
+
+type UploadImportFileResponseData = {
+  uploadImportFile?: UploadImportFileData
+}
+
+type UploadImportFileData = {
+  uploadSignedUrl: string
+  errorCodes?: unknown[]
+}
+
+export async function uploadImportFileRequestMutation(
+  type: UploadImportFileType,
+  contentType: string
+): Promise<UploadImportFileData | undefined> {
+  const mutation = `
+    mutation UploadImportFile($type: UploadImportFileType!, $contentType: String!) {
+      uploadImportFile(type:$type, contentType:$contentType) {
+        ... on UploadImportFileError {
+          errorCodes
+        }
+        ... on UploadImportFileSuccess {
+          uploadSignedUrl
+        }
+      }
+    }`
+
+  const data = await gqlFetcher(mutation, { type, contentType })
+  console.log('UploadImportFile: ', data)
+  const output = data as UploadImportFileResponseData | undefined
+  const error = output?.uploadImportFile?.errorCodes?.find(() => true)
+  console.log('error: ', error)
+  if (error) {
+    throw error
+  }
+  return output?.uploadImportFile
+}

packages/web/package.json

@@ -50,6 +50,7 @@
     "react-hot-toast": "^2.1.1",
     "react-masonry-css": "^1.0.16",
     "react-pro-sidebar": "^0.7.1",
+    "react-spinners": "^0.13.7",
     "react-super-responsive-table": "^5.2.1",
     "react-topbar-progress-indicator": "^4.1.1",
     "react-twitter-widgets": "^1.10.0",

packages/web/pages/tools/import/file.tsx

@@ -0,0 +1,227 @@
+import { ChangeEvent, useCallback, useMemo, useState } from 'react'
+import { Toaster } from 'react-hot-toast'
+
+import { showErrorToast, showSuccessToast } from '../../../lib/toastHelpers'
+import { applyStoredTheme } from '../../../lib/themeUpdater'
+
+import {
+  Box,
+  HStack,
+  VStack,
+} from '../../../components/elements/LayoutPrimitives'
+
+import 'antd/dist/antd.compact.css'
+import { StyledText } from '../../../components/elements/StyledText'
+import { ProfileLayout } from '../../../components/templates/ProfileLayout'
+import {
+  uploadImportFileRequestMutation,
+  UploadImportFileType,
+} from '../../../lib/networking/mutations/uploadImportFileMutation'
+import { Button } from '../../../components/elements/Button'
+import { FormLabel } from '../../../components/elements/FormElements'
+import { Loader } from '../../../components/templates/SavingRequest'
+
+import { SyncLoader } from 'react-spinners'
+import { theme } from '../../../components/tokens/stitches.config'
+
+type UploadState = 'none' | 'uploading' | 'completed'
+
+export default function ImportUploader(): JSX.Element {
+  applyStoredTheme(false)
+
+  const [errorMessage, setErrorMessage] = useState<string | undefined>()
+  const [file, setFile] = useState<File>()
+  const [type, setType] = useState<UploadImportFileType>()
+  const [uploadState, setUploadState] = useState<UploadState>('none')
+
+  const onFinish = (values: unknown) => {
+    console.log(values)
+  }
+
+  const onTypeChange = (e: ChangeEvent<HTMLInputElement>) => {
+    if (e.target.files) {
+      setFile(e.target.files[0])
+    }
+  }
+
+  const handleUploadClick = async () => {
+    if (!file) {
+      setErrorMessage('No file selected.')
+      return
+    }
+
+    if (!type) {
+      setErrorMessage('No file type selected.')
+      return
+    }
+
+    setUploadState('uploading')
+
+    try {
+      const result = await uploadImportFileRequestMutation(type, 'text/csv')
+
+      if (result && result.uploadSignedUrl) {
+        const uploadRes = await fetch(result.uploadSignedUrl, {
+          method: 'PUT',
+          body: file,
+          headers: {
+            'content-type': 'text/csv',
+            'content-length': `${file.size}`,
+          },
+        })
+        setUploadState('completed')
+      } else {
+        setErrorMessage(
+          'Unable to create file upload. Please ensure you are logged in.'
+        )
+        setUploadState('none')
+      }
+    } catch (error) {
+      console.log('caught error', error)
+      if (error == 'UPLOAD_DAILY_LIMIT_EXCEEDED') {
+        setErrorMessage('You have exceeded your maximum daily upload limit.')
+      }
+      setUploadState('none')
+    }
+  }
+
+  return (
+    <ProfileLayout>
+      <VStack
+        alignment="start"
+        css={{
+          padding: '16px',
+          background: 'white',
+          minWidth: '340px',
+          width: '70vw',
+          maxWidth: '576px',
+          borderRadius: '8px',
+          boxShadow: 'rgb(224 224 224) 9px 9px 9px -9px',
+        }}
+      >
+        <StyledText
+          style="modalHeadline"
+          css={{ color: theme.colors.omnivoreGray.toString() }}
+        >
+          Upload a file to import
+        </StyledText>
+        <StyledText
+          style="caption"
+          css={{ pt: '10px', color: theme.colors.omnivoreGray.toString() }}
+        >
+          Omnivore supports uploading a CSV file with one URL per a Row or a
+          _matter_history.csv file for users migrating from the Matter app
+        </StyledText>
+        <StyledText
+          style="caption"
+          css={{ pt: '10px', color: theme.colors.omnivoreGray.toString() }}
+        >
+          <b>Importing from Matter:</b> If you are a Matter user, export your
+          data, unzip the Archive.zip file, select `Matter history` on this
+          page, and upload the _matter_history file that was extracted from the
+          Archive.zip file.
+        </StyledText>
+        <StyledText
+          style="caption"
+          css={{ pt: '10px', color: theme.colors.omnivoreGray.toString() }}
+        >
+          <b>Importing from other apps:</b> Most apps allow you to export your
+          saved items as a CSV file. Once you have extracted a file, ensure the
+          first column of each row is a URL, and choose our `CSV of URLs` option
+          and upload your file.
+        </StyledText>
+        <StyledText
+          style="caption"
+          css={{ pt: '10px', color: theme.colors.omnivoreGray.toString() }}
+        >
+          <b>Note:</b> Please note you are limited to three import uploads per a
+          day, and the maximum file size is 10MB.
+        </StyledText>
+        <VStack css={{ pt: '36px' }}>
+          {uploadState == 'completed' ? (
+            <StyledText
+              style="caption"
+              css={{ pt: '10px', color: theme.colors.omnivoreGray.toString() }}
+            >
+              Your upload has completed. Please note that it can take some time
+              for your library to be updated. You will be sent an email when the
+              process completes.
+            </StyledText>
+          ) : (
+            <>
+              <HStack>
+                <StyledText
+                  style="caption"
+                  css={{
+                    height: '38px',
+                    width: '88px',
+                    fontWeight: 'bold',
+                    color: theme.colors.omnivoreGray.toString(),
+                  }}
+                >
+                  File type:
+                </StyledText>
+                <select
+                  onChange={(event) => {
+                    const changeType: UploadImportFileType =
+                      UploadImportFileType[
+                        event.currentTarget
+                          .value as keyof typeof UploadImportFileType
+                      ]
+                    setType(changeType)
+                  }}
+                  style={{
+                    padding: '8px',
+                    height: '38px',
+                    borderRadius: '6px',
+                    minWidth: '196px',
+                    color: theme.colors.omnivoreGray.toString(),
+                  }}
+                >
+                  <option value="none">Choose file type</option>
+                  <option value="URL_LIST">CSV of URLs</option>
+                  <option value="MATTER">Matter history</option>
+                </select>
+              </HStack>
+
+              <HStack css={{ width: '100%' }}>
+                <FormLabel
+                  css={{
+                    height: '38px',
+                    width: '88px',
+                    color: theme.colors.omnivoreGray.toString(),
+                  }}
+                ></FormLabel>
+                <HStack css={{ py: '16px' }} distribution="start">
+                  <input type="file" onChange={onTypeChange} />
+                  {/* <Box>{file && `${file.name}`}</Box> */}
+                </HStack>
+              </HStack>
+
+              <HStack css={{ width: '100%' }} distribution="start">
+                <FormLabel css={{ height: '38px', width: '88px' }}></FormLabel>
+                {uploadState == 'none' && (
+                  <Button onClick={handleUploadClick} style="ctaDarkYellow">
+                    Upload
+                  </Button>
+                )}
+                {uploadState == 'uploading' && (
+                  <SyncLoader
+                    color={theme.colors.omnivoreGray.toString()}
+                    size={8}
+                  />
+                )}
+              </HStack>
+            </>
+          )}
+          <HStack css={{ width: '100%', pt: '10px' }} distribution="start">
+            <FormLabel css={{ height: '38px', width: '88px' }}></FormLabel>
+            {errorMessage && (
+              <StyledText style="error">{errorMessage}</StyledText>
+            )}
+          </HStack>
+        </VStack>
+      </VStack>
+    </ProfileLayout>
+  )
+}

yarn.lock

@@ -2746,6 +2746,13 @@
   dependencies:
     google-gax "^2.24.1"
 
+"@google-cloud/tasks@^3.0.5":
+  version "3.0.5"
+  resolved "https://registry.yarnpkg.com/@google-cloud/tasks/-/tasks-3.0.5.tgz#4c0c648d10c7cf10b2e1599dd5a48bf8cc78cbd6"
+  integrity sha512-fC7afAV2d+zz0A2TxJWBZyBTHlB8nWBkNr/7BH6ZM/uAgT6gmSLfDho3aQNttnumj7IiI0vwUx31Zu1wG3m7yw==
+  dependencies:
+    google-gax "^3.5.2"
+
 "@graphql-codegen/cli@^2.6.2":
   version "2.6.2"
   resolved "https://registry.yarnpkg.com/@graphql-codegen/cli/-/cli-2.6.2.tgz#a9aa4656141ee0998cae8c7ad7d0bf9ca8e0c9ae"
@@ -19713,11 +19720,6 @@ mocha@^9.0.1:
     yargs-parser "20.2.4"
     yargs-unparser "2.0.0"
 
-modern-random-ua@^1.0.3:
-  version "1.0.3"
-  resolved "https://registry.yarnpkg.com/modern-random-ua/-/modern-random-ua-1.0.3.tgz#c6fca366e60ea9a793c9d99021533744d7f58c41"
-  integrity sha1-xvyjZuYOqaeTydmQIVM3RNf1jEE=
-
 modify-values@^1.0.0:
   version "1.0.1"
   resolved "https://registry.yarnpkg.com/modify-values/-/modify-values-1.0.1.tgz#b3939fa605546474e3e3e3c63d64bd43b4ee6022"
@@ -22890,6 +22892,11 @@ react-slidedown@^2.4.5:
   dependencies:
     tslib "^2.0.0"
 
+react-spinners@^0.13.7:
+  version "0.13.7"
+  resolved "https://registry.yarnpkg.com/react-spinners/-/react-spinners-0.13.7.tgz#0f423c415bfa56765ce9fb36ff604e52a92b37a9"
+  integrity sha512-mRamn56bfxWbGcacif5RT3UbeJaXi2AttjtPwSmomuv2IcxjpbfETCzdTvaQpNDk0E33ENJsStsQeKAZFuJcpA==
+
 react-style-singleton@^2.1.0:
   version "2.1.1"
   resolved "https://registry.yarnpkg.com/react-style-singleton/-/react-style-singleton-2.1.1.tgz#ce7f90b67618be2b6b94902a30aaea152ce52e66"