remove elastic from docker-compose

docker-compose-test.yml

@@ -16,26 +16,6 @@ services:
     expose:
       - 5432
 
-  elastic-test:
-    image: docker.elastic.co/elasticsearch/elasticsearch:7.17.1
-    container_name: "omnivore-elastic-test"
-    healthcheck:
-      test: curl 0.0.0.0:9201/_cat/health >/dev/null || exit 1
-      interval: 2s
-      timeout: 2s
-      retries: 5
-    environment:
-      - discovery.type=single-node
-      - http.cors.allow-origin=*
-      - http.cors.enabled=true
-      - http.cors.allow-headers=X-Requested-With,X-Auth-Token,Content-Type,Content-Length,Authorization
-      - http.cors.allow-credentials=true
-      - http.port=9201
-    volumes:
-      - ./.docker/elastic-test-data:/usr/share/elasticsearch/data
-    ports:
-      - "9201:9201"
-
   api-test:
     build:
       context: .
@@ -49,7 +29,6 @@ services:
       - PG_DB=omnivore_test
       - PG_PORT=5432
       - PG_POOL_MAX=20
-      - ELASTIC_URL=http://elastic-test:9201
       - IMAGE_PROXY_URL=http://localhost:9999
       - IMAGE_PROXY_SECRET=some-secret
       - JWT_SECRET=some_secret
@@ -69,5 +48,3 @@ services:
     depends_on:
       postgres-test:
         condition: service_healthy
-      elastic-test:
-        condition: service_healthy

docker-compose.yml

@@ -28,33 +28,9 @@ services:
       - PG_USER=postgres
       - PG_PASSWORD=postgres
       - PG_DB=omnivore
-      - ELASTIC_URL=http://elastic:9200
-      - ELASTIC_USERNAME=blank
-      - ELASTIC_PASSWORD=blank
     depends_on:
       postgres:
         condition: service_healthy
-      elastic:
-        condition: service_healthy
-
-  elastic:
-    image: docker.elastic.co/elasticsearch/elasticsearch:7.17.1
-    container_name: "omnivore-elastic"
-    healthcheck:
-      test: curl 0.0.0.0:9200/_cat/health >/dev/null || exit 1
-      interval: 2s
-      timeout: 2s
-      retries: 5
-    environment:
-      - discovery.type=single-node
-      - http.cors.allow-origin=*
-      - http.cors.enabled=true
-      - http.cors.allow-headers=X-Requested-With,X-Auth-Token,Content-Type,Content-Length,Authorization
-      - http.cors.allow-credentials=true
-    volumes:
-      - ./.docker/elastic-data:/usr/share/elasticsearch/data
-    ports:
-      - "9200:9200"
 
   api:
     build:
@@ -75,7 +51,6 @@ services:
       - PG_DB=omnivore
       - PG_PORT=5432
       - PG_POOL_MAX=20
-      - ELASTIC_URL=http://elastic:9200
       - JAEGER_HOST=jaeger
       - IMAGE_PROXY_SECRET=some-secret
       - JWT_SECRET=some_secret
@@ -86,8 +61,6 @@ services:
     depends_on:
       migrate:
         condition: service_completed_successfully
-      elastic:
-        condition: service_healthy
 
   web:
     build:

packages/api/.env.example

@@ -24,8 +24,7 @@ GCS_UPLOAD_BUCKET=
 GCS_UPLOAD_SA_KEY_FILE_PATH=
 TWITTER_BEARER_TOKEN=
 PREVIEW_IMAGE_WRAPPER_ID='selected_highlight_wrapper'
-REMINDER_TASK_HANDLER_URL=
-ELASTIC_URL=http://localhost:9200
 SENDER_MESSAGE=msgs@sender.domain
 SENDER_FEEDBACK=feedback@sender.domain
 SENDER_GENERAL=no-reply@sender.domain
+CONTENT_FETCH_URL=http://localhost:9090/

packages/api/.env.test

@@ -26,7 +26,5 @@ GCS_UPLOAD_PRIVATE_BUCKET=
 TWITTER_BEARER_TOKEN=
 PREVIEW_IMAGE_WRAPPER_ID='selected_highlight_wrapper'
 SEGMENT_WRITE_KEY='test'
-REMINDER_TASK_HANDLER_URL=http://localhost:4000/svc/reminders/trigger
 PUBSUB_VERIFICATION_TOKEN='123456'
 CONTENT_FETCH_URL=http://localhost:9090/
-ELASTIC_URL=http://localhost:9200

packages/api/src/apollo.ts

@@ -12,10 +12,10 @@ import { ExpressContext } from 'apollo-server-express/dist/ApolloServer'
 import * as jwt from 'jsonwebtoken'
 import { EntityManager } from 'typeorm'
 import { promisify } from 'util'
-import { AppDataSource } from './data-source'
 import { sanitizeDirectiveTransformer } from './directives'
 import { env } from './env'
 import { createPubSubClient } from './pubsub'
+import { entityManager } from './repository'
 import { functionResolvers } from './resolvers/function_resolvers'
 import { ClaimsToSet, ResolverContext } from './resolvers/types'
 import ScalarResolvers from './scalars'
@@ -76,9 +76,9 @@ const contextFunc: ContextFunction<ExpressContext, ResolverContext> = async ({
       cb: (em: EntityManager) => TResult,
       userRole?: string
     ): Promise<TResult> =>
-      AppDataSource.transaction(async (em) => {
-        await setClaims(em, undefined, userRole)
-        return cb(em)
+      entityManager.transaction(async (tx) => {
+        await setClaims(tx, undefined, userRole)
+        return cb(tx)
       }),
     tracingSpan: tracer.startSpan('apollo.request'),
   }

packages/api/src/data_source.ts

@@ -3,7 +3,7 @@ import { SnakeNamingStrategy } from 'typeorm-naming-strategies'
 import { env } from './env'
 import { CustomTypeOrmLogger } from './utils/logger'
 
-export const AppDataSource = new DataSource({
+export const appDataSource = new DataSource({
   type: 'postgres',
   host: env.pg.host,
   port: env.pg.port,

packages/api/src/entity/api_key.ts

@@ -5,10 +5,12 @@ import {
   JoinColumn,
   ManyToOne,
   PrimaryGeneratedColumn,
+  Unique,
 } from 'typeorm'
 import { User } from './user'
 
 @Entity()
+@Unique('user_id_name', ['user', 'name'])
 export class ApiKey {
   @PrimaryGeneratedColumn('uuid')
   id!: string

packages/api/src/events/highlight_subscriber.ts

@@ -80,7 +80,7 @@ export class HighlightSubscriber
         await this.pubsubClient.entityCreated<Label>(
           EntityType.LABEL,
           event.entity.labels,
-          event.entity.libraryItem.user.id
+          event.databaseEntity.libraryItem.user.id
         )
       }
     }

packages/api/src/events/library_item_subscriber.ts

@@ -65,10 +65,11 @@ export class LibraryItemSubscriber
 
       // publish label added event if a label was added
       if (event.entity.labels) {
+        const labels = event.entity.labels as Label[]
         await event.manager
           .getRepository(LibraryItem)
           .update(event.databaseEntity.id, {
-            labelNames: event.entity.labels,
+            labelNames: labels.map((label) => label.name.toLowerCase()),
           })
 
         await this.pubsubClient.entityCreated<Label>(

packages/api/src/repository/index.ts

@@ -1,6 +1,5 @@
 import { EntityManager, EntityTarget, Repository } from 'typeorm'
-import { AppDataSource } from '../data-source'
-import { ApiKey } from '../entity/api_key'
+import { appDataSource } from '../data_source'
 import { Feature } from '../entity/feature'
 import { Filter } from '../entity/filter'
 import { Follower } from '../entity/follower'
@@ -21,17 +20,16 @@ import { ContentDisplayReport } from '../entity/reports/content_display_report'
 import { Rule } from '../entity/rule'
 import { Subscription } from '../entity/subscription'
 import { UploadFile } from '../entity/upload_file'
-import { User } from '../entity/user'
 import { UserDeviceToken } from '../entity/user_device_tokens'
 import { UserPersonalization } from '../entity/user_personalization'
 import { Webhook } from '../entity/webhook'
 
 export const setClaims = async (
-  t: EntityManager,
+  manager: EntityManager,
   uid: string
 ): Promise<void> => {
   const dbRole = 'omnivore_user'
-  return t
+  return manager
     .query('SELECT * from omnivore.set_claims($1, $2)', [uid, dbRole])
     .then()
 }
@@ -40,9 +38,8 @@ export const getRepository = <T>(entity: EntityTarget<T>): Repository<T> => {
   return entityManager.getRepository(entity)
 }
 
-export const entityManager = AppDataSource.manager
+export const entityManager = appDataSource.manager
 
-export const userRepository = getRepository(User)
 export const uploadFileRepository = getRepository(UploadFile)
 export const reminderRepository = getRepository(Reminder)
 export const libraryItemRepository = getRepository(LibraryItem)
@@ -52,7 +49,6 @@ export const inviteRepository = getRepository(Invite)
 export const abuseReportRepository = getRepository(AbuseReport)
 export const contentDisplayReportRepository =
   getRepository(ContentDisplayReport)
-export const apiKeyRepository = getRepository(ApiKey)
 export const featureRepository = getRepository(Feature)
 export const filterRepository = getRepository(Filter)
 export const followerRepository = getRepository(Follower)

packages/api/src/repository/library_item.ts

@@ -0,0 +1,11 @@
+import { libraryItemRepository } from '.'
+
+export const getLibraryItemById = async (id: string) => {
+  return libraryItemRepository.findOneBy({ id })
+}
+
+export const getLibraryItemByUrl = async (url: string) => {
+  return libraryItemRepository.findOneBy({
+    originalUrl: url,
+  })
+}

packages/api/src/repository/user.ts

@@ -0,0 +1,37 @@
+import { In } from 'typeorm'
+import { entityManager } from '.'
+import { User } from './../entity/user'
+
+const TOP_USERS = [
+  'jacksonh',
+  'nat',
+  'luis',
+  'satindar',
+  'malandrina',
+  'patrick',
+  'alexgutjahr',
+  'hongbowu',
+]
+export const MAX_RECORDS_LIMIT = 1000
+
+export const userRepository = entityManager.getRepository(User).extend({
+  findById(id: string) {
+    return this.findOneBy({ id })
+  },
+
+  findByEmail(email: string) {
+    return this.createQueryBuilder('user')
+      .leftJoinAndSelect('user.profile', 'profile')
+      .where('LOWER(email) = LOWER(:email)', { email }) // case insensitive
+      .getOne()
+  },
+
+  findTopUsers() {
+    return this.createQueryBuilder()
+      .where({
+        profile: { username: In(TOP_USERS) },
+      })
+      .limit(MAX_RECORDS_LIMIT)
+      .getMany()
+  },
+})

packages/api/src/resolvers/api_key/index.ts

@@ -13,37 +13,28 @@ import {
   RevokeApiKeyErrorCode,
   RevokeApiKeySuccess,
 } from '../../generated/graphql'
-import { getRepository, userRepository } from '../../repository'
 import { analytics } from '../../utils/analytics'
 import { generateApiKey, hashApiKey } from '../../utils/auth'
 import { authorized } from '../../utils/helpers'
 
 export const apiKeysResolver = authorized<ApiKeysSuccess, ApiKeysError>(
-  async (_, __, { claims: { uid }, log }) => {
-    log.info('apiKeysResolver')
-
+  async (_, __, { log, authTrx }) => {
     try {
-      const user = await userRepository.findOneBy({ id: uid })
-      if (!user) {
-        return {
-          errorCodes: [ApiKeysErrorCode.Unauthorized],
-        }
-      }
-
-      const apiKeys = await getRepository(ApiKey).find({
-        select: ['id', 'name', 'scopes', 'expiresAt', 'createdAt', 'usedAt'],
-        where: { user: { id: uid } },
-        order: {
-          usedAt: { direction: 'DESC', nulls: 'last' },
-          createdAt: 'DESC',
-        },
+      const apiKeys = await authTrx<Promise<ApiKey[]>>(async (em) => {
+        return em.getRepository(ApiKey).find({
+          select: ['id', 'name', 'scopes', 'expiresAt', 'createdAt', 'usedAt'],
+          order: {
+            usedAt: { direction: 'DESC', nulls: 'last' },
+            createdAt: 'DESC',
+          },
+        })
       })
 
       return {
         apiKeys,
       }
     } catch (e) {
-      log.error(e)
+      log.error('apiKeysResolver error', e)
 
       return {
         errorCodes: [ApiKeysErrorCode.BadRequest],
@@ -56,33 +47,17 @@ export const generateApiKeyResolver = authorized<
   GenerateApiKeySuccess,
   GenerateApiKeyError,
   MutationGenerateApiKeyArgs
->(async (_, { input: { name, expiresAt } }, { claims: { uid }, log }) => {
+>(async (_, { input: { name, expiresAt } }, { authTrx, log, uid }) => {
   try {
-    log.info('generateApiKeyResolver')
-    const user = await userRepository.findOneBy({ id: uid })
-    if (!user) {
-      return {
-        errorCodes: [GenerateApiKeyErrorCode.Unauthorized],
-      }
-    }
-
-    const existingApiKey = await getRepository(ApiKey).findOneBy({
-      user: { id: uid },
-      name,
-    })
-    if (existingApiKey) {
-      return {
-        errorCodes: [GenerateApiKeyErrorCode.AlreadyExists],
-      }
-    }
-
     const exp = new Date(expiresAt)
-    const apiKey = generateApiKey()
-    const apiKeyData = await getRepository(ApiKey).save({
-      user: { id: uid },
-      name,
-      key: hashApiKey(apiKey),
-      expiresAt: exp,
+    const originalKey = generateApiKey()
+    const apiKeyCreated = await authTrx<Promise<ApiKey>>(async (em) => {
+      return em.getRepository(ApiKey).save({
+        user: { id: uid },
+        name,
+        key: hashApiKey(originalKey),
+        expiresAt: exp,
+      })
     })
 
     analytics.track({
@@ -97,12 +72,12 @@ export const generateApiKeyResolver = authorized<
 
     return {
       apiKey: {
-        ...apiKeyData,
-        key: apiKey,
+        ...apiKeyCreated,
+        key: originalKey,
       },
     }
   } catch (error) {
-    log.error(error)
+    log.error('generateApiKeyResolver', error)
 
     return { errorCodes: [GenerateApiKeyErrorCode.BadRequest] }
   }
@@ -112,34 +87,32 @@ export const revokeApiKeyResolver = authorized<
   RevokeApiKeySuccess,
   RevokeApiKeyError,
   MutationRevokeApiKeyArgs
->(async (_, { id }, { claims: { uid }, log }) => {
-  log.info('RevokeApiKeyResolver')
-
+>(async (_, { id }, { claims: { uid }, log, authTrx }) => {
   try {
-    const user = await userRepository.findOneBy({ id: uid })
-    if (!user) {
-      return {
-        errorCodes: [RevokeApiKeyErrorCode.Unauthorized],
+    const deletedApiKey = await authTrx<Promise<ApiKey | null>>(async (em) => {
+      const apiKeyRepository = em.getRepository(ApiKey)
+      const apiKey = await apiKeyRepository.findOneBy({ id })
+      if (!apiKey) {
+        return null
       }
-    }
 
-    const apiKey = await getRepository(ApiKey).findOne({
-      where: { id },
-      relations: ['user'],
+      return apiKeyRepository.remove(apiKey)
     })
-    if (!apiKey) {
+
+    if (!deletedApiKey) {
       return {
         errorCodes: [RevokeApiKeyErrorCode.NotFound],
       }
     }
 
-    if (apiKey.user.id !== uid) {
-      return {
-        errorCodes: [RevokeApiKeyErrorCode.Unauthorized],
-      }
-    }
-
-    const deletedApiKey = await getRepository(ApiKey).remove(apiKey)
+    analytics.track({
+      userId: uid,
+      event: 'api_key_revoked',
+      properties: {
+        id,
+        env: env.server.apiEnv,
+      },
+    })
 
     return {
       apiKey: {
@@ -149,7 +122,7 @@ export const revokeApiKeyResolver = authorized<
       },
     }
   } catch (e) {
-    log.error(e)
+    log.error('revokeApiKeyResolver error', e)
 
     return {
       errorCodes: [RevokeApiKeyErrorCode.BadRequest],

packages/api/src/resolvers/article/index.ts

@@ -6,8 +6,6 @@
 import { Readability } from '@omnivore/readability'
 import graphqlFields from 'graphql-fields'
 import { LibraryItemType } from '../../entity/library_item'
-import { UploadFile } from '../../entity/upload_file'
-import { User } from '../../entity/user'
 import { env } from '../../env'
 import {
   Article,
@@ -61,7 +59,9 @@ import {
   UpdatesSinceErrorCode,
   UpdatesSinceSuccess,
 } from '../../generated/graphql'
-import { getRepository } from '../../repository'
+import { uploadFileRepository } from '../../repository'
+import { getLibraryItemByUrl } from '../../repository/library_item'
+import { getUserById, userRepository } from '../../repository/user'
 import { createPageSaveRequest } from '../../services/create_page_save_request'
 import {
   addLabelToPage,
@@ -167,7 +167,7 @@ export const createArticleResolver = authorized<
       },
     })
 
-    const userData = await getRepository(User).findOneBy({ id: uid })
+    const userData = await userRepository.findById(uid)
     if (!userData) {
       return pageError(
         {
@@ -207,7 +207,7 @@ export const createArticleResolver = authorized<
       let userArticleUrl: string | null = null
       let uploadFileHash = null
       let domContent = null
-      let pageType = LibraryItemType.Unknown
+      let itemType = LibraryItemType.Unknown
 
       const DUMMY_RESPONSE = {
         user,
@@ -249,7 +249,7 @@ export const createArticleResolver = authorized<
         /* We do not trust the values from client, lookup upload file by querying
          * with filtering on user ID and URL to verify client's uploadFileId is valid.
          */
-        const uploadFile = await getRepository(UploadFile).findOneBy({
+        const uploadFile = await uploadFileRepository.findOneBy({
           id: uploadFileId,
           user: { id: uid },
         })
@@ -267,7 +267,7 @@ export const createArticleResolver = authorized<
         uploadFileHash = uploadFileDetails.md5Hash
         userArticleUrl = uploadFileDetails.fileUrl
         canonicalUrl = uploadFile.url
-        pageType = itemTypeForContentType(uploadFile.contentType)
+        itemType = itemTypeForContentType(uploadFile.contentType)
         title = titleForFilePath(uploadFile.url)
       } else if (
         source !== 'puppeteer-parse' &&
@@ -285,7 +285,7 @@ export const createArticleResolver = authorized<
         parsedContent = parseResults.parsedContent
         canonicalUrl = parseResults.canonicalUrl
         domContent = parseResults.domContent
-        pageType = parseResults.pageType as unknown as LibraryItemType
+        itemType = parseResults.pageType as unknown as LibraryItemType
       } else if (!preparedDocument?.document) {
         // We have a URL but no document, so we try to send this to puppeteer
         // and return a dummy response.
@@ -304,7 +304,7 @@ export const createArticleResolver = authorized<
         slug,
         croppedPathname,
         originalHtml: domContent,
-        pageType,
+        itemType,
         preparedDocument,
         uploadFileHash,
         canonicalUrl,
@@ -343,15 +343,13 @@ export const createArticleResolver = authorized<
       // save page's state and labels
       articleToSave.archivedAt = archivedAt
       articleToSave.labels = labels
-      if (
-        pageId ||
-        (pageId = (
-          await getPageByParam({
-            userId: uid,
-            url: articleToSave.url,
-          })
-        )?.id)
-      ) {
+
+      const existingLibraryItem = await getLibraryItemByUrl(
+        articleToSave.originalUrl!,
+        uid
+      )
+      pageId = existingLibraryItem?.id || pageId
+      if (pageId || existingLibraryItem) {
         // update existing page's state from processing to succeeded
         const updated = await updatePage(pageId, articleToSave, {
           ...ctx,
@@ -498,15 +496,6 @@ export const getArticlesResolver = authorized<
 
   const searchQuery = parseSearchQuery(params.query || undefined)
 
-  analytics.track({
-    userId: claims.uid,
-    event: 'get_articles',
-    properties: {
-      env: env.server.apiEnv,
-      ...searchQuery,
-    },
-  })
-
   const [pages, totalCount] = (await searchLibraryItems(
     {
       from: Number(startCursor),
@@ -974,17 +963,6 @@ export const typeaheadSearchResolver = authorized<
   if (!claims?.uid) {
     return { errorCodes: [TypeaheadSearchErrorCode.Unauthorized] }
   }
-
-  analytics.track({
-    userId: claims.uid,
-    event: 'typeahead',
-    properties: {
-      env: env.server.apiEnv,
-      query,
-      first,
-    },
-  })
-
   const results = await searchAsYouType(claims.uid, query, first || undefined)
   const items: TypeaheadSearchItem[] = results.map((r) => ({
     ...r,

packages/api/src/resolvers/filters/index.ts

@@ -1,5 +1,4 @@
 import { Between } from 'typeorm'
-import { AppDataSource } from '../../data-source'
 import { Filter } from '../../entity/filter'
 import { User } from '../../entity/user'
 import { env } from '../../env'
@@ -24,7 +23,7 @@ import {
   UpdateFilterSuccess,
   UpdateFilterErrorCode,
 } from '../../generated/graphql'
-import { getRepository, setClaims } from '../../repository'
+import { entityManager, getRepository, setClaims } from '../../repository'
 import { analytics } from '../../utils/analytics'
 import { env } from '../../env'
 import { isNil, mergeWith } from 'lodash'

packages/api/src/resolvers/labels/index.ts

@@ -1,5 +1,5 @@
 import { Between } from 'typeorm'
-import { AppDataSource } from '../../data-source'
+import { appDataSource } from '../../data_source'
 import { getHighlightById } from '../../elastic/highlights'
 import {
   deleteLabel,
@@ -50,8 +50,6 @@ import { authorized } from '../../utils/helpers'
 
 export const labelsResolver = authorized<LabelsSuccess, LabelsError>(
   async (_obj, _params, { claims: { uid }, log }) => {
-    log.info('labelsResolver')
-
     try {
       const user = await getRepository(User).findOne({
         where: { id: uid },
@@ -170,7 +168,7 @@ export const deleteLabelResolver = authorized<
       }
     }
 
-    const result = await AppDataSource.transaction(async (t) => {
+    const result = await appDataSource.transaction(async (t) => {
       await setClaims(t, uid)
       return t.getRepository(Label).delete(labelId)
     })
@@ -344,7 +342,7 @@ export const updateLabelResolver = authorized<
       },
     })
 
-    const result = await AppDataSource.transaction(async (t) => {
+    const result = await appDataSource.transaction(async (t) => {
       await setClaims(t, uid)
       label.name = name
       label.color = color
@@ -532,7 +530,7 @@ export const moveLabelResolver = authorized<
     const moveUp = newPosition < oldPosition
 
     // move label to the new position
-    const updated = await AppDataSource.transaction(async (t) => {
+    const updated = await appDataSource.transaction(async (t) => {
       await setClaims(t, uid)
 
       // update the position of the other labels

packages/api/src/resolvers/popular_reads/index.ts

@@ -1,4 +1,3 @@
-import { env } from '../../env'
 import {
   AddPopularReadError,
   AddPopularReadErrorCode,
@@ -7,22 +6,12 @@ import {
 } from '../../generated/graphql'
 import { userRepository } from '../../repository'
 import { addPopularRead } from '../../services/popular_reads'
-import { analytics } from '../../utils/analytics'
 import { authorized } from '../../utils/helpers'
 export const addPopularReadResolver = authorized<
   AddPopularReadSuccess,
   AddPopularReadError,
   MutationAddPopularReadArgs
 >(async (_, { name }, { uid }) => {
-  analytics.track({
-    userId: uid,
-    event: 'popular_read_added',
-    properties: {
-      name: name,
-      env: env.server.apiEnv,
-    },
-  })
-
   const user = await userRepository.findOneBy({
     id: uid,
   })

packages/api/src/resolvers/recent_searches/index.ts

@@ -14,16 +14,6 @@ export const recentSearchesResolver = authorized<
   RecentSearchesSuccess,
   RecentSearchesError
 >(async (_obj, _params, { claims: { uid }, log }) => {
-  log.info('recentSearches')
-
-  analytics.track({
-    userId: uid,
-    event: 'recentSearches',
-    properties: {
-      env: env.server.apiEnv,
-    },
-  })
-
   const user = await getRepository(User).findOneBy({ id: uid })
   if (!user) {
     return { errorCodes: [RecentSearchesErrorCode.Unauthorized] }

packages/api/src/resolvers/send_install_instructions/index.ts

@@ -1,3 +1,4 @@
+import { appDataSource } from '../../data_source'
 import { User } from '../../entity/user'
 import { env } from '../../env'
 import {
@@ -5,7 +6,6 @@ import {
   SendInstallInstructionsErrorCode,
   SendInstallInstructionsSuccess,
 } from '../../generated/graphql'
-import { AppDataSource } from '../../data-source'
 import { authorized } from '../../utils/helpers'
 import { sendEmail } from '../../utils/sendEmail'
 
@@ -17,7 +17,7 @@ export const sendInstallInstructionsResolver = authorized<
   SendInstallInstructionsError
 >(async (_parent, _args, { claims, log }) => {
   try {
-    const user = await AppDataSource.getRepository(User).findOneBy({
+    const user = await appDataSource.getRepository(User).findOneBy({
       id: claims.uid,
     })
 

packages/api/src/resolvers/subscriptions/index.ts

@@ -1,6 +1,6 @@
 import Parser from 'rss-parser'
 import { Brackets } from 'typeorm'
-import { AppDataSource } from '../../data-source'
+import { appDataSource } from '../../data_source'
 import { Subscription } from '../../entity/subscription'
 import { User } from '../../entity/user'
 import { env } from '../../env'
@@ -270,7 +270,7 @@ export const subscribeResolver = authorized<
       const feed = await parser.parseURL(input.url)
 
       // limit number of rss subscriptions to 50
-      const newSubscriptions = (await AppDataSource.query(
+      const newSubscriptions = (await appDataSource.query(
         `insert into omnivore.subscriptions (name, url, description, type, user_id, icon) 
         select $1, $2, $3, $4, $5, $6 from omnivore.subscriptions 
         where user_id = $5 and type = 'RSS' and status = 'ACTIVE' 

packages/api/src/resolvers/user/index.ts

@@ -1,6 +1,6 @@
 import * as jwt from 'jsonwebtoken'
 import { RegistrationType } from '../../datalayer/user/model'
-import { AppDataSource } from '../../data-source'
+import { appDataSource } from '../../data_source'
 import { deletePagesByParam } from '../../elastic/pages'
 import { User as UserEntity } from '../../entity/user'
 import { env } from '../../env'
@@ -341,7 +341,7 @@ export const deleteAccountResolver = authorized<
     },
   })
 
-  const result = await AppDataSource.transaction(async (t) => {
+  const result = await appDataSource.transaction(async (t) => {
     await setClaims(t, claims.uid)
     return t.getRepository(UserEntity).delete(userID)
   })

packages/api/src/resolvers/user_personalization/index.ts

@@ -1,4 +1,4 @@
-import { AppDataSource } from '../../data-source'
+import { appDataSource } from '../../data_source'
 import { UserPersonalization } from '../../entity/user_personalization'
 import {
   GetUserPersonalizationError,
@@ -19,7 +19,7 @@ export const setUserPersonalizationResolver = authorized<
 >(async (_, { input }, { claims: { uid }, log }) => {
   log.info('setUserPersonalizationResolver', { uid, input })
 
-  const result = await AppDataSource.transaction(async (entityManager) => {
+  const result = await appDataSource.transaction(async (entityManager) => {
     await setClaims(entityManager, uid)
 
     return entityManager.getRepository(UserPersonalization).upsert(

packages/api/src/routers/auth/auth_router.ts

@@ -14,7 +14,7 @@ import express from 'express'
 import * as jwt from 'jsonwebtoken'
 import url from 'url'
 import { promisify } from 'util'
-import { AppDataSource } from '../../data-source'
+import { appDataSource } from '../../data_source'
 import { RegistrationType, StatusType, User } from '../../entity/user'
 import { env } from '../../env'
 import { LoginErrorCode, SignupErrorCode } from '../../generated/graphql'
@@ -547,7 +547,7 @@ export function authRouter() {
         }
 
         if (user.status === StatusType.Pending) {
-          const updated = await AppDataSource.transaction(
+          const updated = await appDataSource.transaction(
             async (entityManager) => {
               await setClaims(entityManager, user.id)
               return entityManager
@@ -678,7 +678,7 @@ export function authRouter() {
         }
 
         const hashedPassword = await hashPassword(password)
-        const updated = await AppDataSource.transaction(
+        const updated = await appDataSource.transaction(
           async (entityManager) => {
             await setClaims(entityManager, user.id)
             return entityManager.getRepository(User).update(user.id, {

packages/api/src/routers/svc/content.ts

@@ -2,7 +2,7 @@
 /* eslint-disable @typescript-eslint/no-unsafe-assignment */
 /* eslint-disable @typescript-eslint/explicit-module-boundary-types */
 import express from 'express'
-import { AppDataSource } from '../../data-source'
+import { appDataSource } from '../../data_source'
 import { getPageByParam, updatePage } from '../../elastic/pages'
 import { Page } from '../../elastic/types'
 import { ArticleSavingRequestStatus } from '../../generated/graphql'
@@ -73,7 +73,7 @@ export function contentServiceRouter() {
     pageToUpdate.state = ArticleSavingRequestStatus.Succeeded
 
     try {
-      const uploadFileData = await AppDataSource.transaction(async (tx) => {
+      const uploadFileData = await appDataSource.transaction(async (tx) => {
         await setClaims(tx, page.userId)
         return setFileUploadComplete(fileId, tx)
       })

packages/api/src/routers/svc/email_attachment.ts

@@ -1,5 +1,5 @@
 import express from 'express'
-import { AppDataSource } from '../../data-source'
+import { appDataSource } from '../../data_source'
 import { createPage } from '../../elastic/pages'
 import { ArticleSavingRequestStatus, Page } from '../../elastic/types'
 import { env } from '../../env'
@@ -129,7 +129,7 @@ export function emailAttachmentRouter() {
         uploadFile.fileName
       )
 
-      const uploadFileData = await AppDataSource.transaction(async (tx) => {
+      const uploadFileData = await appDataSource.transaction(async (tx) => {
         await setClaims(tx, user.id)
         return setFileUploadComplete(uploadFileId, tx)
       })

packages/api/src/routers/svc/reminders.ts

@@ -1,5 +1,5 @@
 import { MulticastMessage } from 'firebase-admin/messaging'
-import { AppDataSource } from '../../data-source'
+import { appDataSource } from '../../data_source'
 import { updatePage } from '../../elastic/pages'
 import { UserDeviceToken } from '../../entity/user_device_tokens'
 import { homePageURL } from '../../env'
@@ -234,7 +234,7 @@ const updateRemindersStatus = async (
   }
 
   // db update
-  await AppDataSource.transaction(async (tx) => {
+  await appDataSource.transaction(async (tx) => {
     await setClaims(tx, userId)
     await setRemindersComplete(tx, userId, remindAt)
   })

packages/api/src/routers/text_to_speech.ts

@@ -4,7 +4,7 @@
 import { htmlToSpeechFile } from '@omnivore/text-to-speech-handler'
 import cors from 'cors'
 import express from 'express'
-import { AppDataSource } from '../data-source'
+import { appDataSource } from '../data_source'
 import { getPageById } from '../elastic/pages'
 import { ArticleSavingRequestStatus } from '../elastic/types'
 import { Speech, SpeechState } from '../entity/speech'
@@ -151,7 +151,7 @@ export function textToSpeechRouter() {
     }
 
     // set state to completed
-    await AppDataSource.transaction(async (t) => {
+    await appDataSource.transaction(async (t) => {
       await setClaims(t, userId)
       await t.getRepository(Speech).update(speechId, {
         audioFileName: audioFileName,

packages/api/src/server.ts

@@ -13,8 +13,7 @@ import rateLimit from 'express-rate-limit'
 import { createServer, Server } from 'http'
 import { config, loggers } from 'winston'
 import { makeApolloServer } from './apollo'
-import { AppDataSource } from './data-source'
-import { initElasticsearch } from './elastic'
+import { appDataSource } from './data_source'
 import { env } from './env'
 import { articleRouter } from './routers/article_router'
 import { authRouter } from './routers/auth/auth_router'
@@ -144,9 +143,7 @@ const main = async (): Promise<void> => {
   // If creating the DB entities fails, we want this to throw
   // so the container will be restarted and not come online
   // as healthy.
-  await AppDataSource.initialize()
-
-  await initElasticsearch()
+  await appDataSource.initialize()
 
   const { app, apollo, httpServer } = createApp()
 

packages/api/src/services/archive_link.ts

@@ -1,4 +1,4 @@
-import { AppDataSource } from '../data-source'
+import { appDataSource } from '../data_source'
 import { Link } from '../entity/link'
 import { setClaims } from '../repository'
 
@@ -7,7 +7,7 @@ export const setLinkArchived = async (
   linkId: string,
   archived: boolean
 ): Promise<void> => {
-  await AppDataSource.transaction(async (t) => {
+  await appDataSource.transaction(async (t) => {
     await setClaims(t, userId)
     await t.getRepository(Link).update(
       {

packages/api/src/services/create_user.ts

@@ -1,11 +1,12 @@
-import { EntityManager, In } from 'typeorm'
-import { AppDataSource } from '../data-source'
+import { EntityManager } from 'typeorm'
+import { appDataSource } from '../data_source'
 import { GroupMembership } from '../entity/groups/group_membership'
 import { Invite } from '../entity/groups/invite'
 import { Profile } from '../entity/profile'
 import { StatusType, User } from '../entity/user'
 import { SignupErrorCode } from '../generated/graphql'
-import { getRepository, userRepository } from '../repository'
+import { getRepository } from '../repository'
+import { getUserByEmail } from '../repository/user'
 import { AuthProvider } from '../routers/auth/auth_types'
 import { logger } from '../utils/logger'
 import { validateUsername } from '../utils/usernamePolicy'
@@ -71,7 +72,7 @@ export const createUser = async (input: {
     return Promise.reject({ errorCode: SignupErrorCode.InvalidUsername })
   }
 
-  const [user, profile] = await AppDataSource.transaction<[User, Profile]>(
+  const [user, profile] = await appDataSource.transaction<[User, Profile]>(
     async (t) => {
       let hasInvite = false
       let invite: Invite | null = null
@@ -158,7 +159,7 @@ const createDefaultFiltersForUser =
     return t.getRepository(Filter).save(defaultFilters)
   }
 
-// TODO: Maybe this should be moved into a service
+// Maybe this should be moved into a service
 const validateInvite = async (
   entityManager: EntityManager,
   invite: Invite
@@ -175,25 +176,3 @@ const validateInvite = async (
   }
   return true
 }
-
-export const getUserByEmail = async (email: string): Promise<User | null> => {
-  return userRepository
-    .createQueryBuilder('user')
-    .leftJoinAndSelect('user.profile', 'profile')
-    .where('LOWER(email) = LOWER(:email)', { email }) // case insensitive
-    .getOne()
-}
-
-export const getTopUsers = async (): Promise<User[]> => {
-  return userRepository
-    .createQueryBuilder()
-    .where({
-      profile: { username: In(TOP_USERS) },
-    })
-    .limit(MAX_RECORDS_LIMIT)
-    .getMany()
-}
-
-export const getUserById = async (id: string): Promise<User | null> => {
-  return userRepository.findOneBy({ id })
-}

packages/api/src/services/features.ts

@@ -1,6 +1,6 @@
 import * as jwt from 'jsonwebtoken'
 import { IsNull, Not } from 'typeorm'
-import { AppDataSource } from '../data-source'
+import { appDataSource } from '../data_source'
 import { Feature } from '../entity/feature'
 import { env } from '../env'
 import { getRepository } from '../repository'
@@ -42,7 +42,7 @@ const optInUltraRealisticVoice = async (uid: string): Promise<Feature> => {
 
   const MAX_USERS = 1500
   // opt in to feature for the first 1500 users
-  const optedInFeatures = (await AppDataSource.query(
+  const optedInFeatures = (await appDataSource.query(
     `insert into omnivore.features (user_id, name, granted_at) 
     select $1, $2, $3 from omnivore.features 
     where name = $2 and granted_at is not null 

packages/api/src/services/groups.ts

@@ -1,5 +1,5 @@
 import { nanoid } from 'nanoid'
-import { AppDataSource } from '../data-source'
+import { appDataSource } from '../data_source'
 import { Group } from '../entity/groups/group'
 import { GroupMembership } from '../entity/groups/group_membership'
 import { Invite } from '../entity/groups/invite'
@@ -22,7 +22,7 @@ export const createGroup = async (input: {
   onlyAdminCanPost?: boolean | null
   onlyAdminCanSeeMembers?: boolean | null
 }): Promise<[Group, Invite]> => {
-  const [group, invite] = await AppDataSource.transaction<[Group, Invite]>(
+  const [group, invite] = await appDataSource.transaction<[Group, Invite]>(
     async (t) => {
       // Max number of groups a user can create
       const maxGroups = 3
@@ -114,7 +114,7 @@ export const joinGroup = async (
   user: User,
   inviteCode: string
 ): Promise<RecommendationGroup> => {
-  const invite = await AppDataSource.transaction<Invite>(async (t) => {
+  const invite = await appDataSource.transaction<Invite>(async (t) => {
     // Check if the invite exists
     const invite = await t
       .getRepository(Invite)
@@ -175,7 +175,7 @@ export const leaveGroup = async (
   user: User,
   groupId: string
 ): Promise<boolean> => {
-  return AppDataSource.transaction(async (t) => {
+  return appDataSource.transaction(async (t) => {
     const group = await t
       .getRepository(Group)
       .createQueryBuilder('group')

packages/api/src/services/labels.ts

@@ -6,7 +6,7 @@ import { Label } from '../entity/label'
 import { Link } from '../entity/link'
 import { User } from '../entity/user'
 import { CreateLabelInput } from '../generated/graphql'
-import { getRepository } from '../repository'
+import { getRepository, labelRepository } from '../repository'
 import { generateRandomColor } from '../utils/helpers'
 import { logger } from '../utils/logger'
 
@@ -124,7 +124,7 @@ export const createLabels = async (
   ctx: PageContext,
   labels: CreateLabelInput[]
 ): Promise<Label[]> => {
-  const labelEntities = await getRepository(Label)
+  const labelEntities = await labelRepository
     .createQueryBuilder()
     .where({
       user: { id: ctx.uid },
@@ -141,7 +141,7 @@ export const createLabels = async (
     (l) => !existingLabelsInLowerCase.includes(l.name.toLowerCase())
   )
   // create new labels
-  const newLabelEntities = await getRepository(Label).save(
+  const newLabelEntities = await labelRepository.save(
     newLabels.map((l) => ({
       name: l.name,
       description: l.description,

packages/api/src/services/save_file.ts

@@ -1,4 +1,3 @@
-import { AppDataSource } from '../data-source'
 import { updatePage } from '../elastic/pages'
 import { UploadFile } from '../entity/upload_file'
 import { User } from '../entity/user'
@@ -9,7 +8,7 @@ import {
   SaveFileInput,
   SaveResult,
 } from '../generated/graphql'
-import { getRepository } from '../repository'
+import { entityManager, getRepository } from '../repository'
 import { WithDataSourcesContext } from '../resolvers/types'
 import { logger } from '../utils/logger'
 import { getStorageFileDetails } from '../utils/uploads'
@@ -17,7 +16,7 @@ import { createLabels } from './labels'
 
 export const setFileUploadComplete = async (
   id: string,
-  em = AppDataSource.createEntityManager()
+  em = entityManager
 ): Promise<UploadFile | null> => {
   return em.getRepository(UploadFile).save({ id, status: 'COMPLETED' })
 }

packages/api/src/services/user_device_tokens.ts

@@ -1,4 +1,4 @@
-import { AppDataSource } from '../data-source'
+import { appDataSource } from '../data_source'
 import { User } from '../entity/user'
 import { UserDeviceToken } from '../entity/user_device_tokens'
 import { env } from '../env'
@@ -70,7 +70,7 @@ export const deleteDeviceToken = async (
     },
   })
 
-  return AppDataSource.transaction(async (t) => {
+  return appDataSource.transaction(async (t) => {
     await setClaims(t, userId)
     const result = await t.getRepository(UserDeviceToken).delete(id)
 

packages/api/test/db.ts

@@ -1,7 +1,7 @@
 import Postgrator from 'postgrator'
 import { FindOptionsWhere } from 'typeorm'
 import { SnakeNamingStrategy } from 'typeorm-naming-strategies'
-import { getRepository, setClaims, userRepository } from '../src/repository'
+import { appDataSource } from '../src/data_source'
 import { Integration } from '../src/entity/integration'
 import { Label } from '../src/entity/label'
 import { Link } from '../src/entity/link'
@@ -13,7 +13,7 @@ import { Subscription } from '../src/entity/subscription'
 import { User } from '../src/entity/user'
 import { UserDeviceToken } from '../src/entity/user_device_tokens'
 import { SubscriptionStatus, SubscriptionType } from '../src/generated/graphql'
-import { AppDataSource } from '../src/data-source'
+import { getRepository, setClaims, userRepository } from '../src/repository'
 import { createUser } from '../src/services/create_user'
 import { Filter } from "../src/entity/filter"
 
@@ -48,7 +48,7 @@ export const createTestConnection = async (): Promise<void> => {
   // need to manually run migrations before creating the connection
   // await runMigrations()
 
-  AppDataSource.setOptions({
+  appDataSource.setOptions({
     type: 'postgres',
     host: process.env.PG_HOST,
     port: Number(process.env.PG_PORT),
@@ -61,11 +61,11 @@ export const createTestConnection = async (): Promise<void> => {
     subscribers: [__dirname + '/../src/events/**/*{.js,.ts}'],
     namingStrategy: new SnakeNamingStrategy(),
   })
-  await AppDataSource.initialize()
+  await appDataSource.initialize()
 }
 
 export const deleteTestUser = async (userId: string) => {
-  await AppDataSource.transaction(async (t) => {
+  await appDataSource.transaction(async (t) => {
     await setClaims(t, userId)
     await t.getRepository(User).delete(userId)
   })
@@ -231,7 +231,7 @@ export const deleteTestLabels = async (
   userId: string,
   criteria: string[] | FindOptionsWhere<Label>
 ) => {
-  await AppDataSource.transaction(async (t) => {
+  await appDataSource.transaction(async (t) => {
     await setClaims(t, userId)
     await t.getRepository(Label).delete(criteria)
   })
@@ -241,14 +241,14 @@ export const deleteTestIntegrations = async (
   userId: string,
   criteria: string[] | FindOptionsWhere<Integration>
 ) => {
-  await AppDataSource.transaction(async (t) => {
+  await appDataSource.transaction(async (t) => {
     await setClaims(t, userId)
     await t.getRepository(Integration).delete(criteria)
   })
 }
 
 export const updateTestUser = async (userId: string, update: Partial<User>) => {
-  await AppDataSource.transaction(async (t) => {
+  await appDataSource.transaction(async (t) => {
     await setClaims(t, userId)
     await t.getRepository(User).update(userId, update)
   })
@@ -258,7 +258,7 @@ export const deleteTestDeviceTokens = async (
   userId: string,
   criteria: string[] | FindOptionsWhere<UserDeviceToken>
 ) => {
-  await AppDataSource.transaction(async (t) => {
+  await appDataSource.transaction(async (t) => {
     await setClaims(t, userId)
     await t.getRepository(UserDeviceToken).delete(criteria)
   })

packages/api/test/global-setup.ts

@@ -1,14 +1,10 @@
 import { createTestConnection } from './db'
-import { initElasticsearch } from '../src/elastic'
 import { startApolloServer } from './util'
 
 export const mochaGlobalSetup = async () => {
   await createTestConnection()
   console.log('db connection created')
 
-  await initElasticsearch()
-  console.log('elasticsearch initialized')
-
   await startApolloServer()
   console.log('apollo server started')
 }

packages/db/Dockerfile

@@ -14,6 +14,5 @@ RUN yarn install
 
 ADD /packages/db ./packages/db
 ADD /packages/db/setup.sh ./packages/db/setup.sh
-ADD /packages/db/elastic_migrations ./packages/db/elastic_migrations
 
 CMD ["yarn", "workspace", "@omnivore/db", "migrate"]

packages/db/migrate.ts

@@ -1,10 +1,7 @@
 /* eslint-disable @typescript-eslint/naming-convention */
+import chalk from 'chalk'
 import * as dotenv from 'dotenv'
 import Postgrator from 'postgrator'
-import chalk from 'chalk'
-import { Client } from '@elastic/elasticsearch'
-import { readFileSync } from 'fs'
-import { join } from 'path'
 
 dotenv.config()
 
@@ -86,37 +83,6 @@ const logAppliedMigrations = (
   }
 }
 
-export const INDEX_ALIAS = 'pages_alias'
-export const esClient = new Client({
-  node: process.env.ELASTIC_URL || 'http://localhost:9200',
-  requestTimeout: 60000 * 30, // 30 minutes
-  auth: {
-    username: process.env.ELASTIC_USERNAME || '',
-    password: process.env.ELASTIC_PASSWORD || '',
-  },
-})
-// read index settings from file
-const indexSettings = readFileSync(
-  join(__dirname, 'elastic_migrations', 'index_settings.json'),
-  'utf8'
-)
-const INDEX_NAME = 'pages'
-const createIndex = async (): Promise<void> => {
-  // create index
-  await esClient.indices.create({
-    index: INDEX_NAME,
-    body: JSON.parse(indexSettings),
-  })
-}
-
-const updateMappings = async (): Promise<void> => {
-  // update mappings
-  await esClient.indices.putMapping({
-    index: INDEX_ALIAS,
-    body: JSON.parse(indexSettings).mappings,
-  })
-}
-
 // postgres migration
 const postgresMigration = postgrator
   .migrate(targetMigration)
@@ -131,26 +97,4 @@ const postgresMigration = postgrator
     process.exit(1)
   })
 
-// elastic migration
-log('Creating elastic index...')
-const elasticMigration = esClient.indices
-  .exists({ index: INDEX_ALIAS })
-  .then(({ body: exists }) => {
-    if (!exists) {
-      return createIndex().then(() => log('Elastic index created.'))
-    } else {
-      log('Elastic index already exists.')
-    }
-  })
-  .then(() => {
-    log('Updating elastic index mappings...')
-    return updateMappings().then(() => {
-      log('Elastic index mappings updated.')
-    })
-  })
-  .catch((error) => {
-    log(`${chalk.red('Elastic migration failed: ')}${error.message}`, chalk.red)
-    process.exit(1)
-  })
-
-Promise.all([postgresMigration, elasticMigration]).then(() => log('Exiting...'))
+postgresMigration.then(() => log('Exiting...'))