replace saveEmail

packages/api/src/datalayer/links/share_info.ts

@@ -2,7 +2,6 @@
 /* eslint-disable @typescript-eslint/no-unsafe-assignment */
 import { Knex } from 'knex'
 import { LinkShareInfo } from '../../generated/graphql'
-import { DataModels } from '../../resolvers/types'
 import { getPageByParam } from '../../elastic/pages'
 
 // once we have links setup properly in the API we will remove this method
@@ -10,8 +9,7 @@ import { getPageByParam } from '../../elastic/pages'
 export const getShareInfoForArticle = async (
   kx: Knex,
   userId: string,
-  articleId: string,
-  models: DataModels
+  articleId: string
 ): Promise<LinkShareInfo | undefined> => {
   // TEMP: because the old API uses articles instead of Links, we are actually
   // getting an article ID here and need to map it to a link ID. When the API

packages/api/src/entity/library_item.ts

@@ -7,6 +7,7 @@ import {
   ManyToMany,
   OneToOne,
   PrimaryGeneratedColumn,
+  Unique,
   UpdateDateColumn,
 } from 'typeorm'
 import { Label } from './label'
@@ -45,10 +46,11 @@ export enum DirectionalityType {
   RTL = 'RTL',
 }
 
+@Unique('library_item_user_original_url', ['user', 'originalUrl'])
 @Entity({ name: 'library_item' })
 export class LibraryItem {
   @PrimaryGeneratedColumn('uuid')
-  id?: string
+  id!: string
 
   @OneToOne(() => User, { onDelete: 'CASCADE' })
   @JoinColumn({ name: 'user_id' })
@@ -58,7 +60,7 @@ export class LibraryItem {
     enum: LibraryItemState,
     default: LibraryItemState.Succeeded,
   })
-  state?: LibraryItemState
+  state!: LibraryItemState
 
   @Column('text')
   originalUrl!: string
@@ -114,36 +116,36 @@ export class LibraryItem {
   @Column('json', { nullable: true })
   metadata?: Record<string, unknown> | null
 
-  @Column('integer', { nullable: true })
-  readingProgressLastReadAnchor?: number | null
+  @Column('integer')
+  readingProgressLastReadAnchor!: number
 
-  @Column('integer', { nullable: true })
-  readingProgressHighestReadAnchor?: number | null
+  @Column('integer')
+  readingProgressHighestReadAnchor!: number
 
-  @Column('real', { nullable: true })
-  readingProgressTopPercent?: number | null
+  @Column('real')
+  readingProgressTopPercent!: number
 
-  @Column('real', { nullable: true })
-  readingProgressBottomPercent?: number | null
+  @Column('real')
+  readingProgressBottomPercent!: number
 
   @Column('text', { nullable: true })
   thumbnail?: string | null
 
   @Column('enum', { enum: LibraryItemType, default: LibraryItemType.Unknown })
-  itemType?: LibraryItemType
+  itemType!: LibraryItemType
 
   @OneToOne(() => UploadFile, { onDelete: 'CASCADE' })
   @JoinColumn({ name: 'upload_file_id' })
   uploadFile?: UploadFile
 
   @Column('enum', { enum: ContentReaderType, default: ContentReaderType.WEB })
-  contentReader?: ContentReaderType
+  contentReader!: ContentReaderType
 
   @Column('text', { nullable: true })
   originalContent?: string | null
 
-  @Column('text', { nullable: true })
-  readableContent?: string | null
+  @Column('text')
+  readableContent!: string
 
   @Column('text', { nullable: true })
   modelName?: string | null
@@ -158,11 +160,11 @@ export class LibraryItem {
   @Column('text', { nullable: true })
   gcsArchiveId?: string | null
 
-  @OneToOne(() => Subscription, { onDelete: 'CASCADE' })
+  @OneToOne(() => Subscription, { onDelete: 'CASCADE', eager: true })
   @JoinColumn({ name: 'subscription_id' })
   subscription?: Subscription
 
-  @ManyToMany(() => Label)
+  @ManyToMany(() => Label, { eager: true })
   @JoinTable()
   labels?: Label[]
 

packages/api/src/entity/subscription.ts

@@ -46,7 +46,7 @@ export class Subscription {
   unsubscribeHttpUrl?: string
 
   @Column('text', { nullable: true })
-  icon?: string
+  icon?: string | null
 
   @Column('enum', {
     enum: SubscriptionType,

packages/api/src/repository/index.ts

@@ -1,5 +1,6 @@
 import { EntityManager, EntityTarget, Repository } from 'typeorm'
 import { AppDataSource } from '../data-source'
+import { LibraryItem } from '../entity/library_item'
 import { Reminder } from '../entity/reminder'
 import { UploadFile } from '../entity/upload_file'
 import { User } from '../entity/user'
@@ -15,9 +16,12 @@ export const setClaims = async (
 }
 
 export const getRepository = <T>(entity: EntityTarget<T>): Repository<T> => {
-  return AppDataSource.getRepository(entity)
+  return entityManager.getRepository(entity)
 }
 
+export const entityManager = AppDataSource.createEntityManager()
+
 export const userRepository = getRepository(User)
 export const uploadFileRepository = getRepository(UploadFile)
 export const reminderRepository = getRepository(Reminder)
+export const libraryItemRepository = getRepository(LibraryItem)

packages/api/src/resolvers/recent_emails/index.ts

@@ -13,7 +13,7 @@ import {
 } from '../../generated/graphql'
 import { getRepository } from '../../repository'
 import { updateReceivedEmail } from '../../services/received_emails'
-import { saveNewsletterEmail } from '../../services/save_newsletter_email'
+import { saveNewsletter } from '../../services/save_newsletter_email'
 import { authorized } from '../../utils/helpers'
 import { generateUniqueUrl, parseEmailAddress } from '../../utils/parser'
 import { sendEmail } from '../../utils/sendEmail'
@@ -103,7 +103,7 @@ export const markEmailAsItemResolver = authorized<
       }
     }
 
-    const success = await saveNewsletterEmail(
+    const success = await saveNewsletter(
       {
         from: recentEmail.from,
         email: recentEmail.to,

packages/api/src/routers/svc/emails.ts

@@ -1,13 +1,10 @@
 import cors from 'cors'
 import express from 'express'
 import { env } from '../../env'
-import { createPubSubClient, readPushSubscription } from '../../pubsub'
+import { readPushSubscription } from '../../pubsub'
 import { getNewsletterEmail } from '../../services/newsletters'
-import {
-  saveReceivedEmail,
-  updateReceivedEmail,
-} from '../../services/received_emails'
-import { saveEmail } from '../../services/save_email'
+import { saveReceivedEmail } from '../../services/received_emails'
+import { saveNewsletter } from '../../services/save_newsletter_email'
 import { analytics } from '../../utils/analytics'
 import { getClaimsByToken } from '../../utils/auth'
 import { corsConfig } from '../../utils/corsConfig'
@@ -73,7 +70,6 @@ export function emailsServiceRouter() {
         return
       }
       const user = newsletterEmail.user
-      const ctx = { pubsub: createPubSubClient(), uid: user.id }
       const parsedFrom = parseEmailAddress(data.from)
 
       if (
@@ -83,15 +79,21 @@ export function emailsServiceRouter() {
         )
       ) {
         logger.info('handling as article')
-        await saveEmail(ctx, {
-          title: getTitleFromEmailSubject(data.subject),
-          author: parsedFrom.name,
-          url: generateUniqueUrl(),
-          originalContent: data.html || data.text,
-        })
-
-        // update received email type
-        await updateReceivedEmail(data.receivedEmailId, 'article')
+        const savedNewsletter = await saveNewsletter(
+          {
+            title: getTitleFromEmailSubject(data.subject),
+            author: parsedFrom.name,
+            url: generateUniqueUrl(),
+            content: data.html || data.text,
+            receivedEmailId: data.receivedEmailId,
+            email: newsletterEmail.address,
+          },
+          newsletterEmail
+        )
+        if (!savedNewsletter) {
+          logger.info('Failed to save email')
+          return res.status(500).send('Failed to save email')
+        }
 
         res.status(200).send('Article')
         return

packages/api/src/routers/svc/newsletters.ts

@@ -8,7 +8,7 @@ import {
 import { updateReceivedEmail } from '../../services/received_emails'
 import {
   NewsletterMessage,
-  saveNewsletterEmail,
+  saveNewsletter,
 } from '../../services/save_newsletter_email'
 import { saveUrlFromEmail } from '../../services/save_url'
 import { getSubscriptionByNameAndUserId } from '../../services/subscriptions'
@@ -137,7 +137,7 @@ export function newsletterServiceRouter() {
         }
 
         // save newsletter instead
-        const result = await saveNewsletterEmail(data, newsletterEmail, saveCtx)
+        const result = await saveNewsletter(data, newsletterEmail)
         if (!result) {
           logger.info('Error creating newsletter link from data', data)
 

packages/api/src/services/labels.ts

@@ -10,15 +10,22 @@ import { getRepository } from '../repository'
 import { generateRandomColor } from '../utils/helpers'
 import { logger } from '../utils/logger'
 
-const INTERNAL_LABELS_IN_LOWERCASE = [
-  'newsletters',
-  'favorites',
-  'rss',
-  'library',
-]
+const INTERNAL_LABELS_WITH_COLOR = new Map<
+  string,
+  { name: string; color: string }
+>([
+  ['favorites', { name: 'Favorites', color: '#FFD700' }],
+  ['library', { name: 'Library', color: '#584C42' }],
+  ['rss', { name: 'RSS', color: '#F26522' }],
+  ['newsletter', { name: 'Newsletter', color: '#07D2D1' }],
+])
+
+export const getInternalLabelWithColor = (name: string) => {
+  return INTERNAL_LABELS_WITH_COLOR.get(name.toLowerCase())
+}
 
 const isLabelInternal = (name: string): boolean => {
-  return INTERNAL_LABELS_IN_LOWERCASE.includes(name.toLowerCase())
+  return INTERNAL_LABELS_WITH_COLOR.has(name.toLowerCase())
 }
 
 const batchGetLabelsFromLinkIds = async (

packages/api/src/services/library_item.ts

@@ -0,0 +1,32 @@
+import { DeepPartial } from 'typeorm'
+import { LibraryItem } from '../entity/library_item'
+import { entityManager } from '../repository'
+import { wordsCount } from '../utils/helpers'
+import { logger } from '../utils/logger'
+
+const MAX_CONTENT_LENGTH = 10 * 1024 * 1024 // 10MB for readable content
+const CONTENT_LENGTH_ERROR = 'Your page content is too large to be saved.'
+
+export const createLibraryItem = async (
+  libraryItem: DeepPartial<LibraryItem>,
+  em = entityManager
+): Promise<LibraryItem> => {
+  if (
+    libraryItem.readableContent &&
+    libraryItem.readableContent.length > MAX_CONTENT_LENGTH
+  ) {
+    logger.info('page content is too large', {
+      url: libraryItem.originalUrl,
+      contentLength: libraryItem.readableContent.length,
+    })
+
+    libraryItem.readableContent = CONTENT_LENGTH_ERROR
+  }
+
+  return em.getRepository(LibraryItem).save({
+    ...libraryItem,
+    savedAt: libraryItem.savedAt || new Date(),
+    wordCount:
+      libraryItem.wordCount ?? wordsCount(libraryItem.readableContent ?? ''),
+  })
+}

packages/api/src/services/received_emails.ts

@@ -1,5 +1,5 @@
 import { ReceivedEmail } from '../entity/received_email'
-import { getRepository } from '../repository'
+import { entityManager, getRepository } from '../repository'
 
 export const saveReceivedEmail = async (
   from: string,
@@ -23,7 +23,8 @@ export const saveReceivedEmail = async (
 
 export const updateReceivedEmail = async (
   id: string,
-  type: 'article' | 'non-article'
+  type: 'article' | 'non-article',
+  em = entityManager
 ) => {
-  await getRepository(ReceivedEmail).update(id, { type })
+  await em.getRepository(ReceivedEmail).update(id, { type })
 }

packages/api/src/services/save_email.ts

@@ -1,10 +1,14 @@
-import { createPage, getPageByParam, updatePage } from '../elastic/pages'
-import { ArticleSavingRequestStatus, Page } from '../elastic/types'
-import { PubsubClient } from '../pubsub'
+import {
+  LibraryItem,
+  LibraryItemState,
+  LibraryItemType,
+} from '../entity/library_item'
+import { entityManager, libraryItemRepository } from '../repository'
 import { enqueueThumbnailTask } from '../utils/createTask'
 import {
   cleanUrl,
   generateSlug,
+  isBase64Image,
   stringToHash,
   validatedDate,
   wordsCount,
@@ -12,23 +16,24 @@ import {
 import { logger } from '../utils/logger'
 import {
   FAKE_URL_PREFIX,
+  fetchFavicon,
   parsePreparedContent,
   parseUrlMetadata,
 } from '../utils/parser'
-
-export type SaveContext = {
-  pubsub: PubsubClient
-  uid: string
-  refresh?: boolean
-}
+import { getInternalLabelWithColor } from './labels'
+import { createLibraryItem } from './library_item'
+import { updateReceivedEmail } from './received_emails'
 
 export type SaveEmailInput = {
+  userId: string
   originalContent: string
   url: string
   title: string
   author: string
   unsubMailTo?: string
   unsubHttpUrl?: string
+  newsletterEmailId?: string
+  receivedEmailId: string
 }
 
 const isStubUrl = (url: string): boolean => {
@@ -36,9 +41,8 @@ const isStubUrl = (url: string): boolean => {
 }
 
 export const saveEmail = async (
-  ctx: SaveContext,
   input: SaveEmailInput
-): Promise<Page | undefined> => {
+): Promise<LibraryItem | undefined> => {
   const url = input.url
   const parseResult = await parsePreparedContent(
     url,
@@ -51,68 +55,92 @@ export const saveEmail = async (
     null,
     true
   )
+
   const content = parseResult.parsedContent?.content || input.originalContent
   const slug = generateSlug(input.title)
   const metadata = isStubUrl(url) ? undefined : await parseUrlMetadata(url)
-
-  const articleToSave: Page = {
-    id: '',
-    userId: ctx.uid,
-    slug,
-    content,
-    originalHtml: input.originalContent,
-    description: metadata?.description || parseResult.parsedContent?.excerpt,
-    title: input.title,
-    author: input.author,
-    url: cleanUrl(parseResult.canonicalUrl || url),
-    pageType: parseResult.pageType,
-    hash: stringToHash(content),
-    image:
-      metadata?.previewImage ||
-      parseResult.parsedContent?.previewImage ||
-      undefined,
-    publishedAt: validatedDate(
-      parseResult.parsedContent?.publishedDate ?? undefined
-    ),
-    createdAt: new Date(),
-    savedAt: new Date(),
-    readingProgressAnchorIndex: 0,
-    readingProgressPercent: 0,
-    subscription: input.author,
-    state: ArticleSavingRequestStatus.Succeeded,
-    siteIcon: parseResult.parsedContent?.siteIcon ?? undefined,
-    siteName: parseResult.parsedContent?.siteName ?? undefined,
-    wordsCount: wordsCount(content),
+  const cleanedUrl = cleanUrl(parseResult.canonicalUrl || url)
+  let siteIcon = parseResult.parsedContent?.siteIcon
+  if (!siteIcon || isBase64Image(siteIcon)) {
+    // fetch favicon if not already set or is a base64 image
+    siteIcon = await fetchFavicon(url)
   }
 
-  const page = await getPageByParam({
-    userId: ctx.uid,
-    url: articleToSave.url,
-    state: ArticleSavingRequestStatus.Succeeded,
+  const existingLibraryItem = await libraryItemRepository.findOneBy({
+    user: { id: input.userId },
+    originalUrl: cleanedUrl,
+    state: LibraryItemState.Succeeded,
   })
-  if (page) {
-    const result = await updatePage(page.id, { archivedAt: null }, ctx)
-    logger.info('updated page from email', result)
+  if (existingLibraryItem) {
+    const updatedLibraryItem = await libraryItemRepository.save({
+      ...existingLibraryItem,
+      archivedAt: null,
+    })
+    logger.info('updated page from email', updatedLibraryItem)
 
-    return page
+    return updatedLibraryItem
   }
 
-  const pageId = await createPage(articleToSave, ctx)
-  if (!pageId) {
-    logger.info('failed to create new page')
+  const newsletterLabel = getInternalLabelWithColor('newsletter')
 
-    return undefined
-  }
+  // start a transaction to create the library item and update the received email
+  const newLibraryItem = await entityManager.transaction(async (tx) => {
+    const newLibraryItem = await createLibraryItem(
+      {
+        user: { id: input.userId },
+        slug,
+        readableContent: content,
+        originalContent: input.originalContent,
+        description:
+          metadata?.description || parseResult.parsedContent?.excerpt,
+        title: input.title,
+        author: input.author,
+        originalUrl: cleanedUrl,
+        itemType: parseResult.pageType as unknown as LibraryItemType,
+        textContentHash: stringToHash(content),
+        thumbnail:
+          metadata?.previewImage ||
+          parseResult.parsedContent?.previewImage ||
+          undefined,
+        publishedAt: validatedDate(
+          parseResult.parsedContent?.publishedDate ?? undefined
+        ),
+        subscription: {
+          name: input.author,
+          unsubscribeMailTo: input.unsubMailTo,
+          unsubscribeHttpUrl: input.unsubHttpUrl,
+          user: { id: input.userId },
+          newsletterEmail: { id: input.newsletterEmailId },
+          icon: siteIcon,
+          lastFetchedAt: new Date(),
+        },
+        state: LibraryItemState.Succeeded,
+        siteIcon,
+        siteName: parseResult.parsedContent?.siteName ?? undefined,
+        wordCount: wordsCount(content),
+        labels: [
+          {
+            ...newsletterLabel,
+            internal: true,
+            user: { id: input.userId },
+          },
+        ],
+      },
+      tx
+    )
+
+    await updateReceivedEmail(input.receivedEmailId, 'article', tx)
+
+    return newLibraryItem
+  })
 
   // create a task to update thumbnail and pre-cache all images
   try {
-    const taskId = await enqueueThumbnailTask(ctx.uid, slug)
+    const taskId = await enqueueThumbnailTask(input.userId, slug)
     logger.info('Created thumbnail task', { taskId })
   } catch (e) {
     logger.error('Failed to create thumbnail task', e)
   }
 
-  articleToSave.id = pageId
-
-  return articleToSave
+  return newLibraryItem
 }

packages/api/src/services/save_newsletter_email.ts

@@ -1,19 +1,12 @@
 import { MulticastMessage } from 'firebase-admin/messaging'
-import { updatePage } from '../elastic/pages'
 import { Page } from '../elastic/types'
 import { NewsletterEmail } from '../entity/newsletter_email'
 import { UserDeviceToken } from '../entity/user_device_tokens'
 import { env } from '../env'
 import { ContentReader } from '../generated/graphql'
-import { createPubSubClient } from '../pubsub'
 import { analytics } from '../utils/analytics'
-import { isBase64Image } from '../utils/helpers'
 import { logger } from '../utils/logger'
-import { fetchFavicon } from '../utils/parser'
-import { addLabelToPage } from './labels'
-import { SaveContext, saveEmail, SaveEmailInput } from './save_email'
-import { saveSubscription } from './subscriptions'
-import { getDeviceTokensByUserId } from './user_device_tokens'
+import { saveEmail, SaveEmailInput } from './save_email'
 
 export interface NewsletterMessage {
   email: string
@@ -29,10 +22,9 @@ export interface NewsletterMessage {
 
 // Returns true if the link was created successfully. Can still fail to
 // send the push but that is ok and we wont retry in that case.
-export const saveNewsletterEmail = async (
+export const saveNewsletter = async (
   data: NewsletterMessage,
-  newsletterEmail: NewsletterEmail,
-  ctx?: SaveContext
+  newsletterEmail: NewsletterEmail
 ): Promise<boolean> => {
   analytics.track({
     userId: newsletterEmail.user.id,
@@ -50,58 +42,30 @@ export const saveNewsletterEmail = async (
     return false
   }
 
-  const saveCtx = ctx || {
-    pubsub: createPubSubClient(),
-    uid: newsletterEmail.user.id,
-  }
   const input: SaveEmailInput = {
+    userId: newsletterEmail.user.id,
     url: data.url,
     originalContent: data.content,
     title: data.title,
     author: data.author,
     unsubMailTo: data.unsubMailTo,
     unsubHttpUrl: data.unsubHttpUrl,
+    newsletterEmailId: newsletterEmail.id,
+    receivedEmailId: data.receivedEmailId,
   }
-  const page = await saveEmail(saveCtx, input)
-  if (!page) {
+  const savedLibraryItem = await saveEmail(input)
+  if (!savedLibraryItem) {
     logger.info('newsletter not created:', input.title)
 
     return false
   }
 
-  let icon = page.siteIcon
-  if (!icon || isBase64Image(icon)) {
-    // fetch favicon if not already set or is a base64 image
-    icon = await fetchFavicon(page.url)
-    if (icon) {
-      await updatePage(page.id, { siteIcon: icon }, saveCtx)
-    }
-  }
-
-  const subscriptionId = await saveSubscription({
-    userId: newsletterEmail.user.id,
-    name: data.author,
-    newsletterEmail,
-    unsubscribeMailTo: data.unsubMailTo,
-    unsubscribeHttpUrl: data.unsubHttpUrl,
-    icon,
-  })
-  logger.info('subscription saved', subscriptionId)
-
-  // adds newsletters label to page
-  const result = await addLabelToPage(saveCtx, page.id, {
-    name: 'Newsletter',
-    color: '#07D2D1',
-  })
-
-  logger.info('newsletter label added', { result })
-
   // sends push notification
-  const deviceTokens = await getDeviceTokensByUserId(newsletterEmail.user.id)
-  if (!deviceTokens) {
-    logger.info('Device tokens not set:', newsletterEmail.user.id)
-    return true
-  }
+  // const deviceTokens = await getDeviceTokensByUserId(newsletterEmail.user.id)
+  // if (!deviceTokens) {
+  //   logger.info('Device tokens not set:', newsletterEmail.user.id)
+  //   return true
+  // }
 
   // const multicastMessage = messageForLink(page, deviceTokens)
   // await sendMulticastPushNotifications(

packages/api/test/services/save_email.test.ts

@@ -1,11 +1,10 @@
-import 'mocha'
 import { expect } from 'chai'
-import { createTestUser, deleteTestUser } from '../db'
-import { SaveContext, saveEmail } from '../../src/services/save_email'
-import { createPubSubClient } from '../../src/pubsub'
-import { getPageByParam } from '../../src/elastic/pages'
+import 'mocha'
 import nock from 'nock'
+import { getPageByParam } from '../../src/elastic/pages'
 import { User } from '../../src/entity/user'
+import { saveEmail } from '../../src/services/save_email'
+import { createTestUser, deleteTestUser } from '../db'
 
 describe('saveEmail', () => {
   const fakeContent = 'fake content'
@@ -30,26 +29,25 @@ describe('saveEmail', () => {
     const url = 'https://blog.omnivore.app/fake-url'
     const title = 'fake title'
     const author = 'fake author'
-    const ctx: SaveContext = {
-      pubsub: createPubSubClient(),
-      uid: user.id,
-      refresh: true,
-    }
 
-    await saveEmail(ctx, {
+    await saveEmail({
       originalContent: `<html><body>${fakeContent}</body></html>`,
       url,
       title,
       author,
+      userId: user.id,
+      receivedEmailId: 'fakeId',
     })
 
     // This ensures row level security doesnt prevent
     // saving the same URL
-    const secondResult = await saveEmail(ctx, {
+    const secondResult = await saveEmail({
       originalContent: `<html><body>${fakeContent}</body></html>`,
       url,
       title,
       author,
+      userId: user.id,
+      receivedEmailId: 'fakeId',
     })
     expect(secondResult).to.not.be.undefined
 

packages/api/test/services/save_newsletter_email.test.ts

@@ -1,7 +1,6 @@
 import { expect } from 'chai'
 import 'mocha'
 import nock from 'nock'
-import { createPubSubClient } from '../../src/pubsub'
 import { getPageByParam } from '../../src/elastic/pages'
 import { NewsletterEmail } from '../../src/entity/newsletter_email'
 import { ReceivedEmail } from '../../src/entity/received_email'
@@ -9,8 +8,7 @@ import { Subscription } from '../../src/entity/subscription'
 import { User } from '../../src/entity/user'
 import { getRepository } from '../../src/repository'
 import { createNewsletterEmail } from '../../src/services/newsletters'
-import { SaveContext } from '../../src/services/save_email'
-import { saveNewsletterEmail } from '../../src/services/save_newsletter_email'
+import { saveNewsletter } from '../../src/services/save_newsletter_email'
 import { createTestUser, deleteTestUser } from '../db'
 
 describe('saveNewsletterEmail', () => {
@@ -22,17 +20,11 @@ describe('saveNewsletterEmail', () => {
 
   let user: User
   let newsletterEmail: NewsletterEmail
-  let ctx: SaveContext
   let receivedEmail: ReceivedEmail
 
   before(async () => {
     user = await createTestUser('fakeUser')
     newsletterEmail = await createNewsletterEmail(user.id)
-    ctx = {
-      pubsub: createPubSubClient(),
-      refresh: true,
-      uid: user.id,
-    }
     receivedEmail = await getRepository(ReceivedEmail).save({
       user: { id: user.id },
       from,
@@ -53,7 +45,7 @@ describe('saveNewsletterEmail', () => {
     nock('https://blog.omnivore.app').head('/fake-url').reply(200)
     const url = 'https://blog.omnivore.app/fake-url'
 
-    await saveNewsletterEmail(
+    await saveNewsletter(
       {
         from,
         email: newsletterEmail.address,
@@ -64,8 +56,7 @@ describe('saveNewsletterEmail', () => {
         receivedEmailId: receivedEmail.id,
         unsubHttpUrl: 'https://blog.omnivore.app/unsubscribe',
       },
-      newsletterEmail,
-      ctx
+      newsletterEmail
     )
 
     const page = await getPageByParam({ userId: user.id, url })
@@ -90,7 +81,7 @@ describe('saveNewsletterEmail', () => {
       color: '#07D2D1',
     }
 
-    await saveNewsletterEmail(
+    await saveNewsletter(
       {
         email: newsletterEmail.address,
         content: `<html><body>fake content 2</body></html>`,
@@ -100,8 +91,7 @@ describe('saveNewsletterEmail', () => {
         from,
         receivedEmailId: receivedEmail.id,
       },
-      newsletterEmail,
-      ctx
+      newsletterEmail
     )
 
     const page = await getPageByParam({ userId: user.id, url })
@@ -112,7 +102,7 @@ describe('saveNewsletterEmail', () => {
     const url = 'https://omnivore.app/no_url?q=no-unsubscribe'
     nock('https://omnivore.app').get('/no_url?q=no-unsubscribe').reply(404)
 
-    await saveNewsletterEmail(
+    await saveNewsletter(
       {
         email: newsletterEmail.address,
         content: `<html><body>fake content 2</body></html>`,
@@ -122,8 +112,7 @@ describe('saveNewsletterEmail', () => {
         from,
         receivedEmailId: receivedEmail.id,
       },
-      newsletterEmail,
-      ctx
+      newsletterEmail
     )
 
     const subscriptions = await getRepository(Subscription).findBy({

packages/db/migrations/0118.do.library_item.sql

@@ -33,16 +33,16 @@ CREATE TABLE omnivore.library_item (
     site_name text,
     site_icon text,
     metadata JSON,
-    reading_progress_last_read_anchor integer,
-    reading_progress_highest_read_anchor integer,
-    reading_progress_top_percent real,
-    reading_progress_bottom_percent real,
+    reading_progress_last_read_anchor integer NOT NULL DEFAULT 0,
+    reading_progress_highest_read_anchor integer NOT NULL DEFAULT 0,
+    reading_progress_top_percent real NOT NULL DEFAULT 0,
+    reading_progress_bottom_percent real NOT NULL DEFAULT 0,
     thumbnail text,
     item_type library_item_type NOT NULL DEFAULT 'UNKNOWN',
     upload_file_id uuid REFERENCES omnivore.upload_files ON DELETE CASCADE,
     content_reader content_reader_type NOT NULL DEFAULT 'WEB',
     original_content text,
-    readable_content text,
+    readable_content text NOT NULL DEFAULT '',
     content_tsv tsvector,
     site_tsv tsvector,
     title_tsv tsvector,