Merge pull request #367 from omnivore-app/fix/full-email-addresses

packages/api/.env.test

@@ -27,3 +27,5 @@ PREVIEW_IMAGE_WRAPPER_ID='selected_highlight_wrapper'
 SEGMENT_WRITE_KEY='test'
 REMINDER_TASK_HANDLER_URL=http://localhost:4000/svc/reminders/trigger
 PUBSUB_VERIFICATION_TOKEN='123456'
+PUPPETEER_TASK_HANDLER_URL=http://localhost:9090/
+

packages/api/package.json

@@ -84,6 +84,7 @@
     "knex-stringcase": "^1.4.2",
     "luxon": "^1.25.0",
     "nanoid": "^3.1.25",
+    "nodemailer": "^6.7.3",
     "normalize-url": "^6.1.0",
     "oauth": "^0.9.15",
     "pg": "^8.3.3",
@@ -107,6 +108,7 @@
     "@types/analytics-node": "^3.1.7",
     "@types/highlightjs": "^9.12.2",
     "@types/nanoid": "^3.0.0",
+    "@types/nodemailer": "^6.4.4",
     "@types/private-ip": "^1.0.0",
     "chai": "^4.3.4",
     "chai-string": "^1.5.0",

packages/api/src/routers/svc/emails.ts

@@ -4,6 +4,7 @@ import { sendEmail } from '../../utils/sendEmail'
 import { analytics } from '../../utils/analytics'
 import { getNewsletterEmail } from '../../services/newsletters'
 import { env } from '../../env'
+import { v4 as uuid } from 'uuid'
 import { findNewsletterUrl, isProbablyNewsletter } from '../../utils/parser'
 import { saveNewsletterEmail } from '../../services/save_newsletter_email'
 
@@ -59,7 +60,7 @@ export function emailsServiceRouter() {
           author: data.from,
           url:
             (await findNewsletterUrl(data.html)) ||
-            'https://omnivore.app/no_url',
+            'https://omnivore.app/no_url?q' + uuid(),
         })
         res.status(200).send('Newsletter')
         return

packages/api/src/services/newsletters.ts

@@ -4,6 +4,15 @@ import { nanoid } from 'nanoid'
 import { User } from '../entity/user'
 import { CreateNewsletterEmailErrorCode } from '../generated/graphql'
 import { env } from '../env'
+import addressparser from 'nodemailer/lib/addressparser'
+
+const parsedAddress = (emailAddress: string): string | undefined => {
+  const res = addressparser(emailAddress, { flatten: true })
+  if (!res || res.length < 1) {
+    return undefined
+  }
+  return res[0].address
+}
 
 export const createNewsletterEmail = async (
   userId: string
@@ -46,9 +55,10 @@ export const updateConfirmationCode = async (
   emailAddress: string,
   confirmationCode: string
 ): Promise<boolean> => {
+  const address = parsedAddress(emailAddress)
   const result = await getRepository(NewsletterEmail)
     .createQueryBuilder()
-    .where('address ILIKE :address', { address: emailAddress })
+    .where('address ILIKE :address', { address })
     .update({
       confirmationCode: confirmationCode,
     })
@@ -60,10 +70,11 @@ export const updateConfirmationCode = async (
 export const getNewsletterEmail = async (
   emailAddress: string
 ): Promise<NewsletterEmail | undefined> => {
+  const address = parsedAddress(emailAddress)
   return getRepository(NewsletterEmail)
     .createQueryBuilder('newsletter_email')
     .innerJoinAndSelect('newsletter_email.user', 'user')
-    .where('address ILIKE :address', { address: emailAddress })
+    .where('address ILIKE :address', { address })
     .getOne()
 }
 

packages/api/src/services/save_email.ts

@@ -12,6 +12,7 @@ import { createPage, getPageByParam, updatePage } from '../elastic'
 export type SaveContext = {
   pubsub: PubsubClient
   uid: string
+  refresh?: boolean
 }
 
 export type SaveEmailInput = {
@@ -67,7 +68,7 @@ export const saveEmail = async (
     readingProgressPercent: 0,
   }
 
-  const page = await getPageByParam({ url: articleToSave.url })
+  const page = await getPageByParam({ userId: ctx.uid, url: articleToSave.url })
   if (page) {
     const result = await updatePage(page.id, { archivedAt: null }, ctx)
     console.log('updated page from email', result)
@@ -82,7 +83,6 @@ export const saveEmail = async (
     return undefined
   }
 
-  console.log('created new page from email', pageId)
   articleToSave.id = pageId
 
   return articleToSave

packages/api/src/services/save_newsletter_email.ts

@@ -22,7 +22,8 @@ interface NewsletterMessage {
 // Returns true if the link was created successfully. Can still fail to
 // send the push but that is ok and we wont retry in that case.
 export const saveNewsletterEmail = async (
-  data: NewsletterMessage
+  data: NewsletterMessage,
+  ctx?: SaveContext
 ): Promise<boolean> => {
   // get user from newsletter email
   const newsletterEmail = await getNewsletterEmail(data.email)
@@ -43,7 +44,7 @@ export const saveNewsletterEmail = async (
     },
   })
 
-  const ctx: SaveContext = {
+  const saveCtx = ctx || {
     pubsub: createPubSubClient(),
     uid: newsletterEmail.user.id,
   }
@@ -55,14 +56,14 @@ export const saveNewsletterEmail = async (
     author: data.author,
   }
 
-  const page = await saveEmail(ctx, input)
+  const page = await saveEmail(saveCtx, input)
   if (!page) {
     console.log('newsletter not created:', input)
     return false
   }
 
   // add newsletters label to page
-  const result = await addLabelToPage(ctx, page.id, {
+  const result = await addLabelToPage(saveCtx, page.id, {
     name: 'Newsletter',
     color: '#07D2D1',
   })

packages/api/test/elastic/index.test.ts

@@ -113,7 +113,6 @@ describe('elastic api', () => {
   describe('getPageById', () => {
     it('gets a page by id', async () => {
       const pageFound = await getPageById(page.id)
-
       expect(pageFound).not.undefined
     })
   })
@@ -128,7 +127,6 @@ describe('elastic api', () => {
       await updatePage(page.id, updatedPageData, ctx)
 
       const updatedPage = await getPageById(page.id)
-
       expect(updatedPage?.title).to.eql(newTitle)
     })
   })

packages/api/test/resolvers/highlight.test.ts

@@ -137,7 +137,7 @@ describe('Highlights API', () => {
     before(async () => {
       // create test highlight
       highlightId = generateFakeUuid()
-      const shortHighlightId = '_short_id'
+      const shortHighlightId = '_short_id_1'
       const query = createHighlightQuery(
         authToken,
         pageId,
@@ -149,7 +149,7 @@ describe('Highlights API', () => {
 
     it('should not fail', async () => {
       const newHighlightId = generateFakeUuid()
-      const newShortHighlightId = '_short_id_1'
+      const newShortHighlightId = '_short_id_2'
       const query = mergeHighlightQuery(
         pageId,
         newHighlightId,

packages/api/test/resolvers/newsletters.test.ts

@@ -28,11 +28,11 @@ describe('Newsletters API', () => {
     //  create test newsletter emails
     const newsletterEmail1 = await createTestNewsletterEmail(
       user,
-      'Test_email_address_1'
+      'Test_email_address_1@fake-email.com'
     )
     const newsletterEmail2 = await createTestNewsletterEmail(
       user,
-      'Test_email_address_2'
+      'Test_email_address_2@fake-email.com'
     )
     newsletterEmails = [newsletterEmail1, newsletterEmail2]
   })

packages/api/test/routers/article_router.test.ts

@@ -3,6 +3,7 @@ import { request } from '../util'
 import { expect } from 'chai'
 import nock from 'nock'
 import 'mocha'
+import { env } from '../../src/env'
 
 describe('/article/save API', () => {
   const username = 'fakeUser'
@@ -12,7 +13,7 @@ describe('/article/save API', () => {
   // We need to mock the pupeeteer-parse
   // service here because in dev mode the task gets
   // called immediately.
-  nock('http://localhost:8080/').post('/').reply(200)
+  nock(env.queue.puppeteerTaskHanderUrl).post('/').reply(200)
 
   before(async () => {
     // create test user and login

packages/api/test/routers/pdf_attachments.test.ts

@@ -12,7 +12,7 @@ import { getPageById } from '../../src/elastic'
 
 describe('PDF attachments Router', () => {
   const username = 'fakeUser'
-  const newsletterEmail = 'fakeEmail'
+  const newsletterEmail = 'fakeEmail@fake-email.com'
 
   let user: User
   let authToken: string

packages/api/test/services/save_email.test.ts

@@ -17,6 +17,7 @@ describe('saveEmail', () => {
     const ctx: SaveContext = {
       pubsub: createPubSubClient(),
       uid: user.id,
+      refresh: true,
     }
 
     await saveEmail(ctx, {
@@ -36,15 +37,11 @@ describe('saveEmail', () => {
     })
     expect(secondResult).to.not.be.undefined
 
-    setTimeout(async () => {
-      const page = await getPageByParam({ userId: user.id })
-      if (!page) {
-        expect.fail('page not found')
-      }
-      expect(page.url).to.equal('https://example.com')
-      expect(page.title).to.equal('fake title')
-      expect(page.author).to.equal('fake author')
-      expect(page.content).to.contain('fake content')
-    })
+    const page = await getPageByParam({ userId: user.id })
+    expect(page).to.exist
+    expect(page?.url).to.equal('https://example.com')
+    expect(page?.title).to.equal('fake title')
+    expect(page?.author).to.equal('fake author')
+    expect(page?.content).to.contain('fake content')
   })
 })

packages/api/test/util.ts

@@ -4,7 +4,7 @@ import { v4 } from 'uuid'
 import { corsConfig } from '../src/utils/corsConfig'
 import { Page } from '../src/elastic/types'
 import { PageType } from '../src/generated/graphql'
-import { createPage } from '../src/elastic'
+import { createPage, getPageById } from '../src/elastic'
 import { User } from '../src/entity/user'
 import { Label } from '../src/entity/label'
 import { createPubSubClient } from '../src/datalayer/pubsub'
@@ -62,5 +62,11 @@ export const createTestElasticPage = async (
   if (pageId) {
     page.id = pageId
   }
-  return page
+
+  const res = await getPageById(page.id)
+  console.log('got page', res)
+  if (!res) {
+    throw new Error('Failed to create page')
+  }
+  return res
 }

packages/api/test/utils/parser.test.ts

@@ -30,12 +30,22 @@ describe('isProbablyNewsletter', () => {
 
 describe('findNewsletterUrl', async () => {
   it('gets the URL from the header if it is a substack newsletter', async () => {
+    nock('https://newsletter.slowchinese.net')
+    .head('/p/companies-that-eat-people-217?token=eyJ1c2VyX2lkIjoxMTU0MzM0NSwicG9zdF9pZCI6NDg3MjA5NDAsImlhdCI6MTY0NTI1NzQ1MSwiaXNzIjoicHViLTI4MDUzMSIsInN1YiI6InBvc3QtcmVhY3Rpb24ifQ.l5F3Kx6K9tvy9cRAXx3MepobQBCJDJQgAxOpA0INIZA')
+    .reply(200, '');
     const html = load('./test/utils/data/substack-forwarded-newsletter.html')
     const url = await findNewsletterUrl(html)
     // Not sure if the redirects from substack expire, this test could eventually fail
     expect(url).to.startWith('https://newsletter.slowchinese.net/p/companies-that-eat-people-217')
   })
   it('gets the URL from the header if it is a beehiiv newsletter', async () => {
+    nock('https://u23463625.ct.sendgrid.net')
+    .head('/ss/c/AX1lEgEQaxtvFxLaVo0GBo_geajNrlI1TGeIcmMViR3pL3fEDZnbbkoeKcaY62QZk0KPFudUiUXc_uMLerV4nA/3k5/3TFZmreTR0qKSCgowABnVg/h30/zzLik7UXd1H_n4oyd5W8Xu639AYQQB2UXz-CsssSnno')
+    .reply(302, undefined,{
+      'Location': 'https://www.milkroad.com/p/talked-guy-spent-30m-beeple'
+    })
+    .get('/p/talked-guy-spent-30m-beeple')
+    .reply(200, '');
     const html = load('./test/utils/data/beehiiv-newsletter.html')
     const url = await findNewsletterUrl(html)
     expect(url).to.startWith('https://www.milkroad.com/p/talked-guy-spent-30m-beeple')

packages/readabilityjs/Readability.js

@@ -68,7 +68,6 @@ const extractPublishedDateFromAuthor = (author)=> {
  * @param {Object}       options The options object.
  */
 function Readability(doc, options) {
-  console.log("\nOmnivore Inc. v.0.1.9");
   // In some older versions, people passed a URI as the first argument. Cope:
   if (options && options.documentElement) {
     doc = options;

yarn.lock

@@ -6417,6 +6417,13 @@
   resolved "https://registry.yarnpkg.com/@types/node/-/node-14.17.32.tgz#2ca61c9ef8c77f6fa1733be9e623ceb0d372ad96"
   integrity sha512-JcII3D5/OapPGx+eJ+Ik1SQGyt6WvuqdRfh9jUwL6/iHGjmyOriBDciBUu7lEIBTL2ijxwrR70WUnw5AEDmFvQ==
 
+"@types/nodemailer@^6.4.4":
+  version "6.4.4"
+  resolved "https://registry.yarnpkg.com/@types/nodemailer/-/nodemailer-6.4.4.tgz#c265f7e7a51df587597b3a49a023acaf0c741f4b"
+  integrity sha512-Ksw4t7iliXeYGvIQcSIgWQ5BLuC/mljIEbjf615svhZL10PE9t+ei8O9gDaD3FPCasUJn9KTLwz2JFJyiiyuqw==
+  dependencies:
+    "@types/node" "*"
+
 "@types/normalize-package-data@^2.4.0":
   version "2.4.1"
   resolved "https://registry.yarnpkg.com/@types/normalize-package-data/-/normalize-package-data-2.4.1.tgz#d3357479a0fdfdd5907fe67e17e0a85c906e1301"
@@ -15023,6 +15030,11 @@ node-releases@^2.0.1, node-releases@^2.0.2:
   resolved "https://registry.yarnpkg.com/node-releases/-/node-releases-2.0.2.tgz#7139fe71e2f4f11b47d4d2986aaf8c48699e0c01"
   integrity sha512-XxYDdcQ6eKqp/YjI+tb2C5WM2LgjnZrfYg4vgQt49EK268b6gYCHsBLrK2qvJo4FmCtqmKezb0WZFK4fkrZNsg==
 
+nodemailer@^6.7.3:
+  version "6.7.3"
+  resolved "https://registry.yarnpkg.com/nodemailer/-/nodemailer-6.7.3.tgz#b73f9a81b9c8fa8acb4ea14b608f5e725ea8e018"
+  integrity sha512-KUdDsspqx89sD4UUyUKzdlUOper3hRkDVkrKh/89G+d9WKsU5ox51NWS4tB1XR5dPUdR4SP0E3molyEfOvSa3g==
+
 nodemon@^2.0.15:
   version "2.0.15"
   resolved "https://registry.yarnpkg.com/nodemon/-/nodemon-2.0.15.tgz#504516ce3b43d9dc9a955ccd9ec57550a31a8d4e"