Merge pull request #178 from omnivore-app/add-probably-emails

New function to determine if an HTML blob is probably a newsletter based on its content

packages/api/src/routers/svc/emails.ts

@@ -4,6 +4,8 @@ import { sendEmail } from '../../utils/sendEmail'
 import { analytics } from '../../utils/analytics'
 import { getNewsletterEmail } from '../../services/newsletters'
 import { env } from '../../env'
+import { findNewsletterUrl, isProbablyNewsletter } from '../../utils/parser'
+import { saveNewsletterEmail } from '../../services/save_newsletter_email'
 
 interface ForwardEmailMessage {
   from: string
@@ -48,6 +50,21 @@ export function emailsServiceRouter() {
         return
       }
 
+      if (isProbablyNewsletter(data.html)) {
+        console.log('handling as newsletter', data)
+        await saveNewsletterEmail({
+          email: data.to,
+          title: data.subject,
+          content: data.html,
+          author: data.from,
+          url:
+            (await findNewsletterUrl(data.html)) ||
+            'https://omnivore.app/no_url',
+        })
+        res.status(200).send('Newsletter')
+        return
+      }
+
       // get user from newsletter email
       const newsletterEmail = await getNewsletterEmail(data.to)
 

packages/api/src/routers/svc/newsletters.ts

@@ -1,43 +1,13 @@
 import express from 'express'
-import {
-  createPubSubClient,
-  readPushSubscription,
-} from '../../datalayer/pubsub'
-import {
-  getNewsletterEmail,
-  updateConfirmationCode,
-} from '../../services/newsletters'
-import {
-  SaveContext,
-  saveEmail,
-  SaveEmailInput,
-} from '../../services/save_email'
-import { initModels } from '../../server'
-import { kx } from '../../datalayer/knex_config'
-import { analytics } from '../../utils/analytics'
-import { env } from '../../env'
-import { sendMulticastPushNotifications } from '../../utils/sendNotification'
-import { getDeviceTokensByUserId } from '../../services/user_device_tokens'
-import { messaging } from 'firebase-admin'
-import { ContentReader } from '../../generated/graphql'
-import { UserDeviceToken } from '../../entity/user_device_tokens'
-import { UserArticleData } from '../../datalayer/links/model'
-import { ArticleData } from '../../datalayer/article/model'
-import MulticastMessage = messaging.MulticastMessage
+import { readPushSubscription } from '../../datalayer/pubsub'
+import { updateConfirmationCode } from '../../services/newsletters'
+import { saveNewsletterEmail } from '../../services/save_newsletter_email'
 
 interface SetConfirmationCodeMessage {
   emailAddress: string
   confirmationCode: string
 }
 
-interface NewsletterMessage {
-  email: string
-  content: string
-  url: string
-  title: string
-  author: string
-}
-
 export function newsletterServiceRouter() {
   const router = express.Router()
 
@@ -111,7 +81,7 @@ export function newsletterServiceRouter() {
 
     try {
       // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment
-      const data: NewsletterMessage = JSON.parse(message)
+      const data = JSON.parse(message)
 
       if (
         !('email' in data) ||
@@ -124,79 +94,15 @@ export function newsletterServiceRouter() {
         return
       }
 
-      // get user from newsletter email
-      const newsletterEmail = await getNewsletterEmail(data.email)
-
-      if (!newsletterEmail) {
-        console.log('newsletter email not found', data.email)
-        res.status(200).send('Not Found')
-        return
-      }
-
-      analytics.track({
-        userId: newsletterEmail.user.id,
-        event: 'newsletter_email_received',
-        properties: {
-          url: data.url,
-          title: data.title,
-          author: data.author,
-          env: env.server.apiEnv,
-        },
-      })
-
-      const ctx: SaveContext = {
-        models: initModels(kx, false),
-        pubsub: createPubSubClient(),
-      }
-      const input: SaveEmailInput = {
-        url: data.url,
-        originalContent: data.content,
-        title: data.title,
-        author: data.author,
-      }
-
-      const result = await saveEmail(ctx, newsletterEmail.user.id, input)
+      const result = await saveNewsletterEmail(data)
       if (!result) {
-        console.log('newsletter not created:', input)
-        res.status(200).send(result)
+        console.log('Error createing newsletter link from data', data)
+        res.status(500).send('Error creating newsletter link')
         return
       }
 
-      // send push notification
-      const deviceTokens = await getDeviceTokensByUserId(
-        newsletterEmail.user.id
-      )
-
-      if (!deviceTokens) {
-        console.log('Device tokens not set:', newsletterEmail.user.id)
-        res.status(200).send('Device token Not Found')
-        return
-      }
-
-      const link = await ctx.models.userArticle.getForUser(
-        newsletterEmail.user.id,
-        result.articleId
-      )
-
-      if (!link) {
-        console.log(
-          'Newsletter link not found:',
-          newsletterEmail.user.id,
-          result.articleId
-        )
-        res.status(200).send(result)
-        return
-      }
-
-      if (deviceTokens.length) {
-        const multicastMessage = messageForLink(link, deviceTokens)
-        await sendMulticastPushNotifications(
-          newsletterEmail.user.id,
-          multicastMessage,
-          'newsletter'
-        )
-      }
-
+      // We always send 200 if it was a valid message
+      // because we don't want the
       res.status(200).send('newsletter created')
     } catch (e) {
       console.log(e)
@@ -211,43 +117,3 @@ export function newsletterServiceRouter() {
 
   return router
 }
-
-const messageForLink = (
-  link: ArticleData & UserArticleData,
-  deviceTokens: UserDeviceToken[]
-): MulticastMessage => {
-  let title = '📫 - An article was added to your Omnivore Inbox'
-
-  if (link.author) {
-    title = `📫 - ${link.author} has published a new article`
-  }
-
-  const pushData = !link
-    ? undefined
-    : {
-        link: Buffer.from(
-          JSON.stringify({
-            id: link.id,
-            url: link.url,
-            slug: link.slug,
-            title: link.title,
-            image: link.image,
-            author: link.author,
-            isArchived: link.isArchived,
-            contentReader: ContentReader.Web,
-            readingProgressPercent: link.articleReadingProgress,
-            readingProgressAnchorIndex: link.articleReadingProgressAnchorIndex,
-          })
-        ).toString('base64'),
-      }
-
-  return {
-    notification: {
-      title: title,
-      body: link.title,
-      imageUrl: link.image || undefined,
-    },
-    data: pushData,
-    tokens: deviceTokens.map((token) => token.token),
-  }
-}

packages/api/src/services/save_email.ts

@@ -2,7 +2,7 @@ import { PubsubClient } from '../datalayer/pubsub'
 import { DataModels } from '../resolvers/types'
 import { generateSlug, stringToHash, validatedDate } from '../utils/helpers'
 import {
-  parseMetadata,
+  parseUrlMetadata,
   parseOriginalContent,
   parsePreparedContent,
 } from '../utils/parser'
@@ -44,14 +44,15 @@ export const saveEmail = async (
   const slug = generateSlug(title)
 
   const pageType = parseOriginalContent(url, input.originalContent)
-  const metadata = await parseMetadata(url)
+  const metadata = await parseUrlMetadata(url)
 
   const articleToSave = {
     originalHtml: input.originalContent,
     content: content,
     description: metadata?.description || parseResult.parsedContent?.excerpt,
-    title: title,
-    author: input.author,
+    title: metadata?.title || parseResult.parsedContent?.title || title,
+    author:
+      metadata?.author || parseResult.parsedContent?.byline || input.author,
     url: normalizeUrl(parseResult.canonicalUrl || url, {
       stripHash: true,
       stripWWW: false,

packages/api/src/services/save_newsletter_email.ts

@@ -0,0 +1,137 @@
+import { MulticastMessage } from 'firebase-admin/messaging'
+import { ArticleData } from '../datalayer/article/model'
+import { kx } from '../datalayer/knex_config'
+import { UserArticleData } from '../datalayer/links/model'
+import { createPubSubClient } from '../datalayer/pubsub'
+import { UserDeviceToken } from '../entity/user_device_tokens'
+import { env } from '../env'
+import { ContentReader } from '../generated/graphql'
+import { initModels } from '../server'
+import { analytics } from '../utils/analytics'
+import { sendMulticastPushNotifications } from '../utils/sendNotification'
+import { getNewsletterEmail } from './newsletters'
+import { SaveContext, saveEmail, SaveEmailInput } from './save_email'
+import { getDeviceTokensByUserId } from './user_device_tokens'
+
+interface NewsletterMessage {
+  email: string
+  content: string
+  url: string
+  title: string
+  author: string
+}
+
+// Returns true if the link was created successfully. Can still fail to
+// send the push but that is ok and we wont retry in that case.
+export const saveNewsletterEmail = async (
+  data: NewsletterMessage
+): Promise<boolean> => {
+  // get user from newsletter email
+  const newsletterEmail = await getNewsletterEmail(data.email)
+
+  if (!newsletterEmail) {
+    console.log('newsletter email not found', data.email)
+    return false
+  }
+
+  analytics.track({
+    userId: newsletterEmail.user.id,
+    event: 'newsletter_email_received',
+    properties: {
+      url: data.url,
+      title: data.title,
+      author: data.author,
+      env: env.server.apiEnv,
+    },
+  })
+
+  const ctx: SaveContext = {
+    models: initModels(kx, false),
+    pubsub: createPubSubClient(),
+  }
+  const input: SaveEmailInput = {
+    url: data.url,
+    originalContent: data.content,
+    title: data.title,
+    author: data.author,
+  }
+
+  const result = await saveEmail(ctx, newsletterEmail.user.id, input)
+  if (!result) {
+    console.log('newsletter not created:', input)
+    return false
+  }
+
+  // send push notification
+  const deviceTokens = await getDeviceTokensByUserId(newsletterEmail.user.id)
+
+  if (!deviceTokens) {
+    console.log('Device tokens not set:', newsletterEmail.user.id)
+    return true
+  }
+
+  const link = await ctx.models.userArticle.getForUser(
+    newsletterEmail.user.id,
+    result.articleId
+  )
+
+  if (!link) {
+    console.log(
+      'Newsletter link not found:',
+      newsletterEmail.user.id,
+      result.articleId
+    )
+    return true
+  }
+
+  if (deviceTokens.length) {
+    const multicastMessage = messageForLink(link, deviceTokens)
+    await sendMulticastPushNotifications(
+      newsletterEmail.user.id,
+      multicastMessage,
+      'newsletter'
+    )
+  }
+
+  return true
+}
+
+const messageForLink = (
+  link: ArticleData & UserArticleData,
+  deviceTokens: UserDeviceToken[]
+): MulticastMessage => {
+  let title = '📫 - An article was added to your Omnivore Inbox'
+
+  if (link.author) {
+    title = `📫 - ${link.author} has published a new article`
+  }
+
+  const pushData = !link
+    ? undefined
+    : {
+        link: Buffer.from(
+          JSON.stringify({
+            id: link.id,
+            url: link.url,
+            slug: link.slug,
+            title: link.title,
+            image: link.image,
+            author: link.author,
+            isArchived: link.isArchived,
+            contentReader: ContentReader.Web,
+            readingProgressPercent: link.articleReadingProgress,
+            readingProgressAnchorIndex: link.articleReadingProgressAnchorIndex,
+          })
+        ).toString('base64'),
+      }
+
+  return {
+    notification: {
+      title: title,
+      body: link.title,
+      imageUrl: link.image || undefined,
+    },
+    data: pushData,
+    tokens: deviceTokens.map((token) => token.token),
+  }
+}

packages/api/src/utils/axios-handler.ts

@@ -3,6 +3,7 @@ import { DOMWindow } from 'jsdom'
 export class AxiosHandler {
   name = 'axios'
 
+  // eslint-disable-next-line @typescript-eslint/no-unused-vars
   shouldPrehandle = (url: URL, _dom: DOMWindow): boolean => {
     const host = this.name + '.com'
     // check if url ends with axios.com

packages/api/src/utils/createTask.ts

@@ -170,6 +170,7 @@ export const getTask = async (
 
   const request: protos.google.cloud.tasks.v2.GetTaskRequest = {
     responseView: View.FULL,
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
     toJSON(): { [p: string]: any } {
       return {}
     },

packages/api/src/utils/golang-handler.ts

@@ -3,6 +3,7 @@ import { DOMWindow } from 'jsdom'
 export class GolangHandler {
   name = 'golangweekly'
 
+  // eslint-disable-next-line @typescript-eslint/no-unused-vars
   shouldPrehandle = (url: URL, _dom: DOMWindow): boolean => {
     const host = this.name + '.com'
     // check if url ends with golangweekly.com

packages/api/src/utils/parser.ts

@@ -346,16 +346,15 @@ const getJSONLdLinkMetadata = async (
 }
 
 type Metadata = {
+  title?: string
+  author?: string
   description: string
   previewImage: string
 }
 
-export const parseMetadata = async (
-  url: string
-): Promise<Metadata | undefined> => {
+export const parsePageMetadata = (html: string): Metadata | undefined => {
   try {
-    const res = await axios.get(url)
-    const window = new JSDOM(res.data).window
+    const window = new JSDOM(html).window
 
     // get open graph metadata
     const description =
@@ -368,9 +367,99 @@ export const parseMetadata = async (
         .querySelector("head meta[property='og:image']")
         ?.getAttribute('content') || ''
 
-    return { description: description, previewImage: previewImage }
+    const title =
+      window.document
+        .querySelector("head meta[property='og:title']")
+        ?.getAttribute('content') || undefined
+
+    const author =
+      window.document
+        .querySelector("head meta[name='author']")
+        ?.getAttribute('content') || undefined
+
+    // TODO: we should be able to apply the JSONLD metadata
+    // here too
+
+    return { title, author, description, previewImage }
   } catch (e) {
-    console.log('failed to got:', url, e)
+    console.log('failed to parse page:', html, e)
     return undefined
   }
 }
+
+export const parseUrlMetadata = async (
+  url: string
+): Promise<Metadata | undefined> => {
+  try {
+    const res = await axios.get(url)
+    return parsePageMetadata(res.data)
+  } catch (e) {
+    console.log('failed to get:', url, e)
+    return undefined
+  }
+}
+
+// Attempt to determine if an HTML blob is a newsletter
+// based on it's contents.
+// TODO: when we consolidate the handlers we could include this
+// as a utility method on each one.
+export const isProbablyNewsletter = (html: string): boolean => {
+  const dom = new JSDOM(html).window
+  const domCopy = new JSDOM(dom.document.documentElement.outerHTML)
+  const article = new Readability(domCopy.window.document, {
+    debug: false,
+    keepTables: true,
+  }).parse()
+
+  if (!article || !article.content) {
+    return false
+  }
+
+  // substack newsletter emails have tables with a *post-meta class
+  if (dom.document.querySelector('table[class$="post-meta"]')) {
+    return true
+  }
+
+  // If the article has a header link, and substack icons its probably a newsletter
+  const href = findNewsletterHeaderHref(dom.window)
+  const heartIcon = dom.document.querySelector(
+    'table tbody td span a img[src*="HeartIcon"]'
+  )
+  const recommendIcon = dom.document.querySelector(
+    'table tbody td span a img[src*="RecommendIconRounded"]'
+  )
+  if (href && (heartIcon || recommendIcon)) {
+    return true
+  }
+
+  return false
+}
+
+const findNewsletterHeaderHref = (dom: DOMWindow): string | undefined => {
+  const postLink = dom.document.querySelector('h1 a ')
+  if (postLink) {
+    return postLink.getAttribute('href') || undefined
+  }
+  return undefined
+}
+
+// Given an HTML blob tries to find a URL to use for
+// a canonical URL.
+export const findNewsletterUrl = async (
+  html: string
+): Promise<string | undefined> => {
+  const dom = new JSDOM(html).window
+  const href = findNewsletterHeaderHref(dom.window)
+  if (href) {
+    // Try to make a HEAD request so we get the redirected URL, since these
+    // will usually be behind tracking url redirects
+    return axios({
+      method: 'HEAD',
+      url: href,
+    })
+      .then((res) => res.request.res.responseUrl as string | undefined)
+      .catch((e) => href)
+  }
+
+  return undefined
+}

packages/api/src/utils/usernamePolicy.ts

@@ -94,6 +94,7 @@ const RESERVED_NAMES = new Set([
   'mine',
   'mis',
   'news',
+  'no_url',
   'oauth',
   'oauth_clients',
   'offers',

packages/api/src/utils/wikipedia-handler.ts

@@ -3,6 +3,7 @@ import { DOMWindow } from 'jsdom'
 export class WikipediaHandler {
   name = 'wikipedia'
 
+  // eslint-disable-next-line @typescript-eslint/no-unused-vars
   shouldPrehandle = (url: URL, _dom: DOMWindow): boolean => {
     return url.hostname.endsWith('wikipedia.org')
   }

packages/api/test/services/save_newsletter_email.test.ts

@@ -0,0 +1,44 @@
+import 'mocha'
+import { expect } from 'chai'
+import 'chai/register-should'
+import {
+  createTestUser,
+  deleteTestUser,
+} from '../db'
+import { createNewsletterEmail } from '../../src/services/newsletters'
+import { saveNewsletterEmail } from '../../src/services/save_newsletter_email'
+import { getRepository } from 'typeorm'
+import { Link } from '../../src/entity/link'
+
+describe('saveNewsletterEmail', () => {
+  const username = 'fakeUser'
+  after(async () => {
+    await deleteTestUser(username)
+  })
+
+  it('adds the newsletter to the library', async () => {
+    const user = await createTestUser(username)
+    const email = await createNewsletterEmail(user.id)
+
+    await saveNewsletterEmail({
+      email: email.address,
+      content: 'fake content',
+      url: 'https://example.com',
+      title: 'fake title',
+      author: 'fake author',
+    })
+
+    const links = await getRepository(Link).find({
+      where: {
+        user: user,
+      },
+      relations: ['page'],
+    })
+    
+    expect(links.length).to.equal(1)
+    expect(links[0].page.url).to.equal('https://example.com')
+    expect(links[0].page.title).to.equal('fake title')
+    expect(links[0].page.author).to.equal('fake author')
+    expect(links[0].page.content).to.contain('fake content')
+  }).timeout(10000)
+})

packages/api/test/utils/parser.test.ts

@@ -0,0 +1,51 @@
+import 'mocha'
+import { expect } from 'chai'
+import 'chai/register-should'
+import { JSDOM } from 'jsdom'
+import fs from 'fs'
+import { findNewsletterUrl, isProbablyNewsletter, parsePageMetadata } from '../../src/utils/parser'
+
+const load = (path: string): string => {
+  return fs.readFileSync(path, 'utf8')
+}
+
+describe('isProbablyNewsletter', () => {
+  it('returns true for substack newsletter', () => {
+    const html = load('./test/utils/data/substack-forwarded-newsletter.html')
+    isProbablyNewsletter(html).should.be.true
+  })
+  it('returns true for private forwarded substack newsletter', () => {
+    const html = load('./test/utils/data/substack-private-forwarded-newsletter.html')
+    isProbablyNewsletter(html).should.be.true
+  })
+  it('returns false for substack welcome email', () => {
+    const html = load('./test/utils/data/substack-forwarded-welcome-email.html')
+    isProbablyNewsletter(html).should.be.false
+  })
+})
+
+describe('findNewsletterUrl', async () => {
+  it('gets the URL from the header if it is a newsletter', async () => {
+    const html = load('./test/utils/data/substack-forwarded-newsletter.html')
+    const url = await findNewsletterUrl(html)
+    // Not sure if the redirects from substack expire, this test could eventually fail
+    expect(url).to.startWith('https://newsletter.slowchinese.net/p/companies-that-eat-people-217')
+  })
+  it('returns undefined if it is not a newsletter', async () => {
+    const html = load('./test/utils/data/substack-forwarded-welcome-email.html')
+    const url = await findNewsletterUrl(html)
+    expect(url).to.be.undefined
+  })
+})
+
+describe('parseMetadata', async () => {
+  it('gets author, title, image, description', async () => {
+    const html = load('./test/utils/data/substack-post.html')
+    const metadata = await parsePageMetadata(html)
+    expect(metadata?.author).to.deep.equal('Omnivore')
+    expect(metadata?.title).to.deep.equal('Code Block Syntax Highlighting')
+    expect(metadata?.previewImage).to.deep.equal('https://cdn.substack.com/image/fetch/w_1200,h_600,c_fill,f_jpg,q_auto:good,fl_progressive:steep,g_auto/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2F2ab1f7e8-2ca7-4011-8ccb-43d0b3bd244f_1490x2020.png')
+    expect(metadata?.description).to.deep.equal('Highlighted <code> in Omnivore')
+
+  })
+})