Handle newsletter in content-handlers

packages/content-fetch/test/babel-register.js

@@ -0,0 +1,3 @@
+const register = require('@babel/register').default
+
+register({ extensions: ['.ts', '.tsx', '.js', '.jsx'] })

packages/content-fetch/test/stub.test.ts

@@ -0,0 +1,13 @@
+import 'mocha'
+import * as chai from 'chai'
+import { expect } from 'chai'
+import 'chai/register-should'
+import chaiString from 'chai-string'
+
+chai.use(chaiString)
+
+describe('Stub test', () => {
+  it('should pass', () => {
+    expect(true).to.be.true
+  })
+})

packages/content-handler/package.json

@@ -18,9 +18,12 @@
     "eslint-plugin-prettier": "^4.0.0"
   },
   "dependencies": {
+    "addressparser": "^1.0.1",
     "axios": "^0.27.2",
     "linkedom": "^0.14.16",
     "luxon": "^3.0.4",
-    "underscore": "^1.13.6"
+    "rfc2047": "^4.0.1",
+    "underscore": "^1.13.6",
+    "uuid": "^9.0.0"
   }
 }

packages/content-handler/src/axios-handler.ts

@@ -0,0 +1,17 @@
+import { ContentHandler } from './content-handler'
+
+export class AxiosHandler extends ContentHandler {
+  constructor() {
+    super()
+    this.senderRegex = /<.+@axios.com>/
+    this.urlRegex = /View in browser at <a.*>(.*)<\/a>/
+    this.defaultUrl = 'https://axios.com'
+    this.name = 'Axios'
+  }
+
+  isNewsletter(postHeader: string, from: string, unSubHeader: string): boolean {
+    // Axios newsletter is from <xx@axios.com>
+    const re = new RegExp(this.senderRegex)
+    return re.test(from) && (!!postHeader || !!unSubHeader)
+  }
+}

packages/content-handler/src/bloomberg-handler.ts

@@ -5,6 +5,9 @@ import { ContentHandler, PreHandleResult } from './content-handler'
 export class BloombergHandler extends ContentHandler {
   constructor() {
     super()
+    this.senderRegex = /<.+@mail.bloomberg.*.com>/
+    this.urlRegex = /<a class="view-in-browser__url" href=["']([^"']*)["']/
+    this.defaultUrl = 'https://www.bloomberg.com'
     this.name = 'Bloomberg'
   }
 
@@ -38,4 +41,10 @@ export class BloombergHandler extends ContentHandler {
       throw error
     }
   }
+
+  isNewsletter(postHeader: string, from: string, unSubHeader: string): boolean {
+    // Axios newsletter is from <xx@axios.com>
+    const re = new RegExp(this.senderRegex)
+    return re.test(from) && (!!postHeader || !!unSubHeader)
+  }
 }

packages/content-handler/src/content-handler.ts

@@ -7,7 +7,16 @@ interface Unsubscribe {
   httpUrl?: string
 }
 
-interface NewsletterMessage {
+export interface NewsletterInput {
+  postHeader: string
+  from: string
+  unSubHeader: string
+  email: string
+  html: string
+  title: string
+}
+
+export interface NewsletterResult {
   email: string
   content: string
   url: string
@@ -55,9 +64,7 @@ export abstract class ContentHandler {
   }
 
   isNewsletter(postHeader: string, from: string, unSubHeader: string): boolean {
-    // Axios newsletter is from <xx@axios.com>
-    const re = new RegExp(this.senderRegex)
-    return re.test(from) && (!!postHeader || !!unSubHeader)
+    return false
   }
 
   parseNewsletterUrl(_postHeader: string, html: string): string | undefined {
@@ -90,14 +97,14 @@ export abstract class ContentHandler {
     }
   }
 
-  handleNewsletter(
-    email: string,
-    html: string,
-    postHeader: string,
-    title: string,
-    from: string,
-    unSubHeader: string
-  ): NewsletterMessage {
+  handleNewsletter({
+    email,
+    html,
+    postHeader,
+    title,
+    from,
+    unSubHeader,
+  }: NewsletterInput): NewsletterResult {
     console.log('handleNewsletter', email, postHeader, title, from)
 
     if (!email || !html || !title || !from) {

packages/content-handler/src/golang-handler.ts

@@ -0,0 +1,17 @@
+import { ContentHandler } from './content-handler'
+
+export class GolangHandler extends ContentHandler {
+  constructor() {
+    super()
+    this.senderRegex = /<.+@golangweekly.com>/
+    this.urlRegex = /<a href=["']([^"']*)["'].*>Read on the Web<\/a>/
+    this.defaultUrl = 'https://golangweekly.com'
+    this.name = 'Golang Weekly'
+  }
+
+  isNewsletter(postHeader: string, from: string, unSubHeader: string): boolean {
+    // Axios newsletter is from <xx@axios.com>
+    const re = new RegExp(this.senderRegex)
+    return re.test(from) && (!!postHeader || !!unSubHeader)
+  }
+}

packages/content-handler/src/index.ts

@@ -8,7 +8,16 @@ import { ScrapingBeeHandler } from './scrapingBee-handler'
 import { TDotCoHandler } from './t-dot-co-handler'
 import { TwitterHandler } from './twitter-handler'
 import { YoutubeHandler } from './youtube-handler'
-import { ContentHandler, PreHandleResult } from './content-handler'
+import {
+  ContentHandler,
+  NewsletterInput,
+  NewsletterResult,
+  PreHandleResult,
+} from './content-handler'
+import { SubstackHandler } from './substack-handler'
+import { AxiosHandler } from './axios-handler'
+import { GolangHandler } from './golang-handler'
+import { MorningBrewHandler } from './morning-brew-handler'
 
 const validateUrlString = (url: string) => {
   const u = new URL(url)
@@ -37,6 +46,10 @@ const contentHandlers: ContentHandler[] = [
   new TDotCoHandler(),
   new TwitterHandler(),
   new YoutubeHandler(),
+  new SubstackHandler(),
+  new AxiosHandler(),
+  new GolangHandler(),
+  new MorningBrewHandler(),
 ]
 
 export const preHandleContent = async (
@@ -71,6 +84,19 @@ export const preHandleContent = async (
   return undefined
 }
 
+export const handlerNewsletter = (
+  input: NewsletterInput
+): NewsletterResult | undefined => {
+  for (const handler of contentHandlers) {
+    if (handler.isNewsletter(input.postHeader, input.from, input.unSubHeader)) {
+      return handler.handleNewsletter(input)
+    }
+  }
+
+  return undefined
+}
+
 module.exports = {
   preHandleContent,
+  handlerNewsletter,
 }

packages/content-handler/src/morning-brew-handler.ts

@@ -0,0 +1,17 @@
+import { ContentHandler } from './content-handler'
+
+export class MorningBrewHandler extends ContentHandler {
+  constructor() {
+    super()
+    this.senderRegex = /Morning Brew <crew@morningbrew.com>/
+    this.urlRegex = /<a.* href=["']([^"']*)["'].*>View Online<\/a>/
+    this.defaultUrl = 'https://www.morningbrew.com'
+    this.name = 'Morning Brew'
+  }
+
+  isNewsletter(postHeader: string, from: string, unSubHeader: string): boolean {
+    // Axios newsletter is from <xx@axios.com>
+    const re = new RegExp(this.senderRegex)
+    return re.test(from) && (!!postHeader || !!unSubHeader)
+  }
+}

packages/content-handler/src/substack-handler.ts

@@ -0,0 +1,22 @@
+import addressparser from 'addressparser'
+import { ContentHandler } from './content-handler'
+
+export class SubstackHandler extends ContentHandler {
+  constructor() {
+    super()
+    this.defaultUrl = 'https://www.substack.com'
+    this.name = 'Substack'
+  }
+
+  isNewsletter(postHeader: string, from: string, unSubHeader: string): boolean {
+    return !!postHeader
+  }
+
+  parseNewsletterUrl(postHeader: string, html: string): string | undefined {
+    // raw SubStack newsletter url is like <https://hongbo130.substack.com/p/tldr>
+    // we need to get the real url from the raw url
+    return addressparser(postHeader).length > 0
+      ? addressparser(postHeader)[0].name
+      : undefined
+  }
+}

packages/content-handler/test/newsletter.test.ts

@@ -0,0 +1,70 @@
+import { expect } from 'chai'
+import { SubstackHandler } from '../src/substack-handler'
+import { AxiosHandler } from '../src/axios-handler'
+import { BloombergHandler } from '../src/bloomberg-handler'
+import { GolangHandler } from '../src/golang-handler'
+import { MorningBrewHandler } from '../src/morning-brew-handler'
+
+describe('Newsletter email test', () => {
+  describe('#getNewsletterUrl()', () => {
+    it('returns url when email is from SubStack', () => {
+      const rawUrl = '<https://hongbo130.substack.com/p/tldr>'
+
+      expect(new SubstackHandler().parseNewsletterUrl(rawUrl, '')).to.equal(
+        'https://hongbo130.substack.com/p/tldr'
+      )
+    })
+
+    it('returns url when email is from Axios', () => {
+      const url = 'https://axios.com/blog/the-best-way-to-build-a-web-app'
+      const html = `View in browser at <a>${url}</a>`
+
+      expect(new AxiosHandler().parseNewsletterUrl('', html)).to.equal(url)
+    })
+
+    it('returns url when email is from Bloomberg', () => {
+      const url = 'https://www.bloomberg.com/news/google-is-now-a-partner'
+      const html = `
+        <a class="view-in-browser__url" href="${url}">
+        View in browser
+        </a>
+      `
+
+      expect(new BloombergHandler().parseNewsletterUrl('', html)).to.equal(url)
+    })
+
+    it('returns url when email is from Golang Weekly', () => {
+      const url = 'https://www.golangweekly.com/first'
+      const html = `
+        <a href="${url}" style="text-decoration: none">Read on the Web</a>
+      `
+
+      expect(new GolangHandler().parseNewsletterUrl('', html)).to.equal(url)
+    })
+
+    it('returns url when email is from Morning Brew', () => {
+      const url = 'https://www.morningbrew.com/daily/issues/first'
+      const html = `
+        <a style="color: #000000; text-decoration: none;" target="_blank" rel="noopener" href="${url}">View Online</a>
+      `
+
+      expect(new MorningBrewHandler().parseNewsletterUrl('', html)).to.equal(
+        url
+      )
+    })
+  })
+
+  describe('get author from email address', () => {
+    it('returns author when email is from Substack', () => {
+      const from = 'Jackson Harper from Omnivore App <jacksonh@substack.com>'
+      expect(new AxiosHandler().parseAuthor(from)).to.equal(
+        'Jackson Harper from Omnivore App'
+      )
+    })
+
+    it('returns author when email is from Axios', () => {
+      const from = 'Mike Allen <mike@axios.com>'
+      expect(new AxiosHandler().parseAuthor(from)).to.equal('Mike Allen')
+    })
+  })
+})

packages/puppeteer-parse/test/babel-register.js

@@ -0,0 +1,3 @@
+const register = require('@babel/register').default
+
+register({ extensions: ['.ts', '.tsx', '.js', '.jsx'] })

packages/puppeteer-parse/test/stub.test.ts

@@ -0,0 +1,13 @@
+import 'mocha'
+import * as chai from 'chai'
+import { expect } from 'chai'
+import 'chai/register-should'
+import chaiString from 'chai-string'
+
+chai.use(chaiString)
+
+describe('Stub test', () => {
+  it('should pass', () => {
+    expect(true).to.be.true
+  })
+})

yarn.lock

@@ -24933,6 +24933,11 @@ uuid@^8.0.0, uuid@^8.3.0, uuid@^8.3.1, uuid@^8.3.2:
   resolved "https://registry.yarnpkg.com/uuid/-/uuid-8.3.2.tgz#80d5b5ced271bb9af6c445f21a1a04c606cefbe2"
   integrity sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==
 
+uuid@^9.0.0:
+  version "9.0.0"
+  resolved "https://registry.yarnpkg.com/uuid/-/uuid-9.0.0.tgz#592f550650024a38ceb0c562f2f6aa435761efb5"
+  integrity sha512-MXcSTerfPa4uqyzStbRoTgt5XIe3x5+42+q1sDuy3R5MDk66URdLMOZe5aPX/SQd+kuYAh0FdP/pO28IkQyTeg==
+
 v8-compile-cache-lib@^3.0.1:
   version "3.0.1"
   resolved "https://registry.yarnpkg.com/v8-compile-cache-lib/-/v8-compile-cache-lib-3.0.1.tgz#6336e8d71965cb3d35a1bbb7868445a7c05264bf"