diff --git a/packages/content-fetch/test/babel-register.js b/packages/content-fetch/test/babel-register.js new file mode 100644 index 000000000..a6f65f60a --- /dev/null +++ b/packages/content-fetch/test/babel-register.js @@ -0,0 +1,3 @@ +const register = require('@babel/register').default + +register({ extensions: ['.ts', '.tsx', '.js', '.jsx'] }) diff --git a/packages/content-fetch/test/stub.test.ts b/packages/content-fetch/test/stub.test.ts new file mode 100644 index 000000000..173ca4917 --- /dev/null +++ b/packages/content-fetch/test/stub.test.ts @@ -0,0 +1,13 @@ +import 'mocha' +import * as chai from 'chai' +import { expect } from 'chai' +import 'chai/register-should' +import chaiString from 'chai-string' + +chai.use(chaiString) + +describe('Stub test', () => { + it('should pass', () => { + expect(true).to.be.true + }) +}) diff --git a/packages/content-handler/package.json b/packages/content-handler/package.json index bae0e5953..c82d40c5d 100644 --- a/packages/content-handler/package.json +++ b/packages/content-handler/package.json @@ -18,9 +18,12 @@ "eslint-plugin-prettier": "^4.0.0" }, "dependencies": { + "addressparser": "^1.0.1", "axios": "^0.27.2", "linkedom": "^0.14.16", "luxon": "^3.0.4", - "underscore": "^1.13.6" + "rfc2047": "^4.0.1", + "underscore": "^1.13.6", + "uuid": "^9.0.0" } } diff --git a/packages/content-handler/src/axios-handler.ts b/packages/content-handler/src/axios-handler.ts new file mode 100644 index 000000000..73b4b3e8b --- /dev/null +++ b/packages/content-handler/src/axios-handler.ts @@ -0,0 +1,17 @@ +import { ContentHandler } from './content-handler' + +export class AxiosHandler extends ContentHandler { + constructor() { + super() + this.senderRegex = /<.+@axios.com>/ + this.urlRegex = /View in browser at (.*)<\/a>/ + this.defaultUrl = 'https://axios.com' + this.name = 'Axios' + } + + isNewsletter(postHeader: string, from: string, unSubHeader: string): boolean { + // Axios newsletter is from + const re = new RegExp(this.senderRegex) + return re.test(from) && (!!postHeader || !!unSubHeader) + } +} diff --git a/packages/content-handler/src/bloomberg-handler.ts b/packages/content-handler/src/bloomberg-handler.ts index 95670630e..d82941e5a 100644 --- a/packages/content-handler/src/bloomberg-handler.ts +++ b/packages/content-handler/src/bloomberg-handler.ts @@ -5,6 +5,9 @@ import { ContentHandler, PreHandleResult } from './content-handler' export class BloombergHandler extends ContentHandler { constructor() { super() + this.senderRegex = /<.+@mail.bloomberg.*.com>/ + this.urlRegex = / + const re = new RegExp(this.senderRegex) + return re.test(from) && (!!postHeader || !!unSubHeader) + } } diff --git a/packages/content-handler/src/content-handler.ts b/packages/content-handler/src/content-handler.ts index c49987368..b69c3cbfe 100644 --- a/packages/content-handler/src/content-handler.ts +++ b/packages/content-handler/src/content-handler.ts @@ -7,7 +7,16 @@ interface Unsubscribe { httpUrl?: string } -interface NewsletterMessage { +export interface NewsletterInput { + postHeader: string + from: string + unSubHeader: string + email: string + html: string + title: string +} + +export interface NewsletterResult { email: string content: string url: string @@ -55,9 +64,7 @@ export abstract class ContentHandler { } isNewsletter(postHeader: string, from: string, unSubHeader: string): boolean { - // Axios newsletter is from - const re = new RegExp(this.senderRegex) - return re.test(from) && (!!postHeader || !!unSubHeader) + return false } parseNewsletterUrl(_postHeader: string, html: string): string | undefined { @@ -90,14 +97,14 @@ export abstract class ContentHandler { } } - handleNewsletter( - email: string, - html: string, - postHeader: string, - title: string, - from: string, - unSubHeader: string - ): NewsletterMessage { + handleNewsletter({ + email, + html, + postHeader, + title, + from, + unSubHeader, + }: NewsletterInput): NewsletterResult { console.log('handleNewsletter', email, postHeader, title, from) if (!email || !html || !title || !from) { diff --git a/packages/content-handler/src/golang-handler.ts b/packages/content-handler/src/golang-handler.ts new file mode 100644 index 000000000..d7d618f2e --- /dev/null +++ b/packages/content-handler/src/golang-handler.ts @@ -0,0 +1,17 @@ +import { ContentHandler } from './content-handler' + +export class GolangHandler extends ContentHandler { + constructor() { + super() + this.senderRegex = /<.+@golangweekly.com>/ + this.urlRegex = /Read on the Web<\/a>/ + this.defaultUrl = 'https://golangweekly.com' + this.name = 'Golang Weekly' + } + + isNewsletter(postHeader: string, from: string, unSubHeader: string): boolean { + // Axios newsletter is from + const re = new RegExp(this.senderRegex) + return re.test(from) && (!!postHeader || !!unSubHeader) + } +} diff --git a/packages/content-handler/src/index.ts b/packages/content-handler/src/index.ts index 3275a3c89..88509c3e8 100644 --- a/packages/content-handler/src/index.ts +++ b/packages/content-handler/src/index.ts @@ -8,7 +8,16 @@ import { ScrapingBeeHandler } from './scrapingBee-handler' import { TDotCoHandler } from './t-dot-co-handler' import { TwitterHandler } from './twitter-handler' import { YoutubeHandler } from './youtube-handler' -import { ContentHandler, PreHandleResult } from './content-handler' +import { + ContentHandler, + NewsletterInput, + NewsletterResult, + PreHandleResult, +} from './content-handler' +import { SubstackHandler } from './substack-handler' +import { AxiosHandler } from './axios-handler' +import { GolangHandler } from './golang-handler' +import { MorningBrewHandler } from './morning-brew-handler' const validateUrlString = (url: string) => { const u = new URL(url) @@ -37,6 +46,10 @@ const contentHandlers: ContentHandler[] = [ new TDotCoHandler(), new TwitterHandler(), new YoutubeHandler(), + new SubstackHandler(), + new AxiosHandler(), + new GolangHandler(), + new MorningBrewHandler(), ] export const preHandleContent = async ( @@ -71,6 +84,19 @@ export const preHandleContent = async ( return undefined } +export const handlerNewsletter = ( + input: NewsletterInput +): NewsletterResult | undefined => { + for (const handler of contentHandlers) { + if (handler.isNewsletter(input.postHeader, input.from, input.unSubHeader)) { + return handler.handleNewsletter(input) + } + } + + return undefined +} + module.exports = { preHandleContent, + handlerNewsletter, } diff --git a/packages/content-handler/src/morning-brew-handler.ts b/packages/content-handler/src/morning-brew-handler.ts new file mode 100644 index 000000000..4d46e14e4 --- /dev/null +++ b/packages/content-handler/src/morning-brew-handler.ts @@ -0,0 +1,17 @@ +import { ContentHandler } from './content-handler' + +export class MorningBrewHandler extends ContentHandler { + constructor() { + super() + this.senderRegex = /Morning Brew / + this.urlRegex = /View Online<\/a>/ + this.defaultUrl = 'https://www.morningbrew.com' + this.name = 'Morning Brew' + } + + isNewsletter(postHeader: string, from: string, unSubHeader: string): boolean { + // Axios newsletter is from + const re = new RegExp(this.senderRegex) + return re.test(from) && (!!postHeader || !!unSubHeader) + } +} diff --git a/packages/content-handler/src/substack-handler.ts b/packages/content-handler/src/substack-handler.ts new file mode 100644 index 000000000..973ed59ea --- /dev/null +++ b/packages/content-handler/src/substack-handler.ts @@ -0,0 +1,22 @@ +import addressparser from 'addressparser' +import { ContentHandler } from './content-handler' + +export class SubstackHandler extends ContentHandler { + constructor() { + super() + this.defaultUrl = 'https://www.substack.com' + this.name = 'Substack' + } + + isNewsletter(postHeader: string, from: string, unSubHeader: string): boolean { + return !!postHeader + } + + parseNewsletterUrl(postHeader: string, html: string): string | undefined { + // raw SubStack newsletter url is like + // we need to get the real url from the raw url + return addressparser(postHeader).length > 0 + ? addressparser(postHeader)[0].name + : undefined + } +} diff --git a/packages/content-handler/test/newsletter.test.ts b/packages/content-handler/test/newsletter.test.ts new file mode 100644 index 000000000..75a91b438 --- /dev/null +++ b/packages/content-handler/test/newsletter.test.ts @@ -0,0 +1,70 @@ +import { expect } from 'chai' +import { SubstackHandler } from '../src/substack-handler' +import { AxiosHandler } from '../src/axios-handler' +import { BloombergHandler } from '../src/bloomberg-handler' +import { GolangHandler } from '../src/golang-handler' +import { MorningBrewHandler } from '../src/morning-brew-handler' + +describe('Newsletter email test', () => { + describe('#getNewsletterUrl()', () => { + it('returns url when email is from SubStack', () => { + const rawUrl = '' + + expect(new SubstackHandler().parseNewsletterUrl(rawUrl, '')).to.equal( + 'https://hongbo130.substack.com/p/tldr' + ) + }) + + it('returns url when email is from Axios', () => { + const url = 'https://axios.com/blog/the-best-way-to-build-a-web-app' + const html = `View in browser at ${url}` + + expect(new AxiosHandler().parseNewsletterUrl('', html)).to.equal(url) + }) + + it('returns url when email is from Bloomberg', () => { + const url = 'https://www.bloomberg.com/news/google-is-now-a-partner' + const html = ` + + View in browser + + ` + + expect(new BloombergHandler().parseNewsletterUrl('', html)).to.equal(url) + }) + + it('returns url when email is from Golang Weekly', () => { + const url = 'https://www.golangweekly.com/first' + const html = ` + Read on the Web + ` + + expect(new GolangHandler().parseNewsletterUrl('', html)).to.equal(url) + }) + + it('returns url when email is from Morning Brew', () => { + const url = 'https://www.morningbrew.com/daily/issues/first' + const html = ` + View Online + ` + + expect(new MorningBrewHandler().parseNewsletterUrl('', html)).to.equal( + url + ) + }) + }) + + describe('get author from email address', () => { + it('returns author when email is from Substack', () => { + const from = 'Jackson Harper from Omnivore App ' + expect(new AxiosHandler().parseAuthor(from)).to.equal( + 'Jackson Harper from Omnivore App' + ) + }) + + it('returns author when email is from Axios', () => { + const from = 'Mike Allen ' + expect(new AxiosHandler().parseAuthor(from)).to.equal('Mike Allen') + }) + }) +}) diff --git a/packages/puppeteer-parse/test/babel-register.js b/packages/puppeteer-parse/test/babel-register.js new file mode 100644 index 000000000..a6f65f60a --- /dev/null +++ b/packages/puppeteer-parse/test/babel-register.js @@ -0,0 +1,3 @@ +const register = require('@babel/register').default + +register({ extensions: ['.ts', '.tsx', '.js', '.jsx'] }) diff --git a/packages/puppeteer-parse/test/stub.test.ts b/packages/puppeteer-parse/test/stub.test.ts new file mode 100644 index 000000000..173ca4917 --- /dev/null +++ b/packages/puppeteer-parse/test/stub.test.ts @@ -0,0 +1,13 @@ +import 'mocha' +import * as chai from 'chai' +import { expect } from 'chai' +import 'chai/register-should' +import chaiString from 'chai-string' + +chai.use(chaiString) + +describe('Stub test', () => { + it('should pass', () => { + expect(true).to.be.true + }) +}) diff --git a/yarn.lock b/yarn.lock index 3da81da9b..32066d1bb 100644 --- a/yarn.lock +++ b/yarn.lock @@ -24933,6 +24933,11 @@ uuid@^8.0.0, uuid@^8.3.0, uuid@^8.3.1, uuid@^8.3.2: resolved "https://registry.yarnpkg.com/uuid/-/uuid-8.3.2.tgz#80d5b5ced271bb9af6c445f21a1a04c606cefbe2" integrity sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg== +uuid@^9.0.0: + version "9.0.0" + resolved "https://registry.yarnpkg.com/uuid/-/uuid-9.0.0.tgz#592f550650024a38ceb0c562f2f6aa435761efb5" + integrity sha512-MXcSTerfPa4uqyzStbRoTgt5XIe3x5+42+q1sDuy3R5MDk66URdLMOZe5aPX/SQd+kuYAh0FdP/pO28IkQyTeg== + v8-compile-cache-lib@^3.0.1: version "3.0.1" resolved "https://registry.yarnpkg.com/v8-compile-cache-lib/-/v8-compile-cache-lib-3.0.1.tgz#6336e8d71965cb3d35a1bbb7868445a7c05264bf"