From 67082a478e5c92ee763b12a3357667009c18359e Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Tue, 1 Nov 2022 16:41:42 +0800 Subject: [PATCH 1/3] Add support for hey world newsletters --- .../src/newsletters/hey-world-handler.ts | 27 ++ .../test/data/hey-world-newsletter.html | 274 ++++++++++++++++++ .../content-handler/test/newsletter.test.ts | 36 ++- 3 files changed, 335 insertions(+), 2 deletions(-) create mode 100644 packages/content-handler/src/newsletters/hey-world-handler.ts create mode 100644 packages/content-handler/test/data/hey-world-newsletter.html diff --git a/packages/content-handler/src/newsletters/hey-world-handler.ts b/packages/content-handler/src/newsletters/hey-world-handler.ts new file mode 100644 index 000000000..5542bc2ef --- /dev/null +++ b/packages/content-handler/src/newsletters/hey-world-handler.ts @@ -0,0 +1,27 @@ +import { ContentHandler } from '../content-handler' + +export class HeyWorldHandler extends ContentHandler { + constructor() { + super() + this.name = 'hey-world' + this.senderRegex = /<.+@world.hey.com>/ + } + + findNewsletterHeaderHref(dom: Document): string | undefined { + const readOnline = dom.querySelectorAll('a') + let res: string | undefined = undefined + readOnline.forEach((e) => { + if (e.textContent === 'View this post online') { + res = e.getAttribute('href') || undefined + } + }) + return res + } + + async parseNewsletterUrl( + postHeader: string, + html: string + ): Promise { + return this.findNewsletterUrl(html) + } +} diff --git a/packages/content-handler/test/data/hey-world-newsletter.html b/packages/content-handler/test/data/hey-world-newsletter.html new file mode 100644 index 000000000..b774aaef0 --- /dev/null +++ b/packages/content-handler/test/data/hey-world-newsletter.html @@ -0,0 +1,274 @@ + + + + + + + + + + + +
+
+
+ With Twitter under new ownership, it's a great time for us to imagine + how this hellscape might actually get fixed. And by fixed, I mean be a + place where people enjoy hanging out, where the debate over content + moderation is settled, and where Elon Musk might make his money back. + Easy peasy then!

To start, here's my fundamental premise: + The internet is full of the bland, the wonderful, the obscure, the + sacrilegious, the offensive, the infuriating, and the nasty. The + internet, as a whole, is governed by local laws on speech, not by a + council of global content moderators, not by single individuals. This + is a feature not a bug.

An American can sue another + American under US defamation laws for stuff posted on the internet. + But the Thai government won't get an American extradited under its + lèse-majesté laws if that person offends the country's monarch on the + internet. Restrictions on speech are based on jurisdictions (and + Twitter is thankfully liable in one of most liberal speech + jurisdictions around). This is also a feature not a bug.

Twitter + should embrace these features, and accept its role as a fundamental + protocol. Just like HTTP, just like SMTP, just like FTP. All protocols + that can be used to transmit all sorts of disagreeable content, but + where the moderation of said content is left to individuals, not some + all-knowing overlords.

This isn't possible today because + Twitter shoves so much speech in front of you that you never asked to + see. It allows strangers to invade popular threads with no protections + available to the original author.

You know that you can + find the most vile, nasty shit on the internet if you search for the + right words on Google. So most people just don't do that! They manage + their own exposure by moderating their own behavior. This is again a + feature, not a bug.

This could be Twitter!

If + Twitter stopped shoving all this speech you never asked to see in + front of you, there would be no need for an army of content + moderators. The internet at large is not filtered through such an + army, and it's all the better for it. Governments keep trying to + impose such a filter, but so far only the most authoritarian regimes + have succeeded (and up until a few minutes ago, there was broad + consensus in the Western world that this was bad).

This + would mean that Twitter would have to change in a few fundamental + ways:

+
+
    +
  1. + Only speech from people you've asked to follow should appear in your + feed. If any of those people post something disagreeable to your + tastes, you can moderate that content by unfollowing them. +
    2. +
  2. + Authors should be deputized as content moderators of their own + posts. Just like someone hosting a blog with comments on the + internet is. You can either say you don't want comments at all, that + comments need to be pre-approved, that you'll remove the bad ones, + or that you'll just let it flow. +
    3. +
  3. + All the promoted tweets that Twitter wants to showcase to the world + should pass through a positive rather than negative editorial + process. Meaning Twitter employees pick what they like to feature + rather than ban what they don't. +
    4. +
+
+
Thus, like the internet at large, everyone gets to control the + kind of speech they're exposed to. Different people will come to + different conclusions on the kind speech they wish to be exposed to. + This is the fundamental wisdom of the infamous Section 230 protection + that has allowed the internet as we know it to exist.

This + curation should be something you can outsource to trusted delegates. + Let me experience Twitter as a list of authors and threads maintained + by someone I trust.

It'll also mean, like the internet at + large, that you'll be able to find Bad Words, Terrible Words, No Good + Words, if you deliberately seek them out. Twitter will serve as a + portal to this in the same way Google does. These words may be hurtful + or harmful.

But all the content moderators in the world + couldn't put the utopia that a platform for hundreds of millions won't + contain disagreeable speech together again.

Accept that + reality. Accept that the price we pay for an open, free internet is + one which permits the darker sides of humanity. Lean on the local + jurisdictions and laws, democratically controlled as they are, to + police this reality. Like we've done with that beautiful bastard that + is the internet.

Let Twitter be free to ascend to sit + amongst all the other fundamental protocols that constitutes the + pantheon of the internet.

You can do it, Mr Musk! +
+
+ +
+ +
+ +

+ +

+

+ You’re subscribed to + + on + +

+

+ Don’t want these posts anymore? + +

+

+ HEY: PO Box 803338, PMB 53289, Chicago, Illinois 60680 +

+
+ + diff --git a/packages/content-handler/test/newsletter.test.ts b/packages/content-handler/test/newsletter.test.ts index 93e2a0822..b0b28fa37 100644 --- a/packages/content-handler/test/newsletter.test.ts +++ b/packages/content-handler/test/newsletter.test.ts @@ -17,6 +17,7 @@ import { GhostHandler } from '../src/newsletters/ghost-handler' import { CooperPressHandler } from '../src/newsletters/cooper-press-handler' import { getNewsletterHandler } from '../src' import { parseHTML } from 'linkedom' +import { HeyWorldHandler } from '../src/newsletters/hey-world-handler' chai.use(chaiAsPromised) chai.use(chaiString) @@ -144,7 +145,8 @@ describe('Newsletter email test', () => { }) it('fixes up static tweets in Substack newsletters', async () => { - const url = 'https://astralcodexten.substack.com/p/nick-cammarata-on-jhana' + const url = + 'https://astralcodexten.substack.com/p/nick-cammarata-on-jhana' const html = load( './test/data/substack-with-static-tweets-newsletter.html' ) @@ -160,7 +162,9 @@ describe('Newsletter email test', () => { expect(handler?.shouldPreParse(url, dom)).to.be.true const preparsed = await handler?.preParse(url, dom) - const tweets = Array.from(preparsed?.querySelectorAll('div[class="_omnivore-static-tweet"]') ?? []) + const tweets = Array.from( + preparsed?.querySelectorAll('div[class="_omnivore-static-tweet"]') ?? [] + ) expect(tweets.length).to.eq(7) }) @@ -219,6 +223,18 @@ describe('Newsletter email test', () => { }) expect(handler).to.be.instanceOf(CooperPressHandler) }) + + it('returns HeyWorldHandler for hey world newsletter', async () => { + const html = load('./test/data/hey-world-newsletter.html') + const handler = await getNewsletterHandler({ + html, + postHeader: '', + from: 'Hongbo Wu ', + unSubHeader: + '', + }) + expect(handler).to.be.instanceOf(HeyWorldHandler) + }) }) describe('findNewsletterUrl', async () => { @@ -334,6 +350,22 @@ describe('Newsletter email test', () => { expect(url).to.startWith('https://nodeweekly.com/issues/459') }) }) + + context('when email is from hey world', () => { + before(() => { + nock('https://world.hey.com') + .head('/dhh/here-s-how-to-fix-twitter-79632ecb') + .reply(200, '') + }) + + it('gets the URL from the header', async () => { + const html = load('./test/data/hey-world-newsletter.html') + const url = await new HeyWorldHandler().findNewsletterUrl(html) + expect(url).to.startWith( + 'https://world.hey.com/dhh/here-s-how-to-fix-twitter-79632ecb' + ) + }) + }) }) describe('generateUniqueUrl', () => { From ae3ff6f34bfc70625ec4699901ed580fb86e0bcd Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Tue, 1 Nov 2022 17:48:39 +0800 Subject: [PATCH 2/3] Add test case --- packages/content-handler/src/index.ts | 2 + .../test/data/tomasz-tunguz-newsletter.html | 88 +++++++++++++++++++ .../content-handler/test/newsletter.test.ts | 11 +++ 3 files changed, 101 insertions(+) create mode 100644 packages/content-handler/test/data/tomasz-tunguz-newsletter.html diff --git a/packages/content-handler/src/index.ts b/packages/content-handler/src/index.ts index 052642a50..ce61c0a64 100644 --- a/packages/content-handler/src/index.ts +++ b/packages/content-handler/src/index.ts @@ -27,6 +27,7 @@ import { RevueHandler } from './newsletters/revue-handler' import { GhostHandler } from './newsletters/ghost-handler' import { parseHTML } from 'linkedom' import { CooperPressHandler } from './newsletters/cooper-press-handler' +import { HeyWorldHandler } from './newsletters/hey-world-handler' const validateUrlString = (url: string) => { const u = new URL(url) @@ -75,6 +76,7 @@ const newsletterHandlers: ContentHandler[] = [ new RevueHandler(), new GhostHandler(), new CooperPressHandler(), + new HeyWorldHandler(), ] export const preHandleContent = async ( diff --git a/packages/content-handler/test/data/tomasz-tunguz-newsletter.html b/packages/content-handler/test/data/tomasz-tunguz-newsletter.html new file mode 100644 index 000000000..68c35fb17 --- /dev/null +++ b/packages/content-handler/test/data/tomasz-tunguz-newsletter.html @@ -0,0 +1,88 @@ +


---------- Forwarded message ---------
From: Tomasz Tunguz <blog@tomtunguz.com>
Date: Fri, 28 Oct 2022 at 19:15
Subject: Grey Skies in Cloud Earnings
To: <martin@doversten.se>


+ + + + + +
+
+ + +
+
+

Tomasz Tunguz

+

Venture Capitalist at Redpoint

+
+
+

If you were forwarded this newsletter, and you'd like to receive it in the future, subscribe here.

+

Grey Skies in Cloud Earnings

+

+

+

I’m watching public company earnings to identify early weaknesses in the software market. This week Microsoft, Google/ Alphabet, & Amazon reported their third quarter figures.

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
CompanyQ-4 CAGRQ-4 CAGRQ-3 CAGRQ-2 CAGRQ-1 CAGRQ0 CAGR
Microsoft Azure50%51%46%46%40%35%
Google Cloud Platform46%54%45%51%35%38%
Amazon Web Services37%39%40%40%33%27%
+

The meaningful decline in growth rates started four quarters ago is hard to ignore.

+

image + Let’s put these figures into context. Infrastructure revenue growth averaged 33% this quarter, which is astounding considering we’re talking about businesses that sum to more than $50b of revenue per quarter.

+

Customers spent $12b more this quarter - an astounding figure. At a 7x multiple of revenue, that is another $84b of market cap creation, in theory.

+

But, the change in slope is meaningful, too. A year ago, these business units grew at 44% annually. The current recession is evident in the charts: growth rates have declined by 25% in six months’ time.

+

The kink downwards in the red line at Q-2 shows a sudden deceleration in AWS’ growth rate. GCP is more volatile, likely driven by the volatility of bigger deals closing on a smaller revenue base. Meanwhile, Azure has declined in a more steady cadence.

+

The data suggests a broader slowdown in software spending as these companies are indices of software buyers’ behavior.

+

Public software multiples - 4.9x as of last week, not far from the decade low of 3.3x - should depress further given these results & forward guidance. Microsoft stock is down 10%; Amazon fell about 20%; Google about 12%.

+

Lopping another 15% off multiples implies the median software company should trade at 4.2x forward revenues, a far cry from the 15x we observed not twelve months ago.

+ + +

+
+
+
+

If you didn’t enjoy the email you can unsubscribe here.

+

2969 Woodside Road, Woodside, CA 94062

+
+ +
+ + +
+ +
diff --git a/packages/content-handler/test/newsletter.test.ts b/packages/content-handler/test/newsletter.test.ts index b0b28fa37..481abd982 100644 --- a/packages/content-handler/test/newsletter.test.ts +++ b/packages/content-handler/test/newsletter.test.ts @@ -235,6 +235,17 @@ describe('Newsletter email test', () => { }) expect(handler).to.be.instanceOf(HeyWorldHandler) }) + + it('returns ConvertkitHandler for Tomasz Tunguz newsletter', async () => { + const html = load('./test/data/tomasz-tunguz-newsletter.html') + const handler = await getNewsletterHandler({ + html, + postHeader: '', + from: '', + unSubHeader: '', + }) + expect(handler).to.be.instanceOf(ConvertkitHandler) + }) }) describe('findNewsletterUrl', async () => { From 9d753e3c0e8b1182edf9f03523ebdc86244af517 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Wed, 2 Nov 2022 15:27:30 +0800 Subject: [PATCH 3/3] Forward convertkit confirmation email --- .../src/newsletters/convertkit-handler.ts | 18 +- .../test/data/convertkit-confirmation.html | 268 ++++++++++++++++++ .../content-handler/test/newsletter.test.ts | 11 + 3 files changed, 293 insertions(+), 4 deletions(-) create mode 100644 packages/content-handler/test/data/convertkit-confirmation.html diff --git a/packages/content-handler/src/newsletters/convertkit-handler.ts b/packages/content-handler/src/newsletters/convertkit-handler.ts index 15aebae26..cf0b818b5 100644 --- a/packages/content-handler/src/newsletters/convertkit-handler.ts +++ b/packages/content-handler/src/newsletters/convertkit-handler.ts @@ -27,11 +27,21 @@ export class ConvertkitHandler extends ContentHandler { dom: Document }): Promise { const dom = input.dom - return Promise.resolve( - dom.querySelectorAll( - 'img[src*="convertkit.com"], img[src*="convertkit-mail"]' - ).length > 0 + const icons = dom.querySelectorAll( + 'img[src*="convertkit.com"], img[src*="convertkit-mail"]' ) + if (icons.length === 0) { + return Promise.resolve(false) + } + // ignore newsletters that have a confirmation link to the newsletter in the body + const links = dom.querySelectorAll( + 'a[href*="convertkit.com"], a[href*="convertkit-mail"]' + ) + const isConfirmation = Array.from(links).some((e) => { + return e.textContent === 'Confirm your subscription' + }) + + return Promise.resolve(!isConfirmation) } async parseNewsletterUrl( diff --git a/packages/content-handler/test/data/convertkit-confirmation.html b/packages/content-handler/test/data/convertkit-confirmation.html new file mode 100644 index 000000000..99218b36d --- /dev/null +++ b/packages/content-handler/test/data/convertkit-confirmation.html @@ -0,0 +1,268 @@ + + + + + + + +
+ + + + + + + + +
+ + + + diff --git a/packages/content-handler/test/newsletter.test.ts b/packages/content-handler/test/newsletter.test.ts index 481abd982..9d859a1dd 100644 --- a/packages/content-handler/test/newsletter.test.ts +++ b/packages/content-handler/test/newsletter.test.ts @@ -246,6 +246,17 @@ describe('Newsletter email test', () => { }) expect(handler).to.be.instanceOf(ConvertkitHandler) }) + + it('returns undefined for convertkit confirmation email', async () => { + const html = load('./test/data/convertkit-confirmation.html') + const handler = await getNewsletterHandler({ + html, + postHeader: '', + from: '', + unSubHeader: '', + }) + expect(handler).to.be.undefined + }) }) describe('findNewsletterUrl', async () => {