diff --git a/packages/content-handler/src/newsletters/substack-handler.ts b/packages/content-handler/src/newsletters/substack-handler.ts index f94267171..d958ca26d 100644 --- a/packages/content-handler/src/newsletters/substack-handler.ts +++ b/packages/content-handler/src/newsletters/substack-handler.ts @@ -34,6 +34,10 @@ export class SubstackHandler extends ContentHandler { body?.querySelector('.post-cta')?.remove() body?.querySelector('.container-border')?.remove() body?.querySelector('.footer')?.remove() + // this removes the "restack" button + body?.querySelector('.email-ufi-2-bottom')?.remove() + // this removes the "share" button + body?.querySelector('.email-ufi-2-top')?.remove() dom = this.fixupStaticTweets(dom) diff --git a/packages/content-handler/test/data/prepared/substack-newsletter-new.html b/packages/content-handler/test/data/prepared/substack-newsletter-new.html new file mode 100644 index 000000000..31f510030 --- /dev/null +++ b/packages/content-handler/test/data/prepared/substack-newsletter-new.html @@ -0,0 +1,1116 @@ + + + + Title + + + + +
+ subtitle +  ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ +
+ + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/packages/content-handler/test/data/substack-newsletter-new.html b/packages/content-handler/test/data/substack-newsletter-new.html new file mode 100644 index 000000000..8d2c0e0e6 --- /dev/null +++ b/packages/content-handler/test/data/substack-newsletter-new.html @@ -0,0 +1,2172 @@ + + + + Title + + + + +
+ subtitle +  ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ +
+ + + + + + + + + + + + + + + + + diff --git a/packages/content-handler/test/newsletter.test.ts b/packages/content-handler/test/newsletter.test.ts index ac40c139c..7188edbe8 100644 --- a/packages/content-handler/test/newsletter.test.ts +++ b/packages/content-handler/test/newsletter.test.ts @@ -1,27 +1,27 @@ -import 'mocha' import * as chai from 'chai' import { expect } from 'chai' import chaiAsPromised from 'chai-as-promised' import chaiString from 'chai-string' -import { SubstackHandler } from '../src/newsletters/substack-handler' -import { AxiosHandler } from '../src/newsletters/axios-handler' -import { BloombergNewsletterHandler } from '../src/newsletters/bloomberg-newsletter-handler' -import { GolangHandler } from '../src/newsletters/golang-handler' -import { MorningBrewHandler } from '../src/newsletters/morning-brew-handler' -import nock from 'nock' -import { generateUniqueUrl } from '../src/content-handler' import fs from 'fs' -import { BeehiivHandler } from '../src/newsletters/beehiiv-handler' -import { ConvertkitHandler } from '../src/newsletters/convertkit-handler' -import { GhostHandler } from '../src/newsletters/ghost-handler' -import { CooperPressHandler } from '../src/newsletters/cooper-press-handler' -import { getNewsletterHandler } from '../src' import { parseHTML } from 'linkedom' -import { HeyWorldHandler } from '../src/newsletters/hey-world-handler' -import { GenericHandler } from '../src/newsletters/generic-handler' -import { EveryIoHandler } from '../src/newsletters/every-io-handler' +import 'mocha' +import nock from 'nock' +import { getNewsletterHandler } from '../src' +import { generateUniqueUrl } from '../src/content-handler' +import { AxiosHandler } from '../src/newsletters/axios-handler' +import { BeehiivHandler } from '../src/newsletters/beehiiv-handler' +import { BloombergNewsletterHandler } from '../src/newsletters/bloomberg-newsletter-handler' +import { ConvertkitHandler } from '../src/newsletters/convertkit-handler' +import { CooperPressHandler } from '../src/newsletters/cooper-press-handler' import { EnergyWorldHandler } from '../src/newsletters/energy-world' +import { EveryIoHandler } from '../src/newsletters/every-io-handler' +import { GenericHandler } from '../src/newsletters/generic-handler' +import { GhostHandler } from '../src/newsletters/ghost-handler' +import { GolangHandler } from '../src/newsletters/golang-handler' +import { HeyWorldHandler } from '../src/newsletters/hey-world-handler' import { IndiaTimesHandler } from '../src/newsletters/india-times-handler' +import { MorningBrewHandler } from '../src/newsletters/morning-brew-handler' +import { SubstackHandler } from '../src/newsletters/substack-handler' chai.use(chaiAsPromised) chai.use(chaiString) @@ -571,4 +571,20 @@ describe('Newsletter email test', () => { }) }) }) + + describe('preParse', () => { + context('when email is from Substack', () => { + it('removes the Substack footer and header', async () => { + const url = 'https://blog.omnivore.app/p/omnivore-2021-01-31' + const html = load('./test/data/substack-newsletter-new.html') + const dom = parseHTML(html).document + const preparedDom = await new SubstackHandler().preParse(url, dom) + // compare prepared html to the expected html + const expectedHTML = load( + './test/data/prepared/substack-newsletter-new.html' + ) + expect(preparedDom.documentElement.outerHTML).to.eql(expectedHTML) + }) + }) + }) })