From cc91e43572c79dfa34c434aeb6c3ec2aaacf2077 Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Mon, 31 Oct 2022 21:28:31 +0800 Subject: [PATCH] Handle embedded tweets in substack emails This does a few things: - tags static tweets found in substack emails with a special class - upgrades readability to ignore special class names - reduces some readability debug output --- .../src/newsletters/substack-handler.ts | 55 +++++++++++++++++-- .../content-handler/test/newsletter.test.ts | 36 ++++++++++++ packages/readabilityjs/Readability.js | 41 +++++++++++--- 3 files changed, 117 insertions(+), 15 deletions(-) diff --git a/packages/content-handler/src/newsletters/substack-handler.ts b/packages/content-handler/src/newsletters/substack-handler.ts index e82d6d1cf..40a476eb2 100644 --- a/packages/content-handler/src/newsletters/substack-handler.ts +++ b/packages/content-handler/src/newsletters/substack-handler.ts @@ -9,14 +9,15 @@ export class SubstackHandler extends ContentHandler { shouldPreParse(url: string, dom: Document): boolean { const host = this.name + '.com' + const cdnHost = 'substackcdn.com' // check if url ends with substack.com - // or has a profile image hosted at substack.com + // or has a profile image hosted at substack.com or substackcdn.com return ( new URL(url).hostname.endsWith(host) || !!dom .querySelector('.email-body img') ?.getAttribute('src') - ?.includes(host) + ?.includes(host || cdnHost) ) } @@ -34,6 +35,8 @@ export class SubstackHandler extends ContentHandler { body?.querySelector('.container-border')?.remove() body?.querySelector('.footer')?.remove() + dom = this.fixupStaticTweets(dom) + return Promise.resolve(dom) } @@ -66,12 +69,12 @@ export class SubstackHandler extends ContentHandler { // If the article has a header link, and substack icons its probably a newsletter const href = this.findNewsletterHeaderHref(dom) const heartIcon = dom.querySelector( - 'table tbody td span a img[src*="HeartIcon"]' + 'a img[src*="LucideHeart"]' ) - const recommendIcon = dom.querySelector( - 'table tbody td span a img[src*="RecommendIconRounded"]' + const commentsIcon = dom.querySelector( + 'a img[src*="LucideComments"]' ) - return Promise.resolve(!!(href && (heartIcon || recommendIcon))) + return Promise.resolve(!!(href && (heartIcon || commentsIcon))) } async parseNewsletterUrl( @@ -85,4 +88,44 @@ export class SubstackHandler extends ContentHandler { } return this.findNewsletterUrl(html) } + + fixupStaticTweets(dom: Document): Document { + const preClassName = '_omnivore-static-' + const staticTweets = Array.from(dom.querySelectorAll('div[class="tweet static"]')) + + if (staticTweets.length < 1) { + return dom + } + + const recurse = (node: Node, f: (node: Node) => void) => { + for (var i = 0; i < node.childNodes.length; i++) { + var child = node.childNodes[i] + recurse(child, f) + f(child) + } + } + + const isHTMLElement = (node: Node): node is HTMLElement => { + return node.nodeType == 1 + } + + for (const tweet of Array.from(staticTweets)) { + tweet.className = preClassName + 'tweet' + tweet.removeAttribute('style') + + // get all children, rename their class, remove style + // elements (style will be handled in the reader) + recurse(tweet, (n: Node) => { + if (isHTMLElement(n)) { + const className = n.className + if (className.startsWith("tweet-")) { + n.className = preClassName + className + } + n.removeAttribute('style') + } + }) + } + + return dom + } } diff --git a/packages/content-handler/test/newsletter.test.ts b/packages/content-handler/test/newsletter.test.ts index d8eb84d00..e5c1fb5c0 100644 --- a/packages/content-handler/test/newsletter.test.ts +++ b/packages/content-handler/test/newsletter.test.ts @@ -16,6 +16,7 @@ import { ConvertkitHandler } from '../src/newsletters/convertkit-handler' import { GhostHandler } from '../src/newsletters/ghost-handler' import { CooperPressHandler } from '../src/newsletters/cooper-press-handler' import { getNewsletterHandler } from '../src' +import { parseHTML } from 'linkedom' chai.use(chaiAsPromised) chai.use(chaiString) @@ -129,6 +130,41 @@ describe('Newsletter email test', () => { expect(handler).to.be.undefined }) + it('returns SubstackHandler for substack newsletter with static tweets', async () => { + const html = load( + './test/data/substack-with-static-tweets-newsletter.html' + ) + const handler = await getNewsletterHandler({ + html, + postHeader: '', + from: '', + unSubHeader: '', + }) + expect(handler).to.be.instanceOf(SubstackHandler) + }) + + it('fixes up static tweets in Substack newsletters', async () => { + const url = 'https://astralcodexten.substack.com/p/nick-cammarata-on-jhana' + const html = load( + './test/data/substack-with-static-tweets-newsletter.html' + ) + const handler = await getNewsletterHandler({ + html, + postHeader: '', + from: '', + unSubHeader: '', + }) + expect(handler).to.be.instanceOf(SubstackHandler) + + const dom = parseHTML(html).document + expect(handler?.shouldPreParse(url, dom)).to.be + + const preparsed = await handler?.preParse(url, dom) + const tweets = Array.from(preparsed?.querySelectorAll('div[class="_omnivore-static-tweet"]') ?? []) + + expect(tweets.length).to.eq(7) + }) + it('returns BeehiivHandler for beehiiv.com newsletter', async () => { const html = load('./test/data/beehiiv-newsletter.html') const handler = await getNewsletterHandler({ diff --git a/packages/readabilityjs/Readability.js b/packages/readabilityjs/Readability.js index 2957a099f..163200f9f 100644 --- a/packages/readabilityjs/Readability.js +++ b/packages/readabilityjs/Readability.js @@ -414,6 +414,10 @@ Readability.prototype = { * @return void */ _cleanClasses: function (node) { + if (node.className.startsWith("_omnivore")) { + return; + } + if (this.EMBEDS_CLASSES.includes(node.className) || this.hasEmbed(node)) { return; } @@ -598,7 +602,7 @@ Readability.prototype = { continue; } - if (node.parentNode && ["DIV", "SECTION"].includes(node.tagName) && !(node.id && node.id.startsWith("readability"))) { + if (node.parentNode && ["DIV", "SECTION"].includes(node.tagName) && !(node.id && node.id.startsWith("readability") && !this._isOmnivoreNode(node)) { if (this._isElementWithoutContent(node)) { node = this._removeAndGetNext(node); continue; @@ -1136,6 +1140,11 @@ Readability.prototype = { while (node) { var matchString = node.className + " " + node.id; + if (this._isOmnivoreNode(node)) { + node = this._getNextNode(node); + continue; + } + if (!this._isProbablyVisible(node)) { this.log("Removing hidden node - " + matchString); node = this._removeAndGetNext(node); @@ -1337,7 +1346,7 @@ Readability.prototype = { var candidateScore = candidate.readability.contentScore * (1 - this._getLinkDensity(candidate)); candidate.readability.contentScore = candidateScore; - this.log("Candidate:", candidate, "with score " + candidateScore); + this.log("Candidate:", candidate.nodeName, candidate.className, "with score " + candidateScore); for (var t = 0; t < this._nbTopCandidates; t++) { var aTopCandidate = topCandidates[t]; @@ -1465,7 +1474,7 @@ Readability.prototype = { var sibling = siblings[s]; var append = false; - this.log("Looking at sibling node:", sibling, sibling.readability ? ("with score " + sibling.readability.contentScore) : ""); + this.log("Looking at sibling node:", sibling.nodeName, sibling.className, sibling.readability ? ("with score " + sibling.readability.contentScore) : ""); this.log("Sibling has score", sibling.readability ? sibling.readability.contentScore : "Unknown"); if (sibling === topCandidate) { @@ -1496,7 +1505,7 @@ Readability.prototype = { } if (append) { - this.log("Appending node:", sibling); + this.log("Appending node:", sibling.nodeName); if (this.ALTER_TO_DIV_EXCEPTIONS.indexOf(sibling.nodeName) === -1) { // We have a node that isn't a common block level element, like a form or td tag. @@ -2556,12 +2565,12 @@ Readability.prototype = { if (imgHeight && imgWidhth && imgHeight === imgWidhth) { if (elem.tagName.toLowerCase() === 'svg') { if(imgHeight <= 21){ - this.log(`Removing small square SVG: ${imgWidhth}x${imgHeight}`, elem, `className: ${elem.className}`, `src: ${elem.src}`); + this.log(`Removing small square SVG: ${imgWidhth}x${imgHeight}`, `className: ${elem.className}`, `src: ${elem.src}`); elem.parentNode.removeChild(elem); } return; } else if(imgHeight <= 80) { - this.log(`Removing small square image: ${imgWidhth}x${imgHeight}`, elem, `className: ${elem.className}`, `src: ${elem.src}`); + this.log(`Removing small square image: ${imgWidhth}x${imgHeight}`, `className: ${elem.className}`, `src: ${elem.src}`); elem.parentNode.removeChild(elem); return; } @@ -2765,6 +2774,7 @@ Readability.prototype = { } var haveToRemove = + !this._isOmnivoreNode(node) && ( (img > 1 && p / img < 0.5 && !this._hasAncestorTag(node, "figure")) || (!isList && li > p) || (input > Math.floor(p/3)) || @@ -2774,10 +2784,11 @@ Readability.prototype = { // some website like https://substack.com might have their custom styling of tweets // we should omit ignoring their particular case by checking against "tweet" classname (weight >= 25 && linkDensity > 0.5 && !(node.className === "tweet" && linkDensity === 1)) || - ((embedCount === 1 && contentLength < 75) || embedCount > 1); + ((embedCount === 1 && contentLength < 75) || embedCount > 1)) - if (haveToRemove) - this.log("Cleaning Conditionally", { node, className: node.className, children: Array.from(node.children).map(ch => ch.tagName) }); + if (haveToRemove) { + this.log("Cleaning Conditionally", { className: node.className, children: Array.from(node.children).map(ch => ch.tagName) }); + } return haveToRemove; } @@ -2785,6 +2796,18 @@ Readability.prototype = { }); }, + _isOmnivoreNode: function(node) { + var walk = node + + while (walk) { + if (walk.className && node.className.startsWith("_omnivore")) { + return true + } + walk = walk.parentElement + } + return false + }, + /** * Clean out elements that match the specified conditions *