Handle embedded tweets in substack emails

This does a few things:
- tags static tweets found in substack emails with a special class
- upgrades readability to ignore special class names
- reduces some readability debug output

packages/content-handler/src/newsletters/substack-handler.ts

@@ -9,14 +9,15 @@ export class SubstackHandler extends ContentHandler {
 
   shouldPreParse(url: string, dom: Document): boolean {
     const host = this.name + '.com'
+    const cdnHost = 'substackcdn.com'
     // check if url ends with substack.com
-    // or has a profile image hosted at substack.com
+    // or has a profile image hosted at substack.com or substackcdn.com
     return (
       new URL(url).hostname.endsWith(host) ||
       !!dom
         .querySelector('.email-body img')
         ?.getAttribute('src')
-        ?.includes(host)
+        ?.includes(host || cdnHost)
     )
   }
 
@@ -34,6 +35,8 @@ export class SubstackHandler extends ContentHandler {
     body?.querySelector('.container-border')?.remove()
     body?.querySelector('.footer')?.remove()
 
+    dom = this.fixupStaticTweets(dom)
+
     return Promise.resolve(dom)
   }
 
@@ -66,12 +69,12 @@ export class SubstackHandler extends ContentHandler {
     // If the article has a header link, and substack icons its probably a newsletter
     const href = this.findNewsletterHeaderHref(dom)
     const heartIcon = dom.querySelector(
-      'table tbody td span a img[src*="HeartIcon"]'
+      'a img[src*="LucideHeart"]'
     )
-    const recommendIcon = dom.querySelector(
-      'table tbody td span a img[src*="RecommendIconRounded"]'
+    const commentsIcon = dom.querySelector(
+      'a img[src*="LucideComments"]'
     )
-    return Promise.resolve(!!(href && (heartIcon || recommendIcon)))
+    return Promise.resolve(!!(href && (heartIcon || commentsIcon)))
   }
 
   async parseNewsletterUrl(
@@ -85,4 +88,44 @@ export class SubstackHandler extends ContentHandler {
     }
     return this.findNewsletterUrl(html)
   }
+
+  fixupStaticTweets(dom: Document): Document {
+    const preClassName = '_omnivore-static-'
+    const staticTweets = Array.from(dom.querySelectorAll('div[class="tweet static"]'))
+
+    if (staticTweets.length < 1) {
+      return dom
+    }
+
+    const recurse = (node: Node,  f: (node: Node) => void) => {
+      for (var i = 0; i < node.childNodes.length; i++) {
+        var child = node.childNodes[i]
+        recurse(child, f)
+        f(child)
+      }
+    }
+
+    const isHTMLElement = (node: Node): node is HTMLElement => {
+      return node.nodeType == 1
+    }
+
+    for (const tweet of Array.from(staticTweets)) {
+      tweet.className = preClassName + 'tweet'
+      tweet.removeAttribute('style')
+
+      // get all children, rename their class, remove style
+      // elements (style will be handled in the reader)
+      recurse(tweet, (n: Node) => {
+        if (isHTMLElement(n)) {
+          const className = n.className
+          if (className.startsWith("tweet-")) {
+            n.className = preClassName + className
+          }
+          n.removeAttribute('style')
+        }
+      })
+    }
+
+    return dom
+  }
 }

packages/content-handler/test/newsletter.test.ts

@@ -16,6 +16,7 @@ import { ConvertkitHandler } from '../src/newsletters/convertkit-handler'
 import { GhostHandler } from '../src/newsletters/ghost-handler'
 import { CooperPressHandler } from '../src/newsletters/cooper-press-handler'
 import { getNewsletterHandler } from '../src'
+import { parseHTML } from 'linkedom'
 
 chai.use(chaiAsPromised)
 chai.use(chaiString)
@@ -129,6 +130,41 @@ describe('Newsletter email test', () => {
       expect(handler).to.be.undefined
     })
 
+    it('returns SubstackHandler for substack newsletter with static tweets', async () => {
+      const html = load(
+        './test/data/substack-with-static-tweets-newsletter.html'
+      )
+      const handler = await getNewsletterHandler({
+        html,
+        postHeader: '',
+        from: '',
+        unSubHeader: '',
+      })
+      expect(handler).to.be.instanceOf(SubstackHandler)
+    })
+
+    it('fixes up static tweets in Substack newsletters', async () => {
+      const url = 'https://astralcodexten.substack.com/p/nick-cammarata-on-jhana'
+      const html = load(
+        './test/data/substack-with-static-tweets-newsletter.html'
+      )
+      const handler = await getNewsletterHandler({
+        html,
+        postHeader: '',
+        from: '',
+        unSubHeader: '',
+      })
+      expect(handler).to.be.instanceOf(SubstackHandler)
+
+      const dom = parseHTML(html).document
+      expect(handler?.shouldPreParse(url, dom)).to.be
+
+      const preparsed = await handler?.preParse(url, dom)
+      const tweets = Array.from(preparsed?.querySelectorAll('div[class="_omnivore-static-tweet"]') ?? [])
+
+      expect(tweets.length).to.eq(7)
+    })
+
     it('returns BeehiivHandler for beehiiv.com newsletter', async () => {
       const html = load('./test/data/beehiiv-newsletter.html')
       const handler = await getNewsletterHandler({

packages/readabilityjs/Readability.js

@@ -414,6 +414,10 @@ Readability.prototype = {
    * @return void
    */
   _cleanClasses: function (node) {
+    if (node.className.startsWith("_omnivore")) {
+      return;
+    }
+
     if (this.EMBEDS_CLASSES.includes(node.className) || this.hasEmbed(node)) {
       return;
     }
@@ -598,7 +602,7 @@ Readability.prototype = {
         continue;
       }
 
-      if (node.parentNode && ["DIV", "SECTION"].includes(node.tagName) && !(node.id && node.id.startsWith("readability"))) {
+      if (node.parentNode && ["DIV", "SECTION"].includes(node.tagName) && !(node.id && node.id.startsWith("readability") && !this._isOmnivoreNode(node)) {
         if (this._isElementWithoutContent(node)) {
           node = this._removeAndGetNext(node);
           continue;
@@ -1136,6 +1140,11 @@ Readability.prototype = {
       while (node) {
         var matchString = node.className + " " + node.id;
 
+        if (this._isOmnivoreNode(node)) {
+          node = this._getNextNode(node);
+          continue;
+        }
+
         if (!this._isProbablyVisible(node)) {
           this.log("Removing hidden node - " + matchString);
           node = this._removeAndGetNext(node);
@@ -1337,7 +1346,7 @@ Readability.prototype = {
         var candidateScore = candidate.readability.contentScore * (1 - this._getLinkDensity(candidate));
         candidate.readability.contentScore = candidateScore;
 
-        this.log("Candidate:", candidate, "with score " + candidateScore);
+        this.log("Candidate:", candidate.nodeName, candidate.className, "with score " + candidateScore);
 
         for (var t = 0; t < this._nbTopCandidates; t++) {
           var aTopCandidate = topCandidates[t];
@@ -1465,7 +1474,7 @@ Readability.prototype = {
         var sibling = siblings[s];
         var append = false;
 
-        this.log("Looking at sibling node:", sibling, sibling.readability ? ("with score " + sibling.readability.contentScore) : "");
+        this.log("Looking at sibling node:", sibling.nodeName, sibling.className, sibling.readability ? ("with score " + sibling.readability.contentScore) : "");
         this.log("Sibling has score", sibling.readability ? sibling.readability.contentScore : "Unknown");
 
         if (sibling === topCandidate) {
@@ -1496,7 +1505,7 @@ Readability.prototype = {
         }
 
         if (append) {
-          this.log("Appending node:", sibling);
+          this.log("Appending node:", sibling.nodeName);
 
           if (this.ALTER_TO_DIV_EXCEPTIONS.indexOf(sibling.nodeName) === -1) {
             // We have a node that isn't a common block level element, like a form or td tag.
@@ -2556,12 +2565,12 @@ Readability.prototype = {
       if (imgHeight && imgWidhth && imgHeight === imgWidhth) {
         if (elem.tagName.toLowerCase() === 'svg') {
           if(imgHeight <= 21){
-            this.log(`Removing small square SVG: ${imgWidhth}x${imgHeight}`, elem, `className: ${elem.className}`, `src: ${elem.src}`);
+            this.log(`Removing small square SVG: ${imgWidhth}x${imgHeight}`, `className: ${elem.className}`, `src: ${elem.src}`);
             elem.parentNode.removeChild(elem);
           }
           return;
         } else if(imgHeight <= 80) {
-          this.log(`Removing small square image: ${imgWidhth}x${imgHeight}`, elem, `className: ${elem.className}`, `src: ${elem.src}`);
+          this.log(`Removing small square image: ${imgWidhth}x${imgHeight}`, `className: ${elem.className}`, `src: ${elem.src}`);
           elem.parentNode.removeChild(elem);
           return;
         }
@@ -2765,6 +2774,7 @@ Readability.prototype = {
         }
 
         var haveToRemove =
+          !this._isOmnivoreNode(node) && (
           (img > 1 && p / img < 0.5 && !this._hasAncestorTag(node, "figure")) ||
           (!isList && li > p) ||
           (input > Math.floor(p/3)) ||
@@ -2774,10 +2784,11 @@ Readability.prototype = {
           // some website like https://substack.com might have their custom styling of tweets
           // we should omit ignoring their particular case by checking against "tweet" classname
           (weight >= 25 && linkDensity > 0.5 && !(node.className === "tweet" && linkDensity === 1)) ||
-          ((embedCount === 1 && contentLength < 75) || embedCount > 1);
+          ((embedCount === 1 && contentLength < 75) || embedCount > 1))
 
-        if (haveToRemove)
-          this.log("Cleaning Conditionally", { node, className: node.className, children: Array.from(node.children).map(ch => ch.tagName) });
+        if (haveToRemove) {
+          this.log("Cleaning Conditionally", { className: node.className, children: Array.from(node.children).map(ch => ch.tagName) });
+        }
 
         return haveToRemove;
       }
@@ -2785,6 +2796,18 @@ Readability.prototype = {
     });
   },
 
+  _isOmnivoreNode: function(node) {
+    var walk = node
+
+    while (walk) {
+      if (walk.className && node.className.startsWith("_omnivore")) {
+        return true
+      }
+      walk = walk.parentElement
+    }
+    return false
+  },
+
   /**
    * Clean out elements that match the specified conditions
    *