Fetch tweet id from url

packages/readabilityjs/Readability.js

@@ -22,6 +22,7 @@
 
 var parseSrcset = require('parse-srcset');
 var htmlEntities = require('html-entities')
+const axios = require("axios");
 
 /** Checks whether an element is a wrapper for tweet */
 const hasTweetInChildren = element => {
@@ -2204,15 +2205,15 @@ Readability.prototype = {
     }
   },
 
-  _createPlaceholders: function (e) {
-    Array.from(e.getElementsByTagName('a')).forEach(element => {
+  _createPlaceholders: async function (e) {
+    for (const element of Array.from(e.getElementsByTagName('a'))) {
 
       if (this.isEmbed(element)) {
-        return;
+        continue;
       }
 
       // Create tweets placeholders from links
-      if (element.href.includes('twitter.com')) {
+      if (element.href.includes('twitter.com') || element.parentNode.className === 'tweet') {
         const link = element.href;
         const regex = /(https?:\/\/twitter\.com\/\w+\/status\/)(\d+)/gm;
         const match = regex.exec(link);
@@ -2234,6 +2235,22 @@ Readability.prototype = {
           if (tweetParent && tweetParent.className.includes('twitter-tweet')) {
             tweetParent.parentNode.replaceChild(tweet, tweetParent);
           }
+        } else if (element.parentNode.className === 'tweet') {
+          // Create tweets placeholders from classname
+          try {
+            const response = await axios.get(link);
+            const tweetUrl = response.request.res.responseUrl;
+            const match = regex.exec(tweetUrl);
+            if (Array.isArray(match) && typeof match[2] === 'string') {
+              const tweet = this._doc.createElement('div');
+              tweet.innerText = 'Tweet placeholder';
+              tweet.className = 'tweet-placeholder';
+              tweet.setAttribute('data-tweet-id', match[2]);
+              element.parentNode.replaceChild(tweet, element);
+            }
+          } catch (e) {
+            this.log('Error loading tweet: ', link, e);
+          }
         }
       }
 
@@ -2247,7 +2264,7 @@ Readability.prototype = {
           this._createInstagramPostPlaceholder(element, match[2]);
         }
       }
-    });
+    }
 
     Array.from(e.getElementsByTagName('iframe')).forEach(element => {
 

packages/readabilityjs/package.json

@@ -30,7 +30,8 @@
     "mocha": "^8.2.0",
     "puppeteer": "^10.1.0",
     "sinon": "^7.3.2",
-    "linkedom": "^0.14.9"
+    "linkedom": "^0.14.9",
+    "axios": "^0.26.0"
   },
   "dependencies": {
     "html-entities": "^2.3.2",

packages/readabilityjs/test/test-pages/substack-michaelshellenberger/expected-metadata.json

@@ -0,0 +1,10 @@
+{
+  "title": "The 2-minute 20-second video that changes everything",
+  "byline": "Michael Shellenberger",
+  "dir": null,
+  "excerpt": "For decades, people have claimed that homelessness is just a\n                  housing problem. Sure, many also have substance use and mental\n                  illness issues. But if we just give homeless people their own\n                  own studio apartments, and decriminalize public camping,\n                  drugs, and shoplifting, the problem will go away, many\n                  claimed.",
+  "siteName": null,
+  "publishedDate": "2001-05-25T16:00:00.000Z",
+  "language": "English",
+  "readerable": true
+}

packages/readabilityjs/test/test-pages/substack-michaelshellenberger/expected.html

@@ -0,0 +1,38 @@
+<DIV class="page" id="readability-page-1">
+    <div>
+        <div dir="auto">
+            <div>
+                <figure>
+                    <table>
+                        <tbody>
+                            <tr>
+                                <td></td>
+                                <td>
+                                    <a target="_blank" href="https://email.mg2.substack.com/c/eJw1UsmO5CAM_ZriloiwBHLg0Gpp7vMFEYtJUCeQAVLdNV8_pKKRwDbGy5Ofra6wpPxSRyoVXWKurwNUhO-yQa2Q0Vkgz8EpQSkRTBDkFHOD5BKFMvsMsOuwqZpPQMdptmB1DSleGURMDGO0KgmWTVIOgzXES_ByEsy1q7F1cvT-bqxPFyBaUPCE_EoRkE37DrFetdCm1lqP8qAfD_KrnXKaUrX9si72La55wq4XaNpDteulW8WaHuTzz23QjyUl195-m4-clgylhCc0f6kAR8u4OxB-9eBXlybMab-g_ecOMDfGStYZrWnHqIBu4oPstDOUGBBikqwvtNe7_pui_i43rlbjnsvbfIMsbxMPlnIBrZ4kU8dG1ixMWMemgXg7DVZbP3PMfojk_REXFBTBhGBOxqYxJj3tnRDWYelhdBJ7zR8M7wvp_w_nQoCyWlNcTOpESuF3YR20qBBN-ukc7KlPewzPlKHXx3FROF9jP2OorxmiNhu4m916L8mb73mBCLktj5t1VcPIKZ9GNlDC2U1mo4yP48TpIFBD41LLimoPdtWwlRW2DaKBvED-B1rd02I" rel=""><img data-attrs="{&quot;src&quot;:&quot;https://bucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com/public/images/01c357e4-b829-4644-b024-4912fc91cacf_504x285.png&quot;,&quot;fullscreen&quot;:null,&quot;imageSize&quot;:null,&quot;height&quot;:285,&quot;width&quot;:504,&quot;resizeWidth&quot;:null,&quot;bytes&quot;:275557,&quot;alt&quot;:null,&quot;title&quot;:null,&quot;type&quot;:&quot;image/png&quot;,&quot;href&quot;:null}" alt="" width="504" height="285" src="https://substackcdn.com/image/fetch/w_504,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2F01c357e4-b829-4644-b024-4912fc91cacf_504x285.png"></a>
+                                </td>
+                                <td></td>
+                            </tr>
+                        </tbody>
+                    </table>
+                </figure>
+            </div>
+            <p> For decades, people have claimed that homelessness is just a housing problem. Sure, many also have substance use and mental illness issues. But if we just give homeless people their own own studio apartments, and decriminalize public camping, drugs, and shoplifting, the problem will go away, many claimed. </p>
+            <p> That hasn’t happened. Instead, the open drug scenes have worsened. Nationally, drug overdoses and poisonings increased from 17,000 in 2000 to 108,000 in 2021. And California, which pioneered the “Housing First”/decriminalization approach saw its homeless population increase 31% between 2011 and 2020, even as homelessness declined 18% in the rest of the country. </p>
+            <p> I debunked the lies about homelessness in <em>San Fransicko</em>, in hundreds of articles, and on dozens of TV and podcast appearances. But when it comes to educating the public, nothing has been more impactful than the video interviews of homeless people that I’ve conducted over the last few months with my friend Leighton Woodhouse, a documentary filmmaker, as part of my run to become governor of California. </p>
+            <p> Now Leighton has assembled those interviews into a two-minute 20 second video we’ve posted on Twitter. It’s a must-watch. It’s only been on-line for a few hours, and over 130,000 people have seen it. </p>
+            <p> I hope you’ll take a minute to watch it. And, after you do, please consider a donation to Shellenberger for Governor. </p>
+            <p> There’s just 12 days before the primary election. Anyone can vote for anyone. And anyone in the US can donate. Whatever happens, we will make history. </p>
+            <p data-attrs="{&quot;url&quot;:&quot;https://secure.shellenbergerforgovernor.com/list/proc/donation1/?InitiativeKey=KHNHN40ZS8HV&utm_source=proc&utm_medium=web&utm_campaign=donate&quot;,&quot;text&quot;:&quot;Donate to Shellenberger 2022&quot;,&quot;action&quot;:null,&quot;class&quot;:null}">
+                <a href="https://email.mg2.substack.com/c/eJxVUU2L3DAM_TXxbYJjJ3Hm4EOhLFMWFkphD3sJjq1JTGMr-GO2-fd1JntZMBY86UlPT1olmDHscsOYyPGNad9AeviMK6QEgeQIYbRGCs6ZaAUjRramGbqB2DjeA4BTdpUpZCBbnlarVbLoDwYT15ZSskgt2sE0XDdDPykt9KCmhl81VUbRXg_6HKyyseA1SHhA2NED0egc-HT0IqtcUtpixX9U7KW8CDoHqOMC6wp-gjBDuGOYsZA9hrpwS9VqYyphC6hLMOif4pqjBX_55W2yBXjAK-wV__l6e7u9tfTjz3B7r1ifkxsj5qCh5M4OT8yBsdkV7BOmL0grtyk7-wI-ZwCxklHGaMf6EillNa-NENrQ4Q69GehddVVL3czqmKeYlP57KCZBLujnCS8C0f6O7QVKlfUT_rsYcFij8_aBZW-1bYfd42FRLnvsI3g1rWDOS6TzoM_bjDN4CEWUGVWSTd_x7tq3DWddexpf7O36_trxRpCixmBheemsXhSs3wz-D4wTwDA" rel=""><span>Donate to Shellenberger 2022</span></a>
+            </p>
+        </div>
+        <div>
+            <p>
+                <span>You’re a free subscriber to <a href="https://email.mg2.substack.com/c/eJxtkL1uxSAMhZ8mbDfin9yBoUv3PkFEwElQCURAbpu3L2mmSpUse_Gxz_msqbCkfOo9lYquNtZzBx3hqwSoFTI6CuTRO60Yo4oripzmjgxiQL6McwbYjA-65gPQfkzBW1N9ipeCqifHGK0acyFBEGLnmZFZSImZNFw5JYeBcIrvx-ZwHqIFDS_IZ4qAbNo2iPW6hYJea91Lx946-t5q83Y1EMoKIUCcIC-Q-3JMpRr72Tdl20FeU0wpFlS2iTHtWe-Usg4PM0g34NmIjuNtoX-UKOs1xWVKD5WS_yj8AW3Lxyl9PxxsqU9b9K-UoTf7foUeL6NH9PUcIZopgLt51BvrL6FxgQi54XajqZpIwcRTcsKo4Hf8FrKReQpGFGpuXGqqqP-L-QNr9ZSs">Michael Shellenberger</a>. For the full experience, <a href="https://email.mg2.substack.com/c/eJxtUsuO2jAU_RqyI3LsvFhkQcnQgRlCkei8NpFjXxJDYkeOA4Svr1PoolUly5aP7j33cQ6jBkqlh6RVnXHGKzdDC4mES1eDMaCdvgOdC55EhODIj7DDE597cRA7ossPGqChok6M7sFp-6IWjBqh5JiBo5mPkFMlBMMsPlCf05jEJKKRz2PLVASIHgIaRvfCtOcCJIMEzqAHJcFhqmlAmpHLqZPKmLabkPkEL-1pBKso1F0FdQ2yAF2Cdru-6AxlJ9dm2pjxy7QoYEKWRp1ATkgKw9pj-G34wPVpdVRkc9sM2W1HNuIi6PsSWeyaHX_esv0JZ-nT8LpY1_A8F9vjE872n9dtWqLNftWtmvrGhlW4kt-8T4Esxwlt0rmw8Tf-vhJbsT7S729HimceX6xnrpDPL8sD-cr0JQvWZ1OFu3S5bvPo3E5_VC_zj9ehWGz0V5ntJjjsTZN3qtfM9p6O6_kX-zPqA2-Ai74Z5xvleIBMSWP392BghjoiwQhjFODQvghhl7g8ihhH8QFCHqMDDSY-akr81yodnVRKloWaRkqJXedPwUYJWajrlEOjXNVIcVYaXNq2owvyUbleCjPkIGlRA78bxNx99rvHvAQJ2vqP59QkXhiQYBb6HsGBf_eDVT0Iw1lAvMix3XBls2TyP91_AeYL9AI">become a paid subscriber.</a></span>
+            </p>
+            <p>
+                <a role="button" href="https://email.mg2.substack.com/c/eJxtUsuO2jAU_RqyI3LsvFhkQcnQgRlCkei8NpFjXxJDYkeOA4Svr1PoolUly5aP7j33cQ6jBkqlh6RVnXHGKzdDC4mES1eDMaCdvgOdC55EhODIj7DDE597cRA7ossPGqChok6M7sFp-6IWjBqh5JiBo5mPkFMlBMMsPlCf05jEJKKRz2PLVASIHgIaRvfCtOcCJIMEzqAHJcFhqmlAmpHLqZPKmLabkPkEL-1pBKso1F0FdQ2yAF2Cdru-6AxlJ9dm2pjxy7QoYEKWRp1ATkgKw9pj-G34wPVpdVRkc9sM2W1HNuIi6PsSWeyaHX_esv0JZ-nT8LpY1_A8F9vjE872n9dtWqLNftWtmvrGhlW4kt-8T4Esxwlt0rmw8Tf-vhJbsT7S729HimceX6xnrpDPL8sD-cr0JQvWZ1OFu3S5bvPo3E5_VC_zj9ehWGz0V5ntJjjsTZN3qtfM9p6O6_kX-zPqA2-Ai74Z5xvleIBMSWP392BghjoiwQhjFODQvghhl7g8ihhH8QFCHqMDDSY-akr81yodnVRKloWaRkqJXedPwUYJWajrlEOjXNVIcVYaXNq2owvyUbleCjPkIGlRA78bxNx99rvHvAQJ2vqP59QkXhiQYBb6HsGBf_eDVT0Iw1lAvMix3XBls2TyP91_AeYL9AI">Subscribe</a>
+            </p>
+        </div>
+    </div>
+</DIV>

packages/readabilityjs/test/test-pages/substack-michaelshellenberger/url.txt

@@ -0,0 +1 @@
+https://michaelshellenberger.substack.com/p/the-2-minute-20-second-video-that?s=r