From 895e50201a3a14d105a4f47bcef65aaa92e1e33b Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Wed, 15 Mar 2023 19:36:53 +0800 Subject: [PATCH] Fix tests --- packages/readabilityjs/Readability.js | 11 ++++++++++- .../test/test-pages/berthub-2/expected-metadata.json | 2 +- .../bookofhook.blogspot.com/expected-metadata.json | 2 +- .../community.musictribe.com/expected-metadata.json | 2 +- .../test-pages/computer.rip/expected-metadata.json | 2 +- .../test/test-pages/danluu/expected-metadata.json | 2 +- .../test/test-pages/gdcvault/expected-metadata.json | 2 +- .../test/test-pages/gflownet/expected-metadata.json | 2 +- .../test/test-pages/infoproc/expected-metadata.json | 2 +- .../johnhcochrane.blogspot/expected-metadata.json | 2 +- .../test/test-pages/milkroad/expected-metadata.json | 2 +- .../nytimes-podcasts/expected-metadata.json | 2 +- .../test-pages/nytimes.com/expected-metadata.json | 2 +- .../test/test-pages/nytimes/expected-metadata.json | 2 +- .../test-pages/robinwieruch.de/expected-metadata.json | 2 +- .../test-pages/sciencedirect/expected-metadata.json | 2 +- .../test-pages/substack-email/expected-metadata.json | 2 +- .../expected-metadata.json | 2 +- .../test/test-pages/sydney.com/expected-metadata.json | 2 +- .../test-pages/youtube-embed/expected-metadata.json | 2 +- .../test/test-pages/yuyue.com/expected-metadata.json | 2 +- .../test/test-pages/zhihu/expected-metadata.json | 2 +- 22 files changed, 31 insertions(+), 22 deletions(-) diff --git a/packages/readabilityjs/Readability.js b/packages/readabilityjs/Readability.js index 5f5fb96bd..5902ae973 100644 --- a/packages/readabilityjs/Readability.js +++ b/packages/readabilityjs/Readability.js @@ -3009,6 +3009,15 @@ Readability.prototype = { metadata.excerpt = paragraphs[0].textContent.trim(); } } + if (!metadata.siteName) { + // Fallback to hostname + try { + const host = new URL(this._baseURI).hostname; + metadata.siteName = host.replace(/^www\./, ""); + } catch (e) { + // Ignore + } + } var textContent = articleContent.textContent; return { @@ -3019,7 +3028,7 @@ Readability.prototype = { textContent: textContent, length: textContent.length, excerpt: metadata.excerpt, - siteName: metadata.siteName || new URL(this._baseURI).hostname, // Fallback to hostname + siteName: metadata.siteName, siteIcon: metadata.siteIcon, previewImage: metadata.previewImage, publishedDate: metadata.publishedDate || publishedAt || this._articlePublishedDate, diff --git a/packages/readabilityjs/test/test-pages/berthub-2/expected-metadata.json b/packages/readabilityjs/test/test-pages/berthub-2/expected-metadata.json index ade1d4433..3fa4cff99 100644 --- a/packages/readabilityjs/test/test-pages/berthub-2/expected-metadata.json +++ b/packages/readabilityjs/test/test-pages/berthub-2/expected-metadata.json @@ -3,7 +3,7 @@ "byline": null, "dir": null, "excerpt": "All BNT162b2 vaccine data on this page is sourced from this World Health Organization document.", - "siteName": null, + "siteName": "fakehost", "publishedDate": null, "readerable": true } diff --git a/packages/readabilityjs/test/test-pages/bookofhook.blogspot.com/expected-metadata.json b/packages/readabilityjs/test/test-pages/bookofhook.blogspot.com/expected-metadata.json index 42cbfd57e..a114dfad1 100644 --- a/packages/readabilityjs/test/test-pages/bookofhook.blogspot.com/expected-metadata.json +++ b/packages/readabilityjs/test/test-pages/bookofhook.blogspot.com/expected-metadata.json @@ -3,7 +3,7 @@ "byline": null, "dir": "ltr", "excerpt": "Productivity is one of my pet topics, because it's always dogged me a bit, especially early in my career.  I'd pull long days and nights and...", - "siteName": null, + "siteName": "fakehost", "previewImage": "https://lh3.googleusercontent.com/blogger_img_proxy/ABLy4EzWkihPRCY9pIfp0Kyte3jlABLfSMcK5dlFeOQ7OGHfoy_CEMNzvNgmyBoyd1ahGWWJ0UXGSSvUXxouVeOK5l9Fxqlpe5YQQfgFg7-ikDZ0EaToXHZFrA_EmV8V5tKR4bd1HProtJ8=w1200-h630-p-k-no-nu", "publishedDate": "2001-03-01T08:00:00.000Z", "readerable": true diff --git a/packages/readabilityjs/test/test-pages/community.musictribe.com/expected-metadata.json b/packages/readabilityjs/test/test-pages/community.musictribe.com/expected-metadata.json index f673d577b..72e4ea217 100644 --- a/packages/readabilityjs/test/test-pages/community.musictribe.com/expected-metadata.json +++ b/packages/readabilityjs/test/test-pages/community.musictribe.com/expected-metadata.json @@ -3,7 +3,7 @@ "byline": null, "dir": null, "excerpt": "This page displays a discussion entry.", - "siteName": null, + "siteName": "fakehost", "siteIcon": "http://fakehost/test/public/core_file/ec/5d/02/f6baf292a4e76c51e82e87d5cb98ad61.ico?c=5097", "previewImage": "https://community.musictribe.com/", "publishedDate": null, diff --git a/packages/readabilityjs/test/test-pages/computer.rip/expected-metadata.json b/packages/readabilityjs/test/test-pages/computer.rip/expected-metadata.json index 16d576194..efa23a06c 100644 --- a/packages/readabilityjs/test/test-pages/computer.rip/expected-metadata.json +++ b/packages/readabilityjs/test/test-pages/computer.rip/expected-metadata.json @@ -3,7 +3,7 @@ "byline": null, "dir": null, "excerpt": "To start: yes, long time no see. Well, COVID-19 has been like that. Some days I feel accomplished if I successfully check my email. I finally managed to clear out a backlog of an entire handfull of things that needed thoughtful responses, though, and so here I am, screaming into the void instead of at anyone in particular.", - "siteName": null, + "siteName": "fakehost", "publishedDate": "2020-11-28T00:00:00.000Z", "readerable": true } diff --git a/packages/readabilityjs/test/test-pages/danluu/expected-metadata.json b/packages/readabilityjs/test/test-pages/danluu/expected-metadata.json index 1ce25e1c0..51e3c276e 100644 --- a/packages/readabilityjs/test/test-pages/danluu/expected-metadata.json +++ b/packages/readabilityjs/test/test-pages/danluu/expected-metadata.json @@ -3,7 +3,7 @@ "byline": null, "dir": null, "excerpt": "this is an archive of an old article by John Carmack which seems to\n have disappeared off of the internet", - "siteName": null, + "siteName": "fakehost", "siteIcon": "data:;base64,=", "publishedDate": null, "language": "English", diff --git a/packages/readabilityjs/test/test-pages/gdcvault/expected-metadata.json b/packages/readabilityjs/test/test-pages/gdcvault/expected-metadata.json index 5e5525d51..fba5b7d87 100644 --- a/packages/readabilityjs/test/test-pages/gdcvault/expected-metadata.json +++ b/packages/readabilityjs/test/test-pages/gdcvault/expected-metadata.json @@ -3,7 +3,7 @@ "byline": null, "dir": null, "excerpt": "This talk is a detailed walkthrough of the game engine modifications needed to make The Last of Us Remastered run at 60 fps on PlayStation 4. Topics covered will include the fiber-based job system Naughty Dog adopted for the game, the overall...", - "siteName": null, + "siteName": "fakehost", "siteIcon": "http://fakehost/img/favicon.ico", "previewImage": "https://ubm-twvideo01.s3.amazonaws.com/o1/vault/gdc2015/Images/GDC15_Vault-thumb_v1.png", "publishedDate": null, diff --git a/packages/readabilityjs/test/test-pages/gflownet/expected-metadata.json b/packages/readabilityjs/test/test-pages/gflownet/expected-metadata.json index 470e252e0..358ad6299 100644 --- a/packages/readabilityjs/test/test-pages/gflownet/expected-metadata.json +++ b/packages/readabilityjs/test/test-pages/gflownet/expected-metadata.json @@ -3,7 +3,7 @@ "byline": null, "dir": null, "excerpt": "What follows is a high-level overview of this work, for more details refer to our paper. Given a reward \n \n \n \n R\n \n \n (\n \n \n x\n \n \n )\n \n \n \n R(x)\n \n and a deterministic episodic environment where episodes end with a ``generate \n \n \n \n x\n \n \n \n x\n \n '' action, how do we generate diverse and high-reward \n \n \n \n x\n \n \n \n x\n \n s?\n We propose to use Flow Networks to model discrete \n \n \n \n p\n \n \n (\n \n \n x\n \n \n )\n \n \n ∝\n \n \n R\n \n \n (\n \n \n x\n \n \n )\n \n \n \n p(x) \\propto R(x)\n \n from which we can sample sequentially (like episodic RL, rather than iteratively as MCMC methods would). We show that our method, GFlowNet, is very useful on a combinatorial domain, drug molecule synthesis, because unlike RL methods it generates diverse \n \n \n \n x\n \n \n \n x\n \n s by design.", - "siteName": null, + "siteName": "fakehost", "siteIcon": "", "publishedDate": null, "readerable": true diff --git a/packages/readabilityjs/test/test-pages/infoproc/expected-metadata.json b/packages/readabilityjs/test/test-pages/infoproc/expected-metadata.json index c46de6b7f..dcb68175c 100644 --- a/packages/readabilityjs/test/test-pages/infoproc/expected-metadata.json +++ b/packages/readabilityjs/test/test-pages/infoproc/expected-metadata.json @@ -3,7 +3,7 @@ "byline": "View my complete profile", "dir": null, "excerpt": "Theodore A. Postol is professor emeritus of Science, Technology, and International Security at the Massachusetts Institute of Technolog...", - "siteName": null, + "siteName": "fakehost", "siteIcon": "https://infoproc.blogspot.com/favicon.ico", "previewImage": "https://lh3.googleusercontent.com/blogger_img_proxy/ABLy4EwrOqWiqDKUTjOKgPfAJkAH9M0VlZ8ystq9wP0nHvpdfqFpWeootDAtR3o6ZhVbEHf76IEUuztQuEVauUdXxB_jbiK4qOsTNVH3qqz0TtrglbYoOw=w1200-h630-n-k-no-nu", "publishedDate": "2022-05-18T16:00:00.000Z", diff --git a/packages/readabilityjs/test/test-pages/johnhcochrane.blogspot/expected-metadata.json b/packages/readabilityjs/test/test-pages/johnhcochrane.blogspot/expected-metadata.json index e9b9b326b..c43c040a9 100644 --- a/packages/readabilityjs/test/test-pages/johnhcochrane.blogspot/expected-metadata.json +++ b/packages/readabilityjs/test/test-pages/johnhcochrane.blogspot/expected-metadata.json @@ -3,7 +3,7 @@ "byline": null, "dir": null, "excerpt": "Part 1: Who should get the vaccine first? Sell to the highest bidder. The disease and recession go away faster.", - "siteName": null, + "siteName": "fakehost", "previewImage": "https://lh6.googleusercontent.com/proxy/QfszXavmpOM6v0Er22-RrJg7cyNBPk6alnXmZ3MQnqDQ7GWeiytIaO_eIQTeh75iNlsbjFqSoqOytExYaUdlwJyJaj7fb5CEdu2DVI6OsMJvWLGyhA=w1200-h630-p-k-no-nu", "publishedDate": "2020-12-07T08:00:00.000Z", "readerable": true diff --git a/packages/readabilityjs/test/test-pages/milkroad/expected-metadata.json b/packages/readabilityjs/test/test-pages/milkroad/expected-metadata.json index 16ef00bd5..13d56b5f8 100644 --- a/packages/readabilityjs/test/test-pages/milkroad/expected-metadata.json +++ b/packages/readabilityjs/test/test-pages/milkroad/expected-metadata.json @@ -3,7 +3,7 @@ "byline": null, "dir": null, "excerpt": "The newsletter that makes you smarter about web3", - "siteName": null, + "siteName": "fakehost", "siteIcon": "https://media.beehiiv.net/uploads/publication/logo/654e9594-184c-4884-8e02-e6e58a3a6871/thumb_Untitled__1000_x_1000_px___2_.png", "previewImage": "https://media.beehiiv.net/uploads/asset/file/30564/Screenshot_2022-04-08_115750.png", "publishedDate": null, diff --git a/packages/readabilityjs/test/test-pages/nytimes-podcasts/expected-metadata.json b/packages/readabilityjs/test/test-pages/nytimes-podcasts/expected-metadata.json index 9f428f123..61fe09886 100644 --- a/packages/readabilityjs/test/test-pages/nytimes-podcasts/expected-metadata.json +++ b/packages/readabilityjs/test/test-pages/nytimes-podcasts/expected-metadata.json @@ -3,7 +3,7 @@ "byline": "The New York Times", "dir": null, "excerpt": "The Sept. 27, 2022 episode of “The Ezra Klein Show”", - "siteName": null, + "siteName": "fakehost", "siteIcon": "/vi-assets/static-assets/favicon-d2483f10ef688e6f89e23806b9700298.ico", "previewImage": "https://static01.nyt.com/newsgraphics/images/icons/defaultPromoCrop.png", "publishedDate": "2022-09-27T16:25:17.221Z", diff --git a/packages/readabilityjs/test/test-pages/nytimes.com/expected-metadata.json b/packages/readabilityjs/test/test-pages/nytimes.com/expected-metadata.json index c76bb2d55..dfcfb56e8 100644 --- a/packages/readabilityjs/test/test-pages/nytimes.com/expected-metadata.json +++ b/packages/readabilityjs/test/test-pages/nytimes.com/expected-metadata.json @@ -3,7 +3,7 @@ "byline": "Kate Conger", "dir": null, "excerpt": "The social media company went public in 2013. But Elon Musk is taking it private as part of his acquisition of the firm. Here’s what that means.", - "siteName": null, + "siteName": "fakehost", "siteIcon": "http://fakehost/vi-assets/static-assets/favicon-d2483f10ef688e6f89e23806b9700298.ico", "previewImage": "https://static01.nyt.com/images/2022/10/29/business/00jpTWITTER-PRIVATE2-print/00MUSK-TWITTER-facebookJumbo.jpg", "publishedDate": "2022-10-28T09:00:25.000Z", diff --git a/packages/readabilityjs/test/test-pages/nytimes/expected-metadata.json b/packages/readabilityjs/test/test-pages/nytimes/expected-metadata.json index 1bbcdfdb9..882499fc2 100644 --- a/packages/readabilityjs/test/test-pages/nytimes/expected-metadata.json +++ b/packages/readabilityjs/test/test-pages/nytimes/expected-metadata.json @@ -3,7 +3,7 @@ "byline": "Ernesto Londoño, Letícia Casado", "dir": null, "excerpt": "Critics see the recent behavior of Brazil’s president — polarizing in the best of times — as an unnerving sign of a flailing leader. His strategy, if there is one, is difficult to discern.", - "siteName": null, + "siteName": "fakehost", "previewImage": "https://static01.nyt.com/images/2021/03/31/world/31brazil/31brazil-facebookJumbo.jpg", "publishedDate": "2021-03-31T23:26:15.000Z", "readerable": true diff --git a/packages/readabilityjs/test/test-pages/robinwieruch.de/expected-metadata.json b/packages/readabilityjs/test/test-pages/robinwieruch.de/expected-metadata.json index 0054cf92a..5c0d038ab 100644 --- a/packages/readabilityjs/test/test-pages/robinwieruch.de/expected-metadata.json +++ b/packages/readabilityjs/test/test-pages/robinwieruch.de/expected-metadata.json @@ -3,7 +3,7 @@ "byline": "Robin Wieruch", "dir": null, "excerpt": "A tutorial on how to fetch data in React with Hooks from third-party APIs. You will use state and effect hooks for the data request from a real API ...", - "siteName": null, + "siteName": "fakehost", "siteIcon": "http://fakehost/favicon-32x32.png?v=9db82c76a9aaf54925ac42d41f3d384c", "previewImage": "https://www.robinwieruch.de/static/9b13b3546c675d6f1a1e565f5185cab6/9842e/banner.jpg", "publishedDate": null, diff --git a/packages/readabilityjs/test/test-pages/sciencedirect/expected-metadata.json b/packages/readabilityjs/test/test-pages/sciencedirect/expected-metadata.json index ffec318f4..6829c35f1 100644 --- a/packages/readabilityjs/test/test-pages/sciencedirect/expected-metadata.json +++ b/packages/readabilityjs/test/test-pages/sciencedirect/expected-metadata.json @@ -3,7 +3,7 @@ "byline": null, "dir": null, "excerpt": "The “Weak Garden of Eden” model for the origin and dispersal of modern humans (Harpendinget al., 1993) posits that modern humans spread into separate …", - "siteName": null, + "siteName": "fakehost", "siteIcon": "https://sdfestaticassets-eu-west-1.sciencedirectassets.com/shared-assets/13/images/favSD.ico", "previewImage": "https://ars.els-cdn.com/content/image/1-s2.0-S0047248420X00121-cov150h.gif", "publishedDate": null, diff --git a/packages/readabilityjs/test/test-pages/substack-email/expected-metadata.json b/packages/readabilityjs/test/test-pages/substack-email/expected-metadata.json index 469b85b83..6b0bf82df 100644 --- a/packages/readabilityjs/test/test-pages/substack-email/expected-metadata.json +++ b/packages/readabilityjs/test/test-pages/substack-email/expected-metadata.json @@ -3,7 +3,7 @@ "byline": "Andrew Bulkeley", "dir": null, "excerpt": "Hello 20\n Percent,", - "siteName": null, + "siteName": "fakehost", "publishedDate": "2001-01-13T16:00:00.000Z", "readerable": true } diff --git a/packages/readabilityjs/test/test-pages/substack-michaelshellenberger/expected-metadata.json b/packages/readabilityjs/test/test-pages/substack-michaelshellenberger/expected-metadata.json index f41a6b334..ae2fe8c8c 100644 --- a/packages/readabilityjs/test/test-pages/substack-michaelshellenberger/expected-metadata.json +++ b/packages/readabilityjs/test/test-pages/substack-michaelshellenberger/expected-metadata.json @@ -3,7 +3,7 @@ "byline": "Michael Shellenberger", "dir": null, "excerpt": "For decades, people have claimed that homelessness is just a\n housing problem. Sure, many also have substance use and mental\n illness issues. But if we just give homeless people their own\n own studio apartments, and decriminalize public camping,\n drugs, and shoplifting, the problem will go away, many\n claimed.", - "siteName": null, + "siteName": "fakehost", "publishedDate": "2001-05-25T16:00:00.000Z", "language": "English", "readerable": true diff --git a/packages/readabilityjs/test/test-pages/sydney.com/expected-metadata.json b/packages/readabilityjs/test/test-pages/sydney.com/expected-metadata.json index 80d5fd64f..422f596fa 100644 --- a/packages/readabilityjs/test/test-pages/sydney.com/expected-metadata.json +++ b/packages/readabilityjs/test/test-pages/sydney.com/expected-metadata.json @@ -3,7 +3,7 @@ "byline": "author-circle", "dir": null, "excerpt": "Explore our list of the best walks in Sydney where you’ll find stunning coastal views, historical sites, magnificent beaches and fascinating wildlife.", - "siteName": null, + "siteName": "fakehost", "siteIcon": "/sites/sydney/files/favicon-16x16.png", "previewImage": "https://www.sydney.com/sites/sydney/files/styles/open_graph/public/2020-06/Sydney%20Harbour%20from%20Bradleys%20Head%2C%20Mosman.jpg?itok=fJkitzF5", "publishedDate": null, diff --git a/packages/readabilityjs/test/test-pages/youtube-embed/expected-metadata.json b/packages/readabilityjs/test/test-pages/youtube-embed/expected-metadata.json index 1febfa04b..c66965fca 100644 --- a/packages/readabilityjs/test/test-pages/youtube-embed/expected-metadata.json +++ b/packages/readabilityjs/test/test-pages/youtube-embed/expected-metadata.json @@ -3,7 +3,7 @@ "byline": "Noclip - Video Game Documentaries", "dir": null, "excerpt": "The Story of Celeste's Development", - "siteName": null, + "siteName": "fakehost", "siteIcon": "", "previewImage": "https://i.ytimg.com/vi/c3mbELVqAmo/hqdefault.jpg", "publishedDate": null, diff --git a/packages/readabilityjs/test/test-pages/yuyue.com/expected-metadata.json b/packages/readabilityjs/test/test-pages/yuyue.com/expected-metadata.json index 4a1772b83..4e699688b 100644 --- a/packages/readabilityjs/test/test-pages/yuyue.com/expected-metadata.json +++ b/packages/readabilityjs/test/test-pages/yuyue.com/expected-metadata.json @@ -3,7 +3,7 @@ "byline": null, "dir": null, "excerpt": "一句话，用整体性学习，即把新旧知识链起来成网，而不是分...", - "siteName": null, + "siteName": "fakehost", "siteIcon": "https://mdn.alipayobjects.com/huamei_0prmtq/afts/img/A*sRUdR543RjcAAAAAAAAAAAAADvuFAQ/original", "previewImage": "https://cdn.nlark.com/yuque/0/2022/png/22724648/1671339142303-ce7c7caa-57b6-473b-8fb3-bfaf59a79d3b.png", "publishedDate": null, diff --git a/packages/readabilityjs/test/test-pages/zhihu/expected-metadata.json b/packages/readabilityjs/test/test-pages/zhihu/expected-metadata.json index 87e159d31..c9530839b 100644 --- a/packages/readabilityjs/test/test-pages/zhihu/expected-metadata.json +++ b/packages/readabilityjs/test/test-pages/zhihu/expected-metadata.json @@ -3,7 +3,7 @@ "byline": "匿名用户", "dir": null, "excerpt": "二手房，流程", - "siteName": null, + "siteName": "fakehost", "siteIcon": "https://static.zhihu.com/heifetz/assets/apple-touch-icon-152.81060cab.png", "publishedDate": "2016-06-28T07:03:00.000Z", "language": "English",