From 548f317607daed8085a20d7cb1b6305237765ed8 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Wed, 30 Nov 2022 11:04:05 +0800 Subject: [PATCH] Skip cleanning nytimes article first few paragraphs which has too many links --- packages/readabilityjs/Readability.js | 6 +++++- .../test/test-pages/nytimes.com/expected.html | 9 +++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/packages/readabilityjs/Readability.js b/packages/readabilityjs/Readability.js index 1df750d69..409604e9f 100644 --- a/packages/readabilityjs/Readability.js +++ b/packages/readabilityjs/Readability.js @@ -244,6 +244,9 @@ Readability.prototype = { "apos": "'", }, + // These are the classes that we skip when cleaning a tag + CLASSES_TO_SKIP: ["post-body", "StoryBodyCompanionColumn"], + /** * Run any post-process modifications to article content as necessary. * @@ -2782,6 +2785,7 @@ Readability.prototype = { return false; } + var parentClasses = node.parentNode.classList; var haveToRemove = !this._isOmnivoreNode(node) && ( (img > 1 && p / img < 0.5 && !this._hasAncestorTag(node, "figure")) || @@ -2789,7 +2793,7 @@ Readability.prototype = { (input > Math.floor(p/3)) || (!isList && headingDensity < 0.9 && contentLength < 25 && (img === 0 || img > 2) && !this._hasAncestorTag(node, "figure")) || // ignores link density for the links inside the .post-body div (the main content) - (!isList && weight < 25 && linkDensity > 0.2 && !(node.parentElement.className.includes("post-body") && linkDensity === 1)) || + (!isList && weight < 25 && linkDensity > 0.2 && !(this.CLASSES_TO_SKIP.some((c) => parentClasses.contains(c))) )|| // some website like https://substack.com might have their custom styling of tweets // we should omit ignoring their particular case by checking against "tweet" classname (weight >= 25 && linkDensity > 0.5 && !(node.className === "tweet" && linkDensity === 1)) || diff --git a/packages/readabilityjs/test/test-pages/nytimes.com/expected.html b/packages/readabilityjs/test/test-pages/nytimes.com/expected.html index b4d4df352..ceb0aa925 100644 --- a/packages/readabilityjs/test/test-pages/nytimes.com/expected.html +++ b/packages/readabilityjs/test/test-pages/nytimes.com/expected.html @@ -34,6 +34,15 @@

+
+

+ Elon Musk closes his purchase of Twitter and fires several top executives. Follow for the latest news updates. +

+

SAN FRANCISCO — In August 2018, Elon Musk called Michael Dell for advice.

+

Mr. Musk, who was trying to take Tesla, his electric car company, private, quizzed Mr. Dell, who had done that very thing with his eponymous computer company in 2013, about the process and the best lawyers to use for the complicated transaction.

+

“I was really asking him about — did he find being private was good?” Mr. Musk recalled in a deposition about the Tesla effort. “Did he regret going private?”

+

Mr. Dell warned Mr. Musk that it was “a very difficult process” but that he was glad to have done it, according to court filings.

+

Now, Mr. Musk, who did not end up taking Tesla private, is doing so with Twitter. As part of his $44 billion acquisition of the social media service, which closed on Thursday, he is delisting the company’s stock and taking it out of the hands of public shareholders.

Making Twitter a private company gives Mr. Musk some advantages. Unlike publicly traded companies, privately held firms do not have to make quarterly public disclosures about their performance. They are also subject to less regulatory scrutiny and can be more tightly controlled by an owner. That means Mr. Musk can make over Twitter — including tweaking the platform’s content rules, its finances and its priorities — without having to consider the worries of the investing public.