From 01163042844952b61f3155a4cf70158cda414eb1 Mon Sep 17 00:00:00 2001
From: Jackson Harper
Date: Wed, 12 Oct 2022 14:27:28 +0800
Subject: [PATCH 1/4] Dont exclude substacks in article headers
Substack uses this class on header elements like
to give
them anchors in the documents, but we mark content with `header`
in its class name as unlikely.
---
packages/readabilityjs/Readability.js | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/packages/readabilityjs/Readability.js b/packages/readabilityjs/Readability.js
index e7e67d2fe..295bf6597 100644
--- a/packages/readabilityjs/Readability.js
+++ b/packages/readabilityjs/Readability.js
@@ -174,7 +174,7 @@ Readability.prototype = {
unlikelyCandidates: /\bad\b|ai2html|banner|breadcrumbs|breadcrumb|combx|comment|community|cover-wrap|disqus|extra|footer|gdpr|header|legends|menu|related|remark|replies|rss|shoutbox|sidebar|skyscraper|social|sponsor|supplemental|ad-break|agegate|pagination|pager(?!ow)|popup|yom-remote|copyright|keywords|outline|infinite-list|beta|recirculation|site-index|hide-for-print|post-end-share-cta|post-end-cta-full|post-footer|post-head|post-tag|li-date|main-navigation|programtic-ads|outstream_article|hfeed|comment-holder|back-to-top|show-up-next|onward-journey|topic-tracker|list-nav|block-ad-entity|adSpecs|gift-article-button|modal-title|in-story-masthead|share-tools|standard-dock|expanded-dock|margins-h/i,
// okMaybeItsACandidate: /and|article(?!-breadcrumb)|body|column|content|main|shadow|post-header/i,
get okMaybeItsACandidate() {
- return new RegExp(`and|(?
Date: Wed, 12 Oct 2022 14:31:34 +0800
Subject: [PATCH 2/4] Remove the subscriber dialog that was part of the header
before
---
packages/readabilityjs/Readability.js | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/packages/readabilityjs/Readability.js b/packages/readabilityjs/Readability.js
index 295bf6597..86717a1b2 100644
--- a/packages/readabilityjs/Readability.js
+++ b/packages/readabilityjs/Readability.js
@@ -171,7 +171,7 @@ Readability.prototype = {
// Readability-readerable.js. Please keep both copies in sync.
articleNegativeLookBehindCandidates: /breadcrumbs|breadcrumb|utils|trilist/i,
articleNegativeLookAheadCandidates: /outstream(.?)_|sub(.?)_|m_|omeda-promo-|in-article-advert|block-ad-.*/i,
- unlikelyCandidates: /\bad\b|ai2html|banner|breadcrumbs|breadcrumb|combx|comment|community|cover-wrap|disqus|extra|footer|gdpr|header|legends|menu|related|remark|replies|rss|shoutbox|sidebar|skyscraper|social|sponsor|supplemental|ad-break|agegate|pagination|pager(?!ow)|popup|yom-remote|copyright|keywords|outline|infinite-list|beta|recirculation|site-index|hide-for-print|post-end-share-cta|post-end-cta-full|post-footer|post-head|post-tag|li-date|main-navigation|programtic-ads|outstream_article|hfeed|comment-holder|back-to-top|show-up-next|onward-journey|topic-tracker|list-nav|block-ad-entity|adSpecs|gift-article-button|modal-title|in-story-masthead|share-tools|standard-dock|expanded-dock|margins-h/i,
+ unlikelyCandidates: /\bad\b|ai2html|banner|breadcrumbs|breadcrumb|combx|comment|community|cover-wrap|disqus|extra|footer|gdpr|header|legends|menu|related|remark|replies|rss|shoutbox|sidebar|skyscraper|social|sponsor|supplemental|ad-break|agegate|pagination|pager(?!ow)|popup|yom-remote|copyright|keywords|outline|infinite-list|beta|recirculation|site-index|hide-for-print|post-end-share-cta|post-end-cta-full|post-footer|post-head|post-tag|li-date|main-navigation|programtic-ads|outstream_article|hfeed|comment-holder|back-to-top|show-up-next|onward-journey|topic-tracker|list-nav|block-ad-entity|adSpecs|gift-article-button|modal-title|in-story-masthead|share-tools|standard-dock|expanded-dock|margins-h|subscribe-dialog/i,
// okMaybeItsACandidate: /and|article(?!-breadcrumb)|body|column|content|main|shadow|post-header/i,
get okMaybeItsACandidate() {
return new RegExp(`and|(?
Date: Wed, 12 Oct 2022 14:34:36 +0800
Subject: [PATCH 3/4] Add test from the new Omnivore Getting Started article
---
.../expected-metadata.json | 12 +
.../omnivore_getting_started/expected.html | 257 ++++++++++++++
.../omnivore_getting_started/source.html | 321 ++++++++++++++++++
.../omnivore_getting_started/url.txt | 1 +
4 files changed, 591 insertions(+)
create mode 100644 packages/readabilityjs/test/test-pages/omnivore_getting_started/expected-metadata.json
create mode 100644 packages/readabilityjs/test/test-pages/omnivore_getting_started/expected.html
create mode 100644 packages/readabilityjs/test/test-pages/omnivore_getting_started/source.html
create mode 100644 packages/readabilityjs/test/test-pages/omnivore_getting_started/url.txt
diff --git a/packages/readabilityjs/test/test-pages/omnivore_getting_started/expected-metadata.json b/packages/readabilityjs/test/test-pages/omnivore_getting_started/expected-metadata.json
new file mode 100644
index 000000000..7be1e12a8
--- /dev/null
+++ b/packages/readabilityjs/test/test-pages/omnivore_getting_started/expected-metadata.json
@@ -0,0 +1,12 @@
+{
+ "title": "Getting Started with Omnivore",
+ "byline": "Omnivore",
+ "dir": null,
+ "excerpt": "Omnivore is a read-it-later app that lets you save and organize everything you read online.",
+ "siteName": "Omnivore",
+ "siteIcon": "https://bucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com/public/images/8f6c575c-355b-44a6-a8e6-67a1c3745b59/favicon.ico",
+ "previewImage": "https://substackcdn.com/image/fetch/w_1008,h_528,c_fill,f_jpg,q_auto:best,fl_progressive:steep/https%3A%2F%2Fblog.omnivore.app%2Ftwitter%2Fsubscribe-card.svg%3Fv%3De03aaf5cd51e4035f38f2aa621f6a29c%26version%3D7",
+ "publishedDate": null,
+ "language": "English",
+ "readerable": true
+}
diff --git a/packages/readabilityjs/test/test-pages/omnivore_getting_started/expected.html b/packages/readabilityjs/test/test-pages/omnivore_getting_started/expected.html
new file mode 100644
index 000000000..b6f0e25f6
--- /dev/null
+++ b/packages/readabilityjs/test/test-pages/omnivore_getting_started/expected.html
@@ -0,0 +1,257 @@
+
+
+
+
Omnivore is a read-it-later app that lets you save and organize everything you read online.
+
+
+
This guide will show you how to use Omnivore’s basic functions and advanced features, divided into four main activities:
+
+
+
Saving
+
+
+
Reading
+
+
+
Organizing
+
+
+
Integrations
+
+
+
The Library is the center of your Omnivore experience, where you can quickly access any links you have saved. Saved links remain in your Library forever unless you delete them.
+
There are five ways to save links to pages or articles that you wish to read later:
+
+
+
Saving from Your Omnivore Library
+
+
+
Saving from a Browser
+
+
+
Saving from a Phone or Tablet (iOS or Android)
+
+
+
Newsletter Subscriptions via Email
+
+
+
Saving PDFs from a Mac
+
+
+
Saving from Your Omnivore Library
+
1. In the upper right corner of your Library, tap the Add Link button. 2. Enter the URL you wish to save and tap Add Link. 3. The link will appear in your Library the next time you refresh it.
+
Saving from a Browser
+
1. Download and install the Omnivore extension for your browser:
2. Navigate to the page you wish to save and tap the Omnivore button in your browser’s toolbar or Extensions menu. 3. Alternatively, you can right-click (command+click on Mac) on any hyperlink and select Save to Omnivore from the menu. 4. The link will appear in your Library the next time you refresh it.
+
Saving from a Phone or Tablet
+
The best way to save links from your mobile device is via the Omnivore app. You can download the app here:
On your Mac, locate the PDF you wish to save and right-click or ctrl+click on the file name.
+
+
+
Select Share from the menu and choose Omnivore.
+
+
+
The link will appear in your Library the next time you refresh it.
+
+
+
Reading
+
Click any link saved in your Library to enter the Reader view.
+
Omnivore formats pages for easy reading and highlighting, removing ads and clutter for distraction-free reading. The text-focused view also makes articles smaller and quicker to load.
+
While reading, you can:
+
+
+
Change Formatting
+
+
+
Highlight Text
+
+
+
Add Notes
+
+
+
View All Saved Highlights and Notes
+
+
+
Track Reading Progress
+
+
+
Change Formatting
+
+
+
Theme: Tap your photo, initial, or avatar in the top right corner to access the profile menu. Select the white or black thumbnail to choose the Light or Dark theme.
+
+
+
Text Formatting: Tap the Aa icon to adjust the text size, font, margins, and line spacing.
+
+
+
Highlight Text
+
+
+
Select the text you wish to highlight.
+
+
+
Tap the Highlight button.
+
+
+
The text will appear highlighted next time you view the article.
+
+
+
Add Notes
+
+
+
Highlight a section of text where you wish to add a note.
+
+
+
Tap the Note button, type your note, and tap Save.
+
+
+
The Note icon will appear next time you view this article.
+
+
+
View All Saved Highlights and Notes
+
+
+
Tap the Highlight/Note icon to see a list of all the highlighted text and notes you have added to this page.
+
+
+
To remove a note or highlight, select it from the list and tap the Trash icon.
+
+
+
Track Reading Progress
+
Omnivore automatically keeps track of your reading progress across your different devices so you can easily pick up where you left off. A progress bar will appear at the top of each link in your Library after you have started reading.
+
Organizing
+
By default, the Library inbox displays all links you have saved. To manage your list and keep your reading organized, Omnivore provides the following actions:
+
+
+
Archiving
+
+
+
Labels
+
+
+
Search
+
+
+
Filters
+
+
+
Archiving (Web)
+
+
+
Tap the Menu icon next to the link you wish to archive (on the mobile app, long press the link to open the menu).
+
+
+
Select Archive.
+
+
+
The link will disappear from the default Library view, but will show up if you select the Archived filter (see Filters below).
+
+
+
Labels
+
+
+
Tap the Menu icon next to any link and select Set Labels.
+
+
+
Select an existing label from the list or tap Edit Labels to create a new one.
+
+
+
The label will appear next to the link in your Library. Tap it to view all links with the same label.
+
+
+
Omnivore mobile app only: tap Labels to see a complete list of all labels you have used; tap one to view all links with the same label
+
+
+
Note: Omnivore will automatically assign some labels, such as “Newsletters.”
+
+
+
Search
+
+
+
To search through all your saved links, enter a keyword or phrase in the search bar.
Use the Filters menu to refine your Library view (some filters may be visible by default).
+
+
+
Select Read Later to view a list of all your non-archived links except Newsletters.
+
+
+
Select Highlights to view the text selections you have highlighted in all your saved pages.
+
+
+
Select Today to view a list of links you saved today.
+
+
+
Select Newsletters to view links saved via your newsletter subscriptions.
+
+
+
Integrations
+
Omnivore allows integrations with knowledge bases and note-taking apps including:
+
+
+
Logseq
+
+
+
Webhooks
+
+
+
Logseq
+
With Omnivore's Logseq plugin you can sync all your saved articles, highlights, and notes into Logseq, a popular knowledge base. For information on setting up and using the Logseq plugin, please refer to this helpful Omnivore for Logseq Plugin Guide.
+
Webhooks
+
Omnivore can trigger webhooks when you save a link or add highlights to a page you are reading. This example shows webhooks being used to write all saved links to a Google Sheets spreadsheet stored on a Google Drive.
This guide will show you how to use Omnivore’s basic functions and advanced features, divided into four main activities:
Saving
Reading
Organizing
Integrations
The Library is the center of your Omnivore experience, where you can quickly access any links you have saved. Saved links remain in your Library forever unless you delete them.
Saving
There are five ways to save links to pages or articles that you wish to read later:
Saving from Your Omnivore Library
Saving from a Browser
Saving from a Phone or Tablet (iOS or Android)
Newsletter Subscriptions via Email
Saving PDFs from a Mac
Saving from Your Omnivore Library
1. In the upper right corner of your Library, tap the Add Link button. 2. Enter the URL you wish to save and tap Add Link. 3. The link will appear in your Library the next time you refresh it.
Saving from a Browser
1. Download and install the Omnivore extension for your browser:
2. Navigate to the page you wish to save and tap the Omnivore button in your browser’s toolbar or Extensions menu. 3. Alternatively, you can right-click (command+click on Mac) on any hyperlink and select Save to Omnivore from the menu. 4. The link will appear in your Library the next time you refresh it.
Saving from a Phone or Tablet
The best way to save links from your mobile device is via the Omnivore app. You can download the app here:
On your Mac, locate the PDF you wish to save and right-click or ctrl+click on the file name.
Select Share from the menu and choose Omnivore.
The link will appear in your Library the next time you refresh it.
Reading
Click any link saved in your Library to enter the Reader view.
Omnivore formats pages for easy reading and highlighting, removing ads and clutter for distraction-free reading. The text-focused view also makes articles smaller and quicker to load.
While reading, you can:
Change Formatting
Highlight Text
Add Notes
View All Saved Highlights and Notes
Track Reading Progress
Change Formatting
Theme: Tap your photo, initial, or avatar in the top right corner to access the profile menu. Select the white or black thumbnail to choose the Light or Dark theme.
Text Formatting: Tap the Aa icon to adjust the text size, font, margins, and line spacing.
Highlight Text
Select the text you wish to highlight.
Tap the Highlight button.
The text will appear highlighted next time you view the article.
Add Notes
Highlight a section of text where you wish to add a note.
Tap the Note button, type your note, and tap Save.
The Note icon will appear next time you view this article.
View All Saved Highlights and Notes
Tap the Highlight/Note icon to see a list of all the highlighted text and notes you have added to this page.
To remove a note or highlight, select it from the list and tap the Trash icon.
Track Reading Progress
Omnivore automatically keeps track of your reading progress across your different devices so you can easily pick up where you left off. A progress bar will appear at the top of each link in your Library after you have started reading.
Organizing
By default, the Library inbox displays all links you have saved. To manage your list and keep your reading organized, Omnivore provides the following actions:
Archiving
Labels
Search
Filters
Archiving (Web)
Tap the Menu icon next to the link you wish to archive (on the mobile app, long press the link to open the menu).
Select Archive.
The link will disappear from the default Library view, but will show up if you select the Archived filter (see Filters below).
Labels
Tap the Menu icon next to any link and select Set Labels.
Select an existing label from the list or tap Edit Labels to create a new one.
The label will appear next to the link in your Library. Tap it to view all links with the same label.
Omnivore mobile app only: tap Labels to see a complete list of all labels you have used; tap one to view all links with the same label
Note: Omnivore will automatically assign some labels, such as “Newsletters.”
Search
To search through all your saved links, enter a keyword or phrase in the search bar.
Use the Filters menu to refine your Library view (some filters may be visible by default).
Select Read Later to view a list of all your non-archived links except Newsletters.
Select Highlights to view the text selections you have highlighted in all your saved pages.
Select Today to view a list of links you saved today.
Select Newsletters to view links saved via your newsletter subscriptions.
Integrations
Omnivore allows integrations with knowledge bases and note-taking apps including:
Logseq
Webhooks
Logseq
With Omnivore's Logseq plugin you can sync all your saved articles, highlights, and notes into Logseq, a popular knowledge base. For information on setting up and using the Logseq plugin, please refer to this helpful Omnivore for Logseq Plugin Guide.
Webhooks
Omnivore can trigger webhooks when you save a link or add highlights to a page you are reading. This example shows webhooks being used to write all saved links to a Google Sheets spreadsheet stored on a Google Drive.
Subscribe to Omnivore
Launched a year ago
Updates from the Omnivore team
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/packages/readabilityjs/test/test-pages/omnivore_getting_started/url.txt b/packages/readabilityjs/test/test-pages/omnivore_getting_started/url.txt
new file mode 100644
index 000000000..8656d054d
--- /dev/null
+++ b/packages/readabilityjs/test/test-pages/omnivore_getting_started/url.txt
@@ -0,0 +1 @@
+https://blog.omnivore.app/p/d0d30ea5-49aa-4c04-8fae-c004be9c51b9
\ No newline at end of file
From 7af068e8dca2a13269aa601cd73f15e2a69d41f3 Mon Sep 17 00:00:00 2001
From: Jackson Harper
Date: Wed, 12 Oct 2022 14:51:59 +0800
Subject: [PATCH 4/4] Header fix adds the headers back in for this test case
---
.../test/test-pages/computerenhance.com/expected.html | 4 ++++
1 file changed, 4 insertions(+)
diff --git a/packages/readabilityjs/test/test-pages/computerenhance.com/expected.html b/packages/readabilityjs/test/test-pages/computerenhance.com/expected.html
index d8559f1f7..96b3e35c3 100644
--- a/packages/readabilityjs/test/test-pages/computerenhance.com/expected.html
+++ b/packages/readabilityjs/test/test-pages/computerenhance.com/expected.html
@@ -56,6 +56,7 @@
Which brings us to the third item on the list, and the real sticking point. As far as I’m aware, no current or planned future Web API ever lets you do number three. There are many new web “technologies” swarming around the custom packet idea (WebRTC,WebSockets,WebTransport), but to the best of my knowledge, all of them require an HTTPS connection to be made first, so your “custom packet” servers still need to implement all of HTTPS anyway.
+
What is the deployment scenario?
I can imagine someone raising the following objection at this point: “If you don’t support HTTPS on the server, how do you serve the WASM/JavaScript/whatever with the custom packet logic in the first place?”
That’s a reasonable question.
The answer is, the two most logical deployment scenarios I can think of both involve a separate server (or process) for the initial HTTPS transaction.
@@ -64,6 +65,7 @@
The second would be less common, but plausible: you run your own CDN-equivalent, becauseyou’re just that hard core. But you expect that your HTTPS code is more vulnerable than your custom code, since HTTPS is vastly more complicated and has ridiculous things in it like arbitrary text parsing, which no one in their right mind would ever put into a “secure” protocol. So you cabin your HTTPS server instances into their own restricted processes or own machines entirely. This prevents exploits of the HTTPS code from affecting anything other than newly connecting users - existing users (who are only talking to your custom servers) remain unharmed.
In neither scenario do you actually include HTTPS code in any of the processes running your actual secure server.
+
What precedent has been set by other “secure” APIs in the browser?
So that’s the hopefully-at-least-somewhat-convincing explanation of why someone might want raw UDP. Now the question is, can raw UDP be provided by a browser in a way that is “secure”?
I’m putting a lot of these words in scare quotes because browsersaren’tsecure for any serious definition of that word, and hopefully that is overwhelmingly obvious to everyone who has ever used one. But just to be clear about the landscape, there are two different ways browsers are not secure:
@@ -113,6 +115,7 @@
Hopefully we can all agree that this extremely low bar for security is the only hurdle one should have to clear in order to dismiss concerns of “security” as a reason not to implement a feature in a W3C spec. It’s not much, but it issomething.
+
Can raw UDP meet this security standard?
OK, finally, with all that out of the way, this is what I actually wanted someone to point me towhen I asked about this on Twitter. I just wanted to see that someone, somewhere, had worked out exactly why UDP could not be made to fit the same security model considered acceptable across other basic web features already deployed and considered “secure”.
@@ -142,6 +145,7 @@
Nobody seems to be panicked about this. Nobody has pushed the policy that the W3C should standardize on a specific web server deployment that you are forced to use, or a set of n of them made by Google/Mozilla/Apple, or what have you. It is just assumed that everyone is allowed to write their ownserverpacket handling, but that no one is allowed to write their ownclientpacket handling.
So that’s what I would like explained. Internet, justify this!
+
Appendix A.1.2b subsection 12: What about destination consent?
I have seen people mention (but not support) a claim that raw UDP would cause “denial of service” problems because malicious web pages would send UDP packets to random servers in an attempt to overload them. This claim seems completely baseless to me, because there is no reason why you can’t employ the relevant XHR DDoS restrictions to UDP. If DDoS was the concern, just require that UDP packets be sent exclusively within the same domain as the originating code.
Furthermore, you could restrict the port ranges of raw web UDP to some assigned range. A new port range could be explicitly reservedjust for raw web UDPif that makes people more comfortable, so it could literally be discarded at the gateway on any network that doesn’t want to support raw UDP for web, making it easier to deal with than UDP attacks from native code and viruses which can choose their ports at will.