prehistoric-systems/omnivore
1
0
Fork 0
mirror of https://github.com/omnivore-app/omnivore.git synced 2026-03-11 08:54:26 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge pull request #723 from omnivore-app/fix/tweet-not-saved

Browse source 
Fix tweet not saved
This commit is contained in:
Hongbo Wu 2022-06-02 11:20:40 +08:00 committed by GitHub
commit 89164a353f
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
18 changed files with 7227 additions and 125 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
packages/api/package.json
View file

@ -89,6 +89,7 @@
"@types/analytics-node": "^3.1.7",
"@types/bcryptjs": "^2.4.2",
"@types/chai": "^4.2.18",
"@types/chai-as-promised": "^7.1.5",
"@types/chai-string": "^1.4.2",
"@types/cookie": "^0.4.0",
"@types/cookie-parser": "^1.4.2",
@ -109,6 +110,7 @@
"@types/uuid": "^8.3.0",
"@types/voca": "^1.4.0",
"chai": "^4.3.4",
"chai-as-promised": "^7.1.1",
"chai-string": "^1.5.0",
"circular-dependency-plugin": "^5.2.0",
"mocha": "^9.0.1",

2
packages/api/src/readability.d.ts vendored
View file

@ -72,7 +72,7 @@ declare module '@omnivore/readability' {
*
* The response will be null if the processing failed (https://github.com/mozilla/readability/blob/52ab9b5c8916c306a47b2119270dcdabebf9d203/Readability.js#L2038)
*/
parse(): Readability.ParseResult | null
async parse(): Promise<Readability.ParseResult | null>
}
namespace Readability {

2
packages/api/src/routers/svc/emails.ts
View file

@ -51,7 +51,7 @@ export function emailsServiceRouter() {
return
}
if (isProbablyNewsletter(data.html)) {
if (await isProbablyNewsletter(data.html)) {
console.log('handling as newsletter', data)
await saveNewsletterEmail({
email: data.to,

12
packages/api/src/utils/parser.ts
View file

@ -133,12 +133,12 @@ const getPurifiedContent = (html: string): Document => {
return parseHTML(clean).document
}
const getReadabilityResult = (
const getReadabilityResult = async (
url: string,
html: string,
document: Document,
isNewsletter?: boolean
): Readability.ParseResult | null => {
): Promise<Readability.ParseResult | null> => {
// First attempt to read the article as is.
// if that fails attempt to purify then read
const sources = [
@ -157,7 +157,7 @@ const getReadabilityResult = (
}
try {
const article = new Readability(document, {
const article = await new Readability(document, {
debug: DEBUG_MODE,
createImageProxyUrl,
keepTables: isNewsletter,
@ -236,7 +236,7 @@ export const parsePreparedContent = async (
await applyHandlers(url, dom)
try {
article = getReadabilityResult(url, document, dom, isNewsletter)
article = await getReadabilityResult(url, document, dom, isNewsletter)
if (!article?.textContent && allowRetry) {
const newDocument = {
...preparedDocument,
@ -406,10 +406,10 @@ export const parseUrlMetadata = async (
// based on it's contents.
// TODO: when we consolidate the handlers we could include this
// as a utility method on each one.
export const isProbablyNewsletter = (html: string): boolean => {
export const isProbablyNewsletter = async (html: string): Promise<boolean> => {
const dom = parseHTML(html).document
const domCopy = parseHTML(dom.documentElement.outerHTML).document
const article = new Readability(domCopy, {
const article = await new Readability(domCopy, {
debug: false,
keepTables: true,
}).parse()

104
packages/api/test/utils/parser.test.ts
View file

@ -1,54 +1,73 @@
import 'mocha'
import * as chai from 'chai'
import { expect } from 'chai'
import 'chai/register-should'
import fs from 'fs'
import { findNewsletterUrl, isProbablyNewsletter, parsePageMetadata, parsePreparedContent } from '../../src/utils/parser'
import {
findNewsletterUrl,
isProbablyNewsletter,
parsePageMetadata,
parsePreparedContent,
} from '../../src/utils/parser'
import nock from 'nock'
import chaiAsPromised from 'chai-as-promised'
chai.use(chaiAsPromised)
const load = (path: string): string => {
return fs.readFileSync(path, 'utf8')
}
describe('isProbablyNewsletter', () => {
it('returns true for substack newsletter', () => {
it('returns true for substack newsletter', async () => {
const html = load('./test/utils/data/substack-forwarded-newsletter.html')
isProbablyNewsletter(html).should.be.true
await expect(isProbablyNewsletter(html)).to.eventually.be.true
})
it('returns true for private forwarded substack newsletter', () => {
const html = load('./test/utils/data/substack-private-forwarded-newsletter.html')
isProbablyNewsletter(html).should.be.true
it('returns true for private forwarded substack newsletter', async () => {
const html = load(
'./test/utils/data/substack-private-forwarded-newsletter.html'
)
await expect(isProbablyNewsletter(html)).to.eventually.be.true
})
it('returns false for substack welcome email', () => {
it('returns false for substack welcome email', async () => {
const html = load('./test/utils/data/substack-forwarded-welcome-email.html')
isProbablyNewsletter(html).should.be.false
await expect(isProbablyNewsletter(html)).to.eventually.be.false
})
it('returns true for beehiiv.com newsletter', () => {
it('returns true for beehiiv.com newsletter', async () => {
const html = load('./test/utils/data/beehiiv-newsletter.html')
isProbablyNewsletter(html).should.be.true
await expect(isProbablyNewsletter(html)).to.eventually.be.true
})
})
describe('findNewsletterUrl', async () => {
it('gets the URL from the header if it is a substack newsletter', async () => {
nock('https://newsletter.slowchinese.net')
.head('/p/companies-that-eat-people-217?token=eyJ1c2VyX2lkIjoxMTU0MzM0NSwicG9zdF9pZCI6NDg3MjA5NDAsImlhdCI6MTY0NTI1NzQ1MSwiaXNzIjoicHViLTI4MDUzMSIsInN1YiI6InBvc3QtcmVhY3Rpb24ifQ.l5F3Kx6K9tvy9cRAXx3MepobQBCJDJQgAxOpA0INIZA')
.reply(200, '');
.head(
'/p/companies-that-eat-people-217?token=eyJ1c2VyX2lkIjoxMTU0MzM0NSwicG9zdF9pZCI6NDg3MjA5NDAsImlhdCI6MTY0NTI1NzQ1MSwiaXNzIjoicHViLTI4MDUzMSIsInN1YiI6InBvc3QtcmVhY3Rpb24ifQ.l5F3Kx6K9tvy9cRAXx3MepobQBCJDJQgAxOpA0INIZA'
)
.reply(200, '')
const html = load('./test/utils/data/substack-forwarded-newsletter.html')
const url = await findNewsletterUrl(html)
// Not sure if the redirects from substack expire, this test could eventually fail
expect(url).to.startWith('https://newsletter.slowchinese.net/p/companies-that-eat-people-217')
expect(url).to.startWith(
'https://newsletter.slowchinese.net/p/companies-that-eat-people-217'
)
})
it('gets the URL from the header if it is a beehiiv newsletter', async () => {
nock('https://u23463625.ct.sendgrid.net')
.head('/ss/c/AX1lEgEQaxtvFxLaVo0GBo_geajNrlI1TGeIcmMViR3pL3fEDZnbbkoeKcaY62QZk0KPFudUiUXc_uMLerV4nA/3k5/3TFZmreTR0qKSCgowABnVg/h30/zzLik7UXd1H_n4oyd5W8Xu639AYQQB2UXz-CsssSnno')
.reply(302, undefined,{
'Location': 'https://www.milkroad.com/p/talked-guy-spent-30m-beeple'
})
.get('/p/talked-guy-spent-30m-beeple')
.reply(200, '');
.head(
'/ss/c/AX1lEgEQaxtvFxLaVo0GBo_geajNrlI1TGeIcmMViR3pL3fEDZnbbkoeKcaY62QZk0KPFudUiUXc_uMLerV4nA/3k5/3TFZmreTR0qKSCgowABnVg/h30/zzLik7UXd1H_n4oyd5W8Xu639AYQQB2UXz-CsssSnno'
)
.reply(302, undefined, {
Location: 'https://www.milkroad.com/p/talked-guy-spent-30m-beeple',
})
.get('/p/talked-guy-spent-30m-beeple')
.reply(200, '')
const html = load('./test/utils/data/beehiiv-newsletter.html')
const url = await findNewsletterUrl(html)
expect(url).to.startWith('https://www.milkroad.com/p/talked-guy-spent-30m-beeple')
expect(url).to.startWith(
'https://www.milkroad.com/p/talked-guy-spent-30m-beeple'
)
})
it('returns undefined if it is not a newsletter', async () => {
const html = load('./test/utils/data/substack-forwarded-welcome-email.html')
@ -63,31 +82,35 @@ describe('parseMetadata', async () => {
const metadata = await parsePageMetadata(html)
expect(metadata?.author).to.deep.equal('Omnivore')
expect(metadata?.title).to.deep.equal('Code Block Syntax Highlighting')
expect(metadata?.previewImage).to.deep.equal('https://cdn.substack.com/image/fetch/w_1200,h_600,c_fill,f_jpg,q_auto:good,fl_progressive:steep,g_auto/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2F2ab1f7e8-2ca7-4011-8ccb-43d0b3bd244f_1490x2020.png')
expect(metadata?.description).to.deep.equal('Highlighted <code> in Omnivore')
expect(metadata?.previewImage).to.deep.equal(
'https://cdn.substack.com/image/fetch/w_1200,h_600,c_fill,f_jpg,q_auto:good,fl_progressive:steep,g_auto/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2F2ab1f7e8-2ca7-4011-8ccb-43d0b3bd244f_1490x2020.png'
)
expect(metadata?.description).to.deep.equal(
'Highlighted <code> in Omnivore'
)
})
})
describe('parsePreparedContent', async () => {
it('gets published date when JSONLD fails to load', async () => {
const html = load('./test/utils/data/stratechery-blog-post.html')
const result = await parsePreparedContent(
'https://example.com/',
{
document: html,
pageInfo: { }
},
const result = await parsePreparedContent('https://example.com/', {
document: html,
pageInfo: {},
})
expect(result.parsedContent?.publishedDate?.getTime()).to.equal(
new Date('2016-04-05T15:27:51+00:00').getTime()
)
expect(result.parsedContent?.publishedDate?.getTime()).to.equal(new Date('2016-04-05T15:27:51+00:00').getTime())
})
})
describe('parsePreparedContent', async () => {
nock('https://oembeddata').get('/').reply(200, {
"version":"1.0",
"provider_name":"Hippocratic Adventures",
"provider_url":"https:\/\/www.hippocraticadventures.com",
"title":"The Ultimate Guide to Practicing Medicine in Singapore &#8211; Part 2"
version: '1.0',
provider_name: 'Hippocratic Adventures',
provider_url: 'https://www.hippocraticadventures.com',
title:
'The Ultimate Guide to Practicing Medicine in Singapore &#8211; Part 2',
})
it('gets metadata from external JSONLD if available', async () => {
@ -98,13 +121,12 @@ describe('parsePreparedContent', async () => {
</head>
<body>body</body>
</html>`
const result = await parsePreparedContent(
'https://example.com/',
{
document: html,
pageInfo: { }
},
)
expect(result.parsedContent?.title).to.equal('The Ultimate Guide to Practicing Medicine in Singapore – Part 2')
const result = await parsePreparedContent('https://example.com/', {
document: html,
pageInfo: {},
})
expect(result.parsedContent?.title).to.equal(
'The Ultimate Guide to Practicing Medicine in Singapore – Part 2'
)
})
})

75
packages/readabilityjs/Readability.js
View file

@ -22,6 +22,7 @@
var parseSrcset = require('parse-srcset');
var htmlEntities = require('html-entities')
const axios = require("axios");
/** Checks whether an element is a wrapper for tweet */
const hasTweetInChildren = element => {
@ -591,6 +592,12 @@ Readability.prototype = {
continue;
}
// If we have a node with only one child element which has the placeholder class, keep it
if (this._hasSingleTagInsideElement(node, "DIV") && this.PLACEHOLDER_CLASSES.includes(node.firstElementChild.className)) {
node = this._getNextNode(node);
continue;
}
if (node.parentNode && ["DIV", "SECTION"].includes(node.tagName) && !(node.id && node.id.startsWith("readability"))) {
if (this._isElementWithoutContent(node)) {
node = this._removeAndGetNext(node);
@ -823,8 +830,8 @@ Readability.prototype = {
* @param Element
* @return void
**/
_prepArticle: function (articleContent) {
this._createPlaceholders(articleContent);
_prepArticle: async function (articleContent) {
await this._createPlaceholders(articleContent);
this._cleanStyles(articleContent);
// Check for data tables before we continue, to avoid removing items in
// those tables, which will often be isolated even though they're
@ -1098,7 +1105,7 @@ Readability.prototype = {
* @param page a document to run upon. Needs to be a full document, complete with body.
* @return Element
**/
_grabArticle: function (page) {
_grabArticle: async function(page) {
this.log("**** grabArticle ****");
const doc = this._doc;
const isPaging = page !== null;
@ -1141,13 +1148,13 @@ Readability.prototype = {
if (shouldRemoveTitleHeader && this._headerDuplicatesTitle(node)) {
const headingText = node.textContent.trim();
const titleText = this._articleTitle.trim();
this.log("Removing header: ", {headingText, titleText});
this.log("Removing header: ", { headingText, titleText });
shouldRemoveTitleHeader = false;
// Replacing title with the heading if the title includes heading but heading is smaller
// Example article: http://jsomers.net/i-should-have-loved-biology
// Or if there is the specific attribute that we can lean on.
// For example "headline" in this article - https://nymag.com/intelligencer/2020/12/four-seasons-total-landscaping-the-full-est-possible-story.html
if ((titleText !== headingText && titleText.includes(headingText)) || this._someNodeAttribute(node, ({value}) => value === 'headline')) {
if ((titleText !== headingText && titleText.includes(headingText)) || this._someNodeAttribute(node, ({ value }) => value === 'headline')) {
this.log('Replacing title with heading')
this._articleTitle = headingText;
}
@ -1159,8 +1166,8 @@ Readability.prototype = {
if (stripUnlikelyCandidates) {
if (
(this.REGEXPS.unlikelyCandidates.test(matchString) ||
// Checking for the "data-testid" attribute as well for the NYTimes articles
// Example article: https://www.nytimes.com/2021/03/31/world/americas/brazil-coronavirus-bolsonaro.html
// Checking for the "data-testid" attribute as well for the NYTimes articles
// Example article: https://www.nytimes.com/2021/03/31/world/americas/brazil-coronavirus-bolsonaro.html
this.REGEXPS.unlikelyCandidates.test(node.dataset && node.dataset.testid)) &&
!this.REGEXPS.okMaybeItsACandidate.test(matchString) &&
!/tweet(-\w+)?/i.test(matchString) &&
@ -1197,8 +1204,8 @@ Readability.prototype = {
// Remove DIV, SECTION, and HEADER nodes without any content(e.g. text, image, video, or iframe).
if ((node.tagName === "DIV" || node.tagName === "SECTION" || node.tagName === "HEADER" ||
node.tagName === "H1" || node.tagName === "H2" || node.tagName === "H3" ||
node.tagName === "H4" || node.tagName === "H5" || node.tagName === "H6") &&
node.tagName === "H1" || node.tagName === "H2" || node.tagName === "H3" ||
node.tagName === "H4" || node.tagName === "H5" || node.tagName === "H6") &&
this._isElementWithoutContent(node)) {
node = this._removeAndGetNext(node);
continue;
@ -1264,7 +1271,7 @@ Readability.prototype = {
* A score is determined by things like number of commas, class names, etc. Maybe eventually link density.
**/
var candidates = [];
this._forEachNode(elementsToScore, function (elementToScore) {
this._forEachNode(elementsToScore, function(elementToScore) {
if (!elementToScore.parentNode || typeof (elementToScore.parentNode.tagName) === "undefined")
return;
@ -1290,7 +1297,7 @@ Readability.prototype = {
contentScore += Math.min(Math.floor(innerText.length / 100), 3);
// Initialize and score ancestors.
this._forEachNode(ancestors, function (ancestor, level) {
this._forEachNode(ancestors, function(ancestor, level) {
if (!ancestor.tagName || !ancestor.parentNode || typeof (ancestor.parentNode.tagName) === "undefined")
return;
@ -1519,7 +1526,7 @@ Readability.prototype = {
const figures = this._getAllNodesWithTag(headerNode, ['FIGURE']);
this._forEachNode(figures, figure => {
if (!this._someNode(alreadyExistingFigures, existingFigure => existingFigure === figure)) {
this.log(`Prepending figure to the article`, {className: figure.className, scr: figure.src})
this.log(`Prepending figure to the article`, { className: figure.className, scr: figure.src })
articleContent.prepend(figure)
}
})
@ -1528,7 +1535,7 @@ Readability.prototype = {
if (this._debug)
this.log("Article content pre-prep: ", { content: articleContent.innerHTML });
// So we have all of the content that we need. Now we clean it up for presentation.
this._prepArticle(articleContent);
await this._prepArticle(articleContent);
if (this._debug)
this.log("Article content post-prep: ", { content: articleContent.innerHTML });
@ -1567,17 +1574,17 @@ Readability.prototype = {
if (this._flagIsActive(this.FLAG_STRIP_UNLIKELYS)) {
this._removeFlag(this.FLAG_STRIP_UNLIKELYS);
this._attempts.push({articleContent: articleContent, textLength: textLength});
this._attempts.push({ articleContent: articleContent, textLength: textLength });
} else if (this._flagIsActive(this.FLAG_WEIGHT_CLASSES)) {
this._removeFlag(this.FLAG_WEIGHT_CLASSES);
this._attempts.push({articleContent: articleContent, textLength: textLength});
this._attempts.push({ articleContent: articleContent, textLength: textLength });
} else if (this._flagIsActive(this.FLAG_CLEAN_CONDITIONALLY)) {
this._removeFlag(this.FLAG_CLEAN_CONDITIONALLY);
this._attempts.push({articleContent: articleContent, textLength: textLength});
this._attempts.push({ articleContent: articleContent, textLength: textLength });
} else {
this._attempts.push({articleContent: articleContent, textLength: textLength});
this._attempts.push({ articleContent: articleContent, textLength: textLength });
// No luck after removing flags, just return the longest text we found during the different loops
this._attempts.sort(function (a, b) {
this._attempts.sort(function(a, b) {
return b.textLength - a.textLength;
});
@ -1594,7 +1601,7 @@ Readability.prototype = {
if (parseSuccessful) {
// Find out text direction from ancestors of final top candidate.
var ancestors = [parentOfTopCandidate, topCandidate].concat(this._getNodeAncestors(parentOfTopCandidate));
this._someNode(ancestors, function (ancestor) {
this._someNode(ancestors, function(ancestor) {
if (!ancestor.tagName)
return false;
var articleDir = ancestor.getAttribute("dir");
@ -2198,15 +2205,15 @@ Readability.prototype = {
}
},
_createPlaceholders: function (e) {
Array.from(e.getElementsByTagName('a')).forEach(element => {
_createPlaceholders: async function (e) {
for (const element of Array.from(e.getElementsByTagName('a'))) {
if (this.isEmbed(element)) {
return;
continue;
}
// Create tweets placeholders from links
if (element.href.includes('twitter.com')) {
if (element.href.includes('twitter.com') || element.parentNode.className === 'tweet') {
const link = element.href;
const regex = /(https?:\/\/twitter\.com\/\w+\/status\/)(\d+)/gm;
const match = regex.exec(link);
@ -2228,6 +2235,22 @@ Readability.prototype = {
if (tweetParent && tweetParent.className.includes('twitter-tweet')) {
tweetParent.parentNode.replaceChild(tweet, tweetParent);
}
} else if (element.parentNode.className === 'tweet') {
// Create tweets placeholders from classname
try {
const response = await axios.get(link);
const tweetUrl = response.request.res.responseUrl;
const match = regex.exec(tweetUrl);
if (Array.isArray(match) && typeof match[2] === 'string') {
const tweet = this._doc.createElement('div');
tweet.innerText = 'Tweet placeholder';
tweet.className = 'tweet-placeholder';
tweet.setAttribute('data-tweet-id', match[2]);
element.parentNode.replaceWith(tweet);
}
} catch (e) {
this.log('Error loading tweet: ', link, e);
}
}
}
@ -2241,7 +2264,7 @@ Readability.prototype = {
this._createInstagramPostPlaceholder(element, match[2]);
}
}
});
}
Array.from(e.getElementsByTagName('iframe')).forEach(element => {
@ -2857,7 +2880,7 @@ Readability.prototype = {
*
* @return void
**/
parse: function () {
parse: async function() {
// Avoid parsing too large documents, as per configuration option
if (this._maxElemsToParse > 0) {
var numTags = this._doc.getElementsByTagName("*").length;
@ -2882,7 +2905,7 @@ Readability.prototype = {
var metadata = this._getArticleMetadata(jsonLd);
this._articleTitle = metadata.title;
var articleContent = this._grabArticle();
var articleContent = await this._grabArticle();
if (!articleContent)
return null;

6
packages/readabilityjs/package.json
View file

@ -24,13 +24,15 @@
"homepage": "https://github.com/mozilla/readability",
"devDependencies": {
"@c4312/matcha": "^1.3.1",
"axios": "^0.26.0",
"chai": "^2.1.*",
"chai-as-promised": "^7.1.1",
"htmltidy2": "^0.3.0",
"js-beautify": "^1.13.0",
"linkedom": "^0.14.9",
"mocha": "^8.2.0",
"puppeteer": "^10.1.0",
"sinon": "^7.3.2",
"linkedom": "^0.14.9"
"sinon": "^7.3.2"
},
"dependencies": {
"html-entities": "^2.3.2",

12
packages/readabilityjs/test/generate-testcase.js
View file

@ -192,7 +192,7 @@ function onResponseReceived(error, source, destRoot) {
console.log("writing");
}
var sourcePath = path.join(destRoot, "source.html");
fs.writeFile(sourcePath, source, function (err) {
fs.writeFile(sourcePath, source, async function(err) {
if (err) {
console.error("Couldn't write data to source.html!");
console.error(err);
@ -201,11 +201,11 @@ function onResponseReceived(error, source, destRoot) {
if (debug) {
console.log("Running readability stuff");
}
runReadability(source, path.join(destRoot, "expected.html"), path.join(destRoot, "expected-metadata.json"));
await runReadability(source, path.join(destRoot, "expected.html"), path.join(destRoot, "expected-metadata.json"));
});
}
function runReadability(source, destPath, metadataDestPath) {
async function runReadability(source, destPath, metadataDestPath) {
var uri = "http://fakehost/test/page.html";
var myReader, result, readerable;
try {
@ -215,7 +215,7 @@ function runReadability(source, destPath, metadataDestPath) {
// We pass `caption` as a class to check that passing in extra classes works,
// given that it appears in some of the test documents.
myReader = new Readability(jsdom, { classesToPreserve: ["caption"], url: uri });
result = myReader.parse();
result = await myReader.parse();
} catch (ex) {
console.error(ex);
ex.stack.forEach(console.log.bind(console));
@ -225,7 +225,7 @@ function runReadability(source, destPath, metadataDestPath) {
return;
}
fs.writeFile(destPath, prettyPrint(result.content), function (fileWriteErr) {
fs.writeFile(destPath, prettyPrint(result.content), function(fileWriteErr) {
if (fileWriteErr) {
console.error("Couldn't write data to expected.html!");
console.error(fileWriteErr);
@ -240,7 +240,7 @@ function runReadability(source, destPath, metadataDestPath) {
// Add isProbablyReaderable result
result.readerable = readerable;
fs.writeFile(metadataDestPath, JSON.stringify(result, null, 2) + "\n", function (metadataWriteErr) {
fs.writeFile(metadataDestPath, JSON.stringify(result, null, 2) + "\n", function(metadataWriteErr) {
if (metadataWriteErr) {
console.error("Couldn't write data to expected-metadata.json!");
console.error(metadataWriteErr);

12
packages/readabilityjs/test/test-pages/electrek/expected-metadata.json Normal file
View file

@ -0,0 +1,12 @@
{
"title": "Georgia gives US solar panel manufacturing a big boost with a new factory",
"byline": "Michelle Lewis",
"dir": null,
"excerpt": "Solar-cell manufacturing giant Q Cells today announced that it's opening a new solar panel manufacturing facility in Dalton, Georgia.",
"siteName": "Electrek",
"siteIcon": "/favicon.ico",
"previewImage": "https://i0.wp.com/electrek.co/wp-content/uploads/sites/3/2022/05/georgia-solar-manufacturing.jpg?resize=1200%2C628&quality=82&strip=all&ssl=1",
"publishedDate": "2022-05-26T15:57:59.000Z",
"language": "English",
"readerable": true
}

94
packages/readabilityjs/test/test-pages/electrek/expected.html Normal file
View file

@ -0,0 +1,94 @@
<DIV class="page" id="readability-page-1">
<div id="content">
<!-- .news-feed-header -->
<article>
<!-- .elastic-container -->
<div>
<p><img src="https://electrek.co/wp-content/uploads/sites/3/2022/05/georgia-solar-manufacturing.jpg?quality=82&strip=all&w=1200" srcset="
https://i0.wp.com/electrek.co/wp-content/uploads/sites/3/2022/05/georgia-solar-manufacturing.jpg?w=320&quality=82&strip=all&ssl=1 320w,
https://i0.wp.com/electrek.co/wp-content/uploads/sites/3/2022/05/georgia-solar-manufacturing.jpg?w=640&quality=82&strip=all&ssl=1 640w,
https://i0.wp.com/electrek.co/wp-content/uploads/sites/3/2022/05/georgia-solar-manufacturing.jpg?w=1024&quality=82&strip=all&ssl=1 1024w,
https://i0.wp.com/electrek.co/wp-content/uploads/sites/3/2022/05/georgia-solar-manufacturing.jpg?w=1500&quality=82&strip=all&ssl=1 1500w,
https://i0.wp.com/electrek.co/wp-content/uploads/sites/3/2022/05/georgia-solar-manufacturing.jpg?w=2000&quality=82&strip=all&ssl=1 2000w,
https://i0.wp.com/electrek.co/wp-content/uploads/sites/3/2022/05/georgia-solar-manufacturing.jpg?w=2500&quality=82&strip=all&ssl=1 2500w
" width="1200" height="675" alt="Georgia solar manufacturing" loading="lazy">
</p>
<!-- .feat-image -->
<!-- .post-social-mobile -->
</div>
<!-- .feat-image-wrapper -->
<div>
<p> Seoul-headquartered PV solar-cell manufacturing giant <a href="https://www.q-cells.co.uk/" target="_blank" rel="noreferrer noopener">Q Cells</a> today announced that it’s opening a new solar panel manufacturing facility in&nbsp;Dalton, Georgia. </p>
<h2 id="h-georgia-solar-panel-manufacturing-grows-again"> Georgia solar panel manufacturing grows again </h2>
<p> It’s a $171 million expansion of Q Cells’ existing solar module manufacturing plant in Dalton, and that will create 470 additional jobs. Total local Q Cells employees will exceed 1,000 when the expansion is complete. </p>
<p> Groundbreaking is planned for fall 2022 and operation is expected to commence within the first half of 2023. </p>
<p> This latest domestic solar manufacturing expansion will boost production of advanced photovoltaic (PV) modules, and that will help the US work move toward its goal of decarbonizing the electric grid. </p>
<p> The new facility will produce 1.4 gigawatts (GW) of solar modules per year made with Q Cells’ next-gen PV cells, a high-efficiency tunnel oxide passivated contact technology better known as TOPCon. </p>
<p> Combined with the existing 1.7-GW factory, the expansion will bring Q Cells’ total capacity in the US to 3.1 GW; that’s equivalent to one-third of the country’s solar module manufacturing capacity. </p>
<p>Qcells CEO&nbsp;Justin Lee said:</p>
<blockquote>
<p> Georgia&nbsp;has become the clean energy manufacturing heart of America, and we are proud to contribute to the state’s advanced manufacturing economy. </p>
</blockquote>
<p> Q Cells has the largest market share in the US commercial and residential markets and also supplies the utility-scale solar sector. </p>
<p> Senator Jon Ossoff (D-GA)&nbsp;met with Q Cells’ parent company Hanwha&nbsp;in Seoul last year and has been actively pitching and securing additional clean energy investment in Georgia. </p>
<figure>
<div>
<div data-tweet-id="1529811248258371590" class="tweet-placeholder"></div>
</div>
</figure>
<p> Ossoff also recently&nbsp;helped secure&nbsp;<a href="https://electrek.co/2022/05/20/hyundai-motor-group-announces-its-first-dedicated-ev-facilities-coming-to-the-us/" target="_blank" rel="noreferrer noopener">Hyundai’s investment in electric vehicles</a> that will create over 8,000 jobs in Bryan County. </p>
<h2 id="h-washington-georgia-embraces-solar"> Washington, Georgia, embraces solar </h2>
<p> Also in Georgia, <a href="https://wesolarcsp.com/" target="_blank" rel="noreferrer noopener">WeSolar CSP</a>, a minority-owned renewable energy tech and design company headquartered in Princeton, New Jersey, will design a solar farm along with a microgrid that will supply the City of Washington, Georgia, that will replace natural gas use. Washington is 90 miles east of Atlanta and has a population of around 4,000. </p>
<p> The project will comprise both solar panels and a concentrating solar-thermal power (CSP) technology. </p>
<p>WeSolar CSP’s CEO, Steve Anglin, said:</p>
<blockquote>
<p> The citizens of the City of Washington will benefit by having a cleaner environment and experiencing price certainty in the face of the ever-increasing energy costs of fossil fuels. </p>
</blockquote>
<p>
<strong>Read more:</strong>
<a href="https://electrek.co/2022/05/04/here-are-3-vital-insights-installers-shared-about-the-state-of-solar-in-2021/">Here are 3 vital insights installers shared about the state of solar in 2021</a>
</p>
<p><em>Photo: Hanwha Q Cells</em></p>
<hr>
<p>
<em>UnderstandSolar is a free service that links you to top-rated solar installers in your region for personalized solar estimates. Tesla now offers price matching, so it’s important to shop for the best quotes.&nbsp;<a href="https://understandsolarenergy.com/form/?lsid=511&s1=%7Bs1%7D#step1" target="_blank" rel="noreferrer noopener">Click here to learn more and get your quotes</a>. — *ad</em>.
</p>
<p>
<em>FTC: We use income earning auto affiliate links.</em>
<a href="https://electrek.co/about/#affiliate">More.</a>
</p>
<hr>
<p>
<a href="https://www.youtube.com/channel/UCcOIZzJgLCyMPILY7-1Vsdg?sub_confirmation=1">Subscribe to Electrek on YouTube for exclusive videos</a> and subscribe to the <a href="https://www.electrek.co/guides/electrek-podcast">podcast</a>.
<!-- youtube embed -->
</p>
<P>
<iframe title="Recent Videos" src="https://www.youtube.com/embed/wh5M-xjzus4?playlist=mvDHs0LCTf8,N4uJiZmcYbc,4SBwRCHeYpg,dkbUPBzFo90,9Euynq65qus,pV0flQZQhgM,zMQGcVZmqao,R8XFbNWXp34,FRTm8SPoDO8" frameborder="0" allow="accelerometer; autoplay; encrypted-media; gyroscope; picture-in-picture" allowfullscreen width="1000" height="563"></iframe>
</P>
</div>
<!-- .elastic-container -->
<div>
<h2>About the Author</h2>
<div>
<p><a href="https://electrek.co/author/michellelewis/">
<img src="https://secure.gravatar.com/avatar/b2390be790ce625b95e69e27b1a32fe1?s=128&d=identicon&r=g" loading="lazy">
</a></p>
<h3>
<a href="https://electrek.co/author/michellelewis/">
<span>Michelle Lewis</span>
</a>
</h3>
<p>
<a href="https://twitter.com/intent/follow?screen_name=michelle0728&original_referer=https%3A%2F%2Felectrek.co" target="_blank"><span></span>@michelle0728</a>
</p>
<p> Michelle Lewis is a writer and editor on Electrek and an editor on DroneDJ, 9to5Mac, and 9to5Google. She lives in White River Junction, Vermont. She has previously worked for Fast Company, the Guardian, News Deeply, Time, and others. Message Michelle on Twitter or at michelle@9to5mac.com. Check out her personal blog. </p>
</div>
<div>
<h3><span>Michelle Lewis's favorite gear</span></h3>
</div>
</div>
</article>
<!-- .post-content -->
<!-- #comments.comments-area -->
</div>
</DIV>

4880
packages/readabilityjs/test/test-pages/electrek/source.html Normal file
View file

File diff suppressed because it is too large Load diff

1
packages/readabilityjs/test/test-pages/electrek/url.txt Normal file
View file

@ -0,0 +1 @@
https://electrek.co/2022/05/26/georgia-solar-panel-manufacturing/

10
packages/readabilityjs/test/test-pages/substack-michaelshellenberger/expected-metadata.json Normal file
View file

@ -0,0 +1,10 @@
{
"title": "The 2-minute 20-second video that changes everything",
"byline": "Michael Shellenberger",
"dir": null,
"excerpt": "For decades, people have claimed that homelessness is just a\n housing problem. Sure, many also have substance use and mental\n illness issues. But if we just give homeless people their own\n own studio apartments, and decriminalize public camping,\n drugs, and shoplifting, the problem will go away, many\n claimed.",
"siteName": null,
"publishedDate": "2001-05-25T16:00:00.000Z",
"language": "English",
"readerable": true
}

39
packages/readabilityjs/test/test-pages/substack-michaelshellenberger/expected.html Normal file
View file

@ -0,0 +1,39 @@
<DIV class="page" id="readability-page-1">
<div>
<div dir="auto">
<div>
<figure>
<table>
<tbody>
<tr>
<td></td>
<td>
<a target="_blank" href="https://email.mg2.substack.com/c/eJw1UsmO5CAM_ZriloiwBHLg0Gpp7vMFEYtJUCeQAVLdNV8_pKKRwDbGy5Ofra6wpPxSRyoVXWKurwNUhO-yQa2Q0Vkgz8EpQSkRTBDkFHOD5BKFMvsMsOuwqZpPQMdptmB1DSleGURMDGO0KgmWTVIOgzXES_ByEsy1q7F1cvT-bqxPFyBaUPCE_EoRkE37DrFetdCm1lqP8qAfD_KrnXKaUrX9si72La55wq4XaNpDteulW8WaHuTzz23QjyUl195-m4-clgylhCc0f6kAR8u4OxB-9eBXlybMab-g_ecOMDfGStYZrWnHqIBu4oPstDOUGBBikqwvtNe7_pui_i43rlbjnsvbfIMsbxMPlnIBrZ4kU8dG1ixMWMemgXg7DVZbP3PMfojk_REXFBTBhGBOxqYxJj3tnRDWYelhdBJ7zR8M7wvp_w_nQoCyWlNcTOpESuF3YR20qBBN-ukc7KlPewzPlKHXx3FROF9jP2OorxmiNhu4m916L8mb73mBCLktj5t1VcPIKZ9GNlDC2U1mo4yP48TpIFBD41LLimoPdtWwlRW2DaKBvED-B1rd02I" rel=""><img data-attrs="{&quot;src&quot;:&quot;https://bucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com/public/images/01c357e4-b829-4644-b024-4912fc91cacf_504x285.png&quot;,&quot;fullscreen&quot;:null,&quot;imageSize&quot;:null,&quot;height&quot;:285,&quot;width&quot;:504,&quot;resizeWidth&quot;:null,&quot;bytes&quot;:275557,&quot;alt&quot;:null,&quot;title&quot;:null,&quot;type&quot;:&quot;image/png&quot;,&quot;href&quot;:null}" alt="" width="504" height="285" src="https://substackcdn.com/image/fetch/w_504,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2F01c357e4-b829-4644-b024-4912fc91cacf_504x285.png"></a>
</td>
<td></td>
</tr>
</tbody>
</table>
</figure>
</div>
<p> For decades, people have claimed that homelessness is just a housing problem. Sure, many also have substance use and mental illness issues. But if we just give homeless people their own own studio apartments, and decriminalize public camping, drugs, and shoplifting, the problem will go away, many claimed. </p>
<p> That hasn’t happened. Instead, the open drug scenes have worsened. Nationally, drug overdoses and poisonings increased from 17,000 in 2000 to 108,000 in 2021. And California, which pioneered the “Housing First”/decriminalization approach saw its homeless population increase 31% between 2011 and 2020, even as homelessness declined 18% in the rest of the country. </p>
<p> I debunked the lies about homelessness in <em>San Fransicko</em>, in hundreds of articles, and on dozens of TV and podcast appearances. But when it comes to educating the public, nothing has been more impactful than the video interviews of homeless people that I’ve conducted over the last few months with my friend Leighton Woodhouse, a documentary filmmaker, as part of my run to become governor of California. </p>
<p> Now Leighton has assembled those interviews into a two-minute 20 second video we’ve posted on Twitter. It’s a must-watch. It’s only been on-line for a few hours, and over 130,000 people have seen it. </p>
<div data-tweet-id="1529847068138778624" class="tweet-placeholder"></div>
<p> I hope you’ll take a minute to watch it. And, after you do, please consider a donation to Shellenberger for Governor. </p>
<p> There’s just 12 days before the primary election. Anyone can vote for anyone. And anyone in the US can donate. Whatever happens, we will make history. </p>
<p data-attrs="{&quot;url&quot;:&quot;https://secure.shellenbergerforgovernor.com/list/proc/donation1/?InitiativeKey=KHNHN40ZS8HV&utm_source=proc&utm_medium=web&utm_campaign=donate&quot;,&quot;text&quot;:&quot;Donate to Shellenberger 2022&quot;,&quot;action&quot;:null,&quot;class&quot;:null}">
<a href="https://email.mg2.substack.com/c/eJxVUU2L3DAM_TXxbYJjJ3Hm4EOhLFMWFkphD3sJjq1JTGMr-GO2-fd1JntZMBY86UlPT1olmDHscsOYyPGNad9AeviMK6QEgeQIYbRGCs6ZaAUjRramGbqB2DjeA4BTdpUpZCBbnlarVbLoDwYT15ZSskgt2sE0XDdDPykt9KCmhl81VUbRXg_6HKyyseA1SHhA2NED0egc-HT0IqtcUtpixX9U7KW8CDoHqOMC6wp-gjBDuGOYsZA9hrpwS9VqYyphC6hLMOif4pqjBX_55W2yBXjAK-wV__l6e7u9tfTjz3B7r1ifkxsj5qCh5M4OT8yBsdkV7BOmL0grtyk7-wI-ZwCxklHGaMf6EillNa-NENrQ4Q69GehddVVL3czqmKeYlP57KCZBLujnCS8C0f6O7QVKlfUT_rsYcFij8_aBZW-1bYfd42FRLnvsI3g1rWDOS6TzoM_bjDN4CEWUGVWSTd_x7tq3DWddexpf7O36_trxRpCixmBheemsXhSs3wz-D4wTwDA" rel=""><span>Donate to Shellenberger 2022</span></a>
</p>
</div>
<div>
<p>
<span>You’re a free subscriber to <a href="https://email.mg2.substack.com/c/eJxtkL1uxSAMhZ8mbDfin9yBoUv3PkFEwElQCURAbpu3L2mmSpUse_Gxz_msqbCkfOo9lYquNtZzBx3hqwSoFTI6CuTRO60Yo4oripzmjgxiQL6McwbYjA-65gPQfkzBW1N9ipeCqifHGK0acyFBEGLnmZFZSImZNFw5JYeBcIrvx-ZwHqIFDS_IZ4qAbNo2iPW6hYJea91Lx946-t5q83Y1EMoKIUCcIC-Q-3JMpRr72Tdl20FeU0wpFlS2iTHtWe-Usg4PM0g34NmIjuNtoX-UKOs1xWVKD5WS_yj8AW3Lxyl9PxxsqU9b9K-UoTf7foUeL6NH9PUcIZopgLt51BvrL6FxgQi54XajqZpIwcRTcsKo4Hf8FrKReQpGFGpuXGqqqP-L-QNr9ZSs">Michael Shellenberger</a>. For the full experience, <a href="https://email.mg2.substack.com/c/eJxtUsuO2jAU_RqyI3LsvFhkQcnQgRlCkei8NpFjXxJDYkeOA4Svr1PoolUly5aP7j33cQ6jBkqlh6RVnXHGKzdDC4mES1eDMaCdvgOdC55EhODIj7DDE597cRA7ossPGqChok6M7sFp-6IWjBqh5JiBo5mPkFMlBMMsPlCf05jEJKKRz2PLVASIHgIaRvfCtOcCJIMEzqAHJcFhqmlAmpHLqZPKmLabkPkEL-1pBKso1F0FdQ2yAF2Cdru-6AxlJ9dm2pjxy7QoYEKWRp1ATkgKw9pj-G34wPVpdVRkc9sM2W1HNuIi6PsSWeyaHX_esv0JZ-nT8LpY1_A8F9vjE872n9dtWqLNftWtmvrGhlW4kt-8T4Esxwlt0rmw8Tf-vhJbsT7S729HimceX6xnrpDPL8sD-cr0JQvWZ1OFu3S5bvPo3E5_VC_zj9ehWGz0V5ntJjjsTZN3qtfM9p6O6_kX-zPqA2-Ai74Z5xvleIBMSWP392BghjoiwQhjFODQvghhl7g8ihhH8QFCHqMDDSY-akr81yodnVRKloWaRkqJXedPwUYJWajrlEOjXNVIcVYaXNq2owvyUbleCjPkIGlRA78bxNx99rvHvAQJ2vqP59QkXhiQYBb6HsGBf_eDVT0Iw1lAvMix3XBls2TyP91_AeYL9AI">become a paid subscriber.</a></span>
</p>
<p>
<a role="button" href="https://email.mg2.substack.com/c/eJxtUsuO2jAU_RqyI3LsvFhkQcnQgRlCkei8NpFjXxJDYkeOA4Svr1PoolUly5aP7j33cQ6jBkqlh6RVnXHGKzdDC4mES1eDMaCdvgOdC55EhODIj7DDE597cRA7ossPGqChok6M7sFp-6IWjBqh5JiBo5mPkFMlBMMsPlCf05jEJKKRz2PLVASIHgIaRvfCtOcCJIMEzqAHJcFhqmlAmpHLqZPKmLabkPkEL-1pBKso1F0FdQ2yAF2Cdru-6AxlJ9dm2pjxy7QoYEKWRp1ATkgKw9pj-G34wPVpdVRkc9sM2W1HNuIi6PsSWeyaHX_esv0JZ-nT8LpY1_A8F9vjE872n9dtWqLNftWtmvrGhlW4kt-8T4Esxwlt0rmw8Tf-vhJbsT7S729HimceX6xnrpDPL8sD-cr0JQvWZ1OFu3S5bvPo3E5_VC_zj9ehWGz0V5ntJjjsTZN3qtfM9p6O6_kX-zPqA2-Ai74Z5xvleIBMSWP392BghjoiwQhjFODQvghhl7g8ihhH8QFCHqMDDSY-akr81yodnVRKloWaRkqJXedPwUYJWajrlEOjXNVIcVYaXNq2owvyUbleCjPkIGlRA78bxNx99rvHvAQJ2vqP59QkXhiQYBb6HsGBf_eDVT0Iw1lAvMix3XBls2TyP91_AeYL9AI">Subscribe</a>
</p>
</div>
</div>
</DIV>

2000
packages/readabilityjs/test/test-pages/substack-michaelshellenberger/source.html Normal file
View file

File diff suppressed because it is too large Load diff

1
packages/readabilityjs/test/test-pages/substack-michaelshellenberger/url.txt Normal file
View file

@ -0,0 +1 @@
https://michaelshellenberger.substack.com/p/the-2-minute-20-second-video-that?s=r

64
packages/readabilityjs/test/test-readability.js
View file

@ -1,7 +1,9 @@
var chai = require("chai");
var sinon = require("sinon");
var chaiAsPromised = require("chai-as-promised");
const { parseHTML } = require("linkedom");
chai.use(chaiAsPromised);
chai.config.includeStack = true;
var expect = chai.expect;
@ -59,13 +61,13 @@ function runTestsWithItems(label, domGenerationFn, source, expectedContent, expe
var result;
before(function() {
before(async function() {
try {
var doc = domGenerationFn(source);
// Provide one class name to preserve, which we know appears in a few
// of the test documents.
var myReader = new Readability(doc, { classesToPreserve: ["caption"], url: uri });
result = myReader.parse();
result = await myReader.parse();
} catch (err) {
throw reformatError(err);
}
@ -220,70 +222,70 @@ describe("Readability API", function() {
describe("#parse", function() {
var exampleSource = testPages[0].source;
it("shouldn't parse oversized documents as per configuration", function() {
it("shouldn't parse oversized documents as per configuration", async function() {
var doc = new JSDOMParser().parse("<html><div>yo</div></html>");
expect(function() {
new Readability(doc, {maxElemsToParse: 1}).parse();
}).to.Throw("Aborting parsing document; 2 elements found");
await expect(
(new Readability(doc, { maxElemsToParse: 1 })).parse()
).to.be.rejectedWith("Aborting parsing document; 2 elements found");
});
it("should run _cleanClasses with default configuration", function() {
it("should run _cleanClasses with default configuration", async function() {
var doc = parseHTML(exampleSource).document;
var parser = new Readability(doc);
parser._cleanClasses = sinon.fake();
parser.parse();
await parser.parse();
expect(parser._cleanClasses.called).eql(true);
});
it("should run _cleanClasses when option keepClasses = false", function() {
it("should run _cleanClasses when option keepClasses = false", async function() {
var doc = parseHTML(exampleSource).document;
var parser = new Readability(doc, {keepClasses: false});
var parser = new Readability(doc, { keepClasses: false });
parser._cleanClasses = sinon.fake();
parser.parse();
await parser.parse();
expect(parser._cleanClasses.called).eql(true);
});
it("shouldn't run _cleanClasses when option keepClasses = true", function() {
it("shouldn't run _cleanClasses when option keepClasses = true", async function() {
var doc = parseHTML(exampleSource).document;
var parser = new Readability(doc, {keepClasses: true});
var parser = new Readability(doc, { keepClasses: true });
parser._cleanClasses = sinon.fake();
parser.parse();
await parser.parse();
expect(parser._cleanClasses.called).eql(false);
});
xit("should use custom content serializer sent as option", function() {
var dom = new JSDOM("My cat: <img src=''>");
xit("should use custom content serializer sent as option", async function() {
var dom = parseHTML("<html><body>My cat: <img src=''></body></html>");
var expected_xhtml = "<div xmlns=\"http://www.w3.org/1999/xhtml\" id=\"readability-page-1\" class=\"page\">My cat: <img src=\"\" /></div>";
var xml = new dom.window.XMLSerializer();
var content = new Readability(dom.window.document, {
var content = await (new Readability(dom.window.document, {
serializer: function(el) {
return xml.serializeToString(el.firstChild);
}
}).parse().content;
})).parse().content;
expect(content).eql(expected_xhtml);
});
it("should not proxy image with data uri", function() {
it("should not proxy image with data uri", async function() {
var dom = parseHTML("<html><body>My cat: <img src=\"data:image/png;base64, iVBORw0KGgoAAAANSUhEUgAAAAUA" +
"AAAFCAYAAACNbyblAAAAHElEQVQI12P4//8/w38GIAXDIBKE0DHxgljNBAAO9TXL0Y4OHwAAAABJRU5ErkJggg==\"" +
" alt=\"Red dot\" /></body></html>");
var expected_xhtml = "<DIV class=\"page\" id=\"readability-page-1\">My cat: <img src=\"data:image/png;base64," +
" iVBORw0KGgoAAAANSUhEUgAAAAUAAAAFCAYAAACNbyblAAAAHElEQVQI12P4//8/w38GIAXDIBKE0DHxgljNBAAO9TXL0" +
"Y4OHwAAAABJRU5ErkJggg==\" alt=\"Red dot\"></DIV>";
var content = new Readability(dom.document).parse().content;
var content = (await (new Readability(dom.document)).parse()).content;
expect(content).eql(expected_xhtml);
});
it("should handle srcset elements with density descriptors", function() {
it("should handle srcset elements with density descriptors", async function() {
var dom = parseHTML('<html><body>My image: <img src="https://webkit.org/demos/srcset/image-src.png" ' +
'srcset="https://webkit.org/demos/srcset/image-1x.png 1x, ' +
'https://webkit.org/demos/srcset/image-2x.png 2x, ' +
@ -291,29 +293,29 @@ describe("Readability API", function() {
'https://webkit.org/demos/srcset/image-4x.png 4x">' +
'</body></html>');
var expected_xhtml = '<DIV class="page" id="readability-page-1">My image: ' +
'<img src="https://webkit.org/demos/srcset/image-src.png" ' +
'srcset="https://webkit.org/demos/srcset/image-1x.png 1x,' +
'https://webkit.org/demos/srcset/image-2x.png 2x,' +
'https://webkit.org/demos/srcset/image-3x.png 3x,' +
'https://webkit.org/demos/srcset/image-4x.png 4x,"></DIV>';
var content = new Readability(dom.document, {
'<img src="https://webkit.org/demos/srcset/image-src.png" ' +
'srcset="https://webkit.org/demos/srcset/image-1x.png 1x,' +
'https://webkit.org/demos/srcset/image-2x.png 2x,' +
'https://webkit.org/demos/srcset/image-3x.png 3x,' +
'https://webkit.org/demos/srcset/image-4x.png 4x,"></DIV>';
var content = (await (new Readability(dom.document, {
createImageProxyUrl: function(url) {
return url;
}
}).parse().content;
})).parse()).content;
expect(content).eql(expected_xhtml);
});
it("should remove srcset elements that are lazy loading placeholders", function() {
it("should remove srcset elements that are lazy loading placeholders", async function() {
var dom = parseHTML('<html><body>My image: <img class="shrinkToFit jetpack-lazy-image" src="https://i0.wp.com/cdn-images-1.medium.com/max/2000/1*rPXwIczUJRCE54v8FfAHGw.jpeg?resize=900%2C380&#038;ssl=1" alt width="900" height="380" data-recalc-dims="1" data-lazy-src="https://i0.wp.com/cdn-images-1.medium.com/max/2000/1*rPXwIczUJRCE54v8FfAHGw.jpeg?resize=900%2C380&amp;is-pending-load=1#038;ssl=1" srcset="data:image/gif;base64,R0lGODlhAQABAIAAAAAAAP///yH5BAEAAAAALAAAAAABAAEAAAIBRAA7"></body></html>');
var expected_xhtml = '<DIV class="page" id="readability-page-1">' +
'My image: <img src="https://i0.wp.com/cdn-images-1.medium.com/max/2000/1*rPXwIczUJRCE54v8FfAHGw.jpeg?resize=900%2C380&is-pending-load=1#038;ssl=1" alt="" width="900" height="380" data-recalc-dims="1" data-lazy-src="https://i0.wp.com/cdn-images-1.medium.com/max/2000/1*rPXwIczUJRCE54v8FfAHGw.jpeg?resize=900%2C380&is-pending-load=1#038;ssl=1">' +
'</DIV>';
var content = new Readability(dom.document, {
var content = (await (new Readability(dom.document, {
createImageProxyUrl: function(url) {
return url;
}
}).parse().content;
})).parse()).content;
expect(content).eql(expected_xhtml);
});
});

36
yarn.lock
View file

@ -7382,6 +7382,13 @@
resolved "https://registry.yarnpkg.com/@types/caseless/-/caseless-0.12.2.tgz#f65d3d6389e01eeb458bd54dc8f52b95a9463bc8"
integrity sha512-6ckxMjBBD8URvjB6J3NcnuAn5Pkl7t3TizAg+xdlzzQGSPSmBcXf8KoIH0ua/i+tio+ZRUHEXp0HEmvaR4kt0w==
"@types/chai-as-promised@^7.1.5":
version "7.1.5"
resolved "https://registry.yarnpkg.com/@types/chai-as-promised/-/chai-as-promised-7.1.5.tgz#6e016811f6c7a64f2eed823191c3a6955094e255"
integrity sha512-jStwss93SITGBwt/niYrkf2C+/1KTeZCZl1LaeezTlqppAKeoQC7jxyqYuP72sxBGKCIbw7oHgbYssIRzT5FCQ==
dependencies:
"@types/chai" "*"
"@types/chai-string@^1.4.2":
version "1.4.2"
resolved "https://registry.yarnpkg.com/@types/chai-string/-/chai-string-1.4.2.tgz#0f116504a666b6c6a3c42becf86634316c9a19ac"
@ -10321,6 +10328,13 @@ ccount@^1.0.0:
resolved "https://registry.yarnpkg.com/ccount/-/ccount-1.1.0.tgz#246687debb6014735131be8abab2d93898f8d043"
integrity sha512-vlNK021QdI7PNeiUh/lKkC/mNHHfV0m/Ad5JoI0TYtlBnJAslM/JIkm/tGC88bkLIwO6OQ5uV6ztS6kVAtCDlg==
chai-as-promised@^7.1.1:
version "7.1.1"
resolved "https://registry.yarnpkg.com/chai-as-promised/-/chai-as-promised-7.1.1.tgz#08645d825deb8696ee61725dbf590c012eb00ca0"
integrity sha512-azL6xMoi+uxu6z4rhWQ1jbdUhOMhis2PvscD/xjLqNMkv3BPPp2JyyuTHOrf9BOosGpNQ11v6BKv/g57RXbiaA==
dependencies:
check-error "^1.0.2"
chai-string@^1.5.0:
version "1.5.0"
resolved "https://registry.yarnpkg.com/chai-string/-/chai-string-1.5.0.tgz#0bdb2d8a5f1dbe90bc78ec493c1c1c180dd4d3d2"
@ -17133,7 +17147,7 @@ lambdafs@^2.0.3:
resolved "https://registry.yarnpkg.com/lambdafs/-/lambdafs-2.1.1.tgz#4bf8d3037b6c61bbb4a22ab05c73ee47964c25ed"
integrity sha512-x5k8JcoJWkWLvCVBzrl4pzvkEHSgSBqFjg3Dpsc4AcTMq7oUMym4cL/gRTZ6VM4mUMY+M0dIbQ+V1c1tsqqanQ==
dependencies:
tar-fs "*"
tar-fs "^2.1.1"
language-subtag-registry@~0.3.2:
version "0.3.21"
@ -23139,16 +23153,6 @@ tapable@^2.0.0, tapable@^2.1.1, tapable@^2.2.0:
resolved "https://registry.yarnpkg.com/tapable/-/tapable-2.2.1.tgz#1967a73ef4060a82f12ab96af86d52fdb76eeca0"
integrity sha512-GNzQvQTOIP6RyTfE2Qxb8ZVlNmw0n88vp1szwWRimP02mnTsx3Wtn5qRdqY9w2XduFNUgvOwhNnQsjwCp+kqaQ==
tar-fs@*, tar-fs@2.1.1:
version "2.1.1"
resolved "https://registry.yarnpkg.com/tar-fs/-/tar-fs-2.1.1.tgz#489a15ab85f1f0befabb370b7de4f9eb5cbe8784"
integrity sha512-V0r2Y9scmbDRLCNex/+hYzvp/zyYjvFbHPNgVTKfQvVrb6guiE/fxP+XblDNR011utopbkex2nM4dHNV6GDsng==
dependencies:
chownr "^1.1.1"
mkdirp-classic "^0.5.2"
pump "^3.0.0"
tar-stream "^2.1.4"
tar-fs@2.0.0:
version "2.0.0"
resolved "https://registry.yarnpkg.com/tar-fs/-/tar-fs-2.0.0.tgz#677700fc0c8b337a78bee3623fdc235f21d7afad"
@ -23159,6 +23163,16 @@ tar-fs@2.0.0:
pump "^3.0.0"
tar-stream "^2.0.0"
tar-fs@2.1.1, tar-fs@^2.1.1:
version "2.1.1"
resolved "https://registry.yarnpkg.com/tar-fs/-/tar-fs-2.1.1.tgz#489a15ab85f1f0befabb370b7de4f9eb5cbe8784"
integrity sha512-V0r2Y9scmbDRLCNex/+hYzvp/zyYjvFbHPNgVTKfQvVrb6guiE/fxP+XblDNR011utopbkex2nM4dHNV6GDsng==
dependencies:
chownr "^1.1.1"
mkdirp-classic "^0.5.2"
pump "^3.0.0"
tar-stream "^2.1.4"
tar-stream@^2.0.0, tar-stream@^2.1.4:
version "2.2.0"
resolved "https://registry.yarnpkg.com/tar-stream/-/tar-stream-2.2.0.tgz#acad84c284136b060dc3faa64474aa9aebd77287"