prehistoric-systems/omnivore
1
0
Fork 0
mirror of https://github.com/omnivore-app/omnivore.git synced 2026-03-11 08:54:26 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Make async calls to parse()

Browse source
This commit is contained in:
Hongbo Wu 2022-05-31 22:51:00 +08:00
parent 0b0edd3e69
commit 404805e0c0
4 changed files with 42 additions and 42 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
packages/api/src/readability.d.ts vendored
View file

@ -72,7 +72,7 @@ declare module '@omnivore/readability' {
*
* The response will be null if the processing failed (https://github.com/mozilla/readability/blob/52ab9b5c8916c306a47b2119270dcdabebf9d203/Readability.js#L2038)
*/
parse(): Readability.ParseResult | null
async parse(): Promise<Readability.ParseResult | null>
}
namespace Readability {

12
packages/api/src/utils/parser.ts
View file

@ -133,12 +133,12 @@ const getPurifiedContent = (html: string): Document => {
return parseHTML(clean).document
}
const getReadabilityResult = (
const getReadabilityResult = async (
url: string,
html: string,
document: Document,
isNewsletter?: boolean
): Readability.ParseResult | null => {
): Promise<Readability.ParseResult | null> => {
// First attempt to read the article as is.
// if that fails attempt to purify then read
const sources = [
@ -157,7 +157,7 @@ const getReadabilityResult = (
}
try {
const article = new Readability(document, {
const article = await new Readability(document, {
debug: DEBUG_MODE,
createImageProxyUrl,
keepTables: isNewsletter,
@ -236,7 +236,7 @@ export const parsePreparedContent = async (
await applyHandlers(url, dom)
try {
article = getReadabilityResult(url, document, dom, isNewsletter)
article = await getReadabilityResult(url, document, dom, isNewsletter)
if (!article?.textContent && allowRetry) {
const newDocument = {
...preparedDocument,
@ -406,10 +406,10 @@ export const parseUrlMetadata = async (
// based on it's contents.
// TODO: when we consolidate the handlers we could include this
// as a utility method on each one.
export const isProbablyNewsletter = (html: string): boolean => {
export const isProbablyNewsletter = async (html: string): Promise<boolean> => {
const dom = parseHTML(html).document
const domCopy = parseHTML(dom.documentElement.outerHTML).document
const article = new Readability(domCopy, {
const article = await new Readability(domCopy, {
debug: false,
keepTables: true,
}).parse()

12
packages/readabilityjs/test/generate-testcase.js
View file

@ -192,7 +192,7 @@ function onResponseReceived(error, source, destRoot) {
console.log("writing");
}
var sourcePath = path.join(destRoot, "source.html");
fs.writeFile(sourcePath, source, function (err) {
fs.writeFile(sourcePath, source, async function(err) {
if (err) {
console.error("Couldn't write data to source.html!");
console.error(err);
@ -201,11 +201,11 @@ function onResponseReceived(error, source, destRoot) {
if (debug) {
console.log("Running readability stuff");
}
runReadability(source, path.join(destRoot, "expected.html"), path.join(destRoot, "expected-metadata.json"));
await runReadability(source, path.join(destRoot, "expected.html"), path.join(destRoot, "expected-metadata.json"));
});
}
function runReadability(source, destPath, metadataDestPath) {
async function runReadability(source, destPath, metadataDestPath) {
var uri = "http://fakehost/test/page.html";
var myReader, result, readerable;
try {
@ -215,7 +215,7 @@ function runReadability(source, destPath, metadataDestPath) {
// We pass `caption` as a class to check that passing in extra classes works,
// given that it appears in some of the test documents.
myReader = new Readability(jsdom, { classesToPreserve: ["caption"], url: uri });
result = myReader.parse();
result = await myReader.parse();
} catch (ex) {
console.error(ex);
ex.stack.forEach(console.log.bind(console));
@ -225,7 +225,7 @@ function runReadability(source, destPath, metadataDestPath) {
return;
}
fs.writeFile(destPath, prettyPrint(result.content), function (fileWriteErr) {
fs.writeFile(destPath, prettyPrint(result.content), function(fileWriteErr) {
if (fileWriteErr) {
console.error("Couldn't write data to expected.html!");
console.error(fileWriteErr);
@ -240,7 +240,7 @@ function runReadability(source, destPath, metadataDestPath) {
// Add isProbablyReaderable result
result.readerable = readerable;
fs.writeFile(metadataDestPath, JSON.stringify(result, null, 2) + "\n", function (metadataWriteErr) {
fs.writeFile(metadataDestPath, JSON.stringify(result, null, 2) + "\n", function(metadataWriteErr) {
if (metadataWriteErr) {
console.error("Couldn't write data to expected-metadata.json!");
console.error(metadataWriteErr);

58
packages/readabilityjs/test/test-readability.js
View file

@ -59,13 +59,13 @@ function runTestsWithItems(label, domGenerationFn, source, expectedContent, expe
var result;
before(function() {
before(async function() {
try {
var doc = domGenerationFn(source);
// Provide one class name to preserve, which we know appears in a few
// of the test documents.
var myReader = new Readability(doc, { classesToPreserve: ["caption"], url: uri });
result = myReader.parse();
result = await myReader.parse();
} catch (err) {
throw reformatError(err);
}
@ -222,68 +222,68 @@ describe("Readability API", function() {
it("shouldn't parse oversized documents as per configuration", function() {
var doc = new JSDOMParser().parse("<html><div>yo</div></html>");
expect(function() {
new Readability(doc, {maxElemsToParse: 1}).parse();
expect(async function() {
await (new Readability(doc, { maxElemsToParse: 1 })).parse();
}).to.Throw("Aborting parsing document; 2 elements found");
});
it("should run _cleanClasses with default configuration", function() {
it("should run _cleanClasses with default configuration", async function() {
var doc = parseHTML(exampleSource).document;
var parser = new Readability(doc);
parser._cleanClasses = sinon.fake();
parser.parse();
await parser.parse();
expect(parser._cleanClasses.called).eql(true);
});
it("should run _cleanClasses when option keepClasses = false", function() {
it("should run _cleanClasses when option keepClasses = false", async function() {
var doc = parseHTML(exampleSource).document;
var parser = new Readability(doc, {keepClasses: false});
var parser = new Readability(doc, { keepClasses: false });
parser._cleanClasses = sinon.fake();
parser.parse();
await parser.parse();
expect(parser._cleanClasses.called).eql(true);
});
it("shouldn't run _cleanClasses when option keepClasses = true", function() {
it("shouldn't run _cleanClasses when option keepClasses = true", async function() {
var doc = parseHTML(exampleSource).document;
var parser = new Readability(doc, {keepClasses: true});
var parser = new Readability(doc, { keepClasses: true });
parser._cleanClasses = sinon.fake();
parser.parse();
await parser.parse();
expect(parser._cleanClasses.called).eql(false);
});
xit("should use custom content serializer sent as option", function() {
var dom = new JSDOM("My cat: <img src=''>");
xit("should use custom content serializer sent as option", async function() {
var dom = parseHTML("<html><body>My cat: <img src=''></body></html>");
var expected_xhtml = "<div xmlns=\"http://www.w3.org/1999/xhtml\" id=\"readability-page-1\" class=\"page\">My cat: <img src=\"\" /></div>";
var xml = new dom.window.XMLSerializer();
var content = new Readability(dom.window.document, {
var content = await (new Readability(dom.window.document, {
serializer: function(el) {
return xml.serializeToString(el.firstChild);
}
}).parse().content;
})).parse().content;
expect(content).eql(expected_xhtml);
});
it("should not proxy image with data uri", function() {
it("should not proxy image with data uri", async function() {
var dom = parseHTML("<html><body>My cat: <img src=\"data:image/png;base64, iVBORw0KGgoAAAANSUhEUgAAAAUA" +
"AAAFCAYAAACNbyblAAAAHElEQVQI12P4//8/w38GIAXDIBKE0DHxgljNBAAO9TXL0Y4OHwAAAABJRU5ErkJggg==\"" +
" alt=\"Red dot\" /></body></html>");
var expected_xhtml = "<DIV class=\"page\" id=\"readability-page-1\">My cat: <img src=\"data:image/png;base64," +
" iVBORw0KGgoAAAANSUhEUgAAAAUAAAAFCAYAAACNbyblAAAAHElEQVQI12P4//8/w38GIAXDIBKE0DHxgljNBAAO9TXL0" +
"Y4OHwAAAABJRU5ErkJggg==\" alt=\"Red dot\"></DIV>";
var content = new Readability(dom.document).parse().content;
var content = await (new Readability(dom.document)).parse().content;
expect(content).eql(expected_xhtml);
});
it("should handle srcset elements with density descriptors", function() {
it("should handle srcset elements with density descriptors", async function() {
var dom = parseHTML('<html><body>My image: <img src="https://webkit.org/demos/srcset/image-src.png" ' +
'srcset="https://webkit.org/demos/srcset/image-1x.png 1x, ' +
'https://webkit.org/demos/srcset/image-2x.png 2x, ' +
@ -291,29 +291,29 @@ describe("Readability API", function() {
'https://webkit.org/demos/srcset/image-4x.png 4x">' +
'</body></html>');
var expected_xhtml = '<DIV class="page" id="readability-page-1">My image: ' +
'<img src="https://webkit.org/demos/srcset/image-src.png" ' +
'srcset="https://webkit.org/demos/srcset/image-1x.png 1x,' +
'https://webkit.org/demos/srcset/image-2x.png 2x,' +
'https://webkit.org/demos/srcset/image-3x.png 3x,' +
'https://webkit.org/demos/srcset/image-4x.png 4x,"></DIV>';
var content = new Readability(dom.document, {
'<img src="https://webkit.org/demos/srcset/image-src.png" ' +
'srcset="https://webkit.org/demos/srcset/image-1x.png 1x,' +
'https://webkit.org/demos/srcset/image-2x.png 2x,' +
'https://webkit.org/demos/srcset/image-3x.png 3x,' +
'https://webkit.org/demos/srcset/image-4x.png 4x,"></DIV>';
var content = await (new Readability(dom.document, {
createImageProxyUrl: function(url) {
return url;
}
}).parse().content;
})).parse().content;
expect(content).eql(expected_xhtml);
});
it("should remove srcset elements that are lazy loading placeholders", function() {
it("should remove srcset elements that are lazy loading placeholders", async function() {
var dom = parseHTML('<html><body>My image: <img class="shrinkToFit jetpack-lazy-image" src="https://i0.wp.com/cdn-images-1.medium.com/max/2000/1*rPXwIczUJRCE54v8FfAHGw.jpeg?resize=900%2C380&#038;ssl=1" alt width="900" height="380" data-recalc-dims="1" data-lazy-src="https://i0.wp.com/cdn-images-1.medium.com/max/2000/1*rPXwIczUJRCE54v8FfAHGw.jpeg?resize=900%2C380&amp;is-pending-load=1#038;ssl=1" srcset="data:image/gif;base64,R0lGODlhAQABAIAAAAAAAP///yH5BAEAAAAALAAAAAABAAEAAAIBRAA7"></body></html>');
var expected_xhtml = '<DIV class="page" id="readability-page-1">' +
'My image: <img src="https://i0.wp.com/cdn-images-1.medium.com/max/2000/1*rPXwIczUJRCE54v8FfAHGw.jpeg?resize=900%2C380&is-pending-load=1#038;ssl=1" alt="" width="900" height="380" data-recalc-dims="1" data-lazy-src="https://i0.wp.com/cdn-images-1.medium.com/max/2000/1*rPXwIczUJRCE54v8FfAHGw.jpeg?resize=900%2C380&is-pending-load=1#038;ssl=1">' +
'</DIV>';
var content = new Readability(dom.document, {
var content = await (new Readability(dom.document, {
createImageProxyUrl: function(url) {
return url;
}
}).parse().content;
})).parse().content;
expect(content).eql(expected_xhtml);
});
});