diff --git a/packages/api/src/readability.d.ts b/packages/api/src/readability.d.ts index ba8278b14..2587b114e 100644 --- a/packages/api/src/readability.d.ts +++ b/packages/api/src/readability.d.ts @@ -72,7 +72,7 @@ declare module '@omnivore/readability' { * * The response will be null if the processing failed (https://github.com/mozilla/readability/blob/52ab9b5c8916c306a47b2119270dcdabebf9d203/Readability.js#L2038) */ - parse(): Readability.ParseResult | null + async parse(): Promise } namespace Readability { diff --git a/packages/api/src/utils/parser.ts b/packages/api/src/utils/parser.ts index b397365f9..a7a08475b 100644 --- a/packages/api/src/utils/parser.ts +++ b/packages/api/src/utils/parser.ts @@ -133,12 +133,12 @@ const getPurifiedContent = (html: string): Document => { return parseHTML(clean).document } -const getReadabilityResult = ( +const getReadabilityResult = async ( url: string, html: string, document: Document, isNewsletter?: boolean -): Readability.ParseResult | null => { +): Promise => { // First attempt to read the article as is. // if that fails attempt to purify then read const sources = [ @@ -157,7 +157,7 @@ const getReadabilityResult = ( } try { - const article = new Readability(document, { + const article = await new Readability(document, { debug: DEBUG_MODE, createImageProxyUrl, keepTables: isNewsletter, @@ -236,7 +236,7 @@ export const parsePreparedContent = async ( await applyHandlers(url, dom) try { - article = getReadabilityResult(url, document, dom, isNewsletter) + article = await getReadabilityResult(url, document, dom, isNewsletter) if (!article?.textContent && allowRetry) { const newDocument = { ...preparedDocument, @@ -406,10 +406,10 @@ export const parseUrlMetadata = async ( // based on it's contents. // TODO: when we consolidate the handlers we could include this // as a utility method on each one. -export const isProbablyNewsletter = (html: string): boolean => { +export const isProbablyNewsletter = async (html: string): Promise => { const dom = parseHTML(html).document const domCopy = parseHTML(dom.documentElement.outerHTML).document - const article = new Readability(domCopy, { + const article = await new Readability(domCopy, { debug: false, keepTables: true, }).parse() diff --git a/packages/readabilityjs/test/generate-testcase.js b/packages/readabilityjs/test/generate-testcase.js index a757a1786..7658ac7da 100644 --- a/packages/readabilityjs/test/generate-testcase.js +++ b/packages/readabilityjs/test/generate-testcase.js @@ -192,7 +192,7 @@ function onResponseReceived(error, source, destRoot) { console.log("writing"); } var sourcePath = path.join(destRoot, "source.html"); - fs.writeFile(sourcePath, source, function (err) { + fs.writeFile(sourcePath, source, async function(err) { if (err) { console.error("Couldn't write data to source.html!"); console.error(err); @@ -201,11 +201,11 @@ function onResponseReceived(error, source, destRoot) { if (debug) { console.log("Running readability stuff"); } - runReadability(source, path.join(destRoot, "expected.html"), path.join(destRoot, "expected-metadata.json")); + await runReadability(source, path.join(destRoot, "expected.html"), path.join(destRoot, "expected-metadata.json")); }); } -function runReadability(source, destPath, metadataDestPath) { +async function runReadability(source, destPath, metadataDestPath) { var uri = "http://fakehost/test/page.html"; var myReader, result, readerable; try { @@ -215,7 +215,7 @@ function runReadability(source, destPath, metadataDestPath) { // We pass `caption` as a class to check that passing in extra classes works, // given that it appears in some of the test documents. myReader = new Readability(jsdom, { classesToPreserve: ["caption"], url: uri }); - result = myReader.parse(); + result = await myReader.parse(); } catch (ex) { console.error(ex); ex.stack.forEach(console.log.bind(console)); @@ -225,7 +225,7 @@ function runReadability(source, destPath, metadataDestPath) { return; } - fs.writeFile(destPath, prettyPrint(result.content), function (fileWriteErr) { + fs.writeFile(destPath, prettyPrint(result.content), function(fileWriteErr) { if (fileWriteErr) { console.error("Couldn't write data to expected.html!"); console.error(fileWriteErr); @@ -240,7 +240,7 @@ function runReadability(source, destPath, metadataDestPath) { // Add isProbablyReaderable result result.readerable = readerable; - fs.writeFile(metadataDestPath, JSON.stringify(result, null, 2) + "\n", function (metadataWriteErr) { + fs.writeFile(metadataDestPath, JSON.stringify(result, null, 2) + "\n", function(metadataWriteErr) { if (metadataWriteErr) { console.error("Couldn't write data to expected-metadata.json!"); console.error(metadataWriteErr); diff --git a/packages/readabilityjs/test/test-readability.js b/packages/readabilityjs/test/test-readability.js index 1228db774..14be0aa25 100644 --- a/packages/readabilityjs/test/test-readability.js +++ b/packages/readabilityjs/test/test-readability.js @@ -59,13 +59,13 @@ function runTestsWithItems(label, domGenerationFn, source, expectedContent, expe var result; - before(function() { + before(async function() { try { var doc = domGenerationFn(source); // Provide one class name to preserve, which we know appears in a few // of the test documents. var myReader = new Readability(doc, { classesToPreserve: ["caption"], url: uri }); - result = myReader.parse(); + result = await myReader.parse(); } catch (err) { throw reformatError(err); } @@ -222,68 +222,68 @@ describe("Readability API", function() { it("shouldn't parse oversized documents as per configuration", function() { var doc = new JSDOMParser().parse("
yo
"); - expect(function() { - new Readability(doc, {maxElemsToParse: 1}).parse(); + expect(async function() { + await (new Readability(doc, { maxElemsToParse: 1 })).parse(); }).to.Throw("Aborting parsing document; 2 elements found"); }); - it("should run _cleanClasses with default configuration", function() { + it("should run _cleanClasses with default configuration", async function() { var doc = parseHTML(exampleSource).document; var parser = new Readability(doc); parser._cleanClasses = sinon.fake(); - parser.parse(); + await parser.parse(); expect(parser._cleanClasses.called).eql(true); }); - it("should run _cleanClasses when option keepClasses = false", function() { + it("should run _cleanClasses when option keepClasses = false", async function() { var doc = parseHTML(exampleSource).document; - var parser = new Readability(doc, {keepClasses: false}); + var parser = new Readability(doc, { keepClasses: false }); parser._cleanClasses = sinon.fake(); - parser.parse(); + await parser.parse(); expect(parser._cleanClasses.called).eql(true); }); - it("shouldn't run _cleanClasses when option keepClasses = true", function() { + it("shouldn't run _cleanClasses when option keepClasses = true", async function() { var doc = parseHTML(exampleSource).document; - var parser = new Readability(doc, {keepClasses: true}); + var parser = new Readability(doc, { keepClasses: true }); parser._cleanClasses = sinon.fake(); - parser.parse(); + await parser.parse(); expect(parser._cleanClasses.called).eql(false); }); - xit("should use custom content serializer sent as option", function() { - var dom = new JSDOM("My cat: "); + xit("should use custom content serializer sent as option", async function() { + var dom = parseHTML("My cat: "); var expected_xhtml = "
My cat:
"; var xml = new dom.window.XMLSerializer(); - var content = new Readability(dom.window.document, { + var content = await (new Readability(dom.window.document, { serializer: function(el) { return xml.serializeToString(el.firstChild); } - }).parse().content; + })).parse().content; expect(content).eql(expected_xhtml); }); - it("should not proxy image with data uri", function() { + it("should not proxy image with data uri", async function() { var dom = parseHTML("My cat: \"Red"); var expected_xhtml = "
My cat: \"Red
"; - var content = new Readability(dom.document).parse().content; + var content = await (new Readability(dom.document)).parse().content; expect(content).eql(expected_xhtml); }); - it("should handle srcset elements with density descriptors", function() { + it("should handle srcset elements with density descriptors", async function() { var dom = parseHTML('My image: ' + ''); var expected_xhtml = '
My image: ' + - '
'; - var content = new Readability(dom.document, { + ''; + var content = await (new Readability(dom.document, { createImageProxyUrl: function(url) { return url; } - }).parse().content; + })).parse().content; expect(content).eql(expected_xhtml); }); - it("should remove srcset elements that are lazy loading placeholders", function() { + it("should remove srcset elements that are lazy loading placeholders", async function() { var dom = parseHTML('My image: '); var expected_xhtml = '
' + 'My image: ' + '
'; - var content = new Readability(dom.document, { + var content = await (new Readability(dom.document, { createImageProxyUrl: function(url) { return url; } - }).parse().content; + })).parse().content; expect(content).eql(expected_xhtml); }); });