diff --git a/packages/api/src/readability.d.ts b/packages/api/src/readability.d.ts index 2587b114e..15eaab5a9 100644 --- a/packages/api/src/readability.d.ts +++ b/packages/api/src/readability.d.ts @@ -157,9 +157,9 @@ declare module '@omnivore/readability' { /** Article description, or short excerpt from the content */ excerpt: string /** Article site name */ - siteName: string + siteName?: string /** Article site icon */ - siteIcon: string + siteIcon?: string /** Article preview image */ previewImage?: string /** Article published date */ diff --git a/packages/readabilityjs/Readability.js b/packages/readabilityjs/Readability.js index 1c653ef36..3cc8f0641 100644 --- a/packages/readabilityjs/Readability.js +++ b/packages/readabilityjs/Readability.js @@ -23,6 +23,7 @@ var parseSrcset = require('parse-srcset'); var htmlEntities = require('html-entities') const axios = require("axios"); +const { parseHTML } = require("linkedom"); /** Checks whether an element is a wrapper for tweet */ const hasTweetInChildren = element => { @@ -1790,6 +1791,24 @@ Readability.prototype = { return {}; }, + _getFaviconFromDoc: function (doc) { + const favicon = doc.querySelector( + "link[rel='apple-touch-icon'], link[rel='shortcut icon'], link[rel='icon']" + ); + return favicon?.getAttribute('href'); + }, + + _getFaviconFromURL: async function (url) { + try { + const response = await axios.get(url); + const doc = parseHTML(response.data).document; + return this._getFaviconFromDoc(doc); + } catch (e) { + console.log('error parsing favicon url', e); + return undefined; + } + }, + /** * Attempts to get excerpt and byline metadata for the article. * @@ -1798,7 +1817,7 @@ Readability.prototype = { * * @return Object with optional "excerpt" and "byline" properties */ - _getArticleMetadata: function (jsonld) { + _getArticleMetadata: async function (jsonld) { var metadata = {}; var values = {}; var metaElements = this._doc.getElementsByTagName("meta"); @@ -1907,10 +1926,12 @@ Readability.prototype = { values["og:site_name"] || null; // get website icon - const iconLink = this._doc.querySelector( - "link[rel='apple-touch-icon'], link[rel='shortcut icon'], link[rel='icon']" - ); - metadata.siteIcon = iconLink?.href; + metadata.siteIcon = this._getFaviconFromDoc(this._doc); + if (!metadata.siteIcon && this._keepTables) { + // If we're keeping tables, we are likely to parsing a newsletter, so + // we should try to get the site icon from the URL + metadata.siteIcon = await this._getFaviconFromURL(this._baseURI); + } // get published date metadata.publishedDate = jsonld.publishedDate || @@ -2905,7 +2926,7 @@ Readability.prototype = { this._prepDocument(); - var metadata = this._getArticleMetadata(jsonLd); + var metadata = await this._getArticleMetadata(jsonLd); this._articleTitle = metadata.title; var articleContent = await this._grabArticle(); diff --git a/packages/readabilityjs/package.json b/packages/readabilityjs/package.json index 7e248d421..f3fedeb62 100644 --- a/packages/readabilityjs/package.json +++ b/packages/readabilityjs/package.json @@ -36,6 +36,7 @@ }, "dependencies": { "html-entities": "^2.3.2", + "linkedom": "^0.14.9", "modern-random-ua": "^1.0.3", "parse-srcset": "^1.0.2" }