diff --git a/packages/content-handler/src/websites/the-atlantic-handler.ts b/packages/content-handler/src/websites/the-atlantic-handler.ts new file mode 100644 index 000000000..63920fb59 --- /dev/null +++ b/packages/content-handler/src/websites/the-atlantic-handler.ts @@ -0,0 +1,43 @@ +import axios from 'axios' +import { parseHTML } from 'linkedom' +import { ContentHandler, PreHandleResult } from '../content-handler' + +export class TheAtlanticHandler extends ContentHandler { + constructor() { + super() + this.name = 'The Atlantic' + } + + shouldPreHandle(url: string): boolean { + const u = new URL(url) + return u.hostname.endsWith('theatlantic.com'); + } + + + unfurlContent(content: Document): Document { + const articleContentSection = content.querySelector('[data-event-module="article body"]'); + + if (!articleContentSection) { + return content; + } + + const divOverArticle = content.createElement("div"); + divOverArticle.setAttribute('id', 'prehandled') + divOverArticle.innerHTML += articleContentSection.innerHTML + content.insertBefore(divOverArticle, articleContentSection); + + articleContentSection.remove(); + + return content; + } + + async preHandle(url: string): Promise { + // We simply retrieve the article without Javascript enabled using a GET command. + const response = await axios.get(url) + const data = response.data as string + const dom = parseHTML(data).document + const editedDom = this.unfurlContent(dom); + + return { content: editedDom.body.outerHTML, title: dom.title, dom: editedDom }; + } +} diff --git a/packages/puppeteer-parse/index.js b/packages/puppeteer-parse/index.js index 06b9f658d..5915e0fb4 100644 --- a/packages/puppeteer-parse/index.js +++ b/packages/puppeteer-parse/index.js @@ -14,7 +14,7 @@ const signToken = promisify(jwt.sign); const os = require('os'); const { Storage } = require('@google-cloud/storage'); const { parseHTML } = require('linkedom'); -const { preHandleContent, preParseContent } = require("@omnivore/content-handler"); +const { preHandleContent, preParseContent } = require('../content-handler/src/index'); const { Readability } = require("@omnivore/readability"); const puppeteer = require('puppeteer-extra');