Add Paywalled Wired Handler

packages/content-handler/src/websites/wired-handler.ts

@@ -0,0 +1,59 @@
+import axios from 'axios'
+import { parseHTML } from 'linkedom'
+import { ContentHandler, PreHandleResult } from '../content-handler'
+
+export class WiredHandler extends ContentHandler {
+  constructor() {
+    super()
+    this.name = 'Wired'
+  }
+
+  // We check if this is a paywalled document, as paywalled documents will have <p> tags
+  // in the body.
+  isPaywalledContent(document: Document): boolean {
+    return document.getElementsByClassName('paywall').length > 0
+  }
+
+  removeNonArticleNodes(document: Document): Document {
+    const genericCallouts = Array.from(
+      document.querySelectorAll('[data-testid="GenericCallout"]')
+    )
+    const ads = Array.from(document.querySelectorAll('.ad__slot')).map(
+      (it) => it.parentElement
+    )
+    const mostPopularArticles = Array.from(
+      document.querySelectorAll('[data-most-popular-id]')
+    )
+
+    ;[...genericCallouts, ...ads, ...mostPopularArticles].forEach((it) =>
+      it?.remove()
+    )
+
+    return document
+  }
+
+  shouldPreHandle(url: string): boolean {
+    const u = new URL(url)
+    return u.hostname.endsWith('wired.com')
+  }
+
+  async preHandle(url: string): Promise<PreHandleResult> {
+    const response = await axios.get(url)
+    const data = response.data as string
+    const dom = parseHTML(data).document
+
+    if (!this.isPaywalledContent(dom)) {
+      // This is just to ensure that the currently working articles don't break.
+      // Looking further into this, they might all have paywalls?
+      return {}
+    }
+
+    const cleanedArticleDom = this.removeNonArticleNodes(dom)
+
+    return {
+      content: cleanedArticleDom.body.outerHTML,
+      title: dom.title,
+      dom: cleanedArticleDom,
+    }
+  }
+}

packages/content-handler/test/wired.test.ts

@@ -0,0 +1,55 @@
+import { WiredHandler } from '../src/websites/wired-handler'
+import fs from 'fs';
+import nock from 'nock'
+import { expect } from 'chai'
+import { parseHTML } from 'linkedom'
+
+
+describe('Testing Wired Paywalled Article opening', () => {
+  const load = (path: string): string => {
+    return fs.readFileSync(path, 'utf8')
+  }
+  
+  before(() => {
+    const html = load('./test/data/wired-article.html');
+    nock('https://wired.com').persist().get('/article').reply(200, html)
+  })
+
+  it('should parse the title of the wired article.', async () => {
+    const response = await new WiredHandler().preHandle(
+      'https://wired.com/article'
+    );
+
+    // We grab the title from the doucment. 
+    expect(response.title).not.to.be.undefined
+  })
+
+  it('should remove callout', async () => {
+    const response = await new WiredHandler().preHandle(
+      'https://wired.com/article'
+    );
+    const html = load('./test/data/wired-article.html');
+    const priorDom = parseHTML(html).document;
+
+    expect(priorDom.querySelector('[data-testid="GenericCallout"]')).not.to.be.null
+    // Should no longer be the case after pre-rendering.
+    expect(response.dom?.querySelector('[data-testid="GenericCallout"]')).to.be.null
+  });
+
+  it ('should remove any ad placeholders', async() => { 
+    const response = await new WiredHandler().preHandle(
+      'https://wired.com/article'
+    );
+
+    expect(response.dom?.querySelector('.ad__slot')).to.be.null
+  })
+
+  it ('should remove any related content links.', async() => { 
+    const response = await new WiredHandler().preHandle(
+      'https://wired.com/article'
+    );
+
+    // This exists in the HTML, but we remove it when preparsing.
+    expect(response.dom?.querySelector('[data-most-popular-id]')).to.be.null;
+  })
+})