diff --git a/.github/workflows/run-tests.yaml b/.github/workflows/run-tests.yaml index de3fc67ba..3b226c0b0 100644 --- a/.github/workflows/run-tests.yaml +++ b/.github/workflows/run-tests.yaml @@ -88,3 +88,7 @@ jobs: run: 'docker build --file packages/api/Dockerfile .' - name: Build the content-fetch docker image run: 'docker build --file packages/content-fetch/Dockerfile .' + - name: Build the inbound-email-handler docker image + run: 'docker build --file packages/inbound-email-handler/Dockerfile .' + - name: Build the puppeteer-parse docker image + run: 'docker build --file packages/puppeteer-parse/Dockerfile .' diff --git a/packages/api/Dockerfile b/packages/api/Dockerfile index 5f169fbd4..d5271fbfb 100644 --- a/packages/api/Dockerfile +++ b/packages/api/Dockerfile @@ -14,15 +14,17 @@ COPY .eslintrc . COPY /packages/readabilityjs/package.json ./packages/readabilityjs/package.json COPY /packages/api/package.json ./packages/api/package.json COPY /packages/text-to-speech/package.json ./packages/text-to-speech/package.json +COPY /packages/content-handler/package.json ./packages/content-handler/package.json RUN yarn install --pure-lockfile ADD /packages/readabilityjs ./packages/readabilityjs ADD /packages/api ./packages/api ADD /packages/text-to-speech ./packages/text-to-speech +ADD /packages/content-handler ./packages/content-handler -RUN yarn RUN yarn workspace @omnivore/text-to-speech-handler build +RUN yarn workspace @omnivore/content-handler build RUN yarn workspace @omnivore/api build # After building, fetch the production dependencies diff --git a/packages/api/package.json b/packages/api/package.json index 5297027a2..0631570ed 100644 --- a/packages/api/package.json +++ b/packages/api/package.json @@ -18,6 +18,7 @@ "@google-cloud/pubsub": "^2.16.0", "@google-cloud/storage": "^5.18.1", "@google-cloud/tasks": "^2.3.0", + "@omnivore/content-handler": "1.0.0", "@omnivore/readability": "1.0.0", "@omnivore/text-to-speech-handler": "1.0.0", "@opentelemetry/api": "^1.0.1", diff --git a/packages/api/src/utils/axios-handler.ts b/packages/api/src/utils/axios-handler.ts deleted file mode 100644 index 00f6ff86a..000000000 --- a/packages/api/src/utils/axios-handler.ts +++ /dev/null @@ -1,40 +0,0 @@ -export class AxiosHandler { - name = 'axios' - - // eslint-disable-next-line @typescript-eslint/no-unused-vars - shouldPrehandle = (url: URL, _dom: Document): boolean => { - const host = this.name + '.com' - // check if url ends with axios.com - return url.hostname.endsWith(host) - } - - prehandle = (url: URL, dom: Document): Promise => { - const body = dom.querySelector('table') - - let isFooter = false - // this removes ads and replaces table with a div - body?.querySelectorAll('table').forEach((el) => { - // remove the footer and the ads - if (!el.textContent || el.textContent.length < 20 || isFooter) { - el.remove() - } else { - // removes the first few rows of the table (the header) - // remove the last two rows of the table (they are ads) - el.querySelectorAll('tr').forEach((tr, i) => { - if (i <= 7 || i >= el.querySelectorAll('tr').length - 2) { - console.log('removing', tr) - tr.remove() - } - }) - // replace the table with a div - const div = dom.createElement('div') - div.innerHTML = el.innerHTML - el.parentNode?.replaceChild(div, el) - // set the isFooter flag to true because the next table is the footer - isFooter = true - } - }) - - return Promise.resolve(dom) - } -} diff --git a/packages/api/src/utils/bloomberg-handler.ts b/packages/api/src/utils/bloomberg-handler.ts deleted file mode 100644 index c03af3f4b..000000000 --- a/packages/api/src/utils/bloomberg-handler.ts +++ /dev/null @@ -1,30 +0,0 @@ -export class BloombergHandler { - name = 'bloomberg' - - shouldPrehandle = (url: URL, dom: Document): boolean => { - const host = this.name + '.com' - // check if url ends with bloomberg.com - return ( - url.hostname.endsWith(host) || - dom.querySelector('.logo-image')?.getAttribute('alt')?.toLowerCase() === - this.name - ) - } - - prehandle = (_url: URL, dom: Document): Promise => { - const body = dom.querySelector('.wrapper') - - // this removes header - body?.querySelector('.sailthru-variables')?.remove() - body?.querySelector('.preview-text')?.remove() - body?.querySelector('.logo-wrapper')?.remove() - body?.querySelector('.by-the-number-wrapper')?.remove() - // this removes footer - body?.querySelector('.quote-box-wrapper')?.remove() - body?.querySelector('.header-wrapper')?.remove() - body?.querySelector('.component-wrapper')?.remove() - body?.querySelector('.footer')?.remove() - - return Promise.resolve(dom) - } -} diff --git a/packages/api/src/utils/morning-brew-handler.ts b/packages/api/src/utils/morning-brew-handler.ts deleted file mode 100644 index 2cfabf361..000000000 --- a/packages/api/src/utils/morning-brew-handler.ts +++ /dev/null @@ -1,29 +0,0 @@ -export class MorningBrewHandler { - name = 'morningbrew' - - // eslint-disable-next-line @typescript-eslint/no-unused-vars - shouldPrehandle = (url: URL, _dom: Document): boolean => { - const host = this.name + '.com' - // check if url ends with morningbrew.com - return url.hostname.endsWith(host) - } - - prehandle = (url: URL, dom: Document): Promise => { - // retain the width of the cells in the table of market info - dom.querySelectorAll('.markets-arrow-cell').forEach((td) => { - const table = td.closest('table') - if (table) { - const bubbleTable = table.querySelector('.markets-bubble') - if (bubbleTable) { - // replace the nested table with the text - const e = bubbleTable.querySelector('.markets-table-text') - e && bubbleTable.parentNode?.replaceChild(e, bubbleTable) - } - // set custom class for the table - table.className = 'morning-brew-markets' - } - }) - - return Promise.resolve(dom) - } -} diff --git a/packages/api/src/utils/parser.ts b/packages/api/src/utils/parser.ts index 3d6242264..eee86b417 100644 --- a/packages/api/src/utils/parser.ts +++ b/packages/api/src/utils/parser.ts @@ -7,11 +7,6 @@ import { PageType, PreparedDocumentInput } from '../generated/graphql' import { buildLogger, LogRecord } from './logger' import { createImageProxyUrl } from './imageproxy' import axios from 'axios' -import { WikipediaHandler } from './wikipedia-handler' -import { SubstackHandler } from './substack-handler' -import { AxiosHandler } from './axios-handler' -import { BloombergHandler } from './bloomberg-handler' -import { GolangHandler } from './golang-handler' import * as hljs from 'highlightjs' import { decode } from 'html-entities' import { parseHTML } from 'linkedom' @@ -20,7 +15,7 @@ import { User } from '../entity/user' import { ILike } from 'typeorm' import { v4 as uuid } from 'uuid' import addressparser from 'addressparser' -import { MorningBrewHandler } from './morning-brew-handler' +import { preParseContent } from '@omnivore/content-handler' const logger = buildLogger('utils.parse') @@ -47,20 +42,6 @@ const ARTICLE_PREFIX = 'omnivore:' export const FAKE_URL_PREFIX = 'https://omnivore.app/no_url?q=' -interface ContentHandler { - shouldPrehandle: (url: URL, dom: Document) => boolean - prehandle: (url: URL, document: Document) => Promise -} - -const HANDLERS = [ - new WikipediaHandler(), - new SubstackHandler(), - new AxiosHandler(), - new BloombergHandler(), - new GolangHandler(), - new MorningBrewHandler(), -] - /** Hook that prevents DOMPurify from removing youtube iframes */ const domPurifySanitizeHook = ( node: Element, @@ -185,33 +166,6 @@ const getReadabilityResult = async ( return null } -const applyHandlers = async ( - url: string, - document: Document -): Promise => { - try { - const u = new URL(url) - const handler = HANDLERS.find((h) => { - try { - return h.shouldPrehandle(u, document) - } catch (e) { - console.log('error with handler: ', h.name, e) - } - return false - }) - if (handler) { - try { - console.log('pre-handling url or content with handler: ', handler.name) - await handler.prehandle(u, document) - } catch (e) { - console.log('error with handler: ', handler, e) - } - } - } catch (error) { - logger.error('Error prehandling url', url, error) - } -} - export const parsePreparedContent = async ( url: string, preparedDocument: PreparedDocumentInput, @@ -241,9 +195,11 @@ export const parsePreparedContent = async ( } } - const dom = parseHTML(document).document + let dom = parseHTML(document).document - await applyHandlers(url, dom) + // preParse content + const preParsedDom = await preParseContent(url, dom) + preParsedDom && (dom = preParsedDom) try { article = await getReadabilityResult(url, document, dom, isNewsletter) diff --git a/packages/api/src/utils/substack-handler.ts b/packages/api/src/utils/substack-handler.ts deleted file mode 100644 index ea6b69bc3..000000000 --- a/packages/api/src/utils/substack-handler.ts +++ /dev/null @@ -1,33 +0,0 @@ -export class SubstackHandler { - name = 'substack' - - shouldPrehandle = (url: URL, dom: Document): boolean => { - const host = this.name + '.com' - // check if url ends with substack.com - // or has a profile image hosted at substack.com - return ( - url.hostname.endsWith(host) || - !!dom - .querySelector('.email-body img') - ?.getAttribute('src') - ?.includes(host) - ) - } - - prehandle = (url: URL, dom: Document): Promise => { - const body = dom.querySelector('.email-body-container') - - // this removes header and profile avatar - body?.querySelector('.header')?.remove() - body?.querySelector('.preamble')?.remove() - body?.querySelector('.meta-author-wrap')?.remove() - // this removes meta button - body?.querySelector('.post-meta')?.remove() - // this removes footer - body?.querySelector('.post-cta')?.remove() - body?.querySelector('.container-border')?.remove() - body?.querySelector('.footer')?.remove() - - return Promise.resolve(dom) - } -} diff --git a/packages/api/src/utils/wikipedia-handler.ts b/packages/api/src/utils/wikipedia-handler.ts deleted file mode 100644 index ce30517c9..000000000 --- a/packages/api/src/utils/wikipedia-handler.ts +++ /dev/null @@ -1,16 +0,0 @@ -export class WikipediaHandler { - name = 'wikipedia' - - // eslint-disable-next-line @typescript-eslint/no-unused-vars - shouldPrehandle = (url: URL, _dom: Document): boolean => { - return url.hostname.endsWith('wikipedia.org') - } - - prehandle = (url: URL, dom: Document): Promise => { - // This removes the [edit] anchors from wikipedia pages - dom.querySelectorAll('.mw-editsection').forEach((e) => e.remove()) - // this removes the sidebar - dom.querySelector('.infobox')?.remove() - return Promise.resolve(dom) - } -} diff --git a/packages/content-handler/src/content-handler.ts b/packages/content-handler/src/content-handler.ts index 22216fabe..317f3d2d9 100644 --- a/packages/content-handler/src/content-handler.ts +++ b/packages/content-handler/src/content-handler.ts @@ -58,12 +58,20 @@ export abstract class ContentHandler { return Promise.resolve(url) } - shouldPreHandle(url: string, dom?: Document): boolean { + shouldPreHandle(url: string): boolean { return false } - async preHandle(url: string, dom?: Document): Promise { - return Promise.resolve({ url, dom }) + async preHandle(url: string): Promise { + return Promise.resolve({ url }) + } + + shouldPreParse(url: string, dom: Document): boolean { + return false + } + + async preParse(url: string, dom: Document): Promise { + return Promise.resolve(dom) } async isNewsletter(input: { diff --git a/packages/content-handler/src/index.ts b/packages/content-handler/src/index.ts index e41c811c4..ba67d2d9d 100644 --- a/packages/content-handler/src/index.ts +++ b/packages/content-handler/src/index.ts @@ -52,6 +52,11 @@ const contentHandlers: ContentHandler[] = [ new TwitterHandler(), new YoutubeHandler(), new WikipediaHandler(), + new AxiosHandler(), + new GolangHandler(), + new MorningBrewHandler(), + new BloombergNewsletterHandler(), + new SubstackHandler(), ] const newsletterHandlers: ContentHandler[] = [ @@ -60,15 +65,13 @@ const newsletterHandlers: ContentHandler[] = [ new GolangHandler(), new SubstackHandler(), new MorningBrewHandler(), - new SubstackHandler(), new BeehiivHandler(), new ConvertkitHandler(), new RevueHandler(), ] export const preHandleContent = async ( - url: string, - dom?: Document + url: string ): Promise => { // Before we run the regular handlers we check to see if we need tp // pre-resolve the URL. TODO: This should probably happen recursively, @@ -90,9 +93,25 @@ export const preHandleContent = async ( // to perform a prefetch action that can modify our requests. // enumerate the handlers and see if any of them want to handle the request for (const handler of contentHandlers) { - if (handler.shouldPreHandle(url, dom)) { + if (handler.shouldPreHandle(url)) { console.log('preHandleContent', handler.name, url) - return handler.preHandle(url, dom) + return handler.preHandle(url) + } + } + return undefined +} + +export const preParseContent = async ( + url: string, + dom: Document +): Promise => { + // Before we parse the page we check the handlers, to see if they want + // to perform a preParse action that can modify our dom. + // enumerate the handlers and see if any of them want to handle the dom + for (const handler of contentHandlers) { + if (handler.shouldPreParse(url, dom)) { + console.log('preParseContent', handler.name, url) + return handler.preParse(url, dom) } } return undefined @@ -113,4 +132,5 @@ export const handleNewsletter = async ( module.exports = { preHandleContent, handleNewsletter, + preParseContent, } diff --git a/packages/content-handler/src/newsletters/axios-handler.ts b/packages/content-handler/src/newsletters/axios-handler.ts index cd783c30e..0fc6dc3b3 100644 --- a/packages/content-handler/src/newsletters/axios-handler.ts +++ b/packages/content-handler/src/newsletters/axios-handler.ts @@ -1,4 +1,4 @@ -import { ContentHandler, PreHandleResult } from '../content-handler' +import { ContentHandler } from '../content-handler' export class AxiosHandler extends ContentHandler { constructor() { @@ -8,13 +8,13 @@ export class AxiosHandler extends ContentHandler { this.name = 'axios' } - shouldPreHandle(url: string, dom?: Document): boolean { + shouldPreParse(url: string, dom: Document): boolean { const host = this.name + '.com' // check if url ends with axios.com return new URL(url).hostname.endsWith(host) } - async preHandle(url: string, dom: Document): Promise { + async preParse(url: string, dom: Document): Promise { const body = dom.querySelector('table') let isFooter = false @@ -41,6 +41,6 @@ export class AxiosHandler extends ContentHandler { } }) - return Promise.resolve({ dom }) + return Promise.resolve(dom) } } diff --git a/packages/content-handler/src/newsletters/bloomberg-newsletter-handler.ts b/packages/content-handler/src/newsletters/bloomberg-newsletter-handler.ts index a5f84f076..5ccd31466 100644 --- a/packages/content-handler/src/newsletters/bloomberg-newsletter-handler.ts +++ b/packages/content-handler/src/newsletters/bloomberg-newsletter-handler.ts @@ -1,4 +1,4 @@ -import { ContentHandler, PreHandleResult } from '../content-handler' +import { ContentHandler } from '../content-handler' export class BloombergNewsletterHandler extends ContentHandler { constructor() { @@ -8,7 +8,7 @@ export class BloombergNewsletterHandler extends ContentHandler { this.name = 'bloomberg' } - shouldPreHandle(url: string, dom: Document): boolean { + shouldPreParse(url: string, dom: Document): boolean { const host = this.name + '.com' // check if url ends with bloomberg.com return ( @@ -18,7 +18,7 @@ export class BloombergNewsletterHandler extends ContentHandler { ) } - async preHandle(url: string, dom: Document): Promise { + async preParse(url: string, dom: Document): Promise { const body = dom.querySelector('.wrapper') // this removes header @@ -32,6 +32,6 @@ export class BloombergNewsletterHandler extends ContentHandler { body?.querySelector('.component-wrapper')?.remove() body?.querySelector('.footer')?.remove() - return Promise.resolve({ dom }) + return Promise.resolve(dom) } } diff --git a/packages/content-handler/src/newsletters/golang-handler.ts b/packages/content-handler/src/newsletters/golang-handler.ts index 7d4724004..aeafec852 100644 --- a/packages/content-handler/src/newsletters/golang-handler.ts +++ b/packages/content-handler/src/newsletters/golang-handler.ts @@ -1,4 +1,4 @@ -import { ContentHandler, PreHandleResult } from '../content-handler' +import { ContentHandler } from '../content-handler' export class GolangHandler extends ContentHandler { constructor() { @@ -8,13 +8,13 @@ export class GolangHandler extends ContentHandler { this.name = 'golangweekly' } - shouldPreHandle(url: string, dom?: Document): boolean { + shouldPreParse(url: string, dom: Document): boolean { const host = this.name + '.com' // check if url ends with golangweekly.com return new URL(url).hostname.endsWith(host) } - async preHandle(url: string, dom: Document): Promise { + async preParse(url: string, dom: Document): Promise { const body = dom.querySelector('body') // this removes the "Subscribe" button @@ -22,6 +22,6 @@ export class GolangHandler extends ContentHandler { // this removes the title body?.querySelector('.el-masthead')?.remove() - return Promise.resolve({ dom }) + return Promise.resolve(dom) } } diff --git a/packages/content-handler/src/newsletters/morning-brew-handler.ts b/packages/content-handler/src/newsletters/morning-brew-handler.ts index f187ac0dc..0a69292fe 100644 --- a/packages/content-handler/src/newsletters/morning-brew-handler.ts +++ b/packages/content-handler/src/newsletters/morning-brew-handler.ts @@ -1,4 +1,4 @@ -import { ContentHandler, PreHandleResult } from '../content-handler' +import { ContentHandler } from '../content-handler' export class MorningBrewHandler extends ContentHandler { constructor() { @@ -8,13 +8,13 @@ export class MorningBrewHandler extends ContentHandler { this.name = 'morningbrew' } - shouldPreHandle(url: string, dom?: Document): boolean { + shouldPreParse(url: string, dom: Document): boolean { const host = this.name + '.com' // check if url ends with morningbrew.com return new URL(url).hostname.endsWith(host) } - async preHandle(url: string, dom: Document): Promise { + async preParse(url: string, dom: Document): Promise { // retain the width of the cells in the table of market info dom.querySelectorAll('.markets-arrow-cell').forEach((td) => { const table = td.closest('table') @@ -30,6 +30,6 @@ export class MorningBrewHandler extends ContentHandler { } }) - return Promise.resolve({ dom }) + return Promise.resolve(dom) } } diff --git a/packages/content-handler/src/newsletters/substack-handler.ts b/packages/content-handler/src/newsletters/substack-handler.ts index f66a85a52..e90168c8e 100644 --- a/packages/content-handler/src/newsletters/substack-handler.ts +++ b/packages/content-handler/src/newsletters/substack-handler.ts @@ -1,5 +1,5 @@ import addressparser from 'addressparser' -import { ContentHandler, PreHandleResult } from '../content-handler' +import { ContentHandler } from '../content-handler' import { parseHTML } from 'linkedom' export class SubstackHandler extends ContentHandler { @@ -8,7 +8,7 @@ export class SubstackHandler extends ContentHandler { this.name = 'substack' } - shouldPreHandle(url: string, dom: Document): boolean { + shouldPreParse(url: string, dom: Document): boolean { const host = this.name + '.com' // check if url ends with substack.com // or has a profile image hosted at substack.com @@ -21,7 +21,7 @@ export class SubstackHandler extends ContentHandler { ) } - async preHandle(url: string, dom: Document): Promise { + async preParse(url: string, dom: Document): Promise { const body = dom.querySelector('.email-body-container') // this removes header and profile avatar diff --git a/packages/content-handler/src/websites/apple-news-handler.ts b/packages/content-handler/src/websites/apple-news-handler.ts index 0b4026fb6..88a012801 100644 --- a/packages/content-handler/src/websites/apple-news-handler.ts +++ b/packages/content-handler/src/websites/apple-news-handler.ts @@ -8,12 +8,12 @@ export class AppleNewsHandler extends ContentHandler { this.name = 'Apple News' } - shouldPreHandle(url: string, dom?: Document): boolean { + shouldPreHandle(url: string): boolean { const u = new URL(url) return u.hostname === 'apple.news' } - async preHandle(url: string, document?: Document): Promise { + async preHandle(url: string): Promise { const MOBILE_USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.61 Safari/537.36' const response = await axios.get(url, { diff --git a/packages/content-handler/src/websites/bloomberg-handler.ts b/packages/content-handler/src/websites/bloomberg-handler.ts index a867a3503..c7b7a5cb3 100644 --- a/packages/content-handler/src/websites/bloomberg-handler.ts +++ b/packages/content-handler/src/websites/bloomberg-handler.ts @@ -8,13 +8,13 @@ export class BloombergHandler extends ContentHandler { this.name = 'Bloomberg' } - shouldPreHandle(url: string, dom?: Document): boolean { + shouldPreHandle(url: string): boolean { const BLOOMBERG_URL_MATCH = /https?:\/\/(www\.)?bloomberg.[a-zA-Z0-9()]{1,6}\b([-a-zA-Z0-9()@:%_+.~#?&/=]*)/ return BLOOMBERG_URL_MATCH.test(url.toString()) } - async preHandle(url: string, document?: Document): Promise { + async preHandle(url: string): Promise { console.log('prehandling bloomberg url', url) try { diff --git a/packages/content-handler/src/websites/derstandard-handler.ts b/packages/content-handler/src/websites/derstandard-handler.ts index 28742a3e5..783423f82 100644 --- a/packages/content-handler/src/websites/derstandard-handler.ts +++ b/packages/content-handler/src/websites/derstandard-handler.ts @@ -8,12 +8,12 @@ export class DerstandardHandler extends ContentHandler { this.name = 'Derstandard' } - shouldPreHandle(url: string, dom?: Document): boolean { + shouldPreHandle(url: string): boolean { const u = new URL(url) return u.hostname === 'www.derstandard.at' } - async preHandle(url: string, document?: Document): Promise { + async preHandle(url: string): Promise { const response = await axios.get(url, { // set cookie to give consent to get the article headers: { diff --git a/packages/content-handler/src/websites/image-handler.ts b/packages/content-handler/src/websites/image-handler.ts index 068a1cc66..2ccfcd610 100644 --- a/packages/content-handler/src/websites/image-handler.ts +++ b/packages/content-handler/src/websites/image-handler.ts @@ -6,12 +6,12 @@ export class ImageHandler extends ContentHandler { this.name = 'Image' } - shouldPreHandle(url: string, dom?: Document): boolean { + shouldPreHandle(url: string): boolean { const IMAGE_URL_PATTERN = /(https?:\/\/.*\.(?:jpg|jpeg|png|webp))/i return IMAGE_URL_PATTERN.test(url.toString()) } - async preHandle(url: string, document?: Document): Promise { + async preHandle(url: string): Promise { const title = url.toString().split('/').pop() || 'Image' const content = ` diff --git a/packages/content-handler/src/websites/medium-handler.ts b/packages/content-handler/src/websites/medium-handler.ts index 211a30c37..111bbe787 100644 --- a/packages/content-handler/src/websites/medium-handler.ts +++ b/packages/content-handler/src/websites/medium-handler.ts @@ -6,12 +6,12 @@ export class MediumHandler extends ContentHandler { this.name = 'Medium' } - shouldPreHandle(url: string, dom?: Document): boolean { + shouldPreHandle(url: string): boolean { const u = new URL(url) return u.hostname.endsWith('medium.com') } - async preHandle(url: string, document?: Document): Promise { + async preHandle(url: string): Promise { console.log('prehandling medium url', url) try { diff --git a/packages/content-handler/src/websites/pdf-handler.ts b/packages/content-handler/src/websites/pdf-handler.ts index 4c4ef748d..b321bb103 100644 --- a/packages/content-handler/src/websites/pdf-handler.ts +++ b/packages/content-handler/src/websites/pdf-handler.ts @@ -6,13 +6,13 @@ export class PdfHandler extends ContentHandler { this.name = 'PDF' } - shouldPreHandle(url: string, dom?: Document): boolean { + shouldPreHandle(url: string): boolean { const u = new URL(url) const path = u.pathname.replace(u.search, '') return path.endsWith('.pdf') } - async preHandle(_url: string, document?: Document): Promise { + async preHandle(url: string): Promise { return Promise.resolve({ contentType: 'application/pdf' }) } } diff --git a/packages/content-handler/src/websites/scrapingBee-handler.ts b/packages/content-handler/src/websites/scrapingBee-handler.ts index 4c04d00e8..a7818a685 100644 --- a/packages/content-handler/src/websites/scrapingBee-handler.ts +++ b/packages/content-handler/src/websites/scrapingBee-handler.ts @@ -8,14 +8,14 @@ export class ScrapingBeeHandler extends ContentHandler { this.name = 'ScrapingBee' } - shouldPreHandle(url: string, dom?: Document): boolean { + shouldPreHandle(url: string): boolean { const u = new URL(url) const hostnames = ['nytimes.com', 'news.google.com'] return hostnames.some((h) => u.hostname.endsWith(h)) } - async preHandle(url: string, document?: Document): Promise { + async preHandle(url: string): Promise { console.log('prehandling url with scrapingbee', url) try { diff --git a/packages/content-handler/src/websites/twitter-handler.ts b/packages/content-handler/src/websites/twitter-handler.ts index 9be264e5c..108d5785f 100644 --- a/packages/content-handler/src/websites/twitter-handler.ts +++ b/packages/content-handler/src/websites/twitter-handler.ts @@ -140,11 +140,11 @@ export class TwitterHandler extends ContentHandler { this.name = 'Twitter' } - shouldPreHandle(url: string, dom?: Document): boolean { + shouldPreHandle(url: string): boolean { return !!TWITTER_BEARER_TOKEN && TWITTER_URL_MATCH.test(url.toString()) } - async preHandle(url: string, document?: Document): Promise { + async preHandle(url: string): Promise { const tweetId = tweetIdFromStatusUrl(url) if (!tweetId) { throw new Error('could not find tweet id in url') diff --git a/packages/content-handler/src/websites/wikipedia-handler.ts b/packages/content-handler/src/websites/wikipedia-handler.ts index 8c3a176fd..37005a18a 100644 --- a/packages/content-handler/src/websites/wikipedia-handler.ts +++ b/packages/content-handler/src/websites/wikipedia-handler.ts @@ -1,4 +1,4 @@ -import { ContentHandler, PreHandleResult } from '../content-handler' +import { ContentHandler } from '../content-handler' export class WikipediaHandler extends ContentHandler { constructor() { @@ -6,15 +6,15 @@ export class WikipediaHandler extends ContentHandler { this.name = 'wikipedia' } - shouldPreHandle(url: string, dom?: Document): boolean { + shouldPreParse(url: string, dom: Document): boolean { return new URL(url).hostname.endsWith('wikipedia.org') } - async preHandle(url: string, dom: Document): Promise { + async preParse(url: string, dom: Document): Promise { // This removes the [edit] anchors from wikipedia pages dom.querySelectorAll('.mw-editsection').forEach((e) => e.remove()) // this removes the sidebar dom.querySelector('.infobox')?.remove() - return Promise.resolve({ dom }) + return Promise.resolve(dom) } } diff --git a/packages/content-handler/src/websites/youtube-handler.ts b/packages/content-handler/src/websites/youtube-handler.ts index 4cdb7ee98..a1f62f607 100644 --- a/packages/content-handler/src/websites/youtube-handler.ts +++ b/packages/content-handler/src/websites/youtube-handler.ts @@ -24,11 +24,11 @@ export class YoutubeHandler extends ContentHandler { this.name = 'Youtube' } - shouldPreHandle(url: string, dom?: Document): boolean { + shouldPreHandle(url: string): boolean { return YOUTUBE_URL_MATCH.test(url.toString()) } - async preHandle(url: string, document?: Document): Promise { + async preHandle(url: string): Promise { const videoId = getYoutubeVideoId(url) if (!videoId) { return {}