diff --git a/packages/content-handler/src/index.ts b/packages/content-handler/src/index.ts index a75af64df..bf531676f 100644 --- a/packages/content-handler/src/index.ts +++ b/packages/content-handler/src/index.ts @@ -39,6 +39,7 @@ import { WikipediaHandler } from './websites/wikipedia-handler' import { YoutubeHandler } from './websites/youtube-handler' import { ZhihuHandler } from './websites/zhihu-handler' import { TikTokHandler } from './websites/tiktok-handler' +import { RawContentHandler } from './websites/raw-handler' const validateUrlString = (url: string): boolean => { const u = new URL(url) @@ -66,6 +67,7 @@ const contentHandlers: ContentHandler[] = [ new DerstandardHandler(), new ImageHandler(), new MediumHandler(), + new RawContentHandler(), new PdfHandler(), new ScrapingBeeHandler(), new TDotCoHandler(), diff --git a/packages/content-handler/src/websites/medium-handler.ts b/packages/content-handler/src/websites/medium-handler.ts index 111bbe787..8079cfa3f 100644 --- a/packages/content-handler/src/websites/medium-handler.ts +++ b/packages/content-handler/src/websites/medium-handler.ts @@ -1,4 +1,6 @@ import { ContentHandler, PreHandleResult } from '../content-handler' +import axios from 'axios' +import { parseHTML } from 'linkedom' export class MediumHandler extends ContentHandler { constructor() { @@ -17,7 +19,14 @@ export class MediumHandler extends ContentHandler { try { const res = new URL(url) res.searchParams.delete('source') - return Promise.resolve({ url: res.toString() }) + + const response = await axios.get(res.toString()) + const dom = parseHTML(response.data).document + return { + title: dom.title, + content: response.data as string, + url: res.toString(), + } } catch (error) { console.error('error prehandling medium url', error) throw error diff --git a/packages/content-handler/src/websites/raw-handler.ts b/packages/content-handler/src/websites/raw-handler.ts new file mode 100644 index 000000000..bcbd6a10f --- /dev/null +++ b/packages/content-handler/src/websites/raw-handler.ts @@ -0,0 +1,28 @@ +import { ContentHandler, PreHandleResult } from '../content-handler' +import axios from 'axios' +import { parseHTML } from 'linkedom' + +export class RawContentHandler extends ContentHandler { + constructor() { + super() + this.name = 'RawContentHandler' + } + + shouldPreHandle(url: string): boolean { + const u = new URL(url) + const hostnames = ['medium.com', 'fastcompany.com', 'fortelabs.com'] + + return hostnames.some((h) => u.hostname.endsWith(h)) + } + + async preHandle(url: string): Promise { + try { + const response = await axios.get(url) + const dom = parseHTML(response.data).document + return { title: dom.title, content: response.data as string, url: url } + } catch (error) { + console.error('error prehandling URL', error) + throw error + } + } +}