Do raw handlers for Medium

packages/content-handler/src/index.ts

@@ -39,6 +39,7 @@ import { WikipediaHandler } from './websites/wikipedia-handler'
 import { YoutubeHandler } from './websites/youtube-handler'
 import { ZhihuHandler } from './websites/zhihu-handler'
 import { TikTokHandler } from './websites/tiktok-handler'
+import { RawContentHandler } from './websites/raw-handler'
 
 const validateUrlString = (url: string): boolean => {
   const u = new URL(url)
@@ -66,6 +67,7 @@ const contentHandlers: ContentHandler[] = [
   new DerstandardHandler(),
   new ImageHandler(),
   new MediumHandler(),
+  new RawContentHandler(),
   new PdfHandler(),
   new ScrapingBeeHandler(),
   new TDotCoHandler(),

packages/content-handler/src/websites/medium-handler.ts

@@ -1,4 +1,6 @@
 import { ContentHandler, PreHandleResult } from '../content-handler'
+import axios from 'axios'
+import { parseHTML } from 'linkedom'
 
 export class MediumHandler extends ContentHandler {
   constructor() {
@@ -17,7 +19,14 @@ export class MediumHandler extends ContentHandler {
     try {
       const res = new URL(url)
       res.searchParams.delete('source')
-      return Promise.resolve({ url: res.toString() })
+
+      const response = await axios.get(res.toString())
+      const dom = parseHTML(response.data).document
+      return {
+        title: dom.title,
+        content: response.data as string,
+        url: res.toString(),
+      }
     } catch (error) {
       console.error('error prehandling medium url', error)
       throw error

packages/content-handler/src/websites/raw-handler.ts

@@ -0,0 +1,28 @@
+import { ContentHandler, PreHandleResult } from '../content-handler'
+import axios from 'axios'
+import { parseHTML } from 'linkedom'
+
+export class RawContentHandler extends ContentHandler {
+  constructor() {
+    super()
+    this.name = 'RawContentHandler'
+  }
+
+  shouldPreHandle(url: string): boolean {
+    const u = new URL(url)
+    const hostnames = ['medium.com', 'fastcompany.com', 'fortelabs.com']
+
+    return hostnames.some((h) => u.hostname.endsWith(h))
+  }
+
+  async preHandle(url: string): Promise<PreHandleResult> {
+    try {
+      const response = await axios.get(url)
+      const dom = parseHTML(response.data).document
+      return { title: dom.title, content: response.data as string, url: url }
+    } catch (error) {
+      console.error('error prehandling URL', error)
+      throw error
+    }
+  }
+}