diff --git a/packages/content-handler/src/index.ts b/packages/content-handler/src/index.ts index 503b8e063..fa028063c 100644 --- a/packages/content-handler/src/index.ts +++ b/packages/content-handler/src/index.ts @@ -40,6 +40,7 @@ import { YoutubeHandler } from './websites/youtube-handler' import { ZhihuHandler } from './websites/zhihu-handler' import { TikTokHandler } from './websites/tiktok-handler' import { WiredHandler } from './websites/wired-handler' +import { RawContentHandler } from './websites/raw-handler' const validateUrlString = (url: string): boolean => { const u = new URL(url) @@ -68,6 +69,7 @@ const contentHandlers: ContentHandler[] = [ new DerstandardHandler(), new ImageHandler(), new MediumHandler(), + new RawContentHandler(), new PdfHandler(), new ScrapingBeeHandler(), new TDotCoHandler(), diff --git a/packages/content-handler/src/websites/medium-handler.ts b/packages/content-handler/src/websites/medium-handler.ts index 111bbe787..8079cfa3f 100644 --- a/packages/content-handler/src/websites/medium-handler.ts +++ b/packages/content-handler/src/websites/medium-handler.ts @@ -1,4 +1,6 @@ import { ContentHandler, PreHandleResult } from '../content-handler' +import axios from 'axios' +import { parseHTML } from 'linkedom' export class MediumHandler extends ContentHandler { constructor() { @@ -17,7 +19,14 @@ export class MediumHandler extends ContentHandler { try { const res = new URL(url) res.searchParams.delete('source') - return Promise.resolve({ url: res.toString() }) + + const response = await axios.get(res.toString()) + const dom = parseHTML(response.data).document + return { + title: dom.title, + content: response.data as string, + url: res.toString(), + } } catch (error) { console.error('error prehandling medium url', error) throw error diff --git a/packages/content-handler/src/websites/raw-handler.ts b/packages/content-handler/src/websites/raw-handler.ts new file mode 100644 index 000000000..bcbd6a10f --- /dev/null +++ b/packages/content-handler/src/websites/raw-handler.ts @@ -0,0 +1,28 @@ +import { ContentHandler, PreHandleResult } from '../content-handler' +import axios from 'axios' +import { parseHTML } from 'linkedom' + +export class RawContentHandler extends ContentHandler { + constructor() { + super() + this.name = 'RawContentHandler' + } + + shouldPreHandle(url: string): boolean { + const u = new URL(url) + const hostnames = ['medium.com', 'fastcompany.com', 'fortelabs.com'] + + return hostnames.some((h) => u.hostname.endsWith(h)) + } + + async preHandle(url: string): Promise { + try { + const response = await axios.get(url) + const dom = parseHTML(response.data).document + return { title: dom.title, content: response.data as string, url: url } + } catch (error) { + console.error('error prehandling URL', error) + throw error + } + } +} diff --git a/self-hosting/GUIDE.md b/self-hosting/GUIDE.md index 425f0370a..01aea6620 100644 --- a/self-hosting/GUIDE.md +++ b/self-hosting/GUIDE.md @@ -30,7 +30,7 @@ These files provide all you need to get Omnivore up and running on your local en ### 3. Populate the .env file There is a .env.example file located within the docker-compose folder that should give you the necessary environment variables to begin running. -You can use these by `mv .env .example.env` +You can use these by `mv .env.example .env` The following environment variables should be changed to reflect where you are running your application.