From 72e78cd465ade22762d5109828f3b23e5de87740 Mon Sep 17 00:00:00 2001
From: Jackson Harper <jacksonh@gmail.com>
Date: Tue, 19 Mar 2024 13:06:25 +0800
Subject: [PATCH] Revert WIP changes

---
 packages/api/src/jobs/ai-summarize.ts     | 15 ++----
 packages/api/src/services/ai-summaries.ts | 66 -----------------------
 2 files changed, 5 insertions(+), 76 deletions(-)

diff --git a/packages/api/src/jobs/ai-summarize.ts b/packages/api/src/jobs/ai-summarize.ts
index 3658876b6..974a6b347 100644
--- a/packages/api/src/jobs/ai-summarize.ts
+++ b/packages/api/src/jobs/ai-summarize.ts
@@ -4,13 +4,10 @@ import { ChatOpenAI } from '@langchain/openai'
 import { RecursiveCharacterTextSplitter } from 'langchain/text_splitter'
 import { authTrx } from '../repository'
 import { libraryItemRepository } from '../repository/library_item'
+import { htmlToMarkdown } from '../utils/parser'
 import { AISummary } from '../entity/AISummary'
 import { LibraryItemState } from '../entity/library_item'
-import {
-  createSummarizableDocument,
-  getAISummary,
-} from '../services/ai-summaries'
-import { NodeHtmlMarkdown, TranslatorConfigObject } from 'node-html-markdown'
+import { getAISummary } from '../services/ai-summaries'
 
 export interface AISummarizeJobData {
   userId: string
@@ -58,21 +55,19 @@ export const aiSummarize = async (jobData: AISummarizeJobData) => {
       },
     })
     const textSplitter = new RecursiveCharacterTextSplitter({
-      chunkSize: 12000,
+      chunkSize: 2000,
     })
 
-    const document = createSummarizableDocument(libraryItem.readableContent)
+    const document = htmlToMarkdown(libraryItem.readableContent)
     const docs = await textSplitter.createDocuments([document])
     const chain = loadSummarizationChain(llm, {
       type: 'map_reduce', // you can choose from map_reduce, stuff or refine
       verbose: true, // to view the steps in the console
     })
-    const response = await chain.invoke({
+    const response = await chain.call({
       input_documents: docs,
     })
 
-    console.log('summary response: ', JSON.stringify(response))
-
     if (typeof response.text !== 'string') {
       logger.error(`AI summary did not return text`)
       return
diff --git a/packages/api/src/services/ai-summaries.ts b/packages/api/src/services/ai-summaries.ts
index 9b9ebefd8..3c94ca4cd 100644
--- a/packages/api/src/services/ai-summaries.ts
+++ b/packages/api/src/services/ai-summaries.ts
@@ -1,71 +1,5 @@
-import { ChatOpenAI } from '@langchain/openai'
 import { AISummary } from '../entity/AISummary'
 import { authTrx } from '../repository'
-import { RecursiveCharacterTextSplitter } from 'langchain/text_splitter'
-import { loadSummarizationChain } from 'langchain/chains'
-import { logger } from '../utils/logger'
-import { NodeHtmlMarkdown, TranslatorConfigObject } from 'node-html-markdown'
-
-// When creating markdown we remove external links in URLs
-// and images since these often contain per-user trackers
-// that can interfere with caching
-const removeLinksTransformer: TranslatorConfigObject = {
-  a: ({ node, options, visitor }) => {
-    return {
-      postprocess: ({ content }) => {
-        return `[${content}]()`
-      },
-    }
-  },
-  img: ({ node, options, visitor }) => {
-    const alt = node.getAttribute('alt')?.trim()
-    return {
-      content: `![${alt}]()`,
-    }
-  },
-}
-
-export const createSummarizableDocument = (readable: string): string => {
-  const nhm = new NodeHtmlMarkdown(
-    {
-      keepDataImages: false,
-    },
-    removeLinksTransformer
-  )
-  return nhm.translate(readable)
-}
-
-export const createAISummary = async (
-  readableContent: string
-): Promise<string | undefined> => {
-  const llm = new ChatOpenAI({
-    configuration: {
-      apiKey: process.env.OPENAI_API_KEY,
-    },
-  })
-  const textSplitter = new RecursiveCharacterTextSplitter({
-    chunkSize: 12000,
-  })
-
-  const document = createSummarizableDocument(readableContent)
-  const docs = await textSplitter.createDocuments([document])
-  const chain = loadSummarizationChain(llm, {
-    type: 'map_reduce', // you can choose from map_reduce, stuff or refine
-    verbose: true, // to view the steps in the console
-  })
-  const response = await chain.invoke({
-    input_documents: docs,
-  })
-
-  console.log('summary response: ', JSON.stringify(response))
-
-  if (typeof response.text !== 'string') {
-    logger.error(`AI summary did not return text`)
-    return
-  }
-
-  return response.text
-}
 
 export const getAISummary = async (data: {
   userId: string