prehistoric-systems/omnivore
1
0
Fork 0
mirror of https://github.com/omnivore-app/omnivore.git synced 2026-03-11 08:54:26 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Revert WIP changes

Browse source
This commit is contained in:
Jackson Harper 2024-03-19 13:06:25 +08:00
parent 55b6f8197c
commit 72e78cd465
2 changed files with 5 additions and 76 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

15
packages/api/src/jobs/ai-summarize.ts
View file

@ -4,13 +4,10 @@ import { ChatOpenAI } from '@langchain/openai'
import { RecursiveCharacterTextSplitter } from 'langchain/text_splitter'
import { authTrx } from '../repository'
import { libraryItemRepository } from '../repository/library_item'
import { htmlToMarkdown } from '../utils/parser'
import { AISummary } from '../entity/AISummary'
import { LibraryItemState } from '../entity/library_item'
import {
createSummarizableDocument,
getAISummary,
} from '../services/ai-summaries'
import { NodeHtmlMarkdown, TranslatorConfigObject } from 'node-html-markdown'
import { getAISummary } from '../services/ai-summaries'
export interface AISummarizeJobData {
userId: string
@ -58,21 +55,19 @@ export const aiSummarize = async (jobData: AISummarizeJobData) => {
},
})
const textSplitter = new RecursiveCharacterTextSplitter({
chunkSize: 12000,
chunkSize: 2000,
})
const document = createSummarizableDocument(libraryItem.readableContent)
const document = htmlToMarkdown(libraryItem.readableContent)
const docs = await textSplitter.createDocuments([document])
const chain = loadSummarizationChain(llm, {
type: 'map_reduce', // you can choose from map_reduce, stuff or refine
verbose: true, // to view the steps in the console
})
const response = await chain.invoke({
const response = await chain.call({
input_documents: docs,
})
console.log('summary response: ', JSON.stringify(response))
if (typeof response.text !== 'string') {
logger.error(`AI summary did not return text`)
return

66
packages/api/src/services/ai-summaries.ts
View file

@ -1,71 +1,5 @@
import { ChatOpenAI } from '@langchain/openai'
import { AISummary } from '../entity/AISummary'
import { authTrx } from '../repository'
import { RecursiveCharacterTextSplitter } from 'langchain/text_splitter'
import { loadSummarizationChain } from 'langchain/chains'
import { logger } from '../utils/logger'
import { NodeHtmlMarkdown, TranslatorConfigObject } from 'node-html-markdown'
// When creating markdown we remove external links in URLs
// and images since these often contain per-user trackers
// that can interfere with caching
const removeLinksTransformer: TranslatorConfigObject = {
a: ({ node, options, visitor }) => {
return {
postprocess: ({ content }) => {
return `[${content}]()`
},
}
},
img: ({ node, options, visitor }) => {
const alt = node.getAttribute('alt')?.trim()
return {
content: `![${alt}]()`,
}
},
}
export const createSummarizableDocument = (readable: string): string => {
const nhm = new NodeHtmlMarkdown(
{
keepDataImages: false,
},
removeLinksTransformer
)
return nhm.translate(readable)
}
export const createAISummary = async (
readableContent: string
): Promise<string | undefined> => {
const llm = new ChatOpenAI({
configuration: {
apiKey: process.env.OPENAI_API_KEY,
},
})
const textSplitter = new RecursiveCharacterTextSplitter({
chunkSize: 12000,
})
const document = createSummarizableDocument(readableContent)
const docs = await textSplitter.createDocuments([document])
const chain = loadSummarizationChain(llm, {
type: 'map_reduce', // you can choose from map_reduce, stuff or refine
verbose: true, // to view the steps in the console
})
const response = await chain.invoke({
input_documents: docs,
})
console.log('summary response: ', JSON.stringify(response))
if (typeof response.text !== 'string') {
logger.error(`AI summary did not return text`)
return
}
return response.text
}
export const getAISummary = async (data: {
userId: string