From 21335d3ed41e76c5ed5364af3e38ce6b2ee42932 Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Tue, 19 Mar 2024 12:33:40 +0800 Subject: [PATCH 1/3] Fix for pages that dont need auth like terms and conditions page --- packages/api/src/jobs/ai-summarize.ts | 15 +++-- packages/api/src/services/ai-summaries.ts | 66 +++++++++++++++++++ .../components/templates/SettingsLayout.tsx | 1 - packages/web/pages/404.tsx | 6 +- packages/web/pages/500.tsx | 8 +-- packages/web/pages/support.tsx | 6 +- packages/web/pages/terms.tsx | 6 +- 7 files changed, 89 insertions(+), 19 deletions(-) diff --git a/packages/api/src/jobs/ai-summarize.ts b/packages/api/src/jobs/ai-summarize.ts index 974a6b347..3658876b6 100644 --- a/packages/api/src/jobs/ai-summarize.ts +++ b/packages/api/src/jobs/ai-summarize.ts @@ -4,10 +4,13 @@ import { ChatOpenAI } from '@langchain/openai' import { RecursiveCharacterTextSplitter } from 'langchain/text_splitter' import { authTrx } from '../repository' import { libraryItemRepository } from '../repository/library_item' -import { htmlToMarkdown } from '../utils/parser' import { AISummary } from '../entity/AISummary' import { LibraryItemState } from '../entity/library_item' -import { getAISummary } from '../services/ai-summaries' +import { + createSummarizableDocument, + getAISummary, +} from '../services/ai-summaries' +import { NodeHtmlMarkdown, TranslatorConfigObject } from 'node-html-markdown' export interface AISummarizeJobData { userId: string @@ -55,19 +58,21 @@ export const aiSummarize = async (jobData: AISummarizeJobData) => { }, }) const textSplitter = new RecursiveCharacterTextSplitter({ - chunkSize: 2000, + chunkSize: 12000, }) - const document = htmlToMarkdown(libraryItem.readableContent) + const document = createSummarizableDocument(libraryItem.readableContent) const docs = await textSplitter.createDocuments([document]) const chain = loadSummarizationChain(llm, { type: 'map_reduce', // you can choose from map_reduce, stuff or refine verbose: true, // to view the steps in the console }) - const response = await chain.call({ + const response = await chain.invoke({ input_documents: docs, }) + console.log('summary response: ', JSON.stringify(response)) + if (typeof response.text !== 'string') { logger.error(`AI summary did not return text`) return diff --git a/packages/api/src/services/ai-summaries.ts b/packages/api/src/services/ai-summaries.ts index 3c94ca4cd..9b9ebefd8 100644 --- a/packages/api/src/services/ai-summaries.ts +++ b/packages/api/src/services/ai-summaries.ts @@ -1,5 +1,71 @@ +import { ChatOpenAI } from '@langchain/openai' import { AISummary } from '../entity/AISummary' import { authTrx } from '../repository' +import { RecursiveCharacterTextSplitter } from 'langchain/text_splitter' +import { loadSummarizationChain } from 'langchain/chains' +import { logger } from '../utils/logger' +import { NodeHtmlMarkdown, TranslatorConfigObject } from 'node-html-markdown' + +// When creating markdown we remove external links in URLs +// and images since these often contain per-user trackers +// that can interfere with caching +const removeLinksTransformer: TranslatorConfigObject = { + a: ({ node, options, visitor }) => { + return { + postprocess: ({ content }) => { + return `[${content}]()` + }, + } + }, + img: ({ node, options, visitor }) => { + const alt = node.getAttribute('alt')?.trim() + return { + content: `![${alt}]()`, + } + }, +} + +export const createSummarizableDocument = (readable: string): string => { + const nhm = new NodeHtmlMarkdown( + { + keepDataImages: false, + }, + removeLinksTransformer + ) + return nhm.translate(readable) +} + +export const createAISummary = async ( + readableContent: string +): Promise => { + const llm = new ChatOpenAI({ + configuration: { + apiKey: process.env.OPENAI_API_KEY, + }, + }) + const textSplitter = new RecursiveCharacterTextSplitter({ + chunkSize: 12000, + }) + + const document = createSummarizableDocument(readableContent) + const docs = await textSplitter.createDocuments([document]) + const chain = loadSummarizationChain(llm, { + type: 'map_reduce', // you can choose from map_reduce, stuff or refine + verbose: true, // to view the steps in the console + }) + const response = await chain.invoke({ + input_documents: docs, + }) + + console.log('summary response: ', JSON.stringify(response)) + + if (typeof response.text !== 'string') { + logger.error(`AI summary did not return text`) + return + } + + return response.text +} export const getAISummary = async (data: { userId: string diff --git a/packages/web/components/templates/SettingsLayout.tsx b/packages/web/components/templates/SettingsLayout.tsx index 05c91ea8b..dd9be633a 100644 --- a/packages/web/components/templates/SettingsLayout.tsx +++ b/packages/web/components/templates/SettingsLayout.tsx @@ -1,5 +1,4 @@ import { Box, HStack, VStack } from '../elements/LayoutPrimitives' -import { useGetViewerQuery } from '../../lib/networking/queries/useGetViewerQuery' import { navigationCommands } from '../../lib/keyboardShortcuts/navigationShortcuts' import { useKeyboardShortcuts } from '../../lib/keyboardShortcuts/useKeyboardShortcuts' import { useRouter } from 'next/router' diff --git a/packages/web/pages/404.tsx b/packages/web/pages/404.tsx index 80e44ab9d..bdf8e40e4 100644 --- a/packages/web/pages/404.tsx +++ b/packages/web/pages/404.tsx @@ -1,6 +1,6 @@ import Head from 'next/head' import { ErrorLayout } from '../components/templates/ErrorLayout' -import { SettingsLayout } from '../components/templates/SettingsLayout' +import { EmptyLayout } from '../components/templates/EmptyLayout' export default function Custom404(): JSX.Element { return ( @@ -8,9 +8,9 @@ export default function Custom404(): JSX.Element { Page Not Found - + - + ) } diff --git a/packages/web/pages/500.tsx b/packages/web/pages/500.tsx index 75d24835c..e4976faba 100644 --- a/packages/web/pages/500.tsx +++ b/packages/web/pages/500.tsx @@ -1,6 +1,6 @@ import { ErrorLayout } from '../components/templates/ErrorLayout' import Head from 'next/head' -import { SettingsLayout } from '../components/templates/SettingsLayout' +import { EmptyLayout } from '../components/templates/EmptyLayout' export default function Custom500(): JSX.Element { return ( @@ -8,9 +8,9 @@ export default function Custom500(): JSX.Element { An unknown error occurred. - - - + + + ) } diff --git a/packages/web/pages/support.tsx b/packages/web/pages/support.tsx index 98e25f366..edf0d7e09 100644 --- a/packages/web/pages/support.tsx +++ b/packages/web/pages/support.tsx @@ -1,7 +1,7 @@ import { useEffect, useCallback } from 'react' import { Button } from '../components/elements/Button' import { HStack } from '../components/elements/LayoutPrimitives' -import { SettingsLayout } from '../components/templates/SettingsLayout' +import { EmptyLayout } from '../components/templates/EmptyLayout' import { setupAnalytics } from '../lib/analytics' export default function Support(): JSX.Element { @@ -18,7 +18,7 @@ export default function Support(): JSX.Element { }, [initAnalytics]) return ( - + - + ) } diff --git a/packages/web/pages/terms.tsx b/packages/web/pages/terms.tsx index fd267cb75..b018d8d56 100644 --- a/packages/web/pages/terms.tsx +++ b/packages/web/pages/terms.tsx @@ -1,6 +1,6 @@ import { useRouter } from 'next/router' import { TermsAndConditions } from '../components/templates/TermsAndConditions' -import { SettingsLayout } from '../components/templates/SettingsLayout' +import { EmptyLayout } from '../components/templates/DocsLayout' export default function Terms(): JSX.Element { const router = useRouter() @@ -11,9 +11,9 @@ export default function Terms(): JSX.Element { return } else { return ( - + - + ) } } From 55b6f8197cfda363aa71e485253049883bcaf24c Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Tue, 19 Mar 2024 12:39:10 +0800 Subject: [PATCH 2/3] Fixes for empty layout --- .../web/components/templates/EmptyLayout.tsx | 47 +++++++++++++++++++ packages/web/pages/terms.tsx | 2 +- 2 files changed, 48 insertions(+), 1 deletion(-) create mode 100644 packages/web/components/templates/EmptyLayout.tsx diff --git a/packages/web/components/templates/EmptyLayout.tsx b/packages/web/components/templates/EmptyLayout.tsx new file mode 100644 index 000000000..53c655a1d --- /dev/null +++ b/packages/web/components/templates/EmptyLayout.tsx @@ -0,0 +1,47 @@ +import { Box, HStack, VStack } from '../elements/LayoutPrimitives' +import { PageMetaData } from '../patterns/PageMetaData' +import { DEFAULT_HEADER_HEIGHT } from './homeFeed/HeaderSpacer' +import { SettingsDropdown } from './navMenu/SettingsDropdown' + +type EmptyLayoutProps = { + title: string + children: React.ReactNode +} + +export function EmptyLayout(props: EmptyLayoutProps): JSX.Element { + return ( + + + + + + + + + {props.children} + + + + + ) +} diff --git a/packages/web/pages/terms.tsx b/packages/web/pages/terms.tsx index b018d8d56..bed50fe80 100644 --- a/packages/web/pages/terms.tsx +++ b/packages/web/pages/terms.tsx @@ -1,6 +1,6 @@ import { useRouter } from 'next/router' import { TermsAndConditions } from '../components/templates/TermsAndConditions' -import { EmptyLayout } from '../components/templates/DocsLayout' +import { EmptyLayout } from '../components/templates/EmptyLayout' export default function Terms(): JSX.Element { const router = useRouter() From 72e78cd465ade22762d5109828f3b23e5de87740 Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Tue, 19 Mar 2024 13:06:25 +0800 Subject: [PATCH 3/3] Revert WIP changes --- packages/api/src/jobs/ai-summarize.ts | 15 ++---- packages/api/src/services/ai-summaries.ts | 66 ----------------------- 2 files changed, 5 insertions(+), 76 deletions(-) diff --git a/packages/api/src/jobs/ai-summarize.ts b/packages/api/src/jobs/ai-summarize.ts index 3658876b6..974a6b347 100644 --- a/packages/api/src/jobs/ai-summarize.ts +++ b/packages/api/src/jobs/ai-summarize.ts @@ -4,13 +4,10 @@ import { ChatOpenAI } from '@langchain/openai' import { RecursiveCharacterTextSplitter } from 'langchain/text_splitter' import { authTrx } from '../repository' import { libraryItemRepository } from '../repository/library_item' +import { htmlToMarkdown } from '../utils/parser' import { AISummary } from '../entity/AISummary' import { LibraryItemState } from '../entity/library_item' -import { - createSummarizableDocument, - getAISummary, -} from '../services/ai-summaries' -import { NodeHtmlMarkdown, TranslatorConfigObject } from 'node-html-markdown' +import { getAISummary } from '../services/ai-summaries' export interface AISummarizeJobData { userId: string @@ -58,21 +55,19 @@ export const aiSummarize = async (jobData: AISummarizeJobData) => { }, }) const textSplitter = new RecursiveCharacterTextSplitter({ - chunkSize: 12000, + chunkSize: 2000, }) - const document = createSummarizableDocument(libraryItem.readableContent) + const document = htmlToMarkdown(libraryItem.readableContent) const docs = await textSplitter.createDocuments([document]) const chain = loadSummarizationChain(llm, { type: 'map_reduce', // you can choose from map_reduce, stuff or refine verbose: true, // to view the steps in the console }) - const response = await chain.invoke({ + const response = await chain.call({ input_documents: docs, }) - console.log('summary response: ', JSON.stringify(response)) - if (typeof response.text !== 'string') { logger.error(`AI summary did not return text`) return diff --git a/packages/api/src/services/ai-summaries.ts b/packages/api/src/services/ai-summaries.ts index 9b9ebefd8..3c94ca4cd 100644 --- a/packages/api/src/services/ai-summaries.ts +++ b/packages/api/src/services/ai-summaries.ts @@ -1,71 +1,5 @@ -import { ChatOpenAI } from '@langchain/openai' import { AISummary } from '../entity/AISummary' import { authTrx } from '../repository' -import { RecursiveCharacterTextSplitter } from 'langchain/text_splitter' -import { loadSummarizationChain } from 'langchain/chains' -import { logger } from '../utils/logger' -import { NodeHtmlMarkdown, TranslatorConfigObject } from 'node-html-markdown' - -// When creating markdown we remove external links in URLs -// and images since these often contain per-user trackers -// that can interfere with caching -const removeLinksTransformer: TranslatorConfigObject = { - a: ({ node, options, visitor }) => { - return { - postprocess: ({ content }) => { - return `[${content}]()` - }, - } - }, - img: ({ node, options, visitor }) => { - const alt = node.getAttribute('alt')?.trim() - return { - content: `![${alt}]()`, - } - }, -} - -export const createSummarizableDocument = (readable: string): string => { - const nhm = new NodeHtmlMarkdown( - { - keepDataImages: false, - }, - removeLinksTransformer - ) - return nhm.translate(readable) -} - -export const createAISummary = async ( - readableContent: string -): Promise => { - const llm = new ChatOpenAI({ - configuration: { - apiKey: process.env.OPENAI_API_KEY, - }, - }) - const textSplitter = new RecursiveCharacterTextSplitter({ - chunkSize: 12000, - }) - - const document = createSummarizableDocument(readableContent) - const docs = await textSplitter.createDocuments([document]) - const chain = loadSummarizationChain(llm, { - type: 'map_reduce', // you can choose from map_reduce, stuff or refine - verbose: true, // to view the steps in the console - }) - const response = await chain.invoke({ - input_documents: docs, - }) - - console.log('summary response: ', JSON.stringify(response)) - - if (typeof response.text !== 'string') { - logger.error(`AI summary did not return text`) - return - } - - return response.text -} export const getAISummary = async (data: { userId: string