Fix for pages that dont need auth like terms and conditions page

packages/api/src/jobs/ai-summarize.ts

@@ -4,10 +4,13 @@ import { ChatOpenAI } from '@langchain/openai'
 import { RecursiveCharacterTextSplitter } from 'langchain/text_splitter'
 import { authTrx } from '../repository'
 import { libraryItemRepository } from '../repository/library_item'
-import { htmlToMarkdown } from '../utils/parser'
 import { AISummary } from '../entity/AISummary'
 import { LibraryItemState } from '../entity/library_item'
-import { getAISummary } from '../services/ai-summaries'
+import {
+  createSummarizableDocument,
+  getAISummary,
+} from '../services/ai-summaries'
+import { NodeHtmlMarkdown, TranslatorConfigObject } from 'node-html-markdown'
 
 export interface AISummarizeJobData {
   userId: string
@@ -55,19 +58,21 @@ export const aiSummarize = async (jobData: AISummarizeJobData) => {
       },
     })
     const textSplitter = new RecursiveCharacterTextSplitter({
-      chunkSize: 2000,
+      chunkSize: 12000,
     })
 
-    const document = htmlToMarkdown(libraryItem.readableContent)
+    const document = createSummarizableDocument(libraryItem.readableContent)
     const docs = await textSplitter.createDocuments([document])
     const chain = loadSummarizationChain(llm, {
       type: 'map_reduce', // you can choose from map_reduce, stuff or refine
       verbose: true, // to view the steps in the console
     })
-    const response = await chain.call({
+    const response = await chain.invoke({
       input_documents: docs,
     })
 
+    console.log('summary response: ', JSON.stringify(response))
+
     if (typeof response.text !== 'string') {
       logger.error(`AI summary did not return text`)
       return

packages/api/src/services/ai-summaries.ts

@@ -1,5 +1,71 @@
+import { ChatOpenAI } from '@langchain/openai'
 import { AISummary } from '../entity/AISummary'
 import { authTrx } from '../repository'
+import { RecursiveCharacterTextSplitter } from 'langchain/text_splitter'
+import { loadSummarizationChain } from 'langchain/chains'
+import { logger } from '../utils/logger'
+import { NodeHtmlMarkdown, TranslatorConfigObject } from 'node-html-markdown'
+
+// When creating markdown we remove external links in URLs
+// and images since these often contain per-user trackers
+// that can interfere with caching
+const removeLinksTransformer: TranslatorConfigObject = {
+  a: ({ node, options, visitor }) => {
+    return {
+      postprocess: ({ content }) => {
+        return `[${content}]()`
+      },
+    }
+  },
+  img: ({ node, options, visitor }) => {
+    const alt = node.getAttribute('alt')?.trim()
+    return {
+      content: `![${alt}]()`,
+    }
+  },
+}
+
+export const createSummarizableDocument = (readable: string): string => {
+  const nhm = new NodeHtmlMarkdown(
+    {
+      keepDataImages: false,
+    },
+    removeLinksTransformer
+  )
+  return nhm.translate(readable)
+}
+
+export const createAISummary = async (
+  readableContent: string
+): Promise<string | undefined> => {
+  const llm = new ChatOpenAI({
+    configuration: {
+      apiKey: process.env.OPENAI_API_KEY,
+    },
+  })
+  const textSplitter = new RecursiveCharacterTextSplitter({
+    chunkSize: 12000,
+  })
+
+  const document = createSummarizableDocument(readableContent)
+  const docs = await textSplitter.createDocuments([document])
+  const chain = loadSummarizationChain(llm, {
+    type: 'map_reduce', // you can choose from map_reduce, stuff or refine
+    verbose: true, // to view the steps in the console
+  })
+  const response = await chain.invoke({
+    input_documents: docs,
+  })
+
+  console.log('summary response: ', JSON.stringify(response))
+
+  if (typeof response.text !== 'string') {
+    logger.error(`AI summary did not return text`)
+    return
+  }
+
+  return response.text
+}
 
 export const getAISummary = async (data: {
   userId: string

packages/web/components/templates/SettingsLayout.tsx

@@ -1,5 +1,4 @@
 import { Box, HStack, VStack } from '../elements/LayoutPrimitives'
-import { useGetViewerQuery } from '../../lib/networking/queries/useGetViewerQuery'
 import { navigationCommands } from '../../lib/keyboardShortcuts/navigationShortcuts'
 import { useKeyboardShortcuts } from '../../lib/keyboardShortcuts/useKeyboardShortcuts'
 import { useRouter } from 'next/router'

packages/web/pages/404.tsx

@@ -1,6 +1,6 @@
 import Head from 'next/head'
 import { ErrorLayout } from '../components/templates/ErrorLayout'
-import { SettingsLayout } from '../components/templates/SettingsLayout'
+import { EmptyLayout } from '../components/templates/EmptyLayout'
 
 export default function Custom404(): JSX.Element {
   return (
@@ -8,9 +8,9 @@ export default function Custom404(): JSX.Element {
       <Head>
         <title>Page Not Found</title>
       </Head>
-      <SettingsLayout title="Page could not be found">
+      <EmptyLayout title="Page could not be found">
         <ErrorLayout statusCode={404} message="This page could not be found." />
-      </SettingsLayout>
+      </EmptyLayout>
     </>
   )
 }

packages/web/pages/500.tsx

@@ -1,6 +1,6 @@
 import { ErrorLayout } from '../components/templates/ErrorLayout'
 import Head from 'next/head'
-import { SettingsLayout } from '../components/templates/SettingsLayout'
+import { EmptyLayout } from '../components/templates/EmptyLayout'
 
 export default function Custom500(): JSX.Element {
   return (
@@ -8,9 +8,9 @@ export default function Custom500(): JSX.Element {
       <Head>
         <title>An unknown error occurred.</title>
       </Head>
-      <SettingsLayout title="An unknown error occurred">
-       <ErrorLayout statusCode={404} message="An unknown error occurred." />
-      </SettingsLayout>
+      <EmptyLayout title="An unknown error occurred">
+        <ErrorLayout statusCode={404} message="An unknown error occurred." />
+      </EmptyLayout>
     </>
   )
 }

packages/web/pages/support.tsx

@@ -1,7 +1,7 @@
 import { useEffect, useCallback } from 'react'
 import { Button } from '../components/elements/Button'
 import { HStack } from '../components/elements/LayoutPrimitives'
-import { SettingsLayout } from '../components/templates/SettingsLayout'
+import { EmptyLayout } from '../components/templates/EmptyLayout'
 import { setupAnalytics } from '../lib/analytics'
 
 export default function Support(): JSX.Element {
@@ -18,7 +18,7 @@ export default function Support(): JSX.Element {
   }, [initAnalytics])
 
   return (
-    <SettingsLayout title="Support">
+    <EmptyLayout title="Support">
       <HStack
         alignment="center"
         distribution="end"
@@ -46,6 +46,6 @@ export default function Support(): JSX.Element {
           {'Open Chat Window'}
         </Button>
       </HStack>
-    </SettingsLayout>
+    </EmptyLayout>
   )
 }

packages/web/pages/terms.tsx

@@ -1,6 +1,6 @@
 import { useRouter } from 'next/router'
 import { TermsAndConditions } from '../components/templates/TermsAndConditions'
-import { SettingsLayout } from '../components/templates/SettingsLayout'
+import { EmptyLayout } from '../components/templates/DocsLayout'
 
 export default function Terms(): JSX.Element {
   const router = useRouter()
@@ -11,9 +11,9 @@ export default function Terms(): JSX.Element {
     return <TermsAndConditions />
   } else {
     return (
-      <SettingsLayout title="Terms and Conditions">
+      <EmptyLayout title="Terms and Conditions">
         <TermsAndConditions />
-      </SettingsLayout>
+      </EmptyLayout>
     )
   }
 }