From 0cc4daecfe766b9cbd8226c9b9ab1b5c4b36edd4 Mon Sep 17 00:00:00 2001 From: Renzo Robles Date: Thu, 4 May 2023 12:02:08 -0500 Subject: [PATCH] - Adding streaming support --- package-lock.json | 35 +++++++++++++++++++++++++++ package.json | 3 +++ src/routes/ChatRoute.tsx | 33 ++++++++++---------------- src/utils/openai.ts | 51 ++++++++++++++++++++++++++++++++++++++++ 4 files changed, 101 insertions(+), 21 deletions(-) diff --git a/package-lock.json b/package-lock.json index a92396d..8a3c8b8 100644 --- a/package-lock.json +++ b/package-lock.json @@ -23,10 +23,12 @@ "downloadjs": "^1.4.7", "eslint": "8.36.0", "eslint-config-next": "13.2.4", + "gpt-token-utils": "^1.2.0", "lodash": "^4.17.21", "nanoid": "^4.0.1", "next": "13.2.4", "openai": "^3.2.1", + "openai-ext": "^1.2.6", "react": "^18.2.0", "react-dom": "^18.2.0", "react-icons": "^4.8.0", @@ -42,6 +44,7 @@ "@types/react-dom": "^18.0.11", "buffer": "^5.7.1", "parcel": "^2.8.3", + "path-browserify": "^1.0.1", "process": "^0.11.10" } }, @@ -4474,6 +4477,11 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/gpt-token-utils": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/gpt-token-utils/-/gpt-token-utils-1.2.0.tgz", + "integrity": "sha512-s8twaU38UE2Vp65JhQEjz8qvWhWY8KZYvmvYHapxlPT03Ok35Clq+gm9eE27wQILdFisseMVRSiC5lJR9GBklA==" + }, "node_modules/graceful-fs": { "version": "4.2.10", "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.10.tgz", @@ -6681,6 +6689,11 @@ "form-data": "^4.0.0" } }, + "node_modules/openai-ext": { + "version": "1.2.6", + "resolved": "https://registry.npmjs.org/openai-ext/-/openai-ext-1.2.6.tgz", + "integrity": "sha512-bva4Ql5G1lwBGeIBu0vI3mI9FeTyW/l2b7cq5kdEYasujzSAQxxqN9rPi4lSvjNuKofp3C0ravUMdTZkUtKlxw==" + }, "node_modules/optionator": { "version": "0.9.1", "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.9.1.tgz", @@ -6791,6 +6804,12 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/path-browserify": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/path-browserify/-/path-browserify-1.0.1.tgz", + "integrity": "sha512-b7uo2UCUOYZcnF/3ID0lulOJi/bafxa1xPe7ZPsammBSpjSWQkjNxlt635YGS2MiR9GjvuXCtz2emr3jbsz98g==", + "dev": true + }, "node_modules/path-exists": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz", @@ -11457,6 +11476,11 @@ "get-intrinsic": "^1.1.3" } }, + "gpt-token-utils": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/gpt-token-utils/-/gpt-token-utils-1.2.0.tgz", + "integrity": "sha512-s8twaU38UE2Vp65JhQEjz8qvWhWY8KZYvmvYHapxlPT03Ok35Clq+gm9eE27wQILdFisseMVRSiC5lJR9GBklA==" + }, "graceful-fs": { "version": "4.2.10", "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.10.tgz", @@ -12838,6 +12862,11 @@ "form-data": "^4.0.0" } }, + "openai-ext": { + "version": "1.2.6", + "resolved": "https://registry.npmjs.org/openai-ext/-/openai-ext-1.2.6.tgz", + "integrity": "sha512-bva4Ql5G1lwBGeIBu0vI3mI9FeTyW/l2b7cq5kdEYasujzSAQxxqN9rPi4lSvjNuKofp3C0ravUMdTZkUtKlxw==" + }, "optionator": { "version": "0.9.1", "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.9.1.tgz", @@ -12914,6 +12943,12 @@ "lines-and-columns": "^1.1.6" } }, + "path-browserify": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/path-browserify/-/path-browserify-1.0.1.tgz", + "integrity": "sha512-b7uo2UCUOYZcnF/3ID0lulOJi/bafxa1xPe7ZPsammBSpjSWQkjNxlt635YGS2MiR9GjvuXCtz2emr3jbsz98g==", + "dev": true + }, "path-exists": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz", diff --git a/package.json b/package.json index a7d5644..c104ce8 100644 --- a/package.json +++ b/package.json @@ -15,6 +15,7 @@ "@types/react-dom": "^18.0.11", "buffer": "^5.7.1", "parcel": "^2.8.3", + "path-browserify": "^1.0.1", "process": "^0.11.10" }, "dependencies": { @@ -35,10 +36,12 @@ "downloadjs": "^1.4.7", "eslint": "8.36.0", "eslint-config-next": "13.2.4", + "gpt-token-utils": "^1.2.0", "lodash": "^4.17.21", "nanoid": "^4.0.1", "next": "13.2.4", "openai": "^3.2.1", + "openai-ext": "^1.2.6", "react": "^18.2.0", "react-dom": "^18.2.0", "react-icons": "^4.8.0", diff --git a/src/routes/ChatRoute.tsx b/src/routes/ChatRoute.tsx index 9f467ce..6bcf8a0 100644 --- a/src/routes/ChatRoute.tsx +++ b/src/routes/ChatRoute.tsx @@ -26,7 +26,7 @@ import { writingStyles, writingTones, } from "../utils/constants"; -import { createChatCompletion } from "../utils/openai"; +import { createChatCompletion, createStreamChatCompletion } from "../utils/openai"; export function ChatRoute() { const chatId = useChatId(); @@ -96,7 +96,16 @@ export function ChatRoute() { }); setContent(""); - const result = await createChatCompletion(apiKey, [ + const messageId = nanoid() + await db.messages.add({ + id: messageId, + chatId, + content: "█", + role: "assistant", + createdAt: new Date(), + }); + + await createStreamChatCompletion(apiKey, [ { role: "system", content: getSystemMessage(), @@ -106,28 +115,10 @@ export function ChatRoute() { content: message.content, })), { role: "user", content }, - ]); + ], chatId, messageId); - const assistantMessage = result.data.choices[0].message?.content; - if (result.data.usage) { - await db.chats.where({ id: chatId }).modify((chat) => { - if (chat.totalTokens) { - chat.totalTokens += result.data.usage!.total_tokens; - } else { - chat.totalTokens = result.data.usage!.total_tokens; - } - }); - } setSubmitting(false); - await db.messages.add({ - id: nanoid(), - chatId, - content: assistantMessage ?? "unknown reponse", - role: "assistant", - createdAt: new Date(), - }); - if (chat?.description === "New Chat") { const messages = await db.messages .where({ chatId }) diff --git a/src/utils/openai.ts b/src/utils/openai.ts index e0d5cce..f068501 100644 --- a/src/utils/openai.ts +++ b/src/utils/openai.ts @@ -1,6 +1,8 @@ import { ChatCompletionRequestMessage, Configuration, OpenAIApi } from "openai"; import { db } from "../db"; import { defaultModel } from "./constants"; +import { OpenAIExt } from "openai-ext"; +import { encode } from 'gpt-token-utils' function getClient(apiKey: string) { const configuration = new Configuration({ @@ -9,6 +11,55 @@ function getClient(apiKey: string) { return new OpenAIApi(configuration); } +export async function createStreamChatCompletion( + apiKey: string, + messages: ChatCompletionRequestMessage[], + chatId: string, + messageId: string, +) { + const settings = await db.settings.get("general"); + const model = settings?.openAiModel ?? defaultModel; + + return OpenAIExt.streamClientChatCompletion( + { + model, + messages + }, + { + apiKey: apiKey, + handler: { + onContent(content, isFinal, stream) { + setStreamContent(messageId, content, isFinal); + if(isFinal){ + setTotalTokens(chatId, content); + } + }, + onDone(stream) { + }, + onError(error, stream) { + console.error(error); + }, + }, + } + ); +} + +function setStreamContent(messageId:string, content:string, isFinal:boolean){ + content = (isFinal ? content : content + "█") + db.messages.update(messageId, {content: content}) +} + +function setTotalTokens(chatId:string, content:string){ + let total_tokens = encode(content).length; + db.chats.where({ id: chatId }).modify((chat) => { + if (chat.totalTokens) { + chat.totalTokens += total_tokens; + } else { + chat.totalTokens = total_tokens; + } + }); +} + export async function createChatCompletion( apiKey: string, messages: ChatCompletionRequestMessage[]