Merge pull request #3749 from omnivore-app/fix/youtube-model

Some fixes for youtube transcripts

packages/api/package.json

@@ -111,8 +111,7 @@
     "voca": "^1.4.0",
     "winston": "^3.3.3",
     "word-counting": "^1.1.4",
-    "youtubei": "^1.3.4",
-    "youtubei.js": "^9.1.0"
+    "youtubei": "1.3.7"
   },
   "devDependencies": {
     "@babel/register": "^7.14.5",

packages/api/src/jobs/process-youtube-video.ts

@@ -96,7 +96,7 @@ export const createTranscriptHTML = async (
     }
 
     const llm = new OpenAI({
-      modelName: 'gpt-4',
+      modelName: 'gpt-4-0125-preview',
       configuration: {
         apiKey: process.env.OPENAI_API_KEY,
       },
@@ -107,32 +107,11 @@ export const createTranscriptHTML = async (
        {transcriptData}`
     )
     const chain = promptTemplate.pipe(llm)
+    const result = await chain.invoke({
+      transcriptData: transcript.map((item) => item.text).join(' '),
+    })
 
-    let transcriptChunkLength = 0
-    let transcriptChunk: TranscriptProperties[] = []
-    for (const item of transcript) {
-      if (transcriptChunkLength + item.text.length > 8000) {
-        const result = await chain.invoke({
-          transcriptData: transcriptChunk.map((item) => item.text).join(' '),
-        })
-
-        transcriptMarkdown += result
-
-        transcriptChunk = []
-        transcriptChunkLength = 0
-      }
-
-      transcriptChunk.push(item)
-      transcriptChunkLength += item.text.length
-    }
-
-    if (transcriptChunk.length > 0) {
-      const result = await chain.invoke({
-        transcriptData: transcriptChunk.map((item) => item.text).join(' '),
-      })
-
-      transcriptMarkdown += result
-    }
+    transcriptMarkdown = result
   }
 
   // If the LLM didn't give us enough data fallback to the raw template

yarn.lock

@@ -2644,11 +2644,6 @@
   dependencies:
     text-decoding "^1.0.0"
 
-"@fastify/busboy@^2.0.0":
-  version "2.1.1"
-  resolved "https://registry.yarnpkg.com/@fastify/busboy/-/busboy-2.1.1.tgz#b9da6a878a371829a0502c9b6c1c143ef6663f4d"
-  integrity sha512-vBZP4NlzfOlerQTnba4aqZoMhE/a9HY7HRqoOPaETQcSQuWEIyZMHGfVu6w9wGtGK5fED5qRs2DteVCjOH60sA==
-
 "@ffmpeg-installer/darwin-arm64@4.1.5":
   version "4.1.5"
   resolved "https://registry.yarnpkg.com/@ffmpeg-installer/darwin-arm64/-/darwin-arm64-4.1.5.tgz#b7b5c262dd96d1aea4807514e1cdcf6e11f82743"
@@ -19876,13 +19871,6 @@ jest@^27.4.5:
     import-local "^3.0.2"
     jest-cli "^27.5.1"
 
-jintr@^1.1.0:
-  version "1.1.0"
-  resolved "https://registry.yarnpkg.com/jintr/-/jintr-1.1.0.tgz#223a3b07f5e03d410cec6e715c537c8ad1e714c3"
-  integrity sha512-Tu9wk3BpN2v+kb8yT6YBtue+/nbjeLFv4vvVC4PJ7oCidHKbifWhvORrAbQfxVIQZG+67am/mDagpiGSVtvrZg==
-  dependencies:
-    acorn "^8.8.0"
-
 jose@^2.0.5:
   version "2.0.7"
   resolved "https://registry.yarnpkg.com/jose/-/jose-2.0.7.tgz#3aabbaec70bff313c108b9406498a163737b16ba"
@@ -30667,13 +30655,6 @@ undici@^4.9.3:
   resolved "https://registry.yarnpkg.com/undici/-/undici-4.14.1.tgz#7633b143a8a10d6d63335e00511d071e8d52a1d9"
   integrity sha512-WJ+g+XqiZcATcBaUeluCajqy4pEDcQfK1vy+Fo+bC4/mqXI9IIQD/XWHLS70fkGUT6P52Drm7IFslO651OdLPQ==
 
-undici@^5.19.1:
-  version "5.28.3"
-  resolved "https://registry.yarnpkg.com/undici/-/undici-5.28.3.tgz#a731e0eff2c3fcfd41c1169a869062be222d1e5b"
-  integrity sha512-3ItfzbrhDlINjaP0duwnNsKpDQk3acHI3gVJ1z4fmwMK31k5G9OVIAMLSIaP6w4FaGkaAkN6zaQO9LUvZ1t7VA==
-  dependencies:
-    "@fastify/busboy" "^2.0.0"
-
 unfetch@^4.2.0:
   version "4.2.0"
   resolved "https://registry.yarnpkg.com/unfetch/-/unfetch-4.2.0.tgz#7e21b0ef7d363d8d9af0fb929a5555f6ef97a3be"
@@ -32339,19 +32320,10 @@ yocto-queue@^1.0.0:
   resolved "https://registry.yarnpkg.com/yocto-queue/-/yocto-queue-1.0.0.tgz#7f816433fb2cbc511ec8bf7d263c3b58a1a3c251"
   integrity sha512-9bnSc/HEW2uRy67wc+T8UwauLuPJVn28jb+GtJY16iiKWyvmYJRXVT4UamsAEGQfPohgr2q4Tq0sQbQlxTfi1g==
 
-youtubei.js@^9.1.0:
-  version "9.1.0"
-  resolved "https://registry.yarnpkg.com/youtubei.js/-/youtubei.js-9.1.0.tgz#bcf154c9fa21d3c8c1d00a5e10360d0a065c660e"
-  integrity sha512-C5GBJ4LgnS6vGAUkdIdQNOFFb5EZ1p3xBvUELNXmIG3Idr6vxWrKNBNy8ClZT3SuDVXaAJqDgF9b5jvY8lNKcg==
-  dependencies:
-    jintr "^1.1.0"
-    tslib "^2.5.0"
-    undici "^5.19.1"
-
-youtubei@^1.3.4:
-  version "1.3.4"
-  resolved "https://registry.yarnpkg.com/youtubei/-/youtubei-1.3.4.tgz#b9761e33dcc6e0a9569e6628ba1fc48c729636f0"
-  integrity sha512-xN6p2oddcTpreF/ojU2mChwdiUlV+TwwUL6xgP6lXRuxeGS5MokM1tzRdXCgIpxkzYYNNAWpt7xvPuAUQM0PCg==
+youtubei@1.3.7:
+  version "1.3.7"
+  resolved "https://registry.yarnpkg.com/youtubei/-/youtubei-1.3.7.tgz#9cb3cf67b506e09bcf6d6a9cfbdf31ad68948dff"
+  integrity sha512-DBtwjxqDccpDdSzihXltuqXp4NlG4PZoGfilUbbyrfXvRHhMLDEZO91dzTcJdLXwBBv+wovtGr2VA/OikPnw/Q==
   dependencies:
     node-fetch "2.6.7"
     protobufjs "7.2.4"