prehistoric-systems/omnivore
1
0
Fork 0
mirror of https://github.com/omnivore-app/omnivore.git synced 2026-03-11 08:54:26 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Fix logRecord being shared by requests

Browse source
This commit is contained in:
Hongbo Wu 2022-11-17 11:06:56 +08:00
parent d9237723c5
commit c18217e697
2 changed files with 12 additions and 31 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

29
packages/content-handler/src/websites/twitter-handler.ts
View file

@ -2,7 +2,7 @@ import { ContentHandler, PreHandleResult } from '../content-handler'
import axios from 'axios'
import { DateTime } from 'luxon'
import _ from 'underscore'
import { Browser } from 'puppeteer-core'
import { Browser, BrowserContext } from 'puppeteer-core'
interface TweetIncludes {
users: {
@ -209,28 +209,9 @@ const getTweetIds = async (
): Promise<string[]> => {
const pageURL = `https://twitter.com/${author}/status/${tweetId}`
// // Modify this variable to control the size of viewport
// const factor = 0.2
// const height = Math.floor(2000 / factor)
// const width = Math.floor(1700 / factor)
//
// const browser = await puppeteer.launch({
// executablePath: process.env.CHROMIUM_PATH,
// headless: !!process.env.LAUNCH_HEADLESS,
// defaultViewport: {
// width,
// height,
// },
// args: [
// `--force-device-scale-factor=${factor}`,
// `--window-size=${width},${height}`,
// '--no-sandbox',
// '--disable-setuid-sandbox',
// ],
// })
const context = await browser.createIncognitoBrowserContext()
let context: BrowserContext | undefined
try {
context = await browser.createIncognitoBrowserContext()
const page = await context.newPage()
await page.goto(pageURL, {
@ -295,7 +276,9 @@ const getTweetIds = async (
console.log(error)
return []
} finally {
await context.close()
if (context) {
await context.close()
}
}
}

14
packages/puppeteer-parse/index.js
View file

@ -117,8 +117,6 @@ const getBrowserPromise = (async () => {
});
})();
let logRecord, functionStartTime;
const uploadToSignedUrl = async ({ id, uploadSignedUrl }, contentType, contentObjUrl) => {
const stream = await axios.get(contentObjUrl, { responseType: 'stream' });
return await axios.put(uploadSignedUrl, stream.data, {
@ -211,13 +209,13 @@ const saveUploadedPdf = async (userId, url, uploadFileId, articleSavingRequestId
};
async function fetchContent(req, res) {
functionStartTime = Date.now();
let functionStartTime = Date.now();
let url = getUrl(req);
const userId = (req.query ? req.query.userId : undefined) || (req.body ? req.body.userId : undefined);
const articleSavingRequestId = (req.query ? req.query.saveRequestId : undefined) || (req.body ? req.body.saveRequestId : undefined);
logRecord = {
let logRecord = {
url,
userId,
articleSavingRequestId,
@ -253,7 +251,7 @@ async function fetchContent(req, res) {
let context, page, finalUrl;
try {
if ((!content || !title) && contentType !== 'application/pdf') {
const result = await retrievePage(url)
const result = await retrievePage(url, logRecord, functionStartTime);
if (result && result.context) { context = result.context }
if (result && result.page) { page = result.page }
if (result && result.finalUrl) { finalUrl = result.finalUrl }
@ -267,7 +265,7 @@ async function fetchContent(req, res) {
const l = await saveUploadedPdf(userId, finalUrl, uploadedFileId, articleSavingRequestId);
} else {
if (!content || !title) {
const result = await retrieveHtml(page);
const result = await retrieveHtml(page, logRecord);
if (result.isBlocked) {
const sbResult = await fetchContentWithScrapingBee(url)
title = sbResult.title
@ -387,7 +385,7 @@ async function blockResources(client) {
await client.send('Network.setBlockedURLs', { urls: blockedResources });
}
async function retrievePage(url) {
async function retrievePage(url, logRecord, functionStartTime) {
validateUrlString(url);
const browser = await getBrowserPromise;
@ -501,7 +499,7 @@ async function retrievePage(url) {
}
}
async function retrieveHtml(page) {
async function retrieveHtml(page, logRecord) {
let domContent = '', title;
try {
title = await page.title();