prehistoric-systems/omnivore
1
0
Fork 0
mirror of https://github.com/omnivore-app/omnivore.git synced 2026-03-11 08:54:26 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge pull request #1347 from omnivore-app/fix/puppeteer-timeout

Browse source 
Use auth token to identify client request in rate limiter
This commit is contained in:
Hongbo Wu 2022-10-25 17:21:13 +08:00 committed by GitHub
commit 614c6a2b95
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
2 changed files with 21 additions and 13 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
packages/api/src/server.ts
View file

@ -100,6 +100,14 @@ export const createApp = (): {
max: 50, // Limit each IP to 10 requests per `window` (here, per minute)
standardHeaders: true, // Return rate limit info in the `RateLimit-*` headers
legacyHeaders: false, // Disable the `X-RateLimit-*` headers
keyGenerator: (req) => {
return (
req.header('authorization') ||
// eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
(req.cookies['auth'] as string) ||
req.ip
)
},
})
// Apply the rate limiting middleware to API calls only
app.use('/api/', apiLimiter)

26
packages/puppeteer-parse/index.js
View file

@ -304,19 +304,19 @@ exports.puppeteer = Sentry.GCPFunction.wrapHttpFunction(async (req, res) => {
console.log('error with handler: ', e);
}
var context, page, finalUrl;
if ((!content || !title) && contentType !== 'application/pdf') {
const result = await retrievePage(url)
if (result && result.context) { context = result.context }
if (result && result.page) { page = result.page }
if (result && result.finalUrl) { finalUrl = result.finalUrl }
if (result && result.contentType) { contentType = result.contentType }
console.log('context, page, finalUrl, contentType', context, page, finalUrl, contentType);
} else {
finalUrl = url
}
let context, page, finalUrl;
try {
if ((!content || !title) && contentType !== 'application/pdf') {
const result = await retrievePage(url)
if (result && result.context) { context = result.context }
if (result && result.page) { page = result.page }
if (result && result.finalUrl) { finalUrl = result.finalUrl }
if (result && result.contentType) { contentType = result.contentType }
console.log('context, page, finalUrl, contentType', context, page, finalUrl, contentType);
} else {
finalUrl = url
}
if (contentType === 'application/pdf') {
const uploadedFileId = await uploadPdf(finalUrl, userId, articleSavingRequestId);
const l = await saveUploadedPdf(userId, finalUrl, uploadedFileId, articleSavingRequestId);
@ -641,7 +641,7 @@ async function retrievePage(url) {
});
try {
const response = await page.goto(url, { waitUntil: ['networkidle2'] });
const response = await page.goto(url, { timeout: 8 * 1000, waitUntil: ['networkidle2'] });
const finalUrl = response.url();
const contentType = response.headers()['content-type'];