diff --git a/packages/api/package.json b/packages/api/package.json index 7d96bda57..afac267fb 100644 --- a/packages/api/package.json +++ b/packages/api/package.json @@ -73,6 +73,7 @@ "knex": "2.4.2", "knex-stringcase": "^1.4.2", "linkedom": "^0.14.9", + "lodash": "^4.17.21", "luxon": "^3.2.1", "microsoft-cognitiveservices-speech-sdk": "^1.22.0", "nanoid": "^3.1.25", @@ -117,7 +118,6 @@ "@types/highlightjs": "^9.12.2", "@types/intercom-client": "^2.11.8", "@types/jsonwebtoken": "^8.5.0", - "@types/lodash.debounce": "^4.0.6", "@types/luxon": "^1.25.0", "@types/mocha": "^8.2.2", "@types/nanoid": "^3.0.0", diff --git a/packages/api/src/datalayer/helpers.ts b/packages/api/src/datalayer/helpers.ts index 7768d0667..ea0ba4334 100644 --- a/packages/api/src/datalayer/helpers.ts +++ b/packages/api/src/datalayer/helpers.ts @@ -144,11 +144,11 @@ export const edgeLoader = < result.push(keyMap[key] || []) } if (result.length !== keys.length) { - console.error('DataModel error: count mismatch ', keys, result) + logger.error('DataModel error: count mismatch ', keys, result) } return result } catch (e) { - console.error('DataModel error: ', e) + logger.error('DataModel error: ', e) throw e } }, diff --git a/packages/api/src/datalayer/pubsub.ts b/packages/api/src/datalayer/pubsub.ts index d472e8282..82eff598b 100644 --- a/packages/api/src/datalayer/pubsub.ts +++ b/packages/api/src/datalayer/pubsub.ts @@ -2,13 +2,16 @@ import { PubSub } from '@google-cloud/pubsub' import { env } from '../env' import { ReportType } from '../generated/graphql' import express from 'express' +import { buildLogger } from '../utils/logger' + +const logger = buildLogger('pubsub') const client = new PubSub() export const createPubSubClient = (): PubsubClient => { const publish = (topicName: string, msg: Buffer): Promise => { if (env.dev.isLocal) { - console.log(`Publishing ${topicName}`) + logger.info(`Publishing ${topicName}`) return Promise.resolve() } @@ -16,7 +19,7 @@ export const createPubSubClient = (): PubsubClient => { .topic(topicName) .publishMessage({ data: msg }) .catch((err) => { - console.error(`[PubSub] error: ${topicName}`, err) + logger.error(`[PubSub] error: ${topicName}`, err) }) .then(() => { return Promise.resolve() @@ -126,13 +129,13 @@ export const readPushSubscription = ( req: express.Request ): { message: string | undefined; expired: boolean } => { if (req.query.token !== process.env.PUBSUB_VERIFICATION_TOKEN) { - console.log('query does not include valid pubsub token') + logger.info('query does not include valid pubsub token') return { message: undefined, expired: false } } // GCP PubSub sends the request as a base64 encoded string if (!('message' in req.body)) { - console.log('Invalid pubsub message: message not in body') + logger.info('Invalid pubsub message: message not in body') return { message: undefined, expired: false } } diff --git a/packages/api/src/elastic/highlights.ts b/packages/api/src/elastic/highlights.ts index f2d04d58b..8885bfc1b 100644 --- a/packages/api/src/elastic/highlights.ts +++ b/packages/api/src/elastic/highlights.ts @@ -1,5 +1,6 @@ import { errors } from '@elastic/elasticsearch' import { EntityType } from '../datalayer/pubsub' +import { buildLogger } from '../utils/logger' import { SortBy, SortOrder, SortParams } from '../utils/search' import { client, INDEX_ALIAS } from './index' import { @@ -11,6 +12,8 @@ import { SearchResponse, } from './types' +const logger = buildLogger('elasticsearch') + export const addHighlightToPage = async ( id: string, highlight: Highlight, @@ -52,10 +55,10 @@ export const addHighlightToPage = async ( e instanceof errors.ResponseError && e.message === 'document_missing_exception' ) { - console.log('page has been deleted', id) + logger.info('page has been deleted', id) return false } - console.error('failed to add highlight to a page in elastic', e) + logger.error('failed to add highlight to a page in elastic', e) return false } } @@ -90,7 +93,7 @@ export const getHighlightById = async ( // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access,@typescript-eslint/no-unsafe-return return body.hits.hits[0].inner_hits.highlights.hits.hits[0]._source } catch (e) { - console.error('failed to get highlight from a page in elastic', e) + logger.error('failed to get highlight from a page in elastic', e) return undefined } } @@ -146,7 +149,7 @@ export const deleteHighlight = async ( return true } catch (e) { - console.error('failed to delete a highlight in elastic', e) + logger.error('failed to delete a highlight in elastic', e) return false } @@ -223,7 +226,7 @@ export const searchHighlights = async ( ], } - console.log('searching highlights in elastic', JSON.stringify(searchBody)) + logger.info('searching highlights in elastic', JSON.stringify(searchBody)) const response = await client.search>({ index: INDEX_ALIAS, @@ -249,7 +252,7 @@ export const searchHighlights = async ( return [results, response.body.hits.total.value] } catch (e) { - console.error('failed to search highlights in elastic', e) + logger.error('failed to search highlights in elastic', e) return undefined } } @@ -306,7 +309,7 @@ export const updateHighlight = async ( return true } catch (e) { - console.error('failed to update highlight in elastic', e) + logger.error('failed to update highlight in elastic', e) return false } } diff --git a/packages/api/src/elastic/labels.ts b/packages/api/src/elastic/labels.ts index e1035679b..030a0e460 100644 --- a/packages/api/src/elastic/labels.ts +++ b/packages/api/src/elastic/labels.ts @@ -1,8 +1,11 @@ import { errors } from '@elastic/elasticsearch' import { EntityType } from '../datalayer/pubsub' +import { buildLogger } from '../utils/logger' import { client, INDEX_ALIAS } from './index' import { Label, PageContext } from './types' +const logger = buildLogger('elasticsearch') + export const addLabelInPage = async ( pageId: string, label: Label, @@ -46,10 +49,10 @@ export const addLabelInPage = async ( e instanceof errors.ResponseError && e.message === 'document_missing_exception' ) { - console.log('page has been deleted', pageId) + logger.info('page has been deleted', pageId) return false } - console.error('failed to add a label in elastic', e) + logger.error('failed to add a label in elastic', e) return false } } @@ -95,10 +98,10 @@ export const updateLabelsInPage = async ( e instanceof errors.ResponseError && e.message === 'document_missing_exception' ) { - console.log('page has been deleted', pageId) + logger.info('page has been deleted', pageId) return false } - console.error('failed to update labels in elastic', e) + logger.error('failed to update labels in elastic', e) return false } } @@ -174,7 +177,7 @@ export const deleteLabel = async ( return true } catch (e) { - console.error('failed to delete a label in elastic', e) + logger.error('failed to delete a label in elastic', e) return false } } @@ -264,7 +267,7 @@ export const updateLabel = async ( return true } catch (e) { - console.error('failed to update label in elastic', e) + logger.error('failed to update label in elastic', e) return false } @@ -324,10 +327,10 @@ export const setLabelsForHighlight = async ( e instanceof errors.ResponseError && e.message === 'document_missing_exception' ) { - console.log('highlight has been deleted', highlightId) + logger.info('highlight has been deleted', highlightId) return false } - console.error('failed to set labels for highlight in elastic', e) + logger.error('failed to set labels for highlight in elastic', e) return false } } diff --git a/packages/api/src/elastic/pages.ts b/packages/api/src/elastic/pages.ts index 978f19401..a028abcdb 100644 --- a/packages/api/src/elastic/pages.ts +++ b/packages/api/src/elastic/pages.ts @@ -3,6 +3,7 @@ import { BuiltQuery, ESBuilder, esBuilder } from 'elastic-ts' import { EntityType } from '../datalayer/pubsub' import { BulkActionType } from '../generated/graphql' import { wordsCount } from '../utils/helpers' +import { buildLogger } from '../utils/logger' import { DateFilter, FieldFilter, @@ -27,6 +28,8 @@ import { SearchResponse, } from './types' +const logger = buildLogger('elasticsearch') + const appendQuery = (builder: ESBuilder, query: string): ESBuilder => { interface Field { field: string @@ -421,7 +424,7 @@ export const createPage = async ( return page.id } catch (e) { - console.error('failed to create a page in elastic', JSON.stringify(e)) + logger.error('failed to create a page in elastic', JSON.stringify(e)) return undefined } } @@ -461,10 +464,10 @@ export const updatePage = async ( e instanceof errors.ResponseError && e.message === 'document_missing_exception' ) { - console.log('page has been deleted', id) + logger.info('page has been deleted', id) return false } - console.error('failed to update a page in elastic', e) + logger.error('failed to update a page in elastic', e) return false } } @@ -486,10 +489,10 @@ export const deletePage = async ( e instanceof errors.ResponseError && e.message === 'document_missing_exception' ) { - console.log('page has been deleted', id) + logger.info('page has been deleted', id) return false } - console.error('failed to delete a page in elastic', e) + logger.error('failed to delete a page in elastic', e) return false } } @@ -530,7 +533,7 @@ export const getPageByParam = async ( id: body.hits.hits[0]._id, } as Page } catch (e) { - console.error('failed to get page by param in elastic', e) + logger.error('failed to get page by param in elastic', e) return undefined } } @@ -550,10 +553,10 @@ export const getPageById = async (id: string): Promise => { } as Page } catch (e) { if (e instanceof errors.ResponseError && e.statusCode === 404) { - console.log('page has been deleted', id) + logger.info('page has been deleted', id) return undefined } - console.error('failed to get page by id in elastic', e) + logger.error('failed to get page by id in elastic', e) return undefined } } @@ -665,7 +668,7 @@ export const searchPages = async ( }) .build() - console.debug('searching pages in elastic', JSON.stringify(body)) + logger.info('searching pages in elastic', JSON.stringify(body)) const response = await client.search, BuiltQuery>({ index: INDEX_ALIAS, body, @@ -685,10 +688,10 @@ export const searchPages = async ( } catch (e) { if (e instanceof errors.ResponseError) { // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access - console.error('failed to search pages in elastic', e.meta.body.error) + logger.error('failed to search pages in elastic', e.meta.body.error) return undefined } - console.error('failed to search pages in elastic', e) + logger.error('failed to search pages in elastic', e) return undefined } } @@ -726,7 +729,7 @@ export const countByCreatedAt = async ( return body.count as number } catch (e) { - console.error('failed to count pages in elastic', e) + logger.error('failed to count pages in elastic', e) return 0 } } @@ -765,7 +768,7 @@ export const deletePagesByParam = async ( return false } catch (e) { - console.error('failed to delete pages by param in elastic', e) + logger.error('failed to delete pages by param in elastic', e) return false } } @@ -823,7 +826,7 @@ export const searchAsYouType = async ( id: hit._id, })) } catch (e) { - console.error('failed to search as you type in elastic', e) + logger.error('failed to search as you type in elastic', e) return [] } @@ -904,7 +907,7 @@ export const updatePages = async ( .rawOption('script', updatedScript) .build() - console.debug('updating pages in elastic', JSON.stringify(searchBody)) + logger.info('updating pages in elastic', JSON.stringify(searchBody)) try { const { body } = await client.updateByQuery({ @@ -920,21 +923,21 @@ export const updatePages = async ( // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access if (body.failures && body.failures.length > 0) { - console.log('failed to update pages in elastic', body.failures) + logger.info('failed to update pages in elastic', body.failures) return null } // TODO: publish entityUpdated events for each page if (async) { - console.log('update pages task started', body.task) + logger.info('update pages task started', body.task) return body.task as string } - console.log('updated pages in elastic', body.updated) + logger.info('updated pages in elastic', body.updated) return body.updated as string } catch (e) { - console.log('failed to update pages in elastic', e) + logger.info('failed to update pages in elastic', e) return null } } diff --git a/packages/api/src/elastic/recommendation.ts b/packages/api/src/elastic/recommendation.ts index daba6a169..dd327dd6f 100644 --- a/packages/api/src/elastic/recommendation.ts +++ b/packages/api/src/elastic/recommendation.ts @@ -1,10 +1,13 @@ +import { buildLogger } from '../utils/logger' +import { createPage, getPageByParam, updatePage } from './pages' import { ArticleSavingRequestStatus, Page, PageContext, Recommendation, } from './types' -import { createPage, getPageByParam, updatePage } from './pages' + +const logger = buildLogger('app.dispatch') export const addRecommendation = async ( ctx: PageContext, @@ -83,6 +86,6 @@ export const addRecommendation = async ( return createPage(newPage, ctx) } catch (err) { - console.error(err) + logger.error(err) } } diff --git a/packages/api/src/resolvers/api_key/index.ts b/packages/api/src/resolvers/api_key/index.ts index 5ef001fc4..94ec3c753 100644 --- a/packages/api/src/resolvers/api_key/index.ts +++ b/packages/api/src/resolvers/api_key/index.ts @@ -1,3 +1,7 @@ +import { ApiKey } from '../../entity/api_key' +import { User } from '../../entity/user' +import { getRepository } from '../../entity/utils' +import { env } from '../../env' import { ApiKeysError, ApiKeysErrorCode, @@ -12,12 +16,8 @@ import { RevokeApiKeySuccess, } from '../../generated/graphql' import { analytics } from '../../utils/analytics' -import { env } from '../../env' -import { authorized } from '../../utils/helpers' -import { getRepository } from '../../entity/utils' -import { User } from '../../entity/user' -import { ApiKey } from '../../entity/api_key' import { generateApiKey, hashApiKey } from '../../utils/auth' +import { authorized } from '../../utils/helpers' export const apiKeysResolver = authorized( async (_, __, { claims: { uid }, log }) => { @@ -103,7 +103,7 @@ export const generateApiKeyResolver = authorized< }, } } catch (error) { - console.error(error) + log.error(error) return { errorCodes: [GenerateApiKeyErrorCode.BadRequest] } } diff --git a/packages/api/src/resolvers/article/index.ts b/packages/api/src/resolvers/article/index.ts index 1c1bd5888..388425518 100644 --- a/packages/api/src/resolvers/article/index.ts +++ b/packages/api/src/resolvers/article/index.ts @@ -426,7 +426,7 @@ export const getArticleResolver: ResolverFn< Record, WithDataSourcesContext, QueryArticleArgs -> = async (_obj, { slug, format }, { claims }, info) => { +> = async (_obj, { slug, format }, { claims, log }, info) => { try { if (!claims?.uid) { return { errorCodes: [ArticleErrorCode.Unauthorized] } @@ -481,7 +481,7 @@ export const getArticleResolver: ResolverFn< article: { ...page, isArchived: !!page.archivedAt, linkId: page.id }, } } catch (error) { - console.log(error) + log.error(error) return { errorCodes: [ArticleErrorCode.BadData] } } } @@ -495,7 +495,7 @@ export const getArticlesResolver = authorized< PaginatedPartialArticles, ArticlesError, QueryArticlesArgs ->(async (_obj, params, { claims }) => { +>(async (_obj, params, { claims, log }) => { const startCursor = params.after || '' const first = params.first || 10 @@ -526,7 +526,7 @@ export const getArticlesResolver = authorized< const hasNextPage = pages.length > first const endCursor = String(start + pages.length - (hasNextPage ? 1 : 0)) - console.log( + log.info( 'start', start, 'returning end cursor', @@ -881,7 +881,7 @@ export const searchResolver = authorized< SearchSuccess, SearchError, QuerySearchArgs ->(async (_obj, params, { claims }) => { +>(async (_obj, params, { claims, log }) => { const startCursor = params.after || '' const first = params.first || 10 @@ -946,7 +946,7 @@ export const searchResolver = authorized< r.content = converter(r.content, r.highlights) } } catch (error) { - console.log('Error converting content', error) + log.error('Error converting content', error) } } diff --git a/packages/api/src/resolvers/article_saving_request/index.ts b/packages/api/src/resolvers/article_saving_request/index.ts index 381a4a440..55f669093 100644 --- a/packages/api/src/resolvers/article_saving_request/index.ts +++ b/packages/api/src/resolvers/article_saving_request/index.ts @@ -28,7 +28,7 @@ export const createArticleSavingRequestResolver = authorized< CreateArticleSavingRequestSuccess, CreateArticleSavingRequestError, MutationCreateArticleSavingRequestArgs ->(async (_, { input: { url } }, { claims, pubsub }) => { +>(async (_, { input: { url } }, { claims, pubsub, log }) => { analytics.track({ userId: claims.uid, event: 'link_saved', @@ -49,7 +49,7 @@ export const createArticleSavingRequestResolver = authorized< articleSavingRequest: request, } } catch (err) { - console.log('error saving article', err) + log.error('error saving article', err) if (isErrorWithCode(err)) { return { errorCodes: [err.errorCode as CreateArticleSavingRequestErrorCode], diff --git a/packages/api/src/resolvers/recommendations/index.ts b/packages/api/src/resolvers/recommendations/index.ts index 44242f534..68765d194 100644 --- a/packages/api/src/resolvers/recommendations/index.ts +++ b/packages/api/src/resolvers/recommendations/index.ts @@ -1,3 +1,9 @@ +import { In } from 'typeorm' +import { getPageByParam } from '../../elastic/pages' +import { Group } from '../../entity/groups/group' +import { User } from '../../entity/user' +import { getRepository } from '../../entity/utils' +import { env } from '../../env' import { CreateGroupError, CreateGroupErrorCode, @@ -32,15 +38,9 @@ import { joinGroup, leaveGroup, } from '../../services/groups' -import { authorized, userDataToUser } from '../../utils/helpers' -import { getRepository } from '../../entity/utils' -import { User } from '../../entity/user' -import { Group } from '../../entity/groups/group' -import { In } from 'typeorm' -import { getPageByParam } from '../../elastic/pages' -import { enqueueRecommendation } from '../../utils/createTask' -import { env } from '../../env' import { analytics } from '../../utils/analytics' +import { enqueueRecommendation } from '../../utils/createTask' +import { authorized, userDataToUser } from '../../utils/helpers' export const createGroupResolver = authorized< CreateGroupSuccess, @@ -236,7 +236,7 @@ export const recommendResolver = authorized< ) .flat() ) - console.log('taskNames', taskNames) + log.info('taskNames', taskNames) return { success: true, diff --git a/packages/api/src/resolvers/upload_files/index.ts b/packages/api/src/resolvers/upload_files/index.ts index bdc82c9dc..180cc96e3 100644 --- a/packages/api/src/resolvers/upload_files/index.ts +++ b/packages/api/src/resolvers/upload_files/index.ts @@ -1,26 +1,26 @@ /* eslint-disable @typescript-eslint/no-unused-vars */ -import { - ResolverFn, - UploadFileRequestResult, - MutationUploadFileRequestArgs, - UploadFileStatus, - UploadFileRequestErrorCode, - ArticleSavingRequestStatus, -} from '../../generated/graphql' -import { WithDataSourcesContext } from '../types' -import { - generateUploadSignedUrl, - generateUploadFilePathName, - getFilePublicUrl, -} from '../../utils/uploads' -import path from 'path' import normalizeUrl from 'normalize-url' -import { analytics } from '../../utils/analytics' -import { env } from '../../env' +import path from 'path' import { createPage, getPageByParam, updatePage } from '../../elastic/pages' import { PageType } from '../../elastic/types' -import { generateSlug } from '../../utils/helpers' +import { env } from '../../env' +import { + ArticleSavingRequestStatus, + MutationUploadFileRequestArgs, + ResolverFn, + UploadFileRequestErrorCode, + UploadFileRequestResult, + UploadFileStatus, +} from '../../generated/graphql' import { validateUrl } from '../../services/create_page_save_request' +import { analytics } from '../../utils/analytics' +import { generateSlug } from '../../utils/helpers' +import { + generateUploadFilePathName, + generateUploadSignedUrl, + getFilePublicUrl, +} from '../../utils/uploads' +import { WithDataSourcesContext } from '../types' const isFileUrl = (url: string): boolean => { const parsedUrl = new URL(url) @@ -40,7 +40,7 @@ export const uploadFileRequestResolver: ResolverFn< WithDataSourcesContext, MutationUploadFileRequestArgs > = async (_obj, { input }, ctx) => { - const { models, kx, claims } = ctx + const { models, kx, claims, log } = ctx let uploadFileData: { id: string | null } = { id: null, } @@ -79,7 +79,7 @@ export const uploadFileRequestResolver: ResolverFn< try { validateUrl(url) } catch (error) { - console.log('illegal file input url', error) + log.info('illegal file input url', error) return { errorCodes: [UploadFileRequestErrorCode.BadInput], } diff --git a/packages/api/src/routers/notification_router.ts b/packages/api/src/routers/notification_router.ts index bab1da46b..464702269 100644 --- a/packages/api/src/routers/notification_router.ts +++ b/packages/api/src/routers/notification_router.ts @@ -1,7 +1,7 @@ import cors from 'cors' import express from 'express' import * as jwt from 'jsonwebtoken' -import debounce from 'lodash.debounce' +import { debounce } from 'lodash' import { env } from '../env' import { Claims } from '../resolvers/types' import { getDeviceTokensByUserId } from '../services/user_device_tokens' diff --git a/packages/api/src/routers/page_router.ts b/packages/api/src/routers/page_router.ts index d3072874b..c6bc2dcfc 100644 --- a/packages/api/src/routers/page_router.ts +++ b/packages/api/src/routers/page_router.ts @@ -50,7 +50,7 @@ export function pageRouter() { // Get the content type from the query params const { url, clientRequestId } = req.query const contentType = req.headers['content-type'] - console.log( + logger.info( 'contentType', contentType, 'url', @@ -64,7 +64,7 @@ export function pageRouter() { !isString(contentType) || !isString(clientRequestId) ) { - console.log( + logger.info( 'creating page from pdf failed', url, contentType, @@ -74,7 +74,7 @@ export function pageRouter() { } if (!validateUuid(clientRequestId)) { - console.log('creating page from pdf failed invalid uuid') + logger.info('creating page from pdf failed invalid uuid') return res.status(400).send({ errorCode: 'BAD_DATA' }) } @@ -110,7 +110,7 @@ export function pageRouter() { }) if (page) { - console.log('updating page') + logger.info('updating page') await updatePage( page.id, { @@ -120,7 +120,7 @@ export function pageRouter() { ctx ) } else { - console.log('creating page') + logger.info('creating page') const pageId = await createPage( { url: signedUrl, @@ -145,7 +145,7 @@ export function pageRouter() { } } - console.log('redirecting to signed URL', signedUrl) + logger.info('redirecting to signed URL', signedUrl) return res.redirect(signedUrl) }) diff --git a/packages/api/src/routers/svc/content.ts b/packages/api/src/routers/svc/content.ts index 30956b323..13ade7a9b 100644 --- a/packages/api/src/routers/svc/content.ts +++ b/packages/api/src/routers/svc/content.ts @@ -2,16 +2,19 @@ /* eslint-disable @typescript-eslint/no-unsafe-assignment */ /* eslint-disable @typescript-eslint/explicit-module-boundary-types */ import express from 'express' +import { setClaims } from '../../datalayer/helpers' +import { kx } from '../../datalayer/knex_config' import { createPubSubClient, readPushSubscription, } from '../../datalayer/pubsub' -import { Page } from '../../elastic/types' import { getPageByParam, updatePage } from '../../elastic/pages' +import { Page } from '../../elastic/types' import { ArticleSavingRequestStatus } from '../../generated/graphql' import { initModels } from '../../server' -import { kx } from '../../datalayer/knex_config' -import { setClaims } from '../../datalayer/helpers' +import { buildLogger } from '../../utils/logger' + +const logger = buildLogger('app.dispatch') interface UpdateContentMessage { fileId: string @@ -25,9 +28,9 @@ export function contentServiceRouter() { const router = express.Router() router.post('/search', async (req, res) => { - console.log('search req', req.query, req.body) + logger.info('search req', req.query, req.body) const { message: msgStr, expired } = readPushSubscription(req) - console.log('read pubsub message', msgStr, 'has expired', expired) + logger.info('read pubsub message', msgStr, 'has expired', expired) if (!msgStr) { res.status(400).send('Bad Request') @@ -35,14 +38,14 @@ export function contentServiceRouter() { } if (expired) { - console.log('discarding expired message') + logger.info('discarding expired message') res.status(200).send('Expired') return } const data = JSON.parse(msgStr) if (!('fileId' in data) || !('content' in data)) { - console.log('No file id or content found in message') + logger.info('No file id or content found in message') res.status(400).send('Bad Request') return } @@ -52,14 +55,14 @@ export function contentServiceRouter() { const parts = msg.fileId.split('/') const fileId = parts && parts.length > 1 ? parts[1] : undefined if (!fileId) { - console.log('No file id found in message') + logger.info('No file id found in message') res.status(400).send('Bad Request') return } const page = await getPageByParam({ uploadFileId: fileId }) if (!page) { - console.log('No upload file found for id:', fileId) + logger.info('No upload file found for id:', fileId) res.status(400).send('Bad Request') return } @@ -80,16 +83,16 @@ export function contentServiceRouter() { await setClaims(tx, page.userId) return models.uploadFile.setFileUploadComplete(fileId, tx) }) - console.log('updated uploadFileData', uploadFileData) + logger.info('updated uploadFileData', uploadFileData) } catch (error) { - console.log('error marking file upload as completed', error) + logger.info('error marking file upload as completed', error) } const result = await updatePage(page.id, pageToUpdate, { pubsub: createPubSubClient(), uid: page.userId, }) - console.log( + logger.info( 'Updating article text', page.id, result, diff --git a/packages/api/src/routers/svc/email_attachment.ts b/packages/api/src/routers/svc/email_attachment.ts index 3f2f2ebac..7a7f967bf 100644 --- a/packages/api/src/routers/svc/email_attachment.ts +++ b/packages/api/src/routers/svc/email_attachment.ts @@ -12,6 +12,7 @@ import { updateReceivedEmail } from '../../services/received_emails' import { analytics } from '../../utils/analytics' import { getClaimsByToken } from '../../utils/auth' import { generateSlug } from '../../utils/helpers' +import { buildLogger } from '../../utils/logger' import { generateUploadFilePathName, generateUploadSignedUrl, @@ -19,12 +20,14 @@ import { makeStorageFilePublic, } from '../../utils/uploads' +const logger = buildLogger('app.dispatch') + export function emailAttachmentRouter() { const router = express.Router() // eslint-disable-next-line @typescript-eslint/no-misused-promises router.post('/upload', async (req, res) => { - console.log('email-attachment/upload') + logger.info('email-attachment/upload') const { email, fileName, contentType } = req.body as { email: string @@ -79,14 +82,14 @@ export function emailAttachmentRouter() { res.status(400).send('BAD REQUEST') } } catch (err) { - console.error(err) + logger.error(err) return res.status(500).send('INTERNAL_SERVER_ERROR') } }) // eslint-disable-next-line @typescript-eslint/no-misused-promises router.post('/create-article', async (req, res) => { - console.log('email-attachment/create-article') + logger.info('email-attachment/create-article') const { email, uploadFileId, subject, receivedEmailId } = req.body as { email: string @@ -176,7 +179,7 @@ export function emailAttachmentRouter() { res.send({ id: pageId }) } catch (err) { - console.log(err) + logger.info(err) res.status(500).send(err) } }) diff --git a/packages/api/src/routers/svc/integrations.ts b/packages/api/src/routers/svc/integrations.ts index 111e0371f..6f16e2a58 100644 --- a/packages/api/src/routers/svc/integrations.ts +++ b/packages/api/src/routers/svc/integrations.ts @@ -259,7 +259,7 @@ export function integrationsServiceRouter() { syncedAt = retrieved.since || Date.now() retrievedData = retrieved.data - console.debug('retrieved data', { + logger.info('retrieved data', { total: offset, size: retrievedData.length, }) diff --git a/packages/api/src/routers/svc/newsletters.ts b/packages/api/src/routers/svc/newsletters.ts index 537293690..88538a967 100644 --- a/packages/api/src/routers/svc/newsletters.ts +++ b/packages/api/src/routers/svc/newsletters.ts @@ -16,6 +16,9 @@ import { import { saveUrlFromEmail } from '../../services/save_url' import { getSubscriptionByNameAndUserId } from '../../services/subscriptions' import { isUrl } from '../../utils/helpers' +import { buildLogger } from '../../utils/logger' + +const logger = buildLogger('app.dispatch') interface SetConfirmationCodeMessage { emailAddress: string @@ -37,10 +40,10 @@ export function newsletterServiceRouter() { // eslint-disable-next-line @typescript-eslint/no-misused-promises router.post('/confirmation', async (req, res) => { - console.log('setConfirmationCode') + logger.info('setConfirmationCode') const { message, expired } = readPushSubscription(req) - console.log('pubsub message:', message, 'expired:', expired) + logger.info('pubsub message:', message, 'expired:', expired) if (!message) { res.status(400).send('Bad Request') @@ -48,7 +51,7 @@ export function newsletterServiceRouter() { } if (expired) { - console.log('discards expired message:', message) + logger.info('discards expired message:', message) res.status(200).send('Expired') return } @@ -58,7 +61,7 @@ export function newsletterServiceRouter() { const data: SetConfirmationCodeMessage = JSON.parse(message) if (!('emailAddress' in data) || !('confirmationCode' in data)) { - console.log('No email address or confirmation code found in message') + logger.info('No email address or confirmation code found in message') res.status(400).send('Bad Request') return } @@ -68,14 +71,14 @@ export function newsletterServiceRouter() { data.confirmationCode ) if (!result) { - console.log('Newsletter email not found', data.emailAddress) + logger.info('Newsletter email not found', data.emailAddress) res.status(200).send('Not Found') return } res.status(200).send('confirmation code set') } catch (e) { - console.log(e) + logger.info(e) if (e instanceof SyntaxError) { // when message is not a valid json string res.status(400).send(e) @@ -87,7 +90,7 @@ export function newsletterServiceRouter() { // eslint-disable-next-line @typescript-eslint/no-misused-promises router.post('/create', async (req, res) => { - console.log('create') + logger.info('create') const { message, expired } = readPushSubscription(req) if (!message) { @@ -96,7 +99,7 @@ export function newsletterServiceRouter() { } if (expired) { - console.log('discards expired message:', message) + logger.info('discards expired message:', message) res.status(200).send('Expired') return } @@ -104,14 +107,14 @@ export function newsletterServiceRouter() { try { const data = JSON.parse(message) as unknown if (!isNewsletterMessage(data)) { - console.log('invalid newsletter message', data) + logger.info('invalid newsletter message', data) return res.status(400).send('Bad Request') } // get user from newsletter email const newsletterEmail = await getNewsletterEmail(data.email) if (!newsletterEmail) { - console.log('newsletter email not found', data.email) + logger.info('newsletter email not found', data.email) return res.status(200).send('Not Found') } @@ -136,14 +139,14 @@ export function newsletterServiceRouter() { newsletterEmail.user.id ) if (existingSubscription?.status === SubscriptionStatus.Unsubscribed) { - console.log('newsletter already unsubscribed:', data.author) + logger.info('newsletter already unsubscribed:', data.author) return res.status(200).send('newsletter already unsubscribed') } // save newsletter instead const result = await saveNewsletterEmail(data, newsletterEmail, saveCtx) if (!result) { - console.log( + logger.info( 'Error creating newsletter link from data', data.email, data.title, @@ -159,7 +162,7 @@ export function newsletterServiceRouter() { res.status(200).send('newsletter created') } catch (e) { - console.log(e) + logger.info(e) if (e instanceof SyntaxError) { // when message is not a valid json string res.status(400).send(e) diff --git a/packages/api/src/routers/svc/rss_feed.ts b/packages/api/src/routers/svc/rss_feed.ts index 54ff86ae9..e08eae4ca 100644 --- a/packages/api/src/routers/svc/rss_feed.ts +++ b/packages/api/src/routers/svc/rss_feed.ts @@ -5,18 +5,21 @@ import { Subscription } from '../../entity/subscription' import { getRepository } from '../../entity/utils' import { SubscriptionStatus, SubscriptionType } from '../../generated/graphql' import { enqueueRssFeedFetch } from '../../utils/createTask' +import { buildLogger } from '../../utils/logger' + +const logger = buildLogger('app.dispatch') export function rssFeedRouter() { const router = express.Router() router.post('/fetchAll', async (req, res) => { - console.log('fetch all rss feeds') + logger.info('fetch all rss feeds') const { message: msgStr, expired } = readPushSubscription(req) - console.log('read pubsub message', msgStr, 'has expired', expired) + logger.info('read pubsub message', msgStr, 'has expired', expired) if (expired) { - console.log('discarding expired message') + logger.info('discarding expired message') return res.status(200).send('Expired') } @@ -37,14 +40,14 @@ export function rssFeedRouter() { try { return enqueueRssFeedFetch(subscription.user.id, subscription) } catch (error) { - console.log('error creating rss feed fetch task', error) + logger.info('error creating rss feed fetch task', error) } }) ) res.send('OK') } catch (error) { - console.log('error fetching rss feeds', error) + logger.info('error fetching rss feeds', error) res.status(500).send('Internal Server Error') } }) diff --git a/packages/api/src/routers/svc/webhooks.ts b/packages/api/src/routers/svc/webhooks.ts index cc6c689f0..5fda1fcd8 100644 --- a/packages/api/src/routers/svc/webhooks.ts +++ b/packages/api/src/routers/svc/webhooks.ts @@ -6,12 +6,15 @@ import { readPushSubscription } from '../../datalayer/pubsub' import { getRepository } from '../../entity/utils' import { Webhook } from '../../entity/webhook' import axios, { Method } from 'axios' +import { buildLogger } from '../../utils/logger' + +const logger = buildLogger('app.dispatch') export function webhooksServiceRouter() { const router = express.Router() router.post('/trigger/:action', async (req, res) => { - console.log('trigger webhook of action', req.params.action) + logger.info('trigger webhook of action', req.params.action) const { message: msgStr, expired } = readPushSubscription(req) if (!msgStr) { @@ -20,7 +23,7 @@ export function webhooksServiceRouter() { } if (expired) { - console.log('discarding expired message') + logger.info('discarding expired message') res.status(200).send('Expired') return } @@ -29,7 +32,7 @@ export function webhooksServiceRouter() { const data = JSON.parse(msgStr) const { userId, type } = data if (!userId || !type) { - console.log('No userId or type found in message') + logger.info('No userId or type found in message') res.status(400).send('Bad Request') return } @@ -44,7 +47,7 @@ export function webhooksServiceRouter() { .getMany() if (webhooks.length <= 0) { - console.log( + logger.info( 'No active webhook found for user', userId, 'and eventType', @@ -64,7 +67,7 @@ export function webhooksServiceRouter() { [type]: data, }) - console.log('triggering webhook', url) + logger.info('triggering webhook', url) await axios.request({ url, method, @@ -77,7 +80,7 @@ export function webhooksServiceRouter() { res.status(200).send('OK') } catch (err) { - console.log('trigger webhook failed', err) + logger.info('trigger webhook failed', err) res.status(500).send(err) } }) diff --git a/packages/api/src/routers/user_router.ts b/packages/api/src/routers/user_router.ts index ebe954c3a..a3ea891e3 100644 --- a/packages/api/src/routers/user_router.ts +++ b/packages/api/src/routers/user_router.ts @@ -37,7 +37,7 @@ export function userRouter() { subject?: string } if (!subject || !body || !from) { - console.log(subject, body, from) + logger.error('Bad Request', subject, body, from) res.status(400).send('Bad Request') return } diff --git a/packages/api/src/services/integrations/pocket.ts b/packages/api/src/services/integrations/pocket.ts index ff3373ba4..177428d73 100644 --- a/packages/api/src/services/integrations/pocket.ts +++ b/packages/api/src/services/integrations/pocket.ts @@ -1,12 +1,15 @@ import axios from 'axios' import { ArticleSavingRequestStatus } from '../../elastic/types' import { env } from '../../env' +import { buildLogger } from '../../utils/logger' import { IntegrationService, RetrievedResult, RetrieveRequest, } from './integration' +const logger = buildLogger('app.dispatch') + interface PocketResponse { status: number // 1 if success complete: number // 1 if all items have been returned @@ -73,7 +76,7 @@ export class PocketIntegration extends IntegrationService { ) return response.data.access_token } catch (error) { - console.log('error validating pocket token', error) + logger.error('error validating pocket token', error) return null } } @@ -102,10 +105,10 @@ export class PocketIntegration extends IntegrationService { headers: this.headers, } ) - console.debug('pocket data', response.data) + return response.data } catch (error) { - console.log('error retrieving pocket data', error) + logger.error('error retrieving pocket data', error) throw new Error('Error retrieving pocket data') } } diff --git a/packages/api/src/services/integrations/readwise.ts b/packages/api/src/services/integrations/readwise.ts index 1f060e87c..a3bfe40ff 100644 --- a/packages/api/src/services/integrations/readwise.ts +++ b/packages/api/src/services/integrations/readwise.ts @@ -48,7 +48,7 @@ export class ReadwiseIntegration extends IntegrationService { }) return response.status === 204 ? token : null } catch (error) { - console.log('error validating readwise token', error) + console.error('error validating readwise token', error) return null } } @@ -66,7 +66,7 @@ export class ReadwiseIntegration extends IntegrationService { // update integration syncedAt if successful if (result) { - console.log('updating integration syncedAt') + console.info('updating integration syncedAt') await getRepository(Integration).update(integration.id, { syncedAt: new Date(), }) @@ -129,14 +129,14 @@ export class ReadwiseIntegration extends IntegrationService { error.response?.status === 429 && retryCount < 3 ) { - console.log('Readwise API rate limit exceeded, retrying...') + console.info('Readwise API rate limit exceeded, retrying...') // wait for Retry-After seconds in the header if rate limited // max retry count is 3 const retryAfter = error.response?.headers['retry-after'] || '10' // default to 10 seconds await wait(parseInt(retryAfter, 10) * 1000) return this.syncWithReadwise(token, highlights, retryCount + 1) } - console.log('Error creating highlights in Readwise', error) + console.error('Error creating highlights in Readwise', error) return false } } diff --git a/packages/api/src/services/save_email.ts b/packages/api/src/services/save_email.ts index fa258ee44..50da3504d 100644 --- a/packages/api/src/services/save_email.ts +++ b/packages/api/src/services/save_email.ts @@ -9,12 +9,15 @@ import { validatedDate, wordsCount, } from '../utils/helpers' +import { buildLogger } from '../utils/logger' import { FAKE_URL_PREFIX, parsePreparedContent, parseUrlMetadata, } from '../utils/parser' +const logger = buildLogger('app.dispatch') + export type SaveContext = { pubsub: PubsubClient uid: string @@ -91,14 +94,14 @@ export const saveEmail = async ( }) if (page) { const result = await updatePage(page.id, { archivedAt: null }, ctx) - console.log('updated page from email', result) + logger.info('updated page from email', result) return page } const pageId = await createPage(articleToSave, ctx) if (!pageId) { - console.log('failed to create new page') + logger.info('failed to create new page') return undefined } @@ -110,9 +113,9 @@ export const saveEmail = async ( slug, articleToSave.content ) - console.debug('Created thumbnail task', taskId) + logger.info('Created thumbnail task', taskId) } catch (e) { - console.log('Failed to create thumbnail task', e) + logger.error('Failed to create thumbnail task', e) } articleToSave.id = pageId diff --git a/packages/api/src/services/save_page.ts b/packages/api/src/services/save_page.ts index cbc025ba1..518e0d8c8 100644 --- a/packages/api/src/services/save_page.ts +++ b/packages/api/src/services/save_page.ts @@ -22,10 +22,13 @@ import { validatedDate, wordsCount, } from '../utils/helpers' +import { buildLogger } from '../utils/logger' import { parsePreparedContent } from '../utils/parser' import { createPageSaveRequest } from './create_page_save_request' import { createLabels } from './labels' +const logger = buildLogger('app.dispatch') + type SaveContext = { pubsub: PubsubClient models: DataModels @@ -179,9 +182,9 @@ export const savePage = async ( slug, articleToSave.content ) - console.debug('Created thumbnail task', taskId) + logger.info('Created thumbnail task', taskId) } catch (e) { - console.log('Failed to create thumbnail task', e) + logger.error('Failed to create thumbnail task', e) } if (parseResult.highlightData) { diff --git a/packages/api/src/services/subscriptions.ts b/packages/api/src/services/subscriptions.ts index 5eadd22ea..d2063b312 100644 --- a/packages/api/src/services/subscriptions.ts +++ b/packages/api/src/services/subscriptions.ts @@ -3,9 +3,12 @@ import { NewsletterEmail } from '../entity/newsletter_email' import { Subscription } from '../entity/subscription' import { getRepository } from '../entity/utils' import { SubscriptionStatus, SubscriptionType } from '../generated/graphql' +import { buildLogger } from '../utils/logger' import { sendEmail } from '../utils/sendEmail' import { createNewsletterEmail } from './newsletters' +const logger = buildLogger('app.dispatch') + interface SaveSubscriptionInput { userId: string name: string @@ -51,13 +54,13 @@ const sendUnsubscribeEmail = async ( }) if (!sent) { - console.log('Failed to send unsubscribe email', unsubscribeMailTo) + logger.info('Failed to send unsubscribe email', unsubscribeMailTo) return false } return true } catch (error) { - console.log('Failed to send unsubscribe email', error) + logger.info('Failed to send unsubscribe email', error) return false } } @@ -71,9 +74,9 @@ const sendUnsubscribeHttpRequest = async (url: string): Promise => { return true } catch (error) { if (axios.isAxiosError(error)) { - console.log('Failed to send unsubscribe http request', error.message) + logger.info('Failed to send unsubscribe http request', error.message) } else { - console.log('Failed to send unsubscribe http request', error) + logger.info('Failed to send unsubscribe http request', error) } return false } @@ -147,7 +150,7 @@ export const unsubscribe = async (subscription: Subscription) => { if (!unsubscribed) { // update subscription status to unsubscribed if failed to unsubscribe - console.log('Failed to unsubscribe', subscription.id) + logger.info('Failed to unsubscribe', subscription.id) return getRepository(Subscription).update(subscription.id, { status: SubscriptionStatus.Unsubscribed, }) @@ -174,11 +177,11 @@ export const unsubscribeAll = async ( try { await unsubscribe(subscription) } catch (error) { - console.log('Failed to unsubscribe', error) + logger.info('Failed to unsubscribe', error) } } } catch (error) { - console.log('Failed to unsubscribe all', error) + logger.info('Failed to unsubscribe all', error) } } @@ -211,7 +214,7 @@ export class SubscribeHandler { // subscribe to newsletter service const subscribedNames = await this._subscribe(newsletterEmail.address) if (subscribedNames.length === 0) { - console.log('Failed to get subscribe response', name) + logger.info('Failed to get subscribe response', name) return null } @@ -229,7 +232,7 @@ export class SubscribeHandler { return Promise.all(newSubscriptions) } catch (error) { - console.log('Failed to handleSubscribe', error) + logger.info('Failed to handleSubscribe', error) return null } } diff --git a/packages/api/src/utils/createTask.ts b/packages/api/src/utils/createTask.ts index 4982c7384..f6d72bcb7 100644 --- a/packages/api/src/utils/createTask.ts +++ b/packages/api/src/utils/createTask.ts @@ -149,13 +149,13 @@ export const createAppEngineTask = async ({ } } - console.log('Sending task:') - console.log(task) + logger.info('Sending task:') + logger.info(task) // Send create task request. const request = { parent: parent, task: task } const [response] = await client.createTask(request) const name = response.name - console.log(`Created task ${name}`) + logger.info(`Created task ${name}`) return name } @@ -246,7 +246,7 @@ export const enqueueParseRequest = async ({ setTimeout(() => { axios.post(env.queue.contentFetchUrl, payload).catch((error) => { console.error(error) - logger.warning( + console.error( `Error occurred while requesting local puppeteer-parse function\nPlease, ensure your function is set up properly and running using "yarn start" from the "/pkg/gcf/puppeteer-parse" folder` ) }) @@ -399,7 +399,7 @@ export const enqueueTextToSpeech = async ({ // Calling the handler function directly. setTimeout(() => { axios.post(taskHandlerUrl, payload).catch((error) => { - logger.error(error) + console.error(error) }) }, 0) return '' @@ -450,7 +450,7 @@ export const enqueueRecommendation = async ( headers, }) .catch((error) => { - logger.error(error) + console.error(error) }) }, 0) return '' diff --git a/packages/api/src/utils/highlightGenerator.ts b/packages/api/src/utils/highlightGenerator.ts index c7057c040..02da8695d 100644 --- a/packages/api/src/utils/highlightGenerator.ts +++ b/packages/api/src/utils/highlightGenerator.ts @@ -3,6 +3,9 @@ import { parseHTML } from 'linkedom' import { nanoid } from 'nanoid' import { v4 as uuidv4 } from 'uuid' import { interpolationSearch } from './interpolationSearch' +import { buildLogger } from './logger' + +const logger = buildLogger('app.dispatch') const highlightTag = 'omnivore_highlight' export const maxHighlightLength = 2000 @@ -72,7 +75,7 @@ function getTextNodesBetween(rootNode: Node, startNode: Node, endNode: Node) { // If the function takes too long, throw an error if (Date.now() - start > maxTime) { const error = new Error('getTextNodes Timeout') - console.error(error) + logger.error(error) throw error } @@ -164,7 +167,7 @@ export const findEmbeddedHighlight = ( suffix: info.suffix, } } catch (error) { - console.error(error) + logger.error(error) return undefined } } @@ -407,7 +410,7 @@ export function getArticleTextNodes( const rootNode = document.getRootNode() return getTextNodesBetween(rootNode, rootNode, rootNode) } catch (error) { - console.error(error) + logger.error(error) return null } } diff --git a/packages/api/src/utils/logger.ts b/packages/api/src/utils/logger.ts index 0feb934c4..cb3a1f048 100644 --- a/packages/api/src/utils/logger.ts +++ b/packages/api/src/utils/logger.ts @@ -1,5 +1,8 @@ /* eslint-disable @typescript-eslint/no-explicit-any */ /* eslint-disable @typescript-eslint/restrict-template-expressions */ +import { LoggingWinston } from '@google-cloud/logging-winston' +import { cloneDeep, isArray, isObject, isString, truncate } from 'lodash' +import { DateTime } from 'luxon' import { config, format, @@ -8,9 +11,7 @@ import { loggers, transports, } from 'winston' -import { LoggingWinston } from '@google-cloud/logging-winston' -import TransportStream = require('winston-transport') -import { DateTime } from 'luxon' +import TransportStream from 'winston-transport' import { ConsoleTransportOptions } from 'winston/lib/winston/transports' import { env } from '../env' @@ -55,12 +56,41 @@ function localConfig(id: string): ConsoleTransportOptions { } } +// truncate any string values in the object to a given length +const truncateObjectDeep = (object: any, length: number): any => { + const copyObj = cloneDeep(object) as never + + const truncateDeep = (obj: any): any => { + if (isString(obj) && obj.length > length) { + return `${truncate(obj, { length })} [truncated]` + } + + if (isArray(obj)) { + return obj.map((i) => truncateDeep(i) as never) + } + + if (isObject(obj)) { + Object.entries(obj).forEach(([key, value]) => { + obj[key as keyof typeof obj] = truncateDeep(value) as never + }) + + return obj + } + + // return everything else untouched + return obj + } + + return truncateDeep(copyObj) +} + class GcpLoggingTransport extends LoggingWinston { log(info: any, callback: (err: Error | null, apiResponse?: any) => void) { - const infoString = JSON.stringify(info) - if (infoString.length > 250000) { - // max size for a log entry is 256KB - info = infoString.substring(0, 256000) + const sizeInfo = JSON.stringify(info).length + if (sizeInfo > 250000) { + // add a console.trace to help debug + console.trace('GCP Logging API payload too large', sizeInfo) + info = truncateObjectDeep(info, 5000) as never // the max length for string values is 5000 } super.log(info, callback) } diff --git a/packages/api/src/utils/parser.ts b/packages/api/src/utils/parser.ts index d3e4abe8d..841e82085 100644 --- a/packages/api/src/utils/parser.ts +++ b/packages/api/src/utils/parser.ts @@ -183,7 +183,7 @@ const getReadabilityResult = async ( return article } } catch (error) { - console.log('parsing error for url', url, error) + logger.info('parsing error for url', url, error) } } @@ -208,7 +208,7 @@ export const parsePreparedContent = async ( const { document, pageInfo } = preparedDocument if (!document) { - console.log('No document') + logger.info('No document') return { canonicalUrl: url, parsedContent: null, @@ -223,7 +223,7 @@ export const parsePreparedContent = async ( pageInfo.contentType && !ALLOWED_CONTENT_TYPES.includes(pageInfo.contentType) ) { - console.log('Not allowed content type', pageInfo.contentType) + logger.info('Not allowed content type', pageInfo.contentType) return { canonicalUrl: url, parsedContent: null, @@ -348,7 +348,7 @@ export const parsePreparedContent = async ( }) logRecord.parseSuccess = true } catch (error) { - console.log('Error parsing content', error) + logger.info('Error parsing content', error) Object.assign(logRecord, { parseSuccess: false, parseError: error, @@ -443,7 +443,7 @@ export const parsePageMetadata = (html: string): Metadata | undefined => { return { title, author, description, previewImage } } catch (e) { - console.log('failed to parse page:', e) + logger.info('failed to parse page:', e) return undefined } } @@ -455,7 +455,7 @@ export const parseUrlMetadata = async ( const res = await axios.get(url) return parsePageMetadata(res.data) } catch (e) { - console.log('failed to get:', url, e) + logger.info('failed to get:', url, e) return undefined } } @@ -635,7 +635,7 @@ export const htmlToHighlightedMarkdown = ( throw new Error('Invalid html content') } } catch (err) { - console.log(err) + logger.info(err) return nhm.translate(/* html */ html) } @@ -655,7 +655,7 @@ export const htmlToHighlightedMarkdown = ( articleTextNodes ) } catch (err) { - console.log(err) + logger.info(err) } }) html = document.documentElement.outerHTML @@ -674,14 +674,14 @@ export const getDistillerResult = async ( try { const url = process.env.DISTILLER_URL if (!url) { - console.log('No distiller url') + logger.info('No distiller url') return undefined } const exp = Math.floor(Date.now() / 1000) + 60 * 60 // 1 hour const auth = (await signToken({ uid, exp }, env.server.jwtSecret)) as string - console.debug('Parsing by distiller', url) + logger.info('Parsing by distiller', url) const response = await axios.post(url, html, { headers: { Authorization: auth, @@ -690,7 +690,7 @@ export const getDistillerResult = async ( }) return response.data } catch (e) { - console.log('Error parsing by distiller', e) + logger.info('Error parsing by distiller', e) return undefined } } diff --git a/packages/api/test/services/save_newsletter_email.test.ts b/packages/api/test/services/save_newsletter_email.test.ts index af7a0f217..1e369d289 100644 --- a/packages/api/test/services/save_newsletter_email.test.ts +++ b/packages/api/test/services/save_newsletter_email.test.ts @@ -109,8 +109,8 @@ describe('saveNewsletterEmail', () => { }) it('does not create a subscription if no unsubscribe header', async () => { - const url = 'https://blog.omnivore.app/no-unsubscribe' - nock('https://blog.omnivore.app').get('/no-unsubscribe').reply(404) + const url = 'https://omnivore.app/no_url?q=no-unsubscribe' + nock('https://omnivore.app').get('/no_url?q=no-unsubscribe').reply(404) await saveNewsletterEmail( {