mirror of
https://github.com/omnivore-app/omnivore.git
synced 2026-03-11 08:54:26 +00:00
Merge pull request #2548 from omnivore-app/fix/node-module-missing
use lodash instead of standalone lodash.debounce package
This commit is contained in:
Hongbo Wu
GitHub
commit
c0da1f6bcf
31 changed files with 261 additions and 186 deletions
|
|
@ -73,6 +73,7 @@
|
|||
"knex": "2.4.2",
|
||||
"knex-stringcase": "^1.4.2",
|
||||
"linkedom": "^0.14.9",
|
||||
"lodash": "^4.17.21",
|
||||
"luxon": "^3.2.1",
|
||||
"microsoft-cognitiveservices-speech-sdk": "^1.22.0",
|
||||
"nanoid": "^3.1.25",
|
||||
|
|
@ -117,7 +118,6 @@
|
|||
"@types/highlightjs": "^9.12.2",
|
||||
"@types/intercom-client": "^2.11.8",
|
||||
"@types/jsonwebtoken": "^8.5.0",
|
||||
"@types/lodash.debounce": "^4.0.6",
|
||||
"@types/luxon": "^1.25.0",
|
||||
"@types/mocha": "^8.2.2",
|
||||
"@types/nanoid": "^3.0.0",
|
||||
|
|
|
|||
|
|
@ -144,11 +144,11 @@ export const edgeLoader = <
|
|||
result.push(keyMap[key] || [])
|
||||
}
|
||||
if (result.length !== keys.length) {
|
||||
console.error('DataModel error: count mismatch ', keys, result)
|
||||
logger.error('DataModel error: count mismatch ', keys, result)
|
||||
}
|
||||
return result
|
||||
} catch (e) {
|
||||
console.error('DataModel error: ', e)
|
||||
logger.error('DataModel error: ', e)
|
||||
throw e
|
||||
}
|
||||
},
|
||||
|
|
|
|||
|
|
@ -2,13 +2,16 @@ import { PubSub } from '@google-cloud/pubsub'
|
|||
import { env } from '../env'
|
||||
import { ReportType } from '../generated/graphql'
|
||||
import express from 'express'
|
||||
import { buildLogger } from '../utils/logger'
|
||||
|
||||
const logger = buildLogger('pubsub')
|
||||
|
||||
const client = new PubSub()
|
||||
|
||||
export const createPubSubClient = (): PubsubClient => {
|
||||
const publish = (topicName: string, msg: Buffer): Promise<void> => {
|
||||
if (env.dev.isLocal) {
|
||||
console.log(`Publishing ${topicName}`)
|
||||
logger.info(`Publishing ${topicName}`)
|
||||
return Promise.resolve()
|
||||
}
|
||||
|
||||
|
|
@ -16,7 +19,7 @@ export const createPubSubClient = (): PubsubClient => {
|
|||
.topic(topicName)
|
||||
.publishMessage({ data: msg })
|
||||
.catch((err) => {
|
||||
console.error(`[PubSub] error: ${topicName}`, err)
|
||||
logger.error(`[PubSub] error: ${topicName}`, err)
|
||||
})
|
||||
.then(() => {
|
||||
return Promise.resolve()
|
||||
|
|
@ -126,13 +129,13 @@ export const readPushSubscription = (
|
|||
req: express.Request
|
||||
): { message: string | undefined; expired: boolean } => {
|
||||
if (req.query.token !== process.env.PUBSUB_VERIFICATION_TOKEN) {
|
||||
console.log('query does not include valid pubsub token')
|
||||
logger.info('query does not include valid pubsub token')
|
||||
return { message: undefined, expired: false }
|
||||
}
|
||||
|
||||
// GCP PubSub sends the request as a base64 encoded string
|
||||
if (!('message' in req.body)) {
|
||||
console.log('Invalid pubsub message: message not in body')
|
||||
logger.info('Invalid pubsub message: message not in body')
|
||||
return { message: undefined, expired: false }
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
import { errors } from '@elastic/elasticsearch'
|
||||
import { EntityType } from '../datalayer/pubsub'
|
||||
import { buildLogger } from '../utils/logger'
|
||||
import { SortBy, SortOrder, SortParams } from '../utils/search'
|
||||
import { client, INDEX_ALIAS } from './index'
|
||||
import {
|
||||
|
|
@ -11,6 +12,8 @@ import {
|
|||
SearchResponse,
|
||||
} from './types'
|
||||
|
||||
const logger = buildLogger('elasticsearch')
|
||||
|
||||
export const addHighlightToPage = async (
|
||||
id: string,
|
||||
highlight: Highlight,
|
||||
|
|
@ -52,10 +55,10 @@ export const addHighlightToPage = async (
|
|||
e instanceof errors.ResponseError &&
|
||||
e.message === 'document_missing_exception'
|
||||
) {
|
||||
console.log('page has been deleted', id)
|
||||
logger.info('page has been deleted', id)
|
||||
return false
|
||||
}
|
||||
console.error('failed to add highlight to a page in elastic', e)
|
||||
logger.error('failed to add highlight to a page in elastic', e)
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
|
@ -90,7 +93,7 @@ export const getHighlightById = async (
|
|||
// eslint-disable-next-line @typescript-eslint/no-unsafe-member-access,@typescript-eslint/no-unsafe-return
|
||||
return body.hits.hits[0].inner_hits.highlights.hits.hits[0]._source
|
||||
} catch (e) {
|
||||
console.error('failed to get highlight from a page in elastic', e)
|
||||
logger.error('failed to get highlight from a page in elastic', e)
|
||||
return undefined
|
||||
}
|
||||
}
|
||||
|
|
@ -146,7 +149,7 @@ export const deleteHighlight = async (
|
|||
|
||||
return true
|
||||
} catch (e) {
|
||||
console.error('failed to delete a highlight in elastic', e)
|
||||
logger.error('failed to delete a highlight in elastic', e)
|
||||
|
||||
return false
|
||||
}
|
||||
|
|
@ -223,7 +226,7 @@ export const searchHighlights = async (
|
|||
],
|
||||
}
|
||||
|
||||
console.log('searching highlights in elastic', JSON.stringify(searchBody))
|
||||
logger.info('searching highlights in elastic', JSON.stringify(searchBody))
|
||||
|
||||
const response = await client.search<SearchResponse<Page>>({
|
||||
index: INDEX_ALIAS,
|
||||
|
|
@ -249,7 +252,7 @@ export const searchHighlights = async (
|
|||
|
||||
return [results, response.body.hits.total.value]
|
||||
} catch (e) {
|
||||
console.error('failed to search highlights in elastic', e)
|
||||
logger.error('failed to search highlights in elastic', e)
|
||||
return undefined
|
||||
}
|
||||
}
|
||||
|
|
@ -306,7 +309,7 @@ export const updateHighlight = async (
|
|||
|
||||
return true
|
||||
} catch (e) {
|
||||
console.error('failed to update highlight in elastic', e)
|
||||
logger.error('failed to update highlight in elastic', e)
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,8 +1,11 @@
|
|||
import { errors } from '@elastic/elasticsearch'
|
||||
import { EntityType } from '../datalayer/pubsub'
|
||||
import { buildLogger } from '../utils/logger'
|
||||
import { client, INDEX_ALIAS } from './index'
|
||||
import { Label, PageContext } from './types'
|
||||
|
||||
const logger = buildLogger('elasticsearch')
|
||||
|
||||
export const addLabelInPage = async (
|
||||
pageId: string,
|
||||
label: Label,
|
||||
|
|
@ -46,10 +49,10 @@ export const addLabelInPage = async (
|
|||
e instanceof errors.ResponseError &&
|
||||
e.message === 'document_missing_exception'
|
||||
) {
|
||||
console.log('page has been deleted', pageId)
|
||||
logger.info('page has been deleted', pageId)
|
||||
return false
|
||||
}
|
||||
console.error('failed to add a label in elastic', e)
|
||||
logger.error('failed to add a label in elastic', e)
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
|
@ -95,10 +98,10 @@ export const updateLabelsInPage = async (
|
|||
e instanceof errors.ResponseError &&
|
||||
e.message === 'document_missing_exception'
|
||||
) {
|
||||
console.log('page has been deleted', pageId)
|
||||
logger.info('page has been deleted', pageId)
|
||||
return false
|
||||
}
|
||||
console.error('failed to update labels in elastic', e)
|
||||
logger.error('failed to update labels in elastic', e)
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
|
@ -174,7 +177,7 @@ export const deleteLabel = async (
|
|||
|
||||
return true
|
||||
} catch (e) {
|
||||
console.error('failed to delete a label in elastic', e)
|
||||
logger.error('failed to delete a label in elastic', e)
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
|
@ -264,7 +267,7 @@ export const updateLabel = async (
|
|||
|
||||
return true
|
||||
} catch (e) {
|
||||
console.error('failed to update label in elastic', e)
|
||||
logger.error('failed to update label in elastic', e)
|
||||
|
||||
return false
|
||||
}
|
||||
|
|
@ -324,10 +327,10 @@ export const setLabelsForHighlight = async (
|
|||
e instanceof errors.ResponseError &&
|
||||
e.message === 'document_missing_exception'
|
||||
) {
|
||||
console.log('highlight has been deleted', highlightId)
|
||||
logger.info('highlight has been deleted', highlightId)
|
||||
return false
|
||||
}
|
||||
console.error('failed to set labels for highlight in elastic', e)
|
||||
logger.error('failed to set labels for highlight in elastic', e)
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ import { BuiltQuery, ESBuilder, esBuilder } from 'elastic-ts'
|
|||
import { EntityType } from '../datalayer/pubsub'
|
||||
import { BulkActionType } from '../generated/graphql'
|
||||
import { wordsCount } from '../utils/helpers'
|
||||
import { buildLogger } from '../utils/logger'
|
||||
import {
|
||||
DateFilter,
|
||||
FieldFilter,
|
||||
|
|
@ -27,6 +28,8 @@ import {
|
|||
SearchResponse,
|
||||
} from './types'
|
||||
|
||||
const logger = buildLogger('elasticsearch')
|
||||
|
||||
const appendQuery = (builder: ESBuilder, query: string): ESBuilder => {
|
||||
interface Field {
|
||||
field: string
|
||||
|
|
@ -421,7 +424,7 @@ export const createPage = async (
|
|||
|
||||
return page.id
|
||||
} catch (e) {
|
||||
console.error('failed to create a page in elastic', JSON.stringify(e))
|
||||
logger.error('failed to create a page in elastic', JSON.stringify(e))
|
||||
return undefined
|
||||
}
|
||||
}
|
||||
|
|
@ -461,10 +464,10 @@ export const updatePage = async (
|
|||
e instanceof errors.ResponseError &&
|
||||
e.message === 'document_missing_exception'
|
||||
) {
|
||||
console.log('page has been deleted', id)
|
||||
logger.info('page has been deleted', id)
|
||||
return false
|
||||
}
|
||||
console.error('failed to update a page in elastic', e)
|
||||
logger.error('failed to update a page in elastic', e)
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
|
@ -486,10 +489,10 @@ export const deletePage = async (
|
|||
e instanceof errors.ResponseError &&
|
||||
e.message === 'document_missing_exception'
|
||||
) {
|
||||
console.log('page has been deleted', id)
|
||||
logger.info('page has been deleted', id)
|
||||
return false
|
||||
}
|
||||
console.error('failed to delete a page in elastic', e)
|
||||
logger.error('failed to delete a page in elastic', e)
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
|
@ -530,7 +533,7 @@ export const getPageByParam = async <K extends keyof ParamSet>(
|
|||
id: body.hits.hits[0]._id,
|
||||
} as Page
|
||||
} catch (e) {
|
||||
console.error('failed to get page by param in elastic', e)
|
||||
logger.error('failed to get page by param in elastic', e)
|
||||
return undefined
|
||||
}
|
||||
}
|
||||
|
|
@ -550,10 +553,10 @@ export const getPageById = async (id: string): Promise<Page | undefined> => {
|
|||
} as Page
|
||||
} catch (e) {
|
||||
if (e instanceof errors.ResponseError && e.statusCode === 404) {
|
||||
console.log('page has been deleted', id)
|
||||
logger.info('page has been deleted', id)
|
||||
return undefined
|
||||
}
|
||||
console.error('failed to get page by id in elastic', e)
|
||||
logger.error('failed to get page by id in elastic', e)
|
||||
return undefined
|
||||
}
|
||||
}
|
||||
|
|
@ -665,7 +668,7 @@ export const searchPages = async (
|
|||
})
|
||||
.build()
|
||||
|
||||
console.debug('searching pages in elastic', JSON.stringify(body))
|
||||
logger.info('searching pages in elastic', JSON.stringify(body))
|
||||
const response = await client.search<SearchResponse<Page>, BuiltQuery>({
|
||||
index: INDEX_ALIAS,
|
||||
body,
|
||||
|
|
@ -685,10 +688,10 @@ export const searchPages = async (
|
|||
} catch (e) {
|
||||
if (e instanceof errors.ResponseError) {
|
||||
// eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
|
||||
console.error('failed to search pages in elastic', e.meta.body.error)
|
||||
logger.error('failed to search pages in elastic', e.meta.body.error)
|
||||
return undefined
|
||||
}
|
||||
console.error('failed to search pages in elastic', e)
|
||||
logger.error('failed to search pages in elastic', e)
|
||||
return undefined
|
||||
}
|
||||
}
|
||||
|
|
@ -726,7 +729,7 @@ export const countByCreatedAt = async (
|
|||
|
||||
return body.count as number
|
||||
} catch (e) {
|
||||
console.error('failed to count pages in elastic', e)
|
||||
logger.error('failed to count pages in elastic', e)
|
||||
return 0
|
||||
}
|
||||
}
|
||||
|
|
@ -765,7 +768,7 @@ export const deletePagesByParam = async <K extends keyof ParamSet>(
|
|||
|
||||
return false
|
||||
} catch (e) {
|
||||
console.error('failed to delete pages by param in elastic', e)
|
||||
logger.error('failed to delete pages by param in elastic', e)
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
|
@ -823,7 +826,7 @@ export const searchAsYouType = async (
|
|||
id: hit._id,
|
||||
}))
|
||||
} catch (e) {
|
||||
console.error('failed to search as you type in elastic', e)
|
||||
logger.error('failed to search as you type in elastic', e)
|
||||
|
||||
return []
|
||||
}
|
||||
|
|
@ -904,7 +907,7 @@ export const updatePages = async (
|
|||
.rawOption('script', updatedScript)
|
||||
.build()
|
||||
|
||||
console.debug('updating pages in elastic', JSON.stringify(searchBody))
|
||||
logger.info('updating pages in elastic', JSON.stringify(searchBody))
|
||||
|
||||
try {
|
||||
const { body } = await client.updateByQuery({
|
||||
|
|
@ -920,21 +923,21 @@ export const updatePages = async (
|
|||
|
||||
// eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
|
||||
if (body.failures && body.failures.length > 0) {
|
||||
console.log('failed to update pages in elastic', body.failures)
|
||||
logger.info('failed to update pages in elastic', body.failures)
|
||||
return null
|
||||
}
|
||||
|
||||
// TODO: publish entityUpdated events for each page
|
||||
|
||||
if (async) {
|
||||
console.log('update pages task started', body.task)
|
||||
logger.info('update pages task started', body.task)
|
||||
return body.task as string
|
||||
}
|
||||
|
||||
console.log('updated pages in elastic', body.updated)
|
||||
logger.info('updated pages in elastic', body.updated)
|
||||
return body.updated as string
|
||||
} catch (e) {
|
||||
console.log('failed to update pages in elastic', e)
|
||||
logger.info('failed to update pages in elastic', e)
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,10 +1,13 @@
|
|||
import { buildLogger } from '../utils/logger'
|
||||
import { createPage, getPageByParam, updatePage } from './pages'
|
||||
import {
|
||||
ArticleSavingRequestStatus,
|
||||
Page,
|
||||
PageContext,
|
||||
Recommendation,
|
||||
} from './types'
|
||||
import { createPage, getPageByParam, updatePage } from './pages'
|
||||
|
||||
const logger = buildLogger('app.dispatch')
|
||||
|
||||
export const addRecommendation = async (
|
||||
ctx: PageContext,
|
||||
|
|
@ -83,6 +86,6 @@ export const addRecommendation = async (
|
|||
|
||||
return createPage(newPage, ctx)
|
||||
} catch (err) {
|
||||
console.error(err)
|
||||
logger.error(err)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,3 +1,7 @@
|
|||
import { ApiKey } from '../../entity/api_key'
|
||||
import { User } from '../../entity/user'
|
||||
import { getRepository } from '../../entity/utils'
|
||||
import { env } from '../../env'
|
||||
import {
|
||||
ApiKeysError,
|
||||
ApiKeysErrorCode,
|
||||
|
|
@ -12,12 +16,8 @@ import {
|
|||
RevokeApiKeySuccess,
|
||||
} from '../../generated/graphql'
|
||||
import { analytics } from '../../utils/analytics'
|
||||
import { env } from '../../env'
|
||||
import { authorized } from '../../utils/helpers'
|
||||
import { getRepository } from '../../entity/utils'
|
||||
import { User } from '../../entity/user'
|
||||
import { ApiKey } from '../../entity/api_key'
|
||||
import { generateApiKey, hashApiKey } from '../../utils/auth'
|
||||
import { authorized } from '../../utils/helpers'
|
||||
|
||||
export const apiKeysResolver = authorized<ApiKeysSuccess, ApiKeysError>(
|
||||
async (_, __, { claims: { uid }, log }) => {
|
||||
|
|
@ -103,7 +103,7 @@ export const generateApiKeyResolver = authorized<
|
|||
},
|
||||
}
|
||||
} catch (error) {
|
||||
console.error(error)
|
||||
log.error(error)
|
||||
|
||||
return { errorCodes: [GenerateApiKeyErrorCode.BadRequest] }
|
||||
}
|
||||
|
|
|
|||
|
|
@ -426,7 +426,7 @@ export const getArticleResolver: ResolverFn<
|
|||
Record<string, unknown>,
|
||||
WithDataSourcesContext,
|
||||
QueryArticleArgs
|
||||
> = async (_obj, { slug, format }, { claims }, info) => {
|
||||
> = async (_obj, { slug, format }, { claims, log }, info) => {
|
||||
try {
|
||||
if (!claims?.uid) {
|
||||
return { errorCodes: [ArticleErrorCode.Unauthorized] }
|
||||
|
|
@ -481,7 +481,7 @@ export const getArticleResolver: ResolverFn<
|
|||
article: { ...page, isArchived: !!page.archivedAt, linkId: page.id },
|
||||
}
|
||||
} catch (error) {
|
||||
console.log(error)
|
||||
log.error(error)
|
||||
return { errorCodes: [ArticleErrorCode.BadData] }
|
||||
}
|
||||
}
|
||||
|
|
@ -495,7 +495,7 @@ export const getArticlesResolver = authorized<
|
|||
PaginatedPartialArticles,
|
||||
ArticlesError,
|
||||
QueryArticlesArgs
|
||||
>(async (_obj, params, { claims }) => {
|
||||
>(async (_obj, params, { claims, log }) => {
|
||||
const startCursor = params.after || ''
|
||||
const first = params.first || 10
|
||||
|
||||
|
|
@ -526,7 +526,7 @@ export const getArticlesResolver = authorized<
|
|||
const hasNextPage = pages.length > first
|
||||
const endCursor = String(start + pages.length - (hasNextPage ? 1 : 0))
|
||||
|
||||
console.log(
|
||||
log.info(
|
||||
'start',
|
||||
start,
|
||||
'returning end cursor',
|
||||
|
|
@ -881,7 +881,7 @@ export const searchResolver = authorized<
|
|||
SearchSuccess,
|
||||
SearchError,
|
||||
QuerySearchArgs
|
||||
>(async (_obj, params, { claims }) => {
|
||||
>(async (_obj, params, { claims, log }) => {
|
||||
const startCursor = params.after || ''
|
||||
const first = params.first || 10
|
||||
|
||||
|
|
@ -946,7 +946,7 @@ export const searchResolver = authorized<
|
|||
r.content = converter(r.content, r.highlights)
|
||||
}
|
||||
} catch (error) {
|
||||
console.log('Error converting content', error)
|
||||
log.error('Error converting content', error)
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -28,7 +28,7 @@ export const createArticleSavingRequestResolver = authorized<
|
|||
CreateArticleSavingRequestSuccess,
|
||||
CreateArticleSavingRequestError,
|
||||
MutationCreateArticleSavingRequestArgs
|
||||
>(async (_, { input: { url } }, { claims, pubsub }) => {
|
||||
>(async (_, { input: { url } }, { claims, pubsub, log }) => {
|
||||
analytics.track({
|
||||
userId: claims.uid,
|
||||
event: 'link_saved',
|
||||
|
|
@ -49,7 +49,7 @@ export const createArticleSavingRequestResolver = authorized<
|
|||
articleSavingRequest: request,
|
||||
}
|
||||
} catch (err) {
|
||||
console.log('error saving article', err)
|
||||
log.error('error saving article', err)
|
||||
if (isErrorWithCode(err)) {
|
||||
return {
|
||||
errorCodes: [err.errorCode as CreateArticleSavingRequestErrorCode],
|
||||
|
|
|
|||
|
|
@ -1,3 +1,9 @@
|
|||
import { In } from 'typeorm'
|
||||
import { getPageByParam } from '../../elastic/pages'
|
||||
import { Group } from '../../entity/groups/group'
|
||||
import { User } from '../../entity/user'
|
||||
import { getRepository } from '../../entity/utils'
|
||||
import { env } from '../../env'
|
||||
import {
|
||||
CreateGroupError,
|
||||
CreateGroupErrorCode,
|
||||
|
|
@ -32,15 +38,9 @@ import {
|
|||
joinGroup,
|
||||
leaveGroup,
|
||||
} from '../../services/groups'
|
||||
import { authorized, userDataToUser } from '../../utils/helpers'
|
||||
import { getRepository } from '../../entity/utils'
|
||||
import { User } from '../../entity/user'
|
||||
import { Group } from '../../entity/groups/group'
|
||||
import { In } from 'typeorm'
|
||||
import { getPageByParam } from '../../elastic/pages'
|
||||
import { enqueueRecommendation } from '../../utils/createTask'
|
||||
import { env } from '../../env'
|
||||
import { analytics } from '../../utils/analytics'
|
||||
import { enqueueRecommendation } from '../../utils/createTask'
|
||||
import { authorized, userDataToUser } from '../../utils/helpers'
|
||||
|
||||
export const createGroupResolver = authorized<
|
||||
CreateGroupSuccess,
|
||||
|
|
@ -236,7 +236,7 @@ export const recommendResolver = authorized<
|
|||
)
|
||||
.flat()
|
||||
)
|
||||
console.log('taskNames', taskNames)
|
||||
log.info('taskNames', taskNames)
|
||||
|
||||
return {
|
||||
success: true,
|
||||
|
|
|
|||
|
|
@ -1,26 +1,26 @@
|
|||
/* eslint-disable @typescript-eslint/no-unused-vars */
|
||||
import {
|
||||
ResolverFn,
|
||||
UploadFileRequestResult,
|
||||
MutationUploadFileRequestArgs,
|
||||
UploadFileStatus,
|
||||
UploadFileRequestErrorCode,
|
||||
ArticleSavingRequestStatus,
|
||||
} from '../../generated/graphql'
|
||||
import { WithDataSourcesContext } from '../types'
|
||||
import {
|
||||
generateUploadSignedUrl,
|
||||
generateUploadFilePathName,
|
||||
getFilePublicUrl,
|
||||
} from '../../utils/uploads'
|
||||
import path from 'path'
|
||||
import normalizeUrl from 'normalize-url'
|
||||
import { analytics } from '../../utils/analytics'
|
||||
import { env } from '../../env'
|
||||
import path from 'path'
|
||||
import { createPage, getPageByParam, updatePage } from '../../elastic/pages'
|
||||
import { PageType } from '../../elastic/types'
|
||||
import { generateSlug } from '../../utils/helpers'
|
||||
import { env } from '../../env'
|
||||
import {
|
||||
ArticleSavingRequestStatus,
|
||||
MutationUploadFileRequestArgs,
|
||||
ResolverFn,
|
||||
UploadFileRequestErrorCode,
|
||||
UploadFileRequestResult,
|
||||
UploadFileStatus,
|
||||
} from '../../generated/graphql'
|
||||
import { validateUrl } from '../../services/create_page_save_request'
|
||||
import { analytics } from '../../utils/analytics'
|
||||
import { generateSlug } from '../../utils/helpers'
|
||||
import {
|
||||
generateUploadFilePathName,
|
||||
generateUploadSignedUrl,
|
||||
getFilePublicUrl,
|
||||
} from '../../utils/uploads'
|
||||
import { WithDataSourcesContext } from '../types'
|
||||
|
||||
const isFileUrl = (url: string): boolean => {
|
||||
const parsedUrl = new URL(url)
|
||||
|
|
@ -40,7 +40,7 @@ export const uploadFileRequestResolver: ResolverFn<
|
|||
WithDataSourcesContext,
|
||||
MutationUploadFileRequestArgs
|
||||
> = async (_obj, { input }, ctx) => {
|
||||
const { models, kx, claims } = ctx
|
||||
const { models, kx, claims, log } = ctx
|
||||
let uploadFileData: { id: string | null } = {
|
||||
id: null,
|
||||
}
|
||||
|
|
@ -79,7 +79,7 @@ export const uploadFileRequestResolver: ResolverFn<
|
|||
try {
|
||||
validateUrl(url)
|
||||
} catch (error) {
|
||||
console.log('illegal file input url', error)
|
||||
log.info('illegal file input url', error)
|
||||
return {
|
||||
errorCodes: [UploadFileRequestErrorCode.BadInput],
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
import cors from 'cors'
|
||||
import express from 'express'
|
||||
import * as jwt from 'jsonwebtoken'
|
||||
import debounce from 'lodash.debounce'
|
||||
import { debounce } from 'lodash'
|
||||
import { env } from '../env'
|
||||
import { Claims } from '../resolvers/types'
|
||||
import { getDeviceTokensByUserId } from '../services/user_device_tokens'
|
||||
|
|
|
|||
|
|
@ -50,7 +50,7 @@ export function pageRouter() {
|
|||
// Get the content type from the query params
|
||||
const { url, clientRequestId } = req.query
|
||||
const contentType = req.headers['content-type']
|
||||
console.log(
|
||||
logger.info(
|
||||
'contentType',
|
||||
contentType,
|
||||
'url',
|
||||
|
|
@ -64,7 +64,7 @@ export function pageRouter() {
|
|||
!isString(contentType) ||
|
||||
!isString(clientRequestId)
|
||||
) {
|
||||
console.log(
|
||||
logger.info(
|
||||
'creating page from pdf failed',
|
||||
url,
|
||||
contentType,
|
||||
|
|
@ -74,7 +74,7 @@ export function pageRouter() {
|
|||
}
|
||||
|
||||
if (!validateUuid(clientRequestId)) {
|
||||
console.log('creating page from pdf failed invalid uuid')
|
||||
logger.info('creating page from pdf failed invalid uuid')
|
||||
return res.status(400).send({ errorCode: 'BAD_DATA' })
|
||||
}
|
||||
|
||||
|
|
@ -110,7 +110,7 @@ export function pageRouter() {
|
|||
})
|
||||
|
||||
if (page) {
|
||||
console.log('updating page')
|
||||
logger.info('updating page')
|
||||
await updatePage(
|
||||
page.id,
|
||||
{
|
||||
|
|
@ -120,7 +120,7 @@ export function pageRouter() {
|
|||
ctx
|
||||
)
|
||||
} else {
|
||||
console.log('creating page')
|
||||
logger.info('creating page')
|
||||
const pageId = await createPage(
|
||||
{
|
||||
url: signedUrl,
|
||||
|
|
@ -145,7 +145,7 @@ export function pageRouter() {
|
|||
}
|
||||
}
|
||||
|
||||
console.log('redirecting to signed URL', signedUrl)
|
||||
logger.info('redirecting to signed URL', signedUrl)
|
||||
return res.redirect(signedUrl)
|
||||
})
|
||||
|
||||
|
|
|
|||
|
|
@ -2,16 +2,19 @@
|
|||
/* eslint-disable @typescript-eslint/no-unsafe-assignment */
|
||||
/* eslint-disable @typescript-eslint/explicit-module-boundary-types */
|
||||
import express from 'express'
|
||||
import { setClaims } from '../../datalayer/helpers'
|
||||
import { kx } from '../../datalayer/knex_config'
|
||||
import {
|
||||
createPubSubClient,
|
||||
readPushSubscription,
|
||||
} from '../../datalayer/pubsub'
|
||||
import { Page } from '../../elastic/types'
|
||||
import { getPageByParam, updatePage } from '../../elastic/pages'
|
||||
import { Page } from '../../elastic/types'
|
||||
import { ArticleSavingRequestStatus } from '../../generated/graphql'
|
||||
import { initModels } from '../../server'
|
||||
import { kx } from '../../datalayer/knex_config'
|
||||
import { setClaims } from '../../datalayer/helpers'
|
||||
import { buildLogger } from '../../utils/logger'
|
||||
|
||||
const logger = buildLogger('app.dispatch')
|
||||
|
||||
interface UpdateContentMessage {
|
||||
fileId: string
|
||||
|
|
@ -25,9 +28,9 @@ export function contentServiceRouter() {
|
|||
const router = express.Router()
|
||||
|
||||
router.post('/search', async (req, res) => {
|
||||
console.log('search req', req.query, req.body)
|
||||
logger.info('search req', req.query, req.body)
|
||||
const { message: msgStr, expired } = readPushSubscription(req)
|
||||
console.log('read pubsub message', msgStr, 'has expired', expired)
|
||||
logger.info('read pubsub message', msgStr, 'has expired', expired)
|
||||
|
||||
if (!msgStr) {
|
||||
res.status(400).send('Bad Request')
|
||||
|
|
@ -35,14 +38,14 @@ export function contentServiceRouter() {
|
|||
}
|
||||
|
||||
if (expired) {
|
||||
console.log('discarding expired message')
|
||||
logger.info('discarding expired message')
|
||||
res.status(200).send('Expired')
|
||||
return
|
||||
}
|
||||
|
||||
const data = JSON.parse(msgStr)
|
||||
if (!('fileId' in data) || !('content' in data)) {
|
||||
console.log('No file id or content found in message')
|
||||
logger.info('No file id or content found in message')
|
||||
res.status(400).send('Bad Request')
|
||||
return
|
||||
}
|
||||
|
|
@ -52,14 +55,14 @@ export function contentServiceRouter() {
|
|||
const parts = msg.fileId.split('/')
|
||||
const fileId = parts && parts.length > 1 ? parts[1] : undefined
|
||||
if (!fileId) {
|
||||
console.log('No file id found in message')
|
||||
logger.info('No file id found in message')
|
||||
res.status(400).send('Bad Request')
|
||||
return
|
||||
}
|
||||
|
||||
const page = await getPageByParam({ uploadFileId: fileId })
|
||||
if (!page) {
|
||||
console.log('No upload file found for id:', fileId)
|
||||
logger.info('No upload file found for id:', fileId)
|
||||
res.status(400).send('Bad Request')
|
||||
return
|
||||
}
|
||||
|
|
@ -80,16 +83,16 @@ export function contentServiceRouter() {
|
|||
await setClaims(tx, page.userId)
|
||||
return models.uploadFile.setFileUploadComplete(fileId, tx)
|
||||
})
|
||||
console.log('updated uploadFileData', uploadFileData)
|
||||
logger.info('updated uploadFileData', uploadFileData)
|
||||
} catch (error) {
|
||||
console.log('error marking file upload as completed', error)
|
||||
logger.info('error marking file upload as completed', error)
|
||||
}
|
||||
|
||||
const result = await updatePage(page.id, pageToUpdate, {
|
||||
pubsub: createPubSubClient(),
|
||||
uid: page.userId,
|
||||
})
|
||||
console.log(
|
||||
logger.info(
|
||||
'Updating article text',
|
||||
page.id,
|
||||
result,
|
||||
|
|
|
|||
|
|
@ -12,6 +12,7 @@ import { updateReceivedEmail } from '../../services/received_emails'
|
|||
import { analytics } from '../../utils/analytics'
|
||||
import { getClaimsByToken } from '../../utils/auth'
|
||||
import { generateSlug } from '../../utils/helpers'
|
||||
import { buildLogger } from '../../utils/logger'
|
||||
import {
|
||||
generateUploadFilePathName,
|
||||
generateUploadSignedUrl,
|
||||
|
|
@ -19,12 +20,14 @@ import {
|
|||
makeStorageFilePublic,
|
||||
} from '../../utils/uploads'
|
||||
|
||||
const logger = buildLogger('app.dispatch')
|
||||
|
||||
export function emailAttachmentRouter() {
|
||||
const router = express.Router()
|
||||
|
||||
// eslint-disable-next-line @typescript-eslint/no-misused-promises
|
||||
router.post('/upload', async (req, res) => {
|
||||
console.log('email-attachment/upload')
|
||||
logger.info('email-attachment/upload')
|
||||
|
||||
const { email, fileName, contentType } = req.body as {
|
||||
email: string
|
||||
|
|
@ -79,14 +82,14 @@ export function emailAttachmentRouter() {
|
|||
res.status(400).send('BAD REQUEST')
|
||||
}
|
||||
} catch (err) {
|
||||
console.error(err)
|
||||
logger.error(err)
|
||||
return res.status(500).send('INTERNAL_SERVER_ERROR')
|
||||
}
|
||||
})
|
||||
|
||||
// eslint-disable-next-line @typescript-eslint/no-misused-promises
|
||||
router.post('/create-article', async (req, res) => {
|
||||
console.log('email-attachment/create-article')
|
||||
logger.info('email-attachment/create-article')
|
||||
|
||||
const { email, uploadFileId, subject, receivedEmailId } = req.body as {
|
||||
email: string
|
||||
|
|
@ -176,7 +179,7 @@ export function emailAttachmentRouter() {
|
|||
|
||||
res.send({ id: pageId })
|
||||
} catch (err) {
|
||||
console.log(err)
|
||||
logger.info(err)
|
||||
res.status(500).send(err)
|
||||
}
|
||||
})
|
||||
|
|
|
|||
|
|
@ -259,7 +259,7 @@ export function integrationsServiceRouter() {
|
|||
syncedAt = retrieved.since || Date.now()
|
||||
retrievedData = retrieved.data
|
||||
|
||||
console.debug('retrieved data', {
|
||||
logger.info('retrieved data', {
|
||||
total: offset,
|
||||
size: retrievedData.length,
|
||||
})
|
||||
|
|
|
|||
|
|
@ -16,6 +16,9 @@ import {
|
|||
import { saveUrlFromEmail } from '../../services/save_url'
|
||||
import { getSubscriptionByNameAndUserId } from '../../services/subscriptions'
|
||||
import { isUrl } from '../../utils/helpers'
|
||||
import { buildLogger } from '../../utils/logger'
|
||||
|
||||
const logger = buildLogger('app.dispatch')
|
||||
|
||||
interface SetConfirmationCodeMessage {
|
||||
emailAddress: string
|
||||
|
|
@ -37,10 +40,10 @@ export function newsletterServiceRouter() {
|
|||
|
||||
// eslint-disable-next-line @typescript-eslint/no-misused-promises
|
||||
router.post('/confirmation', async (req, res) => {
|
||||
console.log('setConfirmationCode')
|
||||
logger.info('setConfirmationCode')
|
||||
|
||||
const { message, expired } = readPushSubscription(req)
|
||||
console.log('pubsub message:', message, 'expired:', expired)
|
||||
logger.info('pubsub message:', message, 'expired:', expired)
|
||||
|
||||
if (!message) {
|
||||
res.status(400).send('Bad Request')
|
||||
|
|
@ -48,7 +51,7 @@ export function newsletterServiceRouter() {
|
|||
}
|
||||
|
||||
if (expired) {
|
||||
console.log('discards expired message:', message)
|
||||
logger.info('discards expired message:', message)
|
||||
res.status(200).send('Expired')
|
||||
return
|
||||
}
|
||||
|
|
@ -58,7 +61,7 @@ export function newsletterServiceRouter() {
|
|||
const data: SetConfirmationCodeMessage = JSON.parse(message)
|
||||
|
||||
if (!('emailAddress' in data) || !('confirmationCode' in data)) {
|
||||
console.log('No email address or confirmation code found in message')
|
||||
logger.info('No email address or confirmation code found in message')
|
||||
res.status(400).send('Bad Request')
|
||||
return
|
||||
}
|
||||
|
|
@ -68,14 +71,14 @@ export function newsletterServiceRouter() {
|
|||
data.confirmationCode
|
||||
)
|
||||
if (!result) {
|
||||
console.log('Newsletter email not found', data.emailAddress)
|
||||
logger.info('Newsletter email not found', data.emailAddress)
|
||||
res.status(200).send('Not Found')
|
||||
return
|
||||
}
|
||||
|
||||
res.status(200).send('confirmation code set')
|
||||
} catch (e) {
|
||||
console.log(e)
|
||||
logger.info(e)
|
||||
if (e instanceof SyntaxError) {
|
||||
// when message is not a valid json string
|
||||
res.status(400).send(e)
|
||||
|
|
@ -87,7 +90,7 @@ export function newsletterServiceRouter() {
|
|||
|
||||
// eslint-disable-next-line @typescript-eslint/no-misused-promises
|
||||
router.post('/create', async (req, res) => {
|
||||
console.log('create')
|
||||
logger.info('create')
|
||||
|
||||
const { message, expired } = readPushSubscription(req)
|
||||
if (!message) {
|
||||
|
|
@ -96,7 +99,7 @@ export function newsletterServiceRouter() {
|
|||
}
|
||||
|
||||
if (expired) {
|
||||
console.log('discards expired message:', message)
|
||||
logger.info('discards expired message:', message)
|
||||
res.status(200).send('Expired')
|
||||
return
|
||||
}
|
||||
|
|
@ -104,14 +107,14 @@ export function newsletterServiceRouter() {
|
|||
try {
|
||||
const data = JSON.parse(message) as unknown
|
||||
if (!isNewsletterMessage(data)) {
|
||||
console.log('invalid newsletter message', data)
|
||||
logger.info('invalid newsletter message', data)
|
||||
return res.status(400).send('Bad Request')
|
||||
}
|
||||
|
||||
// get user from newsletter email
|
||||
const newsletterEmail = await getNewsletterEmail(data.email)
|
||||
if (!newsletterEmail) {
|
||||
console.log('newsletter email not found', data.email)
|
||||
logger.info('newsletter email not found', data.email)
|
||||
return res.status(200).send('Not Found')
|
||||
}
|
||||
|
||||
|
|
@ -136,14 +139,14 @@ export function newsletterServiceRouter() {
|
|||
newsletterEmail.user.id
|
||||
)
|
||||
if (existingSubscription?.status === SubscriptionStatus.Unsubscribed) {
|
||||
console.log('newsletter already unsubscribed:', data.author)
|
||||
logger.info('newsletter already unsubscribed:', data.author)
|
||||
return res.status(200).send('newsletter already unsubscribed')
|
||||
}
|
||||
|
||||
// save newsletter instead
|
||||
const result = await saveNewsletterEmail(data, newsletterEmail, saveCtx)
|
||||
if (!result) {
|
||||
console.log(
|
||||
logger.info(
|
||||
'Error creating newsletter link from data',
|
||||
data.email,
|
||||
data.title,
|
||||
|
|
@ -159,7 +162,7 @@ export function newsletterServiceRouter() {
|
|||
|
||||
res.status(200).send('newsletter created')
|
||||
} catch (e) {
|
||||
console.log(e)
|
||||
logger.info(e)
|
||||
if (e instanceof SyntaxError) {
|
||||
// when message is not a valid json string
|
||||
res.status(400).send(e)
|
||||
|
|
|
|||
|
|
@ -5,18 +5,21 @@ import { Subscription } from '../../entity/subscription'
|
|||
import { getRepository } from '../../entity/utils'
|
||||
import { SubscriptionStatus, SubscriptionType } from '../../generated/graphql'
|
||||
import { enqueueRssFeedFetch } from '../../utils/createTask'
|
||||
import { buildLogger } from '../../utils/logger'
|
||||
|
||||
const logger = buildLogger('app.dispatch')
|
||||
|
||||
export function rssFeedRouter() {
|
||||
const router = express.Router()
|
||||
|
||||
router.post('/fetchAll', async (req, res) => {
|
||||
console.log('fetch all rss feeds')
|
||||
logger.info('fetch all rss feeds')
|
||||
|
||||
const { message: msgStr, expired } = readPushSubscription(req)
|
||||
console.log('read pubsub message', msgStr, 'has expired', expired)
|
||||
logger.info('read pubsub message', msgStr, 'has expired', expired)
|
||||
|
||||
if (expired) {
|
||||
console.log('discarding expired message')
|
||||
logger.info('discarding expired message')
|
||||
return res.status(200).send('Expired')
|
||||
}
|
||||
|
||||
|
|
@ -37,14 +40,14 @@ export function rssFeedRouter() {
|
|||
try {
|
||||
return enqueueRssFeedFetch(subscription.user.id, subscription)
|
||||
} catch (error) {
|
||||
console.log('error creating rss feed fetch task', error)
|
||||
logger.info('error creating rss feed fetch task', error)
|
||||
}
|
||||
})
|
||||
)
|
||||
|
||||
res.send('OK')
|
||||
} catch (error) {
|
||||
console.log('error fetching rss feeds', error)
|
||||
logger.info('error fetching rss feeds', error)
|
||||
res.status(500).send('Internal Server Error')
|
||||
}
|
||||
})
|
||||
|
|
|
|||
|
|
@ -6,12 +6,15 @@ import { readPushSubscription } from '../../datalayer/pubsub'
|
|||
import { getRepository } from '../../entity/utils'
|
||||
import { Webhook } from '../../entity/webhook'
|
||||
import axios, { Method } from 'axios'
|
||||
import { buildLogger } from '../../utils/logger'
|
||||
|
||||
const logger = buildLogger('app.dispatch')
|
||||
|
||||
export function webhooksServiceRouter() {
|
||||
const router = express.Router()
|
||||
|
||||
router.post('/trigger/:action', async (req, res) => {
|
||||
console.log('trigger webhook of action', req.params.action)
|
||||
logger.info('trigger webhook of action', req.params.action)
|
||||
const { message: msgStr, expired } = readPushSubscription(req)
|
||||
|
||||
if (!msgStr) {
|
||||
|
|
@ -20,7 +23,7 @@ export function webhooksServiceRouter() {
|
|||
}
|
||||
|
||||
if (expired) {
|
||||
console.log('discarding expired message')
|
||||
logger.info('discarding expired message')
|
||||
res.status(200).send('Expired')
|
||||
return
|
||||
}
|
||||
|
|
@ -29,7 +32,7 @@ export function webhooksServiceRouter() {
|
|||
const data = JSON.parse(msgStr)
|
||||
const { userId, type } = data
|
||||
if (!userId || !type) {
|
||||
console.log('No userId or type found in message')
|
||||
logger.info('No userId or type found in message')
|
||||
res.status(400).send('Bad Request')
|
||||
return
|
||||
}
|
||||
|
|
@ -44,7 +47,7 @@ export function webhooksServiceRouter() {
|
|||
.getMany()
|
||||
|
||||
if (webhooks.length <= 0) {
|
||||
console.log(
|
||||
logger.info(
|
||||
'No active webhook found for user',
|
||||
userId,
|
||||
'and eventType',
|
||||
|
|
@ -64,7 +67,7 @@ export function webhooksServiceRouter() {
|
|||
[type]: data,
|
||||
})
|
||||
|
||||
console.log('triggering webhook', url)
|
||||
logger.info('triggering webhook', url)
|
||||
await axios.request({
|
||||
url,
|
||||
method,
|
||||
|
|
@ -77,7 +80,7 @@ export function webhooksServiceRouter() {
|
|||
|
||||
res.status(200).send('OK')
|
||||
} catch (err) {
|
||||
console.log('trigger webhook failed', err)
|
||||
logger.info('trigger webhook failed', err)
|
||||
res.status(500).send(err)
|
||||
}
|
||||
})
|
||||
|
|
|
|||
|
|
@ -37,7 +37,7 @@ export function userRouter() {
|
|||
subject?: string
|
||||
}
|
||||
if (!subject || !body || !from) {
|
||||
console.log(subject, body, from)
|
||||
logger.error('Bad Request', subject, body, from)
|
||||
res.status(400).send('Bad Request')
|
||||
return
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,12 +1,15 @@
|
|||
import axios from 'axios'
|
||||
import { ArticleSavingRequestStatus } from '../../elastic/types'
|
||||
import { env } from '../../env'
|
||||
import { buildLogger } from '../../utils/logger'
|
||||
import {
|
||||
IntegrationService,
|
||||
RetrievedResult,
|
||||
RetrieveRequest,
|
||||
} from './integration'
|
||||
|
||||
const logger = buildLogger('app.dispatch')
|
||||
|
||||
interface PocketResponse {
|
||||
status: number // 1 if success
|
||||
complete: number // 1 if all items have been returned
|
||||
|
|
@ -73,7 +76,7 @@ export class PocketIntegration extends IntegrationService {
|
|||
)
|
||||
return response.data.access_token
|
||||
} catch (error) {
|
||||
console.log('error validating pocket token', error)
|
||||
logger.error('error validating pocket token', error)
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
|
@ -102,10 +105,10 @@ export class PocketIntegration extends IntegrationService {
|
|||
headers: this.headers,
|
||||
}
|
||||
)
|
||||
console.debug('pocket data', response.data)
|
||||
|
||||
return response.data
|
||||
} catch (error) {
|
||||
console.log('error retrieving pocket data', error)
|
||||
logger.error('error retrieving pocket data', error)
|
||||
throw new Error('Error retrieving pocket data')
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -48,7 +48,7 @@ export class ReadwiseIntegration extends IntegrationService {
|
|||
})
|
||||
return response.status === 204 ? token : null
|
||||
} catch (error) {
|
||||
console.log('error validating readwise token', error)
|
||||
console.error('error validating readwise token', error)
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
|
@ -66,7 +66,7 @@ export class ReadwiseIntegration extends IntegrationService {
|
|||
|
||||
// update integration syncedAt if successful
|
||||
if (result) {
|
||||
console.log('updating integration syncedAt')
|
||||
console.info('updating integration syncedAt')
|
||||
await getRepository(Integration).update(integration.id, {
|
||||
syncedAt: new Date(),
|
||||
})
|
||||
|
|
@ -129,14 +129,14 @@ export class ReadwiseIntegration extends IntegrationService {
|
|||
error.response?.status === 429 &&
|
||||
retryCount < 3
|
||||
) {
|
||||
console.log('Readwise API rate limit exceeded, retrying...')
|
||||
console.info('Readwise API rate limit exceeded, retrying...')
|
||||
// wait for Retry-After seconds in the header if rate limited
|
||||
// max retry count is 3
|
||||
const retryAfter = error.response?.headers['retry-after'] || '10' // default to 10 seconds
|
||||
await wait(parseInt(retryAfter, 10) * 1000)
|
||||
return this.syncWithReadwise(token, highlights, retryCount + 1)
|
||||
}
|
||||
console.log('Error creating highlights in Readwise', error)
|
||||
console.error('Error creating highlights in Readwise', error)
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -9,12 +9,15 @@ import {
|
|||
validatedDate,
|
||||
wordsCount,
|
||||
} from '../utils/helpers'
|
||||
import { buildLogger } from '../utils/logger'
|
||||
import {
|
||||
FAKE_URL_PREFIX,
|
||||
parsePreparedContent,
|
||||
parseUrlMetadata,
|
||||
} from '../utils/parser'
|
||||
|
||||
const logger = buildLogger('app.dispatch')
|
||||
|
||||
export type SaveContext = {
|
||||
pubsub: PubsubClient
|
||||
uid: string
|
||||
|
|
@ -91,14 +94,14 @@ export const saveEmail = async (
|
|||
})
|
||||
if (page) {
|
||||
const result = await updatePage(page.id, { archivedAt: null }, ctx)
|
||||
console.log('updated page from email', result)
|
||||
logger.info('updated page from email', result)
|
||||
|
||||
return page
|
||||
}
|
||||
|
||||
const pageId = await createPage(articleToSave, ctx)
|
||||
if (!pageId) {
|
||||
console.log('failed to create new page')
|
||||
logger.info('failed to create new page')
|
||||
|
||||
return undefined
|
||||
}
|
||||
|
|
@ -110,9 +113,9 @@ export const saveEmail = async (
|
|||
slug,
|
||||
articleToSave.content
|
||||
)
|
||||
console.debug('Created thumbnail task', taskId)
|
||||
logger.info('Created thumbnail task', taskId)
|
||||
} catch (e) {
|
||||
console.log('Failed to create thumbnail task', e)
|
||||
logger.error('Failed to create thumbnail task', e)
|
||||
}
|
||||
|
||||
articleToSave.id = pageId
|
||||
|
|
|
|||
|
|
@ -22,10 +22,13 @@ import {
|
|||
validatedDate,
|
||||
wordsCount,
|
||||
} from '../utils/helpers'
|
||||
import { buildLogger } from '../utils/logger'
|
||||
import { parsePreparedContent } from '../utils/parser'
|
||||
import { createPageSaveRequest } from './create_page_save_request'
|
||||
import { createLabels } from './labels'
|
||||
|
||||
const logger = buildLogger('app.dispatch')
|
||||
|
||||
type SaveContext = {
|
||||
pubsub: PubsubClient
|
||||
models: DataModels
|
||||
|
|
@ -179,9 +182,9 @@ export const savePage = async (
|
|||
slug,
|
||||
articleToSave.content
|
||||
)
|
||||
console.debug('Created thumbnail task', taskId)
|
||||
logger.info('Created thumbnail task', taskId)
|
||||
} catch (e) {
|
||||
console.log('Failed to create thumbnail task', e)
|
||||
logger.error('Failed to create thumbnail task', e)
|
||||
}
|
||||
|
||||
if (parseResult.highlightData) {
|
||||
|
|
|
|||
|
|
@ -3,9 +3,12 @@ import { NewsletterEmail } from '../entity/newsletter_email'
|
|||
import { Subscription } from '../entity/subscription'
|
||||
import { getRepository } from '../entity/utils'
|
||||
import { SubscriptionStatus, SubscriptionType } from '../generated/graphql'
|
||||
import { buildLogger } from '../utils/logger'
|
||||
import { sendEmail } from '../utils/sendEmail'
|
||||
import { createNewsletterEmail } from './newsletters'
|
||||
|
||||
const logger = buildLogger('app.dispatch')
|
||||
|
||||
interface SaveSubscriptionInput {
|
||||
userId: string
|
||||
name: string
|
||||
|
|
@ -51,13 +54,13 @@ const sendUnsubscribeEmail = async (
|
|||
})
|
||||
|
||||
if (!sent) {
|
||||
console.log('Failed to send unsubscribe email', unsubscribeMailTo)
|
||||
logger.info('Failed to send unsubscribe email', unsubscribeMailTo)
|
||||
return false
|
||||
}
|
||||
|
||||
return true
|
||||
} catch (error) {
|
||||
console.log('Failed to send unsubscribe email', error)
|
||||
logger.info('Failed to send unsubscribe email', error)
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
|
@ -71,9 +74,9 @@ const sendUnsubscribeHttpRequest = async (url: string): Promise<boolean> => {
|
|||
return true
|
||||
} catch (error) {
|
||||
if (axios.isAxiosError(error)) {
|
||||
console.log('Failed to send unsubscribe http request', error.message)
|
||||
logger.info('Failed to send unsubscribe http request', error.message)
|
||||
} else {
|
||||
console.log('Failed to send unsubscribe http request', error)
|
||||
logger.info('Failed to send unsubscribe http request', error)
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
|
@ -147,7 +150,7 @@ export const unsubscribe = async (subscription: Subscription) => {
|
|||
|
||||
if (!unsubscribed) {
|
||||
// update subscription status to unsubscribed if failed to unsubscribe
|
||||
console.log('Failed to unsubscribe', subscription.id)
|
||||
logger.info('Failed to unsubscribe', subscription.id)
|
||||
return getRepository(Subscription).update(subscription.id, {
|
||||
status: SubscriptionStatus.Unsubscribed,
|
||||
})
|
||||
|
|
@ -174,11 +177,11 @@ export const unsubscribeAll = async (
|
|||
try {
|
||||
await unsubscribe(subscription)
|
||||
} catch (error) {
|
||||
console.log('Failed to unsubscribe', error)
|
||||
logger.info('Failed to unsubscribe', error)
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
console.log('Failed to unsubscribe all', error)
|
||||
logger.info('Failed to unsubscribe all', error)
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -211,7 +214,7 @@ export class SubscribeHandler {
|
|||
// subscribe to newsletter service
|
||||
const subscribedNames = await this._subscribe(newsletterEmail.address)
|
||||
if (subscribedNames.length === 0) {
|
||||
console.log('Failed to get subscribe response', name)
|
||||
logger.info('Failed to get subscribe response', name)
|
||||
return null
|
||||
}
|
||||
|
||||
|
|
@ -229,7 +232,7 @@ export class SubscribeHandler {
|
|||
|
||||
return Promise.all(newSubscriptions)
|
||||
} catch (error) {
|
||||
console.log('Failed to handleSubscribe', error)
|
||||
logger.info('Failed to handleSubscribe', error)
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -149,13 +149,13 @@ export const createAppEngineTask = async ({
|
|||
}
|
||||
}
|
||||
|
||||
console.log('Sending task:')
|
||||
console.log(task)
|
||||
logger.info('Sending task:')
|
||||
logger.info(task)
|
||||
// Send create task request.
|
||||
const request = { parent: parent, task: task }
|
||||
const [response] = await client.createTask(request)
|
||||
const name = response.name
|
||||
console.log(`Created task ${name}`)
|
||||
logger.info(`Created task ${name}`)
|
||||
|
||||
return name
|
||||
}
|
||||
|
|
@ -246,7 +246,7 @@ export const enqueueParseRequest = async ({
|
|||
setTimeout(() => {
|
||||
axios.post(env.queue.contentFetchUrl, payload).catch((error) => {
|
||||
console.error(error)
|
||||
logger.warning(
|
||||
console.error(
|
||||
`Error occurred while requesting local puppeteer-parse function\nPlease, ensure your function is set up properly and running using "yarn start" from the "/pkg/gcf/puppeteer-parse" folder`
|
||||
)
|
||||
})
|
||||
|
|
@ -399,7 +399,7 @@ export const enqueueTextToSpeech = async ({
|
|||
// Calling the handler function directly.
|
||||
setTimeout(() => {
|
||||
axios.post(taskHandlerUrl, payload).catch((error) => {
|
||||
logger.error(error)
|
||||
console.error(error)
|
||||
})
|
||||
}, 0)
|
||||
return ''
|
||||
|
|
@ -450,7 +450,7 @@ export const enqueueRecommendation = async (
|
|||
headers,
|
||||
})
|
||||
.catch((error) => {
|
||||
logger.error(error)
|
||||
console.error(error)
|
||||
})
|
||||
}, 0)
|
||||
return ''
|
||||
|
|
|
|||
|
|
@ -3,6 +3,9 @@ import { parseHTML } from 'linkedom'
|
|||
import { nanoid } from 'nanoid'
|
||||
import { v4 as uuidv4 } from 'uuid'
|
||||
import { interpolationSearch } from './interpolationSearch'
|
||||
import { buildLogger } from './logger'
|
||||
|
||||
const logger = buildLogger('app.dispatch')
|
||||
|
||||
const highlightTag = 'omnivore_highlight'
|
||||
export const maxHighlightLength = 2000
|
||||
|
|
@ -72,7 +75,7 @@ function getTextNodesBetween(rootNode: Node, startNode: Node, endNode: Node) {
|
|||
// If the function takes too long, throw an error
|
||||
if (Date.now() - start > maxTime) {
|
||||
const error = new Error('getTextNodes Timeout')
|
||||
console.error(error)
|
||||
logger.error(error)
|
||||
throw error
|
||||
}
|
||||
|
||||
|
|
@ -164,7 +167,7 @@ export const findEmbeddedHighlight = (
|
|||
suffix: info.suffix,
|
||||
}
|
||||
} catch (error) {
|
||||
console.error(error)
|
||||
logger.error(error)
|
||||
return undefined
|
||||
}
|
||||
}
|
||||
|
|
@ -407,7 +410,7 @@ export function getArticleTextNodes(
|
|||
const rootNode = document.getRootNode()
|
||||
return getTextNodesBetween(rootNode, rootNode, rootNode)
|
||||
} catch (error) {
|
||||
console.error(error)
|
||||
logger.error(error)
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,5 +1,8 @@
|
|||
/* eslint-disable @typescript-eslint/no-explicit-any */
|
||||
/* eslint-disable @typescript-eslint/restrict-template-expressions */
|
||||
import { LoggingWinston } from '@google-cloud/logging-winston'
|
||||
import { cloneDeep, isArray, isObject, isString, truncate } from 'lodash'
|
||||
import { DateTime } from 'luxon'
|
||||
import {
|
||||
config,
|
||||
format,
|
||||
|
|
@ -8,9 +11,7 @@ import {
|
|||
loggers,
|
||||
transports,
|
||||
} from 'winston'
|
||||
import { LoggingWinston } from '@google-cloud/logging-winston'
|
||||
import TransportStream = require('winston-transport')
|
||||
import { DateTime } from 'luxon'
|
||||
import TransportStream from 'winston-transport'
|
||||
import { ConsoleTransportOptions } from 'winston/lib/winston/transports'
|
||||
import { env } from '../env'
|
||||
|
||||
|
|
@ -55,12 +56,41 @@ function localConfig(id: string): ConsoleTransportOptions {
|
|||
}
|
||||
}
|
||||
|
||||
// truncate any string values in the object to a given length
|
||||
const truncateObjectDeep = (object: any, length: number): any => {
|
||||
const copyObj = cloneDeep(object) as never
|
||||
|
||||
const truncateDeep = (obj: any): any => {
|
||||
if (isString(obj) && obj.length > length) {
|
||||
return `${truncate(obj, { length })} [truncated]`
|
||||
}
|
||||
|
||||
if (isArray(obj)) {
|
||||
return obj.map((i) => truncateDeep(i) as never)
|
||||
}
|
||||
|
||||
if (isObject(obj)) {
|
||||
Object.entries(obj).forEach(([key, value]) => {
|
||||
obj[key as keyof typeof obj] = truncateDeep(value) as never
|
||||
})
|
||||
|
||||
return obj
|
||||
}
|
||||
|
||||
// return everything else untouched
|
||||
return obj
|
||||
}
|
||||
|
||||
return truncateDeep(copyObj)
|
||||
}
|
||||
|
||||
class GcpLoggingTransport extends LoggingWinston {
|
||||
log(info: any, callback: (err: Error | null, apiResponse?: any) => void) {
|
||||
const infoString = JSON.stringify(info)
|
||||
if (infoString.length > 250000) {
|
||||
// max size for a log entry is 256KB
|
||||
info = infoString.substring(0, 256000)
|
||||
const sizeInfo = JSON.stringify(info).length
|
||||
if (sizeInfo > 250000) {
|
||||
// add a console.trace to help debug
|
||||
console.trace('GCP Logging API payload too large', sizeInfo)
|
||||
info = truncateObjectDeep(info, 5000) as never // the max length for string values is 5000
|
||||
}
|
||||
super.log(info, callback)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -183,7 +183,7 @@ const getReadabilityResult = async (
|
|||
return article
|
||||
}
|
||||
} catch (error) {
|
||||
console.log('parsing error for url', url, error)
|
||||
logger.info('parsing error for url', url, error)
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -208,7 +208,7 @@ export const parsePreparedContent = async (
|
|||
const { document, pageInfo } = preparedDocument
|
||||
|
||||
if (!document) {
|
||||
console.log('No document')
|
||||
logger.info('No document')
|
||||
return {
|
||||
canonicalUrl: url,
|
||||
parsedContent: null,
|
||||
|
|
@ -223,7 +223,7 @@ export const parsePreparedContent = async (
|
|||
pageInfo.contentType &&
|
||||
!ALLOWED_CONTENT_TYPES.includes(pageInfo.contentType)
|
||||
) {
|
||||
console.log('Not allowed content type', pageInfo.contentType)
|
||||
logger.info('Not allowed content type', pageInfo.contentType)
|
||||
return {
|
||||
canonicalUrl: url,
|
||||
parsedContent: null,
|
||||
|
|
@ -348,7 +348,7 @@ export const parsePreparedContent = async (
|
|||
})
|
||||
logRecord.parseSuccess = true
|
||||
} catch (error) {
|
||||
console.log('Error parsing content', error)
|
||||
logger.info('Error parsing content', error)
|
||||
Object.assign(logRecord, {
|
||||
parseSuccess: false,
|
||||
parseError: error,
|
||||
|
|
@ -443,7 +443,7 @@ export const parsePageMetadata = (html: string): Metadata | undefined => {
|
|||
|
||||
return { title, author, description, previewImage }
|
||||
} catch (e) {
|
||||
console.log('failed to parse page:', e)
|
||||
logger.info('failed to parse page:', e)
|
||||
return undefined
|
||||
}
|
||||
}
|
||||
|
|
@ -455,7 +455,7 @@ export const parseUrlMetadata = async (
|
|||
const res = await axios.get(url)
|
||||
return parsePageMetadata(res.data)
|
||||
} catch (e) {
|
||||
console.log('failed to get:', url, e)
|
||||
logger.info('failed to get:', url, e)
|
||||
return undefined
|
||||
}
|
||||
}
|
||||
|
|
@ -635,7 +635,7 @@ export const htmlToHighlightedMarkdown = (
|
|||
throw new Error('Invalid html content')
|
||||
}
|
||||
} catch (err) {
|
||||
console.log(err)
|
||||
logger.info(err)
|
||||
return nhm.translate(/* html */ html)
|
||||
}
|
||||
|
||||
|
|
@ -655,7 +655,7 @@ export const htmlToHighlightedMarkdown = (
|
|||
articleTextNodes
|
||||
)
|
||||
} catch (err) {
|
||||
console.log(err)
|
||||
logger.info(err)
|
||||
}
|
||||
})
|
||||
html = document.documentElement.outerHTML
|
||||
|
|
@ -674,14 +674,14 @@ export const getDistillerResult = async (
|
|||
try {
|
||||
const url = process.env.DISTILLER_URL
|
||||
if (!url) {
|
||||
console.log('No distiller url')
|
||||
logger.info('No distiller url')
|
||||
return undefined
|
||||
}
|
||||
|
||||
const exp = Math.floor(Date.now() / 1000) + 60 * 60 // 1 hour
|
||||
const auth = (await signToken({ uid, exp }, env.server.jwtSecret)) as string
|
||||
|
||||
console.debug('Parsing by distiller', url)
|
||||
logger.info('Parsing by distiller', url)
|
||||
const response = await axios.post<string>(url, html, {
|
||||
headers: {
|
||||
Authorization: auth,
|
||||
|
|
@ -690,7 +690,7 @@ export const getDistillerResult = async (
|
|||
})
|
||||
return response.data
|
||||
} catch (e) {
|
||||
console.log('Error parsing by distiller', e)
|
||||
logger.info('Error parsing by distiller', e)
|
||||
return undefined
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -109,8 +109,8 @@ describe('saveNewsletterEmail', () => {
|
|||
})
|
||||
|
||||
it('does not create a subscription if no unsubscribe header', async () => {
|
||||
const url = 'https://blog.omnivore.app/no-unsubscribe'
|
||||
nock('https://blog.omnivore.app').get('/no-unsubscribe').reply(404)
|
||||
const url = 'https://omnivore.app/no_url?q=no-unsubscribe'
|
||||
nock('https://omnivore.app').get('/no_url?q=no-unsubscribe').reply(404)
|
||||
|
||||
await saveNewsletterEmail(
|
||||
{
|
||||
|
|
|
|||
Loading…
Reference in a new issue