mirror of
https://github.com/omnivore-app/omnivore.git
synced 2026-03-11 08:54:26 +00:00
use winston logger instead of console log so the log will be truncated if it exceeds the max entry size
This commit is contained in:
Hongbo Wu
parent
66490df72f
commit
07d43812a7
27 changed files with 314 additions and 268 deletions
|
|
@ -144,11 +144,11 @@ export const edgeLoader = <
|
|||
result.push(keyMap[key] || [])
|
||||
}
|
||||
if (result.length !== keys.length) {
|
||||
console.error('DataModel error: count mismatch ', keys, result)
|
||||
logger.error('DataModel error: count mismatch ', keys, result)
|
||||
}
|
||||
return result
|
||||
} catch (e) {
|
||||
console.error('DataModel error: ', e)
|
||||
logger.error('DataModel error: ', e)
|
||||
throw e
|
||||
}
|
||||
},
|
||||
|
|
|
|||
|
|
@ -2,13 +2,16 @@ import { PubSub } from '@google-cloud/pubsub'
|
|||
import { env } from '../env'
|
||||
import { ReportType } from '../generated/graphql'
|
||||
import express from 'express'
|
||||
import { buildLogger } from '../utils/logger'
|
||||
|
||||
const logger = buildLogger('pubsub')
|
||||
|
||||
const client = new PubSub()
|
||||
|
||||
export const createPubSubClient = (): PubsubClient => {
|
||||
const publish = (topicName: string, msg: Buffer): Promise<void> => {
|
||||
if (env.dev.isLocal) {
|
||||
console.log(`Publishing ${topicName}`)
|
||||
logger.info(`Publishing ${topicName}`)
|
||||
return Promise.resolve()
|
||||
}
|
||||
|
||||
|
|
@ -16,7 +19,7 @@ export const createPubSubClient = (): PubsubClient => {
|
|||
.topic(topicName)
|
||||
.publishMessage({ data: msg })
|
||||
.catch((err) => {
|
||||
console.error(`[PubSub] error: ${topicName}`, err)
|
||||
logger.error(`[PubSub] error: ${topicName}`, err)
|
||||
})
|
||||
.then(() => {
|
||||
return Promise.resolve()
|
||||
|
|
@ -126,13 +129,13 @@ export const readPushSubscription = (
|
|||
req: express.Request
|
||||
): { message: string | undefined; expired: boolean } => {
|
||||
if (req.query.token !== process.env.PUBSUB_VERIFICATION_TOKEN) {
|
||||
console.log('query does not include valid pubsub token')
|
||||
logger.info('query does not include valid pubsub token')
|
||||
return { message: undefined, expired: false }
|
||||
}
|
||||
|
||||
// GCP PubSub sends the request as a base64 encoded string
|
||||
if (!('message' in req.body)) {
|
||||
console.log('Invalid pubsub message: message not in body')
|
||||
logger.info('Invalid pubsub message: message not in body')
|
||||
return { message: undefined, expired: false }
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
import { errors } from '@elastic/elasticsearch'
|
||||
import { EntityType } from '../datalayer/pubsub'
|
||||
import { buildLogger } from '../utils/logger'
|
||||
import { SortBy, SortOrder, SortParams } from '../utils/search'
|
||||
import { client, INDEX_ALIAS } from './index'
|
||||
import {
|
||||
|
|
@ -11,6 +12,8 @@ import {
|
|||
SearchResponse,
|
||||
} from './types'
|
||||
|
||||
const logger = buildLogger('elasticsearch')
|
||||
|
||||
export const addHighlightToPage = async (
|
||||
id: string,
|
||||
highlight: Highlight,
|
||||
|
|
@ -52,10 +55,10 @@ export const addHighlightToPage = async (
|
|||
e instanceof errors.ResponseError &&
|
||||
e.message === 'document_missing_exception'
|
||||
) {
|
||||
console.log('page has been deleted', id)
|
||||
logger.info('page has been deleted', id)
|
||||
return false
|
||||
}
|
||||
console.error('failed to add highlight to a page in elastic', e)
|
||||
logger.error('failed to add highlight to a page in elastic', e)
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
|
@ -90,7 +93,7 @@ export const getHighlightById = async (
|
|||
// eslint-disable-next-line @typescript-eslint/no-unsafe-member-access,@typescript-eslint/no-unsafe-return
|
||||
return body.hits.hits[0].inner_hits.highlights.hits.hits[0]._source
|
||||
} catch (e) {
|
||||
console.error('failed to get highlight from a page in elastic', e)
|
||||
logger.error('failed to get highlight from a page in elastic', e)
|
||||
return undefined
|
||||
}
|
||||
}
|
||||
|
|
@ -146,7 +149,7 @@ export const deleteHighlight = async (
|
|||
|
||||
return true
|
||||
} catch (e) {
|
||||
console.error('failed to delete a highlight in elastic', e)
|
||||
logger.error('failed to delete a highlight in elastic', e)
|
||||
|
||||
return false
|
||||
}
|
||||
|
|
@ -223,7 +226,7 @@ export const searchHighlights = async (
|
|||
],
|
||||
}
|
||||
|
||||
console.log('searching highlights in elastic', JSON.stringify(searchBody))
|
||||
logger.info('searching highlights in elastic', JSON.stringify(searchBody))
|
||||
|
||||
const response = await client.search<SearchResponse<Page>>({
|
||||
index: INDEX_ALIAS,
|
||||
|
|
@ -249,7 +252,7 @@ export const searchHighlights = async (
|
|||
|
||||
return [results, response.body.hits.total.value]
|
||||
} catch (e) {
|
||||
console.error('failed to search highlights in elastic', e)
|
||||
logger.error('failed to search highlights in elastic', e)
|
||||
return undefined
|
||||
}
|
||||
}
|
||||
|
|
@ -306,7 +309,7 @@ export const updateHighlight = async (
|
|||
|
||||
return true
|
||||
} catch (e) {
|
||||
console.error('failed to update highlight in elastic', e)
|
||||
logger.error('failed to update highlight in elastic', e)
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,8 +1,11 @@
|
|||
import { errors } from '@elastic/elasticsearch'
|
||||
import { EntityType } from '../datalayer/pubsub'
|
||||
import { buildLogger } from '../utils/logger'
|
||||
import { client, INDEX_ALIAS } from './index'
|
||||
import { Label, PageContext } from './types'
|
||||
|
||||
const logger = buildLogger('elasticsearch')
|
||||
|
||||
export const addLabelInPage = async (
|
||||
pageId: string,
|
||||
label: Label,
|
||||
|
|
@ -46,10 +49,10 @@ export const addLabelInPage = async (
|
|||
e instanceof errors.ResponseError &&
|
||||
e.message === 'document_missing_exception'
|
||||
) {
|
||||
console.log('page has been deleted', pageId)
|
||||
logger.info('page has been deleted', pageId)
|
||||
return false
|
||||
}
|
||||
console.error('failed to add a label in elastic', e)
|
||||
logger.error('failed to add a label in elastic', e)
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
|
@ -95,10 +98,10 @@ export const updateLabelsInPage = async (
|
|||
e instanceof errors.ResponseError &&
|
||||
e.message === 'document_missing_exception'
|
||||
) {
|
||||
console.log('page has been deleted', pageId)
|
||||
logger.info('page has been deleted', pageId)
|
||||
return false
|
||||
}
|
||||
console.error('failed to update labels in elastic', e)
|
||||
logger.error('failed to update labels in elastic', e)
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
|
@ -174,7 +177,7 @@ export const deleteLabel = async (
|
|||
|
||||
return true
|
||||
} catch (e) {
|
||||
console.error('failed to delete a label in elastic', e)
|
||||
logger.error('failed to delete a label in elastic', e)
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
|
@ -264,7 +267,7 @@ export const updateLabel = async (
|
|||
|
||||
return true
|
||||
} catch (e) {
|
||||
console.error('failed to update label in elastic', e)
|
||||
logger.error('failed to update label in elastic', e)
|
||||
|
||||
return false
|
||||
}
|
||||
|
|
@ -324,10 +327,10 @@ export const setLabelsForHighlight = async (
|
|||
e instanceof errors.ResponseError &&
|
||||
e.message === 'document_missing_exception'
|
||||
) {
|
||||
console.log('highlight has been deleted', highlightId)
|
||||
logger.info('highlight has been deleted', highlightId)
|
||||
return false
|
||||
}
|
||||
console.error('failed to set labels for highlight in elastic', e)
|
||||
logger.error('failed to set labels for highlight in elastic', e)
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ import { BuiltQuery, ESBuilder, esBuilder } from 'elastic-ts'
|
|||
import { EntityType } from '../datalayer/pubsub'
|
||||
import { BulkActionType } from '../generated/graphql'
|
||||
import { wordsCount } from '../utils/helpers'
|
||||
import { buildLogger } from '../utils/logger'
|
||||
import {
|
||||
DateFilter,
|
||||
FieldFilter,
|
||||
|
|
@ -27,6 +28,8 @@ import {
|
|||
SearchResponse,
|
||||
} from './types'
|
||||
|
||||
const logger = buildLogger('elasticsearch')
|
||||
|
||||
const appendQuery = (builder: ESBuilder, query: string): ESBuilder => {
|
||||
interface Field {
|
||||
field: string
|
||||
|
|
@ -421,7 +424,7 @@ export const createPage = async (
|
|||
|
||||
return page.id
|
||||
} catch (e) {
|
||||
console.error('failed to create a page in elastic', JSON.stringify(e))
|
||||
logger.error('failed to create a page in elastic', JSON.stringify(e))
|
||||
return undefined
|
||||
}
|
||||
}
|
||||
|
|
@ -461,10 +464,10 @@ export const updatePage = async (
|
|||
e instanceof errors.ResponseError &&
|
||||
e.message === 'document_missing_exception'
|
||||
) {
|
||||
console.log('page has been deleted', id)
|
||||
logger.info('page has been deleted', id)
|
||||
return false
|
||||
}
|
||||
console.error('failed to update a page in elastic', e)
|
||||
logger.error('failed to update a page in elastic', e)
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
|
@ -486,10 +489,10 @@ export const deletePage = async (
|
|||
e instanceof errors.ResponseError &&
|
||||
e.message === 'document_missing_exception'
|
||||
) {
|
||||
console.log('page has been deleted', id)
|
||||
logger.info('page has been deleted', id)
|
||||
return false
|
||||
}
|
||||
console.error('failed to delete a page in elastic', e)
|
||||
logger.error('failed to delete a page in elastic', e)
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
|
@ -530,7 +533,7 @@ export const getPageByParam = async <K extends keyof ParamSet>(
|
|||
id: body.hits.hits[0]._id,
|
||||
} as Page
|
||||
} catch (e) {
|
||||
console.error('failed to get page by param in elastic', e)
|
||||
logger.error('failed to get page by param in elastic', e)
|
||||
return undefined
|
||||
}
|
||||
}
|
||||
|
|
@ -550,10 +553,10 @@ export const getPageById = async (id: string): Promise<Page | undefined> => {
|
|||
} as Page
|
||||
} catch (e) {
|
||||
if (e instanceof errors.ResponseError && e.statusCode === 404) {
|
||||
console.log('page has been deleted', id)
|
||||
logger.info('page has been deleted', id)
|
||||
return undefined
|
||||
}
|
||||
console.error('failed to get page by id in elastic', e)
|
||||
logger.error('failed to get page by id in elastic', e)
|
||||
return undefined
|
||||
}
|
||||
}
|
||||
|
|
@ -665,7 +668,7 @@ export const searchPages = async (
|
|||
})
|
||||
.build()
|
||||
|
||||
console.debug('searching pages in elastic', JSON.stringify(body))
|
||||
logger.info('searching pages in elastic', JSON.stringify(body))
|
||||
const response = await client.search<SearchResponse<Page>, BuiltQuery>({
|
||||
index: INDEX_ALIAS,
|
||||
body,
|
||||
|
|
@ -685,10 +688,10 @@ export const searchPages = async (
|
|||
} catch (e) {
|
||||
if (e instanceof errors.ResponseError) {
|
||||
// eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
|
||||
console.error('failed to search pages in elastic', e.meta.body.error)
|
||||
logger.error('failed to search pages in elastic', e.meta.body.error)
|
||||
return undefined
|
||||
}
|
||||
console.error('failed to search pages in elastic', e)
|
||||
logger.error('failed to search pages in elastic', e)
|
||||
return undefined
|
||||
}
|
||||
}
|
||||
|
|
@ -726,7 +729,7 @@ export const countByCreatedAt = async (
|
|||
|
||||
return body.count as number
|
||||
} catch (e) {
|
||||
console.error('failed to count pages in elastic', e)
|
||||
logger.error('failed to count pages in elastic', e)
|
||||
return 0
|
||||
}
|
||||
}
|
||||
|
|
@ -765,7 +768,7 @@ export const deletePagesByParam = async <K extends keyof ParamSet>(
|
|||
|
||||
return false
|
||||
} catch (e) {
|
||||
console.error('failed to delete pages by param in elastic', e)
|
||||
logger.error('failed to delete pages by param in elastic', e)
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
|
@ -823,7 +826,7 @@ export const searchAsYouType = async (
|
|||
id: hit._id,
|
||||
}))
|
||||
} catch (e) {
|
||||
console.error('failed to search as you type in elastic', e)
|
||||
logger.error('failed to search as you type in elastic', e)
|
||||
|
||||
return []
|
||||
}
|
||||
|
|
@ -904,7 +907,7 @@ export const updatePages = async (
|
|||
.rawOption('script', updatedScript)
|
||||
.build()
|
||||
|
||||
console.debug('updating pages in elastic', JSON.stringify(searchBody))
|
||||
logger.info('updating pages in elastic', JSON.stringify(searchBody))
|
||||
|
||||
try {
|
||||
const { body } = await client.updateByQuery({
|
||||
|
|
@ -920,21 +923,21 @@ export const updatePages = async (
|
|||
|
||||
// eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
|
||||
if (body.failures && body.failures.length > 0) {
|
||||
console.log('failed to update pages in elastic', body.failures)
|
||||
logger.info('failed to update pages in elastic', body.failures)
|
||||
return null
|
||||
}
|
||||
|
||||
// TODO: publish entityUpdated events for each page
|
||||
|
||||
if (async) {
|
||||
console.log('update pages task started', body.task)
|
||||
logger.info('update pages task started', body.task)
|
||||
return body.task as string
|
||||
}
|
||||
|
||||
console.log('updated pages in elastic', body.updated)
|
||||
logger.info('updated pages in elastic', body.updated)
|
||||
return body.updated as string
|
||||
} catch (e) {
|
||||
console.log('failed to update pages in elastic', e)
|
||||
logger.info('failed to update pages in elastic', e)
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,10 +1,13 @@
|
|||
import { buildLogger } from '../utils/logger'
|
||||
import { createPage, getPageByParam, updatePage } from './pages'
|
||||
import {
|
||||
ArticleSavingRequestStatus,
|
||||
Page,
|
||||
PageContext,
|
||||
Recommendation,
|
||||
} from './types'
|
||||
import { createPage, getPageByParam, updatePage } from './pages'
|
||||
|
||||
const logger = buildLogger('app.dispatch')
|
||||
|
||||
export const addRecommendation = async (
|
||||
ctx: PageContext,
|
||||
|
|
@ -83,6 +86,6 @@ export const addRecommendation = async (
|
|||
|
||||
return createPage(newPage, ctx)
|
||||
} catch (err) {
|
||||
console.error(err)
|
||||
logger.error(err)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,3 +1,7 @@
|
|||
import { ApiKey } from '../../entity/api_key'
|
||||
import { User } from '../../entity/user'
|
||||
import { getRepository } from '../../entity/utils'
|
||||
import { env } from '../../env'
|
||||
import {
|
||||
ApiKeysError,
|
||||
ApiKeysErrorCode,
|
||||
|
|
@ -12,12 +16,8 @@ import {
|
|||
RevokeApiKeySuccess,
|
||||
} from '../../generated/graphql'
|
||||
import { analytics } from '../../utils/analytics'
|
||||
import { env } from '../../env'
|
||||
import { authorized } from '../../utils/helpers'
|
||||
import { getRepository } from '../../entity/utils'
|
||||
import { User } from '../../entity/user'
|
||||
import { ApiKey } from '../../entity/api_key'
|
||||
import { generateApiKey, hashApiKey } from '../../utils/auth'
|
||||
import { authorized } from '../../utils/helpers'
|
||||
|
||||
export const apiKeysResolver = authorized<ApiKeysSuccess, ApiKeysError>(
|
||||
async (_, __, { claims: { uid }, log }) => {
|
||||
|
|
@ -103,7 +103,7 @@ export const generateApiKeyResolver = authorized<
|
|||
},
|
||||
}
|
||||
} catch (error) {
|
||||
console.error(error)
|
||||
log.error(error)
|
||||
|
||||
return { errorCodes: [GenerateApiKeyErrorCode.BadRequest] }
|
||||
}
|
||||
|
|
|
|||
|
|
@ -426,7 +426,7 @@ export const getArticleResolver: ResolverFn<
|
|||
Record<string, unknown>,
|
||||
WithDataSourcesContext,
|
||||
QueryArticleArgs
|
||||
> = async (_obj, { slug, format }, { claims }, info) => {
|
||||
> = async (_obj, { slug, format }, { claims, log }, info) => {
|
||||
try {
|
||||
if (!claims?.uid) {
|
||||
return { errorCodes: [ArticleErrorCode.Unauthorized] }
|
||||
|
|
@ -481,7 +481,7 @@ export const getArticleResolver: ResolverFn<
|
|||
article: { ...page, isArchived: !!page.archivedAt, linkId: page.id },
|
||||
}
|
||||
} catch (error) {
|
||||
console.log(error)
|
||||
log.error(error)
|
||||
return { errorCodes: [ArticleErrorCode.BadData] }
|
||||
}
|
||||
}
|
||||
|
|
@ -495,7 +495,7 @@ export const getArticlesResolver = authorized<
|
|||
PaginatedPartialArticles,
|
||||
ArticlesError,
|
||||
QueryArticlesArgs
|
||||
>(async (_obj, params, { claims }) => {
|
||||
>(async (_obj, params, { claims, log }) => {
|
||||
const startCursor = params.after || ''
|
||||
const first = params.first || 10
|
||||
|
||||
|
|
@ -526,7 +526,7 @@ export const getArticlesResolver = authorized<
|
|||
const hasNextPage = pages.length > first
|
||||
const endCursor = String(start + pages.length - (hasNextPage ? 1 : 0))
|
||||
|
||||
console.log(
|
||||
log.info(
|
||||
'start',
|
||||
start,
|
||||
'returning end cursor',
|
||||
|
|
@ -877,105 +877,103 @@ export const getReadingProgressAnchorIndexForArticleResolver: ResolverFn<
|
|||
return articleReadingProgressAnchorIndex || 0
|
||||
}
|
||||
|
||||
export const searchResolver = authorized<
|
||||
SearchSuccess,
|
||||
SearchError,
|
||||
QuerySearchArgs
|
||||
>(async (_obj, params, { claims }) => {
|
||||
const startCursor = params.after || ''
|
||||
const first = params.first || 10
|
||||
export const searchResolver = authorized<SearchSuccess, SearchError, QuerySearchArgs>(
|
||||
async (_obj, params, { claims, log }) => {
|
||||
const startCursor = params.after || ''
|
||||
const first = params.first || 10
|
||||
|
||||
// the query size is limited to 255 characters
|
||||
if (params.query && params.query.length > 255) {
|
||||
return { errorCodes: [SearchErrorCode.QueryTooLong] }
|
||||
}
|
||||
|
||||
const searchQuery = parseSearchQuery(params.query || undefined)
|
||||
|
||||
let results: SearchItemData[]
|
||||
let totalCount: number
|
||||
|
||||
const searchType = searchQuery.typeFilter
|
||||
// search highlights if type:highlights
|
||||
if (searchType === PageType.Highlights) {
|
||||
;[results, totalCount] = (await searchHighlights(
|
||||
{
|
||||
from: Number(startCursor),
|
||||
size: first + 1, // fetch one more item to get next cursor
|
||||
sort: searchQuery.sortParams,
|
||||
query: searchQuery.query,
|
||||
},
|
||||
claims.uid
|
||||
)) || [[], 0]
|
||||
} else {
|
||||
// otherwise, search pages
|
||||
;[results, totalCount] = (await searchPages(
|
||||
{
|
||||
from: Number(startCursor),
|
||||
size: first + 1, // fetch one more item to get next cursor
|
||||
sort: searchQuery.sortParams,
|
||||
includePending: true,
|
||||
includeContent: params.includeContent ?? false,
|
||||
...searchQuery,
|
||||
},
|
||||
claims.uid
|
||||
)) || [[], 0]
|
||||
}
|
||||
|
||||
const start =
|
||||
startCursor && !isNaN(Number(startCursor)) ? Number(startCursor) : 0
|
||||
const hasNextPage = results.length > first
|
||||
const endCursor = String(start + results.length - (hasNextPage ? 1 : 0))
|
||||
|
||||
if (hasNextPage) {
|
||||
// remove an extra if exists
|
||||
results.pop()
|
||||
}
|
||||
|
||||
const edges = results.map((r) => {
|
||||
let siteIcon = r.siteIcon
|
||||
if (siteIcon && !isBase64Image(siteIcon)) {
|
||||
siteIcon = createImageProxyUrl(siteIcon, 128, 128)
|
||||
// the query size is limited to 255 characters
|
||||
if (params.query && params.query.length > 255) {
|
||||
return { errorCodes: [SearchErrorCode.QueryTooLong] }
|
||||
}
|
||||
if (params.includeContent && r.content) {
|
||||
// convert html to the requested format
|
||||
const format = params.format || ArticleFormat.Html
|
||||
try {
|
||||
const converter = contentConverter(format)
|
||||
if (converter) {
|
||||
r.content = converter(r.content, r.highlights)
|
||||
}
|
||||
} catch (error) {
|
||||
console.log('Error converting content', error)
|
||||
|
||||
const searchQuery = parseSearchQuery(params.query || undefined)
|
||||
|
||||
let results: SearchItemData[]
|
||||
let totalCount: number
|
||||
|
||||
const searchType = searchQuery.typeFilter
|
||||
// search highlights if type:highlights
|
||||
if (searchType === PageType.Highlights) {
|
||||
;[results, totalCount] = (await searchHighlights(
|
||||
{
|
||||
from: Number(startCursor),
|
||||
size: first + 1, // fetch one more item to get next cursor
|
||||
sort: searchQuery.sortParams,
|
||||
query: searchQuery.query,
|
||||
},
|
||||
claims.uid
|
||||
)) || [[], 0]
|
||||
} else {
|
||||
// otherwise, search pages
|
||||
;[results, totalCount] = (await searchPages(
|
||||
{
|
||||
from: Number(startCursor),
|
||||
size: first + 1, // fetch one more item to get next cursor
|
||||
sort: searchQuery.sortParams,
|
||||
includePending: true,
|
||||
includeContent: params.includeContent ?? false,
|
||||
...searchQuery,
|
||||
},
|
||||
claims.uid
|
||||
)) || [[], 0]
|
||||
}
|
||||
|
||||
const start =
|
||||
startCursor && !isNaN(Number(startCursor)) ? Number(startCursor) : 0
|
||||
const hasNextPage = results.length > first
|
||||
const endCursor = String(start + results.length - (hasNextPage ? 1 : 0))
|
||||
|
||||
if (hasNextPage) {
|
||||
// remove an extra if exists
|
||||
results.pop()
|
||||
}
|
||||
|
||||
const edges = results.map((r) => {
|
||||
let siteIcon = r.siteIcon
|
||||
if (siteIcon && !isBase64Image(siteIcon)) {
|
||||
siteIcon = createImageProxyUrl(siteIcon, 128, 128)
|
||||
}
|
||||
}
|
||||
if (params.includeContent && r.content) {
|
||||
// convert html to the requested format
|
||||
const format = params.format || ArticleFormat.Html
|
||||
try {
|
||||
const converter = contentConverter(format)
|
||||
if (converter) {
|
||||
r.content = converter(r.content, r.highlights)
|
||||
}
|
||||
} catch (error) {
|
||||
log.error('Error converting content', error)
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
node: {
|
||||
...r,
|
||||
image: r.image && createImageProxyUrl(r.image, 260, 260),
|
||||
isArchived: !!r.archivedAt,
|
||||
contentReader: contentReaderForPage(r.pageType, r.uploadFileId),
|
||||
originalArticleUrl: r.url,
|
||||
publishedAt: validatedDate(r.publishedAt),
|
||||
ownedByViewer: r.userId === claims.uid,
|
||||
siteIcon,
|
||||
} as SearchItem,
|
||||
cursor: endCursor,
|
||||
}
|
||||
})
|
||||
|
||||
return {
|
||||
node: {
|
||||
...r,
|
||||
image: r.image && createImageProxyUrl(r.image, 260, 260),
|
||||
isArchived: !!r.archivedAt,
|
||||
contentReader: contentReaderForPage(r.pageType, r.uploadFileId),
|
||||
originalArticleUrl: r.url,
|
||||
publishedAt: validatedDate(r.publishedAt),
|
||||
ownedByViewer: r.userId === claims.uid,
|
||||
siteIcon,
|
||||
} as SearchItem,
|
||||
cursor: endCursor,
|
||||
edges,
|
||||
pageInfo: {
|
||||
hasPreviousPage: false,
|
||||
startCursor,
|
||||
hasNextPage: hasNextPage,
|
||||
endCursor,
|
||||
totalCount,
|
||||
},
|
||||
}
|
||||
})
|
||||
|
||||
return {
|
||||
edges,
|
||||
pageInfo: {
|
||||
hasPreviousPage: false,
|
||||
startCursor,
|
||||
hasNextPage: hasNextPage,
|
||||
endCursor,
|
||||
totalCount,
|
||||
},
|
||||
}
|
||||
})
|
||||
)
|
||||
|
||||
export const typeaheadSearchResolver = authorized<
|
||||
TypeaheadSearchSuccess,
|
||||
|
|
|
|||
|
|
@ -28,7 +28,7 @@ export const createArticleSavingRequestResolver = authorized<
|
|||
CreateArticleSavingRequestSuccess,
|
||||
CreateArticleSavingRequestError,
|
||||
MutationCreateArticleSavingRequestArgs
|
||||
>(async (_, { input: { url } }, { claims, pubsub }) => {
|
||||
>(async (_, { input: { url } }, { claims, pubsub, log }) => {
|
||||
analytics.track({
|
||||
userId: claims.uid,
|
||||
event: 'link_saved',
|
||||
|
|
@ -49,7 +49,7 @@ export const createArticleSavingRequestResolver = authorized<
|
|||
articleSavingRequest: request,
|
||||
}
|
||||
} catch (err) {
|
||||
console.log('error saving article', err)
|
||||
log.error('error saving article', err)
|
||||
if (isErrorWithCode(err)) {
|
||||
return {
|
||||
errorCodes: [err.errorCode as CreateArticleSavingRequestErrorCode],
|
||||
|
|
|
|||
|
|
@ -1,3 +1,9 @@
|
|||
import { In } from 'typeorm'
|
||||
import { getPageByParam } from '../../elastic/pages'
|
||||
import { Group } from '../../entity/groups/group'
|
||||
import { User } from '../../entity/user'
|
||||
import { getRepository } from '../../entity/utils'
|
||||
import { env } from '../../env'
|
||||
import {
|
||||
CreateGroupError,
|
||||
CreateGroupErrorCode,
|
||||
|
|
@ -32,15 +38,9 @@ import {
|
|||
joinGroup,
|
||||
leaveGroup,
|
||||
} from '../../services/groups'
|
||||
import { authorized, userDataToUser } from '../../utils/helpers'
|
||||
import { getRepository } from '../../entity/utils'
|
||||
import { User } from '../../entity/user'
|
||||
import { Group } from '../../entity/groups/group'
|
||||
import { In } from 'typeorm'
|
||||
import { getPageByParam } from '../../elastic/pages'
|
||||
import { enqueueRecommendation } from '../../utils/createTask'
|
||||
import { env } from '../../env'
|
||||
import { analytics } from '../../utils/analytics'
|
||||
import { enqueueRecommendation } from '../../utils/createTask'
|
||||
import { authorized, userDataToUser } from '../../utils/helpers'
|
||||
|
||||
export const createGroupResolver = authorized<
|
||||
CreateGroupSuccess,
|
||||
|
|
@ -236,7 +236,7 @@ export const recommendResolver = authorized<
|
|||
)
|
||||
.flat()
|
||||
)
|
||||
console.log('taskNames', taskNames)
|
||||
log.info('taskNames', taskNames)
|
||||
|
||||
return {
|
||||
success: true,
|
||||
|
|
|
|||
|
|
@ -1,26 +1,26 @@
|
|||
/* eslint-disable @typescript-eslint/no-unused-vars */
|
||||
import {
|
||||
ResolverFn,
|
||||
UploadFileRequestResult,
|
||||
MutationUploadFileRequestArgs,
|
||||
UploadFileStatus,
|
||||
UploadFileRequestErrorCode,
|
||||
ArticleSavingRequestStatus,
|
||||
} from '../../generated/graphql'
|
||||
import { WithDataSourcesContext } from '../types'
|
||||
import {
|
||||
generateUploadSignedUrl,
|
||||
generateUploadFilePathName,
|
||||
getFilePublicUrl,
|
||||
} from '../../utils/uploads'
|
||||
import path from 'path'
|
||||
import normalizeUrl from 'normalize-url'
|
||||
import { analytics } from '../../utils/analytics'
|
||||
import { env } from '../../env'
|
||||
import path from 'path'
|
||||
import { createPage, getPageByParam, updatePage } from '../../elastic/pages'
|
||||
import { PageType } from '../../elastic/types'
|
||||
import { generateSlug } from '../../utils/helpers'
|
||||
import { env } from '../../env'
|
||||
import {
|
||||
ArticleSavingRequestStatus,
|
||||
MutationUploadFileRequestArgs,
|
||||
ResolverFn,
|
||||
UploadFileRequestErrorCode,
|
||||
UploadFileRequestResult,
|
||||
UploadFileStatus,
|
||||
} from '../../generated/graphql'
|
||||
import { validateUrl } from '../../services/create_page_save_request'
|
||||
import { analytics } from '../../utils/analytics'
|
||||
import { generateSlug } from '../../utils/helpers'
|
||||
import {
|
||||
generateUploadFilePathName,
|
||||
generateUploadSignedUrl,
|
||||
getFilePublicUrl,
|
||||
} from '../../utils/uploads'
|
||||
import { WithDataSourcesContext } from '../types'
|
||||
|
||||
const isFileUrl = (url: string): boolean => {
|
||||
const parsedUrl = new URL(url)
|
||||
|
|
@ -40,7 +40,7 @@ export const uploadFileRequestResolver: ResolverFn<
|
|||
WithDataSourcesContext,
|
||||
MutationUploadFileRequestArgs
|
||||
> = async (_obj, { input }, ctx) => {
|
||||
const { models, kx, claims } = ctx
|
||||
const { models, kx, claims, log } = ctx
|
||||
let uploadFileData: { id: string | null } = {
|
||||
id: null,
|
||||
}
|
||||
|
|
@ -79,7 +79,7 @@ export const uploadFileRequestResolver: ResolverFn<
|
|||
try {
|
||||
validateUrl(url)
|
||||
} catch (error) {
|
||||
console.log('illegal file input url', error)
|
||||
log.info('illegal file input url', error)
|
||||
return {
|
||||
errorCodes: [UploadFileRequestErrorCode.BadInput],
|
||||
}
|
||||
|
|
|
|||
|
|
@ -50,7 +50,7 @@ export function pageRouter() {
|
|||
// Get the content type from the query params
|
||||
const { url, clientRequestId } = req.query
|
||||
const contentType = req.headers['content-type']
|
||||
console.log(
|
||||
logger.info(
|
||||
'contentType',
|
||||
contentType,
|
||||
'url',
|
||||
|
|
@ -64,7 +64,7 @@ export function pageRouter() {
|
|||
!isString(contentType) ||
|
||||
!isString(clientRequestId)
|
||||
) {
|
||||
console.log(
|
||||
logger.info(
|
||||
'creating page from pdf failed',
|
||||
url,
|
||||
contentType,
|
||||
|
|
@ -74,7 +74,7 @@ export function pageRouter() {
|
|||
}
|
||||
|
||||
if (!validateUuid(clientRequestId)) {
|
||||
console.log('creating page from pdf failed invalid uuid')
|
||||
logger.info('creating page from pdf failed invalid uuid')
|
||||
return res.status(400).send({ errorCode: 'BAD_DATA' })
|
||||
}
|
||||
|
||||
|
|
@ -110,7 +110,7 @@ export function pageRouter() {
|
|||
})
|
||||
|
||||
if (page) {
|
||||
console.log('updating page')
|
||||
logger.info('updating page')
|
||||
await updatePage(
|
||||
page.id,
|
||||
{
|
||||
|
|
@ -120,7 +120,7 @@ export function pageRouter() {
|
|||
ctx
|
||||
)
|
||||
} else {
|
||||
console.log('creating page')
|
||||
logger.info('creating page')
|
||||
const pageId = await createPage(
|
||||
{
|
||||
url: signedUrl,
|
||||
|
|
@ -145,7 +145,7 @@ export function pageRouter() {
|
|||
}
|
||||
}
|
||||
|
||||
console.log('redirecting to signed URL', signedUrl)
|
||||
logger.info('redirecting to signed URL', signedUrl)
|
||||
return res.redirect(signedUrl)
|
||||
})
|
||||
|
||||
|
|
|
|||
|
|
@ -2,16 +2,19 @@
|
|||
/* eslint-disable @typescript-eslint/no-unsafe-assignment */
|
||||
/* eslint-disable @typescript-eslint/explicit-module-boundary-types */
|
||||
import express from 'express'
|
||||
import { setClaims } from '../../datalayer/helpers'
|
||||
import { kx } from '../../datalayer/knex_config'
|
||||
import {
|
||||
createPubSubClient,
|
||||
readPushSubscription,
|
||||
} from '../../datalayer/pubsub'
|
||||
import { Page } from '../../elastic/types'
|
||||
import { getPageByParam, updatePage } from '../../elastic/pages'
|
||||
import { Page } from '../../elastic/types'
|
||||
import { ArticleSavingRequestStatus } from '../../generated/graphql'
|
||||
import { initModels } from '../../server'
|
||||
import { kx } from '../../datalayer/knex_config'
|
||||
import { setClaims } from '../../datalayer/helpers'
|
||||
import { buildLogger } from '../../utils/logger'
|
||||
|
||||
const logger = buildLogger('app.dispatch')
|
||||
|
||||
interface UpdateContentMessage {
|
||||
fileId: string
|
||||
|
|
@ -25,9 +28,9 @@ export function contentServiceRouter() {
|
|||
const router = express.Router()
|
||||
|
||||
router.post('/search', async (req, res) => {
|
||||
console.log('search req', req.query, req.body)
|
||||
logger.info('search req', req.query, req.body)
|
||||
const { message: msgStr, expired } = readPushSubscription(req)
|
||||
console.log('read pubsub message', msgStr, 'has expired', expired)
|
||||
logger.info('read pubsub message', msgStr, 'has expired', expired)
|
||||
|
||||
if (!msgStr) {
|
||||
res.status(400).send('Bad Request')
|
||||
|
|
@ -35,14 +38,14 @@ export function contentServiceRouter() {
|
|||
}
|
||||
|
||||
if (expired) {
|
||||
console.log('discarding expired message')
|
||||
logger.info('discarding expired message')
|
||||
res.status(200).send('Expired')
|
||||
return
|
||||
}
|
||||
|
||||
const data = JSON.parse(msgStr)
|
||||
if (!('fileId' in data) || !('content' in data)) {
|
||||
console.log('No file id or content found in message')
|
||||
logger.info('No file id or content found in message')
|
||||
res.status(400).send('Bad Request')
|
||||
return
|
||||
}
|
||||
|
|
@ -52,14 +55,14 @@ export function contentServiceRouter() {
|
|||
const parts = msg.fileId.split('/')
|
||||
const fileId = parts && parts.length > 1 ? parts[1] : undefined
|
||||
if (!fileId) {
|
||||
console.log('No file id found in message')
|
||||
logger.info('No file id found in message')
|
||||
res.status(400).send('Bad Request')
|
||||
return
|
||||
}
|
||||
|
||||
const page = await getPageByParam({ uploadFileId: fileId })
|
||||
if (!page) {
|
||||
console.log('No upload file found for id:', fileId)
|
||||
logger.info('No upload file found for id:', fileId)
|
||||
res.status(400).send('Bad Request')
|
||||
return
|
||||
}
|
||||
|
|
@ -80,16 +83,16 @@ export function contentServiceRouter() {
|
|||
await setClaims(tx, page.userId)
|
||||
return models.uploadFile.setFileUploadComplete(fileId, tx)
|
||||
})
|
||||
console.log('updated uploadFileData', uploadFileData)
|
||||
logger.info('updated uploadFileData', uploadFileData)
|
||||
} catch (error) {
|
||||
console.log('error marking file upload as completed', error)
|
||||
logger.info('error marking file upload as completed', error)
|
||||
}
|
||||
|
||||
const result = await updatePage(page.id, pageToUpdate, {
|
||||
pubsub: createPubSubClient(),
|
||||
uid: page.userId,
|
||||
})
|
||||
console.log(
|
||||
logger.info(
|
||||
'Updating article text',
|
||||
page.id,
|
||||
result,
|
||||
|
|
|
|||
|
|
@ -12,6 +12,7 @@ import { updateReceivedEmail } from '../../services/received_emails'
|
|||
import { analytics } from '../../utils/analytics'
|
||||
import { getClaimsByToken } from '../../utils/auth'
|
||||
import { generateSlug } from '../../utils/helpers'
|
||||
import { buildLogger } from '../../utils/logger'
|
||||
import {
|
||||
generateUploadFilePathName,
|
||||
generateUploadSignedUrl,
|
||||
|
|
@ -19,12 +20,14 @@ import {
|
|||
makeStorageFilePublic,
|
||||
} from '../../utils/uploads'
|
||||
|
||||
const logger = buildLogger('app.dispatch')
|
||||
|
||||
export function emailAttachmentRouter() {
|
||||
const router = express.Router()
|
||||
|
||||
// eslint-disable-next-line @typescript-eslint/no-misused-promises
|
||||
router.post('/upload', async (req, res) => {
|
||||
console.log('email-attachment/upload')
|
||||
logger.info('email-attachment/upload')
|
||||
|
||||
const { email, fileName, contentType } = req.body as {
|
||||
email: string
|
||||
|
|
@ -79,14 +82,14 @@ export function emailAttachmentRouter() {
|
|||
res.status(400).send('BAD REQUEST')
|
||||
}
|
||||
} catch (err) {
|
||||
console.error(err)
|
||||
logger.error(err)
|
||||
return res.status(500).send('INTERNAL_SERVER_ERROR')
|
||||
}
|
||||
})
|
||||
|
||||
// eslint-disable-next-line @typescript-eslint/no-misused-promises
|
||||
router.post('/create-article', async (req, res) => {
|
||||
console.log('email-attachment/create-article')
|
||||
logger.info('email-attachment/create-article')
|
||||
|
||||
const { email, uploadFileId, subject, receivedEmailId } = req.body as {
|
||||
email: string
|
||||
|
|
@ -176,7 +179,7 @@ export function emailAttachmentRouter() {
|
|||
|
||||
res.send({ id: pageId })
|
||||
} catch (err) {
|
||||
console.log(err)
|
||||
logger.info(err)
|
||||
res.status(500).send(err)
|
||||
}
|
||||
})
|
||||
|
|
|
|||
|
|
@ -259,7 +259,7 @@ export function integrationsServiceRouter() {
|
|||
syncedAt = retrieved.since || Date.now()
|
||||
retrievedData = retrieved.data
|
||||
|
||||
console.debug('retrieved data', {
|
||||
logger.info('retrieved data', {
|
||||
total: offset,
|
||||
size: retrievedData.length,
|
||||
})
|
||||
|
|
|
|||
|
|
@ -16,6 +16,9 @@ import {
|
|||
import { saveUrlFromEmail } from '../../services/save_url'
|
||||
import { getSubscriptionByNameAndUserId } from '../../services/subscriptions'
|
||||
import { isUrl } from '../../utils/helpers'
|
||||
import { buildLogger } from '../../utils/logger'
|
||||
|
||||
const logger = buildLogger('app.dispatch')
|
||||
|
||||
interface SetConfirmationCodeMessage {
|
||||
emailAddress: string
|
||||
|
|
@ -37,10 +40,10 @@ export function newsletterServiceRouter() {
|
|||
|
||||
// eslint-disable-next-line @typescript-eslint/no-misused-promises
|
||||
router.post('/confirmation', async (req, res) => {
|
||||
console.log('setConfirmationCode')
|
||||
logger.info('setConfirmationCode')
|
||||
|
||||
const { message, expired } = readPushSubscription(req)
|
||||
console.log('pubsub message:', message, 'expired:', expired)
|
||||
logger.info('pubsub message:', message, 'expired:', expired)
|
||||
|
||||
if (!message) {
|
||||
res.status(400).send('Bad Request')
|
||||
|
|
@ -48,7 +51,7 @@ export function newsletterServiceRouter() {
|
|||
}
|
||||
|
||||
if (expired) {
|
||||
console.log('discards expired message:', message)
|
||||
logger.info('discards expired message:', message)
|
||||
res.status(200).send('Expired')
|
||||
return
|
||||
}
|
||||
|
|
@ -58,7 +61,7 @@ export function newsletterServiceRouter() {
|
|||
const data: SetConfirmationCodeMessage = JSON.parse(message)
|
||||
|
||||
if (!('emailAddress' in data) || !('confirmationCode' in data)) {
|
||||
console.log('No email address or confirmation code found in message')
|
||||
logger.info('No email address or confirmation code found in message')
|
||||
res.status(400).send('Bad Request')
|
||||
return
|
||||
}
|
||||
|
|
@ -68,14 +71,14 @@ export function newsletterServiceRouter() {
|
|||
data.confirmationCode
|
||||
)
|
||||
if (!result) {
|
||||
console.log('Newsletter email not found', data.emailAddress)
|
||||
logger.info('Newsletter email not found', data.emailAddress)
|
||||
res.status(200).send('Not Found')
|
||||
return
|
||||
}
|
||||
|
||||
res.status(200).send('confirmation code set')
|
||||
} catch (e) {
|
||||
console.log(e)
|
||||
logger.info(e)
|
||||
if (e instanceof SyntaxError) {
|
||||
// when message is not a valid json string
|
||||
res.status(400).send(e)
|
||||
|
|
@ -87,7 +90,7 @@ export function newsletterServiceRouter() {
|
|||
|
||||
// eslint-disable-next-line @typescript-eslint/no-misused-promises
|
||||
router.post('/create', async (req, res) => {
|
||||
console.log('create')
|
||||
logger.info('create')
|
||||
|
||||
const { message, expired } = readPushSubscription(req)
|
||||
if (!message) {
|
||||
|
|
@ -96,7 +99,7 @@ export function newsletterServiceRouter() {
|
|||
}
|
||||
|
||||
if (expired) {
|
||||
console.log('discards expired message:', message)
|
||||
logger.info('discards expired message:', message)
|
||||
res.status(200).send('Expired')
|
||||
return
|
||||
}
|
||||
|
|
@ -104,14 +107,14 @@ export function newsletterServiceRouter() {
|
|||
try {
|
||||
const data = JSON.parse(message) as unknown
|
||||
if (!isNewsletterMessage(data)) {
|
||||
console.log('invalid newsletter message', data)
|
||||
logger.info('invalid newsletter message', data)
|
||||
return res.status(400).send('Bad Request')
|
||||
}
|
||||
|
||||
// get user from newsletter email
|
||||
const newsletterEmail = await getNewsletterEmail(data.email)
|
||||
if (!newsletterEmail) {
|
||||
console.log('newsletter email not found', data.email)
|
||||
logger.info('newsletter email not found', data.email)
|
||||
return res.status(200).send('Not Found')
|
||||
}
|
||||
|
||||
|
|
@ -136,14 +139,14 @@ export function newsletterServiceRouter() {
|
|||
newsletterEmail.user.id
|
||||
)
|
||||
if (existingSubscription?.status === SubscriptionStatus.Unsubscribed) {
|
||||
console.log('newsletter already unsubscribed:', data.author)
|
||||
logger.info('newsletter already unsubscribed:', data.author)
|
||||
return res.status(200).send('newsletter already unsubscribed')
|
||||
}
|
||||
|
||||
// save newsletter instead
|
||||
const result = await saveNewsletterEmail(data, newsletterEmail, saveCtx)
|
||||
if (!result) {
|
||||
console.log(
|
||||
logger.info(
|
||||
'Error creating newsletter link from data',
|
||||
data.email,
|
||||
data.title,
|
||||
|
|
@ -159,7 +162,7 @@ export function newsletterServiceRouter() {
|
|||
|
||||
res.status(200).send('newsletter created')
|
||||
} catch (e) {
|
||||
console.log(e)
|
||||
logger.info(e)
|
||||
if (e instanceof SyntaxError) {
|
||||
// when message is not a valid json string
|
||||
res.status(400).send(e)
|
||||
|
|
|
|||
|
|
@ -5,18 +5,21 @@ import { Subscription } from '../../entity/subscription'
|
|||
import { getRepository } from '../../entity/utils'
|
||||
import { SubscriptionStatus, SubscriptionType } from '../../generated/graphql'
|
||||
import { enqueueRssFeedFetch } from '../../utils/createTask'
|
||||
import { buildLogger } from '../../utils/logger'
|
||||
|
||||
const logger = buildLogger('app.dispatch')
|
||||
|
||||
export function rssFeedRouter() {
|
||||
const router = express.Router()
|
||||
|
||||
router.post('/fetchAll', async (req, res) => {
|
||||
console.log('fetch all rss feeds')
|
||||
logger.info('fetch all rss feeds')
|
||||
|
||||
const { message: msgStr, expired } = readPushSubscription(req)
|
||||
console.log('read pubsub message', msgStr, 'has expired', expired)
|
||||
logger.info('read pubsub message', msgStr, 'has expired', expired)
|
||||
|
||||
if (expired) {
|
||||
console.log('discarding expired message')
|
||||
logger.info('discarding expired message')
|
||||
return res.status(200).send('Expired')
|
||||
}
|
||||
|
||||
|
|
@ -37,14 +40,14 @@ export function rssFeedRouter() {
|
|||
try {
|
||||
return enqueueRssFeedFetch(subscription.user.id, subscription)
|
||||
} catch (error) {
|
||||
console.log('error creating rss feed fetch task', error)
|
||||
logger.info('error creating rss feed fetch task', error)
|
||||
}
|
||||
})
|
||||
)
|
||||
|
||||
res.send('OK')
|
||||
} catch (error) {
|
||||
console.log('error fetching rss feeds', error)
|
||||
logger.info('error fetching rss feeds', error)
|
||||
res.status(500).send('Internal Server Error')
|
||||
}
|
||||
})
|
||||
|
|
|
|||
|
|
@ -6,12 +6,15 @@ import { readPushSubscription } from '../../datalayer/pubsub'
|
|||
import { getRepository } from '../../entity/utils'
|
||||
import { Webhook } from '../../entity/webhook'
|
||||
import axios, { Method } from 'axios'
|
||||
import { buildLogger } from '../../utils/logger'
|
||||
|
||||
const logger = buildLogger('app.dispatch')
|
||||
|
||||
export function webhooksServiceRouter() {
|
||||
const router = express.Router()
|
||||
|
||||
router.post('/trigger/:action', async (req, res) => {
|
||||
console.log('trigger webhook of action', req.params.action)
|
||||
logger.info('trigger webhook of action', req.params.action)
|
||||
const { message: msgStr, expired } = readPushSubscription(req)
|
||||
|
||||
if (!msgStr) {
|
||||
|
|
@ -20,7 +23,7 @@ export function webhooksServiceRouter() {
|
|||
}
|
||||
|
||||
if (expired) {
|
||||
console.log('discarding expired message')
|
||||
logger.info('discarding expired message')
|
||||
res.status(200).send('Expired')
|
||||
return
|
||||
}
|
||||
|
|
@ -29,7 +32,7 @@ export function webhooksServiceRouter() {
|
|||
const data = JSON.parse(msgStr)
|
||||
const { userId, type } = data
|
||||
if (!userId || !type) {
|
||||
console.log('No userId or type found in message')
|
||||
logger.info('No userId or type found in message')
|
||||
res.status(400).send('Bad Request')
|
||||
return
|
||||
}
|
||||
|
|
@ -44,7 +47,7 @@ export function webhooksServiceRouter() {
|
|||
.getMany()
|
||||
|
||||
if (webhooks.length <= 0) {
|
||||
console.log(
|
||||
logger.info(
|
||||
'No active webhook found for user',
|
||||
userId,
|
||||
'and eventType',
|
||||
|
|
@ -64,7 +67,7 @@ export function webhooksServiceRouter() {
|
|||
[type]: data,
|
||||
})
|
||||
|
||||
console.log('triggering webhook', url)
|
||||
logger.info('triggering webhook', url)
|
||||
await axios.request({
|
||||
url,
|
||||
method,
|
||||
|
|
@ -77,7 +80,7 @@ export function webhooksServiceRouter() {
|
|||
|
||||
res.status(200).send('OK')
|
||||
} catch (err) {
|
||||
console.log('trigger webhook failed', err)
|
||||
logger.info('trigger webhook failed', err)
|
||||
res.status(500).send(err)
|
||||
}
|
||||
})
|
||||
|
|
|
|||
|
|
@ -37,7 +37,7 @@ export function userRouter() {
|
|||
subject?: string
|
||||
}
|
||||
if (!subject || !body || !from) {
|
||||
console.log(subject, body, from)
|
||||
logger.error('Bad Request', subject, body, from)
|
||||
res.status(400).send('Bad Request')
|
||||
return
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,12 +1,15 @@
|
|||
import axios from 'axios'
|
||||
import { ArticleSavingRequestStatus } from '../../elastic/types'
|
||||
import { env } from '../../env'
|
||||
import { buildLogger } from '../../utils/logger'
|
||||
import {
|
||||
IntegrationService,
|
||||
RetrievedResult,
|
||||
RetrieveRequest,
|
||||
} from './integration'
|
||||
|
||||
const logger = buildLogger('app.dispatch')
|
||||
|
||||
interface PocketResponse {
|
||||
status: number // 1 if success
|
||||
complete: number // 1 if all items have been returned
|
||||
|
|
@ -73,7 +76,7 @@ export class PocketIntegration extends IntegrationService {
|
|||
)
|
||||
return response.data.access_token
|
||||
} catch (error) {
|
||||
console.log('error validating pocket token', error)
|
||||
logger.info('error validating pocket token', error)
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
|
@ -102,10 +105,10 @@ export class PocketIntegration extends IntegrationService {
|
|||
headers: this.headers,
|
||||
}
|
||||
)
|
||||
console.debug('pocket data', response.data)
|
||||
|
||||
return response.data
|
||||
} catch (error) {
|
||||
console.log('error retrieving pocket data', error)
|
||||
logger.info('error retrieving pocket data', error)
|
||||
throw new Error('Error retrieving pocket data')
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -4,9 +4,12 @@ import { Integration } from '../../entity/integration'
|
|||
import { getRepository } from '../../entity/utils'
|
||||
import { env } from '../../env'
|
||||
import { wait } from '../../utils/helpers'
|
||||
import { buildLogger } from '../../utils/logger'
|
||||
import { getHighlightUrl } from '../highlights'
|
||||
import { IntegrationService } from './integration'
|
||||
|
||||
const logger = buildLogger('app.dispatch')
|
||||
|
||||
interface ReadwiseHighlight {
|
||||
// The highlight text, (technically the only field required in a highlight object)
|
||||
text: string
|
||||
|
|
@ -48,7 +51,7 @@ export class ReadwiseIntegration extends IntegrationService {
|
|||
})
|
||||
return response.status === 204 ? token : null
|
||||
} catch (error) {
|
||||
console.log('error validating readwise token', error)
|
||||
logger.info('error validating readwise token', error)
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
|
@ -66,7 +69,7 @@ export class ReadwiseIntegration extends IntegrationService {
|
|||
|
||||
// update integration syncedAt if successful
|
||||
if (result) {
|
||||
console.log('updating integration syncedAt')
|
||||
logger.info('updating integration syncedAt')
|
||||
await getRepository(Integration).update(integration.id, {
|
||||
syncedAt: new Date(),
|
||||
})
|
||||
|
|
@ -129,14 +132,14 @@ export class ReadwiseIntegration extends IntegrationService {
|
|||
error.response?.status === 429 &&
|
||||
retryCount < 3
|
||||
) {
|
||||
console.log('Readwise API rate limit exceeded, retrying...')
|
||||
logger.info('Readwise API rate limit exceeded, retrying...')
|
||||
// wait for Retry-After seconds in the header if rate limited
|
||||
// max retry count is 3
|
||||
const retryAfter = error.response?.headers['retry-after'] || '10' // default to 10 seconds
|
||||
await wait(parseInt(retryAfter, 10) * 1000)
|
||||
return this.syncWithReadwise(token, highlights, retryCount + 1)
|
||||
}
|
||||
console.log('Error creating highlights in Readwise', error)
|
||||
logger.info('Error creating highlights in Readwise', error)
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -9,12 +9,15 @@ import {
|
|||
validatedDate,
|
||||
wordsCount,
|
||||
} from '../utils/helpers'
|
||||
import { buildLogger } from '../utils/logger'
|
||||
import {
|
||||
FAKE_URL_PREFIX,
|
||||
parsePreparedContent,
|
||||
parseUrlMetadata,
|
||||
} from '../utils/parser'
|
||||
|
||||
const logger = buildLogger('app.dispatch')
|
||||
|
||||
export type SaveContext = {
|
||||
pubsub: PubsubClient
|
||||
uid: string
|
||||
|
|
@ -91,14 +94,14 @@ export const saveEmail = async (
|
|||
})
|
||||
if (page) {
|
||||
const result = await updatePage(page.id, { archivedAt: null }, ctx)
|
||||
console.log('updated page from email', result)
|
||||
logger.info('updated page from email', result)
|
||||
|
||||
return page
|
||||
}
|
||||
|
||||
const pageId = await createPage(articleToSave, ctx)
|
||||
if (!pageId) {
|
||||
console.log('failed to create new page')
|
||||
logger.info('failed to create new page')
|
||||
|
||||
return undefined
|
||||
}
|
||||
|
|
@ -110,9 +113,9 @@ export const saveEmail = async (
|
|||
slug,
|
||||
articleToSave.content
|
||||
)
|
||||
console.debug('Created thumbnail task', taskId)
|
||||
logger.info('Created thumbnail task', taskId)
|
||||
} catch (e) {
|
||||
console.log('Failed to create thumbnail task', e)
|
||||
logger.error('Failed to create thumbnail task', e)
|
||||
}
|
||||
|
||||
articleToSave.id = pageId
|
||||
|
|
|
|||
|
|
@ -22,10 +22,13 @@ import {
|
|||
validatedDate,
|
||||
wordsCount,
|
||||
} from '../utils/helpers'
|
||||
import { buildLogger } from '../utils/logger'
|
||||
import { parsePreparedContent } from '../utils/parser'
|
||||
import { createPageSaveRequest } from './create_page_save_request'
|
||||
import { createLabels } from './labels'
|
||||
|
||||
const logger = buildLogger('app.dispatch')
|
||||
|
||||
type SaveContext = {
|
||||
pubsub: PubsubClient
|
||||
models: DataModels
|
||||
|
|
@ -179,9 +182,9 @@ export const savePage = async (
|
|||
slug,
|
||||
articleToSave.content
|
||||
)
|
||||
console.debug('Created thumbnail task', taskId)
|
||||
logger.info('Created thumbnail task', taskId)
|
||||
} catch (e) {
|
||||
console.log('Failed to create thumbnail task', e)
|
||||
logger.error('Failed to create thumbnail task', e)
|
||||
}
|
||||
|
||||
if (parseResult.highlightData) {
|
||||
|
|
|
|||
|
|
@ -3,9 +3,12 @@ import { NewsletterEmail } from '../entity/newsletter_email'
|
|||
import { Subscription } from '../entity/subscription'
|
||||
import { getRepository } from '../entity/utils'
|
||||
import { SubscriptionStatus, SubscriptionType } from '../generated/graphql'
|
||||
import { buildLogger } from '../utils/logger'
|
||||
import { sendEmail } from '../utils/sendEmail'
|
||||
import { createNewsletterEmail } from './newsletters'
|
||||
|
||||
const logger = buildLogger('app.dispatch')
|
||||
|
||||
interface SaveSubscriptionInput {
|
||||
userId: string
|
||||
name: string
|
||||
|
|
@ -51,13 +54,13 @@ const sendUnsubscribeEmail = async (
|
|||
})
|
||||
|
||||
if (!sent) {
|
||||
console.log('Failed to send unsubscribe email', unsubscribeMailTo)
|
||||
logger.info('Failed to send unsubscribe email', unsubscribeMailTo)
|
||||
return false
|
||||
}
|
||||
|
||||
return true
|
||||
} catch (error) {
|
||||
console.log('Failed to send unsubscribe email', error)
|
||||
logger.info('Failed to send unsubscribe email', error)
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
|
@ -71,9 +74,9 @@ const sendUnsubscribeHttpRequest = async (url: string): Promise<boolean> => {
|
|||
return true
|
||||
} catch (error) {
|
||||
if (axios.isAxiosError(error)) {
|
||||
console.log('Failed to send unsubscribe http request', error.message)
|
||||
logger.info('Failed to send unsubscribe http request', error.message)
|
||||
} else {
|
||||
console.log('Failed to send unsubscribe http request', error)
|
||||
logger.info('Failed to send unsubscribe http request', error)
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
|
@ -147,7 +150,7 @@ export const unsubscribe = async (subscription: Subscription) => {
|
|||
|
||||
if (!unsubscribed) {
|
||||
// update subscription status to unsubscribed if failed to unsubscribe
|
||||
console.log('Failed to unsubscribe', subscription.id)
|
||||
logger.info('Failed to unsubscribe', subscription.id)
|
||||
return getRepository(Subscription).update(subscription.id, {
|
||||
status: SubscriptionStatus.Unsubscribed,
|
||||
})
|
||||
|
|
@ -174,11 +177,11 @@ export const unsubscribeAll = async (
|
|||
try {
|
||||
await unsubscribe(subscription)
|
||||
} catch (error) {
|
||||
console.log('Failed to unsubscribe', error)
|
||||
logger.info('Failed to unsubscribe', error)
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
console.log('Failed to unsubscribe all', error)
|
||||
logger.info('Failed to unsubscribe all', error)
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -211,7 +214,7 @@ export class SubscribeHandler {
|
|||
// subscribe to newsletter service
|
||||
const subscribedNames = await this._subscribe(newsletterEmail.address)
|
||||
if (subscribedNames.length === 0) {
|
||||
console.log('Failed to get subscribe response', name)
|
||||
logger.info('Failed to get subscribe response', name)
|
||||
return null
|
||||
}
|
||||
|
||||
|
|
@ -229,7 +232,7 @@ export class SubscribeHandler {
|
|||
|
||||
return Promise.all(newSubscriptions)
|
||||
} catch (error) {
|
||||
console.log('Failed to handleSubscribe', error)
|
||||
logger.info('Failed to handleSubscribe', error)
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -149,13 +149,13 @@ export const createAppEngineTask = async ({
|
|||
}
|
||||
}
|
||||
|
||||
console.log('Sending task:')
|
||||
console.log(task)
|
||||
logger.info('Sending task:')
|
||||
logger.info(task)
|
||||
// Send create task request.
|
||||
const request = { parent: parent, task: task }
|
||||
const [response] = await client.createTask(request)
|
||||
const name = response.name
|
||||
console.log(`Created task ${name}`)
|
||||
logger.info(`Created task ${name}`)
|
||||
|
||||
return name
|
||||
}
|
||||
|
|
@ -245,7 +245,7 @@ export const enqueueParseRequest = async ({
|
|||
// Calling the handler function directly.
|
||||
setTimeout(() => {
|
||||
axios.post(env.queue.contentFetchUrl, payload).catch((error) => {
|
||||
console.error(error)
|
||||
logger.error(error)
|
||||
logger.warning(
|
||||
`Error occurred while requesting local puppeteer-parse function\nPlease, ensure your function is set up properly and running using "yarn start" from the "/pkg/gcf/puppeteer-parse" folder`
|
||||
)
|
||||
|
|
@ -494,7 +494,7 @@ export const enqueueImportFromIntegration = async (
|
|||
headers,
|
||||
})
|
||||
.catch((error) => {
|
||||
console.error(error)
|
||||
logger.error(error)
|
||||
})
|
||||
}, 0)
|
||||
return nanoid()
|
||||
|
|
@ -543,7 +543,7 @@ export const enqueueThumbnailTask = async (
|
|||
headers,
|
||||
})
|
||||
.catch((error) => {
|
||||
console.error(error)
|
||||
logger.error(error)
|
||||
})
|
||||
}, 0)
|
||||
return ''
|
||||
|
|
@ -590,7 +590,7 @@ export const enqueueRssFeedFetch = async (
|
|||
headers,
|
||||
})
|
||||
.catch((error) => {
|
||||
console.error(error)
|
||||
logger.error(error)
|
||||
})
|
||||
}, 0)
|
||||
return nanoid()
|
||||
|
|
|
|||
|
|
@ -3,6 +3,9 @@ import { parseHTML } from 'linkedom'
|
|||
import { nanoid } from 'nanoid'
|
||||
import { v4 as uuidv4 } from 'uuid'
|
||||
import { interpolationSearch } from './interpolationSearch'
|
||||
import { buildLogger } from './logger'
|
||||
|
||||
const logger = buildLogger('app.dispatch')
|
||||
|
||||
const highlightTag = 'omnivore_highlight'
|
||||
export const maxHighlightLength = 2000
|
||||
|
|
@ -72,7 +75,7 @@ function getTextNodesBetween(rootNode: Node, startNode: Node, endNode: Node) {
|
|||
// If the function takes too long, throw an error
|
||||
if (Date.now() - start > maxTime) {
|
||||
const error = new Error('getTextNodes Timeout')
|
||||
console.error(error)
|
||||
logger.error(error)
|
||||
throw error
|
||||
}
|
||||
|
||||
|
|
@ -164,7 +167,7 @@ export const findEmbeddedHighlight = (
|
|||
suffix: info.suffix,
|
||||
}
|
||||
} catch (error) {
|
||||
console.error(error)
|
||||
logger.error(error)
|
||||
return undefined
|
||||
}
|
||||
}
|
||||
|
|
@ -407,7 +410,7 @@ export function getArticleTextNodes(
|
|||
const rootNode = document.getRootNode()
|
||||
return getTextNodesBetween(rootNode, rootNode, rootNode)
|
||||
} catch (error) {
|
||||
console.error(error)
|
||||
logger.error(error)
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -26,7 +26,7 @@ import {
|
|||
findEmbeddedHighlight,
|
||||
getArticleTextNodes,
|
||||
highlightIdAttribute,
|
||||
makeHighlightNodeAttributes,
|
||||
makeHighlightNodeAttributes
|
||||
} from './highlightGenerator'
|
||||
import { createImageProxyUrl } from './imageproxy'
|
||||
import { buildLogger, LogRecord } from './logger'
|
||||
|
|
@ -183,7 +183,7 @@ const getReadabilityResult = async (
|
|||
return article
|
||||
}
|
||||
} catch (error) {
|
||||
console.log('parsing error for url', url, error)
|
||||
logger.info('parsing error for url', url, error)
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -208,7 +208,7 @@ export const parsePreparedContent = async (
|
|||
const { document, pageInfo } = preparedDocument
|
||||
|
||||
if (!document) {
|
||||
console.log('No document')
|
||||
logger.info('No document')
|
||||
return {
|
||||
canonicalUrl: url,
|
||||
parsedContent: null,
|
||||
|
|
@ -223,7 +223,7 @@ export const parsePreparedContent = async (
|
|||
pageInfo.contentType &&
|
||||
!ALLOWED_CONTENT_TYPES.includes(pageInfo.contentType)
|
||||
) {
|
||||
console.log('Not allowed content type', pageInfo.contentType)
|
||||
logger.info('Not allowed content type', pageInfo.contentType)
|
||||
return {
|
||||
canonicalUrl: url,
|
||||
parsedContent: null,
|
||||
|
|
@ -348,7 +348,7 @@ export const parsePreparedContent = async (
|
|||
})
|
||||
logRecord.parseSuccess = true
|
||||
} catch (error) {
|
||||
console.log('Error parsing content', error)
|
||||
logger.info('Error parsing content', error)
|
||||
Object.assign(logRecord, {
|
||||
parseSuccess: false,
|
||||
parseError: error,
|
||||
|
|
@ -443,7 +443,7 @@ export const parsePageMetadata = (html: string): Metadata | undefined => {
|
|||
|
||||
return { title, author, description, previewImage }
|
||||
} catch (e) {
|
||||
console.log('failed to parse page:', e)
|
||||
logger.info('failed to parse page:', e)
|
||||
return undefined
|
||||
}
|
||||
}
|
||||
|
|
@ -455,7 +455,7 @@ export const parseUrlMetadata = async (
|
|||
const res = await axios.get(url)
|
||||
return parsePageMetadata(res.data)
|
||||
} catch (e) {
|
||||
console.log('failed to get:', url, e)
|
||||
logger.info('failed to get:', url, e)
|
||||
return undefined
|
||||
}
|
||||
}
|
||||
|
|
@ -500,7 +500,7 @@ export const fetchFavicon = async (
|
|||
const domain = new URL(realUrl).hostname
|
||||
return `https://api.faviconkit.com/${domain}/128`
|
||||
} catch (e) {
|
||||
console.log('Error fetching favicon', e)
|
||||
logger.info('Error fetching favicon', e)
|
||||
return undefined
|
||||
}
|
||||
}
|
||||
|
|
@ -635,7 +635,7 @@ export const htmlToHighlightedMarkdown = (
|
|||
throw new Error('Invalid html content')
|
||||
}
|
||||
} catch (err) {
|
||||
console.log(err)
|
||||
logger.info(err)
|
||||
return nhm.translate(/* html */ html)
|
||||
}
|
||||
|
||||
|
|
@ -655,7 +655,7 @@ export const htmlToHighlightedMarkdown = (
|
|||
articleTextNodes
|
||||
)
|
||||
} catch (err) {
|
||||
console.log(err)
|
||||
logger.info(err)
|
||||
}
|
||||
})
|
||||
html = document.documentElement.outerHTML
|
||||
|
|
@ -674,14 +674,14 @@ export const getDistillerResult = async (
|
|||
try {
|
||||
const url = process.env.DISTILLER_URL
|
||||
if (!url) {
|
||||
console.log('No distiller url')
|
||||
logger.info('No distiller url')
|
||||
return undefined
|
||||
}
|
||||
|
||||
const exp = Math.floor(Date.now() / 1000) + 60 * 60 // 1 hour
|
||||
const auth = (await signToken({ uid, exp }, env.server.jwtSecret)) as string
|
||||
|
||||
console.debug('Parsing by distiller', url)
|
||||
logger.info('Parsing by distiller', url)
|
||||
const response = await axios.post<string>(url, html, {
|
||||
headers: {
|
||||
Authorization: auth,
|
||||
|
|
@ -690,7 +690,7 @@ export const getDistillerResult = async (
|
|||
})
|
||||
return response.data
|
||||
} catch (e) {
|
||||
console.log('Error parsing by distiller', e)
|
||||
logger.info('Error parsing by distiller', e)
|
||||
return undefined
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue