From c07ada02185e3f325ce3f4262c7c7054caf278a3 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Wed, 22 Mar 2023 15:13:48 +0800 Subject: [PATCH] resolve conflicts --- packages/api/src/resolvers/article/index.ts | 2 +- .../resolvers/article_saving_request/index.ts | 9 +- packages/api/src/routers/article_router.ts | 2 - packages/api/src/routers/svc/integrations.ts | 10 +- .../src/services/create_page_save_request.ts | 16 ++- packages/api/src/services/save_page.ts | 19 +--- packages/api/src/utils/createTask.ts | 107 ++++-------------- packages/api/src/utils/helpers.ts | 24 ++-- packages/api/tsconfig.json | 2 +- 9 files changed, 64 insertions(+), 127 deletions(-) diff --git a/packages/api/src/resolvers/article/index.ts b/packages/api/src/resolvers/article/index.ts index 307f737e6..bc9f6603c 100644 --- a/packages/api/src/resolvers/article/index.ts +++ b/packages/api/src/resolvers/article/index.ts @@ -72,6 +72,7 @@ import { UpdatesSinceSuccess, } from '../../generated/graphql' import { createPageSaveRequest } from '../../services/create_page_save_request' +import { createLabels } from '../../services/labels' import { parsedContentToPage } from '../../services/save_page' import { traceAs } from '../../tracing' import { Merge } from '../../util' @@ -101,7 +102,6 @@ import { makeStorageFilePublic, } from '../../utils/uploads' import { WithDataSourcesContext } from '../types' -import { createLabels } from '../../services/labels' enum ArticleFormat { Markdown = 'markdown', diff --git a/packages/api/src/resolvers/article_saving_request/index.ts b/packages/api/src/resolvers/article_saving_request/index.ts index 2c431dfb0..2417b3d0d 100644 --- a/packages/api/src/resolvers/article_saving_request/index.ts +++ b/packages/api/src/resolvers/article_saving_request/index.ts @@ -1,5 +1,7 @@ /* eslint-disable prefer-const */ import { getPageByParam } from '../../elastic/pages' +import { User } from '../../entity/user' +import { getRepository } from '../../entity/utils' import { env } from '../../env' import { ArticleSavingRequestError, @@ -60,11 +62,14 @@ export const articleSavingRequestResolver = authorized< ArticleSavingRequestSuccess, ArticleSavingRequestError, QueryArticleSavingRequestArgs ->(async (_, { id, url }, { models, claims }) => { +>(async (_, { id, url }, { claims }) => { if (!id && !url) { return { errorCodes: [ArticleSavingRequestErrorCode.BadData] } } - const user = await models.user.get(claims.uid) + const user = await getRepository(User).findOne({ + where: { id: claims.uid }, + relations: ['profile'], + }) if (!user) { return { errorCodes: [ArticleSavingRequestErrorCode.Unauthorized] } } diff --git a/packages/api/src/routers/article_router.ts b/packages/api/src/routers/article_router.ts index dc28281c1..c35b39302 100644 --- a/packages/api/src/routers/article_router.ts +++ b/packages/api/src/routers/article_router.ts @@ -4,7 +4,6 @@ import { htmlToSpeechFile } from '@omnivore/text-to-speech-handler' import cors from 'cors' import express from 'express' import * as jwt from 'jsonwebtoken' -import { kx } from '../datalayer/knex_config' import { createPubSubClient } from '../datalayer/pubsub' import { getPageById, updatePage } from '../elastic/pages' import { Speech, SpeechState } from '../entity/speech' @@ -12,7 +11,6 @@ import { getRepository } from '../entity/utils' import { env } from '../env' import { CreateArticleErrorCode } from '../generated/graphql' import { Claims } from '../resolvers/types' -import { initModels } from '../server' import { createPageSaveRequest } from '../services/create_page_save_request' import { getClaimsByToken } from '../utils/auth' import { isSiteBlockedForParse } from '../utils/blocked' diff --git a/packages/api/src/routers/svc/integrations.ts b/packages/api/src/routers/svc/integrations.ts index 981028576..efc8f85d0 100644 --- a/packages/api/src/routers/svc/integrations.ts +++ b/packages/api/src/routers/svc/integrations.ts @@ -2,19 +2,19 @@ /* eslint-disable @typescript-eslint/no-unsafe-assignment */ /* eslint-disable @typescript-eslint/no-unsafe-member-access */ import express from 'express' +import { DateTime } from 'luxon' +import { v4 as uuidv4 } from 'uuid' import { EntityType, readPushSubscription } from '../../datalayer/pubsub' import { getPageById, searchPages } from '../../elastic/pages' import { Page } from '../../elastic/types' import { Integration, IntegrationType } from '../../entity/integration' import { getRepository } from '../../entity/utils' -import { syncWithIntegration } from '../../services/integrations' +import { Claims } from '../../resolvers/types' +import { getIntegrationService } from '../../services/integrations' +import { getClaimsByToken } from '../../utils/auth' import { buildLogger } from '../../utils/logger' import { DateFilter } from '../../utils/search' -import { DateTime } from 'luxon' import { createGCSFile } from '../../utils/uploads' -import { v4 as uuidv4 } from 'uuid' -import { getClaimsByToken } from '../../utils/auth' -import { Claims } from '../../resolvers/types' export interface Message { type?: EntityType diff --git a/packages/api/src/services/create_page_save_request.ts b/packages/api/src/services/create_page_save_request.ts index 77c3748ee..335421e76 100644 --- a/packages/api/src/services/create_page_save_request.ts +++ b/packages/api/src/services/create_page_save_request.ts @@ -9,17 +9,14 @@ import { updatePage, } from '../elastic/pages' import { ArticleSavingRequestStatus, Label, PageType } from '../elastic/types' +import { User } from '../entity/user' +import { getRepository } from '../entity/utils' import { ArticleSavingRequest, CreateArticleSavingRequestErrorCode, } from '../generated/graphql' -// TODO: switch to a proper Entity instead of using the old data models. -import { DataModels } from '../resolvers/types' import { enqueueParseRequest } from '../utils/createTask' import { generateSlug, pageToArticleSavingRequest } from '../utils/helpers' -import * as privateIpLib from 'private-ip' -import { getRepository } from '../entity/utils' -import { User } from '../entity/user' interface PageSaveRequest { userId: string @@ -157,7 +154,14 @@ export const createPageSaveRequest = async ({ ) } // enqueue task to parse page - await enqueueParseRequest(url, userId, page.id, priority) + await enqueueParseRequest({ + url, + userId, + saveRequestId: page.id, + priority, + archivedAt, + labels, + }) return pageToArticleSavingRequest(user, page) } diff --git a/packages/api/src/services/save_page.ts b/packages/api/src/services/save_page.ts index 78edd5c09..4a43f6e3f 100644 --- a/packages/api/src/services/save_page.ts +++ b/packages/api/src/services/save_page.ts @@ -109,10 +109,10 @@ export const savePage = async ( url: articleToSave.url, }) // save state - const archivedAt = + articleToSave.archivedAt = input.state === ArticleSavingRequestStatus.Archived ? new Date() : null // add labels to page - const labels = input.labels + articleToSave.labels = input.labels ? await createLabels(ctx, input.labels) : undefined @@ -125,11 +125,9 @@ export const savePage = async ( { // update the page with the new content ...articleToSave, - archivedAt, // unarchive if it was archived id: pageId, // we don't want to update the id slug, // we don't want to update the slug createdAt: existingPage.createdAt, // we don't want to update the createdAt - labels, }, ctx )) @@ -146,8 +144,8 @@ export const savePage = async ( url: articleToSave.url, pubsub: ctx.pubsub, articleSavingRequestId: input.clientRequestId, - archivedAt, - labels, + archivedAt: articleToSave.archivedAt, + labels: articleToSave.labels, }) } catch (e) { return { @@ -156,14 +154,7 @@ export const savePage = async ( } } } else { - const newPageId = await createPage( - { - ...articleToSave, - archivedAt, - labels, - }, - ctx - ) + const newPageId = await createPage(articleToSave, ctx) if (!newPageId) { return { errorCodes: [SaveErrorCode.Unknown], diff --git a/packages/api/src/utils/createTask.ts b/packages/api/src/utils/createTask.ts index b6b16262b..85fe44b4d 100644 --- a/packages/api/src/utils/createTask.ts +++ b/packages/api/src/utils/createTask.ts @@ -2,14 +2,14 @@ /* eslint-disable @typescript-eslint/restrict-template-expressions */ // Imports the Google Cloud Tasks library. import { CloudTasksClient, protos } from '@google-cloud/tasks' +import { google } from '@google-cloud/tasks/build/protos/protos' import axios from 'axios' +import { nanoid } from 'nanoid' +import { Label, Recommendation } from '../elastic/types' import { env } from '../env' +import { signFeatureToken } from '../services/features' import { CreateTaskError } from './errors' import { buildLogger } from './logger' -import { nanoid } from 'nanoid' -import { google } from '@google-cloud/tasks/build/protos/protos' -import { signFeatureToken } from '../services/features' -import { Recommendation } from '../elastic/types' import View = google.cloud.tasks.v2.Task.View const logger = buildLogger('app.dispatch') @@ -194,18 +194,30 @@ export const deleteTask = async ( * @param queue - Queue name * @returns Name of the task created */ -export const enqueueParseRequest = async ( - url: string, - userId: string, - saveRequestId: string, - priority: 'low' | 'high' = 'high', - queue = env.queue.name -): Promise => { +export const enqueueParseRequest = async ({ + url, + userId, + saveRequestId, + priority = 'high', + queue = env.queue.name, + archivedAt, + labels, +}: { + url: string + userId: string + saveRequestId: string + priority?: 'low' | 'high' + queue?: string + archivedAt?: Date | null + labels?: Label[] +}): Promise => { const { GOOGLE_CLOUD_PROJECT } = process.env const payload = { url, userId, saveRequestId, + archivedAt, + labels, } // If there is no Google Cloud Project Id exposed, it means that we are in local environment @@ -245,79 +257,6 @@ export const enqueueParseRequest = async ( return createdTasks[0].name } -export const enqueueReminder = async ( - userId: string, - scheduleTime: number -): Promise => { - const { GOOGLE_CLOUD_PROJECT } = process.env - const payload = { - userId, - scheduleTime, - } - - // If there is no Google Cloud Project Id exposed, it means that we are in local environment - if (env.dev.isLocal || !GOOGLE_CLOUD_PROJECT) { - return nanoid() - } - - const createdTasks = await createHttpTaskWithToken({ - project: GOOGLE_CLOUD_PROJECT, - payload, - scheduleTime, - taskHandlerUrl: env.queue.reminderTaskHandlerUrl, - }) - - if (!createdTasks || !createdTasks[0].name) { - logger.error(`Unable to get the name of the task`, { - payload, - createdTasks, - }) - throw new CreateTaskError(`Unable to get the name of the task`) - } - return createdTasks[0].name -} - -export const enqueueSyncWithIntegration = async ( - userId: string, - integrationName: string -): Promise => { - const { GOOGLE_CLOUD_PROJECT, PUBSUB_VERIFICATION_TOKEN } = process.env - // use pubsub data format to send the userId to the task handler - const payload = { - message: { - data: Buffer.from( - JSON.stringify({ - userId, - }) - ).toString('base64'), - publishTime: new Date().toISOString(), - }, - } - - // If there is no Google Cloud Project Id exposed, it means that we are in local environment - if (env.dev.isLocal || !GOOGLE_CLOUD_PROJECT) { - return nanoid() - } - - const createdTasks = await createHttpTaskWithToken({ - project: GOOGLE_CLOUD_PROJECT, - payload, - taskHandlerUrl: `${ - env.queue.integrationTaskHandlerUrl - }/${integrationName.toLowerCase()}/sync_all?token=${PUBSUB_VERIFICATION_TOKEN}`, - priority: 'low', - }) - - if (!createdTasks || !createdTasks[0].name) { - logger.error(`Unable to get the name of the task`, { - payload, - createdTasks, - }) - throw new CreateTaskError(`Unable to get the name of the task`) - } - return createdTasks[0].name -} - export const enqueueTextToSpeech = async ({ userId, text, diff --git a/packages/api/src/utils/helpers.ts b/packages/api/src/utils/helpers.ts index a51e120b1..ecf51afc0 100644 --- a/packages/api/src/utils/helpers.ts +++ b/packages/api/src/utils/helpers.ts @@ -1,4 +1,14 @@ /* eslint-disable @typescript-eslint/no-unsafe-assignment */ +import crypto from 'crypto' +import normalizeUrl from 'normalize-url' +import path from 'path' +import _ from 'underscore' +import slugify from 'voca/slugify' +import wordsCounter from 'word-counting' +import { RegistrationType, UserData } from '../datalayer/user/model' +import { updatePage } from '../elastic/pages' +import { ArticleSavingRequestStatus, Page } from '../elastic/types' +import { User } from '../entity/user' import { ArticleSavingRequest, CreateArticleError, @@ -6,19 +16,9 @@ import { Profile, ResolverFn, } from '../generated/graphql' -import { Claims, WithDataSourcesContext } from '../resolvers/types' -import { RegistrationType, UserData } from '../datalayer/user/model' -import crypto from 'crypto' -import slugify from 'voca/slugify' -import { Merge } from '../util' import { CreateArticlesSuccessPartial } from '../resolvers' -import { ArticleSavingRequestStatus, Page } from '../elastic/types' -import { updatePage } from '../elastic/pages' -import path from 'path' -import normalizeUrl from 'normalize-url' -import wordsCounter from 'word-counting' -import _ from 'underscore' -import { User } from '../entity/user' +import { Claims, WithDataSourcesContext } from '../resolvers/types' +import { Merge } from '../util' interface InputObject { // eslint-disable-next-line @typescript-eslint/no-explicit-any diff --git a/packages/api/tsconfig.json b/packages/api/tsconfig.json index ee1fb782a..7c8caecfd 100644 --- a/packages/api/tsconfig.json +++ b/packages/api/tsconfig.json @@ -7,5 +7,5 @@ "outDir": "dist" }, "include": ["src", "test"], - "exclude": ["./src/generated"] + "exclude": ["./src/generated", "./test"] }