diff --git a/packages/api/src/entity/received_email.ts b/packages/api/src/entity/received_email.ts new file mode 100644 index 000000000..b68289989 --- /dev/null +++ b/packages/api/src/entity/received_email.ts @@ -0,0 +1,44 @@ +import { + Column, + CreateDateColumn, + Entity, + JoinColumn, + ManyToOne, + PrimaryGeneratedColumn, + UpdateDateColumn, +} from 'typeorm' +import { User } from './user' + +@Entity({ name: 'received_emails' }) +export class ReceivedEmail { + @PrimaryGeneratedColumn('uuid') + id!: string + + @ManyToOne(() => User, { onDelete: 'CASCADE' }) + @JoinColumn({ name: 'user_id' }) + user!: User + + @Column('text') + from!: string + + @Column('text') + to!: string + + @Column('text') + subject!: string + + @Column('text') + text!: string + + @Column('text') + html!: string + + @Column('text') + type!: 'article' | 'non-article' + + @CreateDateColumn({ default: () => 'CURRENT_TIMESTAMP' }) + createdAt!: Date + + @UpdateDateColumn({ default: () => 'CURRENT_TIMESTAMP' }) + updatedAt!: Date +} diff --git a/packages/api/src/generated/graphql.ts b/packages/api/src/generated/graphql.ts index 781aa004e..f6cfc874c 100644 --- a/packages/api/src/generated/graphql.ts +++ b/packages/api/src/generated/graphql.ts @@ -1062,6 +1062,24 @@ export type LoginSuccess = { me: User; }; +export type MarkEmailAsItemError = { + __typename?: 'MarkEmailAsItemError'; + errorCodes: Array; +}; + +export enum MarkEmailAsItemErrorCode { + BadRequest = 'BAD_REQUEST', + NotFound = 'NOT_FOUND', + Unauthorized = 'UNAUTHORIZED' +} + +export type MarkEmailAsItemResult = MarkEmailAsItemError | MarkEmailAsItemSuccess; + +export type MarkEmailAsItemSuccess = { + __typename?: 'MarkEmailAsItemSuccess'; + success: Scalars['Boolean']; +}; + export type MergeHighlightError = { __typename?: 'MergeHighlightError'; errorCodes: Array; @@ -1172,6 +1190,7 @@ export type Mutation = { joinGroup: JoinGroupResult; leaveGroup: LeaveGroupResult; logOut: LogOutResult; + markEmailAsItem: MarkEmailAsItemResult; mergeHighlight: MergeHighlightResult; moveFilter: MoveFilterResult; moveLabel: MoveLabelResult; @@ -1338,6 +1357,11 @@ export type MutationLeaveGroupArgs = { }; +export type MutationMarkEmailAsItemArgs = { + recentEmailId: Scalars['ID']; +}; + + export type MutationMergeHighlightArgs = { input: MergeHighlightInput; }; @@ -1669,6 +1693,7 @@ export type Query = { labels: LabelsResult; me?: Maybe; newsletterEmails: NewsletterEmailsResult; + recentEmails: RecentEmailsResult; recentSearches: RecentSearchesResult; reminder: ReminderResult; rules: RulesResult; @@ -1812,6 +1837,35 @@ export type ReadState = { readingTime?: Maybe; }; +export type RecentEmail = { + __typename?: 'RecentEmail'; + createdAt: Scalars['Date']; + from: Scalars['String']; + html?: Maybe; + id: Scalars['ID']; + subject: Scalars['String']; + text: Scalars['String']; + to: Scalars['String']; + type: Scalars['String']; +}; + +export type RecentEmailsError = { + __typename?: 'RecentEmailsError'; + errorCodes: Array; +}; + +export enum RecentEmailsErrorCode { + BadRequest = 'BAD_REQUEST', + Unauthorized = 'UNAUTHORIZED' +} + +export type RecentEmailsResult = RecentEmailsError | RecentEmailsSuccess; + +export type RecentEmailsSuccess = { + __typename?: 'RecentEmailsSuccess'; + recentEmails: Array; +}; + export type RecentSearch = { __typename?: 'RecentSearch'; createdAt: Scalars['Date']; @@ -3316,6 +3370,10 @@ export type ResolversTypes = { LoginErrorCode: LoginErrorCode; LoginResult: ResolversTypes['LoginError'] | ResolversTypes['LoginSuccess']; LoginSuccess: ResolverTypeWrapper; + MarkEmailAsItemError: ResolverTypeWrapper; + MarkEmailAsItemErrorCode: MarkEmailAsItemErrorCode; + MarkEmailAsItemResult: ResolversTypes['MarkEmailAsItemError'] | ResolversTypes['MarkEmailAsItemSuccess']; + MarkEmailAsItemSuccess: ResolverTypeWrapper; MergeHighlightError: ResolverTypeWrapper; MergeHighlightErrorCode: MergeHighlightErrorCode; MergeHighlightInput: MergeHighlightInput; @@ -3353,6 +3411,11 @@ export type ResolversTypes = { Reaction: ResolverTypeWrapper; ReactionType: ReactionType; ReadState: ResolverTypeWrapper; + RecentEmail: ResolverTypeWrapper; + RecentEmailsError: ResolverTypeWrapper; + RecentEmailsErrorCode: RecentEmailsErrorCode; + RecentEmailsResult: ResolversTypes['RecentEmailsError'] | ResolversTypes['RecentEmailsSuccess']; + RecentEmailsSuccess: ResolverTypeWrapper; RecentSearch: ResolverTypeWrapper; RecentSearchesError: ResolverTypeWrapper; RecentSearchesErrorCode: RecentSearchesErrorCode; @@ -3741,6 +3804,9 @@ export type ResolversParentTypes = { LoginError: LoginError; LoginResult: ResolversParentTypes['LoginError'] | ResolversParentTypes['LoginSuccess']; LoginSuccess: LoginSuccess; + MarkEmailAsItemError: MarkEmailAsItemError; + MarkEmailAsItemResult: ResolversParentTypes['MarkEmailAsItemError'] | ResolversParentTypes['MarkEmailAsItemSuccess']; + MarkEmailAsItemSuccess: MarkEmailAsItemSuccess; MergeHighlightError: MergeHighlightError; MergeHighlightInput: MergeHighlightInput; MergeHighlightResult: ResolversParentTypes['MergeHighlightError'] | ResolversParentTypes['MergeHighlightSuccess']; @@ -3771,6 +3837,10 @@ export type ResolversParentTypes = { Query: {}; Reaction: Reaction; ReadState: ReadState; + RecentEmail: RecentEmail; + RecentEmailsError: RecentEmailsError; + RecentEmailsResult: ResolversParentTypes['RecentEmailsError'] | ResolversParentTypes['RecentEmailsSuccess']; + RecentEmailsSuccess: RecentEmailsSuccess; RecentSearch: RecentSearch; RecentSearchesError: RecentSearchesError; RecentSearchesResult: ResolversParentTypes['RecentSearchesError'] | ResolversParentTypes['RecentSearchesSuccess']; @@ -4742,6 +4812,20 @@ export type LoginSuccessResolvers; }; +export type MarkEmailAsItemErrorResolvers = { + errorCodes?: Resolver, ParentType, ContextType>; + __isTypeOf?: IsTypeOfResolverFn; +}; + +export type MarkEmailAsItemResultResolvers = { + __resolveType: TypeResolveFn<'MarkEmailAsItemError' | 'MarkEmailAsItemSuccess', ParentType, ContextType>; +}; + +export type MarkEmailAsItemSuccessResolvers = { + success?: Resolver; + __isTypeOf?: IsTypeOfResolverFn; +}; + export type MergeHighlightErrorResolvers = { errorCodes?: Resolver, ParentType, ContextType>; __isTypeOf?: IsTypeOfResolverFn; @@ -4813,6 +4897,7 @@ export type MutationResolvers>; leaveGroup?: Resolver>; logOut?: Resolver; + markEmailAsItem?: Resolver>; mergeHighlight?: Resolver>; moveFilter?: Resolver>; moveLabel?: Resolver>; @@ -4943,6 +5028,7 @@ export type QueryResolvers; me?: Resolver, ParentType, ContextType>; newsletterEmails?: Resolver; + recentEmails?: Resolver; recentSearches?: Resolver; reminder?: Resolver>; rules?: Resolver>; @@ -4976,6 +5062,32 @@ export type ReadStateResolvers; }; +export type RecentEmailResolvers = { + createdAt?: Resolver; + from?: Resolver; + html?: Resolver, ParentType, ContextType>; + id?: Resolver; + subject?: Resolver; + text?: Resolver; + to?: Resolver; + type?: Resolver; + __isTypeOf?: IsTypeOfResolverFn; +}; + +export type RecentEmailsErrorResolvers = { + errorCodes?: Resolver, ParentType, ContextType>; + __isTypeOf?: IsTypeOfResolverFn; +}; + +export type RecentEmailsResultResolvers = { + __resolveType: TypeResolveFn<'RecentEmailsError' | 'RecentEmailsSuccess', ParentType, ContextType>; +}; + +export type RecentEmailsSuccessResolvers = { + recentEmails?: Resolver, ParentType, ContextType>; + __isTypeOf?: IsTypeOfResolverFn; +}; + export type RecentSearchResolvers = { createdAt?: Resolver; id?: Resolver; @@ -5911,6 +6023,9 @@ export type Resolvers = { LoginError?: LoginErrorResolvers; LoginResult?: LoginResultResolvers; LoginSuccess?: LoginSuccessResolvers; + MarkEmailAsItemError?: MarkEmailAsItemErrorResolvers; + MarkEmailAsItemResult?: MarkEmailAsItemResultResolvers; + MarkEmailAsItemSuccess?: MarkEmailAsItemSuccessResolvers; MergeHighlightError?: MergeHighlightErrorResolvers; MergeHighlightResult?: MergeHighlightResultResolvers; MergeHighlightSuccess?: MergeHighlightSuccessResolvers; @@ -5934,6 +6049,10 @@ export type Resolvers = { Query?: QueryResolvers; Reaction?: ReactionResolvers; ReadState?: ReadStateResolvers; + RecentEmail?: RecentEmailResolvers; + RecentEmailsError?: RecentEmailsErrorResolvers; + RecentEmailsResult?: RecentEmailsResultResolvers; + RecentEmailsSuccess?: RecentEmailsSuccessResolvers; RecentSearch?: RecentSearchResolvers; RecentSearchesError?: RecentSearchesErrorResolvers; RecentSearchesResult?: RecentSearchesResultResolvers; diff --git a/packages/api/src/generated/schema.graphql b/packages/api/src/generated/schema.graphql index c58faa3e7..4154541a5 100644 --- a/packages/api/src/generated/schema.graphql +++ b/packages/api/src/generated/schema.graphql @@ -943,6 +943,22 @@ type LoginSuccess { me: User! } +type MarkEmailAsItemError { + errorCodes: [MarkEmailAsItemErrorCode!]! +} + +enum MarkEmailAsItemErrorCode { + BAD_REQUEST + NOT_FOUND + UNAUTHORIZED +} + +union MarkEmailAsItemResult = MarkEmailAsItemError | MarkEmailAsItemSuccess + +type MarkEmailAsItemSuccess { + success: Boolean! +} + type MergeHighlightError { errorCodes: [MergeHighlightErrorCode!]! } @@ -1046,6 +1062,7 @@ type Mutation { joinGroup(inviteCode: String!): JoinGroupResult! leaveGroup(groupId: ID!): LeaveGroupResult! logOut: LogOutResult! + markEmailAsItem(recentEmailId: ID!): MarkEmailAsItemResult! mergeHighlight(input: MergeHighlightInput!): MergeHighlightResult! moveFilter(input: MoveFilterInput!): MoveFilterResult! moveLabel(input: MoveLabelInput!): MoveLabelResult! @@ -1218,6 +1235,7 @@ type Query { labels: LabelsResult! me: User newsletterEmails: NewsletterEmailsResult! + recentEmails: RecentEmailsResult! recentSearches: RecentSearchesResult! reminder(linkId: ID!): ReminderResult! rules(enabled: Boolean): RulesResult! @@ -1258,6 +1276,32 @@ type ReadState { readingTime: Int } +type RecentEmail { + createdAt: Date! + from: String! + html: String + id: ID! + subject: String! + text: String! + to: String! + type: String! +} + +type RecentEmailsError { + errorCodes: [RecentEmailsErrorCode!]! +} + +enum RecentEmailsErrorCode { + BAD_REQUEST + UNAUTHORIZED +} + +union RecentEmailsResult = RecentEmailsError | RecentEmailsSuccess + +type RecentEmailsSuccess { + recentEmails: [RecentEmail!]! +} + type RecentSearch { createdAt: Date! id: ID! diff --git a/packages/api/src/resolvers/function_resolvers.ts b/packages/api/src/resolvers/function_resolvers.ts index 39dac70c3..ea5ff9ecf 100644 --- a/packages/api/src/resolvers/function_resolvers.ts +++ b/packages/api/src/resolvers/function_resolvers.ts @@ -117,6 +117,7 @@ import { getPageByParam } from '../elastic/pages' import { recentSearchesResolver } from './recent_searches' import { optInFeatureResolver } from './features' import { uploadImportFileResolver } from './importers/uploadImportFileResolver' +import { markEmailAsItemResolver, recentEmailsResolver } from './recent_emails' /* eslint-disable @typescript-eslint/naming-convention */ type ResultResolveType = { @@ -199,6 +200,7 @@ export const functionResolvers = { recommendHighlights: recommendHighlightsResolver, leaveGroup: leaveGroupResolver, uploadImportFile: uploadImportFileResolver, + markEmailAsItem: markEmailAsItemResolver, }, Query: { me: getMeUserResolver, @@ -230,6 +232,7 @@ export const functionResolvers = { deviceTokens: deviceTokensResolver, filters: filtersResolver, groups: groupsResolver, + recentEmails: recentEmailsResolver, }, User: { async sharedArticles( @@ -659,4 +662,6 @@ export const functionResolvers = { ...resultResolveTypeResolver('RecommendHighlights'), ...resultResolveTypeResolver('LeaveGroup'), ...resultResolveTypeResolver('UploadImportFile'), + ...resultResolveTypeResolver('RecentEmails'), + ...resultResolveTypeResolver('MarkEmailAsItem'), } diff --git a/packages/api/src/resolvers/recent_emails/index.ts b/packages/api/src/resolvers/recent_emails/index.ts new file mode 100644 index 000000000..58fccb865 --- /dev/null +++ b/packages/api/src/resolvers/recent_emails/index.ts @@ -0,0 +1,143 @@ +import { + MarkEmailAsItemError, + MarkEmailAsItemErrorCode, + MarkEmailAsItemSuccess, + MutationMarkEmailAsItemArgs, + RecentEmailsError, + RecentEmailsErrorCode, + RecentEmailsSuccess, +} from '../../generated/graphql' +import { authorized } from '../../utils/helpers' +import { getRepository } from '../../entity/utils' +import { ReceivedEmail } from '../../entity/received_email' +import { saveNewsletterEmail } from '../../services/save_newsletter_email' +import { NewsletterEmail } from '../../entity/newsletter_email' +import { generateUniqueUrl, parseEmailAddress } from '../../utils/parser' +import { sendEmail } from '../../utils/sendEmail' +import { env } from '../../env' + +export const recentEmailsResolver = authorized< + RecentEmailsSuccess, + RecentEmailsError +>(async (_, __, { claims, log }) => { + log.info('Getting recent emails', { + labels: { + source: 'resolver', + resolver: 'recentEmailsResolver', + uid: claims.uid, + }, + }) + + try { + const recentEmails = await getRepository(ReceivedEmail).find({ + where: { user: { id: claims.uid } }, + order: { createdAt: 'DESC' }, + take: 20, + }) + + return { + recentEmails, + } + } catch (error) { + log.error('Error getting recent emails', { + error, + labels: { + source: 'resolver', + resolver: 'recentEmailsResolver', + uid: claims.uid, + }, + }) + + return { + errorCodes: [RecentEmailsErrorCode.BadRequest], + } + } +}) + +export const markEmailAsItemResolver = authorized< + MarkEmailAsItemSuccess, + MarkEmailAsItemError, + MutationMarkEmailAsItemArgs +>(async (_, { recentEmailId }, { claims, log }) => { + log.info('Marking email as item', { + recentEmailId, + labels: { + source: 'resolver', + resolver: 'markEmailAsItemResolver', + uid: claims.uid, + }, + }) + + try { + const recentEmail = await getRepository(ReceivedEmail).findOneBy({ + id: recentEmailId, + user: { id: claims.uid }, + type: 'non-article', + }) + if (!recentEmail) { + log.info('no recent email', recentEmailId) + + return { + errorCodes: [MarkEmailAsItemErrorCode.Unauthorized], + } + } + + const newsletterEmail = await getRepository(NewsletterEmail).findOne({ + where: { + address: recentEmail.to, + user: { id: claims.uid }, + }, + relations: ['user'], + }) + if (!newsletterEmail) { + log.info('no newsletter email', recentEmail.to) + + return { + errorCodes: [MarkEmailAsItemErrorCode.NotFound], + } + } + + const success = await saveNewsletterEmail( + { + from: recentEmail.from, + email: recentEmail.to, + title: recentEmail.subject, + content: recentEmail.html, + url: generateUniqueUrl(), + author: parseEmailAddress(recentEmail.from).name, + receivedEmailId: recentEmail.id, + }, + newsletterEmail + ) + + const text = `A recent email marked as a library item + by: ${claims.uid} + from: ${recentEmail.from} + subject: ${recentEmail.subject}` + + // email us to let us know that an email failed to parse as an article + await sendEmail({ + to: env.sender.feedback, + subject: 'A recent email marked as a library item', + text, + from: env.sender.message, + }) + + return { + success, + } + } catch (error) { + log.error('Error marking email as item', { + error, + labels: { + source: 'resolver', + resolver: 'markEmailAsItemResolver', + uid: claims.uid, + }, + }) + + return { + errorCodes: [MarkEmailAsItemErrorCode.BadRequest], + } + } +}) diff --git a/packages/api/src/routers/svc/emails.ts b/packages/api/src/routers/svc/emails.ts index 5fa138812..10a7d949f 100644 --- a/packages/api/src/routers/svc/emails.ts +++ b/packages/api/src/routers/svc/emails.ts @@ -15,8 +15,15 @@ import { } from '../../utils/parser' import { saveEmail } from '../../services/save_email' import { buildLogger } from '../../utils/logger' +import { + saveReceivedEmail, + updateReceivedEmail, +} from '../../services/received_emails' +import cors from 'cors' +import { corsConfig } from '../../utils/corsConfig' +import { getClaimsByToken } from '../../utils/auth' -interface ForwardEmailMessage { +interface EmailMessage { from: string to: string subject: string @@ -25,6 +32,17 @@ interface ForwardEmailMessage { unsubHttpUrl?: string text: string forwardedFrom?: string + receivedEmailId: string +} + +function isEmailMessage(data: any): data is EmailMessage { + return ( + 'from' in data && + 'to' in data && + 'subject' in data && + 'html' in data && + 'text' in data + ) } const logger = buildLogger('app.dispatch') @@ -37,7 +55,6 @@ export function emailsServiceRouter() { logger.info('email forward router') const { message, expired } = readPushSubscription(req) - logger.info('pubsub message:', { message, expired }) if (!message) { res.status(400).send('Bad Request') @@ -51,15 +68,8 @@ export function emailsServiceRouter() { } try { - // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment - const data: ForwardEmailMessage = JSON.parse(message) - - if ( - !('from' in data) || - !('to' in data) || - !('subject' in data) || - (!('html' in data) && !('text' in data)) - ) { + const data = JSON.parse(message) as unknown + if (!isEmailMessage(data)) { logger.error('Invalid message') res.status(400).send('Bad Request') return @@ -90,6 +100,10 @@ export function emailsServiceRouter() { url: generateUniqueUrl(), originalContent: data.html || data.text, }) + + // update received email type + await updateReceivedEmail(data.receivedEmailId, 'article') + res.status(200).send('Article') return } @@ -130,5 +144,54 @@ export function emailsServiceRouter() { } }) + router.post('/save', cors(corsConfig), async (req, res) => { + logger.info('save received email router') + + const token = req?.headers?.authorization + if (!(await getClaimsByToken(token))) { + return res.status(401).send('UNAUTHORIZED') + } + + if (!isEmailMessage(req.body)) { + logger.error('Invalid message') + return res.status(400).send('Bad Request') + } + + try { + // get user from newsletter email + const newsletterEmail = await getNewsletterEmail(req.body.to) + + if (!newsletterEmail) { + logger.info('newsletter email not found', { email: req.body.to }) + res.status(200).send('Not Found') + return + } + + const user = newsletterEmail.user + const receivedEmail = await saveReceivedEmail( + req.body.from, + req.body.to, + req.body.subject, + req.body.text, + req.body.html, + user.id + ) + + analytics.track({ + userId: user.id, + event: 'received_email_saved', + properties: { + env: env.server.apiEnv, + }, + }) + + res.status(200).send({ id: receivedEmail.id }) + } catch (e) { + logger.info(e) + + res.status(500).send(e) + } + }) + return router } diff --git a/packages/api/src/routers/svc/newsletters.ts b/packages/api/src/routers/svc/newsletters.ts index 8edba67a7..48fee62a4 100644 --- a/packages/api/src/routers/svc/newsletters.ts +++ b/packages/api/src/routers/svc/newsletters.ts @@ -1,10 +1,14 @@ import express from 'express' import { readPushSubscription } from '../../datalayer/pubsub' -import { updateConfirmationCode } from '../../services/newsletters' +import { + getNewsletterEmail, + updateConfirmationCode, +} from '../../services/newsletters' import { NewsletterMessage, saveNewsletterEmail, } from '../../services/save_newsletter_email' +import { updateReceivedEmail } from '../../services/received_emails' interface SetConfirmationCodeMessage { emailAddress: string @@ -96,7 +100,14 @@ export function newsletterServiceRouter() { return } - const result = await saveNewsletterEmail(data) + // get user from newsletter email + const newsletterEmail = await getNewsletterEmail(data.email) + if (!newsletterEmail) { + console.log('newsletter email not found', data.email) + return false + } + + const result = await saveNewsletterEmail(data, newsletterEmail) if (!result) { console.log( 'Error creating newsletter link from data', @@ -104,10 +115,14 @@ export function newsletterServiceRouter() { data.title, data.author ) + res.status(500).send('Error creating newsletter link') return } + // update received email type + await updateReceivedEmail(data.receivedEmailId, 'article') + // We always send 200 if it was a valid message // because we don't want the res.status(200).send('newsletter created') diff --git a/packages/api/src/routers/svc/pdf_attachments.ts b/packages/api/src/routers/svc/pdf_attachments.ts index 85ea23d34..6a858ad23 100644 --- a/packages/api/src/routers/svc/pdf_attachments.ts +++ b/packages/api/src/routers/svc/pdf_attachments.ts @@ -17,6 +17,7 @@ import { createPubSubClient } from '../../datalayer/pubsub' import { ArticleSavingRequestStatus, Page } from '../../elastic/types' import { createPage } from '../../elastic/pages' import { getClaimsByToken } from '../../utils/auth' +import { updateReceivedEmail } from '../../services/received_emails' export function pdfAttachmentsRouter() { const router = express.Router() @@ -87,10 +88,11 @@ export function pdfAttachmentsRouter() { router.post('/create-article', async (req, res) => { console.log('pdf-attachments/create-article') - const { email, uploadFileId, subject } = req.body as { + const { email, uploadFileId, subject, receivedEmailId } = req.body as { email: string uploadFileId: string subject: string + receivedEmailId: string } const token = req?.headers?.authorization @@ -149,7 +151,7 @@ export function pdfAttachmentsRouter() { pageType: pageType, hash: uploadFileHash, uploadFileId: uploadFileId, - title: title, + title, content: '', userId: user.id, slug: generateSlug(title), @@ -166,6 +168,9 @@ export function pdfAttachmentsRouter() { uid: user.id, }) + // update received email type + await updateReceivedEmail(receivedEmailId, 'article') + res.send({ id: pageId }) } catch (err) { console.log(err) diff --git a/packages/api/src/schema.ts b/packages/api/src/schema.ts index 6732c45d0..64915652d 100755 --- a/packages/api/src/schema.ts +++ b/packages/api/src/schema.ts @@ -2331,6 +2331,48 @@ const schema = gql` uploadSignedUrl: String } + union RecentEmailsResult = RecentEmailsSuccess | RecentEmailsError + + type RecentEmailsSuccess { + recentEmails: [RecentEmail!]! + } + + type RecentEmailsError { + errorCodes: [RecentEmailsErrorCode!]! + } + + enum RecentEmailsErrorCode { + UNAUTHORIZED + BAD_REQUEST + } + + type RecentEmail { + id: ID! + from: String! + to: String! + subject: String! + type: String! + text: String! + html: String + createdAt: Date! + } + + union MarkEmailAsItemResult = MarkEmailAsItemSuccess | MarkEmailAsItemError + + type MarkEmailAsItemSuccess { + success: Boolean! + } + + type MarkEmailAsItemError { + errorCodes: [MarkEmailAsItemErrorCode!]! + } + + enum MarkEmailAsItemErrorCode { + UNAUTHORIZED + BAD_REQUEST + NOT_FOUND + } + # Mutations type Mutation { googleLogin(input: GoogleLoginInput!): LoginResult! @@ -2419,6 +2461,7 @@ const schema = gql` type: UploadImportFileType! contentType: String! ): UploadImportFileResult! + markEmailAsItem(recentEmailId: ID!): MarkEmailAsItemResult! } # FIXME: remove sort from feedArticles after all cached tabs are closed @@ -2481,6 +2524,7 @@ const schema = gql` deviceTokens: DeviceTokensResult! filters: FiltersResult! groups: GroupsResult! + recentEmails: RecentEmailsResult! } ` diff --git a/packages/api/src/services/received_emails.ts b/packages/api/src/services/received_emails.ts new file mode 100644 index 000000000..a3891bfcc --- /dev/null +++ b/packages/api/src/services/received_emails.ts @@ -0,0 +1,29 @@ +import { ReceivedEmail } from '../entity/received_email' +import { getRepository } from '../entity/utils' + +export const saveReceivedEmail = async ( + from: string, + to: string, + subject: string, + text: string, + html: string, + userId: string, + type: 'article' | 'non-article' = 'non-article' +): Promise => { + return getRepository(ReceivedEmail).save({ + from, + to, + subject, + text, + html, + type, + user: { id: userId }, + }) +} + +export const updateReceivedEmail = async ( + id: string, + type: 'article' | 'non-article' +) => { + await getRepository(ReceivedEmail).update(id, { type }) +} diff --git a/packages/api/src/services/save_newsletter_email.ts b/packages/api/src/services/save_newsletter_email.ts index e81806ec1..81e604d2c 100644 --- a/packages/api/src/services/save_newsletter_email.ts +++ b/packages/api/src/services/save_newsletter_email.ts @@ -4,7 +4,6 @@ import { UserDeviceToken } from '../entity/user_device_tokens' import { env } from '../env' import { ContentReader } from '../generated/graphql' import { analytics } from '../utils/analytics' -import { getNewsletterEmail } from './newsletters' import { SaveContext, saveEmail, SaveEmailInput } from './save_email' import { Page } from '../elastic/types' import { addLabelToPage } from './labels' @@ -13,8 +12,10 @@ import { NewsletterEmail } from '../entity/newsletter_email' import { fetchFavicon } from '../utils/parser' import { updatePage } from '../elastic/pages' import { isBase64Image } from '../utils/helpers' +import { updateReceivedEmail } from './received_emails' export interface NewsletterMessage { + from: string email: string content: string url: string @@ -22,23 +23,16 @@ export interface NewsletterMessage { author: string unsubMailTo?: string unsubHttpUrl?: string - newsletterEmail?: NewsletterEmail + receivedEmailId: string } // Returns true if the link was created successfully. Can still fail to // send the push but that is ok and we wont retry in that case. export const saveNewsletterEmail = async ( data: NewsletterMessage, + newsletterEmail: NewsletterEmail, ctx?: SaveContext ): Promise => { - // get user from newsletter email - const newsletterEmail = - data.newsletterEmail || (await getNewsletterEmail(data.email)) - if (!newsletterEmail) { - console.log('newsletter email not found', data.email) - return false - } - analytics.track({ userId: newsletterEmail.user.id, event: 'newsletter_email_received', @@ -64,10 +58,14 @@ export const saveNewsletterEmail = async ( } const page = await saveEmail(saveCtx, input) if (!page) { - console.log('newsletter not created:', input) + console.log('newsletter not created:', input.title) + return false } + // update received email type + await updateReceivedEmail(data.receivedEmailId, 'article') + if (!page.siteIcon || isBase64Image(page.siteIcon)) { // fetch favicon if not already set or is a base64 image const favicon = await fetchFavicon(page.url) diff --git a/packages/api/test/resolvers/recent_emails.test.ts b/packages/api/test/resolvers/recent_emails.test.ts new file mode 100644 index 000000000..3bae6f7b1 --- /dev/null +++ b/packages/api/test/resolvers/recent_emails.test.ts @@ -0,0 +1,146 @@ +import 'mocha' +import { expect } from 'chai' +import { User } from '../../src/entity/user' +import { createTestUser, deleteTestUser } from '../db' +import { graphqlRequest, request } from '../util' +import { getRepository } from '../../src/entity/utils' +import { ReceivedEmail } from '../../src/entity/received_email' +import { NewsletterEmail } from '../../src/entity/newsletter_email' + +describe('Recent Emails Resolver', () => { + const username = 'fakeUser' + + let user: User + let authToken: string + let newsletterEmail: NewsletterEmail + let newsletterEmail2: NewsletterEmail + + before(async () => { + // create test user and login + user = await createTestUser(username) + const res = await request + .post('/local/debug/fake-user-login') + .send({ fakeEmail: user.email }) + + authToken = res.body.authToken + + // create test newsletter email + newsletterEmail = await getRepository(NewsletterEmail).save({ + user: { id: user.id }, + address: 'fake email address', + }) + newsletterEmail2 = await getRepository(NewsletterEmail).save({ + user: { id: user.id }, + address: 'fake email address 2', + }) + }) + + after(async () => { + // clean up + await deleteTestUser(user.id) + }) + + describe('recentEmails', () => { + const recentEmailsQuery = ` + query { + recentEmails { + ... on RecentEmailsSuccess { + recentEmails { + id + from + to + subject + text + html + } + } + ... on RecentEmailsError { + errorCodes + } + } + } + ` + let recentEmails: ReceivedEmail[] + + before(async () => { + // create fake emails + const recentEmail = await getRepository(ReceivedEmail).save({ + user: { id: user.id }, + from: 'fake from', + subject: 'fake subject', + text: 'fake text', + html: 'fake html', + to: newsletterEmail.address, + type: 'article', + }) + const recentEmail2 = await getRepository(ReceivedEmail).save({ + user: { id: user.id }, + from: 'fake from 2', + subject: 'fake subject 2', + text: 'fake text 2', + html: 'fake html 2', + to: newsletterEmail2.address, + type: 'non-article', + }) + recentEmails = [recentEmail, recentEmail2] + }) + + it('returns recent emails', async () => { + const res = await graphqlRequest(recentEmailsQuery, authToken).expect(200) + const { recentEmails: results } = res.body.data.recentEmails + + expect(results).to.have.lengthOf(2) + expect(results[0].id).to.eql(recentEmails[1].id) + expect(results[1].id).to.eql(recentEmails[0].id) + }) + }) + + describe('markEmailAsItem', () => { + const markEmailAsItemMutation = (recentEmailId: string) => ` + mutation { + markEmailAsItem(recentEmailId: "${recentEmailId}") { + ... on MarkEmailAsItemSuccess { + success + } + ... on MarkEmailAsItemError { + errorCodes + } + } + } + ` + + let recentEmail: ReceivedEmail + + before(async () => { + // create fake email + recentEmail = await getRepository(ReceivedEmail).save({ + user: { id: user.id }, + from: 'Omnivore Newsletter ', + subject: 'fake subject 3', + text: 'fake text 3', + html: 'fake html 3', + to: newsletterEmail.address, + type: 'non-article', + }) + }) + + after(async () => { + // clean up + await getRepository(ReceivedEmail).delete(recentEmail.id) + }) + + it('marks email as item', async () => { + const resp = await graphqlRequest( + markEmailAsItemMutation(recentEmail.id), + authToken + ) + + expect(resp.body.data.markEmailAsItem.success).to.be.true + + const updatedRecentEmail = await getRepository(ReceivedEmail).findOneBy({ + id: recentEmail.id, + }) + expect(updatedRecentEmail?.type).to.eql('article') + }) + }) +}) diff --git a/packages/api/test/routers/emails.test.ts b/packages/api/test/routers/emails.test.ts index e8bdf5f15..1b68043b1 100644 --- a/packages/api/test/routers/emails.test.ts +++ b/packages/api/test/routers/emails.test.ts @@ -11,12 +11,20 @@ import { request } from '../util' import * as parser from '../../src/utils/parser' import * as sendNotification from '../../src/utils/sendNotification' import * as sendEmail from '../../src/utils/sendEmail' +import { getRepository } from '../../src/entity/utils' +import { ReceivedEmail } from '../../src/entity/received_email' +import * as jwt from 'jsonwebtoken' describe('Emails Router', () => { const newsletterEmail = 'fakeUser@omnivore.app' + const from = 'fake from' + const subject = 'fake subject' + const text = 'fake text' + const to = newsletterEmail let user: User let token: string + let receivedEmail: ReceivedEmail before(async () => { // create test user and login @@ -24,6 +32,15 @@ describe('Emails Router', () => { await createTestNewsletterEmail(user, newsletterEmail) token = process.env.PUBSUB_VERIFICATION_TOKEN! + receivedEmail = await getRepository(ReceivedEmail).save({ + user: { id: user.id }, + from, + to, + subject, + text, + html: '', + type: 'non-article', + }) }) after(async () => { @@ -33,10 +50,7 @@ describe('Emails Router', () => { }) describe('forward', () => { - const from = 'from@omnivore.app' - const to = newsletterEmail - const subject = 'test subject' - const html = 'test html' + const html = 'test html' beforeEach(async () => { sinon.replace( @@ -60,7 +74,14 @@ describe('Emails Router', () => { const data = { message: { data: Buffer.from( - JSON.stringify({ from, to, subject, html }) + JSON.stringify({ + from, + to, + subject, + html, + text, + receivedEmailId: receivedEmail.id, + }) ).toString('base64'), publishTime: new Date().toISOString(), }, @@ -82,7 +103,14 @@ describe('Emails Router', () => { const data = { message: { data: Buffer.from( - JSON.stringify({ from, to, subject, html }) + JSON.stringify({ + from, + to, + subject, + html, + text, + receivedEmailId: receivedEmail.id, + }) ).toString('base64'), publishTime: new Date().toISOString(), }, @@ -95,4 +123,30 @@ describe('Emails Router', () => { }) }) }) + + describe('create', () => { + const html = 'test html' + const text = 'test text' + const from = 'fake from' + const subject = 'fake subject' + const authToken = jwt.sign(newsletterEmail, process.env.JWT_SECRET || '') + + it('saves the email in the database', async () => { + const data = { + html, + text, + from, + to: newsletterEmail, + subject, + } + const res = await request + .post('/svc/pubsub/emails/save') + .set('Authorization', `${authToken}`) + .send(data) + .expect(200) + + console.log(res.body) + expect(res.body.id).not.to.be.undefined + }) + }) }) diff --git a/packages/api/test/services/save_newsletter_email.test.ts b/packages/api/test/services/save_newsletter_email.test.ts index a11ead336..e77f64e61 100644 --- a/packages/api/test/services/save_newsletter_email.test.ts +++ b/packages/api/test/services/save_newsletter_email.test.ts @@ -12,24 +12,37 @@ import { getPageByParam } from '../../src/elastic/pages' import nock from 'nock' import { getRepository } from '../../src/entity/utils' import { Subscription } from '../../src/entity/subscription' +import { ReceivedEmail } from '../../src/entity/received_email' describe('saveNewsletterEmail', () => { const fakeContent = 'fake content' const title = 'fake title' const author = 'fake author' + const from = 'fake from' + const text = 'fake text' let user: User - let email: NewsletterEmail + let newsletterEmail: NewsletterEmail let ctx: SaveContext + let receivedEmail: ReceivedEmail before(async () => { user = await createTestUser('fakeUser') - email = await createNewsletterEmail(user.id) + newsletterEmail = await createNewsletterEmail(user.id) ctx = { pubsub: createPubSubClient(), refresh: true, uid: user.id, } + receivedEmail = await getRepository(ReceivedEmail).save({ + user: { id: user.id }, + from, + to: newsletterEmail.address, + subject: title, + text, + html: '', + type: 'non-article', + }) }) after(async () => { @@ -42,12 +55,15 @@ describe('saveNewsletterEmail', () => { await saveNewsletterEmail( { - email: email.address, + from, + email: newsletterEmail.address, content: `${fakeContent}`, url, title, author, + receivedEmailId: receivedEmail.id, }, + newsletterEmail, ctx ) @@ -59,9 +75,15 @@ describe('saveNewsletterEmail', () => { expect(page?.content).to.contain(fakeContent) const subscriptions = await getRepository(Subscription).findBy({ - newsletterEmail: { id: email.id }, + newsletterEmail: { id: newsletterEmail.id }, }) expect(subscriptions).not.to.be.empty + + // check if the received email was updated + const updatedReceivedEmail = await getRepository(ReceivedEmail).findOneBy({ + id: receivedEmail.id, + }) + expect(updatedReceivedEmail?.type).to.equal('article') }) it('should adds a Newsletter label to that page', async () => { @@ -73,12 +95,15 @@ describe('saveNewsletterEmail', () => { await saveNewsletterEmail( { - email: email.address, + email: newsletterEmail.address, content: `fake content 2`, url, title, author, + from, + receivedEmailId: receivedEmail.id, }, + newsletterEmail, ctx ) diff --git a/packages/db/migrations/0108.do.received_emails.sql b/packages/db/migrations/0108.do.received_emails.sql new file mode 100755 index 000000000..a75dbc385 --- /dev/null +++ b/packages/db/migrations/0108.do.received_emails.sql @@ -0,0 +1,44 @@ +-- Type: DO +-- Name: received_emails +-- Description: Create a table for received emails + +BEGIN; + +CREATE TABLE omnivore.received_emails ( + id uuid PRIMARY KEY DEFAULT uuid_generate_v1mc(), + user_id uuid NOT NULL REFERENCES omnivore.user ON DELETE CASCADE, + "from" text NOT NULL, + "to" text NOT NULL, + subject text NOT NULL DEFAULT '', + "text" text NOT NULL, + html text NOT NULL DEFAULT '', + "type" text NOT NULL, + created_at timestamptz NOT NULL DEFAULT current_timestamp, + updated_at timestamptz NOT NULL DEFAULT current_timestamp +); + +CREATE TRIGGER received_emails_modtime BEFORE UPDATE ON omnivore.received_emails + FOR EACH ROW EXECUTE PROCEDURE update_updated_at_column(); + +GRANT SELECT, INSERT, UPDATE, DELETE ON omnivore.received_emails TO omnivore_user; + +-- Create a trigger to keep the most recent 20 emails for each user +CREATE OR REPLACE FUNCTION omnivore.delete_old_received_emails() + RETURNS trigger AS $$ + BEGIN + DELETE FROM omnivore.received_emails + WHERE id NOT IN ( + SELECT id FROM omnivore.received_emails + WHERE user_id = NEW.user_id + ORDER BY created_at DESC + LIMIT 20 + ); + RETURN NEW; + END; +$$ LANGUAGE plpgsql; + +CREATE TRIGGER delete_old_received_emails + AFTER INSERT ON omnivore.received_emails + FOR EACH ROW EXECUTE PROCEDURE omnivore.delete_old_received_emails(); + +COMMIT; diff --git a/packages/db/migrations/0108.undo.received_emails.sql b/packages/db/migrations/0108.undo.received_emails.sql new file mode 100755 index 000000000..cd03d6975 --- /dev/null +++ b/packages/db/migrations/0108.undo.received_emails.sql @@ -0,0 +1,9 @@ +-- Type: UNDO +-- Name: received_emails +-- Description: Create a table for received emails + +BEGIN; + +DROP TABLE IF EXISTS omnivore.received_emails; + +COMMIT; diff --git a/packages/inbound-email-handler/src/index.ts b/packages/inbound-email-handler/src/index.ts index 67d38694d..5b214b1dd 100644 --- a/packages/inbound-email-handler/src/index.ts +++ b/packages/inbound-email-handler/src/index.ts @@ -14,6 +14,15 @@ import { import { PubSub } from '@google-cloud/pubsub' import { handlePdfAttachment } from './pdf' import { handleNewsletter } from '@omnivore/content-handler' +import axios from 'axios' +import { promisify } from 'util' +import * as jwt from 'jsonwebtoken' + +interface SaveReceivedEmailResponse { + id: string +} + +const signToken = promisify(jwt.sign) const NEWSLETTER_EMAIL_RECEIVED_TOPIC = 'newsletterEmailReceived' const NON_NEWSLETTER_EMAIL_TOPIC = 'nonNewsletterEmailReceived' @@ -32,6 +41,33 @@ export const publishMessage = async ( }) } +const saveReceivedEmail = async ( + email: string, + data: any +): Promise => { + if (process.env.JWT_SECRET === undefined) { + throw new Error('JWT_SECRET is not defined') + } + const auth = await signToken(email, process.env.JWT_SECRET) + + if (process.env.INTERNAL_SVC_ENDPOINT === undefined) { + throw new Error('REST_BACKEND_ENDPOINT is not defined') + } + + const response = await axios.post( + `${process.env.INTERNAL_SVC_ENDPOINT}svc/pubsub/emails/save`, + data, + { + headers: { + Authorization: `${auth as string}`, + 'Content-Type': 'application/json', + }, + } + ) + + return response.data as SaveReceivedEmailResponse +} + export const inboundEmailHandler = Sentry.GCPFunction.wrapHttpFunction( async (req, res) => { try { @@ -76,6 +112,14 @@ export const inboundEmailHandler = Sentry.GCPFunction.wrapHttpFunction( const postHeader = headers['list-post']?.toString() const unSubHeader = headers['list-unsubscribe']?.toString() + const { id: receivedEmailId } = await saveReceivedEmail(to, { + from, + to, + subject, + html, + text, + }) + try { // check if it is a confirmation email or forwarding newsletter const newsletterMessage = await handleNewsletter({ @@ -87,10 +131,10 @@ export const inboundEmailHandler = Sentry.GCPFunction.wrapHttpFunction( title: subject, }) if (newsletterMessage) { - await publishMessage( - NEWSLETTER_EMAIL_RECEIVED_TOPIC, - newsletterMessage - ) + await publishMessage(NEWSLETTER_EMAIL_RECEIVED_TOPIC, { + ...newsletterMessage, + receivedEmailId, + }) return res.status(200).send('newsletter received') } @@ -108,7 +152,8 @@ export const inboundEmailHandler = Sentry.GCPFunction.wrapHttpFunction( to, pdfAttachmentName, pdfAttachment, - subject + subject, + receivedEmailId ) return res.send('ok') } @@ -125,6 +170,7 @@ export const inboundEmailHandler = Sentry.GCPFunction.wrapHttpFunction( unsubMailTo: unsubscribe.mailTo, unsubHttpUrl: unsubscribe.httpUrl, forwardedFrom, + receivedEmailId, }, }) @@ -140,6 +186,7 @@ export const inboundEmailHandler = Sentry.GCPFunction.wrapHttpFunction( html, text, forwardedFrom, + receivedEmailId, }, }) } diff --git a/packages/inbound-email-handler/src/pdf.ts b/packages/inbound-email-handler/src/pdf.ts index 85e68cd84..6f60c5d62 100644 --- a/packages/inbound-email-handler/src/pdf.ts +++ b/packages/inbound-email-handler/src/pdf.ts @@ -13,7 +13,8 @@ export const handlePdfAttachment = async ( email: string, fileName: string | undefined, data: Buffer, - subject: string + subject: string, + receivedEmailId: string ): Promise => { console.log('handlePdfAttachment', email, fileName) @@ -26,7 +27,7 @@ export const handlePdfAttachment = async ( return } await uploadToSignedUrl(uploadResult.url, data) - await createArticle(email, uploadResult.id, subject) + await createArticle(email, uploadResult.id, subject, receivedEmailId) } catch (error) { console.error('handlePdfAttachment error', error) } @@ -77,12 +78,14 @@ const uploadToSignedUrl = async ( const createArticle = async ( email: string, uploadFileId: string, - subject: string + subject: string, + receivedEmailId: string ): Promise => { const data = { email, uploadFileId, subject, + receivedEmailId, } if (process.env.JWT_SECRET === undefined) {