Merge pull request #1667 from omnivore-app/recent-emails

recent emails
This commit is contained in:
Jackson Harper 2023-01-25 14:22:00 +08:00 committed by GitHub
commit 506568e0c2
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
18 changed files with 882 additions and 45 deletions

View file

@ -0,0 +1,44 @@
import {
Column,
CreateDateColumn,
Entity,
JoinColumn,
ManyToOne,
PrimaryGeneratedColumn,
UpdateDateColumn,
} from 'typeorm'
import { User } from './user'
@Entity({ name: 'received_emails' })
export class ReceivedEmail {
@PrimaryGeneratedColumn('uuid')
id!: string
@ManyToOne(() => User, { onDelete: 'CASCADE' })
@JoinColumn({ name: 'user_id' })
user!: User
@Column('text')
from!: string
@Column('text')
to!: string
@Column('text')
subject!: string
@Column('text')
text!: string
@Column('text')
html!: string
@Column('text')
type!: 'article' | 'non-article'
@CreateDateColumn({ default: () => 'CURRENT_TIMESTAMP' })
createdAt!: Date
@UpdateDateColumn({ default: () => 'CURRENT_TIMESTAMP' })
updatedAt!: Date
}

View file

@ -1062,6 +1062,24 @@ export type LoginSuccess = {
me: User;
};
export type MarkEmailAsItemError = {
__typename?: 'MarkEmailAsItemError';
errorCodes: Array<MarkEmailAsItemErrorCode>;
};
export enum MarkEmailAsItemErrorCode {
BadRequest = 'BAD_REQUEST',
NotFound = 'NOT_FOUND',
Unauthorized = 'UNAUTHORIZED'
}
export type MarkEmailAsItemResult = MarkEmailAsItemError | MarkEmailAsItemSuccess;
export type MarkEmailAsItemSuccess = {
__typename?: 'MarkEmailAsItemSuccess';
success: Scalars['Boolean'];
};
export type MergeHighlightError = {
__typename?: 'MergeHighlightError';
errorCodes: Array<MergeHighlightErrorCode>;
@ -1172,6 +1190,7 @@ export type Mutation = {
joinGroup: JoinGroupResult;
leaveGroup: LeaveGroupResult;
logOut: LogOutResult;
markEmailAsItem: MarkEmailAsItemResult;
mergeHighlight: MergeHighlightResult;
moveFilter: MoveFilterResult;
moveLabel: MoveLabelResult;
@ -1338,6 +1357,11 @@ export type MutationLeaveGroupArgs = {
};
export type MutationMarkEmailAsItemArgs = {
recentEmailId: Scalars['ID'];
};
export type MutationMergeHighlightArgs = {
input: MergeHighlightInput;
};
@ -1669,6 +1693,7 @@ export type Query = {
labels: LabelsResult;
me?: Maybe<User>;
newsletterEmails: NewsletterEmailsResult;
recentEmails: RecentEmailsResult;
recentSearches: RecentSearchesResult;
reminder: ReminderResult;
rules: RulesResult;
@ -1812,6 +1837,35 @@ export type ReadState = {
readingTime?: Maybe<Scalars['Int']>;
};
export type RecentEmail = {
__typename?: 'RecentEmail';
createdAt: Scalars['Date'];
from: Scalars['String'];
html?: Maybe<Scalars['String']>;
id: Scalars['ID'];
subject: Scalars['String'];
text: Scalars['String'];
to: Scalars['String'];
type: Scalars['String'];
};
export type RecentEmailsError = {
__typename?: 'RecentEmailsError';
errorCodes: Array<RecentEmailsErrorCode>;
};
export enum RecentEmailsErrorCode {
BadRequest = 'BAD_REQUEST',
Unauthorized = 'UNAUTHORIZED'
}
export type RecentEmailsResult = RecentEmailsError | RecentEmailsSuccess;
export type RecentEmailsSuccess = {
__typename?: 'RecentEmailsSuccess';
recentEmails: Array<RecentEmail>;
};
export type RecentSearch = {
__typename?: 'RecentSearch';
createdAt: Scalars['Date'];
@ -3316,6 +3370,10 @@ export type ResolversTypes = {
LoginErrorCode: LoginErrorCode;
LoginResult: ResolversTypes['LoginError'] | ResolversTypes['LoginSuccess'];
LoginSuccess: ResolverTypeWrapper<LoginSuccess>;
MarkEmailAsItemError: ResolverTypeWrapper<MarkEmailAsItemError>;
MarkEmailAsItemErrorCode: MarkEmailAsItemErrorCode;
MarkEmailAsItemResult: ResolversTypes['MarkEmailAsItemError'] | ResolversTypes['MarkEmailAsItemSuccess'];
MarkEmailAsItemSuccess: ResolverTypeWrapper<MarkEmailAsItemSuccess>;
MergeHighlightError: ResolverTypeWrapper<MergeHighlightError>;
MergeHighlightErrorCode: MergeHighlightErrorCode;
MergeHighlightInput: MergeHighlightInput;
@ -3353,6 +3411,11 @@ export type ResolversTypes = {
Reaction: ResolverTypeWrapper<Reaction>;
ReactionType: ReactionType;
ReadState: ResolverTypeWrapper<ReadState>;
RecentEmail: ResolverTypeWrapper<RecentEmail>;
RecentEmailsError: ResolverTypeWrapper<RecentEmailsError>;
RecentEmailsErrorCode: RecentEmailsErrorCode;
RecentEmailsResult: ResolversTypes['RecentEmailsError'] | ResolversTypes['RecentEmailsSuccess'];
RecentEmailsSuccess: ResolverTypeWrapper<RecentEmailsSuccess>;
RecentSearch: ResolverTypeWrapper<RecentSearch>;
RecentSearchesError: ResolverTypeWrapper<RecentSearchesError>;
RecentSearchesErrorCode: RecentSearchesErrorCode;
@ -3741,6 +3804,9 @@ export type ResolversParentTypes = {
LoginError: LoginError;
LoginResult: ResolversParentTypes['LoginError'] | ResolversParentTypes['LoginSuccess'];
LoginSuccess: LoginSuccess;
MarkEmailAsItemError: MarkEmailAsItemError;
MarkEmailAsItemResult: ResolversParentTypes['MarkEmailAsItemError'] | ResolversParentTypes['MarkEmailAsItemSuccess'];
MarkEmailAsItemSuccess: MarkEmailAsItemSuccess;
MergeHighlightError: MergeHighlightError;
MergeHighlightInput: MergeHighlightInput;
MergeHighlightResult: ResolversParentTypes['MergeHighlightError'] | ResolversParentTypes['MergeHighlightSuccess'];
@ -3771,6 +3837,10 @@ export type ResolversParentTypes = {
Query: {};
Reaction: Reaction;
ReadState: ReadState;
RecentEmail: RecentEmail;
RecentEmailsError: RecentEmailsError;
RecentEmailsResult: ResolversParentTypes['RecentEmailsError'] | ResolversParentTypes['RecentEmailsSuccess'];
RecentEmailsSuccess: RecentEmailsSuccess;
RecentSearch: RecentSearch;
RecentSearchesError: RecentSearchesError;
RecentSearchesResult: ResolversParentTypes['RecentSearchesError'] | ResolversParentTypes['RecentSearchesSuccess'];
@ -4742,6 +4812,20 @@ export type LoginSuccessResolvers<ContextType = ResolverContext, ParentType exte
__isTypeOf?: IsTypeOfResolverFn<ParentType, ContextType>;
};
export type MarkEmailAsItemErrorResolvers<ContextType = ResolverContext, ParentType extends ResolversParentTypes['MarkEmailAsItemError'] = ResolversParentTypes['MarkEmailAsItemError']> = {
errorCodes?: Resolver<Array<ResolversTypes['MarkEmailAsItemErrorCode']>, ParentType, ContextType>;
__isTypeOf?: IsTypeOfResolverFn<ParentType, ContextType>;
};
export type MarkEmailAsItemResultResolvers<ContextType = ResolverContext, ParentType extends ResolversParentTypes['MarkEmailAsItemResult'] = ResolversParentTypes['MarkEmailAsItemResult']> = {
__resolveType: TypeResolveFn<'MarkEmailAsItemError' | 'MarkEmailAsItemSuccess', ParentType, ContextType>;
};
export type MarkEmailAsItemSuccessResolvers<ContextType = ResolverContext, ParentType extends ResolversParentTypes['MarkEmailAsItemSuccess'] = ResolversParentTypes['MarkEmailAsItemSuccess']> = {
success?: Resolver<ResolversTypes['Boolean'], ParentType, ContextType>;
__isTypeOf?: IsTypeOfResolverFn<ParentType, ContextType>;
};
export type MergeHighlightErrorResolvers<ContextType = ResolverContext, ParentType extends ResolversParentTypes['MergeHighlightError'] = ResolversParentTypes['MergeHighlightError']> = {
errorCodes?: Resolver<Array<ResolversTypes['MergeHighlightErrorCode']>, ParentType, ContextType>;
__isTypeOf?: IsTypeOfResolverFn<ParentType, ContextType>;
@ -4813,6 +4897,7 @@ export type MutationResolvers<ContextType = ResolverContext, ParentType extends
joinGroup?: Resolver<ResolversTypes['JoinGroupResult'], ParentType, ContextType, RequireFields<MutationJoinGroupArgs, 'inviteCode'>>;
leaveGroup?: Resolver<ResolversTypes['LeaveGroupResult'], ParentType, ContextType, RequireFields<MutationLeaveGroupArgs, 'groupId'>>;
logOut?: Resolver<ResolversTypes['LogOutResult'], ParentType, ContextType>;
markEmailAsItem?: Resolver<ResolversTypes['MarkEmailAsItemResult'], ParentType, ContextType, RequireFields<MutationMarkEmailAsItemArgs, 'recentEmailId'>>;
mergeHighlight?: Resolver<ResolversTypes['MergeHighlightResult'], ParentType, ContextType, RequireFields<MutationMergeHighlightArgs, 'input'>>;
moveFilter?: Resolver<ResolversTypes['MoveFilterResult'], ParentType, ContextType, RequireFields<MutationMoveFilterArgs, 'input'>>;
moveLabel?: Resolver<ResolversTypes['MoveLabelResult'], ParentType, ContextType, RequireFields<MutationMoveLabelArgs, 'input'>>;
@ -4943,6 +5028,7 @@ export type QueryResolvers<ContextType = ResolverContext, ParentType extends Res
labels?: Resolver<ResolversTypes['LabelsResult'], ParentType, ContextType>;
me?: Resolver<Maybe<ResolversTypes['User']>, ParentType, ContextType>;
newsletterEmails?: Resolver<ResolversTypes['NewsletterEmailsResult'], ParentType, ContextType>;
recentEmails?: Resolver<ResolversTypes['RecentEmailsResult'], ParentType, ContextType>;
recentSearches?: Resolver<ResolversTypes['RecentSearchesResult'], ParentType, ContextType>;
reminder?: Resolver<ResolversTypes['ReminderResult'], ParentType, ContextType, RequireFields<QueryReminderArgs, 'linkId'>>;
rules?: Resolver<ResolversTypes['RulesResult'], ParentType, ContextType, Partial<QueryRulesArgs>>;
@ -4976,6 +5062,32 @@ export type ReadStateResolvers<ContextType = ResolverContext, ParentType extends
__isTypeOf?: IsTypeOfResolverFn<ParentType, ContextType>;
};
export type RecentEmailResolvers<ContextType = ResolverContext, ParentType extends ResolversParentTypes['RecentEmail'] = ResolversParentTypes['RecentEmail']> = {
createdAt?: Resolver<ResolversTypes['Date'], ParentType, ContextType>;
from?: Resolver<ResolversTypes['String'], ParentType, ContextType>;
html?: Resolver<Maybe<ResolversTypes['String']>, ParentType, ContextType>;
id?: Resolver<ResolversTypes['ID'], ParentType, ContextType>;
subject?: Resolver<ResolversTypes['String'], ParentType, ContextType>;
text?: Resolver<ResolversTypes['String'], ParentType, ContextType>;
to?: Resolver<ResolversTypes['String'], ParentType, ContextType>;
type?: Resolver<ResolversTypes['String'], ParentType, ContextType>;
__isTypeOf?: IsTypeOfResolverFn<ParentType, ContextType>;
};
export type RecentEmailsErrorResolvers<ContextType = ResolverContext, ParentType extends ResolversParentTypes['RecentEmailsError'] = ResolversParentTypes['RecentEmailsError']> = {
errorCodes?: Resolver<Array<ResolversTypes['RecentEmailsErrorCode']>, ParentType, ContextType>;
__isTypeOf?: IsTypeOfResolverFn<ParentType, ContextType>;
};
export type RecentEmailsResultResolvers<ContextType = ResolverContext, ParentType extends ResolversParentTypes['RecentEmailsResult'] = ResolversParentTypes['RecentEmailsResult']> = {
__resolveType: TypeResolveFn<'RecentEmailsError' | 'RecentEmailsSuccess', ParentType, ContextType>;
};
export type RecentEmailsSuccessResolvers<ContextType = ResolverContext, ParentType extends ResolversParentTypes['RecentEmailsSuccess'] = ResolversParentTypes['RecentEmailsSuccess']> = {
recentEmails?: Resolver<Array<ResolversTypes['RecentEmail']>, ParentType, ContextType>;
__isTypeOf?: IsTypeOfResolverFn<ParentType, ContextType>;
};
export type RecentSearchResolvers<ContextType = ResolverContext, ParentType extends ResolversParentTypes['RecentSearch'] = ResolversParentTypes['RecentSearch']> = {
createdAt?: Resolver<ResolversTypes['Date'], ParentType, ContextType>;
id?: Resolver<ResolversTypes['ID'], ParentType, ContextType>;
@ -5911,6 +6023,9 @@ export type Resolvers<ContextType = ResolverContext> = {
LoginError?: LoginErrorResolvers<ContextType>;
LoginResult?: LoginResultResolvers<ContextType>;
LoginSuccess?: LoginSuccessResolvers<ContextType>;
MarkEmailAsItemError?: MarkEmailAsItemErrorResolvers<ContextType>;
MarkEmailAsItemResult?: MarkEmailAsItemResultResolvers<ContextType>;
MarkEmailAsItemSuccess?: MarkEmailAsItemSuccessResolvers<ContextType>;
MergeHighlightError?: MergeHighlightErrorResolvers<ContextType>;
MergeHighlightResult?: MergeHighlightResultResolvers<ContextType>;
MergeHighlightSuccess?: MergeHighlightSuccessResolvers<ContextType>;
@ -5934,6 +6049,10 @@ export type Resolvers<ContextType = ResolverContext> = {
Query?: QueryResolvers<ContextType>;
Reaction?: ReactionResolvers<ContextType>;
ReadState?: ReadStateResolvers<ContextType>;
RecentEmail?: RecentEmailResolvers<ContextType>;
RecentEmailsError?: RecentEmailsErrorResolvers<ContextType>;
RecentEmailsResult?: RecentEmailsResultResolvers<ContextType>;
RecentEmailsSuccess?: RecentEmailsSuccessResolvers<ContextType>;
RecentSearch?: RecentSearchResolvers<ContextType>;
RecentSearchesError?: RecentSearchesErrorResolvers<ContextType>;
RecentSearchesResult?: RecentSearchesResultResolvers<ContextType>;

View file

@ -943,6 +943,22 @@ type LoginSuccess {
me: User!
}
type MarkEmailAsItemError {
errorCodes: [MarkEmailAsItemErrorCode!]!
}
enum MarkEmailAsItemErrorCode {
BAD_REQUEST
NOT_FOUND
UNAUTHORIZED
}
union MarkEmailAsItemResult = MarkEmailAsItemError | MarkEmailAsItemSuccess
type MarkEmailAsItemSuccess {
success: Boolean!
}
type MergeHighlightError {
errorCodes: [MergeHighlightErrorCode!]!
}
@ -1046,6 +1062,7 @@ type Mutation {
joinGroup(inviteCode: String!): JoinGroupResult!
leaveGroup(groupId: ID!): LeaveGroupResult!
logOut: LogOutResult!
markEmailAsItem(recentEmailId: ID!): MarkEmailAsItemResult!
mergeHighlight(input: MergeHighlightInput!): MergeHighlightResult!
moveFilter(input: MoveFilterInput!): MoveFilterResult!
moveLabel(input: MoveLabelInput!): MoveLabelResult!
@ -1218,6 +1235,7 @@ type Query {
labels: LabelsResult!
me: User
newsletterEmails: NewsletterEmailsResult!
recentEmails: RecentEmailsResult!
recentSearches: RecentSearchesResult!
reminder(linkId: ID!): ReminderResult!
rules(enabled: Boolean): RulesResult!
@ -1258,6 +1276,32 @@ type ReadState {
readingTime: Int
}
type RecentEmail {
createdAt: Date!
from: String!
html: String
id: ID!
subject: String!
text: String!
to: String!
type: String!
}
type RecentEmailsError {
errorCodes: [RecentEmailsErrorCode!]!
}
enum RecentEmailsErrorCode {
BAD_REQUEST
UNAUTHORIZED
}
union RecentEmailsResult = RecentEmailsError | RecentEmailsSuccess
type RecentEmailsSuccess {
recentEmails: [RecentEmail!]!
}
type RecentSearch {
createdAt: Date!
id: ID!

View file

@ -117,6 +117,7 @@ import { getPageByParam } from '../elastic/pages'
import { recentSearchesResolver } from './recent_searches'
import { optInFeatureResolver } from './features'
import { uploadImportFileResolver } from './importers/uploadImportFileResolver'
import { markEmailAsItemResolver, recentEmailsResolver } from './recent_emails'
/* eslint-disable @typescript-eslint/naming-convention */
type ResultResolveType = {
@ -199,6 +200,7 @@ export const functionResolvers = {
recommendHighlights: recommendHighlightsResolver,
leaveGroup: leaveGroupResolver,
uploadImportFile: uploadImportFileResolver,
markEmailAsItem: markEmailAsItemResolver,
},
Query: {
me: getMeUserResolver,
@ -230,6 +232,7 @@ export const functionResolvers = {
deviceTokens: deviceTokensResolver,
filters: filtersResolver,
groups: groupsResolver,
recentEmails: recentEmailsResolver,
},
User: {
async sharedArticles(
@ -659,4 +662,6 @@ export const functionResolvers = {
...resultResolveTypeResolver('RecommendHighlights'),
...resultResolveTypeResolver('LeaveGroup'),
...resultResolveTypeResolver('UploadImportFile'),
...resultResolveTypeResolver('RecentEmails'),
...resultResolveTypeResolver('MarkEmailAsItem'),
}

View file

@ -0,0 +1,143 @@
import {
MarkEmailAsItemError,
MarkEmailAsItemErrorCode,
MarkEmailAsItemSuccess,
MutationMarkEmailAsItemArgs,
RecentEmailsError,
RecentEmailsErrorCode,
RecentEmailsSuccess,
} from '../../generated/graphql'
import { authorized } from '../../utils/helpers'
import { getRepository } from '../../entity/utils'
import { ReceivedEmail } from '../../entity/received_email'
import { saveNewsletterEmail } from '../../services/save_newsletter_email'
import { NewsletterEmail } from '../../entity/newsletter_email'
import { generateUniqueUrl, parseEmailAddress } from '../../utils/parser'
import { sendEmail } from '../../utils/sendEmail'
import { env } from '../../env'
export const recentEmailsResolver = authorized<
RecentEmailsSuccess,
RecentEmailsError
>(async (_, __, { claims, log }) => {
log.info('Getting recent emails', {
labels: {
source: 'resolver',
resolver: 'recentEmailsResolver',
uid: claims.uid,
},
})
try {
const recentEmails = await getRepository(ReceivedEmail).find({
where: { user: { id: claims.uid } },
order: { createdAt: 'DESC' },
take: 20,
})
return {
recentEmails,
}
} catch (error) {
log.error('Error getting recent emails', {
error,
labels: {
source: 'resolver',
resolver: 'recentEmailsResolver',
uid: claims.uid,
},
})
return {
errorCodes: [RecentEmailsErrorCode.BadRequest],
}
}
})
export const markEmailAsItemResolver = authorized<
MarkEmailAsItemSuccess,
MarkEmailAsItemError,
MutationMarkEmailAsItemArgs
>(async (_, { recentEmailId }, { claims, log }) => {
log.info('Marking email as item', {
recentEmailId,
labels: {
source: 'resolver',
resolver: 'markEmailAsItemResolver',
uid: claims.uid,
},
})
try {
const recentEmail = await getRepository(ReceivedEmail).findOneBy({
id: recentEmailId,
user: { id: claims.uid },
type: 'non-article',
})
if (!recentEmail) {
log.info('no recent email', recentEmailId)
return {
errorCodes: [MarkEmailAsItemErrorCode.Unauthorized],
}
}
const newsletterEmail = await getRepository(NewsletterEmail).findOne({
where: {
address: recentEmail.to,
user: { id: claims.uid },
},
relations: ['user'],
})
if (!newsletterEmail) {
log.info('no newsletter email', recentEmail.to)
return {
errorCodes: [MarkEmailAsItemErrorCode.NotFound],
}
}
const success = await saveNewsletterEmail(
{
from: recentEmail.from,
email: recentEmail.to,
title: recentEmail.subject,
content: recentEmail.html,
url: generateUniqueUrl(),
author: parseEmailAddress(recentEmail.from).name,
receivedEmailId: recentEmail.id,
},
newsletterEmail
)
const text = `A recent email marked as a library item
by: ${claims.uid}
from: ${recentEmail.from}
subject: ${recentEmail.subject}`
// email us to let us know that an email failed to parse as an article
await sendEmail({
to: env.sender.feedback,
subject: 'A recent email marked as a library item',
text,
from: env.sender.message,
})
return {
success,
}
} catch (error) {
log.error('Error marking email as item', {
error,
labels: {
source: 'resolver',
resolver: 'markEmailAsItemResolver',
uid: claims.uid,
},
})
return {
errorCodes: [MarkEmailAsItemErrorCode.BadRequest],
}
}
})

View file

@ -15,8 +15,15 @@ import {
} from '../../utils/parser'
import { saveEmail } from '../../services/save_email'
import { buildLogger } from '../../utils/logger'
import {
saveReceivedEmail,
updateReceivedEmail,
} from '../../services/received_emails'
import cors from 'cors'
import { corsConfig } from '../../utils/corsConfig'
import { getClaimsByToken } from '../../utils/auth'
interface ForwardEmailMessage {
interface EmailMessage {
from: string
to: string
subject: string
@ -25,6 +32,17 @@ interface ForwardEmailMessage {
unsubHttpUrl?: string
text: string
forwardedFrom?: string
receivedEmailId: string
}
function isEmailMessage(data: any): data is EmailMessage {
return (
'from' in data &&
'to' in data &&
'subject' in data &&
'html' in data &&
'text' in data
)
}
const logger = buildLogger('app.dispatch')
@ -37,7 +55,6 @@ export function emailsServiceRouter() {
logger.info('email forward router')
const { message, expired } = readPushSubscription(req)
logger.info('pubsub message:', { message, expired })
if (!message) {
res.status(400).send('Bad Request')
@ -51,15 +68,8 @@ export function emailsServiceRouter() {
}
try {
// eslint-disable-next-line @typescript-eslint/no-unsafe-assignment
const data: ForwardEmailMessage = JSON.parse(message)
if (
!('from' in data) ||
!('to' in data) ||
!('subject' in data) ||
(!('html' in data) && !('text' in data))
) {
const data = JSON.parse(message) as unknown
if (!isEmailMessage(data)) {
logger.error('Invalid message')
res.status(400).send('Bad Request')
return
@ -90,6 +100,10 @@ export function emailsServiceRouter() {
url: generateUniqueUrl(),
originalContent: data.html || data.text,
})
// update received email type
await updateReceivedEmail(data.receivedEmailId, 'article')
res.status(200).send('Article')
return
}
@ -130,5 +144,54 @@ export function emailsServiceRouter() {
}
})
router.post('/save', cors<express.Request>(corsConfig), async (req, res) => {
logger.info('save received email router')
const token = req?.headers?.authorization
if (!(await getClaimsByToken(token))) {
return res.status(401).send('UNAUTHORIZED')
}
if (!isEmailMessage(req.body)) {
logger.error('Invalid message')
return res.status(400).send('Bad Request')
}
try {
// get user from newsletter email
const newsletterEmail = await getNewsletterEmail(req.body.to)
if (!newsletterEmail) {
logger.info('newsletter email not found', { email: req.body.to })
res.status(200).send('Not Found')
return
}
const user = newsletterEmail.user
const receivedEmail = await saveReceivedEmail(
req.body.from,
req.body.to,
req.body.subject,
req.body.text,
req.body.html,
user.id
)
analytics.track({
userId: user.id,
event: 'received_email_saved',
properties: {
env: env.server.apiEnv,
},
})
res.status(200).send({ id: receivedEmail.id })
} catch (e) {
logger.info(e)
res.status(500).send(e)
}
})
return router
}

View file

@ -1,10 +1,14 @@
import express from 'express'
import { readPushSubscription } from '../../datalayer/pubsub'
import { updateConfirmationCode } from '../../services/newsletters'
import {
getNewsletterEmail,
updateConfirmationCode,
} from '../../services/newsletters'
import {
NewsletterMessage,
saveNewsletterEmail,
} from '../../services/save_newsletter_email'
import { updateReceivedEmail } from '../../services/received_emails'
interface SetConfirmationCodeMessage {
emailAddress: string
@ -96,7 +100,14 @@ export function newsletterServiceRouter() {
return
}
const result = await saveNewsletterEmail(data)
// get user from newsletter email
const newsletterEmail = await getNewsletterEmail(data.email)
if (!newsletterEmail) {
console.log('newsletter email not found', data.email)
return false
}
const result = await saveNewsletterEmail(data, newsletterEmail)
if (!result) {
console.log(
'Error creating newsletter link from data',
@ -104,10 +115,14 @@ export function newsletterServiceRouter() {
data.title,
data.author
)
res.status(500).send('Error creating newsletter link')
return
}
// update received email type
await updateReceivedEmail(data.receivedEmailId, 'article')
// We always send 200 if it was a valid message
// because we don't want the
res.status(200).send('newsletter created')

View file

@ -17,6 +17,7 @@ import { createPubSubClient } from '../../datalayer/pubsub'
import { ArticleSavingRequestStatus, Page } from '../../elastic/types'
import { createPage } from '../../elastic/pages'
import { getClaimsByToken } from '../../utils/auth'
import { updateReceivedEmail } from '../../services/received_emails'
export function pdfAttachmentsRouter() {
const router = express.Router()
@ -87,10 +88,11 @@ export function pdfAttachmentsRouter() {
router.post('/create-article', async (req, res) => {
console.log('pdf-attachments/create-article')
const { email, uploadFileId, subject } = req.body as {
const { email, uploadFileId, subject, receivedEmailId } = req.body as {
email: string
uploadFileId: string
subject: string
receivedEmailId: string
}
const token = req?.headers?.authorization
@ -149,7 +151,7 @@ export function pdfAttachmentsRouter() {
pageType: pageType,
hash: uploadFileHash,
uploadFileId: uploadFileId,
title: title,
title,
content: '',
userId: user.id,
slug: generateSlug(title),
@ -166,6 +168,9 @@ export function pdfAttachmentsRouter() {
uid: user.id,
})
// update received email type
await updateReceivedEmail(receivedEmailId, 'article')
res.send({ id: pageId })
} catch (err) {
console.log(err)

View file

@ -2331,6 +2331,48 @@ const schema = gql`
uploadSignedUrl: String
}
union RecentEmailsResult = RecentEmailsSuccess | RecentEmailsError
type RecentEmailsSuccess {
recentEmails: [RecentEmail!]!
}
type RecentEmailsError {
errorCodes: [RecentEmailsErrorCode!]!
}
enum RecentEmailsErrorCode {
UNAUTHORIZED
BAD_REQUEST
}
type RecentEmail {
id: ID!
from: String!
to: String!
subject: String!
type: String!
text: String!
html: String
createdAt: Date!
}
union MarkEmailAsItemResult = MarkEmailAsItemSuccess | MarkEmailAsItemError
type MarkEmailAsItemSuccess {
success: Boolean!
}
type MarkEmailAsItemError {
errorCodes: [MarkEmailAsItemErrorCode!]!
}
enum MarkEmailAsItemErrorCode {
UNAUTHORIZED
BAD_REQUEST
NOT_FOUND
}
# Mutations
type Mutation {
googleLogin(input: GoogleLoginInput!): LoginResult!
@ -2419,6 +2461,7 @@ const schema = gql`
type: UploadImportFileType!
contentType: String!
): UploadImportFileResult!
markEmailAsItem(recentEmailId: ID!): MarkEmailAsItemResult!
}
# FIXME: remove sort from feedArticles after all cached tabs are closed
@ -2481,6 +2524,7 @@ const schema = gql`
deviceTokens: DeviceTokensResult!
filters: FiltersResult!
groups: GroupsResult!
recentEmails: RecentEmailsResult!
}
`

View file

@ -0,0 +1,29 @@
import { ReceivedEmail } from '../entity/received_email'
import { getRepository } from '../entity/utils'
export const saveReceivedEmail = async (
from: string,
to: string,
subject: string,
text: string,
html: string,
userId: string,
type: 'article' | 'non-article' = 'non-article'
): Promise<ReceivedEmail> => {
return getRepository(ReceivedEmail).save({
from,
to,
subject,
text,
html,
type,
user: { id: userId },
})
}
export const updateReceivedEmail = async (
id: string,
type: 'article' | 'non-article'
) => {
await getRepository(ReceivedEmail).update(id, { type })
}

View file

@ -4,7 +4,6 @@ import { UserDeviceToken } from '../entity/user_device_tokens'
import { env } from '../env'
import { ContentReader } from '../generated/graphql'
import { analytics } from '../utils/analytics'
import { getNewsletterEmail } from './newsletters'
import { SaveContext, saveEmail, SaveEmailInput } from './save_email'
import { Page } from '../elastic/types'
import { addLabelToPage } from './labels'
@ -13,8 +12,10 @@ import { NewsletterEmail } from '../entity/newsletter_email'
import { fetchFavicon } from '../utils/parser'
import { updatePage } from '../elastic/pages'
import { isBase64Image } from '../utils/helpers'
import { updateReceivedEmail } from './received_emails'
export interface NewsletterMessage {
from: string
email: string
content: string
url: string
@ -22,23 +23,16 @@ export interface NewsletterMessage {
author: string
unsubMailTo?: string
unsubHttpUrl?: string
newsletterEmail?: NewsletterEmail
receivedEmailId: string
}
// Returns true if the link was created successfully. Can still fail to
// send the push but that is ok and we wont retry in that case.
export const saveNewsletterEmail = async (
data: NewsletterMessage,
newsletterEmail: NewsletterEmail,
ctx?: SaveContext
): Promise<boolean> => {
// get user from newsletter email
const newsletterEmail =
data.newsletterEmail || (await getNewsletterEmail(data.email))
if (!newsletterEmail) {
console.log('newsletter email not found', data.email)
return false
}
analytics.track({
userId: newsletterEmail.user.id,
event: 'newsletter_email_received',
@ -64,10 +58,14 @@ export const saveNewsletterEmail = async (
}
const page = await saveEmail(saveCtx, input)
if (!page) {
console.log('newsletter not created:', input)
console.log('newsletter not created:', input.title)
return false
}
// update received email type
await updateReceivedEmail(data.receivedEmailId, 'article')
if (!page.siteIcon || isBase64Image(page.siteIcon)) {
// fetch favicon if not already set or is a base64 image
const favicon = await fetchFavicon(page.url)

View file

@ -0,0 +1,146 @@
import 'mocha'
import { expect } from 'chai'
import { User } from '../../src/entity/user'
import { createTestUser, deleteTestUser } from '../db'
import { graphqlRequest, request } from '../util'
import { getRepository } from '../../src/entity/utils'
import { ReceivedEmail } from '../../src/entity/received_email'
import { NewsletterEmail } from '../../src/entity/newsletter_email'
describe('Recent Emails Resolver', () => {
const username = 'fakeUser'
let user: User
let authToken: string
let newsletterEmail: NewsletterEmail
let newsletterEmail2: NewsletterEmail
before(async () => {
// create test user and login
user = await createTestUser(username)
const res = await request
.post('/local/debug/fake-user-login')
.send({ fakeEmail: user.email })
authToken = res.body.authToken
// create test newsletter email
newsletterEmail = await getRepository(NewsletterEmail).save({
user: { id: user.id },
address: 'fake email address',
})
newsletterEmail2 = await getRepository(NewsletterEmail).save({
user: { id: user.id },
address: 'fake email address 2',
})
})
after(async () => {
// clean up
await deleteTestUser(user.id)
})
describe('recentEmails', () => {
const recentEmailsQuery = `
query {
recentEmails {
... on RecentEmailsSuccess {
recentEmails {
id
from
to
subject
text
html
}
}
... on RecentEmailsError {
errorCodes
}
}
}
`
let recentEmails: ReceivedEmail[]
before(async () => {
// create fake emails
const recentEmail = await getRepository(ReceivedEmail).save({
user: { id: user.id },
from: 'fake from',
subject: 'fake subject',
text: 'fake text',
html: 'fake html',
to: newsletterEmail.address,
type: 'article',
})
const recentEmail2 = await getRepository(ReceivedEmail).save({
user: { id: user.id },
from: 'fake from 2',
subject: 'fake subject 2',
text: 'fake text 2',
html: 'fake html 2',
to: newsletterEmail2.address,
type: 'non-article',
})
recentEmails = [recentEmail, recentEmail2]
})
it('returns recent emails', async () => {
const res = await graphqlRequest(recentEmailsQuery, authToken).expect(200)
const { recentEmails: results } = res.body.data.recentEmails
expect(results).to.have.lengthOf(2)
expect(results[0].id).to.eql(recentEmails[1].id)
expect(results[1].id).to.eql(recentEmails[0].id)
})
})
describe('markEmailAsItem', () => {
const markEmailAsItemMutation = (recentEmailId: string) => `
mutation {
markEmailAsItem(recentEmailId: "${recentEmailId}") {
... on MarkEmailAsItemSuccess {
success
}
... on MarkEmailAsItemError {
errorCodes
}
}
}
`
let recentEmail: ReceivedEmail
before(async () => {
// create fake email
recentEmail = await getRepository(ReceivedEmail).save({
user: { id: user.id },
from: 'Omnivore Newsletter <newsletter@omnivore.app>',
subject: 'fake subject 3',
text: 'fake text 3',
html: '<html><body>fake html 3</body></html>',
to: newsletterEmail.address,
type: 'non-article',
})
})
after(async () => {
// clean up
await getRepository(ReceivedEmail).delete(recentEmail.id)
})
it('marks email as item', async () => {
const resp = await graphqlRequest(
markEmailAsItemMutation(recentEmail.id),
authToken
)
expect(resp.body.data.markEmailAsItem.success).to.be.true
const updatedRecentEmail = await getRepository(ReceivedEmail).findOneBy({
id: recentEmail.id,
})
expect(updatedRecentEmail?.type).to.eql('article')
})
})
})

View file

@ -11,12 +11,20 @@ import { request } from '../util'
import * as parser from '../../src/utils/parser'
import * as sendNotification from '../../src/utils/sendNotification'
import * as sendEmail from '../../src/utils/sendEmail'
import { getRepository } from '../../src/entity/utils'
import { ReceivedEmail } from '../../src/entity/received_email'
import * as jwt from 'jsonwebtoken'
describe('Emails Router', () => {
const newsletterEmail = 'fakeUser@omnivore.app'
const from = 'fake from'
const subject = 'fake subject'
const text = 'fake text'
const to = newsletterEmail
let user: User
let token: string
let receivedEmail: ReceivedEmail
before(async () => {
// create test user and login
@ -24,6 +32,15 @@ describe('Emails Router', () => {
await createTestNewsletterEmail(user, newsletterEmail)
token = process.env.PUBSUB_VERIFICATION_TOKEN!
receivedEmail = await getRepository(ReceivedEmail).save({
user: { id: user.id },
from,
to,
subject,
text,
html: '',
type: 'non-article',
})
})
after(async () => {
@ -33,10 +50,7 @@ describe('Emails Router', () => {
})
describe('forward', () => {
const from = 'from@omnivore.app'
const to = newsletterEmail
const subject = 'test subject'
const html = 'test html'
const html = '<html>test html</html>'
beforeEach(async () => {
sinon.replace(
@ -60,7 +74,14 @@ describe('Emails Router', () => {
const data = {
message: {
data: Buffer.from(
JSON.stringify({ from, to, subject, html })
JSON.stringify({
from,
to,
subject,
html,
text,
receivedEmailId: receivedEmail.id,
})
).toString('base64'),
publishTime: new Date().toISOString(),
},
@ -82,7 +103,14 @@ describe('Emails Router', () => {
const data = {
message: {
data: Buffer.from(
JSON.stringify({ from, to, subject, html })
JSON.stringify({
from,
to,
subject,
html,
text,
receivedEmailId: receivedEmail.id,
})
).toString('base64'),
publishTime: new Date().toISOString(),
},
@ -95,4 +123,30 @@ describe('Emails Router', () => {
})
})
})
describe('create', () => {
const html = '<html>test html</html>'
const text = 'test text'
const from = 'fake from'
const subject = 'fake subject'
const authToken = jwt.sign(newsletterEmail, process.env.JWT_SECRET || '')
it('saves the email in the database', async () => {
const data = {
html,
text,
from,
to: newsletterEmail,
subject,
}
const res = await request
.post('/svc/pubsub/emails/save')
.set('Authorization', `${authToken}`)
.send(data)
.expect(200)
console.log(res.body)
expect(res.body.id).not.to.be.undefined
})
})
})

View file

@ -12,24 +12,37 @@ import { getPageByParam } from '../../src/elastic/pages'
import nock from 'nock'
import { getRepository } from '../../src/entity/utils'
import { Subscription } from '../../src/entity/subscription'
import { ReceivedEmail } from '../../src/entity/received_email'
describe('saveNewsletterEmail', () => {
const fakeContent = 'fake content'
const title = 'fake title'
const author = 'fake author'
const from = 'fake from'
const text = 'fake text'
let user: User
let email: NewsletterEmail
let newsletterEmail: NewsletterEmail
let ctx: SaveContext
let receivedEmail: ReceivedEmail
before(async () => {
user = await createTestUser('fakeUser')
email = await createNewsletterEmail(user.id)
newsletterEmail = await createNewsletterEmail(user.id)
ctx = {
pubsub: createPubSubClient(),
refresh: true,
uid: user.id,
}
receivedEmail = await getRepository(ReceivedEmail).save({
user: { id: user.id },
from,
to: newsletterEmail.address,
subject: title,
text,
html: '',
type: 'non-article',
})
})
after(async () => {
@ -42,12 +55,15 @@ describe('saveNewsletterEmail', () => {
await saveNewsletterEmail(
{
email: email.address,
from,
email: newsletterEmail.address,
content: `<html><body>${fakeContent}</body></html>`,
url,
title,
author,
receivedEmailId: receivedEmail.id,
},
newsletterEmail,
ctx
)
@ -59,9 +75,15 @@ describe('saveNewsletterEmail', () => {
expect(page?.content).to.contain(fakeContent)
const subscriptions = await getRepository(Subscription).findBy({
newsletterEmail: { id: email.id },
newsletterEmail: { id: newsletterEmail.id },
})
expect(subscriptions).not.to.be.empty
// check if the received email was updated
const updatedReceivedEmail = await getRepository(ReceivedEmail).findOneBy({
id: receivedEmail.id,
})
expect(updatedReceivedEmail?.type).to.equal('article')
})
it('should adds a Newsletter label to that page', async () => {
@ -73,12 +95,15 @@ describe('saveNewsletterEmail', () => {
await saveNewsletterEmail(
{
email: email.address,
email: newsletterEmail.address,
content: `<html><body>fake content 2</body></html>`,
url,
title,
author,
from,
receivedEmailId: receivedEmail.id,
},
newsletterEmail,
ctx
)

View file

@ -0,0 +1,44 @@
-- Type: DO
-- Name: received_emails
-- Description: Create a table for received emails
BEGIN;
CREATE TABLE omnivore.received_emails (
id uuid PRIMARY KEY DEFAULT uuid_generate_v1mc(),
user_id uuid NOT NULL REFERENCES omnivore.user ON DELETE CASCADE,
"from" text NOT NULL,
"to" text NOT NULL,
subject text NOT NULL DEFAULT '',
"text" text NOT NULL,
html text NOT NULL DEFAULT '',
"type" text NOT NULL,
created_at timestamptz NOT NULL DEFAULT current_timestamp,
updated_at timestamptz NOT NULL DEFAULT current_timestamp
);
CREATE TRIGGER received_emails_modtime BEFORE UPDATE ON omnivore.received_emails
FOR EACH ROW EXECUTE PROCEDURE update_updated_at_column();
GRANT SELECT, INSERT, UPDATE, DELETE ON omnivore.received_emails TO omnivore_user;
-- Create a trigger to keep the most recent 20 emails for each user
CREATE OR REPLACE FUNCTION omnivore.delete_old_received_emails()
RETURNS trigger AS $$
BEGIN
DELETE FROM omnivore.received_emails
WHERE id NOT IN (
SELECT id FROM omnivore.received_emails
WHERE user_id = NEW.user_id
ORDER BY created_at DESC
LIMIT 20
);
RETURN NEW;
END;
$$ LANGUAGE plpgsql;
CREATE TRIGGER delete_old_received_emails
AFTER INSERT ON omnivore.received_emails
FOR EACH ROW EXECUTE PROCEDURE omnivore.delete_old_received_emails();
COMMIT;

View file

@ -0,0 +1,9 @@
-- Type: UNDO
-- Name: received_emails
-- Description: Create a table for received emails
BEGIN;
DROP TABLE IF EXISTS omnivore.received_emails;
COMMIT;

View file

@ -14,6 +14,15 @@ import {
import { PubSub } from '@google-cloud/pubsub'
import { handlePdfAttachment } from './pdf'
import { handleNewsletter } from '@omnivore/content-handler'
import axios from 'axios'
import { promisify } from 'util'
import * as jwt from 'jsonwebtoken'
interface SaveReceivedEmailResponse {
id: string
}
const signToken = promisify(jwt.sign)
const NEWSLETTER_EMAIL_RECEIVED_TOPIC = 'newsletterEmailReceived'
const NON_NEWSLETTER_EMAIL_TOPIC = 'nonNewsletterEmailReceived'
@ -32,6 +41,33 @@ export const publishMessage = async (
})
}
const saveReceivedEmail = async (
email: string,
data: any
): Promise<SaveReceivedEmailResponse> => {
if (process.env.JWT_SECRET === undefined) {
throw new Error('JWT_SECRET is not defined')
}
const auth = await signToken(email, process.env.JWT_SECRET)
if (process.env.INTERNAL_SVC_ENDPOINT === undefined) {
throw new Error('REST_BACKEND_ENDPOINT is not defined')
}
const response = await axios.post(
`${process.env.INTERNAL_SVC_ENDPOINT}svc/pubsub/emails/save`,
data,
{
headers: {
Authorization: `${auth as string}`,
'Content-Type': 'application/json',
},
}
)
return response.data as SaveReceivedEmailResponse
}
export const inboundEmailHandler = Sentry.GCPFunction.wrapHttpFunction(
async (req, res) => {
try {
@ -76,6 +112,14 @@ export const inboundEmailHandler = Sentry.GCPFunction.wrapHttpFunction(
const postHeader = headers['list-post']?.toString()
const unSubHeader = headers['list-unsubscribe']?.toString()
const { id: receivedEmailId } = await saveReceivedEmail(to, {
from,
to,
subject,
html,
text,
})
try {
// check if it is a confirmation email or forwarding newsletter
const newsletterMessage = await handleNewsletter({
@ -87,10 +131,10 @@ export const inboundEmailHandler = Sentry.GCPFunction.wrapHttpFunction(
title: subject,
})
if (newsletterMessage) {
await publishMessage(
NEWSLETTER_EMAIL_RECEIVED_TOPIC,
newsletterMessage
)
await publishMessage(NEWSLETTER_EMAIL_RECEIVED_TOPIC, {
...newsletterMessage,
receivedEmailId,
})
return res.status(200).send('newsletter received')
}
@ -108,7 +152,8 @@ export const inboundEmailHandler = Sentry.GCPFunction.wrapHttpFunction(
to,
pdfAttachmentName,
pdfAttachment,
subject
subject,
receivedEmailId
)
return res.send('ok')
}
@ -125,6 +170,7 @@ export const inboundEmailHandler = Sentry.GCPFunction.wrapHttpFunction(
unsubMailTo: unsubscribe.mailTo,
unsubHttpUrl: unsubscribe.httpUrl,
forwardedFrom,
receivedEmailId,
},
})
@ -140,6 +186,7 @@ export const inboundEmailHandler = Sentry.GCPFunction.wrapHttpFunction(
html,
text,
forwardedFrom,
receivedEmailId,
},
})
}

View file

@ -13,7 +13,8 @@ export const handlePdfAttachment = async (
email: string,
fileName: string | undefined,
data: Buffer,
subject: string
subject: string,
receivedEmailId: string
): Promise<void> => {
console.log('handlePdfAttachment', email, fileName)
@ -26,7 +27,7 @@ export const handlePdfAttachment = async (
return
}
await uploadToSignedUrl(uploadResult.url, data)
await createArticle(email, uploadResult.id, subject)
await createArticle(email, uploadResult.id, subject, receivedEmailId)
} catch (error) {
console.error('handlePdfAttachment error', error)
}
@ -77,12 +78,14 @@ const uploadToSignedUrl = async (
const createArticle = async (
email: string,
uploadFileId: string,
subject: string
subject: string,
receivedEmailId: string
): Promise<AxiosResponse> => {
const data = {
email,
uploadFileId,
subject,
receivedEmailId,
}
if (process.env.JWT_SECRET === undefined) {