Merge pull request #1389 from omnivore-app/rate-limiting-on-tts

Add a realistic voice api provider
2026-03-11 08:54:26 +00:00 · 2022-11-10 17:25:08 +08:00 · 2022-11-10 17:25:08 +08:00 · f56de45ed0
commit f56de45ed0
parent 120974f348 0f85071af7
21 changed files with 1998 additions and 187 deletions
--- a/apple/OmnivoreKit/Sources/Services/DataService/GQLSchema.swift
+++ b/apple/OmnivoreKit/Sources/Services/DataService/GQLSchema.swift
--- a/packages/api/src/entity/feature.ts
+++ b/packages/api/src/entity/feature.ts
@ -0,0 +1,35 @@
+import {
+  Column,
+  CreateDateColumn,
+  Entity,
+  JoinColumn,
+  ManyToOne,
+  PrimaryGeneratedColumn,
+  UpdateDateColumn,
+} from 'typeorm'
+import { User } from './user'
+
+@Entity({ name: 'features' })
+export class Feature {
+  @PrimaryGeneratedColumn('uuid')
+  id!: string
+
+  @ManyToOne(() => User, { onDelete: 'CASCADE' })
+  @JoinColumn({ name: 'user_id' })
+  user!: User
+
+  @Column('text')
+  name!: string
+
+  @Column('timestamp', { nullable: true })
+  grantedAt?: Date | null
+
+  @Column('timestamp', { nullable: true })
+  expiresAt?: Date | null
+
+  @CreateDateColumn({ default: () => 'CURRENT_TIMESTAMP' })
+  createdAt!: Date
+
+  @UpdateDateColumn({ default: () => 'CURRENT_TIMESTAMP' })
+  updatedAt!: Date
+}
--- a/packages/api/src/entity/user_personalization.ts
+++ b/packages/api/src/entity/user_personalization.ts
@ -39,11 +39,14 @@ export class UserPersonalization {
  @Column('text', { nullable: true })
  speechVoice?: string

-  @Column('integer', { nullable: true })
-  speechRate?: number
+  @Column('text', { nullable: true })
+  speechSecondaryVoice?: string

-  @Column('integer', { nullable: true })
-  speechVolume?: number
+  @Column('text', { nullable: true })
+  speechRate?: string
+
+  @Column('text', { nullable: true })
+  speechVolume?: string

  @CreateDateColumn({ default: () => 'CURRENT_TIMESTAMP' })
  createdAt!: Date
--- a/packages/api/src/generated/graphql.ts
+++ b/packages/api/src/generated/graphql.ts
@ -596,6 +596,17 @@ export type DeviceToken = {
  token: Scalars['String'];
 };

+export type Feature = {
+  __typename?: 'Feature';
+  createdAt: Scalars['Date'];
+  expiresAt?: Maybe<Scalars['Date']>;
+  grantedAt?: Maybe<Scalars['Date']>;
+  id: Scalars['ID'];
+  name: Scalars['String'];
+  token: Scalars['String'];
+  updatedAt: Scalars['Date'];
+};
+
 export type FeedArticle = {
  __typename?: 'FeedArticle';
  annotationsCount?: Maybe<Scalars['Int']>;
@ -971,6 +982,7 @@ export type Mutation = {
  logOut: LogOutResult;
  mergeHighlight: MergeHighlightResult;
  moveLabel: MoveLabelResult;
+  optInFeature: OptInFeatureResult;
  reportItem: ReportItemResult;
  revokeApiKey: RevokeApiKeyResult;
  saveArticleReadingProgress: SaveArticleReadingProgressResult;
@ -1113,6 +1125,11 @@ export type MutationMoveLabelArgs = {
 };


+export type MutationOptInFeatureArgs = {
+  input: OptInFeatureInput;
+};
+
+
 export type MutationReportItemArgs = {
  input: ReportItemInput;
 };
@ -1281,6 +1298,27 @@ export type NewsletterEmailsSuccess = {
  newsletterEmails: Array<NewsletterEmail>;
 };

+export type OptInFeatureError = {
+  __typename?: 'OptInFeatureError';
+  errorCodes: Array<OptInFeatureErrorCode>;
+};
+
+export enum OptInFeatureErrorCode {
+  BadRequest = 'BAD_REQUEST',
+  NotFound = 'NOT_FOUND'
+}
+
+export type OptInFeatureInput = {
+  name: Scalars['String'];
+};
+
+export type OptInFeatureResult = OptInFeatureError | OptInFeatureSuccess;
+
+export type OptInFeatureSuccess = {
+  __typename?: 'OptInFeatureSuccess';
+  feature: Feature;
+};
+
 export type Page = {
  __typename?: 'Page';
  author?: Maybe<Scalars['String']>;
@ -2684,6 +2722,7 @@ export type ResolversTypes = {
  DeleteWebhookResult: ResolversTypes['DeleteWebhookError'] | ResolversTypes['DeleteWebhookSuccess'];
  DeleteWebhookSuccess: ResolverTypeWrapper<DeleteWebhookSuccess>;
  DeviceToken: ResolverTypeWrapper<DeviceToken>;
+  Feature: ResolverTypeWrapper<Feature>;
  FeedArticle: ResolverTypeWrapper<FeedArticle>;
  FeedArticleEdge: ResolverTypeWrapper<FeedArticleEdge>;
  FeedArticlesError: ResolverTypeWrapper<FeedArticlesError>;
@ -2755,6 +2794,11 @@ export type ResolversTypes = {
  NewsletterEmailsErrorCode: NewsletterEmailsErrorCode;
  NewsletterEmailsResult: ResolversTypes['NewsletterEmailsError'] | ResolversTypes['NewsletterEmailsSuccess'];
  NewsletterEmailsSuccess: ResolverTypeWrapper<NewsletterEmailsSuccess>;
+  OptInFeatureError: ResolverTypeWrapper<OptInFeatureError>;
+  OptInFeatureErrorCode: OptInFeatureErrorCode;
+  OptInFeatureInput: OptInFeatureInput;
+  OptInFeatureResult: ResolversTypes['OptInFeatureError'] | ResolversTypes['OptInFeatureSuccess'];
+  OptInFeatureSuccess: ResolverTypeWrapper<OptInFeatureSuccess>;
  Page: ResolverTypeWrapper<Page>;
  PageInfo: ResolverTypeWrapper<PageInfo>;
  PageInfoInput: PageInfoInput;
@ -3045,6 +3089,7 @@ export type ResolversParentTypes = {
  DeleteWebhookResult: ResolversParentTypes['DeleteWebhookError'] | ResolversParentTypes['DeleteWebhookSuccess'];
  DeleteWebhookSuccess: DeleteWebhookSuccess;
  DeviceToken: DeviceToken;
+  Feature: Feature;
  FeedArticle: FeedArticle;
  FeedArticleEdge: FeedArticleEdge;
  FeedArticlesError: FeedArticlesError;
@ -3103,6 +3148,10 @@ export type ResolversParentTypes = {
  NewsletterEmailsError: NewsletterEmailsError;
  NewsletterEmailsResult: ResolversParentTypes['NewsletterEmailsError'] | ResolversParentTypes['NewsletterEmailsSuccess'];
  NewsletterEmailsSuccess: NewsletterEmailsSuccess;
+  OptInFeatureError: OptInFeatureError;
+  OptInFeatureInput: OptInFeatureInput;
+  OptInFeatureResult: ResolversParentTypes['OptInFeatureError'] | ResolversParentTypes['OptInFeatureSuccess'];
+  OptInFeatureSuccess: OptInFeatureSuccess;
  Page: Page;
  PageInfo: PageInfo;
  PageInfoInput: PageInfoInput;
@ -3677,6 +3726,17 @@ export type DeviceTokenResolvers<ContextType = ResolverContext, ParentType exten
  __isTypeOf?: IsTypeOfResolverFn<ParentType, ContextType>;
 };

+export type FeatureResolvers<ContextType = ResolverContext, ParentType extends ResolversParentTypes['Feature'] = ResolversParentTypes['Feature']> = {
+  createdAt?: Resolver<ResolversTypes['Date'], ParentType, ContextType>;
+  expiresAt?: Resolver<Maybe<ResolversTypes['Date']>, ParentType, ContextType>;
+  grantedAt?: Resolver<Maybe<ResolversTypes['Date']>, ParentType, ContextType>;
+  id?: Resolver<ResolversTypes['ID'], ParentType, ContextType>;
+  name?: Resolver<ResolversTypes['String'], ParentType, ContextType>;
+  token?: Resolver<ResolversTypes['String'], ParentType, ContextType>;
+  updatedAt?: Resolver<ResolversTypes['Date'], ParentType, ContextType>;
+  __isTypeOf?: IsTypeOfResolverFn<ParentType, ContextType>;
+};
+
 export type FeedArticleResolvers<ContextType = ResolverContext, ParentType extends ResolversParentTypes['FeedArticle'] = ResolversParentTypes['FeedArticle']> = {
  annotationsCount?: Resolver<Maybe<ResolversTypes['Int']>, ParentType, ContextType>;
  article?: Resolver<ResolversTypes['Article'], ParentType, ContextType>;
@ -3971,6 +4031,7 @@ export type MutationResolvers<ContextType = ResolverContext, ParentType extends
  logOut?: Resolver<ResolversTypes['LogOutResult'], ParentType, ContextType>;
  mergeHighlight?: Resolver<ResolversTypes['MergeHighlightResult'], ParentType, ContextType, RequireFields<MutationMergeHighlightArgs, 'input'>>;
  moveLabel?: Resolver<ResolversTypes['MoveLabelResult'], ParentType, ContextType, RequireFields<MutationMoveLabelArgs, 'input'>>;
+  optInFeature?: Resolver<ResolversTypes['OptInFeatureResult'], ParentType, ContextType, RequireFields<MutationOptInFeatureArgs, 'input'>>;
  reportItem?: Resolver<ResolversTypes['ReportItemResult'], ParentType, ContextType, RequireFields<MutationReportItemArgs, 'input'>>;
  revokeApiKey?: Resolver<ResolversTypes['RevokeApiKeyResult'], ParentType, ContextType, RequireFields<MutationRevokeApiKeyArgs, 'id'>>;
  saveArticleReadingProgress?: Resolver<ResolversTypes['SaveArticleReadingProgressResult'], ParentType, ContextType, RequireFields<MutationSaveArticleReadingProgressArgs, 'input'>>;
@ -4023,6 +4084,20 @@ export type NewsletterEmailsSuccessResolvers<ContextType = ResolverContext, Pare
  __isTypeOf?: IsTypeOfResolverFn<ParentType, ContextType>;
 };

+export type OptInFeatureErrorResolvers<ContextType = ResolverContext, ParentType extends ResolversParentTypes['OptInFeatureError'] = ResolversParentTypes['OptInFeatureError']> = {
+  errorCodes?: Resolver<Array<ResolversTypes['OptInFeatureErrorCode']>, ParentType, ContextType>;
+  __isTypeOf?: IsTypeOfResolverFn<ParentType, ContextType>;
+};
+
+export type OptInFeatureResultResolvers<ContextType = ResolverContext, ParentType extends ResolversParentTypes['OptInFeatureResult'] = ResolversParentTypes['OptInFeatureResult']> = {
+  __resolveType: TypeResolveFn<'OptInFeatureError' | 'OptInFeatureSuccess', ParentType, ContextType>;
+};
+
+export type OptInFeatureSuccessResolvers<ContextType = ResolverContext, ParentType extends ResolversParentTypes['OptInFeatureSuccess'] = ResolversParentTypes['OptInFeatureSuccess']> = {
+  feature?: Resolver<ResolversTypes['Feature'], ParentType, ContextType>;
+  __isTypeOf?: IsTypeOfResolverFn<ParentType, ContextType>;
+};
+
 export type PageResolvers<ContextType = ResolverContext, ParentType extends ResolversParentTypes['Page'] = ResolversParentTypes['Page']> = {
  author?: Resolver<Maybe<ResolversTypes['String']>, ParentType, ContextType>;
  createdAt?: Resolver<ResolversTypes['Date'], ParentType, ContextType>;
@ -4835,6 +4910,7 @@ export type Resolvers<ContextType = ResolverContext> = {
  DeleteWebhookResult?: DeleteWebhookResultResolvers<ContextType>;
  DeleteWebhookSuccess?: DeleteWebhookSuccessResolvers<ContextType>;
  DeviceToken?: DeviceTokenResolvers<ContextType>;
+  Feature?: FeatureResolvers<ContextType>;
  FeedArticle?: FeedArticleResolvers<ContextType>;
  FeedArticleEdge?: FeedArticleEdgeResolvers<ContextType>;
  FeedArticlesError?: FeedArticlesErrorResolvers<ContextType>;
@ -4885,6 +4961,9 @@ export type Resolvers<ContextType = ResolverContext> = {
  NewsletterEmailsError?: NewsletterEmailsErrorResolvers<ContextType>;
  NewsletterEmailsResult?: NewsletterEmailsResultResolvers<ContextType>;
  NewsletterEmailsSuccess?: NewsletterEmailsSuccessResolvers<ContextType>;
+  OptInFeatureError?: OptInFeatureErrorResolvers<ContextType>;
+  OptInFeatureResult?: OptInFeatureResultResolvers<ContextType>;
+  OptInFeatureSuccess?: OptInFeatureSuccessResolvers<ContextType>;
  Page?: PageResolvers<ContextType>;
  PageInfo?: PageInfoResolvers<ContextType>;
  Profile?: ProfileResolvers<ContextType>;
--- a/packages/api/src/generated/schema.graphql
+++ b/packages/api/src/generated/schema.graphql
@ -524,6 +524,16 @@ type DeviceToken {
  token: String!
 }

+type Feature {
+  createdAt: Date!
+  expiresAt: Date
+  grantedAt: Date
+  id: ID!
+  name: String!
+  token: String!
+  updatedAt: Date!
+}
+
 type FeedArticle {
  annotationsCount: Int
  article: Article!
@ -865,6 +875,7 @@ type Mutation {
  logOut: LogOutResult!
  mergeHighlight(input: MergeHighlightInput!): MergeHighlightResult!
  moveLabel(input: MoveLabelInput!): MoveLabelResult!
+  optInFeature(input: OptInFeatureInput!): OptInFeatureResult!
  reportItem(input: ReportItemInput!): ReportItemResult!
  revokeApiKey(id: ID!): RevokeApiKeyResult!
  saveArticleReadingProgress(input: SaveArticleReadingProgressInput!): SaveArticleReadingProgressResult!
@ -917,6 +928,25 @@ type NewsletterEmailsSuccess {
  newsletterEmails: [NewsletterEmail!]!
 }

+type OptInFeatureError {
+  errorCodes: [OptInFeatureErrorCode!]!
+}
+
+enum OptInFeatureErrorCode {
+  BAD_REQUEST
+  NOT_FOUND
+}
+
+input OptInFeatureInput {
+  name: String!
+}
+
+union OptInFeatureResult = OptInFeatureError | OptInFeatureSuccess
+
+type OptInFeatureSuccess {
+  feature: Feature!
+}
+
 type Page {
  author: String
  createdAt: Date!
--- a/packages/api/src/resolvers/features/index.ts
+++ b/packages/api/src/resolvers/features/index.ts
@ -0,0 +1,60 @@
+import { authorized } from '../../utils/helpers'
+import {
+  MutationOptInFeatureArgs,
+  OptInFeatureError,
+  OptInFeatureErrorCode,
+  OptInFeatureSuccess,
+} from '../../generated/graphql'
+import {
+  getFeatureName,
+  optInFeature,
+  signFeatureToken,
+} from '../../services/features'
+
+export const optInFeatureResolver = authorized<
+  OptInFeatureSuccess,
+  OptInFeatureError,
+  MutationOptInFeatureArgs
+>(async (_, { input: { name } }, { claims, log }) => {
+  log.info('Opting in to a feature', {
+    feature: name,
+    labels: {
+      source: 'resolver',
+      resolver: 'optInFeatureResolver',
+      uid: claims.uid,
+    },
+  })
+
+  try {
+    const featureName = getFeatureName(name)
+    if (!featureName) {
+      return {
+        errorCodes: [OptInFeatureErrorCode.NotFound],
+      }
+    }
+
+    const optIn = await optInFeature(featureName, claims.uid)
+    if (!optIn) {
+      return {
+        errorCodes: [OptInFeatureErrorCode.NotFound],
+      }
+    }
+
+    const token = signFeatureToken(optIn)
+
+    return {
+      feature: {
+        ...optIn,
+        token,
+      },
+    }
+  } catch (e) {
+    log.error('Error opting in to a feature', {
+      error: e,
+    })
+
+    return {
+      errorCodes: [OptInFeatureErrorCode.BadRequest],
+    }
+  }
+})
--- a/packages/api/src/resolvers/function_resolvers.ts
+++ b/packages/api/src/resolvers/function_resolvers.ts
@ -5,8 +5,8 @@
 /* eslint-disable @typescript-eslint/explicit-module-boundary-types */
 import { createReactionResolver, deleteReactionResolver } from './reaction'
 import { Claims, WithDataSourcesContext } from './types'
-import { createImageProxyUrl } from './../utils/imageproxy'
-import { userDataToUser, validatedDate } from './../utils/helpers'
+import { createImageProxyUrl } from '../utils/imageproxy'
+import { userDataToUser, validatedDate } from '../utils/helpers'

 import {
  Article,
@ -18,7 +18,7 @@ import {
  Reaction,
  SearchItem,
  User,
-} from './../generated/graphql'
+} from '../generated/graphql'

 import {
  addPopularReadResolver,
@ -101,6 +101,7 @@ import {
 } from '../utils/uploads'
 import { getPageByParam } from '../elastic/pages'
 import { recentSearchesResolver } from './recent_searches'
+import { optInFeatureResolver } from './features'

 /* eslint-disable @typescript-eslint/naming-convention */
 type ResultResolveType = {
@ -171,6 +172,7 @@ export const functionResolvers = {
    moveLabel: moveLabelResolver,
    setIntegration: setIntegrationResolver,
    deleteIntegration: deleteIntegrationResolver,
+    optInFeature: optInFeatureResolver,
  },
  Query: {
    me: getMeUserResolver,
@ -607,4 +609,5 @@ export const functionResolvers = {
  ...resultResolveTypeResolver('Integrations'),
  ...resultResolveTypeResolver('DeleteIntegration'),
  ...resultResolveTypeResolver('RecentSearches'),
+  ...resultResolveTypeResolver('OptInFeature'),
 }
--- a/packages/api/src/routers/text_to_speech.ts
+++ b/packages/api/src/routers/text_to_speech.ts
@ -13,6 +13,9 @@ import { shouldSynthesize } from '../services/speech'
 import { readPushSubscription } from '../datalayer/pubsub'
 import { AppDataSource } from '../server'
 import { enqueueTextToSpeech } from '../utils/createTask'
+import { htmlToSpeechFile } from '@omnivore/text-to-speech-handler'
+import { UserPersonalization } from '../entity/user_personalization'
+import { ArticleSavingRequestStatus } from '../elastic/types'

 const logger = buildLogger('app.dispatch')

@ -34,17 +37,17 @@ export function textToSpeechRouter() {
    }

    try {
-      const data: { userId: string; type: string; id: string; state: string } =
+      const data: { userId: string; type: string; id: string } =
        JSON.parse(msgStr)
-      const { userId, type, id, state } = data
+      const { userId, type, id } = data
      if (!userId || !type || !id) {
        logger.info('Invalid data')
        return res.status(400).send('Bad Request')
      }

-      if (type.toUpperCase() !== 'PAGE' || state !== 'SUCCEEDED') {
-        logger.info('Not a page or not succeeded')
-        return res.status(200).send('Not a page or not succeeded')
+      if (type.toUpperCase() !== 'PAGE') {
+        logger.info('Not a page')
+        return res.status(200).send('Not a page')
      }

      const page = await getPageById(id)
@ -53,25 +56,45 @@ export function textToSpeechRouter() {
        return res.status(200).send('No page found')
      }

+      if (page.state === ArticleSavingRequestStatus.Processing) {
+        logger.info('Page is still processing, try again later', { id })
+        return res.status(400).send('Page is still processing')
+      }
+
      // checks if this page needs to be synthesized automatically
      if (await shouldSynthesize(userId, page)) {
        logger.info('page needs to be synthesized')
-        // initialize state
-        const speech = await getRepository(Speech).save({
-          user: { id: userId },
-          elasticPageId: id,
-          state: SpeechState.INITIALIZED,
-          voice: 'en-US-JennyNeural',
+
+        const userPersonalization = await getRepository(
+          UserPersonalization
+        ).findOneBy({ user: { id: userId } })
+
+        const speechFile = htmlToSpeechFile({
+          title: page.title,
+          content: page.content,
+          options: {
+            primaryVoice: userPersonalization?.speechVoice || 'Axel',
+            secondaryVoice:
+              userPersonalization?.speechSecondaryVoice || 'Evelyn',
+            language: page.language,
+          },
        })
-        // enqueue a task to convert text to speech
-        const taskName = await enqueueTextToSpeech({
-          userId,
-          speechId: speech.id,
-          text: page.content,
-          voice: speech.voice,
-          priority: 'low',
-        })
-        logger.info('Start Text to speech task', { taskName })
+
+        for (const utterance of speechFile.utterances) {
+          // enqueue a task to convert text to speech
+          const taskName = await enqueueTextToSpeech({
+            userId,
+            speechId: utterance.idx,
+            text: utterance.text,
+            voice: utterance.voice || 'Axel',
+            priority: 'high',
+            isUltraRealisticVoice: true,
+            language: speechFile.language,
+            rate: userPersonalization?.speechRate || '1.1',
+          })
+          logger.info('Start Text to speech task', { taskName })
+        }
+
        return res.status(202).send('Text to speech task started')
      }

--- a/packages/api/src/schema.ts
+++ b/packages/api/src/schema.ts
@ -1912,6 +1912,35 @@ const schema = gql`
    BAD_REQUEST
  }

+  input OptInFeatureInput {
+    name: String!
+  }
+
+  union OptInFeatureResult = OptInFeatureSuccess | OptInFeatureError
+
+  type OptInFeatureSuccess {
+    feature: Feature!
+  }
+
+  type Feature {
+    id: ID!
+    name: String!
+    token: String!
+    createdAt: Date!
+    updatedAt: Date!
+    grantedAt: Date
+    expiresAt: Date
+  }
+
+  type OptInFeatureError {
+    errorCodes: [OptInFeatureErrorCode!]!
+  }
+
+  enum OptInFeatureErrorCode {
+    BAD_REQUEST
+    NOT_FOUND
+  }
+
  # Mutations
  type Mutation {
    googleLogin(input: GoogleLoginInput!): LoginResult!
@ -1983,6 +2012,7 @@ const schema = gql`
    moveLabel(input: MoveLabelInput!): MoveLabelResult!
    setIntegration(input: SetIntegrationInput!): SetIntegrationResult!
    deleteIntegration(id: ID!): DeleteIntegrationResult!
+    optInFeature(input: OptInFeatureInput!): OptInFeatureResult!
  }

  # FIXME: remove sort from feedArticles after all cached tabs are closed
--- a/packages/api/src/services/features.ts
+++ b/packages/api/src/services/features.ts
@ -0,0 +1,69 @@
+import { Feature } from '../entity/feature'
+import { getRepository } from '../entity/utils'
+import * as jwt from 'jsonwebtoken'
+import { env } from '../env'
+import { IsNull, Not } from 'typeorm'
+
+enum FeatureName {
+  UltraRealisticVoice = 'ultra-realistic-voice',
+}
+
+export const getFeatureName = (name: string): FeatureName | undefined => {
+  return Object.values(FeatureName).find((v) => v === name)
+}
+
+export const optInFeature = async (
+  name: FeatureName,
+  uid: string
+): Promise<Feature | undefined> => {
+  if (name === FeatureName.UltraRealisticVoice) {
+    return optInUltraRealisticVoice(uid)
+  }
+
+  return undefined
+}
+
+const optInUltraRealisticVoice = async (uid: string): Promise<Feature> => {
+  const feature = await getRepository(Feature).findOne({
+    where: {
+      user: { id: uid },
+      name: FeatureName.UltraRealisticVoice,
+    },
+    relations: ['user'],
+  })
+  if (feature) {
+    // already opted in
+    console.log('already opted in')
+    return feature
+  }
+
+  // opt in to feature for the first 1000 users
+  const count = await getRepository(Feature).countBy({
+    name: FeatureName.UltraRealisticVoice,
+    grantedAt: Not(IsNull()),
+  })
+
+  let grantedAt: Date | null = new Date()
+  if (count >= 1000) {
+    console.log('feature limit reached')
+    grantedAt = null
+  }
+
+  return getRepository(Feature).save({
+    user: { id: uid },
+    name: FeatureName.UltraRealisticVoice,
+    grantedAt,
+  })
+}
+
+export const signFeatureToken = (feature: Feature): string => {
+  return jwt.sign(
+    {
+      uid: feature.user.id,
+      featureName: feature.name,
+      grantedAt: feature.grantedAt ? feature.grantedAt.getTime() / 1000 : null,
+    },
+    env.server.jwtSecret,
+    { expiresIn: '1d' }
+  )
+}
--- a/packages/api/src/utils/createTask.ts
+++ b/packages/api/src/utils/createTask.ts
@ -344,6 +344,9 @@ export const enqueueTextToSpeech = async ({
  bucket = env.fileUpload.gcsUploadBucket,
  queue = 'omnivore-demo-text-to-speech-queue',
  location = env.gcp.location,
+  isUltraRealisticVoice = false,
+  language,
+  rate,
 }: {
  userId: string
  speechId: string
@ -354,6 +357,9 @@ export const enqueueTextToSpeech = async ({
  textType?: 'text' | 'ssml'
  queue?: string
  location?: string
+  isUltraRealisticVoice?: boolean
+  language?: string
+  rate?: string
 }): Promise<string> => {
  const { GOOGLE_CLOUD_PROJECT } = process.env
  const payload = {
@ -362,6 +368,9 @@ export const enqueueTextToSpeech = async ({
    voice,
    bucket,
    textType,
+    isUltraRealisticVoice,
+    language,
+    rate,
  }
  // eslint-disable-next-line @typescript-eslint/ban-ts-comment
  // @ts-ignore
--- a/packages/api/test/resolvers/features.test.ts
+++ b/packages/api/test/resolvers/features.test.ts
@ -0,0 +1,202 @@
+import 'mocha'
+import { expect } from 'chai'
+import { User } from '../../src/entity/user'
+import { createTestUser, deleteTestUser } from '../db'
+import { graphqlRequest, request } from '../util'
+import { getRepository } from '../../src/entity/utils'
+import { Feature } from '../../src/entity/feature'
+import * as jwt from 'jsonwebtoken'
+import sinon, { SinonFakeTimers } from 'sinon'
+import { env } from '../../src/env'
+import { Like } from 'typeorm'
+
+describe('features resolvers', () => {
+  let loginUser: User
+  let authToken: string
+
+  before(async () => {
+    // create test user and login
+    loginUser = await createTestUser('loginUser')
+    const res = await request
+      .post('/local/debug/fake-user-login')
+      .send({ fakeEmail: loginUser.email })
+
+    authToken = res.body.authToken
+  })
+
+  after(async () => {
+    await deleteTestUser(loginUser.name)
+  })
+
+  describe('optInFeature API', () => {
+    const featureName = 'ultra-realistic-voice'
+    const now = new Date()
+    let clock: SinonFakeTimers
+
+    const query = (name: string) => `
+      mutation {
+        optInFeature(input: {
+          name: "${name}"
+        }) {
+          ... on OptInFeatureSuccess {
+            feature {
+              name
+              grantedAt
+              token
+            }
+          }
+          ... on OptInFeatureError {
+            errorCodes
+          }
+        }
+      }
+    `
+
+    before(() => {
+      console.log('opting in to feature')
+      // mock date and ignore milliseconds
+      clock = sinon.useFakeTimers(now.setSeconds(now.getSeconds(), 0))
+    })
+
+    after(() => {
+      clock.restore()
+    })
+
+    context('when user is the first 1000 users', () => {
+      after(async () => {
+        // reset feature
+        await getRepository(Feature).delete({
+          user: { id: loginUser.id },
+        })
+      })
+
+      it('opts in to the feature', async () => {
+        const res = await graphqlRequest(query(featureName), authToken).expect(
+          200
+        )
+
+        const token = jwt.sign(
+          {
+            uid: loginUser.id,
+            featureName,
+            grantedAt: Date.now() / 1000,
+          },
+          env.server.jwtSecret,
+          { expiresIn: '1d' }
+        )
+
+        expect(res.body.data.optInFeature).to.eql({
+          feature: {
+            name: featureName,
+            grantedAt: new Date().toISOString(),
+            token,
+          },
+        })
+      })
+    })
+
+    context('when user is not the first 1000 users', () => {
+      before(async () => {
+        // create 1000 opt-in users
+        const usersToSave = Array.from(Array(1000).keys()).map((i) => {
+          return {
+            name: `user${i}`,
+            source: 'GOOGLE',
+            sourceUserId: `fake-user-id-user${i}`,
+            email: `user${i}@omnivore.app`,
+            username: `user${i}`,
+            bio: `i am user${i}`,
+          }
+        })
+
+        const users = await getRepository(User).save(usersToSave)
+
+        const features = users.map((user) => {
+          return {
+            user: { id: user.id },
+            name: featureName,
+            grantedAt: new Date(),
+          }
+        })
+
+        await getRepository(Feature).save(features)
+      })
+
+      after(async () => {
+        // reset opt-in users
+        await getRepository(User).delete({
+          name: Like(`user%`),
+        })
+        await getRepository(Feature).delete({
+          name: featureName,
+        })
+      })
+
+      it('does not opt in to the feature', async () => {
+        const res = await graphqlRequest(query(featureName), authToken).expect(
+          200
+        )
+
+        const token = jwt.sign(
+          {
+            uid: loginUser.id,
+            featureName,
+            grantedAt: null,
+          },
+          env.server.jwtSecret,
+          { expiresIn: '1d' }
+        )
+
+        expect(res.body.data.optInFeature).to.eql({
+          feature: {
+            name: featureName,
+            grantedAt: null,
+            token,
+          },
+        })
+      })
+    })
+
+    context('when user is already opted in', () => {
+      before(async () => {
+        // opt in
+        await getRepository(Feature).save({
+          user: { id: loginUser.id },
+          name: featureName,
+          grantedAt: new Date(),
+        })
+      })
+
+      after(async () => {
+        // reset feature
+        await getRepository(Feature).delete({
+          user: { id: loginUser.id },
+        })
+      })
+
+      it('returns the feature', async () => {
+        const res = await graphqlRequest(query(featureName), authToken).expect(
+          200
+        )
+
+        const token = jwt.sign(
+          {
+            uid: loginUser.id,
+            featureName,
+            grantedAt: Date.now() / 1000,
+          },
+          env.server.jwtSecret,
+          { expiresIn: '1d' }
+        )
+
+        expect(res.body.data.optInFeature).to.eql({
+          feature: {
+            name: featureName,
+            grantedAt: new Date().toISOString(),
+            token,
+          },
+        })
+      })
+    })
+  })
+})
--- a/packages/db/migrations/0098.do.create_features_table.sql
+++ b/packages/db/migrations/0098.do.create_features_table.sql
@ -0,0 +1,28 @@
+-- Type: DO
+-- Name: create_features_table
+-- Description: Create features table to store opt-in features by users
+
+BEGIN;
+
+CREATE TABLE IF NOT EXISTS omnivore.features (
+    id uuid PRIMARY KEY DEFAULT uuid_generate_v1mc(),
+    user_id uuid NOT NULL REFERENCES omnivore.user ON DELETE CASCADE,
+    name text NOT NULL,
+    granted_at timestamptz,
+    expires_at timestamptz,
+    created_at timestamptz NOT NULL DEFAULT current_timestamp,
+    updated_at timestamptz NOT NULL DEFAULT current_timestamp,
+    UNIQUE (user_id, name)
+);
+
+CREATE TRIGGER features_modtime BEFORE UPDATE ON omnivore.features
+    FOR EACH ROW EXECUTE PROCEDURE update_updated_at_column();
+
+GRANT SELECT, INSERT, UPDATE, DELETE ON omnivore.features TO omnivore_user;
+
+ALTER TABLE omnivore.user_personalization
+    ADD COLUMN IF NOT EXISTS speech_secondary_voice text,
+    ALTER COLUMN speech_rate TYPE text,
+    ALTER COLUMN speech_volume TYPE text;
+
+COMMIT;
--- a/packages/db/migrations/0098.undo.create_features_table.sql
+++ b/packages/db/migrations/0098.undo.create_features_table.sql
@ -0,0 +1,14 @@
+-- Type: UNDO
+-- Name: create_features_table
+-- Description: Create features table to store opt-in features by users
+
+BEGIN;
+
+DROP TABLE IF EXISTS omnivore.features;
+
+ALTER TABLE omnivore.user_personalization
+    DROP COLUMN IF EXISTS speech_secondary_voice,
+    ALTER COLUMN speech_rate TYPE integer USING speech_rate::integer,
+    ALTER COLUMN speech_volume TYPE integer USING speech_volume::integer;
+
+COMMIT;
--- a/packages/text-to-speech/package.json
+++ b/packages/text-to-speech/package.json
@ -21,6 +21,7 @@
    "deploy": "yarn build && yarn gcloud-deploy"
  },
  "devDependencies": {
+    "@types/fluent-ffmpeg": "^2.1.20",
    "@types/html-to-text": "^8.1.1",
    "@types/natural": "^5.1.1",
    "@types/node": "^14.11.2",
@ -30,11 +31,13 @@
    "mocha": "^10.0.0"
  },
  "dependencies": {
+    "@ffmpeg-installer/ffmpeg": "^1.1.0",
    "@google-cloud/functions-framework": "3.1.2",
    "@google-cloud/storage": "^6.4.1",
    "@sentry/serverless": "^6.16.1",
    "axios": "^0.27.2",
    "dotenv": "^16.0.1",
+    "fluent-ffmpeg": "^2.1.2",
    "html-to-text": "^8.2.1",
    "jsonwebtoken": "^8.5.1",
    "linkedom": "^0.14.12",
--- a/packages/text-to-speech/src/azureTextToSpeech.ts
+++ b/packages/text-to-speech/src/azureTextToSpeech.ts
@ -0,0 +1,152 @@
+import {
+  CancellationDetails,
+  CancellationReason,
+  ResultReason,
+  SpeechConfig,
+  SpeechSynthesisOutputFormat,
+  SpeechSynthesisResult,
+  SpeechSynthesizer,
+} from 'microsoft-cognitiveservices-speech-sdk'
+import { endSsml, htmlToSsmlItems, ssmlItemText, startSsml } from './htmlToSsml'
+import * as _ from 'underscore'
+import {
+  SpeechMark,
+  TextToSpeech,
+  TextToSpeechInput,
+  TextToSpeechOutput,
+} from './textToSpeech'
+
+export class AzureTextToSpeech implements TextToSpeech {
+  use(input: TextToSpeechInput): boolean {
+    return !input.isUltraRealisticVoice
+  }
+
+  synthesizeTextToSpeech = async (
+    input: TextToSpeechInput
+  ): Promise<TextToSpeechOutput> => {
+    if (!process.env.AZURE_SPEECH_KEY || !process.env.AZURE_SPEECH_REGION) {
+      throw new Error('Azure Speech Key or Region not set')
+    }
+    const textType = input.textType || 'html'
+    const audioStream = input.audioStream
+    const speechConfig = SpeechConfig.fromSubscription(
+      process.env.AZURE_SPEECH_KEY,
+      process.env.AZURE_SPEECH_REGION
+    )
+    speechConfig.speechSynthesisOutputFormat =
+      SpeechSynthesisOutputFormat.Audio16Khz32KBitRateMonoMp3
+
+    // Create the speech synthesizer.
+    const synthesizer = new SpeechSynthesizer(speechConfig)
+    const speechMarks: SpeechMark[] = []
+    let timeOffset = 0
+    let wordOffset = 0
+
+    synthesizer.synthesizing = function (s, e) {
+      // convert arrayBuffer to stream and write to stream
+      audioStream?.write(Buffer.from(e.result.audioData))
+    }
+
+    // The event synthesis completed signals that the synthesis is completed.
+    synthesizer.synthesisCompleted = (s, e) => {
+      console.info(
+        `(synthesized) Reason: ${ResultReason[e.result.reason]} Audio length: ${
+          e.result.audioData.byteLength
+        }`
+      )
+    }
+
+    // The synthesis started event signals that the synthesis is started.
+    synthesizer.synthesisStarted = (s, e) => {
+      console.info('(synthesis started)')
+    }
+
+    // The event signals that the service has stopped processing speech.
+    // This can happen when an error is encountered.
+    synthesizer.SynthesisCanceled = (s, e) => {
+      const cancellationDetails = CancellationDetails.fromResult(e.result)
+      let str =
+        '(cancel) Reason: ' + CancellationReason[cancellationDetails.reason]
+      if (cancellationDetails.reason === CancellationReason.Error) {
+        str += ': ' + e.result.errorDetails
+      }
+      console.log(str)
+    }
+
+    // The unit of e.audioOffset is tick (1 tick = 100 nanoseconds), divide by 10,000 to convert to milliseconds.
+    synthesizer.wordBoundary = (s, e) => {
+      speechMarks.push({
+        word: e.text,
+        time: (timeOffset + e.audioOffset) / 10000,
+        start: wordOffset + e.textOffset,
+        length: e.wordLength,
+        type: 'word',
+      })
+    }
+
+    synthesizer.bookmarkReached = (s, e) => {
+      speechMarks.push({
+        word: e.text,
+        time: (timeOffset + e.audioOffset) / 10000,
+        type: 'bookmark',
+      })
+    }
+
+    const speakSsmlAsyncPromise = (
+      ssml: string
+    ): Promise<SpeechSynthesisResult> => {
+      return new Promise((resolve, reject) => {
+        synthesizer.speakSsmlAsync(
+          ssml,
+          (result) => {
+            resolve(result)
+          },
+          (error) => {
+            reject(error)
+          }
+        )
+      })
+    }
+
+    try {
+      const ssmlOptions = {
+        primaryVoice: input.voice,
+        secondaryVoice: input.secondaryVoice,
+        language: input.language,
+        rate: input.rate,
+      }
+      if (textType === 'html') {
+        const ssmlItems = htmlToSsmlItems(input.text, ssmlOptions)
+        for (const ssmlItem of ssmlItems) {
+          const ssml = ssmlItemText(ssmlItem)
+          const result = await speakSsmlAsyncPromise(ssml)
+          timeOffset = timeOffset + result.audioDuration
+        }
+        return {
+          speechMarks,
+        }
+      }
+      // for ssml
+      const startSsmlTag = startSsml(ssmlOptions)
+      wordOffset -= startSsmlTag.length
+      const text = _.escape(input.text)
+      const ssml = `${startSsmlTag}${text}${endSsml()}`
+      const result = await speakSsmlAsyncPromise(ssml)
+      if (result.reason === ResultReason.Canceled) {
+        throw new Error(result.errorDetails)
+      }
+
+      return {
+        audioData: Buffer.from(result.audioData),
+        speechMarks,
+      }
+    } catch (error) {
+      console.error('synthesis error:', error)
+      throw error
+    } finally {
+      audioStream?.end()
+      synthesizer.close()
+      console.log('synthesizer closed')
+    }
+  }
+}
--- a/packages/text-to-speech/src/htmlToSsml.ts
+++ b/packages/text-to-speech/src/htmlToSsml.ts
@ -16,7 +16,7 @@ export interface Utterance {
  text: string
  wordOffset: number
  wordCount: number
-  voice?: string
+  voice: string
 }

 export interface SpeechFile {
@ -44,7 +44,7 @@ export type SSMLOptions = {
 const DEFAULT_LANGUAGE = 'en-US'
 const DEFAULT_VOICE = 'en-US-JennyNeural'
 const DEFAULT_SECONDARY_VOICE = 'en-US-GuyNeural'
-const DEFAULT_RATE = '1.0'
+const DEFAULT_RATE = '1.1'

 const ANCHOR_ELEMENTS_BLOCKED_ATTRIBUTES = [
  'omnivore-highlight-id',
@ -269,7 +269,7 @@ const textToUtterances = ({
  idx: string
  textItems: string[]
  wordOffset: number
-  voice?: string
+  voice: string
  isHtml?: boolean
 }): Utterance[] => {
  let text = textItems.join('')
@ -393,6 +393,7 @@ export const htmlToSpeechFile = (htmlInput: HtmlInput): SpeechFile => {
      textItems: [stripEmojis(title)], // title could have emoji
      wordOffset,
      isHtml: false,
+      voice: defaultVoice,
    })[0]
    utterances.push(titleUtterance)
    wordOffset += titleUtterance.wordCount
@ -413,7 +414,9 @@ export const htmlToSpeechFile = (htmlInput: HtmlInput): SpeechFile => {
        textItems,
        wordOffset,
        voice:
-          node.nodeName === 'BLOCKQUOTE' ? options.secondaryVoice : undefined,
+          node.nodeName === 'BLOCKQUOTE'
+            ? options.secondaryVoice || defaultVoice
+            : defaultVoice,
      })
      const wordCount = newUtterances.reduce((acc, u) => acc + u.wordCount, 0)
      wordCount > 0 && utterances.push(...newUtterances)
--- a/packages/text-to-speech/src/index.ts
+++ b/packages/text-to-speech/src/index.ts
@ -7,22 +7,29 @@ import * as Sentry from '@sentry/serverless'
 import axios from 'axios'
 import * as jwt from 'jsonwebtoken'
 import * as dotenv from 'dotenv' // see https://github.com/motdotla/dotenv#how-do-i-use-dotenv-with-import
-import {
-  SpeechMark,
-  synthesizeTextToSpeech,
-  TextToSpeechInput,
-} from './textToSpeech'
+import { AzureTextToSpeech } from './azureTextToSpeech'
 import { File, Storage } from '@google-cloud/storage'
 import { endSsml, htmlToSpeechFile, startSsml } from './htmlToSsml'
 import crypto from 'crypto'
 import { createRedisClient } from './redis'
+import {
+  SpeechMark,
+  TextToSpeechInput,
+  TextToSpeechOutput,
+} from './textToSpeech'
+import { createClient } from 'redis'
+import { RealisticTextToSpeech } from './realisticTextToSpeech'
+
+// explicitly create the return type of RedisClient
+type RedisClient = ReturnType<typeof createClient>

 interface UtteranceInput {
+  text: string
+  idx: string
+  isUltraRealisticVoice?: boolean
  voice?: string
  rate?: string
  language?: string
-  text: string
-  idx: string
 }

 interface HTMLInput {
@ -40,14 +47,38 @@ interface CacheResult {
  speechMarks: SpeechMark[]
 }

+interface Claim {
+  uid: string
+  featureName: string | null
+  grantedAt: number | null
+}
+
 dotenv.config()
 Sentry.GCPFunction.init({
  dsn: process.env.SENTRY_DSN,
  tracesSampleRate: 0,
 })

+const MAX_CHARACTER_COUNT = 50000
 const storage = new Storage()

+const textToSpeechHandlers = [
+  new AzureTextToSpeech(),
+  new RealisticTextToSpeech(),
+]
+
+const synthesizeTextToSpeech = async (
+  input: TextToSpeechInput
+): Promise<TextToSpeechOutput> => {
+  const textToSpeechHandler = textToSpeechHandlers.find((handler) =>
+    handler.use(input)
+  )
+  if (!textToSpeechHandler) {
+    throw new Error('No text to speech handler found')
+  }
+  return textToSpeechHandler.synthesizeTextToSpeech(input)
+}
+
 const uploadToBucket = async (
  filePath: string,
  data: Buffer,
@ -57,7 +88,7 @@ const uploadToBucket = async (
  await storage.bucket(bucket).file(filePath).save(data, options)
 }

-const createGCSFile = (bucket: string, filename: string): File => {
+export const createGCSFile = (bucket: string, filename: string): File => {
  return storage.bucket(bucket).file(filename)
 }

@ -84,14 +115,41 @@ const updateSpeech = async (
  return response.status === 200
 }

+const getCharacterCountFromRedis = async (
+  redisClient: RedisClient,
+  uid: string
+): Promise<number> => {
+  const wordCount = await redisClient.get(`tts:charCount:${uid}`)
+  return wordCount ? parseInt(wordCount) : 0
+}
+
+// store character count of each text to speech request in redis
+// which will be used to rate limit the request
+// expires after 1 day
+const updateCharacterCountInRedis = async (
+  redisClient: RedisClient,
+  uid: string,
+  wordCount: number
+): Promise<void> => {
+  await redisClient.set(`tts:charCount:${uid}`, wordCount.toString(), {
+    EX: 3600 * 24, // in seconds
+    NX: true,
+  })
+}
+
 export const textToSpeechHandler = Sentry.GCPFunction.wrapHttpFunction(
  async (req, res) => {
    console.info('Text to speech request body:', req.body)
-    const token = req.query.token as string
    if (!process.env.JWT_SECRET) {
      console.error('JWT_SECRET not exists')
      return res.status(500).send({ errorCodes: 'JWT_SECRET_NOT_EXISTS' })
    }
+
+    const token = (req.query.token || req.headers.authorization) as string
+    if (!token) {
+      return res.status(401).send({ errorCode: 'INVALID_TOKEN' })
+    }
+
    try {
      jwt.verify(token, process.env.JWT_SECRET)
    } catch (e) {
@ -114,21 +172,28 @@ export const textToSpeechHandler = Sentry.GCPFunction.wrapHttpFunction(
      }) as NodeJS.WriteStream
      // synthesize text to speech
      const startTime = Date.now()
+      // temporary solution to use realistic text to speech
      const { speechMarks } = await synthesizeTextToSpeech({
        ...input,
        textType: 'html',
        audioStream,
+        key: id,
      })
      console.info(
        `Synthesize text to speech completed in ${Date.now() - startTime} ms`
      )
+
      // speech marks file to be saved in GCS
-      const speechMarksFileName = `speech/${id}.json`
-      await uploadToBucket(
-        speechMarksFileName,
-        Buffer.from(JSON.stringify(speechMarks)),
-        bucket
-      )
+      let speechMarksFileName: string | undefined
+      if (speechMarks.length > 0) {
+        speechMarksFileName = `speech/${id}.json`
+        await uploadToBucket(
+          speechMarksFileName,
+          Buffer.from(JSON.stringify(speechMarks)),
+          bucket
+        )
+      }
+
      // update speech state
      const updated = await updateSpeech(
        id,
@ -162,8 +227,11 @@ export const textToSpeechStreamingHandler = Sentry.GCPFunction.wrapHttpFunction(
    if (!token) {
      return res.status(401).send({ errorCode: 'INVALID_TOKEN' })
    }
+
+    let claim: Claim
    try {
      jwt.verify(token, process.env.JWT_SECRET)
+      claim = jwt.decode(token) as Claim
    } catch (e) {
      console.error('Authentication error:', e)
      return res.status(401).send({ errorCode: 'UNAUTHENTICATED' })
@ -177,6 +245,26 @@ export const textToSpeechStreamingHandler = Sentry.GCPFunction.wrapHttpFunction(

    try {
      const utteranceInput = req.body as UtteranceInput
+      if (!utteranceInput.text) {
+        return res.status(400).send('INVALID_INPUT')
+      }
+
+      // validate if user has opted in to use ultra realistic voice feature
+      if (
+        utteranceInput.isUltraRealisticVoice &&
+        (claim.featureName !== 'ultra-realistic-voice' || !claim.grantedAt)
+      ) {
+        return res.status(403).send('UNAUTHORIZED')
+      }
+
+      // validate character count
+      const characterCount =
+        (await getCharacterCountFromRedis(redisClient, claim.uid)) +
+        utteranceInput.text.length
+      if (characterCount > MAX_CHARACTER_COUNT) {
+        return res.status(429).send('RATE_LIMITED')
+      }
+
      const ssmlOptions = {
        primaryVoice: utteranceInput.voice,
        secondaryVoice: utteranceInput.voice,
@ -201,15 +289,50 @@ export const textToSpeechStreamingHandler = Sentry.GCPFunction.wrapHttpFunction(
        return
      }
      console.log('Cache miss')
-      // synthesize text to speech if cache miss
+
+      const bucket = process.env.GCS_UPLOAD_BUCKET
+      if (!bucket) {
+        throw new Error('GCS_UPLOAD_BUCKET not set')
+      }
+
+      // audio file to be saved in GCS
+      const audioFileName = `speech/${cacheKey}.mp3`
+      const speechMarksFileName = `speech/${cacheKey}.json`
+      const audioFile = createGCSFile(bucket, audioFileName)
+      const speechMarksFile = createGCSFile(bucket, speechMarksFileName)
+      // check if audio file already exists
+      const [exists] = await audioFile.exists()
+      if (exists) {
+        console.debug('Audio file already exists')
+        const [audioData] = await audioFile.download()
+        const [speechMarksExists] = await speechMarksFile.exists()
+
+        return {
+          audioData,
+          speechMarks: speechMarksExists
+            ? JSON.parse((await speechMarksFile.download()).toString())
+            : [],
+        }
+      }
+
      const input: TextToSpeechInput = {
        ...utteranceInput,
        textType: 'ssml',
+        key: cacheKey,
      }
+      // synthesize text to speech if cache miss
      const { audioData, speechMarks } = await synthesizeTextToSpeech(input)
      if (!audioData) {
        return res.status(500).send({ errorCode: 'SYNTHESIZER_ERROR' })
      }
+
+      // upload audio data to GCS
+      await audioFile.save(audioData)
+      // upload speech marks to GCS
+      if (speechMarks.length > 0) {
+        await speechMarksFile.save(JSON.stringify(speechMarks))
+      }
+
      const audioDataString = audioData.toString('hex')
      // save audio data to cache for 24 hours for mainly the newsletters
      await redisClient.set(
@ -222,6 +345,9 @@ export const textToSpeechStreamingHandler = Sentry.GCPFunction.wrapHttpFunction(
      )
      console.log('Cache saved')

+      // update character count
+      await updateCharacterCountInRedis(redisClient, claim.uid, characterCount)
+
      res.send({
        idx: utteranceInput.idx,
        audioData: audioDataString,
--- a/packages/text-to-speech/src/realisticTextToSpeech.ts
+++ b/packages/text-to-speech/src/realisticTextToSpeech.ts
@ -0,0 +1,133 @@
+import {
+  TextToSpeech,
+  TextToSpeechInput,
+  TextToSpeechOutput,
+} from './textToSpeech'
+import axios from 'axios'
+import ffmpegPath from '@ffmpeg-installer/ffmpeg'
+import ffmpeg from 'fluent-ffmpeg'
+import { PassThrough } from 'stream'
+
+ffmpeg.setFfmpegPath(ffmpegPath.path)
+
+interface PlayHtConvertResponse {
+  message: string
+  payload: string[]
+}
+
+const convertWavToMp3AndUpload = async (
+  inputStream: PassThrough,
+  outputStream: PassThrough
+) => {
+  return new Promise<void>((resolve, reject) => {
+    ffmpeg(inputStream)
+      .audioCodec('libmp3lame')
+      .format('mp3')
+      .on('error', (err) => {
+        reject(err)
+      })
+      .on('end', () => {
+        console.debug('Finished processing')
+        resolve()
+      })
+      .pipe(outputStream, { end: true })
+  })
+}
+
+export class RealisticTextToSpeech implements TextToSpeech {
+  synthesizeTextToSpeech = async (
+    input: TextToSpeechInput
+  ): Promise<TextToSpeechOutput> => {
+    const apiEndpoint = process.env.REALISTIC_VOICE_API_ENDPOINT
+    const apiKey = process.env.REALISTIC_VOICE_API_KEY
+    const userId = process.env.REALISTIC_VOICE_USER_ID
+    if (!apiEndpoint || !apiKey || !userId) {
+      throw new Error('PlayHT API credentials not set')
+    }
+
+    const inputStream = new PassThrough()
+
+    const HEADERS = {
+      Authorization: apiKey,
+      'X-User-ID': userId,
+      'Content-Type': 'application/json',
+    }
+
+    const data = {
+      voice: input.voice,
+      content: [input.text],
+    }
+
+    // get the download url first
+    const response = await axios.post<PlayHtConvertResponse>(
+      apiEndpoint,
+      data,
+      {
+        headers: HEADERS,
+      }
+    )
+
+    if (response.data.payload.length === 0) {
+      throw new Error('No payload returned')
+    }
+
+    const downloadUrl = response.data.payload[0]
+
+    // polling the download url until the file is ready
+    // timeout after 1 hour
+    const timeout = 60 * 60 * 1000
+    const startTime = Date.now()
+    let isReady = false
+    while (!isReady) {
+      if (Date.now() - startTime > timeout) {
+        throw new Error('Timeout when polling the download url')
+      }
+
+      // download the audio file
+      try {
+        const downloadResponse = await axios.get(downloadUrl, {
+          responseType: 'arraybuffer',
+          headers: {
+            'Content-Type': 'audio/wav',
+          },
+        })
+
+        // write the audio file to the input stream
+        // eslint-disable-next-line @typescript-eslint/no-unsafe-argument
+        inputStream.end(Buffer.from(downloadResponse.data, 'binary'))
+        isReady = true
+      } catch (e) {
+        // ignore error
+        console.debug('checking status of audio file', downloadUrl)
+      }
+    }
+
+    const outputStream = new PassThrough()
+    // transcode the audio file to mp3
+    await convertWavToMp3AndUpload(inputStream, outputStream)
+
+    // convert the buffer stream to a buffer
+    const audioData = await new Promise<Buffer>((resolve, reject) => {
+      const chunks: Buffer[] = []
+      outputStream.on('data', (chunk) => {
+        // eslint-disable-next-line @typescript-eslint/no-unsafe-argument
+        chunks.push(chunk)
+      })
+      outputStream.on('end', () => {
+        resolve(Buffer.concat(chunks))
+      })
+      outputStream.on('error', (err) => {
+        reject(err)
+      })
+    })
+
+    return {
+      audioData,
+      speechMarks: [],
+    }
+  }
+
+  use(input: TextToSpeechInput): boolean {
+    return !!input.isUltraRealisticVoice
+  }
+}
--- a/packages/text-to-speech/src/textToSpeech.ts
+++ b/packages/text-to-speech/src/textToSpeech.ts
@ -1,23 +1,13 @@
-import {
-  CancellationDetails,
-  CancellationReason,
-  ResultReason,
-  SpeechConfig,
-  SpeechSynthesisOutputFormat,
-  SpeechSynthesisResult,
-  SpeechSynthesizer,
-} from 'microsoft-cognitiveservices-speech-sdk'
-import { endSsml, htmlToSsmlItems, ssmlItemText, startSsml } from './htmlToSsml'
-import * as _ from 'underscore'
-
 export interface TextToSpeechInput {
  text: string
+  key: string
  voice?: string
  language?: string
  textType?: 'html' | 'ssml'
  rate?: string
  secondaryVoice?: string
  audioStream?: NodeJS.ReadWriteStream
+  isUltraRealisticVoice?: boolean
 }

 export interface TextToSpeechOutput {
@ -32,132 +22,10 @@ export interface SpeechMark {
  word: string
  type: 'word' | 'bookmark'
 }
+export abstract class TextToSpeech {
+  abstract use(input: TextToSpeechInput): boolean

-export const synthesizeTextToSpeech = async (
-  input: TextToSpeechInput
-): Promise<TextToSpeechOutput> => {
-  if (!process.env.AZURE_SPEECH_KEY || !process.env.AZURE_SPEECH_REGION) {
-    throw new Error('Azure Speech Key or Region not set')
-  }
-  const textType = input.textType || 'html'
-  const audioStream = input.audioStream
-  const speechConfig = SpeechConfig.fromSubscription(
-    process.env.AZURE_SPEECH_KEY,
-    process.env.AZURE_SPEECH_REGION
-  )
-  speechConfig.speechSynthesisOutputFormat =
-    SpeechSynthesisOutputFormat.Audio16Khz32KBitRateMonoMp3
-
-  // Create the speech synthesizer.
-  const synthesizer = new SpeechSynthesizer(speechConfig)
-  const speechMarks: SpeechMark[] = []
-  let timeOffset = 0
-  let wordOffset = 0
-
-  synthesizer.synthesizing = function (s, e) {
-    // convert arrayBuffer to stream and write to stream
-    audioStream?.write(Buffer.from(e.result.audioData))
-  }
-
-  // The event synthesis completed signals that the synthesis is completed.
-  synthesizer.synthesisCompleted = (s, e) => {
-    console.info(
-      `(synthesized) Reason: ${ResultReason[e.result.reason]} Audio length: ${
-        e.result.audioData.byteLength
-      }`
-    )
-  }
-
-  // The synthesis started event signals that the synthesis is started.
-  synthesizer.synthesisStarted = (s, e) => {
-    console.info('(synthesis started)')
-  }
-
-  // The event signals that the service has stopped processing speech.
-  // This can happen when an error is encountered.
-  synthesizer.SynthesisCanceled = (s, e) => {
-    const cancellationDetails = CancellationDetails.fromResult(e.result)
-    let str =
-      '(cancel) Reason: ' + CancellationReason[cancellationDetails.reason]
-    if (cancellationDetails.reason === CancellationReason.Error) {
-      str += ': ' + e.result.errorDetails
-    }
-    console.log(str)
-  }
-
-  // The unit of e.audioOffset is tick (1 tick = 100 nanoseconds), divide by 10,000 to convert to milliseconds.
-  synthesizer.wordBoundary = (s, e) => {
-    speechMarks.push({
-      word: e.text,
-      time: (timeOffset + e.audioOffset) / 10000,
-      start: wordOffset + e.textOffset,
-      length: e.wordLength,
-      type: 'word',
-    })
-  }
-
-  synthesizer.bookmarkReached = (s, e) => {
-    speechMarks.push({
-      word: e.text,
-      time: (timeOffset + e.audioOffset) / 10000,
-      type: 'bookmark',
-    })
-  }
-
-  const speakSsmlAsyncPromise = (
-    ssml: string
-  ): Promise<SpeechSynthesisResult> => {
-    return new Promise((resolve, reject) => {
-      synthesizer.speakSsmlAsync(
-        ssml,
-        (result) => {
-          resolve(result)
-        },
-        (error) => {
-          reject(error)
-        }
-      )
-    })
-  }
-
-  try {
-    const ssmlOptions = {
-      primaryVoice: input.voice,
-      secondaryVoice: input.secondaryVoice,
-      language: input.language,
-      rate: input.rate,
-    }
-    if (textType === 'html') {
-      const ssmlItems = htmlToSsmlItems(input.text, ssmlOptions)
-      for (const ssmlItem of ssmlItems) {
-        const ssml = ssmlItemText(ssmlItem)
-        const result = await speakSsmlAsyncPromise(ssml)
-        timeOffset = timeOffset + result.audioDuration
-      }
-      return {
-        speechMarks,
-      }
-    }
-    // for ssml
-    const startSsmlTag = startSsml(ssmlOptions)
-    wordOffset -= startSsmlTag.length
-    const text = _.escape(input.text)
-    const ssml = `${startSsmlTag}${text}${endSsml()}`
-    const result = await speakSsmlAsyncPromise(ssml)
-    if (result.reason === ResultReason.Canceled) {
-      throw new Error(result.errorDetails)
-    }
-
-    return {
-      audioData: Buffer.from(result.audioData),
-      speechMarks,
-    }
-  } catch (error) {
-    console.error('synthesis error:', error)
-    throw error
-  } finally {
-    audioStream?.end()
-    synthesizer.close()
-    console.log('synthesizer closed')
-  }
+  abstract synthesizeTextToSpeech(
+    input: TextToSpeechInput
+  ): Promise<TextToSpeechOutput>
 }
--- a/yarn.lock
+++ b/yarn.lock
@ -2286,6 +2286,60 @@
    lodash.isundefined "^3.0.1"
    lodash.uniq "^4.5.0"

+"@ffmpeg-installer/darwin-arm64@4.1.5":
+  version "4.1.5"
+  resolved "https://registry.yarnpkg.com/@ffmpeg-installer/darwin-arm64/-/darwin-arm64-4.1.5.tgz#b7b5c262dd96d1aea4807514e1cdcf6e11f82743"
+  integrity sha512-hYqTiP63mXz7wSQfuqfFwfLOfwwFChUedeCVKkBtl/cliaTM7/ePI9bVzfZ2c+dWu3TqCwLDRWNSJ5pqZl8otA==
+
+"@ffmpeg-installer/darwin-x64@4.1.0":
+  version "4.1.0"
+  resolved "https://registry.yarnpkg.com/@ffmpeg-installer/darwin-x64/-/darwin-x64-4.1.0.tgz#48e1706c690e628148482bfb64acb67472089aaa"
+  integrity sha512-Z4EyG3cIFjdhlY8wI9aLUXuH8nVt7E9SlMVZtWvSPnm2sm37/yC2CwjUzyCQbJbySnef1tQwGG2Sx+uWhd9IAw==
+
+"@ffmpeg-installer/ffmpeg@^1.1.0":
+  version "1.1.0"
+  resolved "https://registry.yarnpkg.com/@ffmpeg-installer/ffmpeg/-/ffmpeg-1.1.0.tgz#87fdb9e7d180e8d78f7903f9441e36f978938a90"
+  integrity sha512-Uq4rmwkdGxIa9A6Bd/VqqYbT7zqh1GrT5/rFwCwKM70b42W5gIjWeVETq6SdcL0zXqDtY081Ws/iJWhr1+xvQg==
+  optionalDependencies:
+    "@ffmpeg-installer/darwin-arm64" "4.1.5"
+    "@ffmpeg-installer/darwin-x64" "4.1.0"
+    "@ffmpeg-installer/linux-arm" "4.1.3"
+    "@ffmpeg-installer/linux-arm64" "4.1.4"
+    "@ffmpeg-installer/linux-ia32" "4.1.0"
+    "@ffmpeg-installer/linux-x64" "4.1.0"
+    "@ffmpeg-installer/win32-ia32" "4.1.0"
+    "@ffmpeg-installer/win32-x64" "4.1.0"
+
+"@ffmpeg-installer/linux-arm64@4.1.4":
+  version "4.1.4"
+  resolved "https://registry.yarnpkg.com/@ffmpeg-installer/linux-arm64/-/linux-arm64-4.1.4.tgz#7219f3f901bb67f7926cb060b56b6974a6cad29f"
+  integrity sha512-dljEqAOD0oIM6O6DxBW9US/FkvqvQwgJ2lGHOwHDDwu/pX8+V0YsDL1xqHbj1DMX/+nP9rxw7G7gcUvGspSoKg==
+
+"@ffmpeg-installer/linux-arm@4.1.3":
+  version "4.1.3"
+  resolved "https://registry.yarnpkg.com/@ffmpeg-installer/linux-arm/-/linux-arm-4.1.3.tgz#c554f105ed5f10475ec25d7bec94926ce18db4c1"
+  integrity sha512-NDf5V6l8AfzZ8WzUGZ5mV8O/xMzRag2ETR6+TlGIsMHp81agx51cqpPItXPib/nAZYmo55Bl2L6/WOMI3A5YRg==
+
+"@ffmpeg-installer/linux-ia32@4.1.0":
+  version "4.1.0"
+  resolved "https://registry.yarnpkg.com/@ffmpeg-installer/linux-ia32/-/linux-ia32-4.1.0.tgz#adad70b0d0d9d8d813983d6e683c5a338a75e442"
+  integrity sha512-0LWyFQnPf+Ij9GQGD034hS6A90URNu9HCtQ5cTqo5MxOEc7Rd8gLXrJvn++UmxhU0J5RyRE9KRYstdCVUjkNOQ==
+
+"@ffmpeg-installer/linux-x64@4.1.0":
+  version "4.1.0"
+  resolved "https://registry.yarnpkg.com/@ffmpeg-installer/linux-x64/-/linux-x64-4.1.0.tgz#b4a5d89c4e12e6d9306dbcdc573df716ec1c4323"
+  integrity sha512-Y5BWhGLU/WpQjOArNIgXD3z5mxxdV8c41C+U15nsE5yF8tVcdCGet5zPs5Zy3Ta6bU7haGpIzryutqCGQA/W8A==
+
+"@ffmpeg-installer/win32-ia32@4.1.0":
+  version "4.1.0"
+  resolved "https://registry.yarnpkg.com/@ffmpeg-installer/win32-ia32/-/win32-ia32-4.1.0.tgz#6eac4fb691b64c02e7a116c1e2d167f3e9b40638"
+  integrity sha512-FV2D7RlaZv/lrtdhaQ4oETwoFUsUjlUiasiZLDxhEUPdNDWcH1OU9K1xTvqz+OXLdsmYelUDuBS/zkMOTtlUAw==
+
+"@ffmpeg-installer/win32-x64@4.1.0":
+  version "4.1.0"
+  resolved "https://registry.yarnpkg.com/@ffmpeg-installer/win32-x64/-/win32-x64-4.1.0.tgz#17e8699b5798d4c60e36e2d6326a8ebe5e95a2c5"
+  integrity sha512-Drt5u2vzDnIONf4ZEkKtFlbvwj6rI3kxw1Ck9fpudmtgaZIHD4ucsWB2lCZBXRxJgXR+2IMSti+4rtM4C4rXgg==
+
 "@firebase/app-types@0.7.0":
  version "0.7.0"
  resolved "https://registry.yarnpkg.com/@firebase/app-types/-/app-types-0.7.0.tgz#c9e16d1b8bed1a991840b8d2a725fb58d0b5899f"
@ -7702,6 +7756,13 @@
  resolved "https://registry.yarnpkg.com/@types/fined/-/fined-1.1.3.tgz#83f03e8f0a8d3673dfcafb18fce3571f6250e1bc"
  integrity sha512-CWYnSRnun3CGbt6taXeVo2lCbuaj4mchVJ4UF/BdU5TSuIn3AmS13pGMwCsBUoehGbhZrBrpNJZSZI5EVilXww==

+"@types/fluent-ffmpeg@^2.1.20":
+  version "2.1.20"
+  resolved "https://registry.yarnpkg.com/@types/fluent-ffmpeg/-/fluent-ffmpeg-2.1.20.tgz#3b5f42fc8263761d58284fa46ee6759a64ce54ac"
+  integrity sha512-B+OvhCdJ3LgEq2PhvWNOiB/EfwnXLElfMCgc4Z1K5zXgSfo9I6uGKwR/lqmNPFQuebNnes7re3gqkV77SyypLg==
+  dependencies:
+    "@types/node" "*"
+
 "@types/glob@*":
  version "7.2.0"
  resolved "https://registry.yarnpkg.com/@types/glob/-/glob-7.2.0.tgz#bc1b5bf3aa92f25bd5dd39f35c57361bdce5b2eb"
@ -9590,6 +9651,11 @@ async-retry@^1.2.1, async-retry@^1.3.3:
  dependencies:
    retry "0.13.1"

+async@>=0.2.9:
+  version "3.2.4"
+  resolved "https://registry.yarnpkg.com/async/-/async-3.2.4.tgz#2d22e00f8cddeb5fde5dd33522b56d1cf569a81c"
+  integrity sha512-iAB+JbDEGXhyIUavoDl9WP/Jj106Kz9DEn1DPgYw5ruDn0e3Wgi3sKFm55sASdGBNOQB8F59d9qQ7deqrHA8wQ==
+
 async@^2.6.2:
  version "2.6.4"
  resolved "https://registry.yarnpkg.com/async/-/async-2.6.4.tgz#706b7ff6084664cd7eae713f6f965433b5504221"
@ -14018,6 +14084,14 @@ flatted@^3.1.0:
  resolved "https://registry.yarnpkg.com/flatted/-/flatted-3.2.2.tgz#64bfed5cb68fe3ca78b3eb214ad97b63bedce561"
  integrity sha512-JaTY/wtrcSyvXJl4IMFHPKyFur1sE9AUqc0QnhOaJ0CxHtAoIV8pYDzeEfAaNEtGkOfq4gr3LBFmdXW5mOQFnA==

+fluent-ffmpeg@^2.1.2:
+  version "2.1.2"
+  resolved "https://registry.yarnpkg.com/fluent-ffmpeg/-/fluent-ffmpeg-2.1.2.tgz#c952de2240f812ebda0aa8006d7776ee2acf7d74"
+  integrity sha512-IZTB4kq5GK0DPp7sGQ0q/BWurGHffRtQQwVkiqDgeO6wYJLLV5ZhgNOQ65loZxxuPMKZKZcICCUnaGtlxBiR0Q==
+  dependencies:
+    async ">=0.2.9"
+    which "^1.1.1"
+
 flush-write-stream@^1.0.0:
  version "1.1.1"
  resolved "https://registry.yarnpkg.com/flush-write-stream/-/flush-write-stream-1.1.1.tgz#8dd7d873a1babc207d94ead0c2e0e44276ebf2e8"
@ -25709,7 +25783,7 @@ which@2.0.2, which@^2.0.1, which@^2.0.2:
  dependencies:
    isexe "^2.0.0"

-which@^1.2.14, which@^1.2.9, which@^1.3.1:
+which@^1.1.1, which@^1.2.14, which@^1.2.9, which@^1.3.1:
  version "1.3.1"
  resolved "https://registry.yarnpkg.com/which/-/which-1.3.1.tgz#a45043d54f5805316da8d62f9f50918d3da70b0a"
  integrity sha512-HxJdYWq1MTIQbJ3nw0cqssHoTNU267KlrDuGZ1WYlxDStUtKUhOaJmh112/TZmHxxUfuJqPXSOm7tDyas0OSIQ==