diff --git a/apple/OmnivoreKit/Sources/Services/DataService/GQLSchema.swift b/apple/OmnivoreKit/Sources/Services/DataService/GQLSchema.swift index 1876683b6..f82d4a6fe 100644 --- a/apple/OmnivoreKit/Sources/Services/DataService/GQLSchema.swift +++ b/apple/OmnivoreKit/Sources/Services/DataService/GQLSchema.swift @@ -4673,6 +4673,209 @@ extension Selection where TypeLock == Never, Type == Never { typealias DeviceToken = Selection } +extension Objects { + struct Feature { + let __typename: TypeName = .feature + let createdAt: [String: DateTime] + let expiresAt: [String: DateTime] + let grantedAt: [String: DateTime] + let id: [String: String] + let name: [String: String] + let token: [String: String] + let updatedAt: [String: DateTime] + + enum TypeName: String, Codable { + case feature = "Feature" + } + } +} + +extension Objects.Feature: Decodable { + init(from decoder: Decoder) throws { + let container = try decoder.container(keyedBy: DynamicCodingKeys.self) + + var map = HashMap() + for codingKey in container.allKeys { + if codingKey.isTypenameKey { continue } + + let alias = codingKey.stringValue + let field = GraphQLField.getFieldNameFromAlias(alias) + + switch field { + case "createdAt": + if let value = try container.decode(DateTime?.self, forKey: codingKey) { + map.set(key: field, hash: alias, value: value as Any) + } + case "expiresAt": + if let value = try container.decode(DateTime?.self, forKey: codingKey) { + map.set(key: field, hash: alias, value: value as Any) + } + case "grantedAt": + if let value = try container.decode(DateTime?.self, forKey: codingKey) { + map.set(key: field, hash: alias, value: value as Any) + } + case "id": + if let value = try container.decode(String?.self, forKey: codingKey) { + map.set(key: field, hash: alias, value: value as Any) + } + case "name": + if let value = try container.decode(String?.self, forKey: codingKey) { + map.set(key: field, hash: alias, value: value as Any) + } + case "token": + if let value = try container.decode(String?.self, forKey: codingKey) { + map.set(key: field, hash: alias, value: value as Any) + } + case "updatedAt": + if let value = try container.decode(DateTime?.self, forKey: codingKey) { + map.set(key: field, hash: alias, value: value as Any) + } + default: + throw DecodingError.dataCorrupted( + DecodingError.Context( + codingPath: decoder.codingPath, + debugDescription: "Unknown key \(field)." + ) + ) + } + } + + createdAt = map["createdAt"] + expiresAt = map["expiresAt"] + grantedAt = map["grantedAt"] + id = map["id"] + name = map["name"] + token = map["token"] + updatedAt = map["updatedAt"] + } +} + +extension Fields where TypeLock == Objects.Feature { + func createdAt() throws -> DateTime { + let field = GraphQLField.leaf( + name: "createdAt", + arguments: [] + ) + select(field) + + switch response { + case let .decoding(data): + if let data = data.createdAt[field.alias!] { + return data + } + throw HttpError.badpayload + case .mocking: + return DateTime.mockValue + } + } + + func expiresAt() throws -> DateTime? { + let field = GraphQLField.leaf( + name: "expiresAt", + arguments: [] + ) + select(field) + + switch response { + case let .decoding(data): + return data.expiresAt[field.alias!] + case .mocking: + return nil + } + } + + func grantedAt() throws -> DateTime? { + let field = GraphQLField.leaf( + name: "grantedAt", + arguments: [] + ) + select(field) + + switch response { + case let .decoding(data): + return data.grantedAt[field.alias!] + case .mocking: + return nil + } + } + + func id() throws -> String { + let field = GraphQLField.leaf( + name: "id", + arguments: [] + ) + select(field) + + switch response { + case let .decoding(data): + if let data = data.id[field.alias!] { + return data + } + throw HttpError.badpayload + case .mocking: + return String.mockValue + } + } + + func name() throws -> String { + let field = GraphQLField.leaf( + name: "name", + arguments: [] + ) + select(field) + + switch response { + case let .decoding(data): + if let data = data.name[field.alias!] { + return data + } + throw HttpError.badpayload + case .mocking: + return String.mockValue + } + } + + func token() throws -> String { + let field = GraphQLField.leaf( + name: "token", + arguments: [] + ) + select(field) + + switch response { + case let .decoding(data): + if let data = data.token[field.alias!] { + return data + } + throw HttpError.badpayload + case .mocking: + return String.mockValue + } + } + + func updatedAt() throws -> DateTime { + let field = GraphQLField.leaf( + name: "updatedAt", + arguments: [] + ) + select(field) + + switch response { + case let .decoding(data): + if let data = data.updatedAt[field.alias!] { + return data + } + throw HttpError.badpayload + case .mocking: + return DateTime.mockValue + } + } +} + +extension Selection where TypeLock == Never, Type == Never { + typealias Feature = Selection +} + extension Objects { struct FeedArticle { let __typename: TypeName = .feedArticle @@ -5847,6 +6050,8 @@ extension Objects { let annotation: [String: String] let createdAt: [String: DateTime] let createdByMe: [String: Bool] + let highlightPositionAnchorIndex: [String: Int] + let highlightPositionPercent: [String: Double] let id: [String: String] let patch: [String: String] let prefix: [String: String] @@ -5889,6 +6094,14 @@ extension Objects.Highlight: Decodable { if let value = try container.decode(Bool?.self, forKey: codingKey) { map.set(key: field, hash: alias, value: value as Any) } + case "highlightPositionAnchorIndex": + if let value = try container.decode(Int?.self, forKey: codingKey) { + map.set(key: field, hash: alias, value: value as Any) + } + case "highlightPositionPercent": + if let value = try container.decode(Double?.self, forKey: codingKey) { + map.set(key: field, hash: alias, value: value as Any) + } case "id": if let value = try container.decode(String?.self, forKey: codingKey) { map.set(key: field, hash: alias, value: value as Any) @@ -5946,6 +6159,8 @@ extension Objects.Highlight: Decodable { annotation = map["annotation"] createdAt = map["createdAt"] createdByMe = map["createdByMe"] + highlightPositionAnchorIndex = map["highlightPositionAnchorIndex"] + highlightPositionPercent = map["highlightPositionPercent"] id = map["id"] patch = map["patch"] prefix = map["prefix"] @@ -6012,6 +6227,36 @@ extension Fields where TypeLock == Objects.Highlight { } } + func highlightPositionAnchorIndex() throws -> Int? { + let field = GraphQLField.leaf( + name: "highlightPositionAnchorIndex", + arguments: [] + ) + select(field) + + switch response { + case let .decoding(data): + return data.highlightPositionAnchorIndex[field.alias!] + case .mocking: + return nil + } + } + + func highlightPositionPercent() throws -> Double? { + let field = GraphQLField.leaf( + name: "highlightPositionPercent", + arguments: [] + ) + select(field) + + switch response { + case let .decoding(data): + return data.highlightPositionPercent[field.alias!] + case .mocking: + return nil + } + } + func id() throws -> String { let field = GraphQLField.leaf( name: "id", @@ -8127,6 +8372,7 @@ extension Objects { let logOut: [String: Unions.LogOutResult] let mergeHighlight: [String: Unions.MergeHighlightResult] let moveLabel: [String: Unions.MoveLabelResult] + let optInFeature: [String: Unions.OptInFeatureResult] let reportItem: [String: Objects.ReportItemResult] let revokeApiKey: [String: Unions.RevokeApiKeyResult] let saveArticleReadingProgress: [String: Unions.SaveArticleReadingProgressResult] @@ -8271,6 +8517,10 @@ extension Objects.Mutation: Decodable { if let value = try container.decode(Unions.MoveLabelResult?.self, forKey: codingKey) { map.set(key: field, hash: alias, value: value as Any) } + case "optInFeature": + if let value = try container.decode(Unions.OptInFeatureResult?.self, forKey: codingKey) { + map.set(key: field, hash: alias, value: value as Any) + } case "reportItem": if let value = try container.decode(Objects.ReportItemResult?.self, forKey: codingKey) { map.set(key: field, hash: alias, value: value as Any) @@ -8421,6 +8671,7 @@ extension Objects.Mutation: Decodable { logOut = map["logOut"] mergeHighlight = map["mergeHighlight"] moveLabel = map["moveLabel"] + optInFeature = map["optInFeature"] reportItem = map["reportItem"] revokeApiKey = map["revokeApiKey"] saveArticleReadingProgress = map["saveArticleReadingProgress"] @@ -8910,6 +9161,25 @@ extension Fields where TypeLock == Objects.Mutation { } } + func optInFeature(input: InputObjects.OptInFeatureInput, selection: Selection) throws -> Type { + let field = GraphQLField.composite( + name: "optInFeature", + arguments: [Argument(name: "input", type: "OptInFeatureInput!", value: input)], + selection: selection.selection + ) + select(field) + + switch response { + case let .decoding(data): + if let data = data.optInFeature[field.alias!] { + return try selection.decode(data: data) + } + throw HttpError.badpayload + case .mocking: + return selection.mock() + } + } + func reportItem(input: InputObjects.ReportItemInput, selection: Selection) throws -> Type { let field = GraphQLField.composite( name: "reportItem", @@ -9707,6 +9977,137 @@ extension Selection where TypeLock == Never, Type == Never { typealias NewsletterEmailsSuccess = Selection } +extension Objects { + struct OptInFeatureError { + let __typename: TypeName = .optInFeatureError + let errorCodes: [String: [Enums.OptInFeatureErrorCode]] + + enum TypeName: String, Codable { + case optInFeatureError = "OptInFeatureError" + } + } +} + +extension Objects.OptInFeatureError: Decodable { + init(from decoder: Decoder) throws { + let container = try decoder.container(keyedBy: DynamicCodingKeys.self) + + var map = HashMap() + for codingKey in container.allKeys { + if codingKey.isTypenameKey { continue } + + let alias = codingKey.stringValue + let field = GraphQLField.getFieldNameFromAlias(alias) + + switch field { + case "errorCodes": + if let value = try container.decode([Enums.OptInFeatureErrorCode]?.self, forKey: codingKey) { + map.set(key: field, hash: alias, value: value as Any) + } + default: + throw DecodingError.dataCorrupted( + DecodingError.Context( + codingPath: decoder.codingPath, + debugDescription: "Unknown key \(field)." + ) + ) + } + } + + errorCodes = map["errorCodes"] + } +} + +extension Fields where TypeLock == Objects.OptInFeatureError { + func errorCodes() throws -> [Enums.OptInFeatureErrorCode] { + let field = GraphQLField.leaf( + name: "errorCodes", + arguments: [] + ) + select(field) + + switch response { + case let .decoding(data): + if let data = data.errorCodes[field.alias!] { + return data + } + throw HttpError.badpayload + case .mocking: + return [] + } + } +} + +extension Selection where TypeLock == Never, Type == Never { + typealias OptInFeatureError = Selection +} + +extension Objects { + struct OptInFeatureSuccess { + let __typename: TypeName = .optInFeatureSuccess + let feature: [String: Objects.Feature] + + enum TypeName: String, Codable { + case optInFeatureSuccess = "OptInFeatureSuccess" + } + } +} + +extension Objects.OptInFeatureSuccess: Decodable { + init(from decoder: Decoder) throws { + let container = try decoder.container(keyedBy: DynamicCodingKeys.self) + + var map = HashMap() + for codingKey in container.allKeys { + if codingKey.isTypenameKey { continue } + + let alias = codingKey.stringValue + let field = GraphQLField.getFieldNameFromAlias(alias) + + switch field { + case "feature": + if let value = try container.decode(Objects.Feature?.self, forKey: codingKey) { + map.set(key: field, hash: alias, value: value as Any) + } + default: + throw DecodingError.dataCorrupted( + DecodingError.Context( + codingPath: decoder.codingPath, + debugDescription: "Unknown key \(field)." + ) + ) + } + } + + feature = map["feature"] + } +} + +extension Fields where TypeLock == Objects.OptInFeatureSuccess { + func feature(selection: Selection) throws -> Type { + let field = GraphQLField.composite( + name: "feature", + arguments: [], + selection: selection.selection + ) + select(field) + + switch response { + case let .decoding(data): + if let data = data.feature[field.alias!] { + return try selection.decode(data: data) + } + throw HttpError.badpayload + case .mocking: + return selection.mock() + } + } +} + +extension Selection where TypeLock == Never, Type == Never { + typealias OptInFeatureSuccess = Selection +} + extension Objects { struct Page { let __typename: TypeName = .page @@ -10398,6 +10799,7 @@ extension Objects { let labels: [String: Unions.LabelsResult] let me: [String: Objects.User] let newsletterEmails: [String: Unions.NewsletterEmailsResult] + let recentSearches: [String: Unions.RecentSearchesResult] let reminder: [String: Unions.ReminderResult] let search: [String: Unions.SearchResult] let sendInstallInstructions: [String: Unions.SendInstallInstructionsResult] @@ -10481,6 +10883,10 @@ extension Objects.Query: Decodable { if let value = try container.decode(Unions.NewsletterEmailsResult?.self, forKey: codingKey) { map.set(key: field, hash: alias, value: value as Any) } + case "recentSearches": + if let value = try container.decode(Unions.RecentSearchesResult?.self, forKey: codingKey) { + map.set(key: field, hash: alias, value: value as Any) + } case "reminder": if let value = try container.decode(Unions.ReminderResult?.self, forKey: codingKey) { map.set(key: field, hash: alias, value: value as Any) @@ -10552,6 +10958,7 @@ extension Objects.Query: Decodable { labels = map["labels"] me = map["me"] newsletterEmails = map["newsletterEmails"] + recentSearches = map["recentSearches"] reminder = map["reminder"] search = map["search"] sendInstallInstructions = map["sendInstallInstructions"] @@ -10808,6 +11215,25 @@ extension Fields where TypeLock == Objects.Query { } } + func recentSearches(selection: Selection) throws -> Type { + let field = GraphQLField.composite( + name: "recentSearches", + arguments: [], + selection: selection.selection + ) + select(field) + + switch response { + case let .decoding(data): + if let data = data.recentSearches[field.alias!] { + return try selection.decode(data: data) + } + throw HttpError.badpayload + case .mocking: + return selection.mock() + } + } + func reminder(linkId: String, selection: Selection) throws -> Type { let field = GraphQLField.composite( name: "reminder", @@ -11330,6 +11756,250 @@ extension Selection where TypeLock == Never, Type == Never { typealias ReadState = Selection } +extension Objects { + struct RecentSearch { + let __typename: TypeName = .recentSearch + let createdAt: [String: DateTime] + let id: [String: String] + let term: [String: String] + + enum TypeName: String, Codable { + case recentSearch = "RecentSearch" + } + } +} + +extension Objects.RecentSearch: Decodable { + init(from decoder: Decoder) throws { + let container = try decoder.container(keyedBy: DynamicCodingKeys.self) + + var map = HashMap() + for codingKey in container.allKeys { + if codingKey.isTypenameKey { continue } + + let alias = codingKey.stringValue + let field = GraphQLField.getFieldNameFromAlias(alias) + + switch field { + case "createdAt": + if let value = try container.decode(DateTime?.self, forKey: codingKey) { + map.set(key: field, hash: alias, value: value as Any) + } + case "id": + if let value = try container.decode(String?.self, forKey: codingKey) { + map.set(key: field, hash: alias, value: value as Any) + } + case "term": + if let value = try container.decode(String?.self, forKey: codingKey) { + map.set(key: field, hash: alias, value: value as Any) + } + default: + throw DecodingError.dataCorrupted( + DecodingError.Context( + codingPath: decoder.codingPath, + debugDescription: "Unknown key \(field)." + ) + ) + } + } + + createdAt = map["createdAt"] + id = map["id"] + term = map["term"] + } +} + +extension Fields where TypeLock == Objects.RecentSearch { + func createdAt() throws -> DateTime { + let field = GraphQLField.leaf( + name: "createdAt", + arguments: [] + ) + select(field) + + switch response { + case let .decoding(data): + if let data = data.createdAt[field.alias!] { + return data + } + throw HttpError.badpayload + case .mocking: + return DateTime.mockValue + } + } + + func id() throws -> String { + let field = GraphQLField.leaf( + name: "id", + arguments: [] + ) + select(field) + + switch response { + case let .decoding(data): + if let data = data.id[field.alias!] { + return data + } + throw HttpError.badpayload + case .mocking: + return String.mockValue + } + } + + func term() throws -> String { + let field = GraphQLField.leaf( + name: "term", + arguments: [] + ) + select(field) + + switch response { + case let .decoding(data): + if let data = data.term[field.alias!] { + return data + } + throw HttpError.badpayload + case .mocking: + return String.mockValue + } + } +} + +extension Selection where TypeLock == Never, Type == Never { + typealias RecentSearch = Selection +} + +extension Objects { + struct RecentSearchesError { + let __typename: TypeName = .recentSearchesError + let errorCodes: [String: [Enums.RecentSearchesErrorCode]] + + enum TypeName: String, Codable { + case recentSearchesError = "RecentSearchesError" + } + } +} + +extension Objects.RecentSearchesError: Decodable { + init(from decoder: Decoder) throws { + let container = try decoder.container(keyedBy: DynamicCodingKeys.self) + + var map = HashMap() + for codingKey in container.allKeys { + if codingKey.isTypenameKey { continue } + + let alias = codingKey.stringValue + let field = GraphQLField.getFieldNameFromAlias(alias) + + switch field { + case "errorCodes": + if let value = try container.decode([Enums.RecentSearchesErrorCode]?.self, forKey: codingKey) { + map.set(key: field, hash: alias, value: value as Any) + } + default: + throw DecodingError.dataCorrupted( + DecodingError.Context( + codingPath: decoder.codingPath, + debugDescription: "Unknown key \(field)." + ) + ) + } + } + + errorCodes = map["errorCodes"] + } +} + +extension Fields where TypeLock == Objects.RecentSearchesError { + func errorCodes() throws -> [Enums.RecentSearchesErrorCode] { + let field = GraphQLField.leaf( + name: "errorCodes", + arguments: [] + ) + select(field) + + switch response { + case let .decoding(data): + if let data = data.errorCodes[field.alias!] { + return data + } + throw HttpError.badpayload + case .mocking: + return [] + } + } +} + +extension Selection where TypeLock == Never, Type == Never { + typealias RecentSearchesError = Selection +} + +extension Objects { + struct RecentSearchesSuccess { + let __typename: TypeName = .recentSearchesSuccess + let searches: [String: [Objects.RecentSearch]] + + enum TypeName: String, Codable { + case recentSearchesSuccess = "RecentSearchesSuccess" + } + } +} + +extension Objects.RecentSearchesSuccess: Decodable { + init(from decoder: Decoder) throws { + let container = try decoder.container(keyedBy: DynamicCodingKeys.self) + + var map = HashMap() + for codingKey in container.allKeys { + if codingKey.isTypenameKey { continue } + + let alias = codingKey.stringValue + let field = GraphQLField.getFieldNameFromAlias(alias) + + switch field { + case "searches": + if let value = try container.decode([Objects.RecentSearch]?.self, forKey: codingKey) { + map.set(key: field, hash: alias, value: value as Any) + } + default: + throw DecodingError.dataCorrupted( + DecodingError.Context( + codingPath: decoder.codingPath, + debugDescription: "Unknown key \(field)." + ) + ) + } + } + + searches = map["searches"] + } +} + +extension Fields where TypeLock == Objects.RecentSearchesSuccess { + func searches(selection: Selection) throws -> Type { + let field = GraphQLField.composite( + name: "searches", + arguments: [], + selection: selection.selection + ) + select(field) + + switch response { + case let .decoding(data): + if let data = data.searches[field.alias!] { + return try selection.decode(data: data) + } + throw HttpError.badpayload + case .mocking: + return selection.mock() + } + } +} + +extension Selection where TypeLock == Never, Type == Never { + typealias RecentSearchesSuccess = Selection +} + extension Objects { struct Reminder { let __typename: TypeName = .reminder @@ -21325,6 +21995,154 @@ extension Selection where TypeLock == Never, Type == Never { typealias NewsletterEmailsResult = Selection } +extension Unions { + struct OptInFeatureResult { + let __typename: TypeName + let errorCodes: [String: [Enums.OptInFeatureErrorCode]] + let feature: [String: Objects.Feature] + + enum TypeName: String, Codable { + case optInFeatureError = "OptInFeatureError" + case optInFeatureSuccess = "OptInFeatureSuccess" + } + } +} + +extension Unions.OptInFeatureResult: Decodable { + init(from decoder: Decoder) throws { + let container = try decoder.container(keyedBy: DynamicCodingKeys.self) + + var map = HashMap() + for codingKey in container.allKeys { + if codingKey.isTypenameKey { continue } + + let alias = codingKey.stringValue + let field = GraphQLField.getFieldNameFromAlias(alias) + + switch field { + case "errorCodes": + if let value = try container.decode([Enums.OptInFeatureErrorCode]?.self, forKey: codingKey) { + map.set(key: field, hash: alias, value: value as Any) + } + case "feature": + if let value = try container.decode(Objects.Feature?.self, forKey: codingKey) { + map.set(key: field, hash: alias, value: value as Any) + } + default: + throw DecodingError.dataCorrupted( + DecodingError.Context( + codingPath: decoder.codingPath, + debugDescription: "Unknown key \(field)." + ) + ) + } + } + + __typename = try container.decode(TypeName.self, forKey: DynamicCodingKeys(stringValue: "__typename")!) + + errorCodes = map["errorCodes"] + feature = map["feature"] + } +} + +extension Fields where TypeLock == Unions.OptInFeatureResult { + func on(optInFeatureError: Selection, optInFeatureSuccess: Selection) throws -> Type { + select([GraphQLField.fragment(type: "OptInFeatureError", selection: optInFeatureError.selection), GraphQLField.fragment(type: "OptInFeatureSuccess", selection: optInFeatureSuccess.selection)]) + + switch response { + case let .decoding(data): + switch data.__typename { + case .optInFeatureError: + let data = Objects.OptInFeatureError(errorCodes: data.errorCodes) + return try optInFeatureError.decode(data: data) + case .optInFeatureSuccess: + let data = Objects.OptInFeatureSuccess(feature: data.feature) + return try optInFeatureSuccess.decode(data: data) + } + case .mocking: + return optInFeatureError.mock() + } + } +} + +extension Selection where TypeLock == Never, Type == Never { + typealias OptInFeatureResult = Selection +} + +extension Unions { + struct RecentSearchesResult { + let __typename: TypeName + let errorCodes: [String: [Enums.RecentSearchesErrorCode]] + let searches: [String: [Objects.RecentSearch]] + + enum TypeName: String, Codable { + case recentSearchesError = "RecentSearchesError" + case recentSearchesSuccess = "RecentSearchesSuccess" + } + } +} + +extension Unions.RecentSearchesResult: Decodable { + init(from decoder: Decoder) throws { + let container = try decoder.container(keyedBy: DynamicCodingKeys.self) + + var map = HashMap() + for codingKey in container.allKeys { + if codingKey.isTypenameKey { continue } + + let alias = codingKey.stringValue + let field = GraphQLField.getFieldNameFromAlias(alias) + + switch field { + case "errorCodes": + if let value = try container.decode([Enums.RecentSearchesErrorCode]?.self, forKey: codingKey) { + map.set(key: field, hash: alias, value: value as Any) + } + case "searches": + if let value = try container.decode([Objects.RecentSearch]?.self, forKey: codingKey) { + map.set(key: field, hash: alias, value: value as Any) + } + default: + throw DecodingError.dataCorrupted( + DecodingError.Context( + codingPath: decoder.codingPath, + debugDescription: "Unknown key \(field)." + ) + ) + } + } + + __typename = try container.decode(TypeName.self, forKey: DynamicCodingKeys(stringValue: "__typename")!) + + errorCodes = map["errorCodes"] + searches = map["searches"] + } +} + +extension Fields where TypeLock == Unions.RecentSearchesResult { + func on(recentSearchesError: Selection, recentSearchesSuccess: Selection) throws -> Type { + select([GraphQLField.fragment(type: "RecentSearchesError", selection: recentSearchesError.selection), GraphQLField.fragment(type: "RecentSearchesSuccess", selection: recentSearchesSuccess.selection)]) + + switch response { + case let .decoding(data): + switch data.__typename { + case .recentSearchesError: + let data = Objects.RecentSearchesError(errorCodes: data.errorCodes) + return try recentSearchesError.decode(data: data) + case .recentSearchesSuccess: + let data = Objects.RecentSearchesSuccess(searches: data.searches) + return try recentSearchesSuccess.decode(data: data) + } + case .mocking: + return recentSearchesError.mock() + } + } +} + +extension Selection where TypeLock == Never, Type == Never { + typealias RecentSearchesResult = Selection +} + extension Unions { struct ReminderResult { let __typename: TypeName @@ -24384,6 +25202,15 @@ extension Enums { } } +extension Enums { + /// OptInFeatureErrorCode + enum OptInFeatureErrorCode: String, CaseIterable, Codable { + case badRequest = "BAD_REQUEST" + + case notFound = "NOT_FOUND" + } +} + extension Enums { /// PageType enum PageType: String, CaseIterable, Codable { @@ -24420,6 +25247,15 @@ extension Enums { } } +extension Enums { + /// RecentSearchesErrorCode + enum RecentSearchesErrorCode: String, CaseIterable, Codable { + case badRequest = "BAD_REQUEST" + + case unauthorized = "UNAUTHORIZED" + } +} + extension Enums { /// ReminderErrorCode enum ReminderErrorCode: String, CaseIterable, Codable { @@ -24996,6 +25832,10 @@ extension InputObjects { var articleId: String + var highlightPositionAnchorIndex: OptionalArgument = .absent() + + var highlightPositionPercent: OptionalArgument = .absent() + var id: String var patch: String @@ -25014,6 +25854,8 @@ extension InputObjects { var container = encoder.container(keyedBy: CodingKeys.self) if annotation.hasValue { try container.encode(annotation, forKey: .annotation) } try container.encode(articleId, forKey: .articleId) + if highlightPositionAnchorIndex.hasValue { try container.encode(highlightPositionAnchorIndex, forKey: .highlightPositionAnchorIndex) } + if highlightPositionPercent.hasValue { try container.encode(highlightPositionPercent, forKey: .highlightPositionPercent) } try container.encode(id, forKey: .id) try container.encode(patch, forKey: .patch) if prefix.hasValue { try container.encode(prefix, forKey: .prefix) } @@ -25026,6 +25868,8 @@ extension InputObjects { enum CodingKeys: String, CodingKey { case annotation case articleId + case highlightPositionAnchorIndex + case highlightPositionPercent case id case patch case prefix @@ -25220,6 +26064,10 @@ extension InputObjects { var articleId: String + var highlightPositionAnchorIndex: OptionalArgument = .absent() + + var highlightPositionPercent: OptionalArgument = .absent() + var id: String var overlapHighlightIdList: [String] @@ -25238,6 +26086,8 @@ extension InputObjects { var container = encoder.container(keyedBy: CodingKeys.self) if annotation.hasValue { try container.encode(annotation, forKey: .annotation) } try container.encode(articleId, forKey: .articleId) + if highlightPositionAnchorIndex.hasValue { try container.encode(highlightPositionAnchorIndex, forKey: .highlightPositionAnchorIndex) } + if highlightPositionPercent.hasValue { try container.encode(highlightPositionPercent, forKey: .highlightPositionPercent) } try container.encode(id, forKey: .id) try container.encode(overlapHighlightIdList, forKey: .overlapHighlightIdList) try container.encode(patch, forKey: .patch) @@ -25250,6 +26100,8 @@ extension InputObjects { enum CodingKeys: String, CodingKey { case annotation case articleId + case highlightPositionAnchorIndex + case highlightPositionPercent case id case overlapHighlightIdList case patch @@ -25280,6 +26132,21 @@ extension InputObjects { } } +extension InputObjects { + struct OptInFeatureInput: Encodable, Hashable { + var name: String + + func encode(to encoder: Encoder) throws { + var container = encoder.container(keyedBy: CodingKeys.self) + try container.encode(name, forKey: .name) + } + + enum CodingKeys: String, CodingKey { + case name + } + } +} + extension InputObjects { struct PageInfoInput: Encodable, Hashable { var author: OptionalArgument = .absent() diff --git a/packages/api/src/entity/feature.ts b/packages/api/src/entity/feature.ts new file mode 100644 index 000000000..77ca572e7 --- /dev/null +++ b/packages/api/src/entity/feature.ts @@ -0,0 +1,35 @@ +import { + Column, + CreateDateColumn, + Entity, + JoinColumn, + ManyToOne, + PrimaryGeneratedColumn, + UpdateDateColumn, +} from 'typeorm' +import { User } from './user' + +@Entity({ name: 'features' }) +export class Feature { + @PrimaryGeneratedColumn('uuid') + id!: string + + @ManyToOne(() => User, { onDelete: 'CASCADE' }) + @JoinColumn({ name: 'user_id' }) + user!: User + + @Column('text') + name!: string + + @Column('timestamp', { nullable: true }) + grantedAt?: Date | null + + @Column('timestamp', { nullable: true }) + expiresAt?: Date | null + + @CreateDateColumn({ default: () => 'CURRENT_TIMESTAMP' }) + createdAt!: Date + + @UpdateDateColumn({ default: () => 'CURRENT_TIMESTAMP' }) + updatedAt!: Date +} diff --git a/packages/api/src/entity/user_personalization.ts b/packages/api/src/entity/user_personalization.ts index 8044ef1a7..5e7cc51f8 100644 --- a/packages/api/src/entity/user_personalization.ts +++ b/packages/api/src/entity/user_personalization.ts @@ -39,11 +39,14 @@ export class UserPersonalization { @Column('text', { nullable: true }) speechVoice?: string - @Column('integer', { nullable: true }) - speechRate?: number + @Column('text', { nullable: true }) + speechSecondaryVoice?: string - @Column('integer', { nullable: true }) - speechVolume?: number + @Column('text', { nullable: true }) + speechRate?: string + + @Column('text', { nullable: true }) + speechVolume?: string @CreateDateColumn({ default: () => 'CURRENT_TIMESTAMP' }) createdAt!: Date diff --git a/packages/api/src/generated/graphql.ts b/packages/api/src/generated/graphql.ts index 88a340b40..1f1815b31 100644 --- a/packages/api/src/generated/graphql.ts +++ b/packages/api/src/generated/graphql.ts @@ -596,6 +596,17 @@ export type DeviceToken = { token: Scalars['String']; }; +export type Feature = { + __typename?: 'Feature'; + createdAt: Scalars['Date']; + expiresAt?: Maybe; + grantedAt?: Maybe; + id: Scalars['ID']; + name: Scalars['String']; + token: Scalars['String']; + updatedAt: Scalars['Date']; +}; + export type FeedArticle = { __typename?: 'FeedArticle'; annotationsCount?: Maybe; @@ -971,6 +982,7 @@ export type Mutation = { logOut: LogOutResult; mergeHighlight: MergeHighlightResult; moveLabel: MoveLabelResult; + optInFeature: OptInFeatureResult; reportItem: ReportItemResult; revokeApiKey: RevokeApiKeyResult; saveArticleReadingProgress: SaveArticleReadingProgressResult; @@ -1113,6 +1125,11 @@ export type MutationMoveLabelArgs = { }; +export type MutationOptInFeatureArgs = { + input: OptInFeatureInput; +}; + + export type MutationReportItemArgs = { input: ReportItemInput; }; @@ -1281,6 +1298,27 @@ export type NewsletterEmailsSuccess = { newsletterEmails: Array; }; +export type OptInFeatureError = { + __typename?: 'OptInFeatureError'; + errorCodes: Array; +}; + +export enum OptInFeatureErrorCode { + BadRequest = 'BAD_REQUEST', + NotFound = 'NOT_FOUND' +} + +export type OptInFeatureInput = { + name: Scalars['String']; +}; + +export type OptInFeatureResult = OptInFeatureError | OptInFeatureSuccess; + +export type OptInFeatureSuccess = { + __typename?: 'OptInFeatureSuccess'; + feature: Feature; +}; + export type Page = { __typename?: 'Page'; author?: Maybe; @@ -2684,6 +2722,7 @@ export type ResolversTypes = { DeleteWebhookResult: ResolversTypes['DeleteWebhookError'] | ResolversTypes['DeleteWebhookSuccess']; DeleteWebhookSuccess: ResolverTypeWrapper; DeviceToken: ResolverTypeWrapper; + Feature: ResolverTypeWrapper; FeedArticle: ResolverTypeWrapper; FeedArticleEdge: ResolverTypeWrapper; FeedArticlesError: ResolverTypeWrapper; @@ -2755,6 +2794,11 @@ export type ResolversTypes = { NewsletterEmailsErrorCode: NewsletterEmailsErrorCode; NewsletterEmailsResult: ResolversTypes['NewsletterEmailsError'] | ResolversTypes['NewsletterEmailsSuccess']; NewsletterEmailsSuccess: ResolverTypeWrapper; + OptInFeatureError: ResolverTypeWrapper; + OptInFeatureErrorCode: OptInFeatureErrorCode; + OptInFeatureInput: OptInFeatureInput; + OptInFeatureResult: ResolversTypes['OptInFeatureError'] | ResolversTypes['OptInFeatureSuccess']; + OptInFeatureSuccess: ResolverTypeWrapper; Page: ResolverTypeWrapper; PageInfo: ResolverTypeWrapper; PageInfoInput: PageInfoInput; @@ -3045,6 +3089,7 @@ export type ResolversParentTypes = { DeleteWebhookResult: ResolversParentTypes['DeleteWebhookError'] | ResolversParentTypes['DeleteWebhookSuccess']; DeleteWebhookSuccess: DeleteWebhookSuccess; DeviceToken: DeviceToken; + Feature: Feature; FeedArticle: FeedArticle; FeedArticleEdge: FeedArticleEdge; FeedArticlesError: FeedArticlesError; @@ -3103,6 +3148,10 @@ export type ResolversParentTypes = { NewsletterEmailsError: NewsletterEmailsError; NewsletterEmailsResult: ResolversParentTypes['NewsletterEmailsError'] | ResolversParentTypes['NewsletterEmailsSuccess']; NewsletterEmailsSuccess: NewsletterEmailsSuccess; + OptInFeatureError: OptInFeatureError; + OptInFeatureInput: OptInFeatureInput; + OptInFeatureResult: ResolversParentTypes['OptInFeatureError'] | ResolversParentTypes['OptInFeatureSuccess']; + OptInFeatureSuccess: OptInFeatureSuccess; Page: Page; PageInfo: PageInfo; PageInfoInput: PageInfoInput; @@ -3677,6 +3726,17 @@ export type DeviceTokenResolvers; }; +export type FeatureResolvers = { + createdAt?: Resolver; + expiresAt?: Resolver, ParentType, ContextType>; + grantedAt?: Resolver, ParentType, ContextType>; + id?: Resolver; + name?: Resolver; + token?: Resolver; + updatedAt?: Resolver; + __isTypeOf?: IsTypeOfResolverFn; +}; + export type FeedArticleResolvers = { annotationsCount?: Resolver, ParentType, ContextType>; article?: Resolver; @@ -3971,6 +4031,7 @@ export type MutationResolvers; mergeHighlight?: Resolver>; moveLabel?: Resolver>; + optInFeature?: Resolver>; reportItem?: Resolver>; revokeApiKey?: Resolver>; saveArticleReadingProgress?: Resolver>; @@ -4023,6 +4084,20 @@ export type NewsletterEmailsSuccessResolvers; }; +export type OptInFeatureErrorResolvers = { + errorCodes?: Resolver, ParentType, ContextType>; + __isTypeOf?: IsTypeOfResolverFn; +}; + +export type OptInFeatureResultResolvers = { + __resolveType: TypeResolveFn<'OptInFeatureError' | 'OptInFeatureSuccess', ParentType, ContextType>; +}; + +export type OptInFeatureSuccessResolvers = { + feature?: Resolver; + __isTypeOf?: IsTypeOfResolverFn; +}; + export type PageResolvers = { author?: Resolver, ParentType, ContextType>; createdAt?: Resolver; @@ -4835,6 +4910,7 @@ export type Resolvers = { DeleteWebhookResult?: DeleteWebhookResultResolvers; DeleteWebhookSuccess?: DeleteWebhookSuccessResolvers; DeviceToken?: DeviceTokenResolvers; + Feature?: FeatureResolvers; FeedArticle?: FeedArticleResolvers; FeedArticleEdge?: FeedArticleEdgeResolvers; FeedArticlesError?: FeedArticlesErrorResolvers; @@ -4885,6 +4961,9 @@ export type Resolvers = { NewsletterEmailsError?: NewsletterEmailsErrorResolvers; NewsletterEmailsResult?: NewsletterEmailsResultResolvers; NewsletterEmailsSuccess?: NewsletterEmailsSuccessResolvers; + OptInFeatureError?: OptInFeatureErrorResolvers; + OptInFeatureResult?: OptInFeatureResultResolvers; + OptInFeatureSuccess?: OptInFeatureSuccessResolvers; Page?: PageResolvers; PageInfo?: PageInfoResolvers; Profile?: ProfileResolvers; diff --git a/packages/api/src/generated/schema.graphql b/packages/api/src/generated/schema.graphql index fce0f2f7a..4a4048d46 100644 --- a/packages/api/src/generated/schema.graphql +++ b/packages/api/src/generated/schema.graphql @@ -524,6 +524,16 @@ type DeviceToken { token: String! } +type Feature { + createdAt: Date! + expiresAt: Date + grantedAt: Date + id: ID! + name: String! + token: String! + updatedAt: Date! +} + type FeedArticle { annotationsCount: Int article: Article! @@ -865,6 +875,7 @@ type Mutation { logOut: LogOutResult! mergeHighlight(input: MergeHighlightInput!): MergeHighlightResult! moveLabel(input: MoveLabelInput!): MoveLabelResult! + optInFeature(input: OptInFeatureInput!): OptInFeatureResult! reportItem(input: ReportItemInput!): ReportItemResult! revokeApiKey(id: ID!): RevokeApiKeyResult! saveArticleReadingProgress(input: SaveArticleReadingProgressInput!): SaveArticleReadingProgressResult! @@ -917,6 +928,25 @@ type NewsletterEmailsSuccess { newsletterEmails: [NewsletterEmail!]! } +type OptInFeatureError { + errorCodes: [OptInFeatureErrorCode!]! +} + +enum OptInFeatureErrorCode { + BAD_REQUEST + NOT_FOUND +} + +input OptInFeatureInput { + name: String! +} + +union OptInFeatureResult = OptInFeatureError | OptInFeatureSuccess + +type OptInFeatureSuccess { + feature: Feature! +} + type Page { author: String createdAt: Date! diff --git a/packages/api/src/resolvers/features/index.ts b/packages/api/src/resolvers/features/index.ts new file mode 100644 index 000000000..cb879d71f --- /dev/null +++ b/packages/api/src/resolvers/features/index.ts @@ -0,0 +1,60 @@ +import { authorized } from '../../utils/helpers' +import { + MutationOptInFeatureArgs, + OptInFeatureError, + OptInFeatureErrorCode, + OptInFeatureSuccess, +} from '../../generated/graphql' +import { + getFeatureName, + optInFeature, + signFeatureToken, +} from '../../services/features' + +export const optInFeatureResolver = authorized< + OptInFeatureSuccess, + OptInFeatureError, + MutationOptInFeatureArgs +>(async (_, { input: { name } }, { claims, log }) => { + log.info('Opting in to a feature', { + feature: name, + labels: { + source: 'resolver', + resolver: 'optInFeatureResolver', + uid: claims.uid, + }, + }) + + try { + const featureName = getFeatureName(name) + if (!featureName) { + return { + errorCodes: [OptInFeatureErrorCode.NotFound], + } + } + + const optIn = await optInFeature(featureName, claims.uid) + if (!optIn) { + return { + errorCodes: [OptInFeatureErrorCode.NotFound], + } + } + + const token = signFeatureToken(optIn) + + return { + feature: { + ...optIn, + token, + }, + } + } catch (e) { + log.error('Error opting in to a feature', { + error: e, + }) + + return { + errorCodes: [OptInFeatureErrorCode.BadRequest], + } + } +}) diff --git a/packages/api/src/resolvers/function_resolvers.ts b/packages/api/src/resolvers/function_resolvers.ts index 50cd8580b..8bb5cfb7e 100644 --- a/packages/api/src/resolvers/function_resolvers.ts +++ b/packages/api/src/resolvers/function_resolvers.ts @@ -5,8 +5,8 @@ /* eslint-disable @typescript-eslint/explicit-module-boundary-types */ import { createReactionResolver, deleteReactionResolver } from './reaction' import { Claims, WithDataSourcesContext } from './types' -import { createImageProxyUrl } from './../utils/imageproxy' -import { userDataToUser, validatedDate } from './../utils/helpers' +import { createImageProxyUrl } from '../utils/imageproxy' +import { userDataToUser, validatedDate } from '../utils/helpers' import { Article, @@ -18,7 +18,7 @@ import { Reaction, SearchItem, User, -} from './../generated/graphql' +} from '../generated/graphql' import { addPopularReadResolver, @@ -101,6 +101,7 @@ import { } from '../utils/uploads' import { getPageByParam } from '../elastic/pages' import { recentSearchesResolver } from './recent_searches' +import { optInFeatureResolver } from './features' /* eslint-disable @typescript-eslint/naming-convention */ type ResultResolveType = { @@ -171,6 +172,7 @@ export const functionResolvers = { moveLabel: moveLabelResolver, setIntegration: setIntegrationResolver, deleteIntegration: deleteIntegrationResolver, + optInFeature: optInFeatureResolver, }, Query: { me: getMeUserResolver, @@ -607,4 +609,5 @@ export const functionResolvers = { ...resultResolveTypeResolver('Integrations'), ...resultResolveTypeResolver('DeleteIntegration'), ...resultResolveTypeResolver('RecentSearches'), + ...resultResolveTypeResolver('OptInFeature'), } diff --git a/packages/api/src/routers/text_to_speech.ts b/packages/api/src/routers/text_to_speech.ts index dab71ac4e..e0796a76d 100644 --- a/packages/api/src/routers/text_to_speech.ts +++ b/packages/api/src/routers/text_to_speech.ts @@ -13,6 +13,9 @@ import { shouldSynthesize } from '../services/speech' import { readPushSubscription } from '../datalayer/pubsub' import { AppDataSource } from '../server' import { enqueueTextToSpeech } from '../utils/createTask' +import { htmlToSpeechFile } from '@omnivore/text-to-speech-handler' +import { UserPersonalization } from '../entity/user_personalization' +import { ArticleSavingRequestStatus } from '../elastic/types' const logger = buildLogger('app.dispatch') @@ -34,17 +37,17 @@ export function textToSpeechRouter() { } try { - const data: { userId: string; type: string; id: string; state: string } = + const data: { userId: string; type: string; id: string } = JSON.parse(msgStr) - const { userId, type, id, state } = data + const { userId, type, id } = data if (!userId || !type || !id) { logger.info('Invalid data') return res.status(400).send('Bad Request') } - if (type.toUpperCase() !== 'PAGE' || state !== 'SUCCEEDED') { - logger.info('Not a page or not succeeded') - return res.status(200).send('Not a page or not succeeded') + if (type.toUpperCase() !== 'PAGE') { + logger.info('Not a page') + return res.status(200).send('Not a page') } const page = await getPageById(id) @@ -53,25 +56,45 @@ export function textToSpeechRouter() { return res.status(200).send('No page found') } + if (page.state === ArticleSavingRequestStatus.Processing) { + logger.info('Page is still processing, try again later', { id }) + return res.status(400).send('Page is still processing') + } + // checks if this page needs to be synthesized automatically if (await shouldSynthesize(userId, page)) { logger.info('page needs to be synthesized') - // initialize state - const speech = await getRepository(Speech).save({ - user: { id: userId }, - elasticPageId: id, - state: SpeechState.INITIALIZED, - voice: 'en-US-JennyNeural', + + const userPersonalization = await getRepository( + UserPersonalization + ).findOneBy({ user: { id: userId } }) + + const speechFile = htmlToSpeechFile({ + title: page.title, + content: page.content, + options: { + primaryVoice: userPersonalization?.speechVoice || 'Axel', + secondaryVoice: + userPersonalization?.speechSecondaryVoice || 'Evelyn', + language: page.language, + }, }) - // enqueue a task to convert text to speech - const taskName = await enqueueTextToSpeech({ - userId, - speechId: speech.id, - text: page.content, - voice: speech.voice, - priority: 'low', - }) - logger.info('Start Text to speech task', { taskName }) + + for (const utterance of speechFile.utterances) { + // enqueue a task to convert text to speech + const taskName = await enqueueTextToSpeech({ + userId, + speechId: utterance.idx, + text: utterance.text, + voice: utterance.voice || 'Axel', + priority: 'high', + isUltraRealisticVoice: true, + language: speechFile.language, + rate: userPersonalization?.speechRate || '1.1', + }) + logger.info('Start Text to speech task', { taskName }) + } + return res.status(202).send('Text to speech task started') } diff --git a/packages/api/src/schema.ts b/packages/api/src/schema.ts index 765379a09..7faa4c4a7 100755 --- a/packages/api/src/schema.ts +++ b/packages/api/src/schema.ts @@ -1912,6 +1912,35 @@ const schema = gql` BAD_REQUEST } + input OptInFeatureInput { + name: String! + } + + union OptInFeatureResult = OptInFeatureSuccess | OptInFeatureError + + type OptInFeatureSuccess { + feature: Feature! + } + + type Feature { + id: ID! + name: String! + token: String! + createdAt: Date! + updatedAt: Date! + grantedAt: Date + expiresAt: Date + } + + type OptInFeatureError { + errorCodes: [OptInFeatureErrorCode!]! + } + + enum OptInFeatureErrorCode { + BAD_REQUEST + NOT_FOUND + } + # Mutations type Mutation { googleLogin(input: GoogleLoginInput!): LoginResult! @@ -1983,6 +2012,7 @@ const schema = gql` moveLabel(input: MoveLabelInput!): MoveLabelResult! setIntegration(input: SetIntegrationInput!): SetIntegrationResult! deleteIntegration(id: ID!): DeleteIntegrationResult! + optInFeature(input: OptInFeatureInput!): OptInFeatureResult! } # FIXME: remove sort from feedArticles after all cached tabs are closed diff --git a/packages/api/src/services/features.ts b/packages/api/src/services/features.ts new file mode 100644 index 000000000..ee96064b2 --- /dev/null +++ b/packages/api/src/services/features.ts @@ -0,0 +1,69 @@ +import { Feature } from '../entity/feature' +import { getRepository } from '../entity/utils' +import * as jwt from 'jsonwebtoken' +import { env } from '../env' +import { IsNull, Not } from 'typeorm' + +enum FeatureName { + UltraRealisticVoice = 'ultra-realistic-voice', +} + +export const getFeatureName = (name: string): FeatureName | undefined => { + return Object.values(FeatureName).find((v) => v === name) +} + +export const optInFeature = async ( + name: FeatureName, + uid: string +): Promise => { + if (name === FeatureName.UltraRealisticVoice) { + return optInUltraRealisticVoice(uid) + } + + return undefined +} + +const optInUltraRealisticVoice = async (uid: string): Promise => { + const feature = await getRepository(Feature).findOne({ + where: { + user: { id: uid }, + name: FeatureName.UltraRealisticVoice, + }, + relations: ['user'], + }) + if (feature) { + // already opted in + console.log('already opted in') + return feature + } + + // opt in to feature for the first 1000 users + const count = await getRepository(Feature).countBy({ + name: FeatureName.UltraRealisticVoice, + grantedAt: Not(IsNull()), + }) + + let grantedAt: Date | null = new Date() + if (count >= 1000) { + console.log('feature limit reached') + grantedAt = null + } + + return getRepository(Feature).save({ + user: { id: uid }, + name: FeatureName.UltraRealisticVoice, + grantedAt, + }) +} + +export const signFeatureToken = (feature: Feature): string => { + return jwt.sign( + { + uid: feature.user.id, + featureName: feature.name, + grantedAt: feature.grantedAt ? feature.grantedAt.getTime() / 1000 : null, + }, + env.server.jwtSecret, + { expiresIn: '1d' } + ) +} diff --git a/packages/api/src/utils/createTask.ts b/packages/api/src/utils/createTask.ts index c8102fa02..af9b0e3ad 100644 --- a/packages/api/src/utils/createTask.ts +++ b/packages/api/src/utils/createTask.ts @@ -344,6 +344,9 @@ export const enqueueTextToSpeech = async ({ bucket = env.fileUpload.gcsUploadBucket, queue = 'omnivore-demo-text-to-speech-queue', location = env.gcp.location, + isUltraRealisticVoice = false, + language, + rate, }: { userId: string speechId: string @@ -354,6 +357,9 @@ export const enqueueTextToSpeech = async ({ textType?: 'text' | 'ssml' queue?: string location?: string + isUltraRealisticVoice?: boolean + language?: string + rate?: string }): Promise => { const { GOOGLE_CLOUD_PROJECT } = process.env const payload = { @@ -362,6 +368,9 @@ export const enqueueTextToSpeech = async ({ voice, bucket, textType, + isUltraRealisticVoice, + language, + rate, } // eslint-disable-next-line @typescript-eslint/ban-ts-comment // @ts-ignore diff --git a/packages/api/test/resolvers/features.test.ts b/packages/api/test/resolvers/features.test.ts new file mode 100644 index 000000000..5a60d2da5 --- /dev/null +++ b/packages/api/test/resolvers/features.test.ts @@ -0,0 +1,202 @@ +import 'mocha' +import { expect } from 'chai' +import { User } from '../../src/entity/user' +import { createTestUser, deleteTestUser } from '../db' +import { graphqlRequest, request } from '../util' +import { getRepository } from '../../src/entity/utils' +import { Feature } from '../../src/entity/feature' +import * as jwt from 'jsonwebtoken' +import sinon, { SinonFakeTimers } from 'sinon' +import { env } from '../../src/env' +import { Like } from 'typeorm' + +describe('features resolvers', () => { + let loginUser: User + let authToken: string + + before(async () => { + // create test user and login + loginUser = await createTestUser('loginUser') + const res = await request + .post('/local/debug/fake-user-login') + .send({ fakeEmail: loginUser.email }) + + authToken = res.body.authToken + }) + + after(async () => { + await deleteTestUser(loginUser.name) + }) + + describe('optInFeature API', () => { + const featureName = 'ultra-realistic-voice' + const now = new Date() + let clock: SinonFakeTimers + + const query = (name: string) => ` + mutation { + optInFeature(input: { + name: "${name}" + }) { + ... on OptInFeatureSuccess { + feature { + name + grantedAt + token + } + } + ... on OptInFeatureError { + errorCodes + } + } + } + ` + + before(() => { + console.log('opting in to feature') + // mock date and ignore milliseconds + clock = sinon.useFakeTimers(now.setSeconds(now.getSeconds(), 0)) + }) + + after(() => { + clock.restore() + }) + + context('when user is the first 1000 users', () => { + after(async () => { + // reset feature + await getRepository(Feature).delete({ + user: { id: loginUser.id }, + }) + }) + + it('opts in to the feature', async () => { + const res = await graphqlRequest(query(featureName), authToken).expect( + 200 + ) + + const token = jwt.sign( + { + uid: loginUser.id, + featureName, + grantedAt: Date.now() / 1000, + }, + env.server.jwtSecret, + { expiresIn: '1d' } + ) + + expect(res.body.data.optInFeature).to.eql({ + feature: { + name: featureName, + grantedAt: new Date().toISOString(), + token, + }, + }) + }) + }) + + context('when user is not the first 1000 users', () => { + before(async () => { + // create 1000 opt-in users + const usersToSave = Array.from(Array(1000).keys()).map((i) => { + return { + name: `user${i}`, + source: 'GOOGLE', + sourceUserId: `fake-user-id-user${i}`, + email: `user${i}@omnivore.app`, + username: `user${i}`, + bio: `i am user${i}`, + } + }) + + const users = await getRepository(User).save(usersToSave) + + const features = users.map((user) => { + return { + user: { id: user.id }, + name: featureName, + grantedAt: new Date(), + } + }) + + await getRepository(Feature).save(features) + }) + + after(async () => { + // reset opt-in users + await getRepository(User).delete({ + name: Like(`user%`), + }) + await getRepository(Feature).delete({ + name: featureName, + }) + }) + + it('does not opt in to the feature', async () => { + const res = await graphqlRequest(query(featureName), authToken).expect( + 200 + ) + + const token = jwt.sign( + { + uid: loginUser.id, + featureName, + grantedAt: null, + }, + env.server.jwtSecret, + { expiresIn: '1d' } + ) + + expect(res.body.data.optInFeature).to.eql({ + feature: { + name: featureName, + grantedAt: null, + token, + }, + }) + }) + }) + + context('when user is already opted in', () => { + before(async () => { + // opt in + await getRepository(Feature).save({ + user: { id: loginUser.id }, + name: featureName, + grantedAt: new Date(), + }) + }) + + after(async () => { + // reset feature + await getRepository(Feature).delete({ + user: { id: loginUser.id }, + }) + }) + + it('returns the feature', async () => { + const res = await graphqlRequest(query(featureName), authToken).expect( + 200 + ) + + const token = jwt.sign( + { + uid: loginUser.id, + featureName, + grantedAt: Date.now() / 1000, + }, + env.server.jwtSecret, + { expiresIn: '1d' } + ) + + expect(res.body.data.optInFeature).to.eql({ + feature: { + name: featureName, + grantedAt: new Date().toISOString(), + token, + }, + }) + }) + }) + }) +}) diff --git a/packages/db/migrations/0098.do.create_features_table.sql b/packages/db/migrations/0098.do.create_features_table.sql new file mode 100755 index 000000000..9a5be9b24 --- /dev/null +++ b/packages/db/migrations/0098.do.create_features_table.sql @@ -0,0 +1,28 @@ +-- Type: DO +-- Name: create_features_table +-- Description: Create features table to store opt-in features by users + +BEGIN; + +CREATE TABLE IF NOT EXISTS omnivore.features ( + id uuid PRIMARY KEY DEFAULT uuid_generate_v1mc(), + user_id uuid NOT NULL REFERENCES omnivore.user ON DELETE CASCADE, + name text NOT NULL, + granted_at timestamptz, + expires_at timestamptz, + created_at timestamptz NOT NULL DEFAULT current_timestamp, + updated_at timestamptz NOT NULL DEFAULT current_timestamp, + UNIQUE (user_id, name) +); + +CREATE TRIGGER features_modtime BEFORE UPDATE ON omnivore.features + FOR EACH ROW EXECUTE PROCEDURE update_updated_at_column(); + +GRANT SELECT, INSERT, UPDATE, DELETE ON omnivore.features TO omnivore_user; + +ALTER TABLE omnivore.user_personalization + ADD COLUMN IF NOT EXISTS speech_secondary_voice text, + ALTER COLUMN speech_rate TYPE text, + ALTER COLUMN speech_volume TYPE text; + +COMMIT; diff --git a/packages/db/migrations/0098.undo.create_features_table.sql b/packages/db/migrations/0098.undo.create_features_table.sql new file mode 100755 index 000000000..700211333 --- /dev/null +++ b/packages/db/migrations/0098.undo.create_features_table.sql @@ -0,0 +1,14 @@ +-- Type: UNDO +-- Name: create_features_table +-- Description: Create features table to store opt-in features by users + +BEGIN; + +DROP TABLE IF EXISTS omnivore.features; + +ALTER TABLE omnivore.user_personalization + DROP COLUMN IF EXISTS speech_secondary_voice, + ALTER COLUMN speech_rate TYPE integer USING speech_rate::integer, + ALTER COLUMN speech_volume TYPE integer USING speech_volume::integer; + +COMMIT; diff --git a/packages/text-to-speech/package.json b/packages/text-to-speech/package.json index e63b348b2..2c4e8591b 100644 --- a/packages/text-to-speech/package.json +++ b/packages/text-to-speech/package.json @@ -21,6 +21,7 @@ "deploy": "yarn build && yarn gcloud-deploy" }, "devDependencies": { + "@types/fluent-ffmpeg": "^2.1.20", "@types/html-to-text": "^8.1.1", "@types/natural": "^5.1.1", "@types/node": "^14.11.2", @@ -30,11 +31,13 @@ "mocha": "^10.0.0" }, "dependencies": { + "@ffmpeg-installer/ffmpeg": "^1.1.0", "@google-cloud/functions-framework": "3.1.2", "@google-cloud/storage": "^6.4.1", "@sentry/serverless": "^6.16.1", "axios": "^0.27.2", "dotenv": "^16.0.1", + "fluent-ffmpeg": "^2.1.2", "html-to-text": "^8.2.1", "jsonwebtoken": "^8.5.1", "linkedom": "^0.14.12", diff --git a/packages/text-to-speech/src/azureTextToSpeech.ts b/packages/text-to-speech/src/azureTextToSpeech.ts new file mode 100644 index 000000000..1d972ec1c --- /dev/null +++ b/packages/text-to-speech/src/azureTextToSpeech.ts @@ -0,0 +1,152 @@ +import { + CancellationDetails, + CancellationReason, + ResultReason, + SpeechConfig, + SpeechSynthesisOutputFormat, + SpeechSynthesisResult, + SpeechSynthesizer, +} from 'microsoft-cognitiveservices-speech-sdk' +import { endSsml, htmlToSsmlItems, ssmlItemText, startSsml } from './htmlToSsml' +import * as _ from 'underscore' +import { + SpeechMark, + TextToSpeech, + TextToSpeechInput, + TextToSpeechOutput, +} from './textToSpeech' + +export class AzureTextToSpeech implements TextToSpeech { + use(input: TextToSpeechInput): boolean { + return !input.isUltraRealisticVoice + } + + synthesizeTextToSpeech = async ( + input: TextToSpeechInput + ): Promise => { + if (!process.env.AZURE_SPEECH_KEY || !process.env.AZURE_SPEECH_REGION) { + throw new Error('Azure Speech Key or Region not set') + } + const textType = input.textType || 'html' + const audioStream = input.audioStream + const speechConfig = SpeechConfig.fromSubscription( + process.env.AZURE_SPEECH_KEY, + process.env.AZURE_SPEECH_REGION + ) + speechConfig.speechSynthesisOutputFormat = + SpeechSynthesisOutputFormat.Audio16Khz32KBitRateMonoMp3 + + // Create the speech synthesizer. + const synthesizer = new SpeechSynthesizer(speechConfig) + const speechMarks: SpeechMark[] = [] + let timeOffset = 0 + let wordOffset = 0 + + synthesizer.synthesizing = function (s, e) { + // convert arrayBuffer to stream and write to stream + audioStream?.write(Buffer.from(e.result.audioData)) + } + + // The event synthesis completed signals that the synthesis is completed. + synthesizer.synthesisCompleted = (s, e) => { + console.info( + `(synthesized) Reason: ${ResultReason[e.result.reason]} Audio length: ${ + e.result.audioData.byteLength + }` + ) + } + + // The synthesis started event signals that the synthesis is started. + synthesizer.synthesisStarted = (s, e) => { + console.info('(synthesis started)') + } + + // The event signals that the service has stopped processing speech. + // This can happen when an error is encountered. + synthesizer.SynthesisCanceled = (s, e) => { + const cancellationDetails = CancellationDetails.fromResult(e.result) + let str = + '(cancel) Reason: ' + CancellationReason[cancellationDetails.reason] + if (cancellationDetails.reason === CancellationReason.Error) { + str += ': ' + e.result.errorDetails + } + console.log(str) + } + + // The unit of e.audioOffset is tick (1 tick = 100 nanoseconds), divide by 10,000 to convert to milliseconds. + synthesizer.wordBoundary = (s, e) => { + speechMarks.push({ + word: e.text, + time: (timeOffset + e.audioOffset) / 10000, + start: wordOffset + e.textOffset, + length: e.wordLength, + type: 'word', + }) + } + + synthesizer.bookmarkReached = (s, e) => { + speechMarks.push({ + word: e.text, + time: (timeOffset + e.audioOffset) / 10000, + type: 'bookmark', + }) + } + + const speakSsmlAsyncPromise = ( + ssml: string + ): Promise => { + return new Promise((resolve, reject) => { + synthesizer.speakSsmlAsync( + ssml, + (result) => { + resolve(result) + }, + (error) => { + reject(error) + } + ) + }) + } + + try { + const ssmlOptions = { + primaryVoice: input.voice, + secondaryVoice: input.secondaryVoice, + language: input.language, + rate: input.rate, + } + if (textType === 'html') { + const ssmlItems = htmlToSsmlItems(input.text, ssmlOptions) + for (const ssmlItem of ssmlItems) { + const ssml = ssmlItemText(ssmlItem) + const result = await speakSsmlAsyncPromise(ssml) + timeOffset = timeOffset + result.audioDuration + } + return { + speechMarks, + } + } + // for ssml + const startSsmlTag = startSsml(ssmlOptions) + wordOffset -= startSsmlTag.length + const text = _.escape(input.text) + const ssml = `${startSsmlTag}${text}${endSsml()}` + const result = await speakSsmlAsyncPromise(ssml) + if (result.reason === ResultReason.Canceled) { + throw new Error(result.errorDetails) + } + + return { + audioData: Buffer.from(result.audioData), + speechMarks, + } + } catch (error) { + console.error('synthesis error:', error) + throw error + } finally { + audioStream?.end() + synthesizer.close() + console.log('synthesizer closed') + } + } +} diff --git a/packages/text-to-speech/src/htmlToSsml.ts b/packages/text-to-speech/src/htmlToSsml.ts index 9b1a32d87..be3f1c2e8 100644 --- a/packages/text-to-speech/src/htmlToSsml.ts +++ b/packages/text-to-speech/src/htmlToSsml.ts @@ -16,7 +16,7 @@ export interface Utterance { text: string wordOffset: number wordCount: number - voice?: string + voice: string } export interface SpeechFile { @@ -44,7 +44,7 @@ export type SSMLOptions = { const DEFAULT_LANGUAGE = 'en-US' const DEFAULT_VOICE = 'en-US-JennyNeural' const DEFAULT_SECONDARY_VOICE = 'en-US-GuyNeural' -const DEFAULT_RATE = '1.0' +const DEFAULT_RATE = '1.1' const ANCHOR_ELEMENTS_BLOCKED_ATTRIBUTES = [ 'omnivore-highlight-id', @@ -269,7 +269,7 @@ const textToUtterances = ({ idx: string textItems: string[] wordOffset: number - voice?: string + voice: string isHtml?: boolean }): Utterance[] => { let text = textItems.join('') @@ -393,6 +393,7 @@ export const htmlToSpeechFile = (htmlInput: HtmlInput): SpeechFile => { textItems: [stripEmojis(title)], // title could have emoji wordOffset, isHtml: false, + voice: defaultVoice, })[0] utterances.push(titleUtterance) wordOffset += titleUtterance.wordCount @@ -413,7 +414,9 @@ export const htmlToSpeechFile = (htmlInput: HtmlInput): SpeechFile => { textItems, wordOffset, voice: - node.nodeName === 'BLOCKQUOTE' ? options.secondaryVoice : undefined, + node.nodeName === 'BLOCKQUOTE' + ? options.secondaryVoice || defaultVoice + : defaultVoice, }) const wordCount = newUtterances.reduce((acc, u) => acc + u.wordCount, 0) wordCount > 0 && utterances.push(...newUtterances) diff --git a/packages/text-to-speech/src/index.ts b/packages/text-to-speech/src/index.ts index d38adbf8b..5953478b3 100644 --- a/packages/text-to-speech/src/index.ts +++ b/packages/text-to-speech/src/index.ts @@ -7,22 +7,29 @@ import * as Sentry from '@sentry/serverless' import axios from 'axios' import * as jwt from 'jsonwebtoken' import * as dotenv from 'dotenv' // see https://github.com/motdotla/dotenv#how-do-i-use-dotenv-with-import -import { - SpeechMark, - synthesizeTextToSpeech, - TextToSpeechInput, -} from './textToSpeech' +import { AzureTextToSpeech } from './azureTextToSpeech' import { File, Storage } from '@google-cloud/storage' import { endSsml, htmlToSpeechFile, startSsml } from './htmlToSsml' import crypto from 'crypto' import { createRedisClient } from './redis' +import { + SpeechMark, + TextToSpeechInput, + TextToSpeechOutput, +} from './textToSpeech' +import { createClient } from 'redis' +import { RealisticTextToSpeech } from './realisticTextToSpeech' + +// explicitly create the return type of RedisClient +type RedisClient = ReturnType interface UtteranceInput { + text: string + idx: string + isUltraRealisticVoice?: boolean voice?: string rate?: string language?: string - text: string - idx: string } interface HTMLInput { @@ -40,14 +47,38 @@ interface CacheResult { speechMarks: SpeechMark[] } +interface Claim { + uid: string + featureName: string | null + grantedAt: number | null +} + dotenv.config() Sentry.GCPFunction.init({ dsn: process.env.SENTRY_DSN, tracesSampleRate: 0, }) +const MAX_CHARACTER_COUNT = 50000 const storage = new Storage() +const textToSpeechHandlers = [ + new AzureTextToSpeech(), + new RealisticTextToSpeech(), +] + +const synthesizeTextToSpeech = async ( + input: TextToSpeechInput +): Promise => { + const textToSpeechHandler = textToSpeechHandlers.find((handler) => + handler.use(input) + ) + if (!textToSpeechHandler) { + throw new Error('No text to speech handler found') + } + return textToSpeechHandler.synthesizeTextToSpeech(input) +} + const uploadToBucket = async ( filePath: string, data: Buffer, @@ -57,7 +88,7 @@ const uploadToBucket = async ( await storage.bucket(bucket).file(filePath).save(data, options) } -const createGCSFile = (bucket: string, filename: string): File => { +export const createGCSFile = (bucket: string, filename: string): File => { return storage.bucket(bucket).file(filename) } @@ -84,14 +115,41 @@ const updateSpeech = async ( return response.status === 200 } +const getCharacterCountFromRedis = async ( + redisClient: RedisClient, + uid: string +): Promise => { + const wordCount = await redisClient.get(`tts:charCount:${uid}`) + return wordCount ? parseInt(wordCount) : 0 +} + +// store character count of each text to speech request in redis +// which will be used to rate limit the request +// expires after 1 day +const updateCharacterCountInRedis = async ( + redisClient: RedisClient, + uid: string, + wordCount: number +): Promise => { + await redisClient.set(`tts:charCount:${uid}`, wordCount.toString(), { + EX: 3600 * 24, // in seconds + NX: true, + }) +} + export const textToSpeechHandler = Sentry.GCPFunction.wrapHttpFunction( async (req, res) => { console.info('Text to speech request body:', req.body) - const token = req.query.token as string if (!process.env.JWT_SECRET) { console.error('JWT_SECRET not exists') return res.status(500).send({ errorCodes: 'JWT_SECRET_NOT_EXISTS' }) } + + const token = (req.query.token || req.headers.authorization) as string + if (!token) { + return res.status(401).send({ errorCode: 'INVALID_TOKEN' }) + } + try { jwt.verify(token, process.env.JWT_SECRET) } catch (e) { @@ -114,21 +172,28 @@ export const textToSpeechHandler = Sentry.GCPFunction.wrapHttpFunction( }) as NodeJS.WriteStream // synthesize text to speech const startTime = Date.now() + // temporary solution to use realistic text to speech const { speechMarks } = await synthesizeTextToSpeech({ ...input, textType: 'html', audioStream, + key: id, }) console.info( `Synthesize text to speech completed in ${Date.now() - startTime} ms` ) + // speech marks file to be saved in GCS - const speechMarksFileName = `speech/${id}.json` - await uploadToBucket( - speechMarksFileName, - Buffer.from(JSON.stringify(speechMarks)), - bucket - ) + let speechMarksFileName: string | undefined + if (speechMarks.length > 0) { + speechMarksFileName = `speech/${id}.json` + await uploadToBucket( + speechMarksFileName, + Buffer.from(JSON.stringify(speechMarks)), + bucket + ) + } + // update speech state const updated = await updateSpeech( id, @@ -162,8 +227,11 @@ export const textToSpeechStreamingHandler = Sentry.GCPFunction.wrapHttpFunction( if (!token) { return res.status(401).send({ errorCode: 'INVALID_TOKEN' }) } + + let claim: Claim try { jwt.verify(token, process.env.JWT_SECRET) + claim = jwt.decode(token) as Claim } catch (e) { console.error('Authentication error:', e) return res.status(401).send({ errorCode: 'UNAUTHENTICATED' }) @@ -177,6 +245,26 @@ export const textToSpeechStreamingHandler = Sentry.GCPFunction.wrapHttpFunction( try { const utteranceInput = req.body as UtteranceInput + if (!utteranceInput.text) { + return res.status(400).send('INVALID_INPUT') + } + + // validate if user has opted in to use ultra realistic voice feature + if ( + utteranceInput.isUltraRealisticVoice && + (claim.featureName !== 'ultra-realistic-voice' || !claim.grantedAt) + ) { + return res.status(403).send('UNAUTHORIZED') + } + + // validate character count + const characterCount = + (await getCharacterCountFromRedis(redisClient, claim.uid)) + + utteranceInput.text.length + if (characterCount > MAX_CHARACTER_COUNT) { + return res.status(429).send('RATE_LIMITED') + } + const ssmlOptions = { primaryVoice: utteranceInput.voice, secondaryVoice: utteranceInput.voice, @@ -201,15 +289,50 @@ export const textToSpeechStreamingHandler = Sentry.GCPFunction.wrapHttpFunction( return } console.log('Cache miss') - // synthesize text to speech if cache miss + + const bucket = process.env.GCS_UPLOAD_BUCKET + if (!bucket) { + throw new Error('GCS_UPLOAD_BUCKET not set') + } + + // audio file to be saved in GCS + const audioFileName = `speech/${cacheKey}.mp3` + const speechMarksFileName = `speech/${cacheKey}.json` + const audioFile = createGCSFile(bucket, audioFileName) + const speechMarksFile = createGCSFile(bucket, speechMarksFileName) + // check if audio file already exists + const [exists] = await audioFile.exists() + if (exists) { + console.debug('Audio file already exists') + const [audioData] = await audioFile.download() + const [speechMarksExists] = await speechMarksFile.exists() + + return { + audioData, + speechMarks: speechMarksExists + ? JSON.parse((await speechMarksFile.download()).toString()) + : [], + } + } + const input: TextToSpeechInput = { ...utteranceInput, textType: 'ssml', + key: cacheKey, } + // synthesize text to speech if cache miss const { audioData, speechMarks } = await synthesizeTextToSpeech(input) if (!audioData) { return res.status(500).send({ errorCode: 'SYNTHESIZER_ERROR' }) } + + // upload audio data to GCS + await audioFile.save(audioData) + // upload speech marks to GCS + if (speechMarks.length > 0) { + await speechMarksFile.save(JSON.stringify(speechMarks)) + } + const audioDataString = audioData.toString('hex') // save audio data to cache for 24 hours for mainly the newsletters await redisClient.set( @@ -222,6 +345,9 @@ export const textToSpeechStreamingHandler = Sentry.GCPFunction.wrapHttpFunction( ) console.log('Cache saved') + // update character count + await updateCharacterCountInRedis(redisClient, claim.uid, characterCount) + res.send({ idx: utteranceInput.idx, audioData: audioDataString, diff --git a/packages/text-to-speech/src/realisticTextToSpeech.ts b/packages/text-to-speech/src/realisticTextToSpeech.ts new file mode 100644 index 000000000..94a0a9c7a --- /dev/null +++ b/packages/text-to-speech/src/realisticTextToSpeech.ts @@ -0,0 +1,133 @@ +import { + TextToSpeech, + TextToSpeechInput, + TextToSpeechOutput, +} from './textToSpeech' +import axios from 'axios' +import ffmpegPath from '@ffmpeg-installer/ffmpeg' +import ffmpeg from 'fluent-ffmpeg' +import { PassThrough } from 'stream' + +ffmpeg.setFfmpegPath(ffmpegPath.path) + +interface PlayHtConvertResponse { + message: string + payload: string[] +} + +const convertWavToMp3AndUpload = async ( + inputStream: PassThrough, + outputStream: PassThrough +) => { + return new Promise((resolve, reject) => { + ffmpeg(inputStream) + .audioCodec('libmp3lame') + .format('mp3') + .on('error', (err) => { + reject(err) + }) + .on('end', () => { + console.debug('Finished processing') + resolve() + }) + .pipe(outputStream, { end: true }) + }) +} + +export class RealisticTextToSpeech implements TextToSpeech { + synthesizeTextToSpeech = async ( + input: TextToSpeechInput + ): Promise => { + const apiEndpoint = process.env.REALISTIC_VOICE_API_ENDPOINT + const apiKey = process.env.REALISTIC_VOICE_API_KEY + const userId = process.env.REALISTIC_VOICE_USER_ID + if (!apiEndpoint || !apiKey || !userId) { + throw new Error('PlayHT API credentials not set') + } + + const inputStream = new PassThrough() + + const HEADERS = { + Authorization: apiKey, + 'X-User-ID': userId, + 'Content-Type': 'application/json', + } + + const data = { + voice: input.voice, + content: [input.text], + } + + // get the download url first + const response = await axios.post( + apiEndpoint, + data, + { + headers: HEADERS, + } + ) + + if (response.data.payload.length === 0) { + throw new Error('No payload returned') + } + + const downloadUrl = response.data.payload[0] + + // polling the download url until the file is ready + // timeout after 1 hour + const timeout = 60 * 60 * 1000 + const startTime = Date.now() + let isReady = false + while (!isReady) { + if (Date.now() - startTime > timeout) { + throw new Error('Timeout when polling the download url') + } + + // download the audio file + try { + const downloadResponse = await axios.get(downloadUrl, { + responseType: 'arraybuffer', + headers: { + 'Content-Type': 'audio/wav', + }, + }) + + // write the audio file to the input stream + // eslint-disable-next-line @typescript-eslint/no-unsafe-argument + inputStream.end(Buffer.from(downloadResponse.data, 'binary')) + isReady = true + } catch (e) { + // ignore error + console.debug('checking status of audio file', downloadUrl) + } + } + + const outputStream = new PassThrough() + // transcode the audio file to mp3 + await convertWavToMp3AndUpload(inputStream, outputStream) + + // convert the buffer stream to a buffer + const audioData = await new Promise((resolve, reject) => { + const chunks: Buffer[] = [] + outputStream.on('data', (chunk) => { + // eslint-disable-next-line @typescript-eslint/no-unsafe-argument + chunks.push(chunk) + }) + outputStream.on('end', () => { + resolve(Buffer.concat(chunks)) + }) + outputStream.on('error', (err) => { + reject(err) + }) + }) + + return { + audioData, + speechMarks: [], + } + } + + use(input: TextToSpeechInput): boolean { + return !!input.isUltraRealisticVoice + } +} diff --git a/packages/text-to-speech/src/textToSpeech.ts b/packages/text-to-speech/src/textToSpeech.ts index da6c2e4c2..a446b18b3 100644 --- a/packages/text-to-speech/src/textToSpeech.ts +++ b/packages/text-to-speech/src/textToSpeech.ts @@ -1,23 +1,13 @@ -import { - CancellationDetails, - CancellationReason, - ResultReason, - SpeechConfig, - SpeechSynthesisOutputFormat, - SpeechSynthesisResult, - SpeechSynthesizer, -} from 'microsoft-cognitiveservices-speech-sdk' -import { endSsml, htmlToSsmlItems, ssmlItemText, startSsml } from './htmlToSsml' -import * as _ from 'underscore' - export interface TextToSpeechInput { text: string + key: string voice?: string language?: string textType?: 'html' | 'ssml' rate?: string secondaryVoice?: string audioStream?: NodeJS.ReadWriteStream + isUltraRealisticVoice?: boolean } export interface TextToSpeechOutput { @@ -32,132 +22,10 @@ export interface SpeechMark { word: string type: 'word' | 'bookmark' } +export abstract class TextToSpeech { + abstract use(input: TextToSpeechInput): boolean -export const synthesizeTextToSpeech = async ( - input: TextToSpeechInput -): Promise => { - if (!process.env.AZURE_SPEECH_KEY || !process.env.AZURE_SPEECH_REGION) { - throw new Error('Azure Speech Key or Region not set') - } - const textType = input.textType || 'html' - const audioStream = input.audioStream - const speechConfig = SpeechConfig.fromSubscription( - process.env.AZURE_SPEECH_KEY, - process.env.AZURE_SPEECH_REGION - ) - speechConfig.speechSynthesisOutputFormat = - SpeechSynthesisOutputFormat.Audio16Khz32KBitRateMonoMp3 - - // Create the speech synthesizer. - const synthesizer = new SpeechSynthesizer(speechConfig) - const speechMarks: SpeechMark[] = [] - let timeOffset = 0 - let wordOffset = 0 - - synthesizer.synthesizing = function (s, e) { - // convert arrayBuffer to stream and write to stream - audioStream?.write(Buffer.from(e.result.audioData)) - } - - // The event synthesis completed signals that the synthesis is completed. - synthesizer.synthesisCompleted = (s, e) => { - console.info( - `(synthesized) Reason: ${ResultReason[e.result.reason]} Audio length: ${ - e.result.audioData.byteLength - }` - ) - } - - // The synthesis started event signals that the synthesis is started. - synthesizer.synthesisStarted = (s, e) => { - console.info('(synthesis started)') - } - - // The event signals that the service has stopped processing speech. - // This can happen when an error is encountered. - synthesizer.SynthesisCanceled = (s, e) => { - const cancellationDetails = CancellationDetails.fromResult(e.result) - let str = - '(cancel) Reason: ' + CancellationReason[cancellationDetails.reason] - if (cancellationDetails.reason === CancellationReason.Error) { - str += ': ' + e.result.errorDetails - } - console.log(str) - } - - // The unit of e.audioOffset is tick (1 tick = 100 nanoseconds), divide by 10,000 to convert to milliseconds. - synthesizer.wordBoundary = (s, e) => { - speechMarks.push({ - word: e.text, - time: (timeOffset + e.audioOffset) / 10000, - start: wordOffset + e.textOffset, - length: e.wordLength, - type: 'word', - }) - } - - synthesizer.bookmarkReached = (s, e) => { - speechMarks.push({ - word: e.text, - time: (timeOffset + e.audioOffset) / 10000, - type: 'bookmark', - }) - } - - const speakSsmlAsyncPromise = ( - ssml: string - ): Promise => { - return new Promise((resolve, reject) => { - synthesizer.speakSsmlAsync( - ssml, - (result) => { - resolve(result) - }, - (error) => { - reject(error) - } - ) - }) - } - - try { - const ssmlOptions = { - primaryVoice: input.voice, - secondaryVoice: input.secondaryVoice, - language: input.language, - rate: input.rate, - } - if (textType === 'html') { - const ssmlItems = htmlToSsmlItems(input.text, ssmlOptions) - for (const ssmlItem of ssmlItems) { - const ssml = ssmlItemText(ssmlItem) - const result = await speakSsmlAsyncPromise(ssml) - timeOffset = timeOffset + result.audioDuration - } - return { - speechMarks, - } - } - // for ssml - const startSsmlTag = startSsml(ssmlOptions) - wordOffset -= startSsmlTag.length - const text = _.escape(input.text) - const ssml = `${startSsmlTag}${text}${endSsml()}` - const result = await speakSsmlAsyncPromise(ssml) - if (result.reason === ResultReason.Canceled) { - throw new Error(result.errorDetails) - } - - return { - audioData: Buffer.from(result.audioData), - speechMarks, - } - } catch (error) { - console.error('synthesis error:', error) - throw error - } finally { - audioStream?.end() - synthesizer.close() - console.log('synthesizer closed') - } + abstract synthesizeTextToSpeech( + input: TextToSpeechInput + ): Promise } diff --git a/yarn.lock b/yarn.lock index 27452d1b1..b1b9a214d 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2286,6 +2286,60 @@ lodash.isundefined "^3.0.1" lodash.uniq "^4.5.0" +"@ffmpeg-installer/darwin-arm64@4.1.5": + version "4.1.5" + resolved "https://registry.yarnpkg.com/@ffmpeg-installer/darwin-arm64/-/darwin-arm64-4.1.5.tgz#b7b5c262dd96d1aea4807514e1cdcf6e11f82743" + integrity sha512-hYqTiP63mXz7wSQfuqfFwfLOfwwFChUedeCVKkBtl/cliaTM7/ePI9bVzfZ2c+dWu3TqCwLDRWNSJ5pqZl8otA== + +"@ffmpeg-installer/darwin-x64@4.1.0": + version "4.1.0" + resolved "https://registry.yarnpkg.com/@ffmpeg-installer/darwin-x64/-/darwin-x64-4.1.0.tgz#48e1706c690e628148482bfb64acb67472089aaa" + integrity sha512-Z4EyG3cIFjdhlY8wI9aLUXuH8nVt7E9SlMVZtWvSPnm2sm37/yC2CwjUzyCQbJbySnef1tQwGG2Sx+uWhd9IAw== + +"@ffmpeg-installer/ffmpeg@^1.1.0": + version "1.1.0" + resolved "https://registry.yarnpkg.com/@ffmpeg-installer/ffmpeg/-/ffmpeg-1.1.0.tgz#87fdb9e7d180e8d78f7903f9441e36f978938a90" + integrity sha512-Uq4rmwkdGxIa9A6Bd/VqqYbT7zqh1GrT5/rFwCwKM70b42W5gIjWeVETq6SdcL0zXqDtY081Ws/iJWhr1+xvQg== + optionalDependencies: + "@ffmpeg-installer/darwin-arm64" "4.1.5" + "@ffmpeg-installer/darwin-x64" "4.1.0" + "@ffmpeg-installer/linux-arm" "4.1.3" + "@ffmpeg-installer/linux-arm64" "4.1.4" + "@ffmpeg-installer/linux-ia32" "4.1.0" + "@ffmpeg-installer/linux-x64" "4.1.0" + "@ffmpeg-installer/win32-ia32" "4.1.0" + "@ffmpeg-installer/win32-x64" "4.1.0" + +"@ffmpeg-installer/linux-arm64@4.1.4": + version "4.1.4" + resolved "https://registry.yarnpkg.com/@ffmpeg-installer/linux-arm64/-/linux-arm64-4.1.4.tgz#7219f3f901bb67f7926cb060b56b6974a6cad29f" + integrity sha512-dljEqAOD0oIM6O6DxBW9US/FkvqvQwgJ2lGHOwHDDwu/pX8+V0YsDL1xqHbj1DMX/+nP9rxw7G7gcUvGspSoKg== + +"@ffmpeg-installer/linux-arm@4.1.3": + version "4.1.3" + resolved "https://registry.yarnpkg.com/@ffmpeg-installer/linux-arm/-/linux-arm-4.1.3.tgz#c554f105ed5f10475ec25d7bec94926ce18db4c1" + integrity sha512-NDf5V6l8AfzZ8WzUGZ5mV8O/xMzRag2ETR6+TlGIsMHp81agx51cqpPItXPib/nAZYmo55Bl2L6/WOMI3A5YRg== + +"@ffmpeg-installer/linux-ia32@4.1.0": + version "4.1.0" + resolved "https://registry.yarnpkg.com/@ffmpeg-installer/linux-ia32/-/linux-ia32-4.1.0.tgz#adad70b0d0d9d8d813983d6e683c5a338a75e442" + integrity sha512-0LWyFQnPf+Ij9GQGD034hS6A90URNu9HCtQ5cTqo5MxOEc7Rd8gLXrJvn++UmxhU0J5RyRE9KRYstdCVUjkNOQ== + +"@ffmpeg-installer/linux-x64@4.1.0": + version "4.1.0" + resolved "https://registry.yarnpkg.com/@ffmpeg-installer/linux-x64/-/linux-x64-4.1.0.tgz#b4a5d89c4e12e6d9306dbcdc573df716ec1c4323" + integrity sha512-Y5BWhGLU/WpQjOArNIgXD3z5mxxdV8c41C+U15nsE5yF8tVcdCGet5zPs5Zy3Ta6bU7haGpIzryutqCGQA/W8A== + +"@ffmpeg-installer/win32-ia32@4.1.0": + version "4.1.0" + resolved "https://registry.yarnpkg.com/@ffmpeg-installer/win32-ia32/-/win32-ia32-4.1.0.tgz#6eac4fb691b64c02e7a116c1e2d167f3e9b40638" + integrity sha512-FV2D7RlaZv/lrtdhaQ4oETwoFUsUjlUiasiZLDxhEUPdNDWcH1OU9K1xTvqz+OXLdsmYelUDuBS/zkMOTtlUAw== + +"@ffmpeg-installer/win32-x64@4.1.0": + version "4.1.0" + resolved "https://registry.yarnpkg.com/@ffmpeg-installer/win32-x64/-/win32-x64-4.1.0.tgz#17e8699b5798d4c60e36e2d6326a8ebe5e95a2c5" + integrity sha512-Drt5u2vzDnIONf4ZEkKtFlbvwj6rI3kxw1Ck9fpudmtgaZIHD4ucsWB2lCZBXRxJgXR+2IMSti+4rtM4C4rXgg== + "@firebase/app-types@0.7.0": version "0.7.0" resolved "https://registry.yarnpkg.com/@firebase/app-types/-/app-types-0.7.0.tgz#c9e16d1b8bed1a991840b8d2a725fb58d0b5899f" @@ -7702,6 +7756,13 @@ resolved "https://registry.yarnpkg.com/@types/fined/-/fined-1.1.3.tgz#83f03e8f0a8d3673dfcafb18fce3571f6250e1bc" integrity sha512-CWYnSRnun3CGbt6taXeVo2lCbuaj4mchVJ4UF/BdU5TSuIn3AmS13pGMwCsBUoehGbhZrBrpNJZSZI5EVilXww== +"@types/fluent-ffmpeg@^2.1.20": + version "2.1.20" + resolved "https://registry.yarnpkg.com/@types/fluent-ffmpeg/-/fluent-ffmpeg-2.1.20.tgz#3b5f42fc8263761d58284fa46ee6759a64ce54ac" + integrity sha512-B+OvhCdJ3LgEq2PhvWNOiB/EfwnXLElfMCgc4Z1K5zXgSfo9I6uGKwR/lqmNPFQuebNnes7re3gqkV77SyypLg== + dependencies: + "@types/node" "*" + "@types/glob@*": version "7.2.0" resolved "https://registry.yarnpkg.com/@types/glob/-/glob-7.2.0.tgz#bc1b5bf3aa92f25bd5dd39f35c57361bdce5b2eb" @@ -9590,6 +9651,11 @@ async-retry@^1.2.1, async-retry@^1.3.3: dependencies: retry "0.13.1" +async@>=0.2.9: + version "3.2.4" + resolved "https://registry.yarnpkg.com/async/-/async-3.2.4.tgz#2d22e00f8cddeb5fde5dd33522b56d1cf569a81c" + integrity sha512-iAB+JbDEGXhyIUavoDl9WP/Jj106Kz9DEn1DPgYw5ruDn0e3Wgi3sKFm55sASdGBNOQB8F59d9qQ7deqrHA8wQ== + async@^2.6.2: version "2.6.4" resolved "https://registry.yarnpkg.com/async/-/async-2.6.4.tgz#706b7ff6084664cd7eae713f6f965433b5504221" @@ -14018,6 +14084,14 @@ flatted@^3.1.0: resolved "https://registry.yarnpkg.com/flatted/-/flatted-3.2.2.tgz#64bfed5cb68fe3ca78b3eb214ad97b63bedce561" integrity sha512-JaTY/wtrcSyvXJl4IMFHPKyFur1sE9AUqc0QnhOaJ0CxHtAoIV8pYDzeEfAaNEtGkOfq4gr3LBFmdXW5mOQFnA== +fluent-ffmpeg@^2.1.2: + version "2.1.2" + resolved "https://registry.yarnpkg.com/fluent-ffmpeg/-/fluent-ffmpeg-2.1.2.tgz#c952de2240f812ebda0aa8006d7776ee2acf7d74" + integrity sha512-IZTB4kq5GK0DPp7sGQ0q/BWurGHffRtQQwVkiqDgeO6wYJLLV5ZhgNOQ65loZxxuPMKZKZcICCUnaGtlxBiR0Q== + dependencies: + async ">=0.2.9" + which "^1.1.1" + flush-write-stream@^1.0.0: version "1.1.1" resolved "https://registry.yarnpkg.com/flush-write-stream/-/flush-write-stream-1.1.1.tgz#8dd7d873a1babc207d94ead0c2e0e44276ebf2e8" @@ -25709,7 +25783,7 @@ which@2.0.2, which@^2.0.1, which@^2.0.2: dependencies: isexe "^2.0.0" -which@^1.2.14, which@^1.2.9, which@^1.3.1: +which@^1.1.1, which@^1.2.14, which@^1.2.9, which@^1.3.1: version "1.3.1" resolved "https://registry.yarnpkg.com/which/-/which-1.3.1.tgz#a45043d54f5805316da8d62f9f50918d3da70b0a" integrity sha512-HxJdYWq1MTIQbJ3nw0cqssHoTNU267KlrDuGZ1WYlxDStUtKUhOaJmh112/TZmHxxUfuJqPXSOm7tDyas0OSIQ==