prehistoric-systems/omnivore
1
0
Fork 0
mirror of https://github.com/omnivore-app/omnivore.git synced 2026-03-11 08:54:26 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge branch 'main' into feature/following-screen

Browse source 
# Conflicts:
#	android/Omnivore/app/src/main/graphql/Search.graphql
#	android/Omnivore/app/src/main/java/app/omnivore/omnivore/core/data/LibrarySync.kt
#	android/Omnivore/app/src/main/java/app/omnivore/omnivore/core/network/SearchQuery.kt
#	android/Omnivore/gradle/libs.versions.toml
This commit is contained in:
Stefano Sansone 2024-04-24 22:41:17 +02:00
commit abc1e081bd
42 changed files with 1080 additions and 749 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
android/Omnivore/app/build.gradle.kts
View file

@ -27,8 +27,8 @@ android {
applicationId = "app.omnivore.omnivore"
minSdk = 26
targetSdk = 34
versionCode = 2000050
versionName = "0.200.5"
versionCode = 2000080
versionName = "0.200.8"
testInstrumentationRunner = "androidx.test.runner.AndroidJUnitRunner"
vectorDrawables {

1
android/Omnivore/app/src/main/AndroidManifest.xml
View file

@ -5,6 +5,7 @@
<uses-permission android:name="android.permission.INTERNET" />
<uses-permission android:name="android.permission.ACCESS_NETWORK_STATE"/>
<uses-permission android:name="android.permission.WAKE_LOCK" />
<uses-permission android:name="android.permission.WRITE_EXTERNAL_STORAGE" tools:remove="android:maxSdkVersion" />
<application
android:name=".OmnivoreApplication"

2
android/Omnivore/app/src/main/assets/bundle.js
View file

File diff suppressed because one or more lines are too long

103
android/Omnivore/app/src/main/graphql/Search.graphql
View file

@ -1,55 +1,56 @@
query Search($after: String, $first: Int, $query: String) {
search(first: $first, after: $after, query: $query) {
... on SearchSuccess {
edges {
cursor
node {
id
title
slug
url
pageType
contentReader
createdAt
isArchived
readingProgressPercent
readingProgressAnchorIndex
author
image
description
publishedAt
ownedByViewer
originalArticleUrl
uploadFileId
labels {
...LabelFields
}
highlights {
...HighlightFields
}
pageId
shortId
quote
annotation
state
siteName
subscription
readAt
savedAt
updatedAt
wordsCount
}
}
pageInfo {
hasNextPage
hasPreviousPage
startCursor
endCursor
totalCount
}
}
... on SearchError {
errorCodes
search(first: $first, after: $after, query: $query, includeContent: true) {
... on SearchSuccess {
edges {
cursor
node {
id
title
slug
url
pageType
contentReader
createdAt
isArchived
readingProgressPercent
readingProgressAnchorIndex
author
image
description
publishedAt
ownedByViewer
originalArticleUrl
uploadFileId
labels {
...LabelFields
}
highlights {
...HighlightFields
}
pageId
shortId
quote
annotation
state
siteName
subscription
readAt
savedAt
updatedAt
wordsCount
content
}
}
pageInfo {
hasNextPage
hasPreviousPage
startCursor
endCursor
totalCount
}
}
... on SearchError {
errorCodes
}
}
}

244
android/Omnivore/app/src/main/java/app/omnivore/omnivore/core/data/LibrarySync.kt
View file

@ -1,151 +1,159 @@
package app.omnivore.omnivore.core.data
import android.util.Log
import app.omnivore.omnivore.core.data.model.ServerSyncStatus
import app.omnivore.omnivore.core.database.entities.Highlight
import app.omnivore.omnivore.core.database.entities.SavedItem
import app.omnivore.omnivore.core.database.entities.SavedItemLabel
import app.omnivore.omnivore.core.database.entities.SavedItemWithLabelsAndHighlights
import app.omnivore.omnivore.core.network.savedItem
import app.omnivore.omnivore.core.network.savedItemUpdates
import app.omnivore.omnivore.core.network.search
import app.omnivore.omnivore.core.data.model.ServerSyncStatus
suspend fun DataService.librarySearch(cursor: String?, query: String): SearchResult {
val searchResult = networker.search(cursor = cursor, limit = 10, query = query)
val searchResult = networker.search(cursor = cursor, limit = 10, query = query)
val savedItems = searchResult.items.map {
SavedItemWithLabelsAndHighlights(
savedItem = it.item,
labels = it.labels,
highlights = it.highlights,
)
}
db.savedItemWithLabelsAndHighlightsDao().insertAll(savedItems)
Log.d(
"sync",
"found ${searchResult.items.size} items with search api. Query: $query cursor: $cursor"
val savedItems = searchResult.items.map {
SavedItemWithLabelsAndHighlights(
savedItem = it.item,
labels = it.labels,
highlights = it.highlights,
)
}
return SearchResult(
hasError = false,
hasMoreItems = false,
cursor = searchResult.cursor,
count = searchResult.items.size,
savedItems = savedItems
)
db.savedItemWithLabelsAndHighlightsDao().insertAll(savedItems)
Log.d("sync", "found ${searchResult.items.size} items with search api. Query: $query cursor: $cursor")
return SearchResult(
hasError = false,
hasMoreItems = false,
cursor = searchResult.cursor,
count = searchResult.items.size,
savedItems = savedItems
)
}
suspend fun DataService.sync(since: String, cursor: String?, limit: Int = 20): SavedItemSyncResult {
val syncResult = networker.savedItemUpdates(cursor = cursor, limit = limit, since = since)
?: return SavedItemSyncResult.errorResult
val syncResult = networker.savedItemUpdates(cursor = cursor, limit = limit, since = since)
?: return SavedItemSyncResult.errorResult
if (syncResult.deletedItemIDs.isNotEmpty()) {
db.savedItemDao().deleteByIds(syncResult.deletedItemIDs)
}
if (syncResult.deletedItemIDs.isNotEmpty()) {
db.savedItemDao().deleteByIds(syncResult.deletedItemIDs)
}
val savedItems = syncResult.items.map {
val savedItem = SavedItem(
savedItemId = it.id,
title = it.title,
createdAt = it.createdAt as String,
savedAt = it.savedAt as String,
readAt = it.readAt as String?,
updatedAt = it.updatedAt as String?,
readingProgress = it.readingProgressPercent,
readingProgressAnchor = it.readingProgressAnchorIndex,
imageURLString = it.image,
pageURLString = it.url,
descriptionText = it.description,
publisherURLString = it.originalArticleUrl,
siteName = it.siteName,
author = it.author,
publishDate = it.publishedAt as String?,
slug = it.slug,
isArchived = it.isArchived,
contentReader = it.contentReader.rawValue,
content = null,
wordsCount = it.wordsCount
)
val labels = it.labels?.map { label ->
SavedItemLabel(
savedItemLabelId = label.labelFields.id,
name = label.labelFields.name,
color = label.labelFields.color,
createdAt = null,
labelDescription = null
)
} ?: listOf()
val highlights = it.highlights?.map { highlight ->
Highlight(
type = highlight.highlightFields.type.toString(),
highlightId = highlight.highlightFields.id,
annotation = highlight.highlightFields.annotation,
createdByMe = highlight.highlightFields.createdByMe,
markedForDeletion = false,
patch = highlight.highlightFields.patch,
prefix = highlight.highlightFields.prefix,
quote = highlight.highlightFields.quote,
serverSyncStatus = ServerSyncStatus.IS_SYNCED.rawValue,
shortId = highlight.highlightFields.shortId,
suffix = highlight.highlightFields.suffix,
createdAt = null,
updatedAt = highlight.highlightFields.updatedAt as String?,
color = highlight.highlightFields.color,
highlightPositionPercent = highlight.highlightFields.highlightPositionPercent,
highlightPositionAnchorIndex = highlight.highlightFields.highlightPositionAnchorIndex,
)
} ?: listOf()
SavedItemWithLabelsAndHighlights(
savedItem = savedItem, labels = labels, highlights = highlights
)
}
val savedItems = syncResult.items.map {
val savedItem = SavedItem(
savedItemId = it.id,
title = it.title,
createdAt = it.createdAt as String,
savedAt = it.savedAt as String,
readAt = it.readAt as String?,
updatedAt = it.updatedAt as String?,
readingProgress = it.readingProgressPercent,
readingProgressAnchor = it.readingProgressAnchorIndex,
imageURLString = it.image,
pageURLString = it.url,
descriptionText = it.description,
publisherURLString = it.originalArticleUrl,
siteName = it.siteName,
author = it.author,
publishDate = it.publishedAt as String?,
slug = it.slug,
isArchived = it.isArchived,
contentReader = it.contentReader.rawValue,
wordsCount = it.wordsCount
)
val labels = it.labels?.map { label ->
SavedItemLabel(
savedItemLabelId = label.labelFields.id,
name = label.labelFields.name,
color = label.labelFields.color,
createdAt = null,
labelDescription = null
)
} ?: listOf()
val highlights = it.highlights?.map { highlight ->
Highlight(
type = highlight.highlightFields.type.toString(),
highlightId = highlight.highlightFields.id,
annotation = highlight.highlightFields.annotation,
createdByMe = highlight.highlightFields.createdByMe,
markedForDeletion = false,
patch = highlight.highlightFields.patch,
prefix = highlight.highlightFields.prefix,
quote = highlight.highlightFields.quote,
serverSyncStatus = ServerSyncStatus.IS_SYNCED.rawValue,
shortId = highlight.highlightFields.shortId,
suffix = highlight.highlightFields.suffix,
createdAt = null,
updatedAt = highlight.highlightFields.updatedAt as String?,
color = highlight.highlightFields.color,
highlightPositionPercent = highlight.highlightFields.highlightPositionPercent,
highlightPositionAnchorIndex = highlight.highlightFields.highlightPositionAnchorIndex,
)
} ?: listOf()
SavedItemWithLabelsAndHighlights(
savedItem = savedItem,
labels = labels,
highlights = highlights
)
}
db.savedItemWithLabelsAndHighlightsDao().insertAll(savedItems)
db.savedItemWithLabelsAndHighlightsDao().insertAll(savedItems)
Log.d("sync", "found ${syncResult.items.size} items with sync api. Since: $since")
Log.d("sync", "found ${syncResult.items.size} items with sync api. Since: $since")
return SavedItemSyncResult(hasError = false,
hasMoreItems = syncResult.hasMoreItems,
cursor = syncResult.cursor,
count = syncResult.items.size,
savedItemSlugs = syncResult.items.map { it.slug })
return SavedItemSyncResult(
hasError = false,
hasMoreItems = syncResult.hasMoreItems,
cursor = syncResult.cursor,
count = syncResult.items.size,
savedItemSlugs = syncResult.items.map { it.slug }
)
}
suspend fun DataService.isSavedItemContentStoredInDB(slug: String): Boolean {
val existingItem = db.savedItemDao().getSavedItemWithLabelsAndHighlights(slug)
val content = existingItem?.savedItem?.content ?: ""
return content.length > 10
fun DataService.isSavedItemContentStoredInDB(slug: String): Boolean {
val existingItem = db.savedItemDao().getSavedItemWithLabelsAndHighlights(slug)
val content = existingItem?.savedItem?.content ?: ""
return content.length > 10
}
suspend fun DataService.fetchSavedItemContent(slug: String) {
val syncResult = networker.savedItem(slug)
val savedItem = syncResult.item
savedItem?.let {
val item = SavedItemWithLabelsAndHighlights(
savedItem = savedItem,
labels = syncResult.labels,
highlights = syncResult.highlights
)
db.savedItemWithLabelsAndHighlightsDao().insertAll(listOf(item))
}
}
data class SavedItemSyncResult(
val hasError: Boolean,
val hasMoreItems: Boolean,
val count: Int,
val savedItemSlugs: List<String>,
val cursor: String?
val hasError: Boolean,
val hasMoreItems: Boolean,
val count: Int,
val savedItemSlugs: List<String>,
val cursor: String?
) {
companion object {
val errorResult = SavedItemSyncResult(
hasError = true,
hasMoreItems = true,
cursor = null,
count = 0,
savedItemSlugs = listOf()
)
}
companion object {
val errorResult = SavedItemSyncResult(hasError = true, hasMoreItems = true, cursor = null, count = 0, savedItemSlugs = listOf())
}
}
data class SearchResult(
val hasError: Boolean,
val hasMoreItems: Boolean,
val count: Int,
val savedItems: List<SavedItemWithLabelsAndHighlights>,
val cursor: String?
val hasError: Boolean,
val hasMoreItems: Boolean,
val count: Int,
val savedItems: List<SavedItemWithLabelsAndHighlights>,
val cursor: String?
) {
companion object {
val errorResult = SearchResult(
hasError = true, hasMoreItems = true, cursor = null, count = 0, savedItems = listOf()
)
}
companion object {
val errorResult = SearchResult(hasError = true, hasMoreItems = true, cursor = null, count = 0, savedItems = listOf())
}
}

148
android/Omnivore/app/src/main/java/app/omnivore/omnivore/core/network/SearchQuery.kt
View file

@ -1,90 +1,100 @@
package app.omnivore.omnivore.core.network
import app.omnivore.omnivore.core.data.model.ServerSyncStatus
import app.omnivore.omnivore.core.database.entities.Highlight
import app.omnivore.omnivore.core.database.entities.SavedItem
import app.omnivore.omnivore.core.database.entities.SavedItemLabel
import app.omnivore.omnivore.graphql.generated.SearchQuery
import app.omnivore.omnivore.core.data.model.ServerSyncStatus
import com.apollographql.apollo3.api.Optional
data class LibrarySearchQueryResponse(
val cursor: String?, val items: List<LibrarySearchItem>
val cursor: String?,
val items: List<LibrarySearchItem>
)
data class LibrarySearchItem(
val item: SavedItem, val labels: List<SavedItemLabel>, val highlights: List<Highlight>
val item: SavedItem,
val labels: List<SavedItemLabel>,
val highlights: List<Highlight>
)
suspend fun Networker.search(
cursor: String? = null, limit: Int = 15, query: String
cursor: String? = null,
limit: Int = 15,
query: String
): LibrarySearchQueryResponse {
try {
val result = authenticatedApolloClient().query(
SearchQuery(
after = Optional.presentIfNotNull(cursor),
first = Optional.presentIfNotNull(limit),
query = Optional.presentIfNotNull(query)
)
).execute()
try {
val result = authenticatedApolloClient().query(
SearchQuery(
after = Optional.presentIfNotNull(cursor),
first = Optional.presentIfNotNull(limit),
query = Optional.presentIfNotNull(query)
)
).execute()
val newCursor = result.data?.search?.onSearchSuccess?.pageInfo?.endCursor
val itemList = result.data?.search?.onSearchSuccess?.edges ?: listOf()
val newCursor = result.data?.search?.onSearchSuccess?.pageInfo?.endCursor
val itemList = result.data?.search?.onSearchSuccess?.edges ?: listOf()
val searchItems = itemList.map {
LibrarySearchItem(item = SavedItem(
savedItemId = it.node.id,
title = it.node.title,
createdAt = it.node.createdAt as String,
savedAt = it.node.savedAt as String,
readAt = it.node.readAt as String?,
updatedAt = it.node.updatedAt as String?,
readingProgress = it.node.readingProgressPercent,
readingProgressAnchor = it.node.readingProgressAnchorIndex,
imageURLString = it.node.image,
pageURLString = it.node.url,
descriptionText = it.node.description,
publisherURLString = it.node.originalArticleUrl,
siteName = it.node.siteName,
author = it.node.author,
publishDate = it.node.publishedAt as String?,
slug = it.node.slug,
isArchived = it.node.isArchived,
contentReader = it.node.contentReader.rawValue,
content = null,
wordsCount = it.node.wordsCount,
), labels = (it.node.labels ?: listOf()).map { label ->
SavedItemLabel(
savedItemLabelId = label.labelFields.id,
name = label.labelFields.name,
color = label.labelFields.color,
createdAt = label.labelFields.createdAt as String?,
labelDescription = null
)
}, highlights = (it.node.highlights ?: listOf()).map { highlight ->
Highlight(
highlightId = highlight.highlightFields.id,
type = highlight.highlightFields.type.toString(),
annotation = highlight.highlightFields.annotation,
createdByMe = highlight.highlightFields.createdByMe,
patch = highlight.highlightFields.patch,
prefix = highlight.highlightFields.prefix,
quote = highlight.highlightFields.quote,
serverSyncStatus = ServerSyncStatus.IS_SYNCED.rawValue,
shortId = highlight.highlightFields.shortId,
suffix = highlight.highlightFields.suffix,
updatedAt = highlight.highlightFields.updatedAt as String?,
createdAt = highlight.highlightFields.createdAt as String?,
color = highlight.highlightFields.color,
highlightPositionPercent = highlight.highlightFields.highlightPositionPercent,
highlightPositionAnchorIndex = highlight.highlightFields.highlightPositionAnchorIndex
)
})
val searchItems = itemList.map {
LibrarySearchItem(
item = SavedItem(
savedItemId = it.node.id,
title = it.node.title,
createdAt = it.node.createdAt as String,
savedAt = it.node.savedAt as String,
readAt = it.node.readAt as String?,
updatedAt = it.node.updatedAt as String?,
readingProgress = it.node.readingProgressPercent,
readingProgressAnchor = it.node.readingProgressAnchorIndex,
imageURLString = it.node.image,
pageURLString = it.node.url,
descriptionText = it.node.description,
publisherURLString = it.node.originalArticleUrl,
siteName = it.node.siteName,
author = it.node.author,
publishDate = it.node.publishedAt as String?,
slug = it.node.slug,
isArchived = it.node.isArchived,
contentReader = it.node.contentReader.rawValue,
content = it.node.content,
wordsCount = it.node.wordsCount,
),
labels = (it.node.labels ?: listOf()).map { label ->
SavedItemLabel(
savedItemLabelId = label.labelFields.id,
name = label.labelFields.name,
color = label.labelFields.color,
createdAt = label.labelFields.createdAt as String?,
labelDescription = null
)
},
highlights = (it.node.highlights ?: listOf()).map { highlight ->
Highlight(
highlightId = highlight.highlightFields.id,
type = highlight.highlightFields.type.toString(),
annotation = highlight.highlightFields.annotation,
createdByMe = highlight.highlightFields.createdByMe,
patch = highlight.highlightFields.patch,
prefix = highlight.highlightFields.prefix,
quote = highlight.highlightFields.quote,
serverSyncStatus = ServerSyncStatus.IS_SYNCED.rawValue,
shortId = highlight.highlightFields.shortId,
suffix = highlight.highlightFields.suffix,
updatedAt = highlight.highlightFields.updatedAt as String?,
createdAt = highlight.highlightFields.createdAt as String?,
color = highlight.highlightFields.color,
highlightPositionPercent = highlight.highlightFields.highlightPositionPercent,
highlightPositionAnchorIndex = highlight.highlightFields.highlightPositionAnchorIndex
)
}
return LibrarySearchQueryResponse(
cursor = newCursor, items = searchItems
)
} catch (e: java.lang.Exception) {
return LibrarySearchQueryResponse(null, listOf())
)
}
return LibrarySearchQueryResponse(
cursor = newCursor,
items = searchItems
)
} catch (e: java.lang.Exception) {
return LibrarySearchQueryResponse(null, listOf())
}
}

2
android/Omnivore/gradle/libs.versions.toml
View file

@ -21,7 +21,7 @@ composeMarkdown = "0.3.3"
coreSplashscreen = "1.0.1"
gson = "2.10.1"
hilt = "2.51"
intercom = "15.1.0"
intercom = "15.8.2"
junit4 = "4.13.2"
kotlin = "1.9.22"
ksp = "1.9.22-1.0.18"

37
packages/api/src/apollo.ts
View file

@ -4,16 +4,24 @@
/* eslint-disable @typescript-eslint/no-unsafe-member-access */
/* eslint-disable @typescript-eslint/no-unsafe-assignment */
/* eslint-disable @typescript-eslint/require-await */
import { createPrometheusExporterPlugin } from '@bmatei/apollo-prometheus-exporter'
import { makeExecutableSchema } from '@graphql-tools/schema'
import * as Sentry from '@sentry/node'
import { ContextFunction, PluginDefinition } from 'apollo-server-core'
import { Express } from 'express'
import {
ApolloServerPluginDrainHttpServer,
ContextFunction,
PluginDefinition,
} from 'apollo-server-core'
import { ApolloServer } from 'apollo-server-express'
import { ExpressContext } from 'apollo-server-express/dist/ApolloServer'
import { ApolloServerPlugin } from 'apollo-server-plugin-base'
import { Express } from 'express'
import * as httpContext from 'express-http-context2'
import type http from 'http'
import * as jwt from 'jsonwebtoken'
import { EntityManager } from 'typeorm'
import { promisify } from 'util'
import { ReadingProgressDataSource } from './datasources/reading_progress_data_source'
import { appDataSource } from './data_source'
import { sanitizeDirectiveTransformer } from './directives'
import { env } from './env'
@ -22,17 +30,14 @@ import { functionResolvers } from './resolvers/function_resolvers'
import { ClaimsToSet, RequestContext, ResolverContext } from './resolvers/types'
import ScalarResolvers from './scalars'
import typeDefs from './schema'
import { tracer } from './tracing'
import { getClaimsByToken, setAuthInCookie } from './utils/auth'
import { SetClaimsRole } from './utils/dictionary'
import { logger } from './utils/logger'
import { ReadingProgressDataSource } from './datasources/reading_progress_data_source'
import { createPrometheusExporterPlugin } from '@bmatei/apollo-prometheus-exporter'
import { ApolloServerPlugin } from 'apollo-server-plugin-base'
import {
countDailyServiceUsage,
createServiceUsage,
} from './services/service_usage'
import { tracer } from './tracing'
import { getClaimsByToken, setAuthInCookie } from './utils/auth'
import { SetClaimsRole } from './utils/dictionary'
import { logger } from './utils/logger'
const signToken = promisify(jwt.sign)
const pubsub = createPubSubClient()
@ -100,7 +105,10 @@ const contextFunc: ContextFunction<ExpressContext, ResolverContext> = async ({
return ctx
}
export function makeApolloServer(app: Express): ApolloServer {
export function makeApolloServer(
app: Express,
httpServer: http.Server
): ApolloServer {
let schema = makeExecutableSchema({
resolvers,
typeDefs,
@ -169,7 +177,14 @@ export function makeApolloServer(app: Express): ApolloServer {
const apollo = new ApolloServer({
schema: schema,
context: contextFunc,
plugins: [promExporter, usageLimitPlugin],
plugins: [
// Our httpServer handles incoming requests to our Express app.
// Below, we tell Apollo Server to "drain" this httpServer,
// enabling our servers to shut down gracefully.
ApolloServerPluginDrainHttpServer({ httpServer }),
promExporter,
usageLimitPlugin,
],
formatError: (err) => {
logger.info('server error', err)
Sentry.captureException(err)

196
packages/api/src/jobs/ai/create_digest.ts
View file

@ -1,26 +1,26 @@
import { logger } from '../../utils/logger'
import { v4 as uuid } from 'uuid'
import { OpenAI } from '@langchain/openai'
import { JsonOutputParser } from '@langchain/core/output_parsers'
import { PromptTemplate } from '@langchain/core/prompts'
import { LibraryItem } from '../../entity/library_item'
import { OpenAI } from '@langchain/openai'
import {
htmlToSpeechFile,
SpeechFile,
SSMLOptions,
} from '@omnivore/text-to-speech-handler'
import axios from 'axios'
import showdown from 'showdown'
import yaml from 'yaml'
import { LibraryItem } from '../../entity/library_item'
import { TaskState } from '../../generated/graphql'
import { redisDataSource } from '../../redis_data_source'
import { Digest, writeDigest } from '../../services/digest'
import {
findLibraryItemsByIds,
searchLibraryItems,
} from '../../services/library_item'
import { redisDataSource } from '../../redis_data_source'
import { findDeviceTokensByUserId } from '../../services/user_device_tokens'
import { logger } from '../../utils/logger'
import { htmlToMarkdown } from '../../utils/parser'
import yaml from 'yaml'
import { JsonOutputParser } from '@langchain/core/output_parsers'
import showdown from 'showdown'
import { Digest, writeDigest } from '../../services/digest'
import { TaskState } from '../../generated/graphql'
import { sendMulticastPushNotifications } from '../../utils/sendNotification'
export type CreateDigestJobSchedule = 'daily' | 'weekly'
@ -73,9 +73,18 @@ interface RankedTitle {
}
export const CREATE_DIGEST_JOB = 'create-digest'
export const CRON_PATTERNS = {
// every day at 10:30 UTC
daily: '30 10 * * *',
// every Sunday at 10:30 UTC
weekly: '30 10 * * 7',
}
let digestDefinition: DigestDefinition
export const getCronPattern = (schedule: CreateDigestJobSchedule) =>
CRON_PATTERNS[schedule]
const fetchDigestDefinition = async (): Promise<DigestDefinition> => {
const promptFileUrl = process.env.PROMPT_FILE_URL
if (!promptFileUrl) {
@ -131,7 +140,7 @@ const getPreferencesList = async (userId: string): Promise<LibraryItem[]> => {
// Makes multiple DB queries and combines the results
const getCandidatesList = async (
userId: string,
libraryItemIds?: string[]
selectedLibraryItemIds?: string[]
): Promise<LibraryItem[]> => {
// use the queries from the digest definitions to lookup preferences
// There should be a list of multiple queries we use. For now we can
@ -140,18 +149,25 @@ const getCandidatesList = async (
// count: 100
// reason: "most recent 100 items saved over 500 words
if (libraryItemIds) {
logger.info('Using libraryItemIds')
return findLibraryItemsByIds(libraryItemIds, userId)
if (selectedLibraryItemIds) {
return findLibraryItemsByIds(selectedLibraryItemIds, userId)
}
// get the existing candidate ids from cache
const key = `digest:${userId}:existingCandidateIds`
const existingCandidateIds = await redisDataSource.redisClient?.get(key)
logger.info('existingCandidateIds: ', { existingCandidateIds })
const candidates = await Promise.all(
digestDefinition.candidateSelectors.map(async (selector) => {
// use the selector to fetch items
const results = await searchLibraryItems(
{
includeContent: true,
query: selector.query,
query: existingCandidateIds
? `(${selector.query}) -includes:${existingCandidateIds}` // exclude the existing candidates
: selector.query,
size: selector.count,
},
userId
@ -172,6 +188,23 @@ const getCandidatesList = async (
readableContent: htmlToMarkdown(item.readableContent),
})) // convert the html content to markdown
if (dedupedCandidates.length === 0) {
logger.info('No new candidates found')
if (existingCandidateIds) {
// reuse the existing candidates
const existingIds = existingCandidateIds.split(',')
return findLibraryItemsByIds(existingIds, userId)
}
// return empty array if no existing candidates
return []
}
// store the ids in cache
const candidateIds = dedupedCandidates.map((item) => item.id).join(',')
await redisDataSource.redisClient?.set(key, candidateIds)
return dedupedCandidates
}
@ -203,7 +236,7 @@ const createUserProfile = async (
// it to redis
const findOrCreateUserProfile = async (userId: string): Promise<string> => {
// check redis for user profile, return if found
const key = `userProfile:${userId}`
const key = `digest:${userId}:userProfile`
const existingProfile = await redisDataSource.redisClient?.get(key)
if (existingProfile) {
return existingProfile
@ -266,6 +299,9 @@ const rankCandidates = async (
return rankedItems
}
const filterTopics = (rankedTopics: string[]) =>
rankedTopics.filter((topic) => topic?.length > 0)
// Does some grouping by topic while trying to maintain ranking
// adds some basic topic diversity
const chooseRankedSelections = (rankedCandidates: RankedItem[]) => {
@ -289,7 +325,6 @@ const chooseRankedSelections = (rankedCandidates: RankedItem[]) => {
}
logger.info('rankedTopics: ', rankedTopics)
logger.info('finalSelections: ', selected)
const finalSelections = []
@ -298,9 +333,15 @@ const chooseRankedSelections = (rankedCandidates: RankedItem[]) => {
finalSelections.push(...matches)
}
logger.info('finalSelections: ', finalSelections)
logger.info(
'finalSelections: ',
finalSelections.map((item) => item.libraryItem.title)
)
return { finalSelections, rankedTopics }
return {
finalSelections,
rankedTopics: filterTopics(rankedTopics),
}
}
const summarizeItems = async (
@ -363,7 +404,9 @@ const generateSpeechFiles = (
// we should have a QA step here that does some
// basic checks to make sure the summaries are good.
const filterSummaries = (summaries: RankedItem[]): RankedItem[] => {
return summaries.filter((item) => item.summary.length > 100)
return summaries.filter(
(item) => item.summary.length < item.libraryItem.readableContent.length
)
}
// we can use something more sophisticated to generate titles
@ -376,11 +419,8 @@ const generateDescription = (
summaries: RankedItem[],
rankedTopics: string[]
): string =>
`We selected ${
summaries.length
} articles from your last 24 hours of saved items, covering ${rankedTopics.join(
', '
)}.`
`We selected ${summaries.length} articles from your last 24 hours of saved items` +
(rankedTopics.length ? `, covering ${rankedTopics.join(', ')}.` : '.')
// generate content based on the summaries
const generateContent = (summaries: RankedItem[]): string =>
@ -395,45 +435,77 @@ const generateByline = (summaries: RankedItem[]): string =>
.join(', ')
export const createDigestJob = async (jobData: CreateDigestJobData) => {
digestDefinition = await fetchDigestDefinition()
try {
digestDefinition = await fetchDigestDefinition()
const candidates = await getCandidatesList(
jobData.userId,
jobData.libraryItemIds
)
const userProfile = await findOrCreateUserProfile(jobData.userId)
const rankedCandidates = await rankCandidates(candidates, userProfile)
const { finalSelections, rankedTopics } =
chooseRankedSelections(rankedCandidates)
const candidates = await getCandidatesList(
jobData.userId,
jobData.libraryItemIds
)
if (candidates.length === 0) {
logger.info('No candidates found')
return writeDigest(jobData.userId, {
id: jobData.id,
jobState: TaskState.Succeeded,
title: 'No articles found',
})
}
const summaries = await summarizeItems(finalSelections)
const userProfile = await findOrCreateUserProfile(jobData.userId)
const rankedCandidates = await rankCandidates(candidates, userProfile)
const { finalSelections, rankedTopics } =
chooseRankedSelections(rankedCandidates)
const filteredSummaries = filterSummaries(summaries)
const summaries = await summarizeItems(finalSelections)
const speechFiles = generateSpeechFiles(filteredSummaries, {
...jobData,
primaryVoice: jobData.voices?.[0],
secondaryVoice: jobData.voices?.[1],
})
const title = generateTitle(summaries)
const digest: Digest = {
id: jobData.id,
title,
content: generateContent(summaries),
urlsToAudio: [],
jobState: TaskState.Succeeded,
speechFiles,
chapters: filteredSummaries.map((item, index) => ({
title: item.libraryItem.title,
id: item.libraryItem.id,
url: item.libraryItem.originalUrl,
thumbnail: item.libraryItem.thumbnail ?? undefined,
wordCount: speechFiles[index].wordCount,
})),
createdAt: new Date(),
description: generateDescription(summaries, rankedTopics),
byline: generateByline(summaries),
const filteredSummaries = filterSummaries(summaries)
const speechFiles = generateSpeechFiles(filteredSummaries, {
...jobData,
primaryVoice: jobData.voices?.[0],
secondaryVoice: jobData.voices?.[1],
})
const title = generateTitle(summaries)
const digest: Digest = {
id: jobData.id,
title,
content: generateContent(summaries),
jobState: TaskState.Succeeded,
speechFiles,
chapters: filteredSummaries.map((item, index) => ({
title: item.libraryItem.title,
id: item.libraryItem.id,
url: item.libraryItem.originalUrl,
thumbnail: item.libraryItem.thumbnail ?? undefined,
wordCount: speechFiles[index].wordCount,
})),
createdAt: new Date(),
description: generateDescription(summaries, rankedTopics),
byline: generateByline(summaries),
urlsToAudio: [],
}
await writeDigest(jobData.userId, digest)
} catch (error) {
logger.error('createDigestJob error', error)
await writeDigest(jobData.userId, {
id: jobData.id,
jobState: TaskState.Failed,
})
} finally {
// send notification
const tokens = await findDeviceTokensByUserId(jobData.userId)
if (tokens.length > 0) {
const message = {
notification: {
title: 'Digest ready',
body: 'Your digest is ready to listen',
},
tokens: tokens.map((token) => token.token),
}
await sendMulticastPushNotifications(jobData.userId, message, 'reminder')
}
}
await writeDigest(jobData.userId, digest)
}

7
packages/api/src/jobs/bulk_action.ts
View file

@ -23,9 +23,8 @@ export const bulkAction = async (data: BulkActionData) => {
throw new Error('Queue not initialized')
}
const now = new Date().toISOString()
let offset = 0
do {
for (let offset = 0; offset < count; offset += batchSize) {
const searchArgs = {
size: batchSize,
query: `(${query}) AND updated:*..${now}`, // only process items that have not been updated
@ -36,9 +35,7 @@ export const bulkAction = async (data: BulkActionData) => {
} catch (error) {
logger.error('batch update error', error)
}
offset += batchSize
} while (offset < count)
}
return true
}

10
packages/api/src/jobs/integration/export_all_items.ts
View file

@ -50,9 +50,9 @@ export const exportAllItems = async (jobData: ExportAllItemsJobData) => {
const maxItems = 100
const limit = 10
let offset = 0
let exported = 0
// get max 100 most recent items from the database
while (offset < maxItems) {
for (let offset = 0; offset < maxItems; offset += limit) {
const libraryItems = await findRecentLibraryItems(userId, limit, offset)
if (libraryItems.length === 0) {
logger.info('no library items found', {
@ -92,17 +92,17 @@ export const exportAllItems = async (jobData: ExportAllItemsJobData) => {
updated,
})
offset += libraryItems.length
exported += libraryItems.length
logger.info('exported items', {
...jobData,
offset,
exported,
})
}
logger.info('exported all items', {
...jobData,
offset,
exported,
})
// clear task name in integration

76
packages/api/src/jobs/save_page.ts
View file

@ -11,14 +11,14 @@ import { userRepository } from '../repository/user'
import { saveFile } from '../services/save_file'
import { savePage } from '../services/save_page'
import { uploadFile } from '../services/upload_file'
import { logger } from '../utils/logger'
import { logError, logger } from '../utils/logger'
const signToken = promisify(jwt.sign)
const IMPORTER_METRICS_COLLECTOR_URL = env.queue.importerMetricsUrl
const JWT_SECRET = env.server.jwtSecret
const MAX_ATTEMPTS = 2
const MAX_IMPORT_ATTEMPTS = 1
const REQUEST_TIMEOUT = 30000 // 30 seconds
interface Data {
@ -52,29 +52,32 @@ const uploadToSignedUrl = async (
contentType: string,
contentObjUrl: string
) => {
logger.info('uploading to signed url', {
uploadSignedUrl,
contentType,
const maxContentLength = 10 * 1024 * 1024 // 10MB
logger.info('downloading content', {
contentObjUrl,
})
try {
const stream = await axios.get(contentObjUrl, {
responseType: 'stream',
timeout: REQUEST_TIMEOUT,
})
return await axios.put(uploadSignedUrl, stream.data, {
headers: {
'Content-Type': contentType,
},
maxBodyLength: 1000000000,
maxContentLength: 100000000,
timeout: REQUEST_TIMEOUT,
})
} catch (error) {
logger.error('error uploading to signed url', error)
return null
}
// download the content as stream and max 10MB
const response = await axios.get(contentObjUrl, {
responseType: 'stream',
maxContentLength,
timeout: REQUEST_TIMEOUT,
})
logger.info('uploading to signed url', {
uploadSignedUrl,
contentType,
})
// upload the stream to the signed url
await axios.put(uploadSignedUrl, response.data, {
headers: {
'Content-Type': contentType,
},
maxBodyLength: maxContentLength,
timeout: REQUEST_TIMEOUT,
})
}
const uploadPdf = async (
@ -95,14 +98,13 @@ const uploadPdf = async (
throw new Error('error while getting upload id and signed url')
}
const uploaded = await uploadToSignedUrl(
result.uploadSignedUrl,
'application/pdf',
url
)
if (!uploaded) {
throw new Error('error while uploading pdf')
}
await uploadToSignedUrl(result.uploadSignedUrl, 'application/pdf', url)
logger.info('pdf uploaded successfully', {
url,
uploadFileId: result.id,
itemId: result.createdPageId,
})
return {
uploadFileId: result.id,
@ -134,7 +136,7 @@ const sendImportStatusUpdate = async (
}
)
} catch (e) {
logger.error('error while sending import status update', e)
logError(e)
}
}
@ -268,20 +270,14 @@ export const savePageJob = async (data: Data, attemptsMade: number) => {
isImported = true
isSaved = true
} catch (e) {
if (e instanceof Error) {
logger.error(`error while saving page: ${e.message}`)
} else {
logger.error('error while saving page: unknown error')
}
logError(e)
throw e
} finally {
const lastAttempt = attemptsMade === MAX_ATTEMPTS - 1
if (lastAttempt) {
logger.info(`last attempt reached ${data.url}`)
}
const lastAttempt = attemptsMade + 1 === MAX_IMPORT_ATTEMPTS
if (taskId && (isSaved || lastAttempt)) {
logger.info('sending import status update')
// send import status to update the metrics for importer
await sendImportStatusUpdate(userId, taskId, isImported)
}

34
packages/api/src/queue-processor.ts
View file

@ -19,6 +19,17 @@ import { aiSummarize, AI_SUMMARIZE_JOB_NAME } from './jobs/ai-summarize'
import { createDigestJob, CREATE_DIGEST_JOB } from './jobs/ai/create_digest'
import { bulkAction, BULK_ACTION_JOB_NAME } from './jobs/bulk_action'
import { callWebhook, CALL_WEBHOOK_JOB_NAME } from './jobs/call_webhook'
import {
confirmEmailJob,
CONFIRM_EMAIL_JOB,
forwardEmailJob,
FORWARD_EMAIL_JOB,
saveAttachmentJob,
saveNewsletterJob,
SAVE_ATTACHMENT_JOB,
SAVE_NEWSLETTER_JOB,
} from './jobs/email/inbound_emails'
import { sendEmailJob, SEND_EMAIL_JOB } from './jobs/email/send_email'
import { findThumbnail, THUMBNAIL_JOB } from './jobs/find_thumbnail'
import {
exportAllItems,
@ -37,7 +48,6 @@ import {
import { refreshAllFeeds } from './jobs/rss/refreshAllFeeds'
import { refreshFeed } from './jobs/rss/refreshFeed'
import { savePageJob } from './jobs/save_page'
import { sendEmailJob, SEND_EMAIL_JOB } from './jobs/email/send_email'
import {
syncReadPositionsJob,
SYNC_READ_POSITIONS_JOB_NAME,
@ -54,16 +64,6 @@ import { redisDataSource } from './redis_data_source'
import { CACHED_READING_POSITION_PREFIX } from './services/cached_reading_position'
import { getJobPriority } from './utils/createTask'
import { logger } from './utils/logger'
import {
confirmEmailJob,
CONFIRM_EMAIL_JOB,
forwardEmailJob,
FORWARD_EMAIL_JOB,
saveAttachmentJob,
saveNewsletterJob,
SAVE_ATTACHMENT_JOB,
SAVE_NEWSLETTER_JOB,
} from './jobs/email/inbound_emails'
export const QUEUE_NAME = 'omnivore-backend-queue'
export const JOB_VERSION = 'v001'
@ -188,6 +188,8 @@ export const createWorker = (connection: ConnectionOptions) =>
},
{
connection,
autorun: true, // start processing jobs immediately
lockDuration: 60_000, // 1 minute
}
)
@ -316,6 +318,10 @@ const main = async () => {
console.log('completed job: ', job.jobId)
})
queueEvents.on('failed', async (job) => {
console.log('failed job: ', job.jobId)
})
workerRedisClient.on('error', (error) => {
console.trace('[queue-processor]: redis worker error', { error })
})
@ -337,8 +343,14 @@ const main = async () => {
})
})
await worker.close()
console.log('[queue-processor]: Worker closed')
await redisDataSource.shutdown()
console.log('[queue-processor]: Redis connection closed')
await appDataSource.destroy()
console.log('[queue-processor]: DB connection closed')
process.exit(0)
}

3
packages/api/src/resolvers/article/index.ts
View file

@ -681,7 +681,7 @@ export const searchResolver = authorized<
from: Number(startCursor),
size: first + 1, // fetch one more item to get next cursor
includePending: true,
includeContent: !!params.includeContent,
includeContent: params.includeContent ?? true, // by default include content for offline use for now
includeDeleted: params.query?.includes('in:trash'),
query: params.query,
useFolders: params.query?.includes('use:folders'),
@ -786,6 +786,7 @@ export const updatesSinceResolver = authorized<
size: size + 1, // fetch one more item to get next cursor
includeDeleted: true,
query,
includeContent: true, // by default include content for offline use for now
},
uid
)

28
packages/api/src/routers/auth/auth_router.ts
View file

@ -416,6 +416,7 @@ export function authRouter() {
interface LoginRequest {
email: string
password: string
recaptchaToken?: string
}
function isValidLoginRequest(obj: any): obj is LoginRequest {
return (
@ -430,7 +431,19 @@ export function authRouter() {
`${env.client.url}/auth/email-login?errorCodes=${LoginErrorCode.InvalidCredentials}`
)
}
const { email, password } = req.body
const { email, password, recaptchaToken } = req.body
if (process.env.RECAPTCHA_CHALLENGE_SECRET_KEY) {
const verified =
recaptchaToken && (await verifyChallengeRecaptcha(recaptchaToken))
if (!verified) {
logger.info('recaptcha failed', { recaptchaToken, verified })
return res.redirect(
`${env.client.url}/auth/email-login?errorCodes=UNKNOWN`
)
}
}
try {
const user = await userRepository.findByEmail(email.trim())
if (!user || user.status === StatusType.Deleted) {
@ -514,7 +527,7 @@ export function authRouter() {
const verified =
recaptchaToken && (await verifyChallengeRecaptcha(recaptchaToken))
if (!verified) {
logger.info('recaptcha failed', recaptchaToken, verified)
logger.info('recaptcha failed', { recaptchaToken, verified })
return res.redirect(
`${env.client.url}/auth/email-signup?errorCodes=UNKNOWN`
)
@ -642,6 +655,17 @@ export function authRouter() {
)
}
const captchaToken = req.body.recaptchaToken as string
if (process.env.RECAPTCHA_CHALLENGE_SECRET_KEY) {
const verified = await verifyChallengeRecaptcha(captchaToken)
if (!verified) {
logger.info('recaptcha failed', { captchaToken, verified })
return res.redirect(
`${env.client.url}/auth/forgot-password?errorCodes=UNKNOWN`
)
}
}
try {
const user = await userRepository.findByEmail(email)
if (!user || user.status === StatusType.Deleted) {

12
packages/api/src/routers/digest_router.ts
View file

@ -1,5 +1,6 @@
import cors from 'cors'
import express from 'express'
import { v4 as uuid } from 'uuid'
import { env } from '../env'
import { TaskState } from '../generated/graphql'
import { CreateDigestJobSchedule } from '../jobs/ai/create_digest'
@ -11,7 +12,6 @@ import { getClaimsByToken, getTokenByRequest } from '../utils/auth'
import { corsConfig } from '../utils/corsConfig'
import { enqueueCreateDigest } from '../utils/createTask'
import { logger } from '../utils/logger'
import { v4 as uuid } from 'uuid'
interface Feedback {
digestRating: number
@ -153,15 +153,11 @@ export function digestRouter() {
return res.sendStatus(404)
}
if (digest.jobState === TaskState.Running) {
// if job is running then return job state
return res.send({
jobId: digest.id,
jobState: digest.jobState,
})
if (digest.jobState === TaskState.Failed) {
logger.error(`Digest job failed: ${userId}`)
return res.sendStatus(500)
}
// if job is done then return the digest
return res.send(digest)
} catch (error) {
logger.error('Error while getting digest', error)

23
packages/api/src/server.ts
View file

@ -4,13 +4,12 @@
/* eslint-disable @typescript-eslint/no-misused-promises */
import * as lw from '@google-cloud/logging-winston'
import * as Sentry from '@sentry/node'
import { ApolloServer } from 'apollo-server-express'
import { json, urlencoded } from 'body-parser'
import cookieParser from 'cookie-parser'
import express, { Express } from 'express'
import * as httpContext from 'express-http-context2'
import promBundle from 'express-prom-bundle'
import { createServer, Server } from 'http'
import { createServer } from 'http'
import * as prom from 'prom-client'
import { config, loggers } from 'winston'
import { makeApolloServer } from './apollo'
@ -150,7 +149,8 @@ const main = async (): Promise<void> => {
}
const app = createApp()
const apollo = makeApolloServer(app)
const httpServer = createServer(app)
const apollo = makeApolloServer(app, httpServer)
await apollo.start()
apollo.applyMiddleware({ app, path: '/api/graphql', cors: corsConfig })
@ -159,7 +159,7 @@ const main = async (): Promise<void> => {
const mw = await lw.express.makeMiddleware(mwLogger, transport)
app.use(mw)
const listener = app.listen({ port: PORT }, async () => {
const listener = httpServer.listen({ port: PORT }, async () => {
const logger = buildLogger('app.dispatch')
logger.notice(`🚀 Server ready at ${apollo.graphqlPath}`)
})
@ -176,21 +176,10 @@ const main = async (): Promise<void> => {
const gracefulShutdown = async (signal: string) => {
console.log(`[api]: Received ${signal}, closing server...`)
await apollo.stop()
console.log('[api]: Apollo server stopped')
console.log('[api]: Express server stopped')
console.log('[posthog]: flushing events')
await analytics.shutdownAsync()
console.log('[posthog]: events flushed')
await new Promise<void>((resolve) => {
listener.close((err) => {
console.log('[api]: Express listener closed')
if (err) {
console.log('[api]: error stopping listener', { err })
}
resolve()
})
})
console.log('[api]: Posthog events flushed')
// Shutdown redis before DB because the quit sequence can
// cause appDataSource to get reloaded in the callback

2
packages/api/src/services/features.ts
View file

@ -9,7 +9,7 @@ import { logger } from '../utils/logger'
const MAX_ULTRA_REALISTIC_USERS = 1500
const MAX_YOUTUBE_TRANSCRIPT_USERS = 500
const MAX_NOTION_USERS = 1000
const MAX_AIDIGEST_USERS = 5
const MAX_AIDIGEST_USERS = 10
export enum FeatureName {
AISummaries = 'ai-summaries',

67
packages/api/src/services/library_item.ts
View file

@ -627,7 +627,9 @@ export const buildQuery = (
// select all columns except content
const selects: Select[] = getColumns(libraryItemRepository)
.filter(
(select) => select !== 'readableContent' && select !== 'originalContent'
(select) =>
select !== 'originalContent' && // exclude original content
(args.includeContent || select !== 'readableContent') // exclude content if not requested
)
.map((column) => ({ column: `library_item.${column}` }))
@ -647,20 +649,16 @@ export const buildQuery = (
args.useFolders
)
}
queryBuilder.where('library_item.user_id = :userId', { userId })
// add select
selects.forEach((select, index) => {
if (index === 0) {
queryBuilder.select(select.column, select.alias)
}
queryBuilder.addSelect(select.column, select.alias)
// select must be defined before adding additional selects
index === 0
? queryBuilder.select(select.column, select.alias)
: queryBuilder.addSelect(select.column, select.alias)
})
if (args.includeContent) {
queryBuilder.addSelect('library_item.readableContent')
}
queryBuilder.where('library_item.user_id = :userId', { userId })
if (!args.includePending) {
queryBuilder.andWhere("library_item.state <> 'PROCESSING'")
@ -1117,6 +1115,13 @@ export const batchUpdateLibraryItems = async (
labelIds?: string[] | null,
args?: unknown
) => {
if (!searchArgs.query) {
throw new Error('Search query is required')
}
const searchQuery = parseSearchQuery(searchArgs.query)
const parameters: ObjectLiteral[] = []
const queryString = buildQueryString(searchQuery, parameters)
interface FolderArguments {
folder: string
}
@ -1139,19 +1144,23 @@ export const batchUpdateLibraryItems = async (
const getLibraryItemIds = async (
userId: string,
em: EntityManager
): Promise<{ id: string }[]> => {
em: EntityManager,
forUpdate = false
): Promise<string[]> => {
const queryBuilder = getQueryBuilder(userId, em)
return queryBuilder.select('library_item.id', 'id').getRawMany()
}
if (!searchArgs.query) {
throw new Error('Search query is required')
}
if (forUpdate) {
queryBuilder.setLock('pessimistic_write')
}
const searchQuery = parseSearchQuery(searchArgs.query)
const parameters: ObjectLiteral[] = []
const queryString = buildQueryString(searchQuery, parameters)
const libraryItems = await queryBuilder
.select('library_item.id', 'id')
.take(searchArgs.size)
.skip(searchArgs.from)
.getRawMany<{ id: string }>()
return libraryItems.map((item) => item.id)
}
const now = new Date().toISOString()
// build the script
@ -1174,27 +1183,27 @@ export const batchUpdateLibraryItems = async (
throw new Error('Labels are required for this action')
}
const libraryItems = await authTrx(
const libraryItemIds = await authTrx(
async (tx) => getLibraryItemIds(userId, tx),
undefined,
userId
)
// add labels to library items
for (const libraryItem of libraryItems) {
await addLabelsToLibraryItem(labelIds, libraryItem.id, userId)
for (const libraryItemId of libraryItemIds) {
await addLabelsToLibraryItem(labelIds, libraryItemId, userId)
}
return
}
case BulkActionType.MarkAsRead: {
const libraryItems = await authTrx(
const libraryItemIds = await authTrx(
async (tx) => getLibraryItemIds(userId, tx),
undefined,
userId
)
// update reading progress for library items
for (const libraryItem of libraryItems) {
await markItemAsRead(libraryItem.id, userId)
for (const libraryItemId of libraryItemIds) {
await markItemAsRead(libraryItemId, userId)
}
return
@ -1215,8 +1224,10 @@ export const batchUpdateLibraryItems = async (
}
await authTrx(
async (tx) =>
getQueryBuilder(userId, tx).update(LibraryItem).set(values).execute(),
async (tx) => {
const libraryItemIds = await getLibraryItemIds(userId, tx, true)
await tx.getRepository(LibraryItem).update(libraryItemIds, values)
},
undefined,
userId
)

11
packages/api/src/services/save_email.ts
View file

@ -1,4 +1,8 @@
import { LibraryItem, LibraryItemState } from '../entity/library_item'
import {
DirectionalityType,
LibraryItem,
LibraryItemState,
} from '../entity/library_item'
import { enqueueThumbnailJob } from '../utils/createTask'
import {
cleanUrl,
@ -108,6 +112,11 @@ export const saveEmail = async (
subscription: input.author,
folder: input.folder,
labelNames: labels.map((label) => label.name),
itemLanguage: parseResult.parsedContent?.language,
directionality:
parseResult.parsedContent?.dir?.toLowerCase() === 'rtl'
? DirectionalityType.RTL
: DirectionalityType.LTR, // default to LTR
},
input.userId
)

2
packages/api/src/utils/corsConfig.ts
View file

@ -14,5 +14,7 @@ export const corsConfig = {
env.client.url,
'lsp://logseq.io',
'app://obsidian.md',
'capacitor://localhost',
'http://localhost',
],
}

74
packages/api/src/utils/createTask.ts
View file

@ -22,9 +22,12 @@ import {
CreateDigestJobResponse,
CreateDigestJobSchedule,
CREATE_DIGEST_JOB,
CRON_PATTERNS,
getCronPattern,
} from '../jobs/ai/create_digest'
import { BulkActionData, BULK_ACTION_JOB_NAME } from '../jobs/bulk_action'
import { CallWebhookJobData, CALL_WEBHOOK_JOB_NAME } from '../jobs/call_webhook'
import { SendEmailJobData, SEND_EMAIL_JOB } from '../jobs/email/send_email'
import { THUMBNAIL_JOB } from '../jobs/find_thumbnail'
import { EXPORT_ALL_ITEMS_JOB_NAME } from '../jobs/integration/export_all_items'
import {
@ -42,7 +45,6 @@ import {
REFRESH_ALL_FEEDS_JOB_NAME,
REFRESH_FEED_JOB_NAME,
} from '../jobs/rss/refreshAllFeeds'
import { SendEmailJobData, SEND_EMAIL_JOB } from '../jobs/email/send_email'
import { SYNC_READ_POSITIONS_JOB_NAME } from '../jobs/sync_read_positions'
import { TriggerRuleJobData, TRIGGER_RULE_JOB_NAME } from '../jobs/trigger_rule'
import {
@ -53,13 +55,13 @@ import {
} from '../jobs/update_db'
import { getBackendQueue, JOB_VERSION } from '../queue-processor'
import { redisDataSource } from '../redis_data_source'
import { writeDigest } from '../services/digest'
import { signFeatureToken } from '../services/features'
import { OmnivoreAuthorizationHeader } from './auth'
import { CreateTaskError } from './errors'
import { stringToHash } from './helpers'
import { logger } from './logger'
import { logError, logger } from './logger'
import View = google.cloud.tasks.v2.Task.View
import { writeDigest } from '../services/digest'
// Instantiates a client.
const client = new CloudTasksClient()
@ -104,14 +106,6 @@ export const getJobPriority = (jobName: string): number => {
}
}
const logError = (error: any): void => {
if (axios.isAxiosError(error)) {
logger.error(error.response)
} else {
logger.error(error)
}
}
const createHttpTaskWithToken = async ({
project = process.env.GOOGLE_CLOUD_PROJECT,
queue = env.queue.name,
@ -870,22 +864,22 @@ export const enqueueCreateDigest = async (
throw new Error('No queue found')
}
// enqueue create digest job immediately
const jobId = `${CREATE_DIGEST_JOB}_${data.userId}`
const job = await queue.add(CREATE_DIGEST_JOB, data, {
jobId: data.id, // dedupe by job id
jobId, // dedupe by job id
removeOnComplete: true,
removeOnFail: true,
attempts: 3,
attempts: 1,
priority: getJobPriority(CREATE_DIGEST_JOB),
repeat: schedule
? {
immediately: true, // run immediately
pattern: schedule === 'daily' ? '0 13 * * *' : '0 13 * * 7', // every day or every Sunday at 1PM
utc: true,
}
: undefined,
})
logger.info('create digest job enqueued', { jobId: job.id })
if (!job || !job.id) {
logger.error('Error while enqueuing create digest job', data)
throw new Error('Error while enqueuing create digest job')
}
logger.info('create digest job enqueued', { jobId })
const digest = {
id: data.id,
@ -895,6 +889,44 @@ export const enqueueCreateDigest = async (
// update digest job state in redis
await writeDigest(data.userId, digest)
if (schedule) {
await Promise.all(
Object.keys(CRON_PATTERNS).map(async (key) => {
// remove existing repeated job if any
const isDeleted = await queue.removeRepeatable(
CREATE_DIGEST_JOB,
{
pattern: CRON_PATTERNS[key as keyof typeof CRON_PATTERNS],
tz: 'UTC',
},
jobId
)
if (isDeleted) {
logger.info('existing repeated job removed', { jobId, schedule: key })
}
})
)
// schedule repeated job
const job = await queue.add(CREATE_DIGEST_JOB, data, {
attempts: 1,
priority: getJobPriority(CREATE_DIGEST_JOB),
repeat: {
pattern: getCronPattern(schedule),
jobId,
tz: 'UTC',
},
})
if (!job || !job.id) {
logger.error('Error while scheduling create digest job', data)
throw new Error('Error while scheduling create digest job')
}
logger.info('create digest job scheduled', { jobId, schedule })
}
return {
jobId: digest.id,
jobState: digest.jobState,

2
packages/api/src/utils/highlightGenerator.ts
View file

@ -49,7 +49,7 @@ type FillNodeResponse = {
}
function getTextNodesBetween(rootNode: Node, startNode: Node, endNode: Node) {
const maxTime = 1000 * 60 * 10 // 10 minutes
const maxTime = 1000 * 60 // 60 seconds
const start = Date.now()
let textNodeStartingPoint = 0
let articleText = ''

14
packages/api/src/utils/logger.ts
View file

@ -1,6 +1,7 @@
/* eslint-disable @typescript-eslint/no-explicit-any */
/* eslint-disable @typescript-eslint/restrict-template-expressions */
import { LoggingWinston } from '@google-cloud/logging-winston'
import axios from 'axios'
import jsonStringify from 'fast-safe-stringify'
import { cloneDeep, isArray, isObject, isString, truncate } from 'lodash'
import { DateTime } from 'luxon'
@ -168,6 +169,19 @@ export interface LogRecord {
[key: string]: any
}
export const logError = (error: any): void => {
if (axios.isAxiosError(error)) {
logger.error(error.message, {
response: error.response?.data,
stack: error.stack,
})
} else if (error instanceof Error) {
logger.error(error.message, { stack: error.stack })
} else {
logger.error(error)
}
}
export const logger = buildLogger('app')
export default {}

4
packages/api/test/util.ts
View file

@ -1,4 +1,5 @@
import { ConnectionOptions, Job, QueueEvents, Worker } from 'bullmq'
import { createServer } from 'http'
import { nanoid } from 'nanoid'
import supertest from 'supertest'
import { v4 } from 'uuid'
@ -8,7 +9,8 @@ import { createApp } from '../src/server'
import { corsConfig } from '../src/utils/corsConfig'
const app = createApp()
const apollo = makeApolloServer(app)
const httpServer = createServer(app)
const apollo = makeApolloServer(app, httpServer)
export const request = supertest(app)
let worker: Worker
let queueEvents: QueueEvents

30
packages/content-fetch/src/job.ts
View file

@ -4,9 +4,24 @@ import { redisDataSource } from './redis_data_source'
const QUEUE_NAME = 'omnivore-backend-queue'
const JOB_NAME = 'save-page'
interface savePageJob {
interface SavePageJobData {
userId: string
data: unknown
url: string
finalUrl: string
articleSavingRequestId: string
state?: string
labels?: string[]
source: string
folder?: string
rssFeedUrl?: string
savedAt?: string
publishedAt?: string
taskId?: string
}
interface SavePageJob {
userId: string
data: SavePageJobData
isRss: boolean
isImport: boolean
priority: 'low' | 'high'
@ -16,7 +31,7 @@ const queue = new Queue(QUEUE_NAME, {
connection: redisDataSource.queueRedisClient,
})
const getPriority = (job: savePageJob): number => {
const getPriority = (job: SavePageJob): number => {
// we want to prioritized jobs by the expected time to complete
// lower number means higher priority
// priority 1: jobs that are expected to finish immediately
@ -33,7 +48,7 @@ const getPriority = (job: savePageJob): number => {
return job.priority === 'low' ? 10 : 1
}
const getAttempts = (job: savePageJob): number => {
const getAttempts = (job: SavePageJob): number => {
if (job.isRss || job.isImport) {
// we don't want to retry rss or import jobs
return 1
@ -42,11 +57,8 @@ const getAttempts = (job: savePageJob): number => {
return 3
}
const getOpts = (job: savePageJob): BulkJobOptions => {
const getOpts = (job: SavePageJob): BulkJobOptions => {
return {
// jobId: `${job.userId}-${job.url}`,
// removeOnComplete: true,
// removeOnFail: true,
attempts: getAttempts(job),
priority: getPriority(job),
backoff: {
@ -56,7 +68,7 @@ const getOpts = (job: savePageJob): BulkJobOptions => {
}
}
export const queueSavePageJob = async (savePageJobs: savePageJob[]) => {
export const queueSavePageJob = async (savePageJobs: SavePageJob[]) => {
const jobs = savePageJobs.map((job) => ({
name: JOB_NAME,
data: job.data,

19
packages/web/components/elements/Recaptcha.tsx Normal file
View file

@ -0,0 +1,19 @@
import { GoogleReCaptchaCheckbox } from '@google-recaptcha/react'
type RecaptchaProps = {
setRecaptchaToken: (token: string) => void
}
export const Recaptcha = (props: RecaptchaProps): JSX.Element => {
return (
<>
<GoogleReCaptchaCheckbox
key="recaptcha"
onChange={(token) => {
console.log('recaptcha: ', token)
props.setRecaptchaToken(token)
}}
/>
</>
)
}

56
packages/web/components/templates/AuthLayout.tsx Normal file
View file

@ -0,0 +1,56 @@
import { Box, VStack, HStack } from '../elements/LayoutPrimitives'
import { OmnivoreNameLogo } from '../elements/images/OmnivoreNameLogo'
import { theme } from '../tokens/stitches.config'
import { GoogleReCaptchaProvider } from '@google-recaptcha/react'
type ProfileLayoutProps = {
logoDestination?: string
children: React.ReactNode
}
export function AuthLayout(props: ProfileLayoutProps): JSX.Element {
return (
<>
<VStack
alignment="center"
distribution="center"
css={{
// bg: '$omnivoreYellow',
height: '100vh',
bg: '$omnivoreYellow',
}}
>
{props.children}
</VStack>
<Box
css={{
position: 'absolute',
top: 0,
left: 0,
m: '0',
width: '100%',
}}
>
<HStack
alignment="center"
distribution="between"
css={{
mt: '18px',
ml: '18px',
mr: '0',
'@smDown': {
ml: '8px',
mt: '10px',
},
}}
>
<OmnivoreNameLogo
color={theme.colors.omnivoreGray.toString()}
href={props.logoDestination ?? '/login'}
/>
</HStack>
</Box>
</>
)
}

107
packages/web/components/templates/article/HighlightsLayer.tsx
View file

@ -26,13 +26,12 @@ import { isTouchScreenDevice } from '../../../lib/deviceType'
import { UserBasicData } from '../../../lib/networking/queries/useGetViewerQuery'
import { ReadableItem } from '../../../lib/networking/queries/useGetLibraryItemsQuery'
import { SetHighlightLabelsModalPresenter } from './SetLabelsModalPresenter'
import SlidingPane from 'react-sliding-pane'
import 'react-sliding-pane/dist/react-sliding-pane.css'
import { NotebookContent } from './Notebook'
import { NotebookHeader } from './NotebookHeader'
import useGetWindowDimensions from '../../../lib/hooks/useGetWindowDimensions'
import { ConfirmationModal } from '../../patterns/ConfirmationModal'
import { Resizable } from 're-resizable'
import { ResizableSidebar } from './ResizableSidebar'
type HighlightsLayerProps = {
viewer: UserBasicData
@ -81,15 +80,13 @@ export function HighlightsLayer(props: HighlightsLayerProps): JSX.Element {
const focusedHighlightMousePos = useRef({ pageX: 0, pageY: 0 })
const [currentHighlightIdx, setCurrentHighlightIdx] = useState(0)
const [focusedHighlight, setFocusedHighlight] = useState<
Highlight | undefined
>(undefined)
const [focusedHighlight, setFocusedHighlight] =
useState<Highlight | undefined>(undefined)
const [selectionData, setSelectionData] = useSelection(highlightLocations)
const [labelsTarget, setLabelsTarget] = useState<Highlight | undefined>(
undefined
)
const [labelsTarget, setLabelsTarget] =
useState<Highlight | undefined>(undefined)
const [
confirmDeleteHighlightWithNoteId,
@ -577,7 +574,6 @@ export function HighlightsLayer(props: HighlightsLayerProps): JSX.Element {
selectionData,
setSelectionData,
updateHighlightColor,
confirmDeleteHighlightWithNoteId,
]
)
@ -848,72 +844,41 @@ export function HighlightsLayer(props: HighlightsLayerProps): JSX.Element {
/>
</>
)}
<SlidingPane
className="sliding-pane-class"
isOpen={props.showHighlightsModal}
width="fit-content"
hideHeader={true}
from="right"
overlayClassName="slide-panel-overlay"
onRequestClose={() => {
<ResizableSidebar
isShow={props.showHighlightsModal}
onClose={() => {
props.setShowHighlightsModal(false)
}}
>
<Resizable
onResize={(_e, _direction, ref) => {
if (parseInt(ref.style.width) < 210) {
props.setShowHighlightsModal(false)
}
}}
defaultSize={{
width: windowDimensions.width < 600 ? '100%' : '420px',
height: '100%',
}}
enable={
windowDimensions.width < 600
? false
: {
top: false,
right: false,
bottom: false,
left: true,
topRight: false,
bottomRight: false,
bottomLeft: false,
topLeft: false,
}
}
>
<NotebookHeader
viewer={props.viewer}
item={props.item}
setShowNotebook={props.setShowHighlightsModal}
/>
<NotebookContent
viewer={props.viewer}
item={props.item}
// highlights={highlights}
// onClose={handleCloseNotebook}
viewInReader={(highlightId) => {
// The timeout here is a bit of a hack to work around rerendering
setTimeout(() => {
const target = document.querySelector(
`[omnivore-highlight-id="${highlightId}"]`
)
target?.scrollIntoView({
block: 'center',
behavior: 'auto',
})
}, 1)
history.replaceState(
undefined,
window.location.href,
`#${highlightId}`
<NotebookHeader
viewer={props.viewer}
item={props.item}
setShowNotebook={props.setShowHighlightsModal}
/>
<NotebookContent
viewer={props.viewer}
item={props.item}
// highlights={highlights}
// onClose={handleCloseNotebook}
viewInReader={(highlightId) => {
// The timeout here is a bit of a hack to work around rerendering
setTimeout(() => {
const target = document.querySelector(
`[omnivore-highlight-id="${highlightId}"]`
)
}}
/>
</Resizable>
</SlidingPane>
target?.scrollIntoView({
block: 'center',
behavior: 'auto',
})
}, 1)
history.replaceState(
undefined,
window.location.href,
`#${highlightId}`
)
}}
/>
</ResizableSidebar>
</>
)
}

76
packages/web/components/templates/article/NotebookPresenter.tsx
View file

@ -1,12 +1,11 @@
import { ReadableItem } from '../../../lib/networking/queries/useGetLibraryItemsQuery'
import { UserBasicData } from '../../../lib/networking/queries/useGetViewerQuery'
import SlidingPane from 'react-sliding-pane'
import 'react-sliding-pane/dist/react-sliding-pane.css'
import { NotebookContent } from './Notebook'
import { NotebookHeader } from './NotebookHeader'
import useGetWindowDimensions from '../../../lib/hooks/useGetWindowDimensions'
import { useRouter } from 'next/router'
import { showErrorToast } from '../../../lib/toastHelpers'
import { ResizableSidebar } from './ResizableSidebar'
type NotebookPresenterProps = {
viewer: UserBasicData
@ -18,52 +17,39 @@ type NotebookPresenterProps = {
}
export const NotebookPresenter = (props: NotebookPresenterProps) => {
const windowDimensions = useGetWindowDimensions()
const router = useRouter()
return (
<SlidingPane
className="sliding-pane-class"
isOpen={props.open}
width={windowDimensions.width < 600 ? '100%' : '420px'}
hideHeader={true}
from="right"
overlayClassName="slide-panel-overlay"
onRequestClose={() => {
props.setOpen(false)
}}
>
<>
<NotebookHeader
viewer={props.viewer}
item={props.item}
setShowNotebook={props.setOpen}
/>
<NotebookContent
viewer={props.viewer}
item={props.item}
viewInReader={(highlightId) => {
if (!router || !router.isReady || !props.viewer) {
showErrorToast('Error navigating to highlight')
return
}
router.push(
{
pathname: '/[username]/[slug]',
query: {
username: props.viewer.profile.username,
slug: props.item.slug,
},
hash: highlightId,
<ResizableSidebar isShow={props.open} onClose={() => props.setOpen(false)}>
<NotebookHeader
viewer={props.viewer}
item={props.item}
setShowNotebook={props.setOpen}
/>
<NotebookContent
viewer={props.viewer}
item={props.item}
viewInReader={(highlightId) => {
if (!router || !router.isReady || !props.viewer) {
showErrorToast('Error navigating to highlight')
return
}
router.push(
{
pathname: '/[username]/[slug]',
query: {
username: props.viewer.profile.username,
slug: props.item.slug,
},
`/${props.viewer.profile.username}/${props.item.slug}#${highlightId}`,
{
scroll: false,
}
)
}}
/>
</>
</SlidingPane>
hash: highlightId,
},
`/${props.viewer.profile.username}/${props.item.slug}#${highlightId}`,
{
scroll: false,
}
)
}}
/>
</ResizableSidebar>
)
}

52
packages/web/components/templates/article/PdfArticleContainer.tsx
View file

@ -16,11 +16,11 @@ import { HighlightNoteModal } from './HighlightNoteModal'
import { showErrorToast } from '../../../lib/toastHelpers'
import { DEFAULT_HEADER_HEIGHT } from '../homeFeed/HeaderSpacer'
import { UserBasicData } from '../../../lib/networking/queries/useGetViewerQuery'
import SlidingPane from 'react-sliding-pane'
import 'react-sliding-pane/dist/react-sliding-pane.css'
import { NotebookContent } from './Notebook'
import { NotebookHeader } from './NotebookHeader'
import useWindowDimensions from '../../../lib/hooks/useGetWindowDimensions'
import { ResizableSidebar } from './ResizableSidebar'
export type PdfArticleContainerProps = {
viewer: UserBasicData
@ -35,9 +35,8 @@ export default function PdfArticleContainer(
const containerRef = useRef<HTMLDivElement | null>(null)
const [notebookKey, setNotebookKey] = useState<string>(uuidv4())
const [noteTarget, setNoteTarget] = useState<Highlight | undefined>(undefined)
const [noteTargetPageIndex, setNoteTargetPageIndex] = useState<
number | undefined
>(undefined)
const [noteTargetPageIndex, setNoteTargetPageIndex] =
useState<number | undefined>(undefined)
const highlightsRef = useRef<Highlight[]>([])
const annotationOmnivoreId = (annotation: Annotation): string | undefined => {
@ -595,35 +594,28 @@ export default function PdfArticleContainer(
}}
/>
)}
<SlidingPane
className="sliding-pane-class"
isOpen={props.showHighlightsModal}
width={windowDimensions.width < 600 ? '100%' : '420px'}
hideHeader={true}
from="right"
overlayClassName="slide-panel-overlay"
onRequestClose={() => {
<ResizableSidebar
isShow={props.showHighlightsModal}
onClose={() => {
props.setShowHighlightsModal(false)
}}
>
<>
<NotebookHeader
viewer={props.viewer}
item={props.article}
setShowNotebook={props.setShowHighlightsModal}
/>
<NotebookContent
viewer={props.viewer}
item={props.article}
viewInReader={(highlightId) => {
const event = new CustomEvent('scrollToHighlightId', {
detail: highlightId,
})
document.dispatchEvent(event)
}}
/>
</>
</SlidingPane>
<NotebookHeader
viewer={props.viewer}
item={props.article}
setShowNotebook={props.setShowHighlightsModal}
/>
<NotebookContent
viewer={props.viewer}
item={props.article}
viewInReader={(highlightId) => {
const event = new CustomEvent('scrollToHighlightId', {
detail: highlightId,
})
document.dispatchEvent(event)
}}
/>
</ResizableSidebar>
</Box>
)
}

55
packages/web/components/templates/article/ResizableSidebar.tsx Normal file
View file

@ -0,0 +1,55 @@
import SlidingPane from 'react-sliding-pane'
import { Resizable, ResizeCallback } from 're-resizable'
import useGetWindowDimensions from '../../../lib/hooks/useGetWindowDimensions'
type ResizableSidebarProps = {
isShow: boolean
onClose: () => void
children: React.ReactNode
}
export function ResizableSidebar(props: ResizableSidebarProps): JSX.Element {
const windowDimensions = useGetWindowDimensions()
const handleResize: ResizeCallback = (_e, _direction, ref) => {
if (parseInt(ref.style.width) < 210) {
props.onClose()
}
}
return (
<SlidingPane
className="sliding-pane-class"
isOpen={props.isShow}
width="fit-content"
hideHeader={true}
from="right"
overlayClassName="slide-panel-overlay"
onRequestClose={props.onClose}
>
<Resizable
onResize={handleResize}
defaultSize={{
width: windowDimensions.width < 600 ? '100%' : '420px',
height: '100%',
}}
enable={
windowDimensions.width < 600
? false
: {
top: false,
right: false,
bottom: false,
left: true,
topRight: false,
bottomRight: false,
bottomLeft: false,
topLeft: false,
}
}
>
{props.children}
</Resizable>
</SlidingPane>
)
}

71
packages/web/components/templates/auth/EmailForgotPassword.tsx
View file

@ -1,19 +1,47 @@
import { SpanBox, VStack } from '../../elements/LayoutPrimitives'
import { Button } from '../../elements/Button'
import { StyledText } from '../../elements/StyledText'
import { useEffect, useState } from 'react'
import { useEffect, useRef, useState } from 'react'
import { BorderedFormInput, FormLabel } from '../../elements/FormElements'
import { fetchEndpoint } from '../../../lib/appConfig'
import { logoutMutation } from '../../../lib/networking/mutations/logoutMutation'
import { useRouter } from 'next/router'
import { formatMessage } from '../../../locales/en/messages'
import { parseErrorCodes } from '../../../lib/queryParamParser'
import { Recaptcha } from '../../elements/Recaptcha'
const ForgotPasswordForm = (): JSX.Element => {
const [email, setEmail] = useState<string | undefined>()
return (
<VStack css={{ width: '100%', minWidth: '320px', gap: '16px', pb: '16px' }}>
<SpanBox css={{ width: '100%' }}>
<FormLabel>Email</FormLabel>
<BorderedFormInput
key="email"
type="email"
name="email"
value={email}
placeholder="Email"
autoFocus={true}
css={{ bg: 'white', color: 'black' }}
onChange={(e) => {
e.preventDefault()
setEmail(e.target.value)
}}
/>
</SpanBox>
</VStack>
)
}
export function EmailForgotPassword(): JSX.Element {
const router = useRouter()
const [email, setEmail] = useState<string>('')
const [errorMessage, setErrorMessage] =
useState<string | undefined>(undefined)
const [errorMessage, setErrorMessage] = useState<string | undefined>(
undefined
)
const recaptchaTokenRef = useRef<HTMLInputElement>(null)
useEffect(() => {
if (!router.isReady) return
@ -42,26 +70,27 @@ export function EmailForgotPassword(): JSX.Element {
<StyledText style="subHeadline" css={{ color: '$omnivoreGray' }}>
Reset your password
</StyledText>
<VStack
css={{ width: '100%', minWidth: '320px', gap: '16px', pb: '16px' }}
>
<SpanBox css={{ width: '100%' }}>
<FormLabel>Email</FormLabel>
<BorderedFormInput
key="email"
type="email"
name="email"
value={email}
placeholder="Email"
autoFocus={true}
css={{ bg: 'white', color: 'black' }}
onChange={(e) => {
e.preventDefault()
setEmail(e.target.value)
<ForgotPasswordForm />
{process.env.NEXT_PUBLIC_RECAPTCHA_CHALLENGE_SITE_KEY && (
<>
<Recaptcha
setRecaptchaToken={(token) => {
if (recaptchaTokenRef.current) {
recaptchaTokenRef.current.value = token
} else {
console.log('error updating recaptcha token')
}
}}
/>
</SpanBox>
</VStack>
<input
ref={recaptchaTokenRef}
type="hidden"
name="recaptchaToken"
/>
</>
)}
{errorMessage && <StyledText style="error">{errorMessage}</StyledText>}
<Button type="submit" style="ctaDarkYellow" css={{ my: '$2' }}>

100
packages/web/components/templates/auth/EmailLogin.tsx
View file

@ -1,7 +1,7 @@
import { HStack, SpanBox, VStack } from '../../elements/LayoutPrimitives'
import { Button } from '../../elements/Button'
import { StyledText, StyledTextSpan } from '../../elements/StyledText'
import { useEffect, useState } from 'react'
import { useEffect, useRef, useState } from 'react'
import { BorderedFormInput, FormLabel } from '../../elements/FormElements'
import { fetchEndpoint } from '../../../lib/appConfig'
import { logoutMutation } from '../../../lib/networking/mutations/logoutMutation'
@ -9,13 +9,53 @@ import { useRouter } from 'next/router'
import { parseErrorCodes } from '../../../lib/queryParamParser'
import { formatMessage } from '../../../locales/en/messages'
import Link from 'next/link'
import { Recaptcha } from '../../elements/Recaptcha'
const LoginForm = (): JSX.Element => {
const [email, setEmail] = useState<string | undefined>()
const [password, setPassword] = useState<string | undefined>()
return (
<VStack css={{ width: '100%', minWidth: '320px', gap: '16px', pb: '16px' }}>
<SpanBox css={{ width: '100%' }}>
<FormLabel>Email</FormLabel>
<BorderedFormInput
autoFocus={true}
key="email"
type="email"
name="email"
value={email}
placeholder="Email"
css={{ backgroundColor: 'white', color: 'black' }}
onChange={(e) => {
e.preventDefault()
setEmail(e.target.value)
}}
/>
</SpanBox>
<SpanBox css={{ width: '100%' }}>
<FormLabel>Password</FormLabel>
<BorderedFormInput
key="password"
type="password"
name="password"
value={password}
placeholder="Password"
css={{ bg: 'white', color: 'black' }}
onChange={(e) => setPassword(e.target.value)}
/>
</SpanBox>
</VStack>
)
}
export function EmailLogin(): JSX.Element {
const router = useRouter()
const [email, setEmail] = useState<string | undefined>(undefined)
const [password, setPassword] = useState<string | undefined>(undefined)
const [errorMessage, setErrorMessage] =
useState<string | undefined>(undefined)
const [errorMessage, setErrorMessage] = useState<string | undefined>(
undefined
)
const recaptchaTokenRef = useRef<HTMLInputElement>(null)
useEffect(() => {
if (!router.isReady) return
@ -44,39 +84,27 @@ export function EmailLogin(): JSX.Element {
<StyledText style="subHeadline" css={{ color: '$omnivoreGray' }}>
Login
</StyledText>
<VStack
css={{ width: '100%', minWidth: '320px', gap: '16px', pb: '16px' }}
>
<SpanBox css={{ width: '100%' }}>
<FormLabel>Email</FormLabel>
<BorderedFormInput
autoFocus={true}
key="email"
type="email"
name="email"
value={email}
placeholder="Email"
css={{ backgroundColor: 'white', color: 'black' }}
onChange={(e) => {
e.preventDefault()
setEmail(e.target.value)
<LoginForm />
{process.env.NEXT_PUBLIC_RECAPTCHA_CHALLENGE_SITE_KEY && (
<>
<Recaptcha
setRecaptchaToken={(token) => {
if (recaptchaTokenRef.current) {
recaptchaTokenRef.current.value = token
} else {
console.log('error updating recaptcha token')
}
}}
/>
</SpanBox>
<SpanBox css={{ width: '100%' }}>
<FormLabel>Password</FormLabel>
<BorderedFormInput
key="password"
type="password"
name="password"
value={password}
placeholder="Password"
css={{ bg: 'white', color: 'black' }}
onChange={(e) => setPassword(e.target.value)}
<input
ref={recaptchaTokenRef}
type="hidden"
name="recaptchaToken"
/>
</SpanBox>
</VStack>
</>
)}
{errorMessage && <StyledText style="error">{errorMessage}</StyledText>}
@ -148,5 +176,5 @@ export function EmailLogin(): JSX.Element {
</StyledText>
</VStack>
</form>
);
)
}

28
packages/web/components/templates/auth/EmailSignup.tsx
View file

@ -10,19 +10,17 @@ import { logoutMutation } from '../../../lib/networking/mutations/logoutMutation
import { useRouter } from 'next/router'
import { formatMessage } from '../../../locales/en/messages'
import { parseErrorCodes } from '../../../lib/queryParamParser'
import {
GoogleReCaptchaProvider,
GoogleReCaptchaCheckbox,
} from '@google-recaptcha/react'
import Link from 'next/link'
import { Recaptcha } from '../../elements/Recaptcha'
const SignUpForm = (): JSX.Element => {
const [email, setEmail] = useState<string | undefined>()
const [password, setPassword] = useState<string | undefined>()
const [fullname, setFullname] = useState<string | undefined>()
const [username, setUsername] = useState<string | undefined>()
const [debouncedUsername, setDebouncedUsername] =
useState<string | undefined>()
const [debouncedUsername, setDebouncedUsername] = useState<
string | undefined
>()
const { isUsernameValid, usernameErrorMessage } = useValidateUsernameQuery({
username: debouncedUsername ?? '',
@ -126,24 +124,6 @@ const SignUpForm = (): JSX.Element => {
)
}
type RecaptchaProps = {
setRecaptchaToken: (token: string) => void
}
const Recaptcha = (props: RecaptchaProps): JSX.Element => {
return (
<>
<GoogleReCaptchaCheckbox
key="recaptcha"
onChange={(token) => {
console.log('recaptcha: ', token)
props.setRecaptchaToken(token)
}}
/>
</>
)
}
export function EmailSignup(): JSX.Element {
const router = useRouter()
const recaptchaTokenRef = useRef<HTMLInputElement>(null)

17
packages/web/lib/networking/queries/search.tsx
View file

@ -5,15 +5,27 @@ import { LibraryItemsData } from './useGetLibraryItemsQuery'
export type LibraryItemsQueryInput = {
limit?: number
searchQuery?: string
includeContent?: boolean
}
export async function searchQuery({
limit = 10,
searchQuery,
includeContent = false,
}: LibraryItemsQueryInput): Promise<LibraryItemsData | undefined> {
const query = gql`
query Search($after: String, $first: Int, $query: String) {
search(first: $first, after: $after, query: $query) {
query Search(
$after: String
$first: Int
$query: String
$includeContent: Boolean
) {
search(
first: $first
after: $after
query: $query
includeContent: $includeContent
) {
... on SearchSuccess {
edges {
cursor
@ -69,6 +81,7 @@ export async function searchQuery({
const variables = {
first: limit,
query: searchQuery,
includeContent,
}
try {

31
packages/web/lib/networking/queries/useGetLibraryItemsQuery.tsx
View file

@ -1,20 +1,18 @@
import { gql } from 'graphql-request'
import useSWRInfinite from 'swr/infinite'
import { gqlFetcher } from '../networkHelpers'
import { PageType, State } from '../fragments/articleFragment'
import { ContentReader } from '../fragments/articleFragment'
import { setLinkArchivedMutation } from '../mutations/setLinkArchivedMutation'
import { deleteLinkMutation } from '../mutations/deleteLinkMutation'
import { unsubscribeMutation } from '../mutations/unsubscribeMutation'
import { articleReadingProgressMutation } from '../mutations/articleReadingProgressMutation'
import { Label } from './../fragments/labelFragment'
import {
showErrorToast,
showSuccessToast,
showSuccessToastWithUndo,
} from '../../toastHelpers'
import { ContentReader, PageType, State } from '../fragments/articleFragment'
import { Highlight, highlightFragment } from '../fragments/highlightFragment'
import { articleReadingProgressMutation } from '../mutations/articleReadingProgressMutation'
import { deleteLinkMutation } from '../mutations/deleteLinkMutation'
import { setLinkArchivedMutation } from '../mutations/setLinkArchivedMutation'
import { updatePageMutation } from '../mutations/updatePageMutation'
import { gqlFetcher } from '../networkHelpers'
import { Label } from './../fragments/labelFragment'
export interface ReadableItem {
id: string
@ -27,6 +25,7 @@ export type LibraryItemsQueryInput = {
sortDescending: boolean
searchQuery?: string
cursor?: string
includeContent?: boolean
}
type LibraryItemsQueryResponse = {
@ -148,10 +147,21 @@ export function useGetLibraryItemsQuery({
sortDescending,
searchQuery,
cursor,
includeContent = false,
}: LibraryItemsQueryInput): LibraryItemsQueryResponse {
const query = gql`
query Search($after: String, $first: Int, $query: String) {
search(first: $first, after: $after, query: $query) {
query Search(
$after: String
$first: Int
$query: String
$includeContent: Boolean
) {
search(
first: $first
after: $after
query: $query
includeContent: $includeContent
) {
... on SearchSuccess {
edges {
cursor
@ -227,6 +237,7 @@ export function useGetLibraryItemsQuery({
after: cursor,
first: limit,
query: searchQuery,
includeContent,
}
const { data, error, mutate, size, setSize, isValidating } = useSWRInfinite(

4
packages/web/next.config.js
View file

@ -5,9 +5,9 @@ const ContentSecurityPolicy = `
font-src 'self' data: https://cdn.jsdelivr.net https://js.intercomcdn.com https://fonts.intercomcdn.com;
form-action 'self' ${process.env.NEXT_PUBLIC_SERVER_BASE_URL} https://getpocket.com/auth/authorize https://intercom.help https://api-iam.intercom.io https://api-iam.eu.intercom.io https://api-iam.au.intercom.io https://www.notion.so https://api.notion.com;
frame-ancestors 'none';
frame-src 'self' https://accounts.google.com https://platform.twitter.com https://www.youtube.com https://www.youtube-nocookie.com https://www.google.com/recaptcha/ https://recaptcha.google.com/recaptcha/;
frame-src 'self' https://accounts.google.com https://platform.twitter.com https://www.youtube.com https://www.youtube-nocookie.com https://www.google.com/recaptcha/ https://recaptcha.google.com/recaptcha/ https://www.recaptcha.net;
manifest-src 'self';
script-src 'self' 'unsafe-inline' 'unsafe-eval' accounts.google.com https://widget.intercom.io https://js.intercomcdn.com https://platform.twitter.com https://cdnjs.cloudflare.com https://cdn.jsdelivr.net https://www.google.com/recaptcha/ https://www.gstatic.com/recaptcha/;
script-src 'self' 'unsafe-inline' 'unsafe-eval' accounts.google.com https://widget.intercom.io https://js.intercomcdn.com https://platform.twitter.com https://cdnjs.cloudflare.com https://cdn.jsdelivr.net https://www.google.com/recaptcha/ https://www.gstatic.com/recaptcha/ https://www.recaptcha.net https://www.gstatic.cn/;
style-src 'self' 'unsafe-inline' https://accounts.google.com https://cdnjs.cloudflare.com;
img-src 'self' blob: data: https:;
worker-src 'self' blob:;

35
packages/web/pages/_app.tsx
View file

@ -22,6 +22,8 @@ import {
import { updateTheme } from '../lib/themeUpdater'
import { ThemeId } from '../components/tokens/stitches.config'
import { posthog } from 'posthog-js'
import { GoogleReCaptchaProvider } from '@google-recaptcha/react'
import { Recaptcha } from '../components/elements/Recaptcha'
TopBarProgress.config({
barColors: {
@ -79,19 +81,26 @@ export function OmnivoreApp({ Component, pageProps }: AppProps): JSX.Element {
}, [router.events])
return (
<KBarProvider actions={generateActions(router)}>
<KBarPortal>
<KBarPositioner style={{ zIndex: 100 }}>
<KBarAnimator style={animatorStyle}>
<KBarSearch style={searchStyle} />
<KBarResultsComponents />
</KBarAnimator>
</KBarPositioner>
</KBarPortal>
<IdProvider>
<Component {...pageProps} />
</IdProvider>
</KBarProvider>
<GoogleReCaptchaProvider
type="v2-checkbox"
isEnterprise={true}
host="recaptcha.net"
siteKey={process.env.NEXT_PUBLIC_RECAPTCHA_CHALLENGE_SITE_KEY ?? ''}
>
<KBarProvider actions={generateActions(router)}>
<KBarPortal>
<KBarPositioner style={{ zIndex: 100 }}>
<KBarAnimator style={animatorStyle}>
<KBarSearch style={searchStyle} />
<KBarResultsComponents />
</KBarAnimator>
</KBarPositioner>
</KBarPortal>
<IdProvider>
<Component {...pageProps} />
</IdProvider>
</KBarProvider>
</GoogleReCaptchaProvider>
)
}

10
packages/web/pages/auth/email-login.tsx
View file

@ -1,15 +1,13 @@
import { PageMetaData } from '../../components/patterns/PageMetaData'
import { ProfileLayout } from '../../components/templates/ProfileLayout'
import { AuthLayout } from '../../components/templates/AuthLayout'
import { EmailLogin } from '../../components/templates/auth/EmailLogin'
export default function EmailLoginPage(): JSX.Element {
return (
<>
<AuthLayout>
<PageMetaData title="Login - Omnivore" path="/email-login" />
<ProfileLayout>
<EmailLogin />
</ProfileLayout>
<EmailLogin />
<div data-testid="email-login-page-tag" />
</>
</AuthLayout>
)
}

22
packages/web/pages/auth/email-signup.tsx
View file

@ -1,24 +1,12 @@
import { PageMetaData } from '../../components/patterns/PageMetaData'
import { ProfileLayout } from '../../components/templates/ProfileLayout'
import { AuthLayout } from '../../components/templates/AuthLayout'
import { EmailSignup } from '../../components/templates/auth/EmailSignup'
import { GoogleReCaptchaProvider } from '@google-recaptcha/react'
export default function EmailRegistrationPage(): JSX.Element {
return (
<>
<GoogleReCaptchaProvider
type="v2-checkbox"
isEnterprise={true}
siteKey={process.env.NEXT_PUBLIC_RECAPTCHA_CHALLENGE_SITE_KEY ?? ''}
>
<PageMetaData
title="Sign up with Email - Omnivore"
path="/auth-signup"
/>
<ProfileLayout>
<EmailSignup />
</ProfileLayout>
</GoogleReCaptchaProvider>
</>
<AuthLayout>
<PageMetaData title="Sign up with Email - Omnivore" path="/auth-signup" />
<EmailSignup />
</AuthLayout>
)
}

10
packages/web/pages/auth/forgot-password.tsx
View file

@ -1,11 +1,11 @@
import { PageMetaData } from '../../components/patterns/PageMetaData'
import { ProfileLayout } from '../../components/templates/ProfileLayout'
import { AuthLayout } from '../../components/templates/AuthLayout'
import { EmailForgotPassword } from '../../components/templates/auth/EmailForgotPassword'
import { Toaster } from 'react-hot-toast'
export default function ForgotPassword(): JSX.Element {
return (
<>
<AuthLayout>
<PageMetaData
title="Reset your password - Omnivore"
path="/auth-forgot-password"
@ -15,10 +15,8 @@ export default function ForgotPassword(): JSX.Element {
top: '5rem',
}}
/>
<ProfileLayout>
<EmailForgotPassword />
</ProfileLayout>
<EmailForgotPassword />
<div data-testid="auth-forgot-password-page-tag" />
</>
</AuthLayout>
)
}