diff --git a/packages/api/src/generated/graphql.ts b/packages/api/src/generated/graphql.ts index 92e724e37..52ebd6f6f 100644 --- a/packages/api/src/generated/graphql.ts +++ b/packages/api/src/generated/graphql.ts @@ -2707,6 +2707,7 @@ export type SubscribeError = { export enum SubscribeErrorCode { AlreadySubscribed = 'ALREADY_SUBSCRIBED', BadRequest = 'BAD_REQUEST', + ExceededMaxSubscriptions = 'EXCEEDED_MAX_SUBSCRIPTIONS', NotFound = 'NOT_FOUND', Unauthorized = 'UNAUTHORIZED' } diff --git a/packages/api/src/generated/schema.graphql b/packages/api/src/generated/schema.graphql index 2f967cf8e..9e8474872 100644 --- a/packages/api/src/generated/schema.graphql +++ b/packages/api/src/generated/schema.graphql @@ -2054,6 +2054,7 @@ type SubscribeError { enum SubscribeErrorCode { ALREADY_SUBSCRIBED BAD_REQUEST + EXCEEDED_MAX_SUBSCRIPTIONS NOT_FOUND UNAUTHORIZED } diff --git a/packages/api/src/resolvers/subscriptions/index.ts b/packages/api/src/resolvers/subscriptions/index.ts index fa556cf3e..a74c49a81 100644 --- a/packages/api/src/resolvers/subscriptions/index.ts +++ b/packages/api/src/resolvers/subscriptions/index.ts @@ -25,6 +25,7 @@ import { UpdateSubscriptionErrorCode, UpdateSubscriptionSuccess, } from '../../generated/graphql' +import { AppDataSource } from '../../server' import { getSubscribeHandler, unsubscribe } from '../../services/subscriptions' import { Merge } from '../../util' import { analytics } from '../../utils/analytics' @@ -234,20 +235,34 @@ export const subscribeResolver = authorized< // validate rss feed const feed = await parser.parseURL(input.url) - const newSubscription = await getRepository(Subscription).save({ - name: feed.title, - url: input.url, - user: { id: uid }, - type: SubscriptionType.Rss, - description: feed.description, - icon: feed.image?.url, - }) + // limit number of rss subscriptions to 20 + const newSubscriptions = (await AppDataSource.query( + `insert into omnivore.subscriptions (name, url, description, type, user_id, icon) + select $1, $2, $3, $4, $5, $6 from omnivore.subscriptions + where user_id = $5 and type = 'RSS' and status = 'ACTIVE' + having count(*) < 20 + returning *;`, + [ + feed.title, + input.url, + feed.description || null, + SubscriptionType.Rss, + uid, + feed.image?.url || null, + ] + )) as Subscription[] + + if (newSubscriptions.length === 0) { + return { + errorCodes: [SubscribeErrorCode.ExceededMaxSubscriptions], + } + } // create a cloud task to fetch rss feed item for the new subscription - await enqueueRssFeedFetch(newSubscription) + await enqueueRssFeedFetch(uid, newSubscriptions[0]) return { - subscriptions: [newSubscription], + subscriptions: newSubscriptions, } } diff --git a/packages/api/src/routers/svc/rss_feed.ts b/packages/api/src/routers/svc/rss_feed.ts index 337fe79bb..54ff86ae9 100644 --- a/packages/api/src/routers/svc/rss_feed.ts +++ b/packages/api/src/routers/svc/rss_feed.ts @@ -35,7 +35,7 @@ export function rssFeedRouter() { await Promise.all( subscriptions.map((subscription) => { try { - return enqueueRssFeedFetch(subscription) + return enqueueRssFeedFetch(subscription.user.id, subscription) } catch (error) { console.log('error creating rss feed fetch task', error) } diff --git a/packages/api/src/schema.ts b/packages/api/src/schema.ts index 03deee21b..7d900e0c8 100755 --- a/packages/api/src/schema.ts +++ b/packages/api/src/schema.ts @@ -1694,6 +1694,7 @@ const schema = gql` BAD_REQUEST NOT_FOUND ALREADY_SUBSCRIBED + EXCEEDED_MAX_SUBSCRIPTIONS } union AddPopularReadResult = AddPopularReadSuccess | AddPopularReadError diff --git a/packages/api/src/utils/createTask.ts b/packages/api/src/utils/createTask.ts index 220276ebd..4982c7384 100644 --- a/packages/api/src/utils/createTask.ts +++ b/packages/api/src/utils/createTask.ts @@ -567,6 +567,7 @@ export const enqueueThumbnailTask = async ( } export const enqueueRssFeedFetch = async ( + userId: string, rssFeedSubscription: Subscription ): Promise => { const { GOOGLE_CLOUD_PROJECT } = process.env @@ -577,9 +578,7 @@ export const enqueueRssFeedFetch = async ( } const headers = { - [OmnivoreAuthorizationHeader]: generateVerificationToken( - rssFeedSubscription.user.id - ), + [OmnivoreAuthorizationHeader]: generateVerificationToken(userId), } // If there is no Google Cloud Project Id exposed, it means that we are in local environment diff --git a/packages/rss-handler/src/index.ts b/packages/rss-handler/src/index.ts index b5f1ef38f..49abc27a3 100644 --- a/packages/rss-handler/src/index.ts +++ b/packages/rss-handler/src/index.ts @@ -2,7 +2,7 @@ import * as Sentry from '@sentry/serverless' import axios from 'axios' import * as dotenv from 'dotenv' // see https://github.com/motdotla/dotenv#how-do-i-use-dotenv-with-import import * as jwt from 'jsonwebtoken' -import Parser from 'rss-parser' +import Parser, { Item } from 'rss-parser' import { promisify } from 'util' import { CONTENT_FETCH_URL, createCloudTask } from './task' @@ -12,11 +12,6 @@ interface RssFeedRequest { lastFetchedAt: number // unix timestamp in milliseconds } -interface ValidRssFeedItem { - link: string - isoDate?: string -} - function isRssFeedRequest(body: any): body is RssFeedRequest { return ( 'subscriptionId' in body && 'feedUrl' in body && 'lastFetchedAt' in body @@ -86,7 +81,7 @@ const sendUpdateSubscriptionMutation = async ( const createSavingItemTask = async ( userId: string, feedUrl: string, - item: ValidRssFeedItem + item: Item ) => { const input = { userId, @@ -156,7 +151,7 @@ export const rssHandler = Sentry.GCPFunction.wrapHttpFunction( console.log('Processing feed', feedUrl, lastFetchedAt) let lastItemFetchedAt: Date | null = null - let lastValidItem: ValidRssFeedItem | null = null + let lastValidItem: Item | null = null // fetch feed const feed = await parser.parseURL(feedUrl) @@ -171,27 +166,26 @@ export const rssHandler = Sentry.GCPFunction.wrapHttpFunction( continue } + const publishedAt = item.isoDate ? new Date(item.isoDate) : new Date() // remember the last valid item - lastValidItem = { - link: item.link, - isoDate: item.isoDate, + if ( + !lastValidItem || + (lastValidItem.isoDate && + publishedAt > new Date(lastValidItem.isoDate)) + ) { + lastValidItem = item } // skip old items and items that were published before 24h - const publishedAt = item.isoDate ? new Date(item.isoDate) : new Date() if ( publishedAt < new Date(lastFetchedAt) || publishedAt < new Date(Date.now() - 24 * 60 * 60 * 1000) ) { - console.log('Skipping old feed item', lastValidItem.link) + console.log('Skipping old feed item', item.link) continue } - const created = await createSavingItemTask( - userId, - feedUrl, - lastValidItem - ) + const created = await createSavingItemTask(userId, feedUrl, item) if (!created) { console.error('Failed to create task for feed item', item.link) continue diff --git a/packages/web/locales/en/messages.ts b/packages/web/locales/en/messages.ts index 9b505ffe4..247e25297 100644 --- a/packages/web/locales/en/messages.ts +++ b/packages/web/locales/en/messages.ts @@ -28,6 +28,7 @@ const errorMessages: Record = { 'error.INVALID_PASSWORD': 'Invalid password. Password must be at least 8 chars.', 'error.ALREADY_SUBSCRIBED': 'You are already subscribed to this feed', 'error.BAD_REQUEST': 'Bad request', + 'error.EXCEEDED_MAX_SUBSCRIPTIONS': 'Exceeded max subscriptions', } const loginPageMessages: Record = {