From a7773bf00bc25334f94fe14139d3b06ea1279c96 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Tue, 4 Jul 2023 22:13:31 +0800 Subject: [PATCH 01/19] test rss-parse --- packages/rss-handler/.dockerignore | 5 ++++ packages/rss-handler/.eslintignore | 2 ++ packages/rss-handler/.eslintrc | 6 +++++ packages/rss-handler/.gcloudignore | 16 +++++++++++ packages/rss-handler/Dockerfile | 26 ++++++++++++++++++ packages/rss-handler/mocha-config.json | 5 ++++ packages/rss-handler/package.json | 30 +++++++++++++++++++++ packages/rss-handler/src/index.ts | 30 +++++++++++++++++++++ packages/rss-handler/test/babel-register.js | 3 +++ packages/rss-handler/test/stub.test.ts | 8 ++++++ packages/rss-handler/tsconfig.json | 8 ++++++ yarn.lock | 18 ++++++++++++- 12 files changed, 156 insertions(+), 1 deletion(-) create mode 100644 packages/rss-handler/.dockerignore create mode 100644 packages/rss-handler/.eslintignore create mode 100644 packages/rss-handler/.eslintrc create mode 100644 packages/rss-handler/.gcloudignore create mode 100644 packages/rss-handler/Dockerfile create mode 100644 packages/rss-handler/mocha-config.json create mode 100644 packages/rss-handler/package.json create mode 100644 packages/rss-handler/src/index.ts create mode 100644 packages/rss-handler/test/babel-register.js create mode 100644 packages/rss-handler/test/stub.test.ts create mode 100644 packages/rss-handler/tsconfig.json diff --git a/packages/rss-handler/.dockerignore b/packages/rss-handler/.dockerignore new file mode 100644 index 000000000..d8aea4ee6 --- /dev/null +++ b/packages/rss-handler/.dockerignore @@ -0,0 +1,5 @@ +node_modules +build +.env* +Dockerfile +.dockerignore diff --git a/packages/rss-handler/.eslintignore b/packages/rss-handler/.eslintignore new file mode 100644 index 000000000..b38db2f29 --- /dev/null +++ b/packages/rss-handler/.eslintignore @@ -0,0 +1,2 @@ +node_modules/ +build/ diff --git a/packages/rss-handler/.eslintrc b/packages/rss-handler/.eslintrc new file mode 100644 index 000000000..e006282a6 --- /dev/null +++ b/packages/rss-handler/.eslintrc @@ -0,0 +1,6 @@ +{ + "extends": "../../.eslintrc", + "parserOptions": { + "project": "tsconfig.json" + } +} \ No newline at end of file diff --git a/packages/rss-handler/.gcloudignore b/packages/rss-handler/.gcloudignore new file mode 100644 index 000000000..ccc4eb240 --- /dev/null +++ b/packages/rss-handler/.gcloudignore @@ -0,0 +1,16 @@ +# This file specifies files that are *not* uploaded to Google Cloud Platform +# using gcloud. It follows the same syntax as .gitignore, with the addition of +# "#!include" directives (which insert the entries of the given .gitignore-style +# file at that point). +# +# For more information, run: +# $ gcloud topic gcloudignore +# +.gcloudignore +# If you would like to upload your .git directory, .gitignore file or files +# from your .gitignore file, remove the corresponding line +# below: +.git +.gitignore + +node_modules diff --git a/packages/rss-handler/Dockerfile b/packages/rss-handler/Dockerfile new file mode 100644 index 000000000..8411d67c6 --- /dev/null +++ b/packages/rss-handler/Dockerfile @@ -0,0 +1,26 @@ +FROM node:14.18-alpine + +# Run everything after as non-privileged user. +WORKDIR /app + +COPY package.json . +COPY yarn.lock . +COPY tsconfig.json . +COPY .eslintrc . + +COPY /packages/rss-handler/package.json ./packages/rss-handler/package.json + +RUN yarn install --pure-lockfile + +ADD /packages/rss-handler ./packages/rss-handler +RUN yarn workspace @omnivore/rss-handler build + +# After building, fetch the production dependencies +RUN rm -rf /app/packages/rss-handler/node_modules +RUN rm -rf /app/node_modules +RUN yarn install --pure-lockfile --production + +EXPOSE 8080 + +CMD ["yarn", "workspace", "@omnivore/rss-handler", "start"] + diff --git a/packages/rss-handler/mocha-config.json b/packages/rss-handler/mocha-config.json new file mode 100644 index 000000000..44d1d24c1 --- /dev/null +++ b/packages/rss-handler/mocha-config.json @@ -0,0 +1,5 @@ +{ + "extension": ["ts"], + "spec": "test/**/*.test.ts", + "require": "test/babel-register.js" + } \ No newline at end of file diff --git a/packages/rss-handler/package.json b/packages/rss-handler/package.json new file mode 100644 index 000000000..e24080ae1 --- /dev/null +++ b/packages/rss-handler/package.json @@ -0,0 +1,30 @@ +{ + "name": "@omnivore/rss-handler", + "version": "1.0.0", + "main": "build/src/index.js", + "files": [ + "build/src" + ], + "license": "Apache-2.0", + "scripts": { + "test": "yarn mocha -r ts-node/register --config mocha-config.json", + "lint": "eslint src --ext ts,js,tsx,jsx", + "compile": "tsc", + "build": "tsc", + "start": "functions-framework --target=rssHandler", + "dev": "concurrently \"tsc -w\" \"nodemon --watch ./build/ --exec npm run start\"" + }, + "devDependencies": { + "chai": "^4.3.6", + "eslint-plugin-prettier": "^4.0.0", + "mocha": "^10.0.0" + }, + "dependencies": { + "@google-cloud/functions-framework": "3.1.2", + "@sentry/serverless": "^6.16.1", + "axios": "^1.4.0", + "dotenv": "^16.0.1", + "jsonwebtoken": "^8.5.1", + "rss-parser": "^3.13.0" + } +} diff --git a/packages/rss-handler/src/index.ts b/packages/rss-handler/src/index.ts new file mode 100644 index 000000000..c6459bbff --- /dev/null +++ b/packages/rss-handler/src/index.ts @@ -0,0 +1,30 @@ +import * as Sentry from '@sentry/serverless' +import * as dotenv from 'dotenv' // see https://github.com/motdotla/dotenv#how-do-i-use-dotenv-with-import +import Parser from 'rss-parser' + +dotenv.config() +Sentry.GCPFunction.init({ + dsn: process.env.SENTRY_DSN, + tracesSampleRate: 0, +}) + +const parser = new Parser() + +export const rssHandler = Sentry.GCPFunction.wrapHttpFunction( + async (req, res) => { + try { + const feed = await parser.parseURL('https://www.reddit.com/.rss') + console.log(feed.title) + + feed.items.forEach((item) => { + // eslint-disable-next-line @typescript-eslint/restrict-template-expressions + console.log(`${item.title}:${item.link}`) + }) + + res.send('ok') + } catch (e) { + console.error('Error while parsing RSS feed', e) + res.status(500).send('INTERNAL_SERVER_ERROR') + } + } +) diff --git a/packages/rss-handler/test/babel-register.js b/packages/rss-handler/test/babel-register.js new file mode 100644 index 000000000..a6f65f60a --- /dev/null +++ b/packages/rss-handler/test/babel-register.js @@ -0,0 +1,3 @@ +const register = require('@babel/register').default + +register({ extensions: ['.ts', '.tsx', '.js', '.jsx'] }) diff --git a/packages/rss-handler/test/stub.test.ts b/packages/rss-handler/test/stub.test.ts new file mode 100644 index 000000000..24ad25c8f --- /dev/null +++ b/packages/rss-handler/test/stub.test.ts @@ -0,0 +1,8 @@ +import 'mocha' +import { expect } from 'chai' + +describe('stub test', () => { + it('should pass', () => { + expect(true).to.be.true + }) +}) diff --git a/packages/rss-handler/tsconfig.json b/packages/rss-handler/tsconfig.json new file mode 100644 index 000000000..7ebe093f6 --- /dev/null +++ b/packages/rss-handler/tsconfig.json @@ -0,0 +1,8 @@ +{ + "extends": "./../../tsconfig.json", + "compilerOptions": { + "outDir": "build", + "rootDir": "." + }, + "include": ["src"] +} diff --git a/yarn.lock b/yarn.lock index c304c0ecc..0053463a6 100644 --- a/yarn.lock +++ b/yarn.lock @@ -13762,7 +13762,7 @@ ent@^2.2.0: resolved "https://registry.yarnpkg.com/ent/-/ent-2.2.0.tgz#e964219325a21d05f44466a2f686ed6ce5f5dd1d" integrity sha1-6WQhkyWiHQX0RGai9obtbOX13R0= -entities@^2.0.0: +entities@^2.0.0, entities@^2.0.3: version "2.2.0" resolved "https://registry.yarnpkg.com/entities/-/entities-2.2.0.tgz#098dc90ebb83d8dffa089d55256b351d34c4da55" integrity sha512-p92if5Nz619I0w+akJrLZH0MX0Pb5DX39XOwQTtXSdQQOaYH03S1uIQp4mhOZtAXrxq4ViO67YTiLBo2638o9A== @@ -24814,6 +24814,14 @@ rollup@2.78.0: optionalDependencies: fsevents "~2.3.2" +rss-parser@^3.13.0: + version "3.13.0" + resolved "https://registry.yarnpkg.com/rss-parser/-/rss-parser-3.13.0.tgz#f1f83b0a85166b8310ec531da6fbaa53ff0f50f0" + integrity sha512-7jWUBV5yGN3rqMMj7CZufl/291QAhvrrGpDNE4k/02ZchL0npisiYYqULF71jCEKoIiHvK/Q2e6IkDwPziT7+w== + dependencies: + entities "^2.0.3" + xml2js "^0.5.0" + rsvp@^4.8.4, rsvp@^4.8.5: version "4.8.5" resolved "https://registry.yarnpkg.com/rsvp/-/rsvp-4.8.5.tgz#c8f155311d167f68f21e168df71ec5b083113734" @@ -28459,6 +28467,14 @@ xml2js@^0.4.23: sax ">=0.6.0" xmlbuilder "~11.0.0" +xml2js@^0.5.0: + version "0.5.0" + resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.5.0.tgz#d9440631fbb2ed800203fad106f2724f62c493b7" + integrity sha512-drPFnkQJik/O+uPKpqSgr22mpuFHqKdbS835iAQrUC73L2F5WkboIRd63ai/2Yg6I1jzifPFKH2NTK+cfglkIA== + dependencies: + sax ">=0.6.0" + xmlbuilder "~11.0.0" + xmlbuilder@~11.0.0: version "11.0.1" resolved "https://registry.yarnpkg.com/xmlbuilder/-/xmlbuilder-11.0.1.tgz#be9bae1c8a046e76b31127726347d0ad7002beb3" From d9130879d8f81bdd8b85b8d41f7652678adcebf9 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Wed, 5 Jul 2023 16:35:52 +0800 Subject: [PATCH 02/19] create rss_subscription db table --- .../migrations/0114.do.rss_subscription.sql | 20 +++++++++++++++++++ .../migrations/0114.undo.rss_subscription.sql | 9 +++++++++ 2 files changed, 29 insertions(+) create mode 100755 packages/db/migrations/0114.do.rss_subscription.sql create mode 100755 packages/db/migrations/0114.undo.rss_subscription.sql diff --git a/packages/db/migrations/0114.do.rss_subscription.sql b/packages/db/migrations/0114.do.rss_subscription.sql new file mode 100755 index 000000000..91f8d2f29 --- /dev/null +++ b/packages/db/migrations/0114.do.rss_subscription.sql @@ -0,0 +1,20 @@ +-- Type: DO +-- Name: rss_subscription +-- Description: Create a table for RSS subscriptions + +BEGIN; + +CREATE TABLE omnivore.rss_subscription ( + id uuid PRIMARY KEY DEFAULT uuid_generate_v1mc(), + user_id uuid NOT NULL REFERENCES omnivore.user ON DELETE CASCADE, + title character varying(255) NOT NULL, + description TEXT, + url TEXT NOT NULL, + image_url TEXT, + count integer NOT NULL DEFAULT 0, + last_updated timestamptz, + created_at timestamptz NOT NULL DEFAULT current_timestamp, + updated_at timestamptz NOT NULL DEFAULT current_timestamp, +); + +COMMIT; diff --git a/packages/db/migrations/0114.undo.rss_subscription.sql b/packages/db/migrations/0114.undo.rss_subscription.sql new file mode 100755 index 000000000..3a3ef4f5a --- /dev/null +++ b/packages/db/migrations/0114.undo.rss_subscription.sql @@ -0,0 +1,9 @@ +-- Type: UNDO +-- Name: rss_subscription +-- Description: Create a table for RSS subscriptions + +BEGIN; + +DROP TABLE IF EXISTS omnivore.rss_subscription; + +COMMIT; From 9f7796e2f005d3e439f24292691b401c763a365d Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Wed, 5 Jul 2023 16:38:31 +0800 Subject: [PATCH 03/19] fix typo in sql --- packages/db/migrations/0114.do.rss_subscription.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/db/migrations/0114.do.rss_subscription.sql b/packages/db/migrations/0114.do.rss_subscription.sql index 91f8d2f29..6fc4f9b4d 100755 --- a/packages/db/migrations/0114.do.rss_subscription.sql +++ b/packages/db/migrations/0114.do.rss_subscription.sql @@ -14,7 +14,7 @@ CREATE TABLE omnivore.rss_subscription ( count integer NOT NULL DEFAULT 0, last_updated timestamptz, created_at timestamptz NOT NULL DEFAULT current_timestamp, - updated_at timestamptz NOT NULL DEFAULT current_timestamp, + updated_at timestamptz NOT NULL DEFAULT current_timestamp ); COMMIT; From a6192ed86bbc45a125f3d40056b868d4146abe7a Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Wed, 5 Jul 2023 16:55:08 +0800 Subject: [PATCH 04/19] add rss subscription entity class --- packages/api/src/entity/rss_subscription.ts | 44 +++++++++++++++++++ .../migrations/0114.do.rss_subscription.sql | 2 +- 2 files changed, 45 insertions(+), 1 deletion(-) create mode 100644 packages/api/src/entity/rss_subscription.ts diff --git a/packages/api/src/entity/rss_subscription.ts b/packages/api/src/entity/rss_subscription.ts new file mode 100644 index 000000000..d5f181b06 --- /dev/null +++ b/packages/api/src/entity/rss_subscription.ts @@ -0,0 +1,44 @@ +import { + Column, + CreateDateColumn, + Entity, + JoinColumn, + ManyToOne, + PrimaryGeneratedColumn, + UpdateDateColumn, +} from 'typeorm' +import { User } from './user' + +@Entity({ name: 'rss_subscription' }) +export class RssSubscription { + @PrimaryGeneratedColumn('uuid') + id!: string + + @ManyToOne(() => User, { onDelete: 'CASCADE' }) + @JoinColumn({ name: 'user_id' }) + user!: User + + @Column('varchar', { length: 255 }) + title!: string + + @Column('text', { nullable: true }) + description?: string | null + + @Column('text') + url!: string + + @Column('text', { nullable: true }) + imageUrl?: string | null + + @Column('integer', { default: 0 }) + count!: number + + @Column('timestamp', { nullable: true }) + lastFetchedAt?: Date | null + + @CreateDateColumn({ default: () => 'CURRENT_TIMESTAMP' }) + createdAt!: Date + + @UpdateDateColumn({ default: () => 'CURRENT_TIMESTAMP' }) + updatedAt!: Date +} diff --git a/packages/db/migrations/0114.do.rss_subscription.sql b/packages/db/migrations/0114.do.rss_subscription.sql index 6fc4f9b4d..bf4b4f1a4 100755 --- a/packages/db/migrations/0114.do.rss_subscription.sql +++ b/packages/db/migrations/0114.do.rss_subscription.sql @@ -12,7 +12,7 @@ CREATE TABLE omnivore.rss_subscription ( url TEXT NOT NULL, image_url TEXT, count integer NOT NULL DEFAULT 0, - last_updated timestamptz, + last_fetched_at timestamptz, created_at timestamptz NOT NULL DEFAULT current_timestamp, updated_at timestamptz NOT NULL DEFAULT current_timestamp ); From 2adf753f8f0232a80922567af262c332d9054b65 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Wed, 5 Jul 2023 18:23:01 +0800 Subject: [PATCH 05/19] update graphql schema --- packages/api/src/entity/rss_subscription.ts | 44 ------------------- packages/api/src/entity/subscription.ts | 21 ++++++--- packages/api/src/generated/graphql.ts | 16 ++++++- packages/api/src/generated/schema.graphql | 10 ++++- packages/api/src/schema.ts | 10 ++++- .../0114.do.add_type_to_subscriptions.sql | 15 +++++++ .../migrations/0114.do.rss_subscription.sql | 20 --------- .../0114.undo.add_type_to_subscriptions.sql | 15 +++++++ .../migrations/0114.undo.rss_subscription.sql | 9 ---- 9 files changed, 77 insertions(+), 83 deletions(-) delete mode 100644 packages/api/src/entity/rss_subscription.ts create mode 100755 packages/db/migrations/0114.do.add_type_to_subscriptions.sql delete mode 100755 packages/db/migrations/0114.do.rss_subscription.sql create mode 100755 packages/db/migrations/0114.undo.add_type_to_subscriptions.sql delete mode 100755 packages/db/migrations/0114.undo.rss_subscription.sql diff --git a/packages/api/src/entity/rss_subscription.ts b/packages/api/src/entity/rss_subscription.ts deleted file mode 100644 index d5f181b06..000000000 --- a/packages/api/src/entity/rss_subscription.ts +++ /dev/null @@ -1,44 +0,0 @@ -import { - Column, - CreateDateColumn, - Entity, - JoinColumn, - ManyToOne, - PrimaryGeneratedColumn, - UpdateDateColumn, -} from 'typeorm' -import { User } from './user' - -@Entity({ name: 'rss_subscription' }) -export class RssSubscription { - @PrimaryGeneratedColumn('uuid') - id!: string - - @ManyToOne(() => User, { onDelete: 'CASCADE' }) - @JoinColumn({ name: 'user_id' }) - user!: User - - @Column('varchar', { length: 255 }) - title!: string - - @Column('text', { nullable: true }) - description?: string | null - - @Column('text') - url!: string - - @Column('text', { nullable: true }) - imageUrl?: string | null - - @Column('integer', { default: 0 }) - count!: number - - @Column('timestamp', { nullable: true }) - lastFetchedAt?: Date | null - - @CreateDateColumn({ default: () => 'CURRENT_TIMESTAMP' }) - createdAt!: Date - - @UpdateDateColumn({ default: () => 'CURRENT_TIMESTAMP' }) - updatedAt!: Date -} diff --git a/packages/api/src/entity/subscription.ts b/packages/api/src/entity/subscription.ts index ade5f4b57..a0e311a19 100644 --- a/packages/api/src/entity/subscription.ts +++ b/packages/api/src/entity/subscription.ts @@ -5,15 +5,13 @@ import { JoinColumn, ManyToOne, PrimaryGeneratedColumn, - Unique, UpdateDateColumn, } from 'typeorm' -import { User } from './user' -import { SubscriptionStatus } from '../generated/graphql' +import { SubscriptionStatus, SubscriptionType } from '../generated/graphql' import { NewsletterEmail } from './newsletter_email' +import { User } from './user' @Entity({ name: 'subscriptions' }) -@Unique(['name', 'user']) export class Subscription { @PrimaryGeneratedColumn('uuid') id!: string @@ -50,9 +48,20 @@ export class Subscription { @Column('text', { nullable: true }) icon?: string - @CreateDateColumn() + @Column('enum', { + enum: SubscriptionType, + }) + type!: SubscriptionType + + @Column('integer', { default: 0 }) + count!: number + + @Column('timestamp', { nullable: true }) + lastFetchedAt?: Date | null + + @CreateDateColumn({ default: () => 'CURRENT_TIMESTAMP' }) createdAt!: Date - @UpdateDateColumn() + @UpdateDateColumn({ default: () => 'CURRENT_TIMESTAMP' }) updatedAt!: Date } diff --git a/packages/api/src/generated/graphql.ts b/packages/api/src/generated/graphql.ts index 82e0206d4..fa49a1a0f 100644 --- a/packages/api/src/generated/graphql.ts +++ b/packages/api/src/generated/graphql.ts @@ -2706,13 +2706,16 @@ export type SubscribeSuccess = { export type Subscription = { __typename?: 'Subscription'; + count: Scalars['Int']; createdAt: Scalars['Date']; description?: Maybe; icon?: Maybe; id: Scalars['ID']; + lastFetchedAt?: Maybe; name: Scalars['String']; - newsletterEmail: Scalars['String']; + newsletterEmail?: Maybe; status: SubscriptionStatus; + type: SubscriptionType; unsubscribeHttpUrl?: Maybe; unsubscribeMailTo?: Maybe; updatedAt: Scalars['Date']; @@ -2725,6 +2728,11 @@ export enum SubscriptionStatus { Unsubscribed = 'UNSUBSCRIBED' } +export enum SubscriptionType { + Newsletter = 'NEWSLETTER', + Rss = 'RSS' +} + export type SubscriptionsError = { __typename?: 'SubscriptionsError'; errorCodes: Array; @@ -3691,6 +3699,7 @@ export type ResolversTypes = { SubscribeSuccess: ResolverTypeWrapper; Subscription: ResolverTypeWrapper<{}>; SubscriptionStatus: SubscriptionStatus; + SubscriptionType: SubscriptionType; SubscriptionsError: ResolverTypeWrapper; SubscriptionsErrorCode: SubscriptionsErrorCode; SubscriptionsResult: ResolversTypes['SubscriptionsError'] | ResolversTypes['SubscriptionsSuccess']; @@ -5745,13 +5754,16 @@ export type SubscribeSuccessResolvers = { + count?: SubscriptionResolver; createdAt?: SubscriptionResolver; description?: SubscriptionResolver, "description", ParentType, ContextType>; icon?: SubscriptionResolver, "icon", ParentType, ContextType>; id?: SubscriptionResolver; + lastFetchedAt?: SubscriptionResolver, "lastFetchedAt", ParentType, ContextType>; name?: SubscriptionResolver; - newsletterEmail?: SubscriptionResolver; + newsletterEmail?: SubscriptionResolver, "newsletterEmail", ParentType, ContextType>; status?: SubscriptionResolver; + type?: SubscriptionResolver; unsubscribeHttpUrl?: SubscriptionResolver, "unsubscribeHttpUrl", ParentType, ContextType>; unsubscribeMailTo?: SubscriptionResolver, "unsubscribeMailTo", ParentType, ContextType>; updatedAt?: SubscriptionResolver; diff --git a/packages/api/src/generated/schema.graphql b/packages/api/src/generated/schema.graphql index 010eca42a..433c99924 100644 --- a/packages/api/src/generated/schema.graphql +++ b/packages/api/src/generated/schema.graphql @@ -2058,13 +2058,16 @@ type SubscribeSuccess { } type Subscription { + count: Int! createdAt: Date! description: String icon: String id: ID! + lastFetchedAt: Date name: String! - newsletterEmail: String! + newsletterEmail: String status: SubscriptionStatus! + type: SubscriptionType! unsubscribeHttpUrl: String unsubscribeMailTo: String updatedAt: Date! @@ -2077,6 +2080,11 @@ enum SubscriptionStatus { UNSUBSCRIBED } +enum SubscriptionType { + NEWSLETTER + RSS +} + type SubscriptionsError { errorCodes: [SubscriptionsErrorCode!]! } diff --git a/packages/api/src/schema.ts b/packages/api/src/schema.ts index 4800d8da4..5ac31e3bb 100755 --- a/packages/api/src/schema.ts +++ b/packages/api/src/schema.ts @@ -1619,16 +1619,24 @@ const schema = gql` subscriptions: [Subscription!]! } + enum SubscriptionType { + RSS + NEWSLETTER + } + type Subscription { id: ID! name: String! - newsletterEmail: String! + newsletterEmail: String url: String description: String status: SubscriptionStatus! unsubscribeMailTo: String unsubscribeHttpUrl: String icon: String + type: SubscriptionType! + count: Int! + lastFetchedAt: Date createdAt: Date! updatedAt: Date! } diff --git a/packages/db/migrations/0114.do.add_type_to_subscriptions.sql b/packages/db/migrations/0114.do.add_type_to_subscriptions.sql new file mode 100755 index 000000000..4a154fec9 --- /dev/null +++ b/packages/db/migrations/0114.do.add_type_to_subscriptions.sql @@ -0,0 +1,15 @@ +-- Type: DO +-- Name: add_type_to_subscriptions +-- Description: Add type, count and last_fetched_at fields to subscriptions table + +BEGIN; + +CREATE TYPE subscription_type AS ENUM ('NEWSLETTER', 'RSS'); + +ALTER TABLE omnivore.subscriptions + ADD COLUMN "type" subscription_type NOT NULL DEFAULT 'NEWSLETTER', + ADD COLUMN count INTEGER NOT NULL DEFAULT 0, + ADD COLUMN last_fetched_at timestamptz, + DROP CONSTRAINT subscriptions_user_id_name_key; -- Drop unique constraint on user_id and name + +COMMIT; diff --git a/packages/db/migrations/0114.do.rss_subscription.sql b/packages/db/migrations/0114.do.rss_subscription.sql deleted file mode 100755 index bf4b4f1a4..000000000 --- a/packages/db/migrations/0114.do.rss_subscription.sql +++ /dev/null @@ -1,20 +0,0 @@ --- Type: DO --- Name: rss_subscription --- Description: Create a table for RSS subscriptions - -BEGIN; - -CREATE TABLE omnivore.rss_subscription ( - id uuid PRIMARY KEY DEFAULT uuid_generate_v1mc(), - user_id uuid NOT NULL REFERENCES omnivore.user ON DELETE CASCADE, - title character varying(255) NOT NULL, - description TEXT, - url TEXT NOT NULL, - image_url TEXT, - count integer NOT NULL DEFAULT 0, - last_fetched_at timestamptz, - created_at timestamptz NOT NULL DEFAULT current_timestamp, - updated_at timestamptz NOT NULL DEFAULT current_timestamp -); - -COMMIT; diff --git a/packages/db/migrations/0114.undo.add_type_to_subscriptions.sql b/packages/db/migrations/0114.undo.add_type_to_subscriptions.sql new file mode 100755 index 000000000..1d6b9f048 --- /dev/null +++ b/packages/db/migrations/0114.undo.add_type_to_subscriptions.sql @@ -0,0 +1,15 @@ +-- Type: UNDO +-- Name: add_type_to_subscriptions +-- Description: Add type, count and last_fetched_at fields to subscriptions table + +BEGIN; + +ALTER TABLE omnivore.subscriptions + ADD CONSTRAINT subscriptions_user_id_name_key UNIQUE (user_id, name), + DROP COLUMN last_fetched_at, + DROP COLUMN count, + DROP COLUMN "type"; + +DROP TYPE omnivore.subscription_type; + +COMMIT; diff --git a/packages/db/migrations/0114.undo.rss_subscription.sql b/packages/db/migrations/0114.undo.rss_subscription.sql deleted file mode 100755 index 3a3ef4f5a..000000000 --- a/packages/db/migrations/0114.undo.rss_subscription.sql +++ /dev/null @@ -1,9 +0,0 @@ --- Type: UNDO --- Name: rss_subscription --- Description: Create a table for RSS subscriptions - -BEGIN; - -DROP TABLE IF EXISTS omnivore.rss_subscription; - -COMMIT; From 7075375e2ba0a4613adf7bccf24a25cbf31328c7 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Wed, 5 Jul 2023 20:48:07 +0800 Subject: [PATCH 06/19] fix test --- packages/api/src/services/subscriptions.ts | 45 +++++++++++++++------- 1 file changed, 32 insertions(+), 13 deletions(-) diff --git a/packages/api/src/services/subscriptions.ts b/packages/api/src/services/subscriptions.ts index 59efb74ed..a43b22dc7 100644 --- a/packages/api/src/services/subscriptions.ts +++ b/packages/api/src/services/subscriptions.ts @@ -2,7 +2,7 @@ import axios from 'axios' import { NewsletterEmail } from '../entity/newsletter_email' import { Subscription } from '../entity/subscription' import { getRepository } from '../entity/utils' -import { SubscriptionStatus } from '../generated/graphql' +import { SubscriptionStatus, SubscriptionType } from '../generated/graphql' import { sendEmail } from '../utils/sendEmail' import { createNewsletterEmail } from './newsletters' @@ -13,6 +13,7 @@ interface SaveSubscriptionInput { unsubscribeMailTo?: string unsubscribeHttpUrl?: string icon?: string + from?: string } export const UNSUBSCRIBE_EMAIL_TEXT = @@ -85,6 +86,7 @@ export const getSubscriptionByNameAndUserId = async ( return getRepository(Subscription).findOneBy({ name, user: { id: userId }, + type: SubscriptionType.Newsletter, }) } @@ -96,19 +98,36 @@ export const saveSubscription = async ({ unsubscribeHttpUrl, icon, }: SaveSubscriptionInput): Promise => { - const result = await getRepository(Subscription).upsert( - { - name, - newsletterEmail: { id: newsletterEmail.id }, - user: { id: userId }, - unsubscribeHttpUrl, - unsubscribeMailTo, - icon, - }, - ['name', 'user'] - ) + const subscriptionData = { + unsubscribeHttpUrl, + unsubscribeMailTo, + icon, + lastFetchedAt: new Date(), + } - return result.identifiers[0].id as string + const existingSubscription = await getSubscriptionByNameAndUserId( + name, + userId + ) + if (existingSubscription) { + // update subscription if already exists + await getRepository(Subscription).update( + existingSubscription.id, + subscriptionData + ) + + return existingSubscription.id + } + + const result = await getRepository(Subscription).save({ + ...subscriptionData, + name, + newsletterEmail: { id: newsletterEmail.id }, + user: { id: userId }, + type: SubscriptionType.Newsletter, + }) + + return result.id } export const unsubscribe = async (subscription: Subscription) => { From 574636451dfc567beddf0217dc64dae792f78cf4 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Thu, 6 Jul 2023 19:49:03 +0800 Subject: [PATCH 07/19] update api to accommodate rss feed subscriptions --- packages/api/package.json | 1 + packages/api/src/entity/subscription.ts | 4 +- packages/api/src/generated/graphql.ts | 14 ++- packages/api/src/generated/schema.graphql | 12 +- .../api/src/resolvers/subscriptions/index.ts | 114 +++++++++++++----- packages/api/src/schema.ts | 15 ++- packages/api/src/services/subscriptions.ts | 38 +++--- packages/api/test/db.ts | 35 +++--- .../api/test/resolvers/subscriptions.test.ts | 23 ++-- 9 files changed, 173 insertions(+), 83 deletions(-) diff --git a/packages/api/package.json b/packages/api/package.json index ec47f1bcc..a1a042dc3 100644 --- a/packages/api/package.json +++ b/packages/api/package.json @@ -83,6 +83,7 @@ "pg": "^8.3.3", "postgrator": "^4.2.0", "private-ip": "^2.3.3", + "rss-parser": "^3.13.0", "sanitize-html": "^2.3.2", "search-query-parser": "^1.6.0", "snake-case": "^3.0.3", diff --git a/packages/api/src/entity/subscription.ts b/packages/api/src/entity/subscription.ts index a0e311a19..3fa5a600c 100644 --- a/packages/api/src/entity/subscription.ts +++ b/packages/api/src/entity/subscription.ts @@ -29,9 +29,9 @@ export class Subscription { }) status!: SubscriptionStatus - @ManyToOne(() => NewsletterEmail) + @ManyToOne(() => NewsletterEmail, { nullable: true }) @JoinColumn({ name: 'newsletter_email_id' }) - newsletterEmail!: NewsletterEmail + newsletterEmail?: NewsletterEmail | null @Column('text', { nullable: true }) description?: string diff --git a/packages/api/src/generated/graphql.ts b/packages/api/src/generated/graphql.ts index fa49a1a0f..051bae52b 100644 --- a/packages/api/src/generated/graphql.ts +++ b/packages/api/src/generated/graphql.ts @@ -1574,12 +1574,13 @@ export type MutationSetWebhookArgs = { export type MutationSubscribeArgs = { - name: Scalars['String']; + input: SubscribeInput; }; export type MutationUnsubscribeArgs = { name: Scalars['String']; + subscriptionId?: InputMaybe; }; @@ -1869,6 +1870,7 @@ export type QuerySharedArticleArgs = { export type QuerySubscriptionsArgs = { sort?: InputMaybe; + type?: InputMaybe; }; @@ -2697,6 +2699,12 @@ export enum SubscribeErrorCode { Unauthorized = 'UNAUTHORIZED' } +export type SubscribeInput = { + name?: InputMaybe; + subscriptionType?: InputMaybe; + url?: InputMaybe; +}; + export type SubscribeResult = SubscribeError | SubscribeSuccess; export type SubscribeSuccess = { @@ -3695,6 +3703,7 @@ export type ResolversTypes = { String: ResolverTypeWrapper; SubscribeError: ResolverTypeWrapper; SubscribeErrorCode: SubscribeErrorCode; + SubscribeInput: SubscribeInput; SubscribeResult: ResolversTypes['SubscribeError'] | ResolversTypes['SubscribeSuccess']; SubscribeSuccess: ResolverTypeWrapper; Subscription: ResolverTypeWrapper<{}>; @@ -4102,6 +4111,7 @@ export type ResolversParentTypes = { SortParams: SortParams; String: Scalars['String']; SubscribeError: SubscribeError; + SubscribeInput: SubscribeInput; SubscribeResult: ResolversParentTypes['SubscribeError'] | ResolversParentTypes['SubscribeSuccess']; SubscribeSuccess: SubscribeSuccess; Subscription: {}; @@ -5123,7 +5133,7 @@ export type MutationResolvers>; setUserPersonalization?: Resolver>; setWebhook?: Resolver>; - subscribe?: Resolver>; + subscribe?: Resolver>; unsubscribe?: Resolver>; updateHighlight?: Resolver>; updateHighlightReply?: Resolver>; diff --git a/packages/api/src/generated/schema.graphql b/packages/api/src/generated/schema.graphql index 433c99924..8a2ebbeff 100644 --- a/packages/api/src/generated/schema.graphql +++ b/packages/api/src/generated/schema.graphql @@ -1151,8 +1151,8 @@ type Mutation { setShareHighlight(input: SetShareHighlightInput!): SetShareHighlightResult! setUserPersonalization(input: SetUserPersonalizationInput!): SetUserPersonalizationResult! setWebhook(input: SetWebhookInput!): SetWebhookResult! - subscribe(name: String!): SubscribeResult! - unsubscribe(name: String!): UnsubscribeResult! + subscribe(input: SubscribeInput!): SubscribeResult! + unsubscribe(name: String!, subscriptionId: ID): UnsubscribeResult! updateHighlight(input: UpdateHighlightInput!): UpdateHighlightResult! updateHighlightReply(input: UpdateHighlightReplyInput!): UpdateHighlightReplyResult! updateLabel(input: UpdateLabelInput!): UpdateLabelResult! @@ -1308,7 +1308,7 @@ type Query { search(after: String, first: Int, format: String, includeContent: Boolean, query: String): SearchResult! sendInstallInstructions: SendInstallInstructionsResult! sharedArticle(selectedHighlightId: String, slug: String!, username: String!): SharedArticleResult! - subscriptions(sort: SortParams): SubscriptionsResult! + subscriptions(sort: SortParams, type: SubscriptionType): SubscriptionsResult! typeaheadSearch(first: Int, query: String!): TypeaheadSearchResult! updatesSince(after: String, first: Int, since: Date!, sort: SortParams): UpdatesSinceResult! user(userId: ID, username: String): UserResult! @@ -2051,6 +2051,12 @@ enum SubscribeErrorCode { UNAUTHORIZED } +input SubscribeInput { + name: String + subscriptionType: SubscriptionType + url: String +} + union SubscribeResult = SubscribeError | SubscribeSuccess type SubscribeSuccess { diff --git a/packages/api/src/resolvers/subscriptions/index.ts b/packages/api/src/resolvers/subscriptions/index.ts index a7aa15392..e4a112e15 100644 --- a/packages/api/src/resolvers/subscriptions/index.ts +++ b/packages/api/src/resolvers/subscriptions/index.ts @@ -1,4 +1,4 @@ -import { ILike } from 'typeorm' +import Parser from 'rss-parser' import { Subscription } from '../../entity/subscription' import { User } from '../../entity/user' import { getRepository } from '../../entity/utils' @@ -16,6 +16,7 @@ import { SubscriptionsErrorCode, SubscriptionsSuccess, SubscriptionStatus, + SubscriptionType, UnsubscribeError, UnsubscribeErrorCode, UnsubscribeSuccess, @@ -25,11 +26,13 @@ import { analytics } from '../../utils/analytics' import { authorized } from '../../utils/helpers' import { createImageProxyUrl } from '../../utils/imageproxy' +const parser = new Parser() + export const subscriptionsResolver = authorized< SubscriptionsSuccess, SubscriptionsError, QuerySubscriptionsArgs ->(async (_obj, { sort }, { claims: { uid }, log }) => { +>(async (_obj, { sort, type: subscriptionType }, { claims: { uid }, log }) => { log.info('subscriptionsResolver') analytics.track({ @@ -41,7 +44,8 @@ export const subscriptionsResolver = authorized< }) try { - const sortBy = sort?.by === SortBy.UpdatedTime ? 'updatedAt' : 'createdAt' + const sortBy = + sort?.by === SortBy.UpdatedTime ? 'lastFetchedAt' : 'createdAt' const sortOrder = sort?.order === SortOrder.Ascending ? 'ASC' : 'DESC' const user = await getRepository(User).findOneBy({ id: uid }) if (!user) { @@ -52,10 +56,11 @@ export const subscriptionsResolver = authorized< const subscriptions = await getRepository(Subscription) .createQueryBuilder('subscription') - .innerJoinAndSelect('subscription.newsletterEmail', 'newsletterEmail') + .leftJoinAndSelect('subscription.newsletterEmail', 'newsletterEmail') .where({ user: { id: uid }, status: SubscriptionStatus.Active, + type: subscriptionType || SubscriptionType.Newsletter, // default to newsletter }) .orderBy('subscription.' + sortBy, sortOrder) .getMany() @@ -64,7 +69,7 @@ export const subscriptionsResolver = authorized< subscriptions: subscriptions.map((s) => ({ ...s, icon: s.icon && createImageProxyUrl(s.icon, 128, 128), - newsletterEmail: s.newsletterEmail.address, + newsletterEmail: s.newsletterEmail?.address, })), } } catch (error) { @@ -79,7 +84,7 @@ export const unsubscribeResolver = authorized< UnsubscribeSuccess, UnsubscribeError, MutationUnsubscribeArgs ->(async (_, { name }, { claims: { uid }, log }) => { +>(async (_, { name, subscriptionId }, { claims: { uid }, log }) => { log.info('unsubscribeResolver') try { @@ -90,13 +95,20 @@ export const unsubscribeResolver = authorized< } } - const subscription = await getRepository(Subscription) + const queryBuilder = getRepository(Subscription) .createQueryBuilder('subscription') - .innerJoinAndSelect('subscription.newsletterEmail', 'newsletterEmail') + .leftJoinAndSelect('subscription.newsletterEmail', 'newsletterEmail') .where({ user: { id: uid } }) - .andWhere('LOWER(name) = LOWER(:name)', { name }) // case insensitive - .getOne() + if (subscriptionId) { + // if subscriptionId is provided, ignore name + queryBuilder.andWhere({ id: subscriptionId }) + } else { + // if subscriptionId is not provided, use name for old clients + queryBuilder.andWhere({ name }) + } + + const subscription = await queryBuilder.getOne() if (!subscription) { return { errorCodes: [UnsubscribeErrorCode.NotFound], @@ -128,7 +140,7 @@ export const unsubscribeResolver = authorized< return { subscription: { ...subscription, - newsletterEmail: subscription.newsletterEmail.address, + newsletterEmail: subscription.newsletterEmail?.address, }, } } catch (error) { @@ -143,7 +155,7 @@ export const subscribeResolver = authorized< SubscribeSuccess, SubscribeError, MutationSubscribeArgs ->(async (_, { name }, { claims: { uid }, log }) => { +>(async (_, { input }, { claims: { uid }, log }) => { log.info('subscribeResolver') try { @@ -154,10 +166,13 @@ export const subscribeResolver = authorized< } } + // find existing subscription const subscription = await getRepository(Subscription).findOneBy({ - name: ILike(name), + url: input.url || undefined, + name: input.name || undefined, user: { id: uid }, status: SubscriptionStatus.Active, + type: input.subscriptionType || SubscriptionType.Rss, // default to rss }) if (subscription) { return { @@ -165,34 +180,69 @@ export const subscribeResolver = authorized< } } - const subscribeHandler = getSubscribeHandler(name) - if (!subscribeHandler) { - return { - errorCodes: [SubscribeErrorCode.NotFound], - } - } - - const newSubscriptions = await subscribeHandler.handleSubscribe(uid, name) - if (!newSubscriptions) { - return { - errorCodes: [SubscribeErrorCode.BadRequest], - } - } - analytics.track({ userId: uid, event: 'subscribed', properties: { - name, + ...input, env: env.server.apiEnv, }, }) + // create new newsletter subscription + if (input.name && input.subscriptionType === SubscriptionType.Newsletter) { + const subscribeHandler = getSubscribeHandler(input.name) + if (!subscribeHandler) { + return { + errorCodes: [SubscribeErrorCode.NotFound], + } + } + + const newSubscriptions = await subscribeHandler.handleSubscribe( + uid, + input.name + ) + if (!newSubscriptions) { + return { + errorCodes: [SubscribeErrorCode.BadRequest], + } + } + + return { + subscriptions: newSubscriptions.map((s) => ({ + ...s, + newsletterEmail: s.newsletterEmail?.address, + })), + } + } + + // create new rss subscription + if (input.url) { + // validate rss feed + const feed = await parser.parseURL(input.url) + + const newSubscription = await getRepository(Subscription).save({ + name: feed.title, + url: input.url, + user: { id: uid }, + type: SubscriptionType.Rss, + description: feed.description, + icon: feed.image?.url, + }) + + return { + subscriptions: [ + { + ...newSubscription, + newsletterEmail: null, + }, + ], + } + } + + log.info('missing url or name') return { - subscriptions: newSubscriptions.map((s) => ({ - ...s, - newsletterEmail: s.newsletterEmail.address, - })), + errorCodes: [SubscribeErrorCode.BadRequest], } } catch (error) { log.error('failed to subscribe', error) diff --git a/packages/api/src/schema.ts b/packages/api/src/schema.ts index 5ac31e3bb..3a4be3243 100755 --- a/packages/api/src/schema.ts +++ b/packages/api/src/schema.ts @@ -2486,6 +2486,12 @@ const schema = gql` ALREADY_EXISTS } + input SubscribeInput { + url: String + name: String + subscriptionType: SubscriptionType + } + # Mutations type Mutation { googleLogin(input: GoogleLoginInput!): LoginResult! @@ -2547,8 +2553,8 @@ const schema = gql` deleteLabel(id: ID!): DeleteLabelResult! setLabels(input: SetLabelsInput!): SetLabelsResult! generateApiKey(input: GenerateApiKeyInput!): GenerateApiKeyResult! - unsubscribe(name: String!): UnsubscribeResult! - subscribe(name: String!): SubscribeResult! + unsubscribe(name: String!, subscriptionId: ID): UnsubscribeResult! + subscribe(input: SubscribeInput!): SubscribeResult! addPopularRead(name: String!): AddPopularReadResult! setWebhook(input: SetWebhookInput!): SetWebhookResult! deleteWebhook(id: ID!): DeleteWebhookResult! @@ -2628,7 +2634,10 @@ const schema = gql` includeContent: Boolean format: String ): SearchResult! - subscriptions(sort: SortParams): SubscriptionsResult! + subscriptions( + sort: SortParams + type: SubscriptionType + ): SubscriptionsResult! sendInstallInstructions: SendInstallInstructionsResult! webhooks: WebhooksResult! webhook(id: ID!): WebhookResult! diff --git a/packages/api/src/services/subscriptions.ts b/packages/api/src/services/subscriptions.ts index a43b22dc7..5eadd22ea 100644 --- a/packages/api/src/services/subscriptions.ts +++ b/packages/api/src/services/subscriptions.ts @@ -131,26 +131,30 @@ export const saveSubscription = async ({ } export const unsubscribe = async (subscription: Subscription) => { - let unsubscribed = false - if (subscription.unsubscribeMailTo) { - // unsubscribe by sending email - unsubscribed = await sendUnsubscribeEmail( - subscription.unsubscribeMailTo, - subscription.newsletterEmail.address - ) - } - // TODO: find a good way to unsubscribe by url if email fails or not provided - // because it often requires clicking a button on the page to unsubscribe + // unsubscribe from newsletter + if (subscription.type === SubscriptionType.Newsletter) { + let unsubscribed = false - if (!unsubscribed) { - // update subscription status to unsubscribed if failed to unsubscribe - console.log('Failed to unsubscribe', subscription.id) - return getRepository(Subscription).update(subscription.id, { - status: SubscriptionStatus.Unsubscribed, - }) + if (subscription.unsubscribeMailTo && subscription.newsletterEmail) { + // unsubscribe by sending email + unsubscribed = await sendUnsubscribeEmail( + subscription.unsubscribeMailTo, + subscription.newsletterEmail.address + ) + } + // TODO: find a good way to unsubscribe by url if email fails or not provided + // because it often requires clicking a button on the page to unsubscribe + + if (!unsubscribed) { + // update subscription status to unsubscribed if failed to unsubscribe + console.log('Failed to unsubscribe', subscription.id) + return getRepository(Subscription).update(subscription.id, { + status: SubscriptionStatus.Unsubscribed, + }) + } } - // delete the subscription if successfully unsubscribed + // delete the subscription if successfully unsubscribed or it's an rss feed await getRepository(Subscription).delete(subscription.id) } diff --git a/packages/api/test/db.ts b/packages/api/test/db.ts index 37d1b90d6..773b57167 100644 --- a/packages/api/test/db.ts +++ b/packages/api/test/db.ts @@ -1,20 +1,20 @@ import Postgrator from 'postgrator' -import { User } from '../src/entity/user' -import { Profile } from '../src/entity/profile' -import { Page } from '../src/entity/page' -import { Link } from '../src/entity/link' -import { Reminder } from '../src/entity/reminder' -import { NewsletterEmail } from '../src/entity/newsletter_email' -import { UserDeviceToken } from '../src/entity/user_device_tokens' -import { Label } from '../src/entity/label' -import { Subscription } from '../src/entity/subscription' -import { AppDataSource } from '../src/server' -import { getRepository, setClaims } from '../src/entity/utils' -import { createUser } from '../src/services/create_user' -import { SnakeNamingStrategy } from 'typeorm-naming-strategies' -import { SubscriptionStatus } from '../src/generated/graphql' -import { Integration } from '../src/entity/integration' import { FindOptionsWhere } from 'typeorm' +import { SnakeNamingStrategy } from 'typeorm-naming-strategies' +import { Integration } from '../src/entity/integration' +import { Label } from '../src/entity/label' +import { Link } from '../src/entity/link' +import { NewsletterEmail } from '../src/entity/newsletter_email' +import { Page } from '../src/entity/page' +import { Profile } from '../src/entity/profile' +import { Reminder } from '../src/entity/reminder' +import { Subscription } from '../src/entity/subscription' +import { User } from '../src/entity/user' +import { UserDeviceToken } from '../src/entity/user_device_tokens' +import { getRepository, setClaims } from '../src/entity/utils' +import { SubscriptionStatus, SubscriptionType } from '../src/generated/graphql' +import { AppDataSource } from '../src/server' +import { createUser } from '../src/services/create_user' const runMigrations = async () => { const migrationDirectory = __dirname + '/../../db/migrations' @@ -199,7 +199,8 @@ export const createTestSubscription = async ( name: string, newsletterEmail?: NewsletterEmail, status = SubscriptionStatus.Active, - unsubscribeMailTo?: string + unsubscribeMailTo?: string, + subscriptionType = SubscriptionType.Newsletter ): Promise => { return getRepository(Subscription).save({ user, @@ -207,6 +208,8 @@ export const createTestSubscription = async ( newsletterEmail, status, unsubscribeMailTo, + lastFetchedAt: new Date(), + type: subscriptionType, }) } diff --git a/packages/api/test/resolvers/subscriptions.test.ts b/packages/api/test/resolvers/subscriptions.test.ts index d4843c362..a809128ea 100644 --- a/packages/api/test/resolvers/subscriptions.test.ts +++ b/packages/api/test/resolvers/subscriptions.test.ts @@ -6,7 +6,10 @@ import { NewsletterEmail } from '../../src/entity/newsletter_email' import { Subscription } from '../../src/entity/subscription' import { User } from '../../src/entity/user' import { getRepository } from '../../src/entity/utils' -import { SubscriptionStatus } from '../../src/generated/graphql' +import { + SubscriptionStatus, + SubscriptionType, +} from '../../src/generated/graphql' import { UNSUBSCRIBE_EMAIL_TEXT } from '../../src/services/subscriptions' import * as sendEmail from '../../src/utils/sendEmail' import { createTestSubscription, createTestUser, deleteTestUser } from '../db' @@ -35,7 +38,7 @@ describe('Subscriptions API', () => { confirmationCode: 'test', }) - // create testing subscriptions + // create testing newsletter subscriptions const sub1 = await createTestSubscription(user, 'sub_1', newsletterEmail) const sub2 = await createTestSubscription(user, 'sub_2', newsletterEmail) // create a unsubscribed subscription @@ -45,8 +48,15 @@ describe('Subscriptions API', () => { newsletterEmail, SubscriptionStatus.Unsubscribed ) - // create a subscription without a newsletter email - await createTestSubscription(user, 'sub_4') + // create an rss feed subscription + await createTestSubscription( + user, + 'sub_4', + undefined, + SubscriptionStatus.Active, + undefined, + SubscriptionType.Rss + ) subscriptions = [sub2, sub1] }) @@ -143,10 +153,7 @@ describe('Subscriptions API', () => { sinon.fake.resolves(true) ) - const res = await graphqlRequest( - query(name.toUpperCase()), - authToken - ).expect(200) + const res = await graphqlRequest(query(name), authToken).expect(200) expect(res.body.data.unsubscribe.subscription).to.eql({ id: subscription.id, From 66819f8cb3503aec794e35d4d8bf712587356738 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Thu, 6 Jul 2023 19:51:05 +0800 Subject: [PATCH 08/19] fix migration sql --- packages/db/migrations/0114.undo.add_type_to_subscriptions.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/db/migrations/0114.undo.add_type_to_subscriptions.sql b/packages/db/migrations/0114.undo.add_type_to_subscriptions.sql index 1d6b9f048..e15b0698b 100755 --- a/packages/db/migrations/0114.undo.add_type_to_subscriptions.sql +++ b/packages/db/migrations/0114.undo.add_type_to_subscriptions.sql @@ -10,6 +10,6 @@ ALTER TABLE omnivore.subscriptions DROP COLUMN count, DROP COLUMN "type"; -DROP TYPE omnivore.subscription_type; +DROP TYPE subscription_type; COMMIT; From 0ad434ada584fd81b7b42e7c1088d9b0d0b3ac43 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Thu, 6 Jul 2023 19:52:31 +0800 Subject: [PATCH 09/19] resolve conflicts --- ...to_subscriptions.sql => 0115.do.add_type_to_subscriptions.sql} | 0 ..._subscriptions.sql => 0115.undo.add_type_to_subscriptions.sql} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename packages/db/migrations/{0114.do.add_type_to_subscriptions.sql => 0115.do.add_type_to_subscriptions.sql} (100%) rename packages/db/migrations/{0114.undo.add_type_to_subscriptions.sql => 0115.undo.add_type_to_subscriptions.sql} (100%) diff --git a/packages/db/migrations/0114.do.add_type_to_subscriptions.sql b/packages/db/migrations/0115.do.add_type_to_subscriptions.sql similarity index 100% rename from packages/db/migrations/0114.do.add_type_to_subscriptions.sql rename to packages/db/migrations/0115.do.add_type_to_subscriptions.sql diff --git a/packages/db/migrations/0114.undo.add_type_to_subscriptions.sql b/packages/db/migrations/0115.undo.add_type_to_subscriptions.sql similarity index 100% rename from packages/db/migrations/0114.undo.add_type_to_subscriptions.sql rename to packages/db/migrations/0115.undo.add_type_to_subscriptions.sql From 68667053c868e002a98af76c780dfcdba28d5d9b Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Fri, 7 Jul 2023 21:15:42 +0800 Subject: [PATCH 10/19] create an endpoint to fetch all rss feeds --- packages/api/src/routers/svc/rss_feed.ts | 52 ++++++++++++++++++++++++ packages/api/src/server.ts | 2 + packages/api/src/util.ts | 5 ++- packages/api/src/utils/createTask.ts | 28 +++++++++++++ 4 files changed, 86 insertions(+), 1 deletion(-) create mode 100644 packages/api/src/routers/svc/rss_feed.ts diff --git a/packages/api/src/routers/svc/rss_feed.ts b/packages/api/src/routers/svc/rss_feed.ts new file mode 100644 index 000000000..a23f1acf0 --- /dev/null +++ b/packages/api/src/routers/svc/rss_feed.ts @@ -0,0 +1,52 @@ +/* eslint-disable @typescript-eslint/no-misused-promises */ +import express from 'express' +import { readPushSubscription } from '../../datalayer/pubsub' +import { Subscription } from '../../entity/subscription' +import { getRepository } from '../../entity/utils' +import { SubscriptionStatus, SubscriptionType } from '../../generated/graphql' +import { enqueueRssFeedFetch } from '../../utils/createTask' + +export function rssFeedRouter() { + const router = express.Router() + + router.post('/fetchAll', async (req, res) => { + console.log('fetch all rss feeds') + + const { message: msgStr, expired } = readPushSubscription(req) + console.log('read pubsub message', msgStr, 'has expired', expired) + + if (expired) { + console.log('discarding expired message') + return res.status(200).send('Expired') + } + + try { + // get all active rss feed subscriptions + const subscriptions = await getRepository(Subscription).find({ + where: { + type: SubscriptionType.Rss, + status: SubscriptionStatus.Active, + }, + relations: ['user'], + }) + + // create a cloud taks to fetch rss feed item for each subscription + await Promise.all( + subscriptions.map((subscription) => { + try { + return enqueueRssFeedFetch(subscription) + } catch (error) { + console.log('error creating rss feed fetch task', error) + } + }) + ) + + res.send('OK') + } catch (error) { + console.log('error fetching rss feeds', error) + res.status(500).send('Internal Server Error') + } + }) + + return router +} diff --git a/packages/api/src/server.ts b/packages/api/src/server.ts index 31789b6ec..bc87360ae 100755 --- a/packages/api/src/server.ts +++ b/packages/api/src/server.ts @@ -43,6 +43,7 @@ import { integrationsServiceRouter } from './routers/svc/integrations' import { linkServiceRouter } from './routers/svc/links' import { newsletterServiceRouter } from './routers/svc/newsletters' import { remindersServiceRouter } from './routers/svc/reminders' +import { rssFeedRouter } from './routers/svc/rss_feed' import { uploadServiceRouter } from './routers/svc/upload' import { webhooksServiceRouter } from './routers/svc/webhooks' import { textToSpeechRouter } from './routers/text_to_speech' @@ -159,6 +160,7 @@ export const createApp = (): { app.use('/svc/pubsub/integrations', integrationsServiceRouter()) app.use('/svc/reminders', remindersServiceRouter()) app.use('/svc/email-attachment', emailAttachmentRouter()) + app.use('/svc/rss-feed', rssFeedRouter()) if (env.dev.isLocal) { app.use('/local/debug', localDebugRouter()) diff --git a/packages/api/src/util.ts b/packages/api/src/util.ts index 2f2c9a5ef..039785d9e 100755 --- a/packages/api/src/util.ts +++ b/packages/api/src/util.ts @@ -1,8 +1,8 @@ /* eslint-disable @typescript-eslint/no-unsafe-return */ /* eslint-disable @typescript-eslint/naming-convention */ /* eslint-disable @typescript-eslint/no-explicit-any */ -import os from 'os' import * as dotenv from 'dotenv' +import os from 'os' interface BackendEnv { pg: { @@ -67,6 +67,7 @@ interface BackendEnv { textToSpeechTaskHandlerUrl: string recommendationTaskHandlerUrl: string thumbnailTaskHandlerUrl: string + rssFeedTaskHandlerUrl: string } fileUpload: { gcsUploadBucket: string @@ -161,6 +162,7 @@ const nullableEnvVars = [ 'RECOMMENDATION_TASK_HANDLER_URL', 'POCKET_CONSUMER_KEY', 'THUMBNAIL_TASK_HANDLER_URL', + 'RSS_FEED_TASK_HANDLER_URL', ] // Allow some vars to be null/empty /* If not in GAE and Prod/QA/Demo env (f.e. on localhost/dev env), allow following env vars to be null */ @@ -248,6 +250,7 @@ export function getEnv(): BackendEnv { textToSpeechTaskHandlerUrl: parse('TEXT_TO_SPEECH_TASK_HANDLER_URL'), recommendationTaskHandlerUrl: parse('RECOMMENDATION_TASK_HANDLER_URL'), thumbnailTaskHandlerUrl: parse('THUMBNAIL_TASK_HANDLER_URL'), + rssFeedTaskHandlerUrl: parse('RSS_FEED_TASK_HANDLER_URL'), } const imageProxy = { url: parse('IMAGE_PROXY_URL'), diff --git a/packages/api/src/utils/createTask.ts b/packages/api/src/utils/createTask.ts index 03752913e..f928cf1c0 100644 --- a/packages/api/src/utils/createTask.ts +++ b/packages/api/src/utils/createTask.ts @@ -6,6 +6,7 @@ import { google } from '@google-cloud/tasks/build/protos/protos' import axios from 'axios' import { nanoid } from 'nanoid' import { Recommendation } from '../elastic/types' +import { Subscription } from '../entity/subscription' import { env } from '../env' import { ArticleSavingRequestStatus, @@ -559,4 +560,31 @@ export const enqueueThumbnailTask = async ( return createdTasks[0].name } +export const enqueueRssFeedFetch = async ( + rssFeedSubscription: Subscription +): Promise => { + const { GOOGLE_CLOUD_PROJECT } = process.env + const payload = { + subscriptionId: rssFeedSubscription.id, + userId: rssFeedSubscription.user.id, + feedUrl: rssFeedSubscription.url, + } + + const createdTasks = await createHttpTaskWithToken({ + project: GOOGLE_CLOUD_PROJECT, + queue: 'omnivore-rss-feed-queue', + payload, + taskHandlerUrl: env.queue.rssFeedTaskHandlerUrl, + }) + + if (!createdTasks || !createdTasks[0].name) { + logger.error(`Unable to get the name of the task`, { + payload, + createdTasks, + }) + throw new CreateTaskError(`Unable to get the name of the task`) + } + return createdTasks[0].name +} + export default createHttpTaskWithToken From abd59f05277b4772645b22f4e00aa7cd6e9f7676 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Fri, 7 Jul 2023 21:48:16 +0800 Subject: [PATCH 11/19] save each item in a feed --- packages/rss-handler/package.json | 1 + packages/rss-handler/src/index.ts | 114 ++++++++++++++++++++++++++++-- 2 files changed, 108 insertions(+), 7 deletions(-) diff --git a/packages/rss-handler/package.json b/packages/rss-handler/package.json index e24080ae1..82b87db8c 100644 --- a/packages/rss-handler/package.json +++ b/packages/rss-handler/package.json @@ -21,6 +21,7 @@ }, "dependencies": { "@google-cloud/functions-framework": "3.1.2", + "@google-cloud/tasks": "^3.0.5", "@sentry/serverless": "^6.16.1", "axios": "^1.4.0", "dotenv": "^16.0.1", diff --git a/packages/rss-handler/src/index.ts b/packages/rss-handler/src/index.ts index c6459bbff..895af2855 100644 --- a/packages/rss-handler/src/index.ts +++ b/packages/rss-handler/src/index.ts @@ -1,6 +1,70 @@ import * as Sentry from '@sentry/serverless' +import axios from 'axios' import * as dotenv from 'dotenv' // see https://github.com/motdotla/dotenv#how-do-i-use-dotenv-with-import +import * as jwt from 'jsonwebtoken' import Parser from 'rss-parser' +import { promisify } from 'util' + +interface RssFeedRequest { + subscriptionId: string + userId: string + feedUrl: string +} + +function isRssFeedRequest(body: any): body is RssFeedRequest { + return 'subscriptionId' in body && 'userId' in body && 'feedUrl' in body +} + +const sendSavePageMutation = async (userId: string, input: unknown) => { + const JWT_SECRET = process.env.JWT_SECRET + const REST_BACKEND_ENDPOINT = process.env.REST_BACKEND_ENDPOINT + + if (!JWT_SECRET || !REST_BACKEND_ENDPOINT) { + throw 'Environment not configured correctly' + } + + const data = JSON.stringify({ + query: `mutation SavePage ($input: SavePageInput!){ + savePage(input:$input){ + ... on SaveSuccess{ + url + clientRequestId + } + ... on SaveError{ + errorCodes + } + } + }`, + variables: { + input: Object.assign({}, input, { source: 'puppeteer-parse' }), + }, + }) + + const auth = (await signToken({ uid: userId }, JWT_SECRET)) as string + try { + const response = await axios.post( + `${REST_BACKEND_ENDPOINT}/graphql`, + data, + { + headers: { + Cookie: `auth=${auth};`, + 'Content-Type': 'application/json', + }, + timeout: 30000, // 30s + } + ) + + /* eslint-disable @typescript-eslint/no-unsafe-member-access */ + return !!response.data.data.savePage + } catch (error) { + if (axios.isAxiosError(error)) { + console.error('save page mutation error', error.message) + } else { + console.error(error) + } + return false + } +} dotenv.config() Sentry.GCPFunction.init({ @@ -8,18 +72,54 @@ Sentry.GCPFunction.init({ tracesSampleRate: 0, }) +const signToken = promisify(jwt.sign) const parser = new Parser() export const rssHandler = Sentry.GCPFunction.wrapHttpFunction( async (req, res) => { - try { - const feed = await parser.parseURL('https://www.reddit.com/.rss') - console.log(feed.title) + if (!process.env.JWT_SECRET) { + console.error('Missing JWT_SECRET in environment') + return res.status(500).send('INTERNAL_SERVER_ERROR') + } - feed.items.forEach((item) => { - // eslint-disable-next-line @typescript-eslint/restrict-template-expressions - console.log(`${item.title}:${item.link}`) - }) + try { + if (!isRssFeedRequest(req.body)) { + console.error('Invalid request body', req.body) + return res.status(400).send('INVALID_REQUEST_BODY') + } + + const { userId, feedUrl } = req.body + // fetch feed + const feed = await parser.parseURL(feedUrl) + console.debug(feed.title) + + // save each item in the feed + await Promise.all( + feed.items.map((item) => { + if (!item.link || !item.title || !item.content) { + console.log('Invalid feed item', item) + return + } + + const input = { + source: 'rss-feeder', + url: item.link, + saveRequestId: '', + labels: [{ name: 'RSS' }], + title: item.title, + originalContent: item.content, + } + + try { + // save page + return sendSavePageMutation(userId, input) + } catch (error) { + console.error('Error while saving page', error) + } + }) + ) + + // TODO: update subscription lastFetchedAt res.send('ok') } catch (e) { From 56a27878dce00284ebe2867cf8dd8b4f6cd7c2e3 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Mon, 10 Jul 2023 15:24:33 +0800 Subject: [PATCH 12/19] add updateSubscription api --- packages/api/src/generated/graphql.ts | 58 +++++++++ packages/api/src/generated/schema.graphql | 24 ++++ .../api/src/resolvers/function_resolvers.ts | 14 +++ .../api/src/resolvers/subscriptions/index.ts | 119 ++++++++++++++---- packages/api/src/schema.ts | 28 +++++ 5 files changed, 218 insertions(+), 25 deletions(-) diff --git a/packages/api/src/generated/graphql.ts b/packages/api/src/generated/graphql.ts index 051bae52b..dab3bc754 100644 --- a/packages/api/src/generated/graphql.ts +++ b/packages/api/src/generated/graphql.ts @@ -1292,6 +1292,7 @@ export type Mutation = { updatePage: UpdatePageResult; updateReminder: UpdateReminderResult; updateSharedComment: UpdateSharedCommentResult; + updateSubscription: UpdateSubscriptionResult; updateUser: UpdateUserResult; updateUserProfile: UpdateUserProfileResult; uploadFileRequest: UploadFileRequestResult; @@ -1619,6 +1620,11 @@ export type MutationUpdateSharedCommentArgs = { }; +export type MutationUpdateSubscriptionArgs = { + input: UpdateSubscriptionInput; +}; + + export type MutationUpdateUserArgs = { input: UpdateUserInput; }; @@ -2995,6 +3001,31 @@ export type UpdateSharedCommentSuccess = { sharedComment: Scalars['String']; }; +export type UpdateSubscriptionError = { + __typename?: 'UpdateSubscriptionError'; + errorCodes: Array; +}; + +export enum UpdateSubscriptionErrorCode { + BadRequest = 'BAD_REQUEST', + NotFound = 'NOT_FOUND', + Unauthorized = 'UNAUTHORIZED' +} + +export type UpdateSubscriptionInput = { + description?: InputMaybe; + id: Scalars['ID']; + lastFetchedAt?: InputMaybe; + name?: InputMaybe; +}; + +export type UpdateSubscriptionResult = UpdateSubscriptionError | UpdateSubscriptionSuccess; + +export type UpdateSubscriptionSuccess = { + __typename?: 'UpdateSubscriptionSuccess'; + subscription: Subscription; +}; + export type UpdateUserError = { __typename?: 'UpdateUserError'; errorCodes: Array; @@ -3759,6 +3790,11 @@ export type ResolversTypes = { UpdateSharedCommentInput: UpdateSharedCommentInput; UpdateSharedCommentResult: ResolversTypes['UpdateSharedCommentError'] | ResolversTypes['UpdateSharedCommentSuccess']; UpdateSharedCommentSuccess: ResolverTypeWrapper; + UpdateSubscriptionError: ResolverTypeWrapper; + UpdateSubscriptionErrorCode: UpdateSubscriptionErrorCode; + UpdateSubscriptionInput: UpdateSubscriptionInput; + UpdateSubscriptionResult: ResolversTypes['UpdateSubscriptionError'] | ResolversTypes['UpdateSubscriptionSuccess']; + UpdateSubscriptionSuccess: ResolverTypeWrapper; UpdateUserError: ResolverTypeWrapper; UpdateUserErrorCode: UpdateUserErrorCode; UpdateUserInput: UpdateUserInput; @@ -4154,6 +4190,10 @@ export type ResolversParentTypes = { UpdateSharedCommentInput: UpdateSharedCommentInput; UpdateSharedCommentResult: ResolversParentTypes['UpdateSharedCommentError'] | ResolversParentTypes['UpdateSharedCommentSuccess']; UpdateSharedCommentSuccess: UpdateSharedCommentSuccess; + UpdateSubscriptionError: UpdateSubscriptionError; + UpdateSubscriptionInput: UpdateSubscriptionInput; + UpdateSubscriptionResult: ResolversParentTypes['UpdateSubscriptionError'] | ResolversParentTypes['UpdateSubscriptionSuccess']; + UpdateSubscriptionSuccess: UpdateSubscriptionSuccess; UpdateUserError: UpdateUserError; UpdateUserInput: UpdateUserInput; UpdateUserProfileError: UpdateUserProfileError; @@ -5142,6 +5182,7 @@ export type MutationResolvers>; updateReminder?: Resolver>; updateSharedComment?: Resolver>; + updateSubscription?: Resolver>; updateUser?: Resolver>; updateUserProfile?: Resolver>; uploadFileRequest?: Resolver>; @@ -5938,6 +5979,20 @@ export type UpdateSharedCommentSuccessResolvers; }; +export type UpdateSubscriptionErrorResolvers = { + errorCodes?: Resolver, ParentType, ContextType>; + __isTypeOf?: IsTypeOfResolverFn; +}; + +export type UpdateSubscriptionResultResolvers = { + __resolveType: TypeResolveFn<'UpdateSubscriptionError' | 'UpdateSubscriptionSuccess', ParentType, ContextType>; +}; + +export type UpdateSubscriptionSuccessResolvers = { + subscription?: Resolver; + __isTypeOf?: IsTypeOfResolverFn; +}; + export type UpdateUserErrorResolvers = { errorCodes?: Resolver, ParentType, ContextType>; __isTypeOf?: IsTypeOfResolverFn; @@ -6405,6 +6460,9 @@ export type Resolvers = { UpdateSharedCommentError?: UpdateSharedCommentErrorResolvers; UpdateSharedCommentResult?: UpdateSharedCommentResultResolvers; UpdateSharedCommentSuccess?: UpdateSharedCommentSuccessResolvers; + UpdateSubscriptionError?: UpdateSubscriptionErrorResolvers; + UpdateSubscriptionResult?: UpdateSubscriptionResultResolvers; + UpdateSubscriptionSuccess?: UpdateSubscriptionSuccessResolvers; UpdateUserError?: UpdateUserErrorResolvers; UpdateUserProfileError?: UpdateUserProfileErrorResolvers; UpdateUserProfileResult?: UpdateUserProfileResultResolvers; diff --git a/packages/api/src/generated/schema.graphql b/packages/api/src/generated/schema.graphql index 8a2ebbeff..11766f7b8 100644 --- a/packages/api/src/generated/schema.graphql +++ b/packages/api/src/generated/schema.graphql @@ -1160,6 +1160,7 @@ type Mutation { updatePage(input: UpdatePageInput!): UpdatePageResult! updateReminder(input: UpdateReminderInput!): UpdateReminderResult! updateSharedComment(input: UpdateSharedCommentInput!): UpdateSharedCommentResult! + updateSubscription(input: UpdateSubscriptionInput!): UpdateSubscriptionResult! updateUser(input: UpdateUserInput!): UpdateUserResult! updateUserProfile(input: UpdateUserProfileInput!): UpdateUserProfileResult! uploadFileRequest(input: UploadFileRequestInput!): UploadFileRequestResult! @@ -2323,6 +2324,29 @@ type UpdateSharedCommentSuccess { sharedComment: String! } +type UpdateSubscriptionError { + errorCodes: [UpdateSubscriptionErrorCode!]! +} + +enum UpdateSubscriptionErrorCode { + BAD_REQUEST + NOT_FOUND + UNAUTHORIZED +} + +input UpdateSubscriptionInput { + description: String + id: ID! + lastFetchedAt: Date + name: String +} + +union UpdateSubscriptionResult = UpdateSubscriptionError | UpdateSubscriptionSuccess + +type UpdateSubscriptionSuccess { + subscription: Subscription! +} + type UpdateUserError { errorCodes: [UpdateUserErrorCode!]! } diff --git a/packages/api/src/resolvers/function_resolvers.ts b/packages/api/src/resolvers/function_resolvers.ts index 49e9b6c2e..559673b21 100644 --- a/packages/api/src/resolvers/function_resolvers.ts +++ b/packages/api/src/resolvers/function_resolvers.ts @@ -5,6 +5,7 @@ /* eslint-disable @typescript-eslint/explicit-module-boundary-types */ import { getShareInfoForArticle } from '../datalayer/links/share_info' import { getPageByParam } from '../elastic/pages' +import { Subscription } from '../entity/subscription' import { Article, ArticleHighlightsInput, @@ -109,6 +110,7 @@ import { updateReminderResolver, updateSharedCommentResolver, updatesSinceResolver, + updateSubscriptionResolver, updateUserProfileResolver, updateUserResolver, uploadFileRequestResolver, @@ -206,6 +208,7 @@ export const functionResolvers = { bulkAction: bulkActionResolver, importFromIntegration: importFromIntegrationResolver, setFavoriteArticle: setFavoriteArticleResolver, + updateSubscription: updateSubscriptionResolver, }, Query: { me: getMeUserResolver, @@ -573,6 +576,16 @@ export const functionResolvers = { return item.pageType || PageType.Unknown }, }, + Subscription: { + newsletterEmail(subscription: Subscription) { + return subscription.newsletterEmail?.address + }, + icon(subscription: Subscription) { + return ( + subscription.icon && createImageProxyUrl(subscription.icon, 128, 128) + ) + }, + }, ...resultResolveTypeResolver('Login'), ...resultResolveTypeResolver('LogOut'), ...resultResolveTypeResolver('GoogleSignup'), @@ -662,4 +675,5 @@ export const functionResolvers = { ...resultResolveTypeResolver('BulkAction'), ...resultResolveTypeResolver('ImportFromIntegration'), ...resultResolveTypeResolver('SetFavoriteArticle'), + ...resultResolveTypeResolver('UpdateSubscription'), } diff --git a/packages/api/src/resolvers/subscriptions/index.ts b/packages/api/src/resolvers/subscriptions/index.ts index e4a112e15..378b0d111 100644 --- a/packages/api/src/resolvers/subscriptions/index.ts +++ b/packages/api/src/resolvers/subscriptions/index.ts @@ -6,6 +6,7 @@ import { env } from '../../env' import { MutationSubscribeArgs, MutationUnsubscribeArgs, + MutationUpdateSubscriptionArgs, QuerySubscriptionsArgs, SortBy, SortOrder, @@ -20,16 +21,25 @@ import { UnsubscribeError, UnsubscribeErrorCode, UnsubscribeSuccess, + UpdateSubscriptionError, + UpdateSubscriptionErrorCode, + UpdateSubscriptionSuccess, } from '../../generated/graphql' import { getSubscribeHandler, unsubscribe } from '../../services/subscriptions' +import { Merge } from '../../util' import { analytics } from '../../utils/analytics' import { authorized } from '../../utils/helpers' -import { createImageProxyUrl } from '../../utils/imageproxy' + +type PartialSubscription = Omit const parser = new Parser() -export const subscriptionsResolver = authorized< +export type SubscriptionsSuccessPartial = Merge< SubscriptionsSuccess, + { subscriptions: PartialSubscription[] } +> +export const subscriptionsResolver = authorized< + SubscriptionsSuccessPartial, SubscriptionsError, QuerySubscriptionsArgs >(async (_obj, { sort, type: subscriptionType }, { claims: { uid }, log }) => { @@ -66,11 +76,7 @@ export const subscriptionsResolver = authorized< .getMany() return { - subscriptions: subscriptions.map((s) => ({ - ...s, - icon: s.icon && createImageProxyUrl(s.icon, 128, 128), - newsletterEmail: s.newsletterEmail?.address, - })), + subscriptions, } } catch (error) { log.error(error) @@ -80,8 +86,12 @@ export const subscriptionsResolver = authorized< } }) -export const unsubscribeResolver = authorized< +export type UnsubscribeSuccessPartial = Merge< UnsubscribeSuccess, + { subscription: PartialSubscription } +> +export const unsubscribeResolver = authorized< + UnsubscribeSuccessPartial, UnsubscribeError, MutationUnsubscribeArgs >(async (_, { name, subscriptionId }, { claims: { uid }, log }) => { @@ -122,8 +132,12 @@ export const unsubscribeResolver = authorized< } } - if (!subscription.unsubscribeMailTo && !subscription.unsubscribeHttpUrl) { - log.info('No unsubscribe method found') + if ( + subscription.type === SubscriptionType.Newsletter && + !subscription.unsubscribeMailTo && + !subscription.unsubscribeHttpUrl + ) { + log.info('No unsubscribe method found for newsletter subscription') } await unsubscribe(subscription) @@ -138,10 +152,7 @@ export const unsubscribeResolver = authorized< }) return { - subscription: { - ...subscription, - newsletterEmail: subscription.newsletterEmail?.address, - }, + subscription, } } catch (error) { log.error('failed to unsubscribe', error) @@ -151,8 +162,12 @@ export const unsubscribeResolver = authorized< } }) -export const subscribeResolver = authorized< +export type SubscribeSuccessPartial = Merge< SubscribeSuccess, + { subscriptions: PartialSubscription[] } +> +export const subscribeResolver = authorized< + SubscribeSuccessPartial, SubscribeError, MutationSubscribeArgs >(async (_, { input }, { claims: { uid }, log }) => { @@ -209,10 +224,7 @@ export const subscribeResolver = authorized< } return { - subscriptions: newSubscriptions.map((s) => ({ - ...s, - newsletterEmail: s.newsletterEmail?.address, - })), + subscriptions: newSubscriptions, } } @@ -231,12 +243,7 @@ export const subscribeResolver = authorized< }) return { - subscriptions: [ - { - ...newSubscription, - newsletterEmail: null, - }, - ], + subscriptions: [newSubscription], } } @@ -251,3 +258,65 @@ export const subscribeResolver = authorized< } } }) + +export type UpdateSubscriptionSuccessPartial = Merge< + UpdateSubscriptionSuccess, + { subscription: PartialSubscription } +> +export const updateSubscriptionResolver = authorized< + UpdateSubscriptionSuccessPartial, + UpdateSubscriptionError, + MutationUpdateSubscriptionArgs +>(async (_, { input }, { claims: { uid }, log }) => { + log.info('updateSubscriptionResolver') + + try { + analytics.track({ + userId: uid, + event: 'update_subscription', + properties: { + ...input, + env: env.server.apiEnv, + }, + }) + + const user = await getRepository(User).findOneBy({ id: uid }) + if (!user) { + return { + errorCodes: [UpdateSubscriptionErrorCode.Unauthorized], + } + } + + // find existing subscription + const subscription = await getRepository(Subscription).findOneBy({ + id: input.id, + user: { id: uid }, + status: SubscriptionStatus.Active, + }) + if (!subscription) { + log.info('subscription not found') + return { + errorCodes: [UpdateSubscriptionErrorCode.NotFound], + } + } + + // update subscription + const updatedSubscription = await getRepository(Subscription).save({ + id: input.id, + name: input.name || undefined, + description: input.description || undefined, + lastFetchedAt: input.lastFetchedAt + ? new Date(input.lastFetchedAt) + : undefined, + }) + + return { + subscription: updatedSubscription, + } + } catch (error) { + log.error('failed to update subscription', error) + return { + errorCodes: [UpdateSubscriptionErrorCode.BadRequest], + } + } +}) diff --git a/packages/api/src/schema.ts b/packages/api/src/schema.ts index 3a4be3243..4c72d7d3a 100755 --- a/packages/api/src/schema.ts +++ b/packages/api/src/schema.ts @@ -2492,6 +2492,31 @@ const schema = gql` subscriptionType: SubscriptionType } + input UpdateSubscriptionInput { + id: ID! + name: String + description: String + lastFetchedAt: Date + } + + union UpdateSubscriptionResult = + UpdateSubscriptionSuccess + | UpdateSubscriptionError + + type UpdateSubscriptionSuccess { + subscription: Subscription! + } + + type UpdateSubscriptionError { + errorCodes: [UpdateSubscriptionErrorCode!]! + } + + enum UpdateSubscriptionErrorCode { + UNAUTHORIZED + BAD_REQUEST + NOT_FOUND + } + # Mutations type Mutation { googleLogin(input: GoogleLoginInput!): LoginResult! @@ -2590,6 +2615,9 @@ const schema = gql` ): BulkActionResult! importFromIntegration(integrationId: ID!): ImportFromIntegrationResult! setFavoriteArticle(id: ID!): SetFavoriteArticleResult! + updateSubscription( + input: UpdateSubscriptionInput! + ): UpdateSubscriptionResult! } # FIXME: remove sort from feedArticles after all cached tabs are closed From c39799fb67aa0ccc28ad0ffe5006f473ed38bf86 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Mon, 10 Jul 2023 15:54:21 +0800 Subject: [PATCH 13/19] update subscription lastFetchedAt after fetching all items --- packages/rss-handler/src/index.ts | 71 ++++++++++++++++++++++++++++++- 1 file changed, 69 insertions(+), 2 deletions(-) diff --git a/packages/rss-handler/src/index.ts b/packages/rss-handler/src/index.ts index 895af2855..b85fe9179 100644 --- a/packages/rss-handler/src/index.ts +++ b/packages/rss-handler/src/index.ts @@ -66,6 +66,66 @@ const sendSavePageMutation = async (userId: string, input: unknown) => { } } +const sendUpdateSubscriptionMutation = async ( + userId: string, + subscriptionId: string, + lastFetchedAt: Date +) => { + const JWT_SECRET = process.env.JWT_SECRET + const REST_BACKEND_ENDPOINT = process.env.REST_BACKEND_ENDPOINT + + if (!JWT_SECRET || !REST_BACKEND_ENDPOINT) { + throw 'Environment not configured correctly' + } + + const data = JSON.stringify({ + query: `mutation UpdateSubscription($input: UpdateSubscriptionInput!){ + updateSubscription(input:$input){ + ... on UpdateSubscriptionSuccess{ + subscription{ + id + lastFetchedAt + } + } + ... on UpdateSubscriptionError{ + errorCodes + } + } + }`, + variables: { + input: { + id: subscriptionId, + lastFetchedAt, + }, + }, + }) + + const auth = (await signToken({ uid: userId }, JWT_SECRET)) as string + try { + const response = await axios.post( + `${REST_BACKEND_ENDPOINT}/graphql`, + data, + { + headers: { + Cookie: `auth=${auth};`, + 'Content-Type': 'application/json', + }, + timeout: 30000, // 30s + } + ) + + /* eslint-disable @typescript-eslint/no-unsafe-member-access */ + return !!response.data.data.savePage + } catch (error) { + if (axios.isAxiosError(error)) { + console.error('update subscription mutation error', error.message) + } else { + console.error(error) + } + return false + } +} + dotenv.config() Sentry.GCPFunction.init({ dsn: process.env.SENTRY_DSN, @@ -91,7 +151,7 @@ export const rssHandler = Sentry.GCPFunction.wrapHttpFunction( const { userId, feedUrl } = req.body // fetch feed const feed = await parser.parseURL(feedUrl) - console.debug(feed.title) + console.log('Fetched feed', feed.title) // save each item in the feed await Promise.all( @@ -111,6 +171,7 @@ export const rssHandler = Sentry.GCPFunction.wrapHttpFunction( } try { + console.log('Saving page', input.title) // save page return sendSavePageMutation(userId, input) } catch (error) { @@ -119,7 +180,13 @@ export const rssHandler = Sentry.GCPFunction.wrapHttpFunction( }) ) - // TODO: update subscription lastFetchedAt + // update subscription lastFetchedAt + const updatedSubscription = await sendUpdateSubscriptionMutation( + userId, + req.body.subscriptionId, + new Date() + ) + console.log('Updated subscription', updatedSubscription) res.send('ok') } catch (e) { From 08182adbe109565d4058d321332c862930b0254b Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Mon, 10 Jul 2023 16:46:13 +0800 Subject: [PATCH 14/19] remove cloud task --- packages/rss-handler/package.json | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/rss-handler/package.json b/packages/rss-handler/package.json index 82b87db8c..e24080ae1 100644 --- a/packages/rss-handler/package.json +++ b/packages/rss-handler/package.json @@ -21,7 +21,6 @@ }, "dependencies": { "@google-cloud/functions-framework": "3.1.2", - "@google-cloud/tasks": "^3.0.5", "@sentry/serverless": "^6.16.1", "axios": "^1.4.0", "dotenv": "^16.0.1", From b88cf7a4e87c0cf3f3ab2a2a2c329061523c2ed1 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Mon, 10 Jul 2023 18:55:52 +0800 Subject: [PATCH 15/19] add rssFeedUrl field in elasticsearch index --- packages/db/elastic_migrations/index_settings.json | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/packages/db/elastic_migrations/index_settings.json b/packages/db/elastic_migrations/index_settings.json index 6b14fd5ce..6f0ec9bc3 100644 --- a/packages/db/elastic_migrations/index_settings.json +++ b/packages/db/elastic_migrations/index_settings.json @@ -159,6 +159,10 @@ "type": "keyword", "normalizer": "lowercase_normalizer" }, + "rssFeedUrl": { + "type": "keyword", + "normalizer": "lowercase_normalizer" + }, "state": { "type": "keyword" }, From 44473ba0898a9af4d9aa89ba68c798015b3cd2ae Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Tue, 11 Jul 2023 12:22:49 +0800 Subject: [PATCH 16/19] upload feed subscriptions in cloud storage in cronjob --- packages/api/src/elastic/types.ts | 1 + packages/api/src/routers/svc/rss_feed.ts | 41 +++++++++++------ packages/rss-handler/package.json | 1 + packages/rss-handler/src/index.ts | 54 +++++++++++------------ packages/rss-handler/src/task.ts | 56 ++++++++++++++++++++++++ 5 files changed, 113 insertions(+), 40 deletions(-) create mode 100644 packages/rss-handler/src/task.ts diff --git a/packages/api/src/elastic/types.ts b/packages/api/src/elastic/types.ts index 320604c3b..b6e5bf41b 100644 --- a/packages/api/src/elastic/types.ts +++ b/packages/api/src/elastic/types.ts @@ -162,6 +162,7 @@ export interface Page { listenedAt?: Date wordsCount?: number recommendations?: Recommendation[] + rssFeedUrl?: string } export interface SearchItem { diff --git a/packages/api/src/routers/svc/rss_feed.ts b/packages/api/src/routers/svc/rss_feed.ts index a23f1acf0..e5a96af71 100644 --- a/packages/api/src/routers/svc/rss_feed.ts +++ b/packages/api/src/routers/svc/rss_feed.ts @@ -1,10 +1,12 @@ /* eslint-disable @typescript-eslint/no-misused-promises */ +import { stringify } from 'csv-stringify/.' import express from 'express' +import { DateTime } from 'luxon' import { readPushSubscription } from '../../datalayer/pubsub' import { Subscription } from '../../entity/subscription' import { getRepository } from '../../entity/utils' import { SubscriptionStatus, SubscriptionType } from '../../generated/graphql' -import { enqueueRssFeedFetch } from '../../utils/createTask' +import { createGCSFile } from '../../utils/uploads' export function rssFeedRouter() { const router = express.Router() @@ -20,9 +22,11 @@ export function rssFeedRouter() { return res.status(200).send('Expired') } + let writeStream: NodeJS.WritableStream | undefined try { // get all active rss feed subscriptions const subscriptions = await getRepository(Subscription).find({ + select: ['id', 'url', 'user'], where: { type: SubscriptionType.Rss, status: SubscriptionStatus.Active, @@ -30,22 +34,33 @@ export function rssFeedRouter() { relations: ['user'], }) - // create a cloud taks to fetch rss feed item for each subscription - await Promise.all( - subscriptions.map((subscription) => { - try { - return enqueueRssFeedFetch(subscription) - } catch (error) { - console.log('error creating rss feed fetch task', error) - } - }) - ) + // write the list of subscriptions to a csv file and upload it to gcs + // path style: rss/.csv + const dateStr = DateTime.now().toISODate() + const fullPath = `rss/${dateStr}.csv` + // open a write_stream to the file + const file = createGCSFile(fullPath) + writeStream = file.createWriteStream({ + contentType: 'text/csv', + }) + // stringify the data and pipe it to the write_stream + const stringifier = stringify({ + header: false, + columns: ['subscriptionId', 'userId', 'feedUrl'], + }) + stringifier.pipe(writeStream) - res.send('OK') + subscriptions.forEach((sub) => { + stringifier.write([sub.id, sub.user.id, sub.url]) + }) } catch (error) { console.log('error fetching rss feeds', error) - res.status(500).send('Internal Server Error') + return res.status(500).send('Internal Server Error') + } finally { + writeStream?.end() } + + res.send('OK') }) return router diff --git a/packages/rss-handler/package.json b/packages/rss-handler/package.json index e24080ae1..82b87db8c 100644 --- a/packages/rss-handler/package.json +++ b/packages/rss-handler/package.json @@ -21,6 +21,7 @@ }, "dependencies": { "@google-cloud/functions-framework": "3.1.2", + "@google-cloud/tasks": "^3.0.5", "@sentry/serverless": "^6.16.1", "axios": "^1.4.0", "dotenv": "^16.0.1", diff --git a/packages/rss-handler/src/index.ts b/packages/rss-handler/src/index.ts index b85fe9179..933a9703b 100644 --- a/packages/rss-handler/src/index.ts +++ b/packages/rss-handler/src/index.ts @@ -148,43 +148,43 @@ export const rssHandler = Sentry.GCPFunction.wrapHttpFunction( return res.status(400).send('INVALID_REQUEST_BODY') } - const { userId, feedUrl } = req.body + const { userId, feedUrl, subscriptionId } = req.body // fetch feed const feed = await parser.parseURL(feedUrl) - console.log('Fetched feed', feed.title) + const lastFetchedAt = new Date() + console.log('Fetched feed', feed.title, lastFetchedAt) // save each item in the feed - await Promise.all( - feed.items.map((item) => { - if (!item.link || !item.title || !item.content) { - console.log('Invalid feed item', item) - return - } + for (const item of feed.items) { + if (!item.link || !item.title || !item.content) { + console.log('Invalid feed item', item) + continue + } - const input = { - source: 'rss-feeder', - url: item.link, - saveRequestId: '', - labels: [{ name: 'RSS' }], - title: item.title, - originalContent: item.content, - } + const input = { + source: 'rss-feeder', + url: item.link, + saveRequestId: '', + labels: [{ name: 'RSS' }], + title: item.title, + originalContent: item.content, + } - try { - console.log('Saving page', input.title) - // save page - return sendSavePageMutation(userId, input) - } catch (error) { - console.error('Error while saving page', error) - } - }) - ) + try { + console.log('Saving page', input.title) + // save page + const result = await sendSavePageMutation(userId, input) + console.log('Saved page', result) + } catch (error) { + console.error('Error while saving page', error) + } + } // update subscription lastFetchedAt const updatedSubscription = await sendUpdateSubscriptionMutation( userId, - req.body.subscriptionId, - new Date() + subscriptionId, + lastFetchedAt ) console.log('Updated subscription', updatedSubscription) diff --git a/packages/rss-handler/src/task.ts b/packages/rss-handler/src/task.ts new file mode 100644 index 000000000..94b3352cc --- /dev/null +++ b/packages/rss-handler/src/task.ts @@ -0,0 +1,56 @@ +/* eslint-disable @typescript-eslint/restrict-template-expressions */ +import { CloudTasksClient, protos } from '@google-cloud/tasks' + +const cloudTask = new CloudTasksClient() + +export const emailUserUrl = () => { + const envar = process.env.INTERNAL_SVC_ENDPOINT + if (envar) { + return envar + 'api/user/email' + } + throw 'INTERNAL_SVC_ENDPOINT not set' +} + +export const CONTENT_FETCH_URL = process.env.CONTENT_FETCH_GCF_URL + +export const createCloudTask = async ( + taskHandlerUrl: string | undefined, + payload: unknown, + requestHeaders?: Record, + queue = 'omnivore-import-queue' +) => { + const location = process.env.GCP_LOCATION + const project = process.env.GCP_PROJECT_ID + + if (!project || !location || !queue || !taskHandlerUrl) { + throw `Environment not configured: ${project}, ${location}, ${queue}, ${taskHandlerUrl}` + } + + const serviceAccountEmail = `${project}@appspot.gserviceaccount.com` + + const parent = cloudTask.queuePath(project, location, queue) + const convertedPayload = JSON.stringify(payload) + const body = Buffer.from(convertedPayload).toString('base64') + const task: protos.google.cloud.tasks.v2.ITask = { + httpRequest: { + httpMethod: 'POST', + url: taskHandlerUrl, + headers: { + 'Content-Type': 'application/json', + ...requestHeaders, + }, + body, + ...(serviceAccountEmail + ? { + oidcToken: { + serviceAccountEmail, + }, + } + : null), + }, + } + + return cloudTask.createTask({ parent, task }).then((result) => { + return result[0].name ?? undefined + }) +} From eb9a3eddd0be7b26627335033a36fcca0a95a972 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Tue, 11 Jul 2023 12:59:55 +0800 Subject: [PATCH 17/19] skip old item --- packages/api/src/routers/svc/rss_feed.ts | 40 ++++++----------- packages/rss-handler/package.json | 1 - packages/rss-handler/src/index.ts | 23 +++++++--- packages/rss-handler/src/task.ts | 56 ------------------------ 4 files changed, 30 insertions(+), 90 deletions(-) delete mode 100644 packages/rss-handler/src/task.ts diff --git a/packages/api/src/routers/svc/rss_feed.ts b/packages/api/src/routers/svc/rss_feed.ts index e5a96af71..e19436bee 100644 --- a/packages/api/src/routers/svc/rss_feed.ts +++ b/packages/api/src/routers/svc/rss_feed.ts @@ -1,12 +1,10 @@ /* eslint-disable @typescript-eslint/no-misused-promises */ -import { stringify } from 'csv-stringify/.' import express from 'express' -import { DateTime } from 'luxon' import { readPushSubscription } from '../../datalayer/pubsub' import { Subscription } from '../../entity/subscription' import { getRepository } from '../../entity/utils' import { SubscriptionStatus, SubscriptionType } from '../../generated/graphql' -import { createGCSFile } from '../../utils/uploads' +import { enqueueRssFeedFetch } from '../../utils/createTask' export function rssFeedRouter() { const router = express.Router() @@ -22,7 +20,6 @@ export function rssFeedRouter() { return res.status(200).send('Expired') } - let writeStream: NodeJS.WritableStream | undefined try { // get all active rss feed subscriptions const subscriptions = await getRepository(Subscription).find({ @@ -34,33 +31,22 @@ export function rssFeedRouter() { relations: ['user'], }) - // write the list of subscriptions to a csv file and upload it to gcs - // path style: rss/.csv - const dateStr = DateTime.now().toISODate() - const fullPath = `rss/${dateStr}.csv` - // open a write_stream to the file - const file = createGCSFile(fullPath) - writeStream = file.createWriteStream({ - contentType: 'text/csv', - }) - // stringify the data and pipe it to the write_stream - const stringifier = stringify({ - header: false, - columns: ['subscriptionId', 'userId', 'feedUrl'], - }) - stringifier.pipe(writeStream) + // create a cloud taks to fetch rss feed item for each subscription + await Promise.all( + subscriptions.map((subscription) => { + try { + return enqueueRssFeedFetch(subscription) + } catch (error) { + console.log('error creating rss feed fetch task', error) + } + }) + ) - subscriptions.forEach((sub) => { - stringifier.write([sub.id, sub.user.id, sub.url]) - }) + res.send('OK') } catch (error) { console.log('error fetching rss feeds', error) - return res.status(500).send('Internal Server Error') - } finally { - writeStream?.end() + res.status(500).send('Internal Server Error') } - - res.send('OK') }) return router diff --git a/packages/rss-handler/package.json b/packages/rss-handler/package.json index 82b87db8c..e24080ae1 100644 --- a/packages/rss-handler/package.json +++ b/packages/rss-handler/package.json @@ -21,7 +21,6 @@ }, "dependencies": { "@google-cloud/functions-framework": "3.1.2", - "@google-cloud/tasks": "^3.0.5", "@sentry/serverless": "^6.16.1", "axios": "^1.4.0", "dotenv": "^16.0.1", diff --git a/packages/rss-handler/src/index.ts b/packages/rss-handler/src/index.ts index 933a9703b..c4e36fbb0 100644 --- a/packages/rss-handler/src/index.ts +++ b/packages/rss-handler/src/index.ts @@ -9,10 +9,16 @@ interface RssFeedRequest { subscriptionId: string userId: string feedUrl: string + lastFetchedAt: Date } function isRssFeedRequest(body: any): body is RssFeedRequest { - return 'subscriptionId' in body && 'userId' in body && 'feedUrl' in body + return ( + 'subscriptionId' in body && + 'userId' in body && + 'feedUrl' in body && + 'lastFetchedAt' in body + ) } const sendSavePageMutation = async (userId: string, input: unknown) => { @@ -148,19 +154,24 @@ export const rssHandler = Sentry.GCPFunction.wrapHttpFunction( return res.status(400).send('INVALID_REQUEST_BODY') } - const { userId, feedUrl, subscriptionId } = req.body + const { userId, feedUrl, subscriptionId, lastFetchedAt } = req.body // fetch feed const feed = await parser.parseURL(feedUrl) - const lastFetchedAt = new Date() - console.log('Fetched feed', feed.title, lastFetchedAt) + const newFetchedAt = new Date() + console.log('Fetched feed', feed.title, newFetchedAt) // save each item in the feed for (const item of feed.items) { - if (!item.link || !item.title || !item.content) { + if (!item.link || !item.title || !item.content || !item.isoDate) { console.log('Invalid feed item', item) continue } + if (new Date(item.isoDate) <= lastFetchedAt) { + console.log('Skipping old feed item', item.title) + continue + } + const input = { source: 'rss-feeder', url: item.link, @@ -184,7 +195,7 @@ export const rssHandler = Sentry.GCPFunction.wrapHttpFunction( const updatedSubscription = await sendUpdateSubscriptionMutation( userId, subscriptionId, - lastFetchedAt + newFetchedAt ) console.log('Updated subscription', updatedSubscription) diff --git a/packages/rss-handler/src/task.ts b/packages/rss-handler/src/task.ts deleted file mode 100644 index 94b3352cc..000000000 --- a/packages/rss-handler/src/task.ts +++ /dev/null @@ -1,56 +0,0 @@ -/* eslint-disable @typescript-eslint/restrict-template-expressions */ -import { CloudTasksClient, protos } from '@google-cloud/tasks' - -const cloudTask = new CloudTasksClient() - -export const emailUserUrl = () => { - const envar = process.env.INTERNAL_SVC_ENDPOINT - if (envar) { - return envar + 'api/user/email' - } - throw 'INTERNAL_SVC_ENDPOINT not set' -} - -export const CONTENT_FETCH_URL = process.env.CONTENT_FETCH_GCF_URL - -export const createCloudTask = async ( - taskHandlerUrl: string | undefined, - payload: unknown, - requestHeaders?: Record, - queue = 'omnivore-import-queue' -) => { - const location = process.env.GCP_LOCATION - const project = process.env.GCP_PROJECT_ID - - if (!project || !location || !queue || !taskHandlerUrl) { - throw `Environment not configured: ${project}, ${location}, ${queue}, ${taskHandlerUrl}` - } - - const serviceAccountEmail = `${project}@appspot.gserviceaccount.com` - - const parent = cloudTask.queuePath(project, location, queue) - const convertedPayload = JSON.stringify(payload) - const body = Buffer.from(convertedPayload).toString('base64') - const task: protos.google.cloud.tasks.v2.ITask = { - httpRequest: { - httpMethod: 'POST', - url: taskHandlerUrl, - headers: { - 'Content-Type': 'application/json', - ...requestHeaders, - }, - body, - ...(serviceAccountEmail - ? { - oidcToken: { - serviceAccountEmail, - }, - } - : null), - }, - } - - return cloudTask.createTask({ parent, task }).then((result) => { - return result[0].name ?? undefined - }) -} From c500997693c5499d743cc2742abc9664e854a577 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Tue, 11 Jul 2023 13:05:27 +0800 Subject: [PATCH 18/19] add lastFetchedAt to the cloud task payload --- packages/api/src/routers/svc/rss_feed.ts | 2 +- packages/api/src/utils/createTask.ts | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/api/src/routers/svc/rss_feed.ts b/packages/api/src/routers/svc/rss_feed.ts index e19436bee..337fe79bb 100644 --- a/packages/api/src/routers/svc/rss_feed.ts +++ b/packages/api/src/routers/svc/rss_feed.ts @@ -23,7 +23,7 @@ export function rssFeedRouter() { try { // get all active rss feed subscriptions const subscriptions = await getRepository(Subscription).find({ - select: ['id', 'url', 'user'], + select: ['id', 'url', 'user', 'lastFetchedAt'], where: { type: SubscriptionType.Rss, status: SubscriptionStatus.Active, diff --git a/packages/api/src/utils/createTask.ts b/packages/api/src/utils/createTask.ts index f928cf1c0..2820c864f 100644 --- a/packages/api/src/utils/createTask.ts +++ b/packages/api/src/utils/createTask.ts @@ -568,6 +568,7 @@ export const enqueueRssFeedFetch = async ( subscriptionId: rssFeedSubscription.id, userId: rssFeedSubscription.user.id, feedUrl: rssFeedSubscription.url, + lastFetchedAt: rssFeedSubscription.lastFetchedAt, } const createdTasks = await createHttpTaskWithToken({ From b1b4a2e3fd882f22114a0e4545e52d0ee88290ee Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Tue, 11 Jul 2023 13:14:43 +0800 Subject: [PATCH 19/19] save rss feed url in the index --- packages/api/src/generated/graphql.ts | 1 + packages/api/src/generated/schema.graphql | 1 + packages/api/src/schema.ts | 1 + packages/api/src/services/save_page.ts | 4 ++++ packages/rss-handler/src/index.ts | 1 + 5 files changed, 8 insertions(+) diff --git a/packages/api/src/generated/graphql.ts b/packages/api/src/generated/graphql.ts index dab3bc754..63ca8ee91 100644 --- a/packages/api/src/generated/graphql.ts +++ b/packages/api/src/generated/graphql.ts @@ -2257,6 +2257,7 @@ export type SavePageInput = { labels?: InputMaybe>; originalContent: Scalars['String']; parseResult?: InputMaybe; + rssFeedUrl?: InputMaybe; source: Scalars['String']; state?: InputMaybe; title?: InputMaybe; diff --git a/packages/api/src/generated/schema.graphql b/packages/api/src/generated/schema.graphql index 11766f7b8..d015776ab 100644 --- a/packages/api/src/generated/schema.graphql +++ b/packages/api/src/generated/schema.graphql @@ -1637,6 +1637,7 @@ input SavePageInput { labels: [CreateLabelInput!] originalContent: String! parseResult: ParseResult + rssFeedUrl: String source: String! state: ArticleSavingRequestStatus title: String diff --git a/packages/api/src/schema.ts b/packages/api/src/schema.ts index 4c72d7d3a..5c9fd1163 100755 --- a/packages/api/src/schema.ts +++ b/packages/api/src/schema.ts @@ -561,6 +561,7 @@ const schema = gql` parseResult: ParseResult state: ArticleSavingRequestStatus labels: [CreateLabelInput!] + rssFeedUrl: String } input SaveUrlInput { diff --git a/packages/api/src/services/save_page.ts b/packages/api/src/services/save_page.ts index 560b6d3f0..158e9eb10 100644 --- a/packages/api/src/services/save_page.ts +++ b/packages/api/src/services/save_page.ts @@ -103,6 +103,7 @@ export const savePage = async ( pageType: parseResult.pageType, originalHtml: parseResult.domContent, canonicalUrl: parseResult.canonicalUrl, + rssFeedUrl: input.rssFeedUrl, }) // save state @@ -221,6 +222,7 @@ export const parsedContentToPage = ({ uploadFileHash, uploadFileId, saveTime, + rssFeedUrl, }: { url: string userId: string @@ -236,6 +238,7 @@ export const parsedContentToPage = ({ uploadFileHash?: string | null uploadFileId?: string | null saveTime?: Date + rssFeedUrl?: string | null }): Page => { return { id: pageId || '', @@ -267,5 +270,6 @@ export const parsedContentToPage = ({ language: parsedContent?.language ?? undefined, siteIcon: parsedContent?.siteIcon ?? undefined, wordsCount: wordsCount(parsedContent?.textContent || ''), + rssFeedUrl: rssFeedUrl || undefined, } } diff --git a/packages/rss-handler/src/index.ts b/packages/rss-handler/src/index.ts index c4e36fbb0..6133d67c5 100644 --- a/packages/rss-handler/src/index.ts +++ b/packages/rss-handler/src/index.ts @@ -179,6 +179,7 @@ export const rssHandler = Sentry.GCPFunction.wrapHttpFunction( labels: [{ name: 'RSS' }], title: item.title, originalContent: item.content, + rssFeedUrl: feedUrl, } try {