From 56698ae065df416ac8bb1d8ed866e0cd11fe6909 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Mon, 21 Aug 2023 20:30:20 +0800 Subject: [PATCH 001/144] add create library_item sql --- .../db/migrations/0118.do.library_item.sql | 59 +++++++++++++++++++ .../db/migrations/0118.undo.library_item.sql | 17 ++++++ 2 files changed, 76 insertions(+) create mode 100755 packages/db/migrations/0118.do.library_item.sql create mode 100755 packages/db/migrations/0118.undo.library_item.sql diff --git a/packages/db/migrations/0118.do.library_item.sql b/packages/db/migrations/0118.do.library_item.sql new file mode 100755 index 000000000..3742c4c17 --- /dev/null +++ b/packages/db/migrations/0118.do.library_item.sql @@ -0,0 +1,59 @@ +-- Type: DO +-- Name: library_item +-- Description: Create library_item table + +BEGIN; + +CREATE EXTENSION vector; + +CREATE TYPE library_item_state AS ENUM ('SUCCEEDED', 'FAILED', 'PROCESSING', 'ARCHIVED', 'DELETED'); + +CREATE TYPE content_reader_type AS ENUM ('WEB', 'PDF', 'EPUB'); + +CREATE TABLE omnivore.library_item ( + id uuid PRIMARY KEY DEFAULT uuid_generate_v1mc(), + user_id uuid NOT NULL REFERENCES omnivore.user ON DELETE CASCADE, + state library_item_state NOT NULL DEFAULT 'SUCCEEDED', + original_url text NOT NULL, + download_url text, + slug text NOT NULL, + title text NOT NULL, + author text, + description text, + saved_at timestamptz NOT NULL DEFAULT current_timestamp, + created_at timestamptz NOT NULL DEFAULT current_timestamp, + published_at timestamptz, + archived_at timestamptz, + deleted_at timestamptz, + read_at timestamptz, + updated_at timestamptz NOT NULL DEFAULT current_timestamp, + language text, + words_count integer, + site_name text, + site_icon text, + metadata JSON, + reading_progress_last_read_anchor integer, + reading_progress_highest_read_anchor integer, + reading_progress_top_percent real, + reading_progress_bottom_percent real, + thumbnail text, + page_type text, + upload_file_id uuid REFERENCES omnivore.upload_files ON DELETE CASCADE, + content_reader content_reader_type, + original_content text, + readable_content text, + content_tsv tsvector, + site_tsv tsvector, + title_tsv tsvector, + author_tsv tsvector, + description_tsv tsvector, + search_tsv tsvector, + model_name text, + embedding vector(768), + text_content_hash text, + gcs_archive_id text +); + +CREATE TRIGGER update_library_item_modtime BEFORE UPDATE ON omnivore.library_item FOR EACH ROW EXECUTE PROCEDURE update_updated_at_column(); + +COMMIT; diff --git a/packages/db/migrations/0118.undo.library_item.sql b/packages/db/migrations/0118.undo.library_item.sql new file mode 100755 index 000000000..ab53082ae --- /dev/null +++ b/packages/db/migrations/0118.undo.library_item.sql @@ -0,0 +1,17 @@ +-- Type: UNDO +-- Name: library_item +-- Description: Create library_item table + +BEGIN; + +DROP TRIGGER update_library_item_modtime ON omnivore.library_item; + +DROP TABLE omnivore.library_item; + +DROP TYPE content_reader_type; + +DROP TYPE library_item_state; + +DROP EXTENSION vector; + +COMMIT; From 9d2ca1f9345bd755df2d0481f4a800c0d6619b97 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Mon, 21 Aug 2023 20:45:30 +0800 Subject: [PATCH 002/144] change page_type to enum --- packages/db/migrations/0118.do.library_item.sql | 6 ++++-- packages/db/migrations/0118.undo.library_item.sql | 2 ++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/packages/db/migrations/0118.do.library_item.sql b/packages/db/migrations/0118.do.library_item.sql index 3742c4c17..1dbb1a8d0 100755 --- a/packages/db/migrations/0118.do.library_item.sql +++ b/packages/db/migrations/0118.do.library_item.sql @@ -10,6 +10,8 @@ CREATE TYPE library_item_state AS ENUM ('SUCCEEDED', 'FAILED', 'PROCESSING', 'AR CREATE TYPE content_reader_type AS ENUM ('WEB', 'PDF', 'EPUB'); +CREATE TYPE library_item_type AS ENUM ('ARTICLE', 'BOOK', 'FILE', 'PROFILE', 'WEBSITE', 'TWEET', 'VIDEO', 'IMAGE', 'UNKNOWN'); + CREATE TABLE omnivore.library_item ( id uuid PRIMARY KEY DEFAULT uuid_generate_v1mc(), user_id uuid NOT NULL REFERENCES omnivore.user ON DELETE CASCADE, @@ -27,7 +29,7 @@ CREATE TABLE omnivore.library_item ( deleted_at timestamptz, read_at timestamptz, updated_at timestamptz NOT NULL DEFAULT current_timestamp, - language text, + item_language text, words_count integer, site_name text, site_icon text, @@ -37,7 +39,7 @@ CREATE TABLE omnivore.library_item ( reading_progress_top_percent real, reading_progress_bottom_percent real, thumbnail text, - page_type text, + item_type library_item_type, upload_file_id uuid REFERENCES omnivore.upload_files ON DELETE CASCADE, content_reader content_reader_type, original_content text, diff --git a/packages/db/migrations/0118.undo.library_item.sql b/packages/db/migrations/0118.undo.library_item.sql index ab53082ae..991cfb401 100755 --- a/packages/db/migrations/0118.undo.library_item.sql +++ b/packages/db/migrations/0118.undo.library_item.sql @@ -8,6 +8,8 @@ DROP TRIGGER update_library_item_modtime ON omnivore.library_item; DROP TABLE omnivore.library_item; +DROP TYPE library_item_type; + DROP TYPE content_reader_type; DROP TYPE library_item_state; From 927414fe182401acafe75c0b2e8eb705415a5d41 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Mon, 21 Aug 2023 20:52:15 +0800 Subject: [PATCH 003/144] add sql to create library_item_subscriptions db table --- .../0119.do.library_item_subscriptions.sql | 13 +++++++++++++ .../0119.undo.library_item_subscriptions.sql | 9 +++++++++ 2 files changed, 22 insertions(+) create mode 100755 packages/db/migrations/0119.do.library_item_subscriptions.sql create mode 100755 packages/db/migrations/0119.undo.library_item_subscriptions.sql diff --git a/packages/db/migrations/0119.do.library_item_subscriptions.sql b/packages/db/migrations/0119.do.library_item_subscriptions.sql new file mode 100755 index 000000000..0700299ef --- /dev/null +++ b/packages/db/migrations/0119.do.library_item_subscriptions.sql @@ -0,0 +1,13 @@ +-- Type: DO +-- Name: library_item_subscriptions +-- Description: Create a join table for library item and subscriptions + +BEGIN; + +CREATE TABLE omnivore.library_item_subscriptions ( + library_item_id uuid NOT NULL REFERENCES omnivore.library_item(id) ON DELETE CASCADE, + subscription_id uuid NOT NULL REFERENCES omnivore.subscriptions(id) ON DELETE CASCADE, + PRIMARY KEY (library_item_id, subscription_id) +); + +COMMIT; diff --git a/packages/db/migrations/0119.undo.library_item_subscriptions.sql b/packages/db/migrations/0119.undo.library_item_subscriptions.sql new file mode 100755 index 000000000..dbedc0b21 --- /dev/null +++ b/packages/db/migrations/0119.undo.library_item_subscriptions.sql @@ -0,0 +1,9 @@ +-- Type: UNDO +-- Name: library_item_subscriptions +-- Description: Create a join table for library item and subscriptions + +BEGIN; + +DROP TABLE omnivore.library_item_subscriptions; + +COMMIT; From e722b0e2485df1a8a57dfbc5daffd45bd9f27e20 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Mon, 21 Aug 2023 21:15:26 +0800 Subject: [PATCH 004/144] add sql to create library_item_labels db table --- .../db/migrations/0120.do.library_item_labels.sql | 13 +++++++++++++ .../db/migrations/0120.undo.library_item_labels.sql | 9 +++++++++ 2 files changed, 22 insertions(+) create mode 100755 packages/db/migrations/0120.do.library_item_labels.sql create mode 100755 packages/db/migrations/0120.undo.library_item_labels.sql diff --git a/packages/db/migrations/0120.do.library_item_labels.sql b/packages/db/migrations/0120.do.library_item_labels.sql new file mode 100755 index 000000000..f9bcba00a --- /dev/null +++ b/packages/db/migrations/0120.do.library_item_labels.sql @@ -0,0 +1,13 @@ +-- Type: DO +-- Name: library_item_labels +-- Description: Create table library_item_labels + +BEGIN; + +CREATE TABLE omnivore.library_item_labels ( + library_item_id uuid NOT NULL REFERENCES omnivore.library_item(id) ON DELETE CASCADE, + label_id uuid NOT NULL REFERENCES omnivore.labels(id) ON DELETE CASCADE, + PRIMARY KEY (library_item_id, label_id) +); + +COMMIT; diff --git a/packages/db/migrations/0120.undo.library_item_labels.sql b/packages/db/migrations/0120.undo.library_item_labels.sql new file mode 100755 index 000000000..ec7ac7cb7 --- /dev/null +++ b/packages/db/migrations/0120.undo.library_item_labels.sql @@ -0,0 +1,9 @@ +-- Type: UNDO +-- Name: library_item_labels +-- Description: Create table library_item_labels + +BEGIN; + +DROP TABLE omnivore.library_item_labels; + +COMMIT; From 37b7d9a4c7e6f853847b67f02d8ed74018e78389 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Mon, 21 Aug 2023 21:27:15 +0800 Subject: [PATCH 005/144] add sql for creating library_item_preview db table --- .../0121.do.library_item_preview.sql | 21 +++++++++++++++++++ .../0121.undo.library_item_preview.sql | 11 ++++++++++ 2 files changed, 32 insertions(+) create mode 100755 packages/db/migrations/0121.do.library_item_preview.sql create mode 100755 packages/db/migrations/0121.undo.library_item_preview.sql diff --git a/packages/db/migrations/0121.do.library_item_preview.sql b/packages/db/migrations/0121.do.library_item_preview.sql new file mode 100755 index 000000000..4b8a0189a --- /dev/null +++ b/packages/db/migrations/0121.do.library_item_preview.sql @@ -0,0 +1,21 @@ +-- Type: DO +-- Name: library_item_preview +-- Description: Create library_item_preview table + +BEGIN; + +CREATE TABLE omnivore.library_item_preview ( + id uuid PRIMARY KEY DEFAULT uuid_generate_v1mc(), + sender_id uuid NOT NULL REFERENCES omnivore.user ON DELETE CASCADE, + recipient_ids uuid[] NOT NULL, -- Array of user ids + library_item_id uuid NOT NULL REFERENCES omnivore.library_item(id) ON DELETE CASCADE, + thumbnail text, + includes_note bool NOT NULL DEFAULT false, + includes_highlight bool NOT NULL DEFAULT false, + created_at timestamptz NOT NULL DEFAULT current_timestamp, + updated_at timestamptz NOT NULL DEFAULT current_timestamp +); + +CREATE TRIGGER update_library_item_preview_modtime BEFORE UPDATE ON omnivore.library_item_preview FOR EACH ROW EXECUTE PROCEDURE update_updated_at_column(); + +COMMIT; diff --git a/packages/db/migrations/0121.undo.library_item_preview.sql b/packages/db/migrations/0121.undo.library_item_preview.sql new file mode 100755 index 000000000..c55d96584 --- /dev/null +++ b/packages/db/migrations/0121.undo.library_item_preview.sql @@ -0,0 +1,11 @@ +-- Type: UNDO +-- Name: library_item_preview +-- Description: Create library_item_preview table + +BEGIN; + +DROP TRIGGER update_library_item_preview_modtime; + +DROP TABLE omnivore.library_item_preview; + +COMMIT; From e1af53ab486a2d231d511b55b7ba6605ae87f318 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Mon, 21 Aug 2023 23:27:11 +0800 Subject: [PATCH 006/144] add libraryItem entity class --- packages/api/src/entity/library_item.ts | 169 ++++++++++++++++++ packages/api/tsconfig.json | 4 +- .../db/migrations/0118.do.library_item.sql | 37 +++- .../db/migrations/0118.undo.library_item.sql | 14 +- .../0121.undo.library_item_preview.sql | 2 +- 5 files changed, 217 insertions(+), 9 deletions(-) create mode 100644 packages/api/src/entity/library_item.ts diff --git a/packages/api/src/entity/library_item.ts b/packages/api/src/entity/library_item.ts new file mode 100644 index 000000000..730f6c4a6 --- /dev/null +++ b/packages/api/src/entity/library_item.ts @@ -0,0 +1,169 @@ +import { + Column, + CreateDateColumn, + Entity, + JoinColumn, + OneToOne, + PrimaryGeneratedColumn, + UpdateDateColumn, +} from 'typeorm' +import { UploadFile } from './upload_file' +import { User } from './user' + +export enum LibraryItemState { + Failed = 'FAILED', + Processing = 'PROCESSING', + Succeeded = 'SUCCEEDED', + Deleted = 'DELETED', + Archived = 'ARCHIVED', +} + +export enum LibraryItemType { + Article = 'ARTICLE', + Book = 'BOOK', + File = 'FILE', + Profile = 'PROFILE', + Website = 'WEBSITE', + Tweet = 'TWEET', + Video = 'VIDEO', + Image = 'IMAGE', + Unknown = 'UNKNOWN', +} + +export enum ContentReaderType { + WEB = 'WEB', + PDF = 'PDF', + EPUB = 'EPUB', +} + +@Entity({ name: 'library_item' }) +export class LibraryItem { + @PrimaryGeneratedColumn('uuid') + id?: string + + @OneToOne(() => User, { onDelete: 'CASCADE' }) + @JoinColumn({ name: 'user_id' }) + user!: User + + @Column('enum', { + enum: LibraryItemState, + default: LibraryItemState.Succeeded, + }) + state?: LibraryItemState + + @Column('text') + originalUrl!: string + + @Column('text', { nullable: true }) + downloadUrl?: string + + @Column('text') + slug!: string + + @Column('text') + title!: string + + @Column('text', { nullable: true }) + author?: string + + @Column('text', { nullable: true }) + description?: string + + @Column('timestamptz') + savedAt?: Date + + @CreateDateColumn() + createdAt?: Date + + @Column('timestamptz', { nullable: true }) + publishedAt?: Date + + @Column('timestamptz') + archivedAt?: Date + + @Column('timestamptz') + deletedAt?: Date + + @Column('timestamptz') + readAt?: Date + + @UpdateDateColumn() + updatedAt?: Date + + @Column('text', { nullable: true }) + itemLanguage?: string + + @Column('integer', { nullable: true }) + wordCount?: number + + @Column('text', { nullable: true }) + siteName?: string + + @Column('text', { nullable: true }) + siteIcon?: string + + @Column('json', { nullable: true }) + metadata?: Record + + @Column('integer', { nullable: true }) + readingProgressLastReadAnchor?: number + + @Column('integer', { nullable: true }) + readingProgressHighestReadAnchor?: number + + @Column('real', { nullable: true }) + readingProgressTopPercent?: number + + @Column('real', { nullable: true }) + readingProgressBottomPercent?: number + + @Column('text', { nullable: true }) + thumbnail?: string + + @Column('enum', { enum: LibraryItemType, default: LibraryItemType.Unknown }) + itemType?: LibraryItemType + + @OneToOne(() => UploadFile, { onDelete: 'CASCADE' }) + @JoinColumn({ name: 'upload_file_id' }) + uploadFile?: UploadFile + + @Column('enum', { enum: ContentReaderType, default: ContentReaderType.WEB }) + contentReader?: ContentReaderType + + @Column('text', { nullable: true }) + originalContent?: string + + @Column('text', { nullable: true }) + readableContent?: string + + @Column('tsvector', { nullable: true }) + contentTsv?: string + + @Column('tsvector', { nullable: true }) + siteTsv?: string + + @Column('tsvector', { nullable: true }) + titleTsv?: string + + @Column('tsvector', { nullable: true }) + authorTsv?: string + + @Column('tsvector', { nullable: true }) + descriptionTsv?: string + + @Column('tsvector', { nullable: true }) + searchTsv?: string + + @Column('text', { nullable: true }) + modelName?: string + + // NOT SUPPORTED IN TYPEORM + // @Column('vector', { nullable: true }) + // embedding?: number[] + + @Column('text', { nullable: true }) + textContentHash?: string + + @Column('text', { nullable: true }) + gcsArchiveId?: string +} diff --git a/packages/api/tsconfig.json b/packages/api/tsconfig.json index 7c8caecfd..9d64d8069 100644 --- a/packages/api/tsconfig.json +++ b/packages/api/tsconfig.json @@ -4,7 +4,9 @@ "files": true }, "compilerOptions": { - "outDir": "dist" + "outDir": "dist", + "emitDecoratorMetadata": true, + "experimentalDecorators": true }, "include": ["src", "test"], "exclude": ["./src/generated", "./test"] diff --git a/packages/db/migrations/0118.do.library_item.sql b/packages/db/migrations/0118.do.library_item.sql index 1dbb1a8d0..5d3ec8765 100755 --- a/packages/db/migrations/0118.do.library_item.sql +++ b/packages/db/migrations/0118.do.library_item.sql @@ -30,7 +30,7 @@ CREATE TABLE omnivore.library_item ( read_at timestamptz, updated_at timestamptz NOT NULL DEFAULT current_timestamp, item_language text, - words_count integer, + word_count integer, site_name text, site_icon text, metadata JSON, @@ -39,9 +39,9 @@ CREATE TABLE omnivore.library_item ( reading_progress_top_percent real, reading_progress_bottom_percent real, thumbnail text, - item_type library_item_type, + item_type library_item_type NOT NULL DEFAULT 'UNKNOWN', upload_file_id uuid REFERENCES omnivore.upload_files ON DELETE CASCADE, - content_reader content_reader_type, + content_reader content_reader_type NOT NULL DEFAULT 'WEB', original_content text, readable_content text, content_tsv tsvector, @@ -58,4 +58,35 @@ CREATE TABLE omnivore.library_item ( CREATE TRIGGER update_library_item_modtime BEFORE UPDATE ON omnivore.library_item FOR EACH ROW EXECUTE PROCEDURE update_updated_at_column(); +CREATE INDEX library_item_content_tsv_idx ON omnivore.library_item USING GIN (content_tsv); +CREATE INDEX library_item_site_tsv_idx ON omnivore.library_item USING GIN (site_tsv); +CREATE INDEX library_item_title_tsv_idx ON omnivore.library_item USING GIN (title_tsv); +CREATE INDEX library_item_author_tsv_idx ON omnivore.library_item USING GIN (author_tsv); +CREATE INDEX library_item_description_tsv_idx ON omnivore.library_item USING GIN (description_tsv); +CREATE INDEX library_item_search_tsv_idx ON omnivore.library_item USING GIN (search_tsv); + +CREATE OR REPLACE FUNCTION update_library_item_tsv() RETURNS trigger AS $$ +begin + new.content_tsv := to_tsvector('pg_catalog.english', coalesce(new.readable_content, '')); + new.site_tsv := to_tsvector('pg_catalog.english', coalesce(new.site_name, '')); + new.title_tsv := to_tsvector('pg_catalog.english', coalesce(new.title, '')); + new.author_tsv := to_tsvector('pg_catalog.english', coalesce(new.author, '')); + new.description_tsv := to_tsvector('pg_catalog.english', coalesce(new.description, '')); + new.search_tsv := + setweight(new.title_tsv, 'A') || + setweight(new.author_tsv, 'A') || + setweight(new.site_tsv, 'A') || + setweight(new.description_tsv, 'A') || + -- full hostname (eg www.omnivore.app) + setweight(to_tsvector('pg_catalog.english', coalesce(regexp_replace(new.url, '^((http[s]?):\/)?\/?([^:\/\s]+)((\/\w+)*\/)([\w\-\.]+[^#?\s]+)(.*)?(#[\w\-]+)?$', '\3'), '')), 'A') || + -- secondary hostname (eg omnivore) + setweight(to_tsvector('pg_catalog.english', coalesce(regexp_replace(new.url, '^((http[s]?):\/)?\/?(.*\.)?([^:\/\s]+)(\..*)((\/+)*\/)?([\w\-\.]+[^#?\s]+)(.*)?(#[\w\-]+)?$', '\4'), '')), 'A') || + setweight(new.content_tsv, 'B'); + return new; +end +$$ LANGUAGE plpgsql; + +CREATE TRIGGER library_item_tsv_update BEFORE INSERT OR UPDATE + ON omnivore.library_item FOR EACH ROW EXECUTE PROCEDURE update_library_item_tsv(); + COMMIT; diff --git a/packages/db/migrations/0118.undo.library_item.sql b/packages/db/migrations/0118.undo.library_item.sql index 991cfb401..6197c6c44 100755 --- a/packages/db/migrations/0118.undo.library_item.sql +++ b/packages/db/migrations/0118.undo.library_item.sql @@ -4,14 +4,20 @@ BEGIN; +DROP TRIGGER library_item_tsv_update ON omnivore.library_item; +DROP FUNCTION update_library_item_tsv(); + +DROP INDEX library_item_search_tsv_idx; +DROP INDEX library_item_description_tsv_idx; +DROP INDEX library_item_author_tsv_idx; +DROP INDEX library_item_title_tsv_idx; +DROP INDEX library_item_site_tsv_idx; +DROP INDEX library_item_content_tsv_idx; + DROP TRIGGER update_library_item_modtime ON omnivore.library_item; - DROP TABLE omnivore.library_item; - DROP TYPE library_item_type; - DROP TYPE content_reader_type; - DROP TYPE library_item_state; DROP EXTENSION vector; diff --git a/packages/db/migrations/0121.undo.library_item_preview.sql b/packages/db/migrations/0121.undo.library_item_preview.sql index c55d96584..0644e036d 100755 --- a/packages/db/migrations/0121.undo.library_item_preview.sql +++ b/packages/db/migrations/0121.undo.library_item_preview.sql @@ -4,7 +4,7 @@ BEGIN; -DROP TRIGGER update_library_item_preview_modtime; +DROP TRIGGER update_library_item_preview_modtime ON omnivore.library_item_preview; DROP TABLE omnivore.library_item_preview; From 928ed2e6a9a298b398f0aa57b3f04479c38ff6a7 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Tue, 22 Aug 2023 00:02:38 +0800 Subject: [PATCH 007/144] add libraryItemPreview entity class --- packages/api/src/entity/label.ts | 7 +++ packages/api/src/entity/library_item.ts | 20 +++++++++ .../api/src/entity/library_item_preview.ts | 43 +++++++++++++++++++ packages/api/tsconfig.json | 4 +- .../db/migrations/0118.do.library_item.sql | 8 ++-- .../db/migrations/0118.undo.library_item.sql | 13 +++--- ...ls.sql => 0119.do.library_item_labels.sql} | 0 .../0119.do.library_item_subscriptions.sql | 13 ------ ....sql => 0119.undo.library_item_labels.sql} | 0 .../0119.undo.library_item_subscriptions.sql | 9 ---- ...w.sql => 0120.do.library_item_preview.sql} | 0 ...sql => 0120.undo.library_item_preview.sql} | 0 12 files changed, 83 insertions(+), 34 deletions(-) create mode 100644 packages/api/src/entity/library_item_preview.ts rename packages/db/migrations/{0120.do.library_item_labels.sql => 0119.do.library_item_labels.sql} (100%) delete mode 100755 packages/db/migrations/0119.do.library_item_subscriptions.sql rename packages/db/migrations/{0120.undo.library_item_labels.sql => 0119.undo.library_item_labels.sql} (100%) delete mode 100755 packages/db/migrations/0119.undo.library_item_subscriptions.sql rename packages/db/migrations/{0121.do.library_item_preview.sql => 0120.do.library_item_preview.sql} (100%) rename packages/db/migrations/{0121.undo.library_item_preview.sql => 0120.undo.library_item_preview.sql} (100%) diff --git a/packages/api/src/entity/label.ts b/packages/api/src/entity/label.ts index dd0c65079..384cf4ed9 100644 --- a/packages/api/src/entity/label.ts +++ b/packages/api/src/entity/label.ts @@ -3,9 +3,12 @@ import { CreateDateColumn, Entity, JoinColumn, + JoinTable, + ManyToMany, ManyToOne, PrimaryGeneratedColumn, } from 'typeorm' +import { LibraryItem } from './library_item' import { User } from './user' @Entity({ name: 'labels' }) @@ -34,4 +37,8 @@ export class Label { @Column('boolean', { default: false }) internal!: boolean + + @ManyToMany(() => LibraryItem, (libraryItem) => libraryItem.labels) + @JoinTable() + libraryItems?: LibraryItem[] } diff --git a/packages/api/src/entity/library_item.ts b/packages/api/src/entity/library_item.ts index 730f6c4a6..04228c67f 100644 --- a/packages/api/src/entity/library_item.ts +++ b/packages/api/src/entity/library_item.ts @@ -3,10 +3,14 @@ import { CreateDateColumn, Entity, JoinColumn, + JoinTable, + ManyToMany, OneToOne, PrimaryGeneratedColumn, UpdateDateColumn, } from 'typeorm' +import { Label } from './label' +import { Subscription } from './subscription' import { UploadFile } from './upload_file' import { User } from './user' @@ -36,6 +40,11 @@ export enum ContentReaderType { EPUB = 'EPUB', } +export enum DirectionalityType { + LTR = 'LTR', + RTL = 'RTL', +} + @Entity({ name: 'library_item' }) export class LibraryItem { @PrimaryGeneratedColumn('uuid') @@ -166,4 +175,15 @@ export class LibraryItem { @Column('text', { nullable: true }) gcsArchiveId?: string + + @OneToOne(() => Subscription, { onDelete: 'CASCADE' }) + @JoinColumn({ name: 'subscription_id' }) + subscription?: Subscription + + @ManyToMany(() => Label, (label) => label.libraryItems) + @JoinTable() + labels?: Label[] + + @Column('enum', { enum: DirectionalityType, default: DirectionalityType.LTR }) + directionality?: DirectionalityType } diff --git a/packages/api/src/entity/library_item_preview.ts b/packages/api/src/entity/library_item_preview.ts new file mode 100644 index 000000000..cf2a9b7dd --- /dev/null +++ b/packages/api/src/entity/library_item_preview.ts @@ -0,0 +1,43 @@ +import { + Column, + CreateDateColumn, + Entity, + JoinColumn, + OneToOne, + PrimaryGeneratedColumn, + UpdateDateColumn, +} from 'typeorm' +import { LibraryItem } from './library_item' +import { User } from './user' + +@Entity({ name: 'library_item_preview' }) +export class LibraryItemPreview { + @PrimaryGeneratedColumn('uuid') + id?: string + + @OneToOne(() => User, { onDelete: 'CASCADE' }) + @JoinColumn({ name: 'sender_id' }) + sender!: User + + @Column('array', { name: 'recipient_ids' }) + recipientIds!: string[] + + @OneToOne(() => LibraryItem, { onDelete: 'CASCADE' }) + @JoinColumn({ name: 'library_item_id' }) + libraryItem!: LibraryItem + + @Column('text') + thumbnail?: string + + @Column('bool', { default: false }) + includesNote?: boolean + + @Column('bool', { default: false }) + includesHighlight?: boolean + + @CreateDateColumn() + createdAt?: Date + + @UpdateDateColumn() + updatedAt?: Date +} diff --git a/packages/api/tsconfig.json b/packages/api/tsconfig.json index 9d64d8069..7c8caecfd 100644 --- a/packages/api/tsconfig.json +++ b/packages/api/tsconfig.json @@ -4,9 +4,7 @@ "files": true }, "compilerOptions": { - "outDir": "dist", - "emitDecoratorMetadata": true, - "experimentalDecorators": true + "outDir": "dist" }, "include": ["src", "test"], "exclude": ["./src/generated", "./test"] diff --git a/packages/db/migrations/0118.do.library_item.sql b/packages/db/migrations/0118.do.library_item.sql index 5d3ec8765..5d8e6a459 100755 --- a/packages/db/migrations/0118.do.library_item.sql +++ b/packages/db/migrations/0118.do.library_item.sql @@ -7,10 +7,9 @@ BEGIN; CREATE EXTENSION vector; CREATE TYPE library_item_state AS ENUM ('SUCCEEDED', 'FAILED', 'PROCESSING', 'ARCHIVED', 'DELETED'); - CREATE TYPE content_reader_type AS ENUM ('WEB', 'PDF', 'EPUB'); - CREATE TYPE library_item_type AS ENUM ('ARTICLE', 'BOOK', 'FILE', 'PROFILE', 'WEBSITE', 'TWEET', 'VIDEO', 'IMAGE', 'UNKNOWN'); +CREATE TYPE directionality_type AS ENUM ('LTR', 'RTL'); CREATE TABLE omnivore.library_item ( id uuid PRIMARY KEY DEFAULT uuid_generate_v1mc(), @@ -53,7 +52,10 @@ CREATE TABLE omnivore.library_item ( model_name text, embedding vector(768), text_content_hash text, - gcs_archive_id text + gcs_archive_id text, + directionality directionality_type NOT NULL DEFAULT 'LTR', + subscription_id uuid REFERENCES omnivore.subscriptions ON DELETE CASCADE, + UNIQUE (user_id, original_url) ); CREATE TRIGGER update_library_item_modtime BEFORE UPDATE ON omnivore.library_item FOR EACH ROW EXECUTE PROCEDURE update_updated_at_column(); diff --git a/packages/db/migrations/0118.undo.library_item.sql b/packages/db/migrations/0118.undo.library_item.sql index 6197c6c44..202b831dd 100755 --- a/packages/db/migrations/0118.undo.library_item.sql +++ b/packages/db/migrations/0118.undo.library_item.sql @@ -7,15 +7,16 @@ BEGIN; DROP TRIGGER library_item_tsv_update ON omnivore.library_item; DROP FUNCTION update_library_item_tsv(); -DROP INDEX library_item_search_tsv_idx; -DROP INDEX library_item_description_tsv_idx; -DROP INDEX library_item_author_tsv_idx; -DROP INDEX library_item_title_tsv_idx; -DROP INDEX library_item_site_tsv_idx; -DROP INDEX library_item_content_tsv_idx; +DROP INDEX omnivore.library_item_search_tsv_idx; +DROP INDEX omnivore.library_item_description_tsv_idx; +DROP INDEX omnivore.library_item_author_tsv_idx; +DROP INDEX omnivore.library_item_title_tsv_idx; +DROP INDEX omnivore.library_item_site_tsv_idx; +DROP INDEX omnivore.library_item_content_tsv_idx; DROP TRIGGER update_library_item_modtime ON omnivore.library_item; DROP TABLE omnivore.library_item; +DROP TYPE directionality_type; DROP TYPE library_item_type; DROP TYPE content_reader_type; DROP TYPE library_item_state; diff --git a/packages/db/migrations/0120.do.library_item_labels.sql b/packages/db/migrations/0119.do.library_item_labels.sql similarity index 100% rename from packages/db/migrations/0120.do.library_item_labels.sql rename to packages/db/migrations/0119.do.library_item_labels.sql diff --git a/packages/db/migrations/0119.do.library_item_subscriptions.sql b/packages/db/migrations/0119.do.library_item_subscriptions.sql deleted file mode 100755 index 0700299ef..000000000 --- a/packages/db/migrations/0119.do.library_item_subscriptions.sql +++ /dev/null @@ -1,13 +0,0 @@ --- Type: DO --- Name: library_item_subscriptions --- Description: Create a join table for library item and subscriptions - -BEGIN; - -CREATE TABLE omnivore.library_item_subscriptions ( - library_item_id uuid NOT NULL REFERENCES omnivore.library_item(id) ON DELETE CASCADE, - subscription_id uuid NOT NULL REFERENCES omnivore.subscriptions(id) ON DELETE CASCADE, - PRIMARY KEY (library_item_id, subscription_id) -); - -COMMIT; diff --git a/packages/db/migrations/0120.undo.library_item_labels.sql b/packages/db/migrations/0119.undo.library_item_labels.sql similarity index 100% rename from packages/db/migrations/0120.undo.library_item_labels.sql rename to packages/db/migrations/0119.undo.library_item_labels.sql diff --git a/packages/db/migrations/0119.undo.library_item_subscriptions.sql b/packages/db/migrations/0119.undo.library_item_subscriptions.sql deleted file mode 100755 index dbedc0b21..000000000 --- a/packages/db/migrations/0119.undo.library_item_subscriptions.sql +++ /dev/null @@ -1,9 +0,0 @@ --- Type: UNDO --- Name: library_item_subscriptions --- Description: Create a join table for library item and subscriptions - -BEGIN; - -DROP TABLE omnivore.library_item_subscriptions; - -COMMIT; diff --git a/packages/db/migrations/0121.do.library_item_preview.sql b/packages/db/migrations/0120.do.library_item_preview.sql similarity index 100% rename from packages/db/migrations/0121.do.library_item_preview.sql rename to packages/db/migrations/0120.do.library_item_preview.sql diff --git a/packages/db/migrations/0121.undo.library_item_preview.sql b/packages/db/migrations/0120.undo.library_item_preview.sql similarity index 100% rename from packages/db/migrations/0121.undo.library_item_preview.sql rename to packages/db/migrations/0120.undo.library_item_preview.sql From 425864ce010faddfd6bfb39b8b20a0706fd3c058 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Tue, 22 Aug 2023 22:00:48 +0800 Subject: [PATCH 008/144] remove knex --- packages/api/src/apollo.ts | 21 +- .../api/src/entity/{utils.ts => index.ts} | 7 + packages/api/src/entity/label.ts | 7 - packages/api/src/entity/library_item.ts | 64 +- packages/api/src/entity/profile.ts | 8 +- packages/api/src/entity/user.ts | 19 +- packages/api/src/events/user/user_created.ts | 2 +- packages/api/src/resolvers/api_key/index.ts | 2 +- packages/api/src/resolvers/article/index.ts | 157 ++--- .../resolvers/article_saving_request/index.ts | 2 +- packages/api/src/resolvers/filters/index.ts | 11 +- .../api/src/resolvers/function_resolvers.ts | 398 ++++++------ .../importers/uploadImportFileResolver.ts | 2 +- .../api/src/resolvers/integrations/index.ts | 2 +- packages/api/src/resolvers/labels/index.ts | 2 +- packages/api/src/resolvers/links/index.ts | 71 +-- .../api/src/resolvers/newsletters/index.ts | 2 +- packages/api/src/resolvers/reaction/index.ts | 228 ++++--- .../api/src/resolvers/recent_emails/index.ts | 2 +- .../src/resolvers/recent_searches/index.ts | 10 +- .../src/resolvers/recommendations/index.ts | 2 +- packages/api/src/resolvers/reminders/index.ts | 571 +++++++++--------- packages/api/src/resolvers/rules/index.ts | 8 +- packages/api/src/resolvers/save/index.ts | 35 +- .../api/src/resolvers/subscriptions/index.ts | 2 +- packages/api/src/resolvers/types.ts | 38 +- .../api/src/resolvers/upload_files/index.ts | 50 +- packages/api/src/resolvers/user/index.ts | 181 +++--- .../src/resolvers/user_feed_article/index.ts | 256 ++++---- .../api/src/resolvers/user_friends/index.ts | 199 +++--- .../resolvers/user_personalization/index.ts | 14 +- packages/api/src/resolvers/webhooks/index.ts | 10 +- packages/api/src/routers/article_router.ts | 2 +- packages/api/src/routers/auth/auth_router.ts | 2 +- packages/api/src/routers/page_router.ts | 10 +- packages/api/src/routers/svc/content.ts | 11 +- .../api/src/routers/svc/email_attachment.ts | 18 +- packages/api/src/routers/svc/integrations.ts | 2 +- packages/api/src/routers/svc/reminders.ts | 199 +++--- packages/api/src/routers/svc/rss_feed.ts | 2 +- packages/api/src/routers/svc/webhooks.ts | 2 +- packages/api/src/routers/text_to_speech.ts | 2 +- packages/api/src/routers/user_router.ts | 2 +- packages/api/src/schema.ts | 70 +-- packages/api/src/server.ts | 32 +- packages/api/src/services/archive_link.ts | 2 +- .../src/services/create_page_save_request.ts | 2 +- packages/api/src/services/create_user.ts | 28 +- packages/api/src/services/features.ts | 2 +- packages/api/src/services/followers.ts | 4 +- packages/api/src/services/groups.ts | 2 +- .../api/src/services/integrations/readwise.ts | 2 +- packages/api/src/services/labels.ts | 14 +- packages/api/src/services/newsletters.ts | 6 +- packages/api/src/services/received_emails.ts | 2 +- packages/api/src/services/reminders.ts | 24 +- packages/api/src/services/reports.ts | 2 +- packages/api/src/services/rules.ts | 4 +- packages/api/src/services/save_file.ts | 35 +- packages/api/src/services/save_page.ts | 42 +- packages/api/src/services/save_url.ts | 4 +- packages/api/src/services/search_history.ts | 2 +- packages/api/src/services/subscriptions.ts | 2 +- .../api/src/services/user_device_tokens.ts | 8 +- packages/api/src/utils/auth.ts | 2 +- packages/api/src/utils/helpers.ts | 11 +- packages/api/src/utils/parser.ts | 2 +- packages/api/test/db.ts | 2 +- packages/api/test/resolvers/api_key.test.ts | 2 +- packages/api/test/resolvers/article.test.ts | 2 +- packages/api/test/resolvers/features.test.ts | 2 +- .../api/test/resolvers/integrations.test.ts | 2 +- packages/api/test/resolvers/labels.test.ts | 2 +- .../api/test/resolvers/newsletters.test.ts | 2 +- .../api/test/resolvers/recent_emails.test.ts | 2 +- .../test/resolvers/recent_searches.test.ts | 2 +- packages/api/test/resolvers/report.test.ts | 2 +- packages/api/test/resolvers/rules.test.ts | 2 +- .../api/test/resolvers/subscriptions.test.ts | 2 +- .../test/resolvers/user_device_tokens.test.ts | 2 +- .../test/resolvers/user_feed_article.test.ts | 2 +- packages/api/test/resolvers/webhooks.test.ts | 2 +- packages/api/test/routers/auth.test.ts | 2 +- packages/api/test/routers/emails.test.ts | 2 +- .../api/test/routers/integrations.test.ts | 2 +- packages/api/test/routers/webhooks.test.ts | 2 +- .../create_content_display_report.test.ts | 5 +- .../api/test/services/create_user.test.ts | 2 +- packages/api/test/services/labels.test.ts | 2 +- .../services/save_newsletter_email.test.ts | 2 +- 90 files changed, 1431 insertions(+), 1557 deletions(-) rename packages/api/src/entity/{utils.ts => index.ts} (61%) diff --git a/packages/api/src/apollo.ts b/packages/api/src/apollo.ts index 4ea00c319..ea6f75d7f 100644 --- a/packages/api/src/apollo.ts +++ b/packages/api/src/apollo.ts @@ -10,9 +10,8 @@ import { ContextFunction } from 'apollo-server-core' import { ApolloServer } from 'apollo-server-express' import { ExpressContext } from 'apollo-server-express/dist/ApolloServer' import * as jwt from 'jsonwebtoken' -import { Knex } from 'knex' +import { EntityManager } from 'typeorm' import { promisify } from 'util' -import { kx } from './datalayer/knex_config' import { createPubSubClient } from './datalayer/pubsub' import { sanitizeDirectiveTransformer } from './directives' import { env } from './env' @@ -20,7 +19,7 @@ import { functionResolvers } from './resolvers/function_resolvers' import { ClaimsToSet, ResolverContext } from './resolvers/types' import ScalarResolvers from './scalars' import typeDefs from './schema' -import { initModels } from './server' +import { AppDataSource } from './server' import { tracer } from './tracing' import { getClaimsByToken, setAuthInCookie } from './utils/auth' import { SetClaimsRole } from './utils/dictionary' @@ -47,7 +46,7 @@ const contextFunc: ContextFunction = async ({ const claims = await getClaimsByToken(token) async function setClaims( - tx: Knex.Transaction, + em: EntityManager, uuid?: string, userRole?: string ): Promise { @@ -55,18 +54,14 @@ const contextFunc: ContextFunction = async ({ (claims && claims.uid) || uuid || '00000000-0000-0000-0000-000000000000' const dbRole = userRole === SetClaimsRole.ADMIN ? 'omnivore_admin' : 'omnivore_user' - return tx.raw('SELECT * from omnivore.set_claims(?, ?)', [uid, dbRole]) + return em.query('SELECT * from omnivore.set_claims($1, $2)', [uid, dbRole]) } const ctx = { log: logger, claims, - kx, pubsub, // no caching for subscriptions - // TODO: create per request caching for connections - // eslint-disable-next-line @typescript-eslint/no-explicit-any - models: initModels(kx, true), clearAuth: () => { res.clearCookie('auth') res.clearCookie('pendingUserAuth') @@ -78,12 +73,12 @@ const contextFunc: ContextFunction = async ({ ) => await setAuthInCookie(claims, res, secret), setClaims, authTrx: ( - cb: (tx: Knex.Transaction) => TResult, + cb: (em: EntityManager) => TResult, userRole?: string ): Promise => - kx.transaction(async (tx) => { - await setClaims(tx, undefined, userRole) - return cb(tx) + AppDataSource.transaction(async (em) => { + await setClaims(em, undefined, userRole) + return cb(em) }), tracingSpan: tracer.startSpan('apollo.request'), } diff --git a/packages/api/src/entity/utils.ts b/packages/api/src/entity/index.ts similarity index 61% rename from packages/api/src/entity/utils.ts rename to packages/api/src/entity/index.ts index 1ee7933cc..0c87ae2d2 100644 --- a/packages/api/src/entity/utils.ts +++ b/packages/api/src/entity/index.ts @@ -1,5 +1,8 @@ import { EntityManager, EntityTarget, Repository } from 'typeorm' import { AppDataSource } from '../server' +import { Reminder } from './reminder' +import { UploadFile } from './upload_file' +import { User } from './user' export const setClaims = async ( t: EntityManager, @@ -14,3 +17,7 @@ export const setClaims = async ( export const getRepository = (entity: EntityTarget): Repository => { return AppDataSource.getRepository(entity) } + +export const userRepository = getRepository(User) +export const uploadFileRepository = getRepository(UploadFile) +export const reminderRepository = getRepository(Reminder) diff --git a/packages/api/src/entity/label.ts b/packages/api/src/entity/label.ts index 384cf4ed9..dd0c65079 100644 --- a/packages/api/src/entity/label.ts +++ b/packages/api/src/entity/label.ts @@ -3,12 +3,9 @@ import { CreateDateColumn, Entity, JoinColumn, - JoinTable, - ManyToMany, ManyToOne, PrimaryGeneratedColumn, } from 'typeorm' -import { LibraryItem } from './library_item' import { User } from './user' @Entity({ name: 'labels' }) @@ -37,8 +34,4 @@ export class Label { @Column('boolean', { default: false }) internal!: boolean - - @ManyToMany(() => LibraryItem, (libraryItem) => libraryItem.labels) - @JoinTable() - libraryItems?: LibraryItem[] } diff --git a/packages/api/src/entity/library_item.ts b/packages/api/src/entity/library_item.ts index 04228c67f..6a8325a9b 100644 --- a/packages/api/src/entity/library_item.ts +++ b/packages/api/src/entity/library_item.ts @@ -64,7 +64,7 @@ export class LibraryItem { originalUrl!: string @Column('text', { nullable: true }) - downloadUrl?: string + downloadUrl?: string | null @Column('text') slug!: string @@ -73,10 +73,10 @@ export class LibraryItem { title!: string @Column('text', { nullable: true }) - author?: string + author?: string | null @Column('text', { nullable: true }) - description?: string + description?: string | null @Column('timestamptz') savedAt?: Date @@ -85,49 +85,49 @@ export class LibraryItem { createdAt?: Date @Column('timestamptz', { nullable: true }) - publishedAt?: Date + publishedAt?: Date | null @Column('timestamptz') - archivedAt?: Date + archivedAt?: Date | null @Column('timestamptz') - deletedAt?: Date + deletedAt?: Date | null @Column('timestamptz') - readAt?: Date + readAt?: Date | null @UpdateDateColumn() updatedAt?: Date @Column('text', { nullable: true }) - itemLanguage?: string + itemLanguage?: string | null @Column('integer', { nullable: true }) - wordCount?: number + wordCount?: number | null @Column('text', { nullable: true }) - siteName?: string + siteName?: string | null @Column('text', { nullable: true }) - siteIcon?: string + siteIcon?: string | null @Column('json', { nullable: true }) - metadata?: Record + metadata?: Record | null @Column('integer', { nullable: true }) - readingProgressLastReadAnchor?: number + readingProgressLastReadAnchor?: number | null @Column('integer', { nullable: true }) - readingProgressHighestReadAnchor?: number + readingProgressHighestReadAnchor?: number | null @Column('real', { nullable: true }) - readingProgressTopPercent?: number + readingProgressTopPercent?: number | null @Column('real', { nullable: true }) - readingProgressBottomPercent?: number + readingProgressBottomPercent?: number | null @Column('text', { nullable: true }) - thumbnail?: string + thumbnail?: string | null @Column('enum', { enum: LibraryItemType, default: LibraryItemType.Unknown }) itemType?: LibraryItemType @@ -140,47 +140,29 @@ export class LibraryItem { contentReader?: ContentReaderType @Column('text', { nullable: true }) - originalContent?: string + originalContent?: string | null @Column('text', { nullable: true }) - readableContent?: string - - @Column('tsvector', { nullable: true }) - contentTsv?: string - - @Column('tsvector', { nullable: true }) - siteTsv?: string - - @Column('tsvector', { nullable: true }) - titleTsv?: string - - @Column('tsvector', { nullable: true }) - authorTsv?: string - - @Column('tsvector', { nullable: true }) - descriptionTsv?: string - - @Column('tsvector', { nullable: true }) - searchTsv?: string + readableContent?: string | null @Column('text', { nullable: true }) - modelName?: string + modelName?: string | null // NOT SUPPORTED IN TYPEORM // @Column('vector', { nullable: true }) // embedding?: number[] @Column('text', { nullable: true }) - textContentHash?: string + textContentHash?: string | null @Column('text', { nullable: true }) - gcsArchiveId?: string + gcsArchiveId?: string | null @OneToOne(() => Subscription, { onDelete: 'CASCADE' }) @JoinColumn({ name: 'subscription_id' }) subscription?: Subscription - @ManyToMany(() => Label, (label) => label.libraryItems) + @ManyToMany(() => Label) @JoinTable() labels?: Label[] diff --git a/packages/api/src/entity/profile.ts b/packages/api/src/entity/profile.ts index b5f04b48d..9fdc96b76 100644 --- a/packages/api/src/entity/profile.ts +++ b/packages/api/src/entity/profile.ts @@ -18,11 +18,11 @@ export class Profile { @Column('text') username!: string - @Column('text') - bio!: string + @Column('text', { nullable: true }) + bio?: string | null - @Column('text') - pictureUrl!: string + @Column('text', { nullable: true }) + pictureUrl?: string | null @OneToOne(() => User, (user) => user.profile) @JoinColumn({ name: 'user_id' }) diff --git a/packages/api/src/entity/user.ts b/packages/api/src/entity/user.ts index 60859dc91..a926b5a89 100644 --- a/packages/api/src/entity/user.ts +++ b/packages/api/src/entity/user.ts @@ -7,13 +7,23 @@ import { PrimaryGeneratedColumn, UpdateDateColumn, } from 'typeorm' -import { RegistrationType, StatusType } from '../datalayer/user/model' +import { Label } from './label' import { NewsletterEmail } from './newsletter_email' import { Profile } from './profile' -import { Label } from './label' import { Subscription } from './subscription' import { UserPersonalization } from './user_personalization' +export enum RegistrationType { + Google = 'GOOGLE', + Apple = 'APPLE', + Email = 'EMAIL', +} + +export enum StatusType { + Active = 'ACTIVE', + Pending = 'PENDING', +} + @Entity() export class User { @PrimaryGeneratedColumn('uuid') @@ -40,7 +50,10 @@ export class User { @OneToMany(() => NewsletterEmail, (newsletterEmail) => newsletterEmail.user) newsletterEmails?: NewsletterEmail[] - @OneToOne(() => Profile, (profile) => profile.user, { eager: true }) + @OneToOne(() => Profile, (profile) => profile.user, { + eager: true, + cascade: true, + }) profile!: Profile @Column('varchar', { length: 255, nullable: true }) diff --git a/packages/api/src/events/user/user_created.ts b/packages/api/src/events/user/user_created.ts index 13c46aae0..d73383dae 100644 --- a/packages/api/src/events/user/user_created.ts +++ b/packages/api/src/events/user/user_created.ts @@ -26,7 +26,7 @@ export class CreateIntercomAccount email: profile.user.email, externalId: profile.user.id, name: profile.user.name, - avatar: profile.pictureUrl, + avatar: profile.pictureUrl || undefined, customAttributes: customAttributes, signedUpAt: Math.floor(Date.now() / 1000), }) diff --git a/packages/api/src/resolvers/api_key/index.ts b/packages/api/src/resolvers/api_key/index.ts index 94ec3c753..8b5cc6a5b 100644 --- a/packages/api/src/resolvers/api_key/index.ts +++ b/packages/api/src/resolvers/api_key/index.ts @@ -1,6 +1,6 @@ +import { getRepository } from '../../entity' import { ApiKey } from '../../entity/api_key' import { User } from '../../entity/user' -import { getRepository } from '../../entity/utils' import { env } from '../../env' import { ApiKeysError, diff --git a/packages/api/src/resolvers/article/index.ts b/packages/api/src/resolvers/article/index.ts index b05a80643..d50532d44 100644 --- a/packages/api/src/resolvers/article/index.ts +++ b/packages/api/src/resolvers/article/index.ts @@ -20,6 +20,9 @@ import { PageType, SearchItem as SearchItemData, } from '../../elastic/types' +import { getRepository } from '../../entity' +import { UploadFile } from '../../entity/upload_file' +import { User } from '../../entity/user' import { env } from '../../env' import { Article, @@ -42,7 +45,6 @@ import { MutationSaveArticleReadingProgressArgs, MutationSetBookmarkArticleArgs, MutationSetFavoriteArticleArgs, - MutationSetShareArticleArgs, PageInfo, QueryArticleArgs, QueryArticlesArgs, @@ -63,8 +65,6 @@ import { SetFavoriteArticleError, SetFavoriteArticleErrorCode, SetFavoriteArticleSuccess, - SetShareArticleError, - SetShareArticleErrorCode, SetShareArticleSuccess, SortParams, TypeaheadSearchError, @@ -82,6 +82,7 @@ import { createLabels, getLabelsByIds, } from '../../services/labels' +import { setFileUploadComplete } from '../../services/save_file' import { parsedContentToPage } from '../../services/save_page' import { traceAs } from '../../tracing' import { Merge } from '../../util' @@ -167,12 +168,7 @@ export const createArticleResolver = authorized< }, ctx ) => { - const { - models, - authTrx, - claims: { uid }, - log, - } = ctx + const { authTrx, log, uid } = ctx analytics.track({ userId: uid, @@ -184,7 +180,18 @@ export const createArticleResolver = authorized< }, }) - const user = userDataToUser(await models.user.get(uid)) + const userData = await getRepository(User).findOneBy({ id: uid }) + if (!userData) { + return pageError( + { + errorCodes: [CreateArticleErrorCode.Unauthorized], + }, + ctx, + pageId + ) + } + const user = userDataToUser(userData) + try { if (isSiteBlockedForParse(url)) { return pageError( @@ -235,14 +242,17 @@ export const createArticleResolver = authorized< }, } // save state - let archivedAt = + const archivedAt = state === ArticleSavingRequestStatus.Archived ? new Date() : null - if (pageId) { - const reminder = await models.reminder.getByRequestId(uid, pageId) - if (reminder && reminder.archiveUntil) { - archivedAt = new Date() - } - } + // if (pageId) { + // const reminder = await getRepository(Reminder).findOneBy({ + // articleSavingRequest: pageId, + // user: { id: uid }, + // }) + // if (reminder && reminder.archiveUntil) { + // archivedAt = new Date() + // } + // } // add labels to page const labels = inputLabels ? await createLabels(ctx, inputLabels) @@ -252,9 +262,9 @@ export const createArticleResolver = authorized< /* We do not trust the values from client, lookup upload file by querying * with filtering on user ID and URL to verify client's uploadFileId is valid. */ - const uploadFile = await models.uploadFile.getWhere({ + const uploadFile = await getRepository(UploadFile).findOneBy({ id: uploadFileId, - userId: uid, + user: { id: uid }, }) if (!uploadFile) { return pageError( @@ -330,7 +340,7 @@ export const createArticleResolver = authorized< if (uploadFileId) { const uploadFileData = await authTrx(async (tx) => { - return models.uploadFile.setFileUploadComplete(uploadFileId, tx) + return setFileUploadComplete(uploadFileId, tx) }) if (!uploadFileData || !uploadFileData.id || !uploadFileData.fileName) { return pageError( @@ -579,64 +589,65 @@ export type SetShareArticleSuccessPartial = Merge< } > -export const setShareArticleResolver = authorized< - SetShareArticleSuccessPartial, - SetShareArticleError, - MutationSetShareArticleArgs ->( - async ( - _, - { input: { articleID, share, sharedComment, sharedWithHighlights } }, - { models, authTrx, claims: { uid }, log } - ) => { - const article = await models.article.get(articleID) - if (!article) { - return { errorCodes: [SetShareArticleErrorCode.NotFound] } - } +// TODO: not implemented yet +// export const setShareArticleResolver = authorized< +// SetShareArticleSuccessPartial, +// SetShareArticleError, +// MutationSetShareArticleArgs +// >( +// async ( +// _, +// { input: { articleID, share, sharedComment, sharedWithHighlights } }, +// { models, authTrx, claims: { uid }, log } +// ) => { +// const article = await models.article.get(articleID) +// if (!article) { +// return { errorCodes: [SetShareArticleErrorCode.NotFound] } +// } - const sharedAt = share ? new Date() : null +// const sharedAt = share ? new Date() : null - log.info(`${share ? 'S' : 'Uns'}haring an article`, { - article: Object.assign({}, article, { - content: undefined, - originalHtml: undefined, - sharedAt, - }), - labels: { - source: 'resolver', - resolver: 'setShareArticleResolver', - articleId: article.id, - userId: uid, - }, - }) +// log.info(`${share ? 'S' : 'Uns'}haring an article`, { +// article: Object.assign({}, article, { +// content: undefined, +// originalHtml: undefined, +// sharedAt, +// }), +// labels: { +// source: 'resolver', +// resolver: 'setShareArticleResolver', +// articleId: article.id, +// userId: uid, +// }, +// }) - const result = await authTrx((tx) => - models.userArticle.updateByArticleId( - uid, - articleID, - { sharedAt, sharedComment, sharedWithHighlights }, - tx - ) - ) +// const result = await authTrx((tx) => +// models.userArticle.updateByArticleId( +// uid, +// articleID, +// { sharedAt, sharedComment, sharedWithHighlights }, +// tx +// ) +// ) - if (!result) { - return { errorCodes: [SetShareArticleErrorCode.NotFound] } - } +// if (!result) { +// return { errorCodes: [SetShareArticleErrorCode.NotFound] } +// } - // Make sure article.id instead of userArticle.id has passed. We use it for cache updates - const updatedArticle = { - ...result, - ...article, - postedByViewer: !!sharedAt, - } - const updatedFeedArticle = sharedAt ? { ...result, sharedAt } : undefined - return { - updatedFeedArticleId: result.id, - updatedFeedArticle, - updatedArticle, - } - } -) +// // Make sure article.id instead of userArticle.id has passed. We use it for cache updates +// const updatedArticle = { +// ...result, +// ...article, +// postedByViewer: !!sharedAt, +// } +// const updatedFeedArticle = sharedAt ? { ...result, sharedAt } : undefined +// return { +// updatedFeedArticleId: result.id, +// updatedFeedArticle, +// updatedArticle, +// } +// } +// ) export type SetBookmarkArticleSuccessPartial = Merge< SetBookmarkArticleSuccess, diff --git a/packages/api/src/resolvers/article_saving_request/index.ts b/packages/api/src/resolvers/article_saving_request/index.ts index 55f669093..ee935665b 100644 --- a/packages/api/src/resolvers/article_saving_request/index.ts +++ b/packages/api/src/resolvers/article_saving_request/index.ts @@ -1,7 +1,7 @@ /* eslint-disable prefer-const */ import { getPageByParam } from '../../elastic/pages' +import { getRepository } from '../../entity' import { User } from '../../entity/user' -import { getRepository } from '../../entity/utils' import { env } from '../../env' import { ArticleSavingRequestError, diff --git a/packages/api/src/resolvers/filters/index.ts b/packages/api/src/resolvers/filters/index.ts index fd6193a6c..483b86902 100644 --- a/packages/api/src/resolvers/filters/index.ts +++ b/packages/api/src/resolvers/filters/index.ts @@ -1,4 +1,8 @@ -import { authorized } from '../../utils/helpers' +import { Between } from 'typeorm' +import { getRepository, setClaims } from '../../entity' +import { Filter } from '../../entity/filter' +import { User } from '../../entity/user' +import { env } from '../../env' import { DeleteFilterError, DeleteFilterErrorCode, @@ -20,14 +24,11 @@ import { UpdateFilterSuccess, UpdateFilterErrorCode, } from '../../generated/graphql' -import { Filter } from '../../entity/filter' -import { getRepository, setClaims } from '../../entity/utils' -import { User } from '../../entity/user' import { AppDataSource } from '../../server' -import { Between } from 'typeorm' import { analytics } from '../../utils/analytics' import { env } from '../../env' import { isNil, mergeWith } from 'lodash' +import { authorized } from '../../utils/helpers' export const saveFilterResolver = authorized< SaveFilterSuccess, diff --git a/packages/api/src/resolvers/function_resolvers.ts b/packages/api/src/resolvers/function_resolvers.ts index 9c62aeccf..e635ba08a 100644 --- a/packages/api/src/resolvers/function_resolvers.ts +++ b/packages/api/src/resolvers/function_resolvers.ts @@ -3,23 +3,21 @@ /* eslint-disable @typescript-eslint/no-unsafe-return */ /* eslint-disable @typescript-eslint/no-unsafe-member-access */ /* eslint-disable @typescript-eslint/explicit-module-boundary-types */ -import { getShareInfoForArticle } from '../datalayer/links/share_info' import { getPageByParam } from '../elastic/pages' +import { getRepository } from '../entity' import { Subscription } from '../entity/subscription' +import { UploadFile } from '../entity/upload_file' +import { User } from '../entity/user' import { Article, ArticleHighlightsInput, Highlight, HighlightType, - LinkShareInfo, PageType, - Reaction, SearchItem, - User, } from '../generated/graphql' import { userDataToUser, validatedDate, wordsCount } from '../utils/helpers' import { createImageProxyUrl } from '../utils/imageproxy' -import { logger } from '../utils/logger' import { contentReaderForPage, generateDownloadSignedUrl, @@ -38,14 +36,14 @@ import { createHighlightResolver, createLabelResolver, createNewsletterEmailResolver, - createReminderResolver, + // createReminderResolver, deleteAccountResolver, deleteFilterResolver, deleteHighlightResolver, deleteIntegrationResolver, deleteLabelResolver, deleteNewsletterEmailResolver, - deleteReminderResolver, + // deleteReminderResolver, deleteRuleResolver, deleteWebhookResolver, deviceTokensResolver, @@ -54,11 +52,11 @@ import { getAllUsersResolver, getArticleResolver, getArticlesResolver, - getFollowersResolver, - getFollowingResolver, + // getFollowersResolver, + // getFollowingResolver, getMeUserResolver, - getSharedArticleResolver, - getUserFeedArticlesResolver, + // getSharedArticleResolver, + // getUserFeedArticlesResolver, getUserPersonalizationResolver, getUserResolver, googleLoginResolver, @@ -76,7 +74,7 @@ import { newsletterEmailsResolver, recommendHighlightsResolver, recommendResolver, - reminderResolver, + // reminderResolver, reportItemResolver, revokeApiKeyResolver, rulesResolver, @@ -90,13 +88,13 @@ import { setBookmarkArticleResolver, setDeviceTokenResolver, setFavoriteArticleResolver, - setFollowResolver, + // setFollowResolver, setIntegrationResolver, setLabelsForHighlightResolver, setLabelsResolver, setLinkArchivedResolver, setRuleResolver, - setShareArticleResolver, + // setShareArticleResolver, setShareHighlightResolver, setUserPersonalizationResolver, setWebhookResolver, @@ -107,10 +105,10 @@ import { updateFilterResolver, updateHighlightResolver, updateLabelResolver, - updateLinkShareInfoResolver, + // updateLinkShareInfoResolver, updatePageResolver, - updateReminderResolver, - updateSharedCommentResolver, + // updateReminderResolver, + // updateSharedCommentResolver, updatesSinceResolver, updateSubscriptionResolver, updateUserProfileResolver, @@ -120,7 +118,6 @@ import { webhookResolver, webhooksResolver, } from './index' -import { createReactionResolver, deleteReactionResolver } from './reaction' import { markEmailAsItemResolver, recentEmailsResolver } from './recent_emails' import { recentSearchesResolver } from './recent_searches' import { Claims, WithDataSourcesContext } from './types' @@ -154,30 +151,30 @@ export const functionResolvers = { updateUserProfile: updateUserProfileResolver, createArticle: createArticleResolver, createHighlight: createHighlightResolver, - createReaction: createReactionResolver, - deleteReaction: deleteReactionResolver, + // createReaction: createReactionResolver, + // deleteReaction: deleteReactionResolver, mergeHighlight: mergeHighlightResolver, updateHighlight: updateHighlightResolver, deleteHighlight: deleteHighlightResolver, uploadFileRequest: uploadFileRequestResolver, - setShareArticle: setShareArticleResolver, - updateSharedComment: updateSharedCommentResolver, - setFollow: setFollowResolver, + // setShareArticle: setShareArticleResolver, + // updateSharedComment: updateSharedCommentResolver, + // setFollow: setFollowResolver, setBookmarkArticle: setBookmarkArticleResolver, setUserPersonalization: setUserPersonalizationResolver, createArticleSavingRequest: createArticleSavingRequestResolver, setShareHighlight: setShareHighlightResolver, reportItem: reportItemResolver, - updateLinkShareInfo: updateLinkShareInfoResolver, + // updateLinkShareInfo: updateLinkShareInfoResolver, setLinkArchived: setLinkArchivedResolver, createNewsletterEmail: createNewsletterEmailResolver, deleteNewsletterEmail: deleteNewsletterEmailResolver, saveUrl: saveUrlResolver, savePage: savePageResolver, saveFile: saveFileResolver, - createReminder: createReminderResolver, - updateReminder: updateReminderResolver, - deleteReminder: deleteReminderResolver, + // createReminder: createReminderResolver, + // updateReminder: updateReminderResolver, + // deleteReminder: deleteReminderResolver, setDeviceToken: setDeviceTokenResolver, createLabel: createLabelResolver, updateLabel: updateLabelResolver, @@ -221,15 +218,15 @@ export const functionResolvers = { users: getAllUsersResolver, validateUsername: validateUsernameResolver, article: getArticleResolver, - sharedArticle: getSharedArticleResolver, + // sharedArticle: getSharedArticleResolver, articles: getArticlesResolver, - feedArticles: getUserFeedArticlesResolver, - getFollowers: getFollowersResolver, - getFollowing: getFollowingResolver, + // feedArticles: getUserFeedArticlesResolver, + // getFollowers: getFollowersResolver, + // getFollowing: getFollowingResolver, getUserPersonalization: getUserPersonalizationResolver, articleSavingRequest: articleSavingRequestResolver, newsletterEmails: newsletterEmailsResolver, - reminder: reminderResolver, + // reminder: reminderResolver, labels: labelsResolver, search: searchResolver, subscriptions: subscriptionsResolver, @@ -247,141 +244,141 @@ export const functionResolvers = { groups: groupsResolver, recentEmails: recentEmailsResolver, }, - User: { - async sharedArticles( - user: User, - __: Record, - ctx: WithDataSourcesContext - ) { - return ctx.models.userArticle.getUserSharedArticles(user.id, ctx.kx) - }, - async sharedArticlesCount( - user: { id: string; sharedArticlesCount?: number }, - __: Record, - ctx: WithDataSourcesContext - ) { - if (user.sharedArticlesCount) return user.sharedArticlesCount - return ctx.models.userArticle.getSharedArticlesCount(user.id, ctx.kx) - }, - async sharedHighlightsCount( - user: { id: string; sharedHighlightsCount?: number }, - _: unknown, - ctx: WithDataSourcesContext - ) { - // #TODO: restructure highlightStats and sharedArticlesCount in order to get it within a single query - if (user.sharedHighlightsCount) return user.sharedHighlightsCount - const { sharedHighlightsCount } = - await ctx.models.user.getSharedHighlightsStats(user.id) - return sharedHighlightsCount - }, - async sharedNotesCount( - user: User, - _: unknown, - ctx: WithDataSourcesContext - ) { - if (user.sharedNotesCount) return user.sharedNotesCount - const { sharedNotesCount } = - await ctx.models.user.getSharedHighlightsStats(user.id) - return sharedNotesCount - }, - }, - FeedArticle: { - async article( - feedArticle: { articleId: string; userId: string; article?: Article }, - __: unknown, - ctx: WithDataSourcesContext - ) { - if (feedArticle.article) return feedArticle.article + // User: { + // async sharedArticles( + // user: User, + // __: Record, + // ctx: WithDataSourcesContext + // ) { + // return ctx.models.userArticle.getUserSharedArticles(user.id, ctx.kx) + // }, + // async sharedArticlesCount( + // user: { id: string; sharedArticlesCount?: number }, + // __: Record, + // ctx: WithDataSourcesContext + // ) { + // if (user.sharedArticlesCount) return user.sharedArticlesCount + // return ctx.models.userArticle.getSharedArticlesCount(user.id, ctx.kx) + // }, + // async sharedHighlightsCount( + // user: { id: string; sharedHighlightsCount?: number }, + // _: unknown, + // ctx: WithDataSourcesContext + // ) { + // // #TODO: restructure highlightStats and sharedArticlesCount in order to get it within a single query + // if (user.sharedHighlightsCount) return user.sharedHighlightsCount + // const { sharedHighlightsCount } = + // await ctx.models.user.getSharedHighlightsStats(user.id) + // return sharedHighlightsCount + // }, + // async sharedNotesCount( + // user: User, + // _: unknown, + // ctx: WithDataSourcesContext + // ) { + // if (user.sharedNotesCount) return user.sharedNotesCount + // const { sharedNotesCount } = + // await ctx.models.user.getSharedHighlightsStats(user.id) + // return sharedNotesCount + // }, + // }, + // FeedArticle: { + // async article( + // feedArticle: { articleId: string; userId: string; article?: Article }, + // __: unknown, + // ctx: WithDataSourcesContext + // ) { + // if (feedArticle.article) return feedArticle.article - // eslint-disable-next-line @typescript-eslint/no-explicit-any - let a: any + // // eslint-disable-next-line @typescript-eslint/no-explicit-any + // let a: any - const savedArticle = - ctx.claims?.uid && - (await ctx.models.userArticle.getForUser( - ctx.claims?.uid, - feedArticle.articleId, - ctx.kx - )) + // const savedArticle = + // ctx.claims?.uid && + // (await ctx.models.userArticle.getForUser( + // ctx.claims?.uid, + // feedArticle.articleId, + // ctx.kx + // )) - if (savedArticle) { - // If user has saved the article, use his version (slug) then - a = { - ...savedArticle, - savedByViewer: true, - postedByViewer: !!savedArticle.sharedAt, - } - } else { - a = await ctx.models.userArticle.getForUser( - feedArticle.userId, - feedArticle.articleId, - ctx.kx - ) - } + // if (savedArticle) { + // // If user has saved the article, use his version (slug) then + // a = { + // ...savedArticle, + // savedByViewer: true, + // postedByViewer: !!savedArticle.sharedAt, + // } + // } else { + // a = await ctx.models.userArticle.getForUser( + // feedArticle.userId, + // feedArticle.articleId, + // ctx.kx + // ) + // } - if (a && a.image) { - a.image = createImageProxyUrl(a.image, 0, 180) - } else { - logger.info( - 'error getting article for feedItem', - feedArticle.userId, - feedArticle.articleId - ) - } + // if (a && a.image) { + // a.image = createImageProxyUrl(a.image, 0, 180) + // } else { + // logger.info( + // 'error getting article for feedItem', + // feedArticle.userId, + // feedArticle.articleId + // ) + // } - return a - }, - async sharedBy( - feedArticle: { userId: string; sharedBy?: User }, - __: unknown, - ctx: WithDataSourcesContext - ) { - if (feedArticle.sharedBy) return feedArticle.sharedBy - return userDataToUser(await ctx.models.user.get(feedArticle.userId)) - }, - async highlight( - feedArticle: { highlightId?: string; highlight?: Highlight }, - _: unknown, - ctx: WithDataSourcesContext - ) { - if (feedArticle.highlight) return feedArticle.highlight - return feedArticle.highlightId - ? await ctx.models.highlight.get(feedArticle.highlightId) - : null - }, - async reactions( - feedArticle: { id: string; reactions?: Reaction[] }, - _: unknown, - ctx: WithDataSourcesContext - ) { - const { reactions, id } = feedArticle - if (reactions) return reactions + // return a + // }, + // async sharedBy( + // feedArticle: { userId: string; sharedBy?: User }, + // __: unknown, + // ctx: WithDataSourcesContext + // ) { + // if (feedArticle.sharedBy) return feedArticle.sharedBy + // return userDataToUser(await ctx.models.user.get(feedArticle.userId)) + // }, + // async highlight( + // feedArticle: { highlightId?: string; highlight?: Highlight }, + // _: unknown, + // ctx: WithDataSourcesContext + // ) { + // if (feedArticle.highlight) return feedArticle.highlight + // return feedArticle.highlightId + // ? await ctx.models.highlight.get(feedArticle.highlightId) + // : null + // }, + // async reactions( + // feedArticle: { id: string; reactions?: Reaction[] }, + // _: unknown, + // ctx: WithDataSourcesContext + // ) { + // const { reactions, id } = feedArticle + // if (reactions) return reactions - return await ctx.models.reaction.batchGetFromArticle(id) - }, - async highlightsCount( - feedArticle: { id: string; highlightsCount?: number }, - _: unknown, - ctx: WithDataSourcesContext - ) { - if (feedArticle.highlightsCount) return feedArticle.highlightsCount - const { highlightsCount } = await ctx.models.userArticle.getStats( - feedArticle.id - ) - return highlightsCount - }, - async annotationsCount( - feedArticle: { id: string; annotationsCount?: number }, - _: unknown, - ctx: WithDataSourcesContext - ) { - if (feedArticle.annotationsCount) return feedArticle.annotationsCount - const { annotationsCount } = await ctx.models.userArticle.getStats( - feedArticle.id - ) - return annotationsCount - }, - }, + // return await ctx.models.reaction.batchGetFromArticle(id) + // }, + // async highlightsCount( + // feedArticle: { id: string; highlightsCount?: number }, + // _: unknown, + // ctx: WithDataSourcesContext + // ) { + // if (feedArticle.highlightsCount) return feedArticle.highlightsCount + // const { highlightsCount } = await ctx.models.userArticle.getStats( + // feedArticle.id + // ) + // return highlightsCount + // }, + // async annotationsCount( + // feedArticle: { id: string; annotationsCount?: number }, + // _: unknown, + // ctx: WithDataSourcesContext + // ) { + // if (feedArticle.annotationsCount) return feedArticle.annotationsCount + // const { annotationsCount } = await ctx.models.userArticle.getStats( + // feedArticle.id + // ) + // return annotationsCount + // }, + // }, Article: { async url(article: Article, _: unknown, ctx: WithDataSourcesContext) { if ( @@ -390,7 +387,9 @@ export const functionResolvers = { ctx.claims && article.uploadFileId ) { - const upload = await ctx.models.uploadFile.get(article.uploadFileId) + const upload = await getRepository(UploadFile).findOneBy({ + id: article.uploadFileId, + }) if (!upload || !upload.fileName) { return undefined } @@ -490,20 +489,20 @@ export const functionResolvers = { ) { return article.highlights || [] }, - async shareInfo( - article: { id: string; sharedBy?: User; shareInfo?: LinkShareInfo }, - __: unknown, - ctx: WithDataSourcesContext - ): Promise { - if (article.shareInfo) return article.shareInfo - if (!ctx.claims?.uid) return undefined - return getShareInfoForArticle( - ctx.kx, - ctx.claims?.uid, - article.id, - ctx.models - ) - }, + // async shareInfo( + // article: { id: string; sharedBy?: User; shareInfo?: LinkShareInfo }, + // __: unknown, + // ctx: WithDataSourcesContext + // ): Promise { + // if (article.shareInfo) return article.shareInfo + // if (!ctx.claims?.uid) return undefined + // return getShareInfoForArticle( + // ctx.kx, + // ctx.claims?.uid, + // article.id, + // ctx.models + // ) + // }, image(article: { image?: string }): string | undefined { return article.image && createImageProxyUrl(article.image, 320, 320) }, @@ -528,18 +527,23 @@ export const functionResolvers = { __: unknown, ctx: WithDataSourcesContext ) { - return userDataToUser(await ctx.models.user.get(highlight.userId)) - }, - async reactions( - highlight: { id: string; reactions?: Reaction[] }, - _: unknown, - ctx: WithDataSourcesContext - ) { - const { reactions, id } = highlight - if (reactions) return reactions + const userData = await getRepository(User).findOneBy({ + id: highlight.userId, + }) + if (!userData) return null - return await ctx.models.reaction.batchGetFromHighlight(id) + return userDataToUser(userData) }, + // async reactions( + // highlight: { id: string; reactions?: Reaction[] }, + // _: unknown, + // ctx: WithDataSourcesContext + // ) { + // const { reactions, id } = highlight + // if (reactions) return reactions + + // return await ctx.models.reaction.batchGetFromHighlight(id) + // }, async createdByMe( highlight: { userId: string; createdByMe?: boolean }, __: unknown, @@ -551,15 +555,15 @@ export const functionResolvers = { return highlight.type || HighlightType.Highlight }, }, - Reaction: { - async user( - reaction: { userId: string }, - __: unknown, - ctx: WithDataSourcesContext - ) { - return userDataToUser(await ctx.models.user.get(reaction.userId)) - }, - }, + // Reaction: { + // async user( + // reaction: { userId: string }, + // __: unknown, + // ctx: WithDataSourcesContext + // ) { + // return userDataToUser(await ctx.models.user.get(reaction.userId)) + // }, + // }, SearchItem: { async url(item: SearchItem, _: unknown, ctx: WithDataSourcesContext) { if ( @@ -567,7 +571,9 @@ export const functionResolvers = { ctx.claims && item.uploadFileId ) { - const upload = await ctx.models.uploadFile.get(item.uploadFileId) + const upload = await getRepository(UploadFile).findOneBy({ + id: item.uploadFileId, + }) if (!upload || !upload.fileName) { return undefined } diff --git a/packages/api/src/resolvers/importers/uploadImportFileResolver.ts b/packages/api/src/resolvers/importers/uploadImportFileResolver.ts index 122ef69ee..175942813 100644 --- a/packages/api/src/resolvers/importers/uploadImportFileResolver.ts +++ b/packages/api/src/resolvers/importers/uploadImportFileResolver.ts @@ -1,7 +1,7 @@ import { DateTime } from 'luxon' import { v4 as uuidv4 } from 'uuid' +import { getRepository } from '../../entity' import { User } from '../../entity/user' -import { getRepository } from '../../entity/utils' import { env } from '../../env' import { MutationUploadImportFileArgs, diff --git a/packages/api/src/resolvers/integrations/index.ts b/packages/api/src/resolvers/integrations/index.ts index 35f697d39..1a480f020 100644 --- a/packages/api/src/resolvers/integrations/index.ts +++ b/packages/api/src/resolvers/integrations/index.ts @@ -1,6 +1,6 @@ +import { getRepository } from '../../entity' import { Integration, IntegrationType } from '../../entity/integration' import { User } from '../../entity/user' -import { getRepository } from '../../entity/utils' import { env } from '../../env' import { DeleteIntegrationError, diff --git a/packages/api/src/resolvers/labels/index.ts b/packages/api/src/resolvers/labels/index.ts index b58a1128e..530793914 100644 --- a/packages/api/src/resolvers/labels/index.ts +++ b/packages/api/src/resolvers/labels/index.ts @@ -8,9 +8,9 @@ import { updateLabelsInPage, } from '../../elastic/labels' import { getPageById } from '../../elastic/pages' +import { getRepository, setClaims } from '../../entity' import { Label } from '../../entity/label' import { User } from '../../entity/user' -import { getRepository, setClaims } from '../../entity/utils' import { env } from '../../env' import { CreateLabelError, diff --git a/packages/api/src/resolvers/links/index.ts b/packages/api/src/resolvers/links/index.ts index f0105554a..da3afa3bd 100644 --- a/packages/api/src/resolvers/links/index.ts +++ b/packages/api/src/resolvers/links/index.ts @@ -1,4 +1,3 @@ -import { createOrUpdateLinkShareInfo } from '../../datalayer/links/share_info' import { updatePage } from '../../elastic/pages' import { env } from '../../env' import { @@ -6,52 +5,48 @@ import { ArchiveLinkErrorCode, ArchiveLinkSuccess, MutationSetLinkArchivedArgs, - MutationUpdateLinkShareInfoArgs, - UpdateLinkShareInfoError, - UpdateLinkShareInfoErrorCode, - UpdateLinkShareInfoSuccess, } from '../../generated/graphql' import { analytics } from '../../utils/analytics' import { authorized } from '../../utils/helpers' -export const updateLinkShareInfoResolver = authorized< - UpdateLinkShareInfoSuccess, - UpdateLinkShareInfoError, - MutationUpdateLinkShareInfoArgs ->(async (_obj, args, { models, claims, authTrx, log }) => { - const { title, description } = args.input +// export const updateLinkShareInfoResolver = authorized< +// UpdateLinkShareInfoSuccess, +// UpdateLinkShareInfoError, +// MutationUpdateLinkShareInfoArgs +// >(async (_obj, args, { models, claims, authTrx, log }) => { +// const { title, description } = args.input - log.info('updateLinkShareInfoResolver', args.input.linkId, title, description) +// log.info('updateLinkShareInfoResolver', args.input.linkId, title, description) - // TEMP: because the old API uses articles instead of Links, we are actually - // getting an article ID here and need to map it to a link ID. When the API - // is updated to use Links instead of Articles this will be removed. - const link = await authTrx((tx) => - models.userArticle.getByArticleId(claims.uid, args.input.linkId, tx) - ) +// // TEMP: because the old API uses articles instead of Links, we are actually +// // getting an article ID here and need to map it to a link ID. When the API +// // is updated to use Links instead of Articles this will be removed. +// const link = await authTrx((tx) => +// models.userArticle.getByArticleId(claims.uid, args.input.linkId, tx) +// ) - if (!link?.id) { - return { - __typename: 'UpdateLinkShareInfoError', - errorCodes: [UpdateLinkShareInfoErrorCode.Unauthorized], - } - } +// if (!link?.id) { +// return { +// __typename: 'UpdateLinkShareInfoError', +// errorCodes: [UpdateLinkShareInfoErrorCode.Unauthorized], +// } +// } - const result = await authTrx((tx) => - createOrUpdateLinkShareInfo(tx, link.id, title, description) - ) - if (!result) { - return { - __typename: 'UpdateLinkShareInfoError', - errorCodes: [UpdateLinkShareInfoErrorCode.BadRequest], - } - } +// const result = await authTrx((tx) => +// createOrUpdateLinkShareInfo(tx, link.id, title, description) +// ) +// if (!result) { +// return { +// __typename: 'UpdateLinkShareInfoError', +// errorCodes: [UpdateLinkShareInfoErrorCode.BadRequest], +// } +// } - return { - __typename: 'UpdateLinkShareInfoSuccess', - message: 'Updated Share Information', - } -}) +// return { +// __typename: 'UpdateLinkShareInfoSuccess', +// message: 'Updated Share Information', +// } +// }) export const setLinkArchivedResolver = authorized< ArchiveLinkSuccess, diff --git a/packages/api/src/resolvers/newsletters/index.ts b/packages/api/src/resolvers/newsletters/index.ts index 58919c985..af7d72dd1 100644 --- a/packages/api/src/resolvers/newsletters/index.ts +++ b/packages/api/src/resolvers/newsletters/index.ts @@ -1,6 +1,6 @@ +import { getRepository } from '../../entity' import { NewsletterEmail } from '../../entity/newsletter_email' import { User } from '../../entity/user' -import { getRepository } from '../../entity/utils' import { env } from '../../env' import { CreateNewsletterEmailError, diff --git a/packages/api/src/resolvers/reaction/index.ts b/packages/api/src/resolvers/reaction/index.ts index 25687a55f..aec16962a 100644 --- a/packages/api/src/resolvers/reaction/index.ts +++ b/packages/api/src/resolvers/reaction/index.ts @@ -1,15 +1,5 @@ import { Merge } from '../../util' -import { authorized } from '../../utils/helpers' -import { - CreateReactionError, - CreateReactionErrorCode, - CreateReactionSuccess, - DeleteReactionError, - DeleteReactionErrorCode, - MutationCreateReactionArgs, - MutationDeleteReactionArgs, - Reaction, -} from './../../generated/graphql' +import { CreateReactionSuccess, Reaction } from './../../generated/graphql' export type PartialReaction = Omit @@ -18,125 +8,125 @@ export type PartialCreateReactionSuccess = Merge< { reaction: PartialReaction } > -export const createReactionResolver = authorized< - PartialCreateReactionSuccess, - CreateReactionError, - MutationCreateReactionArgs ->(async (_, { input }, { models, claims, log, authTrx }) => { - const { userArticleId, highlightId } = input +// export const createReactionResolver = authorized< +// PartialCreateReactionSuccess, +// CreateReactionError, +// MutationCreateReactionArgs +// >(async (_, { input }, { models, claims, log, authTrx }) => { +// const { userArticleId, highlightId } = input - if ((!userArticleId && !highlightId) || (userArticleId && highlightId)) { - // One reaction target is required - // Higlight replies hasn't supported yet - return { - errorCodes: [CreateReactionErrorCode.BadTarget], - } - } +// if ((!userArticleId && !highlightId) || (userArticleId && highlightId)) { +// // One reaction target is required +// // Higlight replies hasn't supported yet +// return { +// errorCodes: [CreateReactionErrorCode.BadTarget], +// } +// } - if (input.code && input.code.length > 50) { - return { - errorCodes: [CreateReactionErrorCode.BadCode], - } - } +// if (input.code && input.code.length > 50) { +// return { +// errorCodes: [CreateReactionErrorCode.BadCode], +// } +// } - if (userArticleId) { - if (!(await models.userArticle.get(userArticleId))) { - return { - errorCodes: [CreateReactionErrorCode.BadTarget], - } - } - } else if (highlightId) { - if (!(await models.highlight.get(highlightId))) { - return { - errorCodes: [CreateReactionErrorCode.BadTarget], - } - } - } +// if (userArticleId) { +// if (!(await models.userArticle.get(userArticleId))) { +// return { +// errorCodes: [CreateReactionErrorCode.BadTarget], +// } +// } +// } else if (highlightId) { +// if (!(await models.highlight.get(highlightId))) { +// return { +// errorCodes: [CreateReactionErrorCode.BadTarget], +// } +// } +// } - try { - const previousReaction = userArticleId - ? await models.reaction.getByUserAndParam(claims.uid, { userArticleId }) - : await models.reaction.getByUserAndParam(claims.uid, { highlightId }) +// try { +// const previousReaction = userArticleId +// ? await models.reaction.getByUserAndParam(claims.uid, { userArticleId }) +// : await models.reaction.getByUserAndParam(claims.uid, { highlightId }) - let reaction - if (!previousReaction) { - reaction = await authTrx((tx) => - models.reaction.create({ ...input, userId: claims.uid }, tx) - ) - } else { - reaction = await authTrx((tx) => - models.reaction.update( - previousReaction.id, - { - code: input.code, - }, - tx - ) - ) - } +// let reaction +// if (!previousReaction) { +// reaction = await authTrx((tx) => +// models.reaction.create({ ...input, userId: claims.uid }, tx) +// ) +// } else { +// reaction = await authTrx((tx) => +// models.reaction.update( +// previousReaction.id, +// { +// code: input.code, +// }, +// tx +// ) +// ) +// } - if (!reaction) { - return { - errorCodes: [CreateReactionErrorCode.NotFound], - } - } - log.info(`${previousReaction ? 'Updating' : 'Creating'} a new reaction`, { - reaction, - labels: { - source: 'resolver', - resolver: 'createReactionResolver', - uid: claims.uid, - }, - }) +// if (!reaction) { +// return { +// errorCodes: [CreateReactionErrorCode.NotFound], +// } +// } +// log.info(`${previousReaction ? 'Updating' : 'Creating'} a new reaction`, { +// reaction, +// labels: { +// source: 'resolver', +// resolver: 'createReactionResolver', +// uid: claims.uid, +// }, +// }) - return { - reaction: reaction as PartialReaction, - } - } catch (err) { - log.info(err) - return { - errorCodes: [CreateReactionErrorCode.NotFound], - } - } -}) +// return { +// reaction: reaction as PartialReaction, +// } +// } catch (err) { +// log.info(err) +// return { +// errorCodes: [CreateReactionErrorCode.NotFound], +// } +// } +// }) -export const deleteReactionResolver = authorized< - PartialCreateReactionSuccess, - DeleteReactionError, - MutationDeleteReactionArgs ->(async (_, { id }, { authTrx, models, claims, log }) => { - const reaction = await models.reaction.get(id) +// export const deleteReactionResolver = authorized< +// PartialCreateReactionSuccess, +// DeleteReactionError, +// MutationDeleteReactionArgs +// >(async (_, { id }, { authTrx, models, claims, log }) => { +// const reaction = await models.reaction.get(id) - if (!reaction?.id) { - return { - errorCodes: [DeleteReactionErrorCode.NotFound], - } - } +// if (!reaction?.id) { +// return { +// errorCodes: [DeleteReactionErrorCode.NotFound], +// } +// } - if (reaction.userId !== claims.uid) { - return { - errorCodes: [DeleteReactionErrorCode.Forbidden], - } - } +// if (reaction.userId !== claims.uid) { +// return { +// errorCodes: [DeleteReactionErrorCode.Forbidden], +// } +// } - const deleted = await authTrx((tx) => models.reaction.delete(id, tx)) +// const deleted = await authTrx((tx) => models.reaction.delete(id, tx)) - if ('error' in deleted) { - return { - errorCodes: [DeleteReactionErrorCode.NotFound], - } - } +// if ('error' in deleted) { +// return { +// errorCodes: [DeleteReactionErrorCode.NotFound], +// } +// } - log.info('Deleting a highlight', { - deleted, - labels: { - source: 'resolver', - resolver: 'deleteHighlightResolver', - uid: claims.uid, - }, - }) +// log.info('Deleting a highlight', { +// deleted, +// labels: { +// source: 'resolver', +// resolver: 'deleteHighlightResolver', +// uid: claims.uid, +// }, +// }) - return { - reaction: reaction as PartialReaction, - } -}) +// return { +// reaction: reaction as PartialReaction, +// } +// }) diff --git a/packages/api/src/resolvers/recent_emails/index.ts b/packages/api/src/resolvers/recent_emails/index.ts index ef4955ec0..0c0266a1e 100644 --- a/packages/api/src/resolvers/recent_emails/index.ts +++ b/packages/api/src/resolvers/recent_emails/index.ts @@ -1,7 +1,7 @@ import { ILike } from 'typeorm' +import { getRepository } from '../../entity' import { NewsletterEmail } from '../../entity/newsletter_email' import { ReceivedEmail } from '../../entity/received_email' -import { getRepository } from '../../entity/utils' import { env } from '../../env' import { MarkEmailAsItemError, diff --git a/packages/api/src/resolvers/recent_searches/index.ts b/packages/api/src/resolvers/recent_searches/index.ts index d8efadf9e..5f109e5b6 100644 --- a/packages/api/src/resolvers/recent_searches/index.ts +++ b/packages/api/src/resolvers/recent_searches/index.ts @@ -1,14 +1,14 @@ -import { authorized } from '../../utils/helpers' +import { getRepository } from '../../entity' +import { User } from '../../entity/user' +import { env } from '../../env' import { RecentSearchesError, RecentSearchesErrorCode, RecentSearchesSuccess, } from '../../generated/graphql' -import { analytics } from '../../utils/analytics' -import { env } from '../../env' -import { getRepository } from '../../entity/utils' -import { User } from '../../entity/user' import { getRecentSearches } from '../../services/search_history' +import { analytics } from '../../utils/analytics' +import { authorized } from '../../utils/helpers' export const recentSearchesResolver = authorized< RecentSearchesSuccess, diff --git a/packages/api/src/resolvers/recommendations/index.ts b/packages/api/src/resolvers/recommendations/index.ts index 68765d194..09d43eb61 100644 --- a/packages/api/src/resolvers/recommendations/index.ts +++ b/packages/api/src/resolvers/recommendations/index.ts @@ -1,8 +1,8 @@ import { In } from 'typeorm' import { getPageByParam } from '../../elastic/pages' +import { getRepository } from '../../entity' import { Group } from '../../entity/groups/group' import { User } from '../../entity/user' -import { getRepository } from '../../entity/utils' import { env } from '../../env' import { CreateGroupError, diff --git a/packages/api/src/resolvers/reminders/index.ts b/packages/api/src/resolvers/reminders/index.ts index 9b33134b6..4f311e6e8 100644 --- a/packages/api/src/resolvers/reminders/index.ts +++ b/packages/api/src/resolvers/reminders/index.ts @@ -1,31 +1,4 @@ import { DateTime } from 'luxon' -import { getPageById } from '../../elastic/pages' -import { Page } from '../../elastic/types' -import { env } from '../../env' -import { - CreateReminderError, - CreateReminderErrorCode, - CreateReminderSuccess, - DeleteReminderError, - DeleteReminderErrorCode, - DeleteReminderSuccess, - MutationCreateReminderArgs, - MutationDeleteReminderArgs, - MutationUpdateReminderArgs, - QueryReminderArgs, - ReminderError, - ReminderErrorCode, - ReminderSuccess, - UpdateReminderError, - UpdateReminderErrorCode, - UpdateReminderSuccess, -} from '../../generated/graphql' -import { setLinkArchived } from '../../services/archive_link' -import { analytics } from '../../utils/analytics' -import { deleteTask, enqueueReminder } from '../../utils/createTask' -import { authorized } from '../../utils/helpers' -import { logger } from '../../utils/logger' -import { DataModels } from '../types' const validScheduleTime = (str: string): Date | undefined => { const scheduleTime = DateTime.fromISO(str, { setZone: true }).set({ @@ -39,320 +12,320 @@ const validScheduleTime = (str: string): Date | undefined => { return scheduleTime.toJSDate() } -export const createReminderResolver = authorized< - CreateReminderSuccess, - CreateReminderError, - MutationCreateReminderArgs ->(async (_, { input }, { models, claims: { uid }, log }) => { - log.info('createReminderResolver') +// export const createReminderResolver = authorized< +// CreateReminderSuccess, +// CreateReminderError, +// MutationCreateReminderArgs +// >(async (_, { input }, { models, claims: { uid }, log }) => { +// log.info('createReminderResolver') - const { clientRequestId, linkId, archiveUntil, sendNotification } = input +// const { clientRequestId, linkId, archiveUntil, sendNotification } = input - const scheduledTime = validScheduleTime(input.remindAt) - if (!scheduledTime) { - log.error('Invalid scheduled time', input.remindAt) - return { - errorCodes: [CreateReminderErrorCode.BadRequest], - } - } +// const scheduledTime = validScheduleTime(input.remindAt) +// if (!scheduledTime) { +// log.error('Invalid scheduled time', input.remindAt) +// return { +// errorCodes: [CreateReminderErrorCode.BadRequest], +// } +// } - const pageId = linkId || clientRequestId - if (!pageId) { - log.error('client request id or link id is required') - return { - errorCodes: [CreateReminderErrorCode.BadRequest], - } - } +// const pageId = linkId || clientRequestId +// if (!pageId) { +// log.error('client request id or link id is required') +// return { +// errorCodes: [CreateReminderErrorCode.BadRequest], +// } +// } - analytics.track({ - userId: uid, - event: 'reminder_created', - properties: { - clientRequestId, - remindAt: scheduledTime, - archiveUntil, - sendNotification, - linkId, - env: env.server.apiEnv, - }, - }) +// analytics.track({ +// userId: uid, +// event: 'reminder_created', +// properties: { +// clientRequestId, +// remindAt: scheduledTime, +// archiveUntil, +// sendNotification, +// linkId, +// env: env.server.apiEnv, +// }, +// }) - try { - // saving from web - const page = await getPageById(pageId) - if (!page) { - log.error('page not found', pageId) +// try { +// // saving from web +// const page = await getPageById(pageId) +// if (!page) { +// log.error('page not found', pageId) - return { - errorCodes: [CreateReminderErrorCode.NotFound], - } - } +// return { +// errorCodes: [CreateReminderErrorCode.NotFound], +// } +// } - if (page.userId !== uid) { - log.error('user not authorized', uid) +// if (page.userId !== uid) { +// log.error('user not authorized', uid) - return { - errorCodes: [CreateReminderErrorCode.Unauthorized], - } - } - if (archiveUntil) { - await archivePage(uid, page) - } +// return { +// errorCodes: [CreateReminderErrorCode.Unauthorized], +// } +// } +// if (archiveUntil) { +// await archivePage(uid, page) +// } - const taskName = await groupReminders(scheduledTime, uid, models) - log.info('scheduled task name', taskName) +// const taskName = await groupReminders(scheduledTime, uid, models) +// log.info('scheduled task name', taskName) - // insert reminder to db - const reminder = await models.reminder.create({ - userId: uid, - taskName: taskName, - archiveUntil: archiveUntil, - sendNotification: sendNotification, - createdAt: new Date(), - remindAt: scheduledTime, - elasticPageId: pageId, - }) - log.info('created reminder', reminder) +// // insert reminder to db +// const reminder = await models.reminder.create({ +// userId: uid, +// taskName: taskName, +// archiveUntil: archiveUntil, +// sendNotification: sendNotification, +// createdAt: new Date(), +// remindAt: scheduledTime, +// elasticPageId: pageId, +// }) +// log.info('created reminder', reminder) - return { - reminder: { - id: reminder.id, - archiveUntil, - sendNotification, - remindAt: scheduledTime, - }, - } - } catch (e) { - log.info('error creating reminder', e) +// return { +// reminder: { +// id: reminder.id, +// archiveUntil, +// sendNotification, +// remindAt: scheduledTime, +// }, +// } +// } catch (e) { +// log.info('error creating reminder', e) - return { - errorCodes: [CreateReminderErrorCode.BadRequest], - } - } -}) +// return { +// errorCodes: [CreateReminderErrorCode.BadRequest], +// } +// } +// }) -// Attempts to find a link and archive it if it exists. -// It is possible that the link has not been created -// yet if it is still in the saving process. In that -// case it will be archived when the link is created. -const archivePage = async (uid: string, page: Page) => { - try { - await setLinkArchived(uid, page.id, true) - } catch (e) { - logger.info('error archiving link', e) - } -} +// // Attempts to find a link and archive it if it exists. +// // It is possible that the link has not been created +// // yet if it is still in the saving process. In that +// // case it will be archived when the link is created. +// const archivePage = async (uid: string, page: Page) => { +// try { +// await setLinkArchived(uid, page.id, true) +// } catch (e) { +// logger.info('error archiving link', e) +// } +// } -export const reminderResolver = authorized< - ReminderSuccess, - ReminderError, - QueryReminderArgs ->(async (_, { linkId: pageId }, { models, claims: { uid }, log }) => { - log.info('reminderResolver') +// export const reminderResolver = authorized< +// ReminderSuccess, +// ReminderError, +// QueryReminderArgs +// >(async (_, { linkId: pageId }, { models, claims: { uid }, log }) => { +// log.info('reminderResolver') - analytics.track({ - userId: uid, - event: 'reminder', - properties: { - linkId: pageId, - env: env.server.apiEnv, - }, - }) +// analytics.track({ +// userId: uid, +// event: 'reminder', +// properties: { +// linkId: pageId, +// env: env.server.apiEnv, +// }, +// }) - try { - // get page from articleId - const page = await getPageById(pageId) - if (!page) { - return { - errorCodes: [ReminderErrorCode.NotFound], - } - } - if (page.userId !== uid) { - return { - errorCodes: [ReminderErrorCode.Unauthorized], - } - } - const reminder = await models.reminder.getCreatedByParameters(uid, { - elasticPageId: page.id, - }) +// try { +// // get page from articleId +// const page = await getPageById(pageId) +// if (!page) { +// return { +// errorCodes: [ReminderErrorCode.NotFound], +// } +// } +// if (page.userId !== uid) { +// return { +// errorCodes: [ReminderErrorCode.Unauthorized], +// } +// } +// const reminder = await models.reminder.getCreatedByParameters(uid, { +// elasticPageId: page.id, +// }) - if (!reminder) { - log.error('reminder not found: pageId: ', pageId) +// if (!reminder) { +// log.error('reminder not found: pageId: ', pageId) - return { - errorCodes: [ReminderErrorCode.NotFound], - } - } +// return { +// errorCodes: [ReminderErrorCode.NotFound], +// } +// } - return { - reminder: { - id: reminder.id, - archiveUntil: reminder.archiveUntil || false, - sendNotification: reminder.sendNotification || true, - remindAt: reminder.remindAt, - }, - } - } catch (e) { - log.error(e) +// return { +// reminder: { +// id: reminder.id, +// archiveUntil: reminder.archiveUntil || false, +// sendNotification: reminder.sendNotification || true, +// remindAt: reminder.remindAt, +// }, +// } +// } catch (e) { +// log.error(e) - return { - errorCodes: [ReminderErrorCode.BadRequest], - } - } -}) +// return { +// errorCodes: [ReminderErrorCode.BadRequest], +// } +// } +// }) -export const updateReminderResolver = authorized< - UpdateReminderSuccess, - UpdateReminderError, - MutationUpdateReminderArgs ->(async (_, { input }, { models, claims: { uid }, log, authTrx }) => { - log.info('updateReminderResolver') +// export const updateReminderResolver = authorized< +// UpdateReminderSuccess, +// UpdateReminderError, +// MutationUpdateReminderArgs +// >(async (_, { input }, { models, claims: { uid }, log, authTrx }) => { +// log.info('updateReminderResolver') - const { id, archiveUntil, sendNotification } = input +// const { id, archiveUntil, sendNotification } = input - const scheduledTime = validScheduleTime(input.remindAt) - if (!scheduledTime) { - log.error('Invalid scheduled time', input.remindAt) - return { - errorCodes: [UpdateReminderErrorCode.BadRequest], - } - } +// const scheduledTime = validScheduleTime(input.remindAt) +// if (!scheduledTime) { +// log.error('Invalid scheduled time', input.remindAt) +// return { +// errorCodes: [UpdateReminderErrorCode.BadRequest], +// } +// } - analytics.track({ - userId: uid, - event: 'reminder_updated', - properties: { - id, - remindAt: scheduledTime, - archiveUntil, - sendNotification, - env: env.server.apiEnv, - }, - }) +// analytics.track({ +// userId: uid, +// event: 'reminder_updated', +// properties: { +// id, +// remindAt: scheduledTime, +// archiveUntil, +// sendNotification, +// env: env.server.apiEnv, +// }, +// }) - try { - const reminder = await models.reminder.getCreated(id) +// try { +// const reminder = await models.reminder.getCreated(id) - if (!reminder) { - log.error('reminder not found:', id) +// if (!reminder) { +// log.error('reminder not found:', id) - return { - errorCodes: [UpdateReminderErrorCode.NotFound], - } - } +// return { +// errorCodes: [UpdateReminderErrorCode.NotFound], +// } +// } - if (reminder.userId !== uid) { - return { - errorCodes: [UpdateReminderErrorCode.Unauthorized], - } - } +// if (reminder.userId !== uid) { +// return { +// errorCodes: [UpdateReminderErrorCode.Unauthorized], +// } +// } - // delete old google cloud task - if (reminder.taskName) { - await deleteTask(reminder.taskName) - } +// // delete old google cloud task +// if (reminder.taskName) { +// await deleteTask(reminder.taskName) +// } - const taskName = await groupReminders(scheduledTime, uid, models) +// const taskName = await groupReminders(scheduledTime, uid, models) - // update db - await authTrx((tx) => - models.reminder.update( - id, - { - taskName: taskName, - archiveUntil, - sendNotification, - remindAt: scheduledTime, - }, - tx - ) - ) +// // update db +// await authTrx((tx) => +// models.reminder.update( +// id, +// { +// taskName: taskName, +// archiveUntil, +// sendNotification, +// remindAt: scheduledTime, +// }, +// tx +// ) +// ) - return { - reminder: { - id: reminder.id, - archiveUntil, - sendNotification, - remindAt: scheduledTime, - }, - } - } catch (e) { - log.error(e) +// return { +// reminder: { +// id: reminder.id, +// archiveUntil, +// sendNotification, +// remindAt: scheduledTime, +// }, +// } +// } catch (e) { +// log.error(e) - return { - errorCodes: [UpdateReminderErrorCode.BadRequest], - } - } -}) +// return { +// errorCodes: [UpdateReminderErrorCode.BadRequest], +// } +// } +// }) -export const deleteReminderResolver = authorized< - DeleteReminderSuccess, - DeleteReminderError, - MutationDeleteReminderArgs ->(async (_, { id }, { models, claims: { uid }, log, authTrx }) => { - log.info('deleteReminderResolver') +// export const deleteReminderResolver = authorized< +// DeleteReminderSuccess, +// DeleteReminderError, +// MutationDeleteReminderArgs +// >(async (_, { id }, { models, claims: { uid }, log, authTrx }) => { +// log.info('deleteReminderResolver') - analytics.track({ - userId: uid, - event: 'reminder_deleted', - properties: { - id: id, - env: env.server.apiEnv, - }, - }) +// analytics.track({ +// userId: uid, +// event: 'reminder_deleted', +// properties: { +// id: id, +// env: env.server.apiEnv, +// }, +// }) - try { - const reminder = await models.reminder.getCreated(id) +// try { +// const reminder = await models.reminder.getCreated(id) - if (!reminder) { - log.error('reminder not found:', id) +// if (!reminder) { +// log.error('reminder not found:', id) - return { - errorCodes: [DeleteReminderErrorCode.NotFound], - } - } +// return { +// errorCodes: [DeleteReminderErrorCode.NotFound], +// } +// } - if (reminder.userId !== uid) { - return { - errorCodes: [DeleteReminderErrorCode.Unauthorized], - } - } +// if (reminder.userId !== uid) { +// return { +// errorCodes: [DeleteReminderErrorCode.Unauthorized], +// } +// } - // update db - await authTrx((tx) => models.reminder.delete(id, tx)) +// // update db +// await authTrx((tx) => models.reminder.delete(id, tx)) - return { - reminder: { - id: reminder.id, - archiveUntil: reminder.archiveUntil || false, - sendNotification: reminder.sendNotification || true, - remindAt: reminder.remindAt, - }, - } - } catch (e) { - log.error(e) +// return { +// reminder: { +// id: reminder.id, +// archiveUntil: reminder.archiveUntil || false, +// sendNotification: reminder.sendNotification || true, +// remindAt: reminder.remindAt, +// }, +// } +// } catch (e) { +// log.error(e) - return { - errorCodes: [DeleteReminderErrorCode.BadRequest], - } - } -}) +// return { +// errorCodes: [DeleteReminderErrorCode.BadRequest], +// } +// } +// }) -// check if there exists reminders for the same user at the same time -// create a Google cloud task if no existing task and return task name -const groupReminders = async ( - scheduledTime: Date, - userId: string, - models: DataModels -): Promise => { - const exists = await models.reminder.existByUserAndRemindAt( - userId, - scheduledTime - ) +// // check if there exists reminders for the same user at the same time +// // create a Google cloud task if no existing task and return task name +// const groupReminders = async ( +// scheduledTime: Date, +// userId: string, +// models: DataModels +// ): Promise => { +// const exists = await models.reminder.existByUserAndRemindAt( +// userId, +// scheduledTime +// ) - if (!exists) { - return enqueueReminder(userId, scheduledTime.getTime()) - } +// if (!exists) { +// return enqueueReminder(userId, scheduledTime.getTime()) +// } - return undefined -} +// return undefined +// } diff --git a/packages/api/src/resolvers/rules/index.ts b/packages/api/src/resolvers/rules/index.ts index 456f127a0..b26e4b569 100644 --- a/packages/api/src/resolvers/rules/index.ts +++ b/packages/api/src/resolvers/rules/index.ts @@ -1,4 +1,6 @@ -import { authorized } from '../../utils/helpers' +import { getRepository } from '../../entity' +import { Rule } from '../../entity/rule' +import { User } from '../../entity/user' import { DeleteRuleError, DeleteRuleErrorCode, @@ -13,9 +15,7 @@ import { SetRuleErrorCode, SetRuleSuccess, } from '../../generated/graphql' -import { getRepository } from '../../entity/utils' -import { User } from '../../entity/user' -import { Rule } from '../../entity/rule' +import { authorized } from '../../utils/helpers' export const setRuleResolver = authorized< SetRuleSuccess, diff --git a/packages/api/src/resolvers/save/index.ts b/packages/api/src/resolvers/save/index.ts index 27e7ace38..3c737d493 100644 --- a/packages/api/src/resolvers/save/index.ts +++ b/packages/api/src/resolvers/save/index.ts @@ -1,5 +1,5 @@ +import { getRepository } from '../../entity' import { User } from '../../entity/user' -import { getRepository } from '../../entity/utils' import { env } from '../../env' import { MutationSaveFileArgs, @@ -13,19 +13,15 @@ import { saveFile } from '../../services/save_file' import { savePage } from '../../services/save_page' import { saveUrl } from '../../services/save_url' import { analytics } from '../../utils/analytics' -import { authorized, userDataToUser } from '../../utils/helpers' +import { authorized } from '../../utils/helpers' export const savePageResolver = authorized< SaveSuccess, SaveError, MutationSavePageArgs >(async (_, { input }, ctx) => { - const { - models, - claims: { uid }, - } = ctx analytics.track({ - userId: uid, + userId: ctx.uid, event: 'link_saved', properties: { url: input.url, @@ -35,16 +31,14 @@ export const savePageResolver = authorized< }, }) - const user = userDataToUser(await models.user.get(uid)) + const user = await getRepository(User).findOneBy({ + id: ctx.uid, + }) if (!user) { return { errorCodes: [SaveErrorCode.Unauthorized] } } - return savePage( - { ...ctx, uid, refresh: true }, - { userId: user.id, username: user.profile.username }, - input - ) + return savePage(ctx, user, input) }) export const saveUrlResolver = authorized< @@ -74,7 +68,7 @@ export const saveUrlResolver = authorized< return { errorCodes: [SaveErrorCode.Unauthorized] } } - return (await saveUrl({ ...ctx, uid }, user, input)) as SaveSuccess + return saveUrl(ctx, user, input) }) export const saveFileResolver = authorized< @@ -82,13 +76,8 @@ export const saveFileResolver = authorized< SaveError, MutationSaveFileArgs >(async (_, { input }, ctx) => { - const { - models, - claims: { uid }, - } = ctx - analytics.track({ - userId: uid, + userId: ctx.uid, event: 'link_saved', properties: { url: input.url, @@ -98,10 +87,12 @@ export const saveFileResolver = authorized< }, }) - const user = userDataToUser(await models.user.get(uid)) + const user = await getRepository(User).findOneBy({ + id: ctx.uid, + }) if (!user) { return { errorCodes: [SaveErrorCode.Unauthorized] } } - return (await saveFile({ ...ctx, uid }, user, input)) as SaveSuccess + return saveFile(ctx, user, input) }) diff --git a/packages/api/src/resolvers/subscriptions/index.ts b/packages/api/src/resolvers/subscriptions/index.ts index 4665fdd4d..9fe846258 100644 --- a/packages/api/src/resolvers/subscriptions/index.ts +++ b/packages/api/src/resolvers/subscriptions/index.ts @@ -1,8 +1,8 @@ import Parser from 'rss-parser' import { Brackets } from 'typeorm' +import { getRepository } from '../../entity' import { Subscription } from '../../entity/subscription' import { User } from '../../entity/user' -import { getRepository } from '../../entity/utils' import { env } from '../../env' import { MutationSubscribeArgs, diff --git a/packages/api/src/resolvers/types.ts b/packages/api/src/resolvers/types.ts index 2a26a3ff4..17cf31258 100644 --- a/packages/api/src/resolvers/types.ts +++ b/packages/api/src/resolvers/types.ts @@ -1,20 +1,10 @@ /* eslint-disable @typescript-eslint/ban-types */ -import { Context as ApolloContext } from 'apollo-server-core' -import winston from 'winston' -import { Knex } from 'knex' -import UserModel from '../datalayer/user' -import ArticleModel from '../datalayer/article' -import UserArticleModel from '../datalayer/links' -import UserFriendModel from '../datalayer/user_friends' -import UserPersonalizationModel from '../datalayer/user_personalization' -import ArticleSavingRequestModel from '../datalayer/article_saving_request' -import UploadFileDataModel from '../datalayer/upload_files' -import * as jwt from 'jsonwebtoken' import { Span } from '@opentelemetry/api' -import HighlightModel from '../datalayer/highlight' -import ReactionModel from '../datalayer/reaction' +import { Context as ApolloContext } from 'apollo-server-core' +import * as jwt from 'jsonwebtoken' +import { EntityManager } from 'typeorm' +import winston from 'winston' import { PubsubClient } from '../datalayer/pubsub' -import ReminderModel from '../datalayer/reminders' export interface Claims { uid: string @@ -30,35 +20,19 @@ export type ClaimsToSet = { userRole?: string | null } -export type DataModels = { - user: UserModel - article: ArticleModel - userArticle: UserArticleModel - userFriends: UserFriendModel - userPersonalization: UserPersonalizationModel - articleSavingRequest: ArticleSavingRequestModel - uploadFile: UploadFileDataModel - highlight: HighlightModel - reaction: ReactionModel - reminder: ReminderModel -} - export interface RequestContext { log: winston.Logger claims: Claims | undefined - kx: Knex pubsub: PubsubClient - models: DataModels setAuth: (claims: ClaimsToSet, secret?: string) => Promise clearAuth: () => void - setClaims: (tx: Knex.Transaction, uuid?: string | undefined) => Promise - // eslint-disable-next-line @typescript-eslint/ban-types + setClaims: (em: EntityManager, uuid?: string | undefined) => Promise signToken: ( arg1: string | object | Buffer, arg2: jwt.Secret ) => Promise authTrx: ( - cb: (tx: Knex.Transaction) => TResult, + cb: (em: EntityManager) => TResult, userRole?: string ) => Promise tracingSpan: Span diff --git a/packages/api/src/resolvers/upload_files/index.ts b/packages/api/src/resolvers/upload_files/index.ts index b529d2412..2fcd49980 100644 --- a/packages/api/src/resolvers/upload_files/index.ts +++ b/packages/api/src/resolvers/upload_files/index.ts @@ -3,24 +3,25 @@ import normalizeUrl from 'normalize-url' import path from 'path' import { createPage, getPageByParam, updatePage } from '../../elastic/pages' import { PageType } from '../../elastic/types' +import { uploadFileRepository } from '../../entity' +import { UploadFile } from '../../entity/upload_file' import { env } from '../../env' import { ArticleSavingRequestStatus, MutationUploadFileRequestArgs, - ResolverFn, + UploadFileRequestError, UploadFileRequestErrorCode, - UploadFileRequestResult, + UploadFileRequestSuccess, UploadFileStatus, } from '../../generated/graphql' import { validateUrl } from '../../services/create_page_save_request' import { analytics } from '../../utils/analytics' -import { generateSlug } from '../../utils/helpers' +import { authorized, generateSlug } from '../../utils/helpers' import { generateUploadFilePathName, generateUploadSignedUrl, getFilePublicUrl, } from '../../utils/uploads' -import { WithDataSourcesContext } from '../types' const isFileUrl = (url: string): boolean => { const parsedUrl = new URL(url) @@ -34,23 +35,18 @@ export const pageTypeForContentType = (contentType: string): PageType => { return PageType.File } -export const uploadFileRequestResolver: ResolverFn< - UploadFileRequestResult, - unknown, - WithDataSourcesContext, +export const uploadFileRequestResolver = authorized< + UploadFileRequestSuccess, + UploadFileRequestError, MutationUploadFileRequestArgs -> = async (_obj, { input }, ctx) => { - const { models, authTrx, claims, log } = ctx +>(async (_, { input }, ctx) => { + const { authTrx, uid, log } = ctx let uploadFileData: { id: string | null } = { id: null, } - if (!claims?.uid) { - return { errorCodes: [UploadFileRequestErrorCode.Unauthorized] } - } - analytics.track({ - userId: claims.uid, + userId: uid, event: 'file_upload_request', properties: { url: input.url, @@ -89,10 +85,10 @@ export const uploadFileRequestResolver: ResolverFn< return { errorCodes: [UploadFileRequestErrorCode.BadInput] } } - uploadFileData = await models.uploadFile.create({ + uploadFileData = await uploadFileRepository.save({ url: input.url, - userId: claims.uid, - fileName: fileName, + userId: uid, + fileName, status: UploadFileStatus.Initialized, contentType: input.contentType, }) @@ -113,14 +109,10 @@ export const uploadFileRequestResolver: ResolverFn< // If this is a file URL, we swap in the GCS public URL if (isFileUrl(input.url)) { await authTrx(async (tx) => { - await models.uploadFile.update( - uploadFileId, - { - url: publicUrl, - status: UploadFileStatus.Initialized, - }, - tx - ) + await tx.getRepository(UploadFile).update(uploadFileId, { + url: publicUrl, + status: UploadFileStatus.Initialized, + }) }) } @@ -131,7 +123,7 @@ export const uploadFileRequestResolver: ResolverFn< // new item. const page = isFileUrl(input.url) ? await getPageByParam({ - userId: claims.uid, + userId: uid, url: input.url, }) : undefined @@ -155,7 +147,7 @@ export const uploadFileRequestResolver: ResolverFn< { url: isFileUrl(input.url) ? publicUrl : input.url, id: input.clientRequestId || '', - userId: claims.uid, + userId: uid, title: title, hash: uploadFilePathName, content: '', @@ -185,4 +177,4 @@ export const uploadFileRequestResolver: ResolverFn< } else { return { errorCodes: [UploadFileRequestErrorCode.FailedCreate] } } -} +}) diff --git a/packages/api/src/resolvers/user/index.ts b/packages/api/src/resolvers/user/index.ts index 62b24a794..2591a6269 100644 --- a/packages/api/src/resolvers/user/index.ts +++ b/packages/api/src/resolvers/user/index.ts @@ -1,8 +1,8 @@ import * as jwt from 'jsonwebtoken' import { RegistrationType } from '../../datalayer/user/model' import { deletePagesByParam } from '../../elastic/pages' +import { setClaims, userRepository } from '../../entity' import { User as UserEntity } from '../../entity/user' -import { getRepository, setClaims } from '../../entity/utils' import { env } from '../../env' import { DeleteAccountError, @@ -39,7 +39,7 @@ import { UsersSuccess, } from '../../generated/graphql' import { AppDataSource } from '../../server' -import { createUser } from '../../services/create_user' +import { createUser, getTopUsers } from '../../services/create_user' import { sendVerificationEmail } from '../../services/send_emails' import { authorized, userDataToUser } from '../../utils/helpers' import { validateUsername } from '../../utils/usernamePolicy' @@ -49,8 +49,10 @@ export const updateUserResolver = authorized< UpdateUserSuccess, UpdateUserError, MutationUpdateUserArgs ->(async (_, { input: { name, bio } }, { models, authTrx, claims }) => { - const user = await models.user.get(claims.uid) +>(async (_, { input: { name, bio } }, { uid }) => { + const user = await userRepository.findOneBy({ + id: uid, + }) if (!user) { return { errorCodes: [UpdateUserErrorCode.UserNotFound] } } @@ -69,16 +71,14 @@ export const updateUserResolver = authorized< return { errorCodes } } - const updatedUser = await authTrx(async (tx) => { - const [updatedRecord, profile] = await Promise.all([ - models.user.update( - claims.uid, - { name, source: user.source, sourceUserId: user.sourceUserId }, - tx - ), - models.user.updateProfile(claims.uid, { bio }, tx), - ]) - return { ...updatedRecord, profile } + const updatedUser = await userRepository.save({ + id: uid, + name, + source: user.source, + sourceUserId: user.sourceUserId, + profile: { + bio, + }, }) return { user: userDataToUser(updatedUser) } @@ -88,69 +88,63 @@ export const updateUserProfileResolver = authorized< UpdateUserProfileSuccess, UpdateUserProfileError, MutationUpdateUserProfileArgs ->( - async ( - _, - { input: { userId, username, pictureUrl } }, - { models, authTrx, claims } - ) => { - const user = await models.user.get(userId) - if (user.id !== claims.uid) { - return { - errorCodes: [UpdateUserProfileErrorCode.Forbidden], - } - } - - if (!(username || pictureUrl)) { - return { - errorCodes: [UpdateUserProfileErrorCode.BadData], - } - } - - const lowerCasedUsername = username?.toLowerCase() - if (lowerCasedUsername) { - const existingUser = await models.user.getWhere({ - username: lowerCasedUsername, - }) - if (existingUser?.id) { - return { - errorCodes: [UpdateUserProfileErrorCode.UsernameExists], - } - } - - if (!validateUsername(lowerCasedUsername)) { - return { - errorCodes: [UpdateUserProfileErrorCode.BadUsername], - } - } - } - - const updatedProfile = await authTrx((tx) => - models.user.updateProfile( - userId, - { - username: lowerCasedUsername, - picture_url: pictureUrl, - }, - tx - ) - ) - - user.profile = { - ...updatedProfile, - picture_url: updatedProfile.pictureUrl, - } - - return { user: userDataToUser(user) } +>(async (_, { input: { userId, username, pictureUrl } }, { uid }) => { + const user = await userRepository.findOneBy({ + id: userId, + }) + if (!user) { + return { errorCodes: [UpdateUserProfileErrorCode.Unauthorized] } } -) + + if (user.id !== uid) { + return { + errorCodes: [UpdateUserProfileErrorCode.Forbidden], + } + } + + if (!(username || pictureUrl)) { + return { + errorCodes: [UpdateUserProfileErrorCode.BadData], + } + } + + const lowerCasedUsername = username?.toLowerCase() + if (lowerCasedUsername) { + const existingUser = await userRepository.findOneBy({ + profile: { + username: lowerCasedUsername, + }, + }) + if (existingUser?.id) { + return { + errorCodes: [UpdateUserProfileErrorCode.UsernameExists], + } + } + + if (!validateUsername(lowerCasedUsername)) { + return { + errorCodes: [UpdateUserProfileErrorCode.BadUsername], + } + } + } + + const updatedUser = await userRepository.save({ + id: uid, + profile: { + username: lowerCasedUsername, + pictureUrl, + }, + }) + + return { user: userDataToUser(updatedUser) } +}) export const googleLoginResolver: ResolverFn< LoginResult, unknown, WithDataSourcesContext, MutationGoogleLoginArgs -> = async (_obj, { input }, { models, setAuth }) => { +> = async (_obj, { input }, { setAuth }) => { const { email, secret } = input try { @@ -159,7 +153,7 @@ export const googleLoginResolver: ResolverFn< return { errorCodes: [LoginErrorCode.AuthFailed] } } - const user = await models.user.getWhere({ + const user = await userRepository.findOneBy({ email, }) if (!user?.id) { @@ -176,13 +170,18 @@ export const validateUsernameResolver: ResolverFn< Record, WithDataSourcesContext, QueryValidateUsernameArgs -> = async (_obj, { username }, { models }) => { +> = async (_obj, { username }) => { const lowerCasedUsername = username.toLowerCase() if (!validateUsername(lowerCasedUsername)) { return false } - return !(await models.user.exists({ username: lowerCasedUsername })) + const user = await userRepository.findOneBy({ + profile: { + username: lowerCasedUsername, + }, + }) + return !user } export const googleSignupResolver: ResolverFn< @@ -245,11 +244,20 @@ export const getMeUserResolver: ResolverFn< unknown, WithDataSourcesContext, unknown -> = async (_obj, __, { models, claims }) => { +> = async (_obj, __, { claims }) => { try { - return claims?.uid - ? userDataToUser(await models.user.get(claims.uid)) - : undefined + if (!claims?.uid) { + return undefined + } + + const user = await userRepository.findOneBy({ + id: claims.uid, + }) + if (!user) { + return undefined + } + + return userDataToUser(user) } catch (error) { return undefined } @@ -260,18 +268,20 @@ export const getUserResolver: ResolverFn< unknown, WithDataSourcesContext, QueryUserArgs -> = async (_obj, { userId: id, username }, { models, claims }) => { +> = async (_obj, { userId: id, username }, { uid }) => { if (!(id || username)) { return { errorCodes: [UserErrorCode.BadRequest] } } const userId = - id || (username && (await models.user.getWhere({ username }))?.id) + id || + (username && + (await userRepository.findOneBy({ profile: { username } }))?.id) if (!userId) { return { errorCodes: [UserErrorCode.UserNotFound] } } - const userRecord = await models.user.getUserDetails(claims?.uid, userId) + const userRecord = await userRepository.findOneBy({ id: userId }) if (!userRecord) { return { errorCodes: [UserErrorCode.UserNotFound] } } @@ -280,9 +290,8 @@ export const getUserResolver: ResolverFn< } export const getAllUsersResolver = authorized( - async (_obj, _params, { models, claims, authTrx }) => { - const users = - (await authTrx((tx) => models.user.getTopUsers(claims.uid, tx))) || [] + async (_obj, _params) => { + const users = await getTopUsers() const result = { users: users.map((userData) => userDataToUser(userData)) } return result } @@ -303,8 +312,10 @@ export const deleteAccountResolver = authorized< DeleteAccountSuccess, DeleteAccountError, MutationDeleteAccountArgs ->(async (_, { userID }, { models, claims, log, pubsub }) => { - const user = await models.user.get(userID) +>(async (_, { userID }, { claims, log, pubsub }) => { + const user = await userRepository.findOneBy({ + id: userID, + }) if (!user) { return { errorCodes: [DeleteAccountErrorCode.UserNotFound], diff --git a/packages/api/src/resolvers/user_feed_article/index.ts b/packages/api/src/resolvers/user_feed_article/index.ts index acfc154a5..724896678 100644 --- a/packages/api/src/resolvers/user_feed_article/index.ts +++ b/packages/api/src/resolvers/user_feed_article/index.ts @@ -1,24 +1,12 @@ /* eslint-disable @typescript-eslint/require-await */ /* eslint-disable @typescript-eslint/no-unused-vars */ import { PartialArticle } from '..' -import { getShareInfoForArticle } from '../../datalayer/links/share_info' import { FeedArticle, - MutationUpdateSharedCommentArgs, PageInfo, - QueryFeedArticlesArgs, - QuerySharedArticleArgs, - ResolverFn, - SharedArticleError, - SharedArticleErrorCode, SharedArticleSuccess, - UpdateSharedCommentError, - UpdateSharedCommentErrorCode, - UpdateSharedCommentSuccess, } from '../../generated/graphql' import { Merge } from '../../util' -import { authorized } from '../../utils/helpers' -import { WithDataSourcesContext } from '../types' export type PartialFeedArticle = Omit< FeedArticle, @@ -35,139 +23,139 @@ export type SharedArticleSuccessPartial = Merge< { article: PartialArticle } > -export const getSharedArticleResolver: ResolverFn< - SharedArticleSuccessPartial | SharedArticleError, - Record, - WithDataSourcesContext, - QuerySharedArticleArgs -> = async (_obj, { username, slug, selectedHighlightId }, { kx, models }) => { - try { - const user = await models.user.getWhere({ username }) - if (!user) { - return { - errorCodes: [SharedArticleErrorCode.NotFound], - } - } +// export const getSharedArticleResolver: ResolverFn< +// SharedArticleSuccessPartial | SharedArticleError, +// Record, +// WithDataSourcesContext, +// QuerySharedArticleArgs +// > = async (_obj, { username, slug, selectedHighlightId }, { kx, models }) => { +// try { +// const user = await models.user.getWhere({ username }) +// if (!user) { +// return { +// errorCodes: [SharedArticleErrorCode.NotFound], +// } +// } - const article = await models.userArticle.getBySlug(username, slug) - if (!article || !article.sharedAt) { - return { - errorCodes: [SharedArticleErrorCode.NotFound], - } - } +// const article = await models.userArticle.getBySlug(username, slug) +// if (!article || !article.sharedAt) { +// return { +// errorCodes: [SharedArticleErrorCode.NotFound], +// } +// } - if (selectedHighlightId) { - const highlightResult = await models.highlight.getWhereIn('shortId', [ - selectedHighlightId, - ]) - if (!highlightResult || !highlightResult[0].sharedAt) { - return { - errorCodes: [SharedArticleErrorCode.NotFound], - } - } - } +// if (selectedHighlightId) { +// const highlightResult = await models.highlight.getWhereIn('shortId', [ +// selectedHighlightId, +// ]) +// if (!highlightResult || !highlightResult[0].sharedAt) { +// return { +// errorCodes: [SharedArticleErrorCode.NotFound], +// } +// } +// } - const shareInfo = await getShareInfoForArticle( - kx, - user.id, - article.id, - models - ) +// const shareInfo = await getShareInfoForArticle( +// kx, +// user.id, +// article.id, +// models +// ) - return { article: { ...article, userId: user.id, shareInfo: shareInfo } } - } catch (error) { - return { errorCodes: [SharedArticleErrorCode.NotFound] } - } -} +// return { article: { ...article, userId: user.id, shareInfo: shareInfo } } +// } catch (error) { +// return { errorCodes: [SharedArticleErrorCode.NotFound] } +// } +// } -export const getUserFeedArticlesResolver: ResolverFn< - PaginatedFeedArticlesSuccessPartial, - unknown, - WithDataSourcesContext, - QueryFeedArticlesArgs -> = async ( - _obj, - { after: _startCursor, first: _first, sharedByUser }, - { models, claims, authTrx } -) => { - if (!(sharedByUser || claims?.uid)) { - return { - edges: [], - pageInfo: { - startCursor: '', - endCursor: '', - hasNextPage: false, - hasPreviousPage: false, - }, - } - } +// export const getUserFeedArticlesResolver: ResolverFn< +// PaginatedFeedArticlesSuccessPartial, +// unknown, +// WithDataSourcesContext, +// QueryFeedArticlesArgs +// > = async ( +// _obj, +// { after: _startCursor, first: _first, sharedByUser }, +// { models, claims, authTrx } +// ) => { +// if (!(sharedByUser || claims?.uid)) { +// return { +// edges: [], +// pageInfo: { +// startCursor: '', +// endCursor: '', +// hasNextPage: false, +// hasPreviousPage: false, +// }, +// } +// } - const first = _first || 0 - const startCursor = _startCursor || '' +// const first = _first || 0 +// const startCursor = _startCursor || '' - const feedArticles = - (await authTrx((tx) => - models.userArticle.getUserFeedArticlesPaginatedWithHighlights( - { cursor: startCursor, first: first + 1, sharedByUser }, // fetch one more item to get next cursor - claims?.uid || '', - tx - ) - )) || [] +// const feedArticles = +// (await authTrx((tx) => +// models.userArticle.getUserFeedArticlesPaginatedWithHighlights( +// { cursor: startCursor, first: first + 1, sharedByUser }, // fetch one more item to get next cursor +// claims?.uid || '', +// tx +// ) +// )) || [] - const endCursor = feedArticles[feedArticles.length - 1]?.sharedAt - .getTime() - ?.toString() - const hasNextPage = feedArticles.length > first +// const endCursor = feedArticles[feedArticles.length - 1]?.sharedAt +// .getTime() +// ?.toString() +// const hasNextPage = feedArticles.length > first - if (hasNextPage) { - // remove an extra if exists - feedArticles.pop() - } +// if (hasNextPage) { +// // remove an extra if exists +// feedArticles.pop() +// } - const edges = feedArticles.map((fa) => { - return { - node: fa, - cursor: fa.sharedAt.getTime()?.toString(), - } - }) +// const edges = feedArticles.map((fa) => { +// return { +// node: fa, +// cursor: fa.sharedAt.getTime()?.toString(), +// } +// }) - return { - edges, - pageInfo: { - hasPreviousPage: false, - startCursor: '', - hasNextPage, - endCursor, - }, - } -} +// return { +// edges, +// pageInfo: { +// hasPreviousPage: false, +// startCursor: '', +// hasNextPage, +// endCursor, +// }, +// } +// } -export const updateSharedCommentResolver = authorized< - UpdateSharedCommentSuccess, - UpdateSharedCommentError, - MutationUpdateSharedCommentArgs ->( - async ( - _, - { input: { articleID, sharedComment } }, - { models, authTrx, claims: { uid } } - ) => { - const ua = await authTrx((tx) => - models.userArticle.getByParameters(uid, { articleId: articleID }, tx) - ) - if (!ua) { - return { errorCodes: [UpdateSharedCommentErrorCode.NotFound] } - } +// export const updateSharedCommentResolver = authorized< +// UpdateSharedCommentSuccess, +// UpdateSharedCommentError, +// MutationUpdateSharedCommentArgs +// >( +// async ( +// _, +// { input: { articleID, sharedComment } }, +// { models, authTrx, claims: { uid } } +// ) => { +// const ua = await authTrx((tx) => +// models.userArticle.getByParameters(uid, { articleId: articleID }, tx) +// ) +// if (!ua) { +// return { errorCodes: [UpdateSharedCommentErrorCode.NotFound] } +// } - await authTrx((tx) => - models.userArticle.updateByArticleId( - uid, - articleID, - { sharedComment }, - tx - ) - ) +// await authTrx((tx) => +// models.userArticle.updateByArticleId( +// uid, +// articleID, +// { sharedComment }, +// tx +// ) +// ) - return { articleID, sharedComment } - } -) +// return { articleID, sharedComment } +// } +// ) diff --git a/packages/api/src/resolvers/user_friends/index.ts b/packages/api/src/resolvers/user_friends/index.ts index 39da6e5d8..823184066 100644 --- a/packages/api/src/resolvers/user_friends/index.ts +++ b/packages/api/src/resolvers/user_friends/index.ts @@ -1,118 +1,101 @@ -import { Knex } from 'knex' -import { UserData } from '../../datalayer/user/model' -import { - GetFollowersResult, - GetFollowingResult, - MutationSetFollowArgs, - QueryGetFollowersArgs, - QueryGetFollowingArgs, - ResolverFn, - SetFollowError, - SetFollowErrorCode, - SetFollowSuccess, - User, -} from '../../generated/graphql' -import { authorized, userDataToUser } from '../../utils/helpers' -import { DataModels, WithDataSourcesContext } from '../types' +// export const setFollowResolver = authorized< +// SetFollowSuccess, +// SetFollowError, +// MutationSetFollowArgs +// >( +// async ( +// _, +// { input: { userId: friendUserId, follow } }, +// { models, authTrx, claims: { uid } } +// ) => { +// const user = await models.user.getUserDetails(uid, friendUserId) +// if (!user) return { errorCodes: [SetFollowErrorCode.NotFound] } -export const setFollowResolver = authorized< - SetFollowSuccess, - SetFollowError, - MutationSetFollowArgs ->( - async ( - _, - { input: { userId: friendUserId, follow } }, - { models, authTrx, claims: { uid } } - ) => { - const user = await models.user.getUserDetails(uid, friendUserId) - if (!user) return { errorCodes: [SetFollowErrorCode.NotFound] } +// const userFriendRecord = await authTrx((tx) => +// models.userFriends.getByUserFriendId(uid, friendUserId, tx) +// ) - const userFriendRecord = await authTrx((tx) => - models.userFriends.getByUserFriendId(uid, friendUserId, tx) - ) +// if (follow) { +// if (!userFriendRecord) { +// await authTrx((tx) => +// models.userFriends.create({ friendUserId, userId: uid }, tx) +// ) +// } +// } else if (userFriendRecord) { +// await authTrx((tx) => models.userFriends.delete(userFriendRecord.id, tx)) +// } - if (follow) { - if (!userFriendRecord) { - await authTrx((tx) => - models.userFriends.create({ friendUserId, userId: uid }, tx) - ) - } - } else if (userFriendRecord) { - await authTrx((tx) => models.userFriends.delete(userFriendRecord.id, tx)) - } +// const updatedUser = await models.user.getUserDetails(uid, friendUserId) +// if (!updatedUser) return { errorCodes: [SetFollowErrorCode.NotFound] } - const updatedUser = await models.user.getUserDetails(uid, friendUserId) - if (!updatedUser) return { errorCodes: [SetFollowErrorCode.NotFound] } +// return { +// updatedUser: { +// ...userDataToUser(updatedUser), +// isFriend: updatedUser.viewerIsFollowing, +// }, +// } +// } +// ) - return { - updatedUser: { - ...userDataToUser(updatedUser), - isFriend: updatedUser.viewerIsFollowing, - }, - } - } -) +// const getUserList = async ( +// uid: string, +// users: UserData[], +// models: DataModels, +// authTrx: ( +// cb: (tx: Knex.Transaction) => TResult, +// userRole?: string +// ) => Promise +// ): Promise => { +// const usersIds = users.map(({ id }) => id) +// const friends = await authTrx((tx) => +// models.userFriends.getByFriendIds(uid, usersIds, tx) +// ) -const getUserList = async ( - uid: string, - users: UserData[], - models: DataModels, - authTrx: ( - cb: (tx: Knex.Transaction) => TResult, - userRole?: string - ) => Promise -): Promise => { - const usersIds = users.map(({ id }) => id) - const friends = await authTrx((tx) => - models.userFriends.getByFriendIds(uid, usersIds, tx) - ) +// const friendsIds = friends.map(({ friendUserId }) => friendUserId) +// users = users.map((f) => ({ +// ...f, +// isFriend: friendsIds.includes(f.id), +// viewerIsFollowing: friendsIds.includes(f.id), +// })) - const friendsIds = friends.map(({ friendUserId }) => friendUserId) - users = users.map((f) => ({ - ...f, - isFriend: friendsIds.includes(f.id), - viewerIsFollowing: friendsIds.includes(f.id), - })) +// return users.map((u) => userDataToUser(u)) +// } - return users.map((u) => userDataToUser(u)) -} +// export const getFollowersResolver: ResolverFn< +// GetFollowersResult, +// unknown, +// WithDataSourcesContext, +// QueryGetFollowersArgs +// > = async (_parent, { userId }, { models, claims, authTrx }) => { +// const followers = userId +// ? await authTrx((tx) => models.user.getUserFollowersList(userId, tx)) +// : [] +// if (!claims?.uid) return { followers: usersWithNoFriends(followers) } +// return { +// followers: await getUserList(claims?.uid, followers, models, authTrx), +// } +// } -export const getFollowersResolver: ResolverFn< - GetFollowersResult, - unknown, - WithDataSourcesContext, - QueryGetFollowersArgs -> = async (_parent, { userId }, { models, claims, authTrx }) => { - const followers = userId - ? await authTrx((tx) => models.user.getUserFollowersList(userId, tx)) - : [] - if (!claims?.uid) return { followers: usersWithNoFriends(followers) } - return { - followers: await getUserList(claims?.uid, followers, models, authTrx), - } -} +// export const getFollowingResolver: ResolverFn< +// GetFollowingResult, +// unknown, +// WithDataSourcesContext, +// QueryGetFollowingArgs +// > = async (_parent, { userId }, { models, claims, authTrx }) => { +// const following = userId +// ? await authTrx((tx) => models.user.getUserFollowingList(userId, tx)) +// : [] +// if (!claims?.uid) return { following: usersWithNoFriends(following) } +// return { +// following: await getUserList(claims?.uid, following, models, authTrx), +// } +// } -export const getFollowingResolver: ResolverFn< - GetFollowingResult, - unknown, - WithDataSourcesContext, - QueryGetFollowingArgs -> = async (_parent, { userId }, { models, claims, authTrx }) => { - const following = userId - ? await authTrx((tx) => models.user.getUserFollowingList(userId, tx)) - : [] - if (!claims?.uid) return { following: usersWithNoFriends(following) } - return { - following: await getUserList(claims?.uid, following, models, authTrx), - } -} - -const usersWithNoFriends = (users: UserData[]): User[] => { - return users.map((f) => - userDataToUser({ - ...f, - isFriend: false, - } as UserData) - ) -} +// const usersWithNoFriends = (users: UserData[]): User[] => { +// return users.map((f) => +// userDataToUser({ +// ...f, +// isFriend: false, +// } as UserData) +// ) +// } diff --git a/packages/api/src/resolvers/user_personalization/index.ts b/packages/api/src/resolvers/user_personalization/index.ts index 955dc16d4..54c179bdf 100644 --- a/packages/api/src/resolvers/user_personalization/index.ts +++ b/packages/api/src/resolvers/user_personalization/index.ts @@ -1,3 +1,5 @@ +import { getRepository, setClaims } from '../../entity' +import { UserPersonalization } from '../../entity/user_personalization' import { GetUserPersonalizationError, GetUserPersonalizationResult, @@ -7,10 +9,8 @@ import { SetUserPersonalizationSuccess, SortOrder, } from '../../generated/graphql' -import { authorized } from '../../utils/helpers' -import { UserPersonalization } from '../../entity/user_personalization' import { AppDataSource } from '../../server' -import { getRepository, setClaims } from '../../entity/utils' +import { authorized } from '../../utils/helpers' export const setUserPersonalizationResolver = authorized< SetUserPersonalizationSuccess, @@ -58,8 +58,12 @@ export const setUserPersonalizationResolver = authorized< export const getUserPersonalizationResolver = authorized< GetUserPersonalizationResult, GetUserPersonalizationError ->(async (_parent, _args, { models, claims: { uid } }) => { - const userPersonalization = await models.userPersonalization.getByUserId(uid) +>(async (_parent, _args, { uid }) => { + const userPersonalization = await getRepository( + UserPersonalization + ).findOneBy({ + user: { id: uid }, + }) // Cast SortOrder from string to enum const librarySortOrder = userPersonalization?.librarySortOrder as diff --git a/packages/api/src/resolvers/webhooks/index.ts b/packages/api/src/resolvers/webhooks/index.ts index 5ec23d9fe..0d2d6b0d4 100644 --- a/packages/api/src/resolvers/webhooks/index.ts +++ b/packages/api/src/resolvers/webhooks/index.ts @@ -1,4 +1,7 @@ -import { authorized } from '../../utils/helpers' +import { getRepository } from '../../entity' +import { User } from '../../entity/user' +import { Webhook } from '../../entity/webhook' +import { env } from '../../env' import { DeleteWebhookError, DeleteWebhookErrorCode, @@ -18,11 +21,8 @@ import { WebhooksSuccess, WebhookSuccess, } from '../../generated/graphql' -import { getRepository } from '../../entity/utils' -import { User } from '../../entity/user' -import { Webhook } from '../../entity/webhook' import { analytics } from '../../utils/analytics' -import { env } from '../../env' +import { authorized } from '../../utils/helpers' export const webhooksResolver = authorized( async (_obj, _params, { claims: { uid }, log }) => { diff --git a/packages/api/src/routers/article_router.ts b/packages/api/src/routers/article_router.ts index 2444f7c8d..f87951765 100644 --- a/packages/api/src/routers/article_router.ts +++ b/packages/api/src/routers/article_router.ts @@ -6,8 +6,8 @@ import express from 'express' import * as jwt from 'jsonwebtoken' import { createPubSubClient } from '../datalayer/pubsub' import { getPageById, updatePage } from '../elastic/pages' +import { getRepository } from '../entity' import { Speech, SpeechState } from '../entity/speech' -import { getRepository } from '../entity/utils' import { env } from '../env' import { CreateArticleErrorCode } from '../generated/graphql' import { Claims } from '../resolvers/types' diff --git a/packages/api/src/routers/auth/auth_router.ts b/packages/api/src/routers/auth/auth_router.ts index a2efc46c2..60d02be9d 100644 --- a/packages/api/src/routers/auth/auth_router.ts +++ b/packages/api/src/routers/auth/auth_router.ts @@ -21,8 +21,8 @@ import { StatusType, UserData, } from '../../datalayer/user/model' +import { getRepository, setClaims } from '../../entity' import { User } from '../../entity/user' -import { getRepository, setClaims } from '../../entity/utils' import { env } from '../../env' import { LoginErrorCode, SignupErrorCode } from '../../generated/graphql' import { isErrorWithCode } from '../../resolvers' diff --git a/packages/api/src/routers/page_router.ts b/packages/api/src/routers/page_router.ts index f321c07d4..b00091c61 100644 --- a/packages/api/src/routers/page_router.ts +++ b/packages/api/src/routers/page_router.ts @@ -5,11 +5,11 @@ import cors from 'cors' import express from 'express' import * as jwt from 'jsonwebtoken' -import { kx } from '../datalayer/knex_config' import { createPubSubClient } from '../datalayer/pubsub' import { createPage, getPageByParam, updatePage } from '../elastic/pages' import { addRecommendation } from '../elastic/recommendation' import { Recommendation } from '../elastic/types' +import { uploadFileRepository } from '../entity' import { env } from '../env' import { ArticleSavingRequestStatus, @@ -17,7 +17,6 @@ import { UploadFileStatus, } from '../generated/graphql' import { Claims } from '../resolvers/types' -import { initModels } from '../server' import { getTokenByRequest } from '../utils/auth' import { corsConfig } from '../utils/corsConfig' import { @@ -76,7 +75,6 @@ export function pageRouter() { return res.status(400).send({ errorCode: 'BAD_DATA' }) } - const models = initModels(kx, false) const ctx = { uid: claims.uid, pubsub: createPubSubClient(), @@ -84,10 +82,10 @@ export function pageRouter() { const title = titleForFilePath(url) const fileName = fileNameForFilePath(url) - const uploadFileData = await models.uploadFile.create({ - url: url, + const uploadFileData = await uploadFileRepository.save({ + url, userId: claims.uid, - fileName: fileName, + fileName, status: UploadFileStatus.Initialized, contentType: 'application/pdf', }) diff --git a/packages/api/src/routers/svc/content.ts b/packages/api/src/routers/svc/content.ts index a991d3062..e52a89700 100644 --- a/packages/api/src/routers/svc/content.ts +++ b/packages/api/src/routers/svc/content.ts @@ -2,16 +2,16 @@ /* eslint-disable @typescript-eslint/no-unsafe-assignment */ /* eslint-disable @typescript-eslint/explicit-module-boundary-types */ import express from 'express' -import { setClaims } from '../../datalayer/helpers' -import { kx } from '../../datalayer/knex_config' import { createPubSubClient, readPushSubscription, } from '../../datalayer/pubsub' import { getPageByParam, updatePage } from '../../elastic/pages' import { Page } from '../../elastic/types' +import { setClaims } from '../../entity' import { ArticleSavingRequestStatus } from '../../generated/graphql' -import { initModels } from '../../server' +import { AppDataSource } from '../../server' +import { setFileUploadComplete } from '../../services/save_file' import { logger } from '../../utils/logger' interface UpdateContentMessage { @@ -76,10 +76,9 @@ export function contentServiceRouter() { pageToUpdate.state = ArticleSavingRequestStatus.Succeeded try { - const models = initModels(kx, false) - const uploadFileData = await kx.transaction(async (tx) => { + const uploadFileData = await AppDataSource.transaction(async (tx) => { await setClaims(tx, page.userId) - return models.uploadFile.setFileUploadComplete(fileId, tx) + return setFileUploadComplete(fileId, tx) }) logger.info('updated uploadFileData', uploadFileData) } catch (error) { diff --git a/packages/api/src/routers/svc/email_attachment.ts b/packages/api/src/routers/svc/email_attachment.ts index 86403aa24..62da08e7d 100644 --- a/packages/api/src/routers/svc/email_attachment.ts +++ b/packages/api/src/routers/svc/email_attachment.ts @@ -1,14 +1,14 @@ import express from 'express' -import { setClaims } from '../../datalayer/helpers' -import { kx } from '../../datalayer/knex_config' import { createPubSubClient } from '../../datalayer/pubsub' import { createPage } from '../../elastic/pages' import { ArticleSavingRequestStatus, Page } from '../../elastic/types' +import { setClaims, uploadFileRepository } from '../../entity' import { env } from '../../env' import { PageType, UploadFileStatus } from '../../generated/graphql' -import { initModels } from '../../server' +import { AppDataSource } from '../../server' import { getNewsletterEmail } from '../../services/newsletters' import { updateReceivedEmail } from '../../services/received_emails' +import { setFileUploadComplete } from '../../services/save_file' import { analytics } from '../../utils/analytics' import { getClaimsByToken } from '../../utils/auth' import { generateSlug } from '../../utils/helpers' @@ -54,8 +54,7 @@ export function emailAttachmentRouter() { }) try { - const models = initModels(kx, false) - const uploadFileData = await models.uploadFile.create({ + const uploadFileData = await uploadFileRepository.save({ url: '', userId: user.id, fileName: fileName, @@ -117,10 +116,9 @@ export function emailAttachmentRouter() { }) try { - const models = initModels(kx, false) - const uploadFile = await models.uploadFile.getWhere({ + const uploadFile = await uploadFileRepository.findOneBy({ id: uploadFileId, - userId: user.id, + user: { id: user.id }, }) if (!uploadFile) { return res.status(400).send('BAD REQUEST') @@ -131,9 +129,9 @@ export function emailAttachmentRouter() { uploadFile.fileName ) - const uploadFileData = await kx.transaction(async (tx) => { + const uploadFileData = await AppDataSource.transaction(async (tx) => { await setClaims(tx, user.id) - return models.uploadFile.setFileUploadComplete(uploadFileId, tx) + return setFileUploadComplete(uploadFileId, tx) }) if (!uploadFileData || !uploadFileData.id || !uploadFileData.fileName) { return res.status(400).send('BAD REQUEST') diff --git a/packages/api/src/routers/svc/integrations.ts b/packages/api/src/routers/svc/integrations.ts index bb09b9d8f..67b9840f1 100644 --- a/packages/api/src/routers/svc/integrations.ts +++ b/packages/api/src/routers/svc/integrations.ts @@ -8,8 +8,8 @@ import { v4 as uuidv4 } from 'uuid' import { EntityType, readPushSubscription } from '../../datalayer/pubsub' import { getPageById, searchPages } from '../../elastic/pages' import { Page } from '../../elastic/types' +import { getRepository } from '../../entity' import { Integration, IntegrationType } from '../../entity/integration' -import { getRepository } from '../../entity/utils' import { Claims } from '../../resolvers/types' import { getIntegrationService } from '../../services/integrations' import { getClaimsByToken } from '../../utils/auth' diff --git a/packages/api/src/routers/svc/reminders.ts b/packages/api/src/routers/svc/reminders.ts index 390608376..6212ff721 100644 --- a/packages/api/src/routers/svc/reminders.ts +++ b/packages/api/src/routers/svc/reminders.ts @@ -1,20 +1,12 @@ -import express from 'express' import { MulticastMessage } from 'firebase-admin/messaging' -import { setClaims } from '../../datalayer/helpers' -import { kx } from '../../datalayer/knex_config' import { createPubSubClient } from '../../datalayer/pubsub' import { updatePage } from '../../elastic/pages' +import { setClaims } from '../../entity' import { UserDeviceToken } from '../../entity/user_device_tokens' -import { env, homePageURL } from '../../env' +import { homePageURL } from '../../env' import { ContentReader } from '../../generated/graphql' -import { DataModels } from '../../resolvers/types' -import { initModels } from '../../server' -import { getPagesWithReminder, PageReminder } from '../../services/reminders' -import { getDeviceTokensByUserId } from '../../services/user_device_tokens' -import { analytics } from '../../utils/analytics' -import { logger } from '../../utils/logger' -import { sendEmail } from '../../utils/sendEmail' -import { sendMulticastPushNotifications } from '../../utils/sendNotification' +import { AppDataSource } from '../../server' +import { PageReminder, setRemindersComplete } from '../../services/reminders' interface PageToNotify { title: string @@ -24,116 +16,116 @@ interface PageToNotify { image: string | undefined | null } -export function remindersServiceRouter() { - const router = express.Router() +// export function remindersServiceRouter() { +// const router = express.Router() - // eslint-disable-next-line @typescript-eslint/no-misused-promises - router.post('/trigger', async (req, res) => { - logger.info('reminders/trigger') +// // eslint-disable-next-line @typescript-eslint/no-misused-promises +// router.post('/trigger', async (req, res) => { +// logger.info('reminders/trigger') - const { userId, scheduleTime } = req.body as { - userId?: string - scheduleTime?: number - } +// const { userId, scheduleTime } = req.body as { +// userId?: string +// scheduleTime?: number +// } - analytics.track({ - userId: userId, - event: 'reminder_triggered', - properties: { - env: env.server.apiEnv, - }, - }) +// analytics.track({ +// userId: userId, +// event: 'reminder_triggered', +// properties: { +// env: env.server.apiEnv, +// }, +// }) - if (!userId || !scheduleTime) { - res.status(400).send('Bad Request') - return - } +// if (!userId || !scheduleTime) { +// res.status(400).send('Bad Request') +// return +// } - try { - const remindAt = new Date(scheduleTime) +// try { +// const remindAt = new Date(scheduleTime) - // get all reminders by userid and scheduled time - const models = initModels(kx, false) +// // get all reminders by userid and scheduled time +// const models = initModels(kx, false) - const user = await models.user.get(userId) - if (!user || !user.email) { - logger.info('user not found', userId) - res.status(400).send('User Not Found') - return - } +// const user = await models.user.get(userId) +// if (!user || !user.email) { +// logger.info('user not found', userId) +// res.status(400).send('User Not Found') +// return +// } - const pageReminders = await getPagesWithReminder(userId, remindAt) +// const pageReminders = await getPagesWithReminder(userId, remindAt) - if (!pageReminders) { - logger.info('pages with reminders not found', userId, scheduleTime) - res.status(200).send('Reminders Not Found') - return - } +// if (!pageReminders) { +// logger.info('pages with reminders not found', userId, scheduleTime) +// res.status(200).send('Reminders Not Found') +// return +// } - logger.info('page with reminders:', pageReminders) +// logger.info('page with reminders:', pageReminders) - const [pagesToNotify, pagesToUnarchive] = getPagesToNotifyAndUnarchive( - pageReminders, - user.profile.username - ) +// const [pagesToNotify, pagesToUnarchive] = getPagesToNotifyAndUnarchive( +// pageReminders, +// user.profile.username +// ) - // If none of the fetch reminders have sendNotification - // set to true, then we should not send an email or notification - if (pagesToNotify.length > 0) { - // we have configured Sendgrid to send a template - if (!process.env.SENDGRID_REMINDER_TEMPLATE_ID) { - logger.info('Sendgrid reminder email template_id not set') +// // If none of the fetch reminders have sendNotification +// // set to true, then we should not send an email or notification +// if (pagesToNotify.length > 0) { +// // we have configured Sendgrid to send a template +// if (!process.env.SENDGRID_REMINDER_TEMPLATE_ID) { +// logger.info('Sendgrid reminder email template_id not set') - await updateRemindersStatus( - models, - userId, - pagesToUnarchive, - remindAt - ) - res.status(200).send('Template Id Not Found') - return - } +// await updateRemindersStatus( +// models, +// userId, +// pagesToUnarchive, +// remindAt +// ) +// res.status(200).send('Template Id Not Found') +// return +// } - const dynamicTemplateData = { - subject: `Omnivore Reminder Service`, - title: `Hey ${user.name}, you have ${pagesToNotify.length} article(s) to read on Omnivore`, - articles: pagesToNotify, - } +// const dynamicTemplateData = { +// subject: `Omnivore Reminder Service`, +// title: `Hey ${user.name}, you have ${pagesToNotify.length} article(s) to read on Omnivore`, +// articles: pagesToNotify, +// } - logger.info('dynamic template data:', dynamicTemplateData) +// logger.info('dynamic template data:', dynamicTemplateData) - await sendEmail({ - from: env.sender.message, - dynamicTemplateData: dynamicTemplateData, - templateId: env.sendgrid.reminderTemplateId, - to: user.email, - }) +// await sendEmail({ +// from: env.sender.message, +// dynamicTemplateData: dynamicTemplateData, +// templateId: env.sendgrid.reminderTemplateId, +// to: user.email, +// }) - // send push notifications - const deviceTokens = await getDeviceTokensByUserId(userId) - if (deviceTokens && deviceTokens.length > 0) { - const message = messageForPages(pageReminders, deviceTokens) - await sendMulticastPushNotifications(userId, message, 'reminder') - } +// // send push notifications +// const deviceTokens = await getDeviceTokensByUserId(userId) +// if (deviceTokens && deviceTokens.length > 0) { +// const message = messageForPages(pageReminders, deviceTokens) +// await sendMulticastPushNotifications(userId, message, 'reminder') +// } - if (!deviceTokens) { - logger.info('Device tokens not set:', userId) +// if (!deviceTokens) { +// logger.info('Device tokens not set:', userId) - res.status(400).send('Device token Not Found') - return - } - } +// res.status(400).send('Device token Not Found') +// return +// } +// } - await updateRemindersStatus(models, userId, pagesToUnarchive, remindAt) - res.status(200).send('Reminders triggered') - } catch (e) { - logger.info(e) - res.status(500).send(e) - } - }) +// await updateRemindersStatus(models, userId, pagesToUnarchive, remindAt) +// res.status(200).send('Reminders triggered') +// } catch (e) { +// logger.info(e) +// res.status(500).send(e) +// } +// }) - return router -} +// return router +// } const getPagesToNotifyAndUnarchive = ( pageReminders: PageReminder[], @@ -221,7 +213,6 @@ const messageForPages = ( } const updateRemindersStatus = async ( - models: DataModels, userId: string, pagesToUnarchive: string[], remindAt: Date @@ -243,8 +234,8 @@ const updateRemindersStatus = async ( } // db update - await kx.transaction(async (tx) => { + await AppDataSource.transaction(async (tx) => { await setClaims(tx, userId) - await models.reminder.setRemindersComplete(userId, remindAt, tx) + await setRemindersComplete(tx, userId, remindAt) }) } diff --git a/packages/api/src/routers/svc/rss_feed.ts b/packages/api/src/routers/svc/rss_feed.ts index c53ee726e..78e30cc0e 100644 --- a/packages/api/src/routers/svc/rss_feed.ts +++ b/packages/api/src/routers/svc/rss_feed.ts @@ -1,8 +1,8 @@ /* eslint-disable @typescript-eslint/no-misused-promises */ import express from 'express' import { readPushSubscription } from '../../datalayer/pubsub' +import { getRepository } from '../../entity' import { Subscription } from '../../entity/subscription' -import { getRepository } from '../../entity/utils' import { SubscriptionStatus, SubscriptionType } from '../../generated/graphql' import { enqueueRssFeedFetch } from '../../utils/createTask' import { logger } from '../../utils/logger' diff --git a/packages/api/src/routers/svc/webhooks.ts b/packages/api/src/routers/svc/webhooks.ts index c2cdd26e0..00937eeea 100644 --- a/packages/api/src/routers/svc/webhooks.ts +++ b/packages/api/src/routers/svc/webhooks.ts @@ -4,7 +4,7 @@ import axios, { Method } from 'axios' import express from 'express' import { readPushSubscription } from '../../datalayer/pubsub' -import { getRepository } from '../../entity/utils' +import { getRepository } from '../../entity' import { Webhook } from '../../entity/webhook' import { logger } from '../../utils/logger' diff --git a/packages/api/src/routers/text_to_speech.ts b/packages/api/src/routers/text_to_speech.ts index 5fee7cc60..1afd8e819 100644 --- a/packages/api/src/routers/text_to_speech.ts +++ b/packages/api/src/routers/text_to_speech.ts @@ -7,9 +7,9 @@ import express from 'express' import { readPushSubscription } from '../datalayer/pubsub' import { getPageById } from '../elastic/pages' import { ArticleSavingRequestStatus } from '../elastic/types' +import { getRepository, setClaims } from '../entity' import { Speech, SpeechState } from '../entity/speech' import { UserPersonalization } from '../entity/user_personalization' -import { getRepository, setClaims } from '../entity/utils' import { AppDataSource } from '../server' import { FeatureName, getFeature } from '../services/features' import { shouldSynthesize } from '../services/speech' diff --git a/packages/api/src/routers/user_router.ts b/packages/api/src/routers/user_router.ts index d2feb11fe..b80ba0bb1 100644 --- a/packages/api/src/routers/user_router.ts +++ b/packages/api/src/routers/user_router.ts @@ -2,8 +2,8 @@ /* eslint-disable @typescript-eslint/no-unsafe-assignment */ import cors from 'cors' import express from 'express' +import { getRepository } from '../entity' import { User } from '../entity/user' -import { getRepository } from '../entity/utils' import { env } from '../env' import { getClaimsByToken, getTokenByRequest } from '../utils/auth' import { corsConfig } from '../utils/corsConfig' diff --git a/packages/api/src/schema.ts b/packages/api/src/schema.ts index d0b161eb9..94f719594 100755 --- a/packages/api/src/schema.ts +++ b/packages/api/src/schema.ts @@ -2604,24 +2604,24 @@ const schema = gql` mergeHighlight(input: MergeHighlightInput!): MergeHighlightResult! updateHighlight(input: UpdateHighlightInput!): UpdateHighlightResult! deleteHighlight(highlightId: ID!): DeleteHighlightResult! - createHighlightReply( - input: CreateHighlightReplyInput! - ): CreateHighlightReplyResult! - updateHighlightReply( - input: UpdateHighlightReplyInput! - ): UpdateHighlightReplyResult! - deleteHighlightReply(highlightReplyId: ID!): DeleteHighlightReplyResult! - createReaction(input: CreateReactionInput!): CreateReactionResult! - deleteReaction(id: ID!): DeleteReactionResult! + # createHighlightReply( + # input: CreateHighlightReplyInput! + # ): CreateHighlightReplyResult! + # updateHighlightReply( + # input: UpdateHighlightReplyInput! + # ): UpdateHighlightReplyResult! + # deleteHighlightReply(highlightReplyId: ID!): DeleteHighlightReplyResult! + # createReaction(input: CreateReactionInput!): CreateReactionResult! + # deleteReaction(id: ID!): DeleteReactionResult! uploadFileRequest(input: UploadFileRequestInput!): UploadFileRequestResult! saveArticleReadingProgress( input: SaveArticleReadingProgressInput! ): SaveArticleReadingProgressResult! - setShareArticle(input: SetShareArticleInput!): SetShareArticleResult! - updateSharedComment( - input: UpdateSharedCommentInput! - ): UpdateSharedCommentResult! - setFollow(input: SetFollowInput!): SetFollowResult! + # setShareArticle(input: SetShareArticleInput!): SetShareArticleResult! + # updateSharedComment( + # input: UpdateSharedCommentInput! + # ): UpdateSharedCommentResult! + # setFollow(input: SetFollowInput!): SetFollowResult! setBookmarkArticle( input: SetBookmarkArticleInput! ): SetBookmarkArticleResult! @@ -2631,11 +2631,11 @@ const schema = gql` createArticleSavingRequest( input: CreateArticleSavingRequestInput! ): CreateArticleSavingRequestResult! - setShareHighlight(input: SetShareHighlightInput!): SetShareHighlightResult! + # setShareHighlight(input: SetShareHighlightInput!): SetShareHighlightResult! reportItem(input: ReportItemInput!): ReportItemResult! - updateLinkShareInfo( - input: UpdateLinkShareInfoInput! - ): UpdateLinkShareInfoResult! + # updateLinkShareInfo( + # input: UpdateLinkShareInfoInput! + # ): UpdateLinkShareInfoResult! setLinkArchived(input: ArchiveLinkInput!): ArchiveLinkResult! createNewsletterEmail: CreateNewsletterEmailResult! deleteNewsletterEmail(newsletterEmailId: ID!): DeleteNewsletterEmailResult! @@ -2643,9 +2643,9 @@ const schema = gql` savePage(input: SavePageInput!): SaveResult! updatePage(input: UpdatePageInput!): UpdatePageResult! saveFile(input: SaveFileInput!): SaveResult! - createReminder(input: CreateReminderInput!): CreateReminderResult! - updateReminder(input: UpdateReminderInput!): UpdateReminderResult! - deleteReminder(id: ID!): DeleteReminderResult! + # createReminder(input: CreateReminderInput!): CreateReminderResult! + # updateReminder(input: UpdateReminderInput!): UpdateReminderResult! + # deleteReminder(id: ID!): DeleteReminderResult! setDeviceToken(input: SetDeviceTokenInput!): SetDeviceTokenResult! createLabel(input: CreateLabelInput!): CreateLabelResult! updateLabel(input: UpdateLabelInput!): UpdateLabelResult! @@ -2710,25 +2710,25 @@ const schema = gql` includePending: Boolean ): ArticlesResult! article(username: String!, slug: String!, format: String): ArticleResult! - sharedArticle( - username: String! - slug: String! - selectedHighlightId: String - ): SharedArticleResult! - feedArticles( - after: String - first: Int - sort: SortParams - sharedByUser: ID - ): FeedArticlesResult! + # sharedArticle( + # username: String! + # slug: String! + # selectedHighlightId: String + # ): SharedArticleResult! + # feedArticles( + # after: String + # first: Int + # sort: SortParams + # sharedByUser: ID + # ): FeedArticlesResult! users: UsersResult! validateUsername(username: String!): Boolean! - getFollowers(userId: ID): GetFollowersResult! - getFollowing(userId: ID): GetFollowingResult! + # getFollowers(userId: ID): GetFollowersResult! + # getFollowing(userId: ID): GetFollowingResult! getUserPersonalization: GetUserPersonalizationResult! articleSavingRequest(id: ID, url: String): ArticleSavingRequestResult! newsletterEmails: NewsletterEmailsResult! - reminder(linkId: ID!): ReminderResult! + # reminder(linkId: ID!): ReminderResult! labels: LabelsResult! search( after: String diff --git a/packages/api/src/server.ts b/packages/api/src/server.ts index 7178cb427..77eeaa7df 100755 --- a/packages/api/src/server.ts +++ b/packages/api/src/server.ts @@ -11,24 +11,12 @@ import express, { Express } from 'express' import * as httpContext from 'express-http-context2' import rateLimit from 'express-rate-limit' import { createServer, Server } from 'http' -import { Knex } from 'knex' import { DataSource } from 'typeorm' import { SnakeNamingStrategy } from 'typeorm-naming-strategies' import { config, loggers } from 'winston' import { makeApolloServer } from './apollo' -import ArticleModel from './datalayer/article' -import ArticleSavingRequestModel from './datalayer/article_saving_request' -import HighlightModel from './datalayer/highlight' -import UserArticleModel from './datalayer/links' -import ReactionModel from './datalayer/reaction' -import ReminderModel from './datalayer/reminders' -import UploadFileDataModel from './datalayer/upload_files' -import UserModel from './datalayer/user' -import UserFriendModel from './datalayer/user_friends' -import UserPersonalizationModel from './datalayer/user_personalization' import { initElasticsearch } from './elastic' import { env } from './env' -import { DataModels } from './resolvers/types' import { articleRouter } from './routers/article_router' import { authRouter } from './routers/auth/auth_router' import { mobileAuthRouter } from './routers/auth/mobile/mobile_auth_router' @@ -42,7 +30,7 @@ import { emailAttachmentRouter } from './routers/svc/email_attachment' import { integrationsServiceRouter } from './routers/svc/integrations' import { linkServiceRouter } from './routers/svc/links' import { newsletterServiceRouter } from './routers/svc/newsletters' -import { remindersServiceRouter } from './routers/svc/reminders' +// import { remindersServiceRouter } from './routers/svc/reminders' import { rssFeedRouter } from './routers/svc/rss_feed' import { uploadServiceRouter } from './routers/svc/upload' import { webhooksServiceRouter } from './routers/svc/webhooks' @@ -59,19 +47,6 @@ import { const PORT = process.env.PORT || 4000 -export const initModels = (kx: Knex, cache = true): DataModels => ({ - user: new UserModel(kx, cache), - article: new ArticleModel(kx, cache), - userArticle: new UserArticleModel(kx, cache), - userFriends: new UserFriendModel(kx, cache), - userPersonalization: new UserPersonalizationModel(kx, cache), - articleSavingRequest: new ArticleSavingRequestModel(kx, cache), - uploadFile: new UploadFileDataModel(kx, cache), - highlight: new HighlightModel(kx, cache), - reaction: new ReactionModel(kx, cache), - reminder: new ReminderModel(kx, cache), -}) - export const AppDataSource = new DataSource({ type: 'postgres', host: env.pg.host, @@ -85,6 +60,9 @@ export const AppDataSource = new DataSource({ subscribers: [__dirname + '/events/**/*{.js,.ts}'], namingStrategy: new SnakeNamingStrategy(), logger: new CustomTypeOrmLogger(), + cache: true, + connectTimeoutMS: 60000, // 60 seconds + maxQueryExecutionTime: 60000, // 60 seconds }) export const createApp = (): { @@ -164,7 +142,7 @@ export const createApp = (): { app.use('/svc/pubsub/webhooks', webhooksServiceRouter()) app.use('/svc/pubsub/integrations', integrationsServiceRouter()) app.use('/svc/pubsub/rss-feed', rssFeedRouter()) - app.use('/svc/reminders', remindersServiceRouter()) + // app.use('/svc/reminders', remindersServiceRouter()) app.use('/svc/email-attachment', emailAttachmentRouter()) if (env.dev.isLocal) { diff --git a/packages/api/src/services/archive_link.ts b/packages/api/src/services/archive_link.ts index 226df4672..8d74de5e5 100644 --- a/packages/api/src/services/archive_link.ts +++ b/packages/api/src/services/archive_link.ts @@ -1,5 +1,5 @@ +import { setClaims } from '../entity' import { Link } from '../entity/link' -import { setClaims } from '../entity/utils' import { AppDataSource } from '../server' export const setLinkArchived = async ( diff --git a/packages/api/src/services/create_page_save_request.ts b/packages/api/src/services/create_page_save_request.ts index f51510148..6c7b74408 100644 --- a/packages/api/src/services/create_page_save_request.ts +++ b/packages/api/src/services/create_page_save_request.ts @@ -8,8 +8,8 @@ import { updatePage, } from '../elastic/pages' import { ArticleSavingRequestStatus, Label, PageType } from '../elastic/types' +import { getRepository } from '../entity' import { User } from '../entity/user' -import { getRepository } from '../entity/utils' import { ArticleSavingRequest, CreateArticleSavingRequestErrorCode, diff --git a/packages/api/src/services/create_user.ts b/packages/api/src/services/create_user.ts index 649e04b34..7b49e0f40 100644 --- a/packages/api/src/services/create_user.ts +++ b/packages/api/src/services/create_user.ts @@ -1,10 +1,10 @@ -import { EntityManager } from 'typeorm' +import { EntityManager, In } from 'typeorm' import { StatusType } from '../datalayer/user/model' +import { getRepository, userRepository } from '../entity' import { GroupMembership } from '../entity/groups/group_membership' import { Invite } from '../entity/groups/invite' import { Profile } from '../entity/profile' import { User } from '../entity/user' -import { getRepository } from '../entity/utils' import { SignupErrorCode } from '../generated/graphql' import { AuthProvider } from '../routers/auth/auth_types' import { AppDataSource } from '../server' @@ -15,6 +15,18 @@ import { Filter } from '../entity/filter' import { analytics } from '../utils/analytics' import { env } from '../env' +const TOP_USERS = [ + 'jacksonh', + 'nat', + 'luis', + 'satindar', + 'malandrina', + 'patrick', + 'alexgutjahr', + 'hongbowu', +] +export const MAX_RECORDS_LIMIT = 1000 + export const createUser = async (input: { provider: AuthProvider sourceUserId?: string @@ -166,9 +178,19 @@ const validateInvite = async ( } export const getUserByEmail = async (email: string): Promise => { - return getRepository(User) + return userRepository .createQueryBuilder('user') .leftJoinAndSelect('user.profile', 'profile') .where('LOWER(email) = LOWER(:email)', { email }) // case insensitive .getOne() } + +export const getTopUsers = async (): Promise => { + return userRepository + .createQueryBuilder() + .where({ + profile: { username: In(TOP_USERS) }, + }) + .limit(MAX_RECORDS_LIMIT) + .getMany() +} diff --git a/packages/api/src/services/features.ts b/packages/api/src/services/features.ts index 7d3158d70..36495fc07 100644 --- a/packages/api/src/services/features.ts +++ b/packages/api/src/services/features.ts @@ -1,7 +1,7 @@ import * as jwt from 'jsonwebtoken' import { IsNull, Not } from 'typeorm' +import { getRepository } from '../entity' import { Feature } from '../entity/feature' -import { getRepository } from '../entity/utils' import { env } from '../env' import { AppDataSource } from '../server' import { logger } from '../utils/logger' diff --git a/packages/api/src/services/followers.ts b/packages/api/src/services/followers.ts index 0b93c2e81..84543a24a 100644 --- a/packages/api/src/services/followers.ts +++ b/packages/api/src/services/followers.ts @@ -1,6 +1,6 @@ -import { User } from '../entity/user' +import { getRepository } from '../entity' import { Follower } from '../entity/follower' -import { getRepository } from '../entity/utils' +import { User } from '../entity/user' export const getUserFollowers = async ( user: User, diff --git a/packages/api/src/services/groups.ts b/packages/api/src/services/groups.ts index 3f2979182..a7b49ff6e 100644 --- a/packages/api/src/services/groups.ts +++ b/packages/api/src/services/groups.ts @@ -1,10 +1,10 @@ import { nanoid } from 'nanoid' +import { getRepository } from '../entity' import { Group } from '../entity/groups/group' import { GroupMembership } from '../entity/groups/group_membership' import { Invite } from '../entity/groups/invite' import { RuleActionType } from '../entity/rule' import { User } from '../entity/user' -import { getRepository } from '../entity/utils' import { homePageURL } from '../env' import { RecommendationGroup, User as GraphqlUser } from '../generated/graphql' import { AppDataSource } from '../server' diff --git a/packages/api/src/services/integrations/readwise.ts b/packages/api/src/services/integrations/readwise.ts index b7487c22f..70d7a3151 100644 --- a/packages/api/src/services/integrations/readwise.ts +++ b/packages/api/src/services/integrations/readwise.ts @@ -1,7 +1,7 @@ import axios from 'axios' import { HighlightType, Page } from '../../elastic/types' +import { getRepository } from '../../entity' import { Integration } from '../../entity/integration' -import { getRepository } from '../../entity/utils' import { env } from '../../env' import { wait } from '../../utils/helpers' import { logger } from '../../utils/logger' diff --git a/packages/api/src/services/labels.ts b/packages/api/src/services/labels.ts index 5837ddf72..135514e9c 100644 --- a/packages/api/src/services/labels.ts +++ b/packages/api/src/services/labels.ts @@ -2,10 +2,10 @@ import DataLoader from 'dataloader' import { In } from 'typeorm' import { addLabelInPage } from '../elastic/labels' import { PageContext } from '../elastic/types' +import { getRepository } from '../entity' import { Label } from '../entity/label' import { Link } from '../entity/link' import { User } from '../entity/user' -import { getRepository } from '../entity/utils' import { CreateLabelInput } from '../generated/graphql' import { generateRandomColor } from '../utils/helpers' import { logger } from '../utils/logger' @@ -117,18 +117,10 @@ export const createLabels = async ( ctx: PageContext, labels: CreateLabelInput[] ): Promise => { - const user = await getRepository(User).findOneBy({ - id: ctx.uid, - }) - if (!user) { - logger.error('user not found') - return [] - } - const labelEntities = await getRepository(Label) .createQueryBuilder() .where({ - user: { id: user.id }, + user: { id: ctx.uid }, }) .andWhere('LOWER(name) IN (:...names)', { names: labels.map((l) => l.name.toLowerCase()), @@ -148,7 +140,7 @@ export const createLabels = async ( description: l.description, color: l.color || generateRandomColor(), internal: isLabelInternal(l.name), - user, + user: { id: ctx.uid }, })) ) return [...labelEntities, ...newLabelEntities] diff --git a/packages/api/src/services/newsletters.ts b/packages/api/src/services/newsletters.ts index 79ddef432..20345b1f3 100644 --- a/packages/api/src/services/newsletters.ts +++ b/packages/api/src/services/newsletters.ts @@ -1,12 +1,12 @@ -import { NewsletterEmail } from '../entity/newsletter_email' import { nanoid } from 'nanoid' +import { getRepository } from '../entity' +import { NewsletterEmail } from '../entity/newsletter_email' import { User } from '../entity/user' +import { env } from '../env' import { CreateNewsletterEmailErrorCode, SubscriptionStatus, } from '../generated/graphql' -import { env } from '../env' -import { getRepository } from '../entity/utils' import addressparser = require('nodemailer/lib/addressparser') const parsedAddress = (emailAddress: string): string | undefined => { diff --git a/packages/api/src/services/received_emails.ts b/packages/api/src/services/received_emails.ts index a3891bfcc..2995f7346 100644 --- a/packages/api/src/services/received_emails.ts +++ b/packages/api/src/services/received_emails.ts @@ -1,5 +1,5 @@ +import { getRepository } from '../entity' import { ReceivedEmail } from '../entity/received_email' -import { getRepository } from '../entity/utils' export const saveReceivedEmail = async ( from: string, diff --git a/packages/api/src/services/reminders.ts b/packages/api/src/services/reminders.ts index d94e065a4..95168735c 100644 --- a/packages/api/src/services/reminders.ts +++ b/packages/api/src/services/reminders.ts @@ -1,7 +1,7 @@ -import { IsNull, Not } from 'typeorm' +import { EntityManager, IsNull, Not } from 'typeorm' import { getPageById } from '../elastic/pages' +import { getRepository } from '../entity' import { Reminder } from '../entity/reminder' -import { getRepository } from '../entity/utils' import { logger } from '../utils/logger' export interface PageReminder { @@ -52,3 +52,23 @@ export const getPagesWithReminder = async ( return results } + +export const setRemindersComplete = async ( + tx: EntityManager, + userId: string, + remindAt: Date +): Promise => { + const updateResult = await tx + .createQueryBuilder() + .where({ userId, remindAt }) + .update(Reminder) + .set({ status: 'COMPLETED' }) + .returning('*') + .execute() + + if (updateResult.generatedMaps.length === 0) { + return null + } + + return updateResult.generatedMaps[0] as Reminder +} diff --git a/packages/api/src/services/reports.ts b/packages/api/src/services/reports.ts index 1495af024..96590e94e 100644 --- a/packages/api/src/services/reports.ts +++ b/packages/api/src/services/reports.ts @@ -1,7 +1,7 @@ import { getPageById } from '../elastic/pages' +import { getRepository } from '../entity' import { AbuseReport } from '../entity/reports/abuse_report' import { ContentDisplayReport } from '../entity/reports/content_display_report' -import { getRepository } from '../entity/utils' import { ReportItemInput, ReportType } from '../generated/graphql' import { logger } from '../utils/logger' diff --git a/packages/api/src/services/rules.ts b/packages/api/src/services/rules.ts index c4fa6e2a3..94fd3f4eb 100644 --- a/packages/api/src/services/rules.ts +++ b/packages/api/src/services/rules.ts @@ -1,6 +1,6 @@ -import { Rule, RuleAction } from '../entity/rule' -import { getRepository } from '../entity/utils' import { ILike } from 'typeorm' +import { getRepository } from '../entity' +import { Rule, RuleAction } from '../entity/rule' export const createRule = async ( userId: string, diff --git a/packages/api/src/services/save_file.ts b/packages/api/src/services/save_file.ts index 235670839..5fa56b106 100644 --- a/packages/api/src/services/save_file.ts +++ b/packages/api/src/services/save_file.ts @@ -1,7 +1,7 @@ -import { Knex } from 'knex' -import { PubsubClient } from '../datalayer/pubsub' import { UserData } from '../datalayer/user/model' import { updatePage } from '../elastic/pages' +import { getRepository } from '../entity' +import { UploadFile } from '../entity/upload_file' import { homePageURL } from '../env' import { ArticleSavingRequestStatus, @@ -9,33 +9,30 @@ import { SaveFileInput, SaveResult, } from '../generated/graphql' -import { DataModels } from '../resolvers/types' +import { WithDataSourcesContext } from '../resolvers/types' +import { AppDataSource } from '../server' import { logger } from '../utils/logger' import { getStorageFileDetails } from '../utils/uploads' import { createLabels } from './labels' -type SaveContext = { - pubsub: PubsubClient - models: DataModels - authTrx: ( - cb: (tx: Knex.Transaction) => TResult, - userRole?: string - ) => Promise - uid: string +export const setFileUploadComplete = async ( + id: string, + em = AppDataSource.createEntityManager() +): Promise => { + return em.getRepository(UploadFile).save({ id, status: 'COMPLETED' }) } export const saveFile = async ( - ctx: SaveContext, - saver: UserData, + ctx: WithDataSourcesContext, + user: UserData, input: SaveFileInput ): Promise => { logger.info('saving file with input', input) const pageId = input.clientRequestId - const uploadFile = await ctx.models.uploadFile.getWhere({ + const uploadFile = await getRepository(UploadFile).findOneBy({ id: input.uploadFileId, - userId: saver.id, + user: { id: ctx.uid }, }) - if (!uploadFile) { return { errorCodes: [SaveErrorCode.Unauthorized], @@ -45,7 +42,7 @@ export const saveFile = async ( await getStorageFileDetails(input.uploadFileId, uploadFile.fileName) const uploadFileData = await ctx.authTrx(async (tx) => { - return ctx.models.uploadFile.setFileUploadComplete(input.uploadFileId, tx) + return setFileUploadComplete(input.uploadFileId, tx) }) if (!uploadFileData) { @@ -59,7 +56,7 @@ export const saveFile = async ( input.state === ArticleSavingRequestStatus.Archived ? new Date() : null // add labels to page const labels = input.labels - ? await createLabels({ ...ctx, uid: saver.id }, input.labels) + ? await createLabels(ctx, input.labels) : undefined if (input.state || input.labels) { const updated = await updatePage( @@ -80,7 +77,7 @@ export const saveFile = async ( return { clientRequestId: input.clientRequestId, - url: `${homePageURL()}/${saver.profile.username}/links/${ + url: `${homePageURL()}/${user.profile.username}/links/${ input.clientRequestId }`, } diff --git a/packages/api/src/services/save_page.ts b/packages/api/src/services/save_page.ts index 2acaa6755..1bf439803 100644 --- a/packages/api/src/services/save_page.ts +++ b/packages/api/src/services/save_page.ts @@ -1,8 +1,8 @@ import { Readability } from '@omnivore/readability' -import { PubsubClient } from '../datalayer/pubsub' import { addHighlightToPage } from '../elastic/highlights' import { createPage, getPageByParam, updatePage } from '../elastic/pages' import { ArticleSavingRequestStatus, Page, PageType } from '../elastic/types' +import { User } from '../entity/user' import { homePageURL } from '../env' import { HighlightType, @@ -12,7 +12,7 @@ import { SavePageInput, SaveResult, } from '../generated/graphql' -import { DataModels } from '../resolvers/types' +import { WithDataSourcesContext } from '../resolvers/types' import { enqueueThumbnailTask } from '../utils/createTask' import { cleanUrl, @@ -27,18 +27,6 @@ import { parsePreparedContent } from '../utils/parser' import { createPageSaveRequest } from './create_page_save_request' import { createLabels } from './labels' -type SaveContext = { - pubsub: PubsubClient - models: DataModels - uid: string - refresh?: boolean -} - -type SaverUserData = { - userId: string - username: string -} - // where we can use APIs to fetch their underlying content. const FORCE_PUPPETEER_URLS = [ TWEET_URL_REGEX, @@ -65,19 +53,9 @@ const shouldParseInBackend = (input: SavePageInput): boolean => { ) } -export const createSavingRequest = ( - ctx: SaveContext, - clientRequestId: string -) => { - return ctx.models.articleSavingRequest.create({ - userId: ctx.uid, - id: clientRequestId, - }) -} - export const savePage = async ( - ctx: SaveContext, - saver: SaverUserData, + ctx: WithDataSourcesContext, + user: User, input: SavePageInput ): Promise => { const parseResult = await parsePreparedContent( @@ -97,7 +75,7 @@ export const savePage = async ( const articleToSave = parsedContentToPage({ url: input.url, title: input.title, - userId: saver.userId, + userId: user.id, pageId, slug, croppedPathname, @@ -124,7 +102,7 @@ export const savePage = async ( if (shouldParseInBackend(input)) { try { await createPageSaveRequest({ - userId: saver.userId, + userId: user.id, url: articleToSave.url, pubsub: ctx.pubsub, articleSavingRequestId: input.clientRequestId, @@ -140,7 +118,7 @@ export const savePage = async ( } else { // check if the page already exists const existingPage = await getPageByParam({ - userId: saver.userId, + userId: user.id, url: articleToSave.url, }) if (existingPage) { @@ -151,7 +129,7 @@ export const savePage = async ( ) { return { clientRequestId: pageId, - url: `${homePageURL()}/${saver.username}/${slug}`, + url: `${homePageURL()}/${user.profile.username}/${slug}`, } } @@ -195,7 +173,7 @@ export const savePage = async ( if (!isImported) { try { // create a task to update thumbnail and pre-cache all images - const taskId = await enqueueThumbnailTask(saver.userId, slug) + const taskId = await enqueueThumbnailTask(user.id, slug) logger.info('Created thumbnail task', { taskId }) } catch (e) { logger.error('Failed to create thumbnail task', e) @@ -222,7 +200,7 @@ export const savePage = async ( return { clientRequestId: pageId, - url: `${homePageURL()}/${saver.username}/${slug}`, + url: `${homePageURL()}/${user.profile.username}/${slug}`, } } diff --git a/packages/api/src/services/save_url.ts b/packages/api/src/services/save_url.ts index f42f34115..7c9267867 100644 --- a/packages/api/src/services/save_url.ts +++ b/packages/api/src/services/save_url.ts @@ -1,7 +1,7 @@ import { PubsubClient } from '../datalayer/pubsub' import { ArticleSavingRequestStatus } from '../elastic/types' +import { getRepository } from '../entity' import { User } from '../entity/user' -import { getRepository } from '../entity/utils' import { homePageURL } from '../env' import { SaveErrorCode, SaveResult, SaveUrlInput } from '../generated/graphql' import { logger } from '../utils/logger' @@ -24,7 +24,7 @@ export const saveUrl = async ( input.state === ArticleSavingRequestStatus.Archived ? new Date() : null // add labels to page const labels = input.labels - ? await createLabels({ ...ctx, uid: ctx.uid }, input.labels) + ? await createLabels(ctx, input.labels) : undefined const pageSaveRequest = await createPageSaveRequest({ diff --git a/packages/api/src/services/search_history.ts b/packages/api/src/services/search_history.ts index 59da78836..582048c3c 100644 --- a/packages/api/src/services/search_history.ts +++ b/packages/api/src/services/search_history.ts @@ -1,5 +1,5 @@ +import { getRepository } from '../entity' import { SearchHistory } from '../entity/search_history' -import { getRepository } from '../entity/utils' export const getRecentSearches = async ( userId: string diff --git a/packages/api/src/services/subscriptions.ts b/packages/api/src/services/subscriptions.ts index c950dfee7..ecc0db916 100644 --- a/packages/api/src/services/subscriptions.ts +++ b/packages/api/src/services/subscriptions.ts @@ -1,7 +1,7 @@ import axios from 'axios' +import { getRepository } from '../entity' import { NewsletterEmail } from '../entity/newsletter_email' import { Subscription } from '../entity/subscription' -import { getRepository } from '../entity/utils' import { SubscriptionStatus, SubscriptionType } from '../generated/graphql' import { logger } from '../utils/logger' import { sendEmail } from '../utils/sendEmail' diff --git a/packages/api/src/services/user_device_tokens.ts b/packages/api/src/services/user_device_tokens.ts index 4cc740a4b..f050a5388 100644 --- a/packages/api/src/services/user_device_tokens.ts +++ b/packages/api/src/services/user_device_tokens.ts @@ -1,10 +1,10 @@ -import { UserDeviceToken } from '../entity/user_device_tokens' +import { getRepository, setClaims } from '../entity' import { User } from '../entity/user' -import { SetDeviceTokenErrorCode } from '../generated/graphql' -import { getRepository, setClaims } from '../entity/utils' -import { analytics } from '../utils/analytics' +import { UserDeviceToken } from '../entity/user_device_tokens' import { env } from '../env' +import { SetDeviceTokenErrorCode } from '../generated/graphql' import { AppDataSource } from '../server' +import { analytics } from '../utils/analytics' export const getDeviceToken = async ( id: string diff --git a/packages/api/src/utils/auth.ts b/packages/api/src/utils/auth.ts index 951bf6300..1f410a4a4 100644 --- a/packages/api/src/utils/auth.ts +++ b/packages/api/src/utils/auth.ts @@ -4,8 +4,8 @@ import express from 'express' import * as jwt from 'jsonwebtoken' import { promisify } from 'util' import { v4 as uuidv4 } from 'uuid' +import { getRepository } from '../entity' import { ApiKey } from '../entity/api_key' -import { getRepository } from '../entity/utils' import { env } from '../env' import { Claims, ClaimsToSet } from '../resolvers/types' import { logger } from './logger' diff --git a/packages/api/src/utils/helpers.ts b/packages/api/src/utils/helpers.ts index 74607ab33..ea72d7ffc 100644 --- a/packages/api/src/utils/helpers.ts +++ b/packages/api/src/utils/helpers.ts @@ -5,10 +5,9 @@ import path from 'path' import _ from 'underscore' import slugify from 'voca/slugify' import wordsCounter from 'word-counting' -import { RegistrationType, UserData } from '../datalayer/user/model' import { updatePage } from '../elastic/pages' import { ArticleSavingRequestStatus, Page } from '../elastic/types' -import { User } from '../entity/user' +import { RegistrationType, User } from '../entity/user' import { ArticleSavingRequest, CreateArticleError, @@ -116,7 +115,7 @@ export const findDelimiter = ( // FIXME: Remove this Date stub after nullable types will be fixed export const userDataToUser = ( user: Merge< - UserData, + User, { isFriend?: boolean followersCount?: number @@ -156,15 +155,11 @@ export const userDataToUser = ( followersCount: user.followersCount || 0, isFullUser: true, viewerIsFollowing: user.viewerIsFollowing || user.isFriend || false, - picture: user.profile.picture_url, + picture: user.profile.pictureUrl, sharedArticles: [], sharedArticlesCount: user.sharedArticlesCount || 0, sharedHighlightsCount: user.sharedHighlightsCount || 0, sharedNotesCount: user.sharedNotesCount || 0, - profile: { - ...user.profile, - pictureUrl: user.profile.picture_url, - }, }) export const generateSlug = (title: string): string => { diff --git a/packages/api/src/utils/parser.ts b/packages/api/src/utils/parser.ts index 329bd977a..b280ed784 100644 --- a/packages/api/src/utils/parser.ts +++ b/packages/api/src/utils/parser.ts @@ -16,8 +16,8 @@ import { ILike } from 'typeorm' import { promisify } from 'util' import { v4 as uuid } from 'uuid' import { Highlight } from '../elastic/types' +import { getRepository } from '../entity' import { User } from '../entity/user' -import { getRepository } from '../entity/utils' import { env } from '../env' import { PageType, PreparedDocumentInput } from '../generated/graphql' import { ArticleFormat } from '../resolvers/article' diff --git a/packages/api/test/db.ts b/packages/api/test/db.ts index 6faf1803e..22da1a403 100644 --- a/packages/api/test/db.ts +++ b/packages/api/test/db.ts @@ -1,6 +1,7 @@ import Postgrator from 'postgrator' import { FindOptionsWhere } from 'typeorm' import { SnakeNamingStrategy } from 'typeorm-naming-strategies' +import { getRepository, setClaims } from '../src/entity' import { Integration } from '../src/entity/integration' import { Label } from '../src/entity/label' import { Link } from '../src/entity/link' @@ -11,7 +12,6 @@ import { Reminder } from '../src/entity/reminder' import { Subscription } from '../src/entity/subscription' import { User } from '../src/entity/user' import { UserDeviceToken } from '../src/entity/user_device_tokens' -import { getRepository, setClaims } from '../src/entity/utils' import { SubscriptionStatus, SubscriptionType } from '../src/generated/graphql' import { AppDataSource } from '../src/server' import { createUser } from '../src/services/create_user' diff --git a/packages/api/test/resolvers/api_key.test.ts b/packages/api/test/resolvers/api_key.test.ts index e1f85f3d8..2bdc840fb 100644 --- a/packages/api/test/resolvers/api_key.test.ts +++ b/packages/api/test/resolvers/api_key.test.ts @@ -3,7 +3,7 @@ import { createTestUser, deleteTestUser } from '../db' import { graphqlRequest, request } from '../util' import { expect } from 'chai' import supertest from 'supertest' -import { getRepository } from '../../src/entity/utils' +import { getRepository } from '../../src/entity' import { ApiKey } from '../../src/entity/api_key' const testAPIKey = (apiKey: string): supertest.Test => { diff --git a/packages/api/test/resolvers/article.test.ts b/packages/api/test/resolvers/article.test.ts index 10a94e06a..43678497e 100644 --- a/packages/api/test/resolvers/article.test.ts +++ b/packages/api/test/resolvers/article.test.ts @@ -22,9 +22,9 @@ import { PageContext, PageType, } from '../../src/elastic/types' +import { getRepository } from '../../src/entity' import { UploadFile } from '../../src/entity/upload_file' import { User } from '../../src/entity/user' -import { getRepository } from '../../src/entity/utils' import { BulkActionType, SyncUpdatedItemEdge, diff --git a/packages/api/test/resolvers/features.test.ts b/packages/api/test/resolvers/features.test.ts index d4781612c..2988cf90c 100644 --- a/packages/api/test/resolvers/features.test.ts +++ b/packages/api/test/resolvers/features.test.ts @@ -2,9 +2,9 @@ import { expect } from 'chai' import * as jwt from 'jsonwebtoken' import 'mocha' import sinon, { SinonFakeTimers } from 'sinon' +import { getRepository } from '../../src/entity' import { Feature } from '../../src/entity/feature' import { User } from '../../src/entity/user' -import { getRepository } from '../../src/entity/utils' import { env } from '../../src/env' import { createTestUser, deleteTestUser } from '../db' import { graphqlRequest, request } from '../util' diff --git a/packages/api/test/resolvers/integrations.test.ts b/packages/api/test/resolvers/integrations.test.ts index 79c130294..a802a7092 100644 --- a/packages/api/test/resolvers/integrations.test.ts +++ b/packages/api/test/resolvers/integrations.test.ts @@ -4,7 +4,7 @@ import { createTestUser, deleteTestIntegrations, deleteTestUser } from '../db' import { generateFakeUuid, graphqlRequest, request } from '../util' import { SetIntegrationErrorCode } from '../../src/generated/graphql' import chai, { expect } from 'chai' -import { getRepository } from '../../src/entity/utils' +import { getRepository } from '../../src/entity' import { Integration } from '../../src/entity/integration' import nock from 'nock' import { READWISE_API_URL } from '../../src/services/integrations/readwise' diff --git a/packages/api/test/resolvers/labels.test.ts b/packages/api/test/resolvers/labels.test.ts index 4717c820e..9868d6036 100644 --- a/packages/api/test/resolvers/labels.test.ts +++ b/packages/api/test/resolvers/labels.test.ts @@ -20,7 +20,7 @@ import { Page, PageContext, } from '../../src/elastic/types' -import { getRepository } from '../../src/entity/utils' +import { getRepository } from '../../src/entity' import { deletePage, getPageById } from '../../src/elastic/pages' import { createPubSubClient } from '../../src/datalayer/pubsub' import { diff --git a/packages/api/test/resolvers/newsletters.test.ts b/packages/api/test/resolvers/newsletters.test.ts index 14eee11b7..54690b8f0 100644 --- a/packages/api/test/resolvers/newsletters.test.ts +++ b/packages/api/test/resolvers/newsletters.test.ts @@ -14,7 +14,7 @@ import { SubscriptionStatus, } from '../../src/generated/graphql' import 'mocha' -import { getRepository } from '../../src/entity/utils' +import { getRepository } from '../../src/entity' describe('Newsletters API', () => { let user: User diff --git a/packages/api/test/resolvers/recent_emails.test.ts b/packages/api/test/resolvers/recent_emails.test.ts index 7d87414be..28c96d2d3 100644 --- a/packages/api/test/resolvers/recent_emails.test.ts +++ b/packages/api/test/resolvers/recent_emails.test.ts @@ -3,7 +3,7 @@ import { expect } from 'chai' import { User } from '../../src/entity/user' import { createTestUser, deleteTestUser } from '../db' import { graphqlRequest, request } from '../util' -import { getRepository } from '../../src/entity/utils' +import { getRepository } from '../../src/entity' import { ReceivedEmail } from '../../src/entity/received_email' import { NewsletterEmail } from '../../src/entity/newsletter_email' import sinon from 'sinon' diff --git a/packages/api/test/resolvers/recent_searches.test.ts b/packages/api/test/resolvers/recent_searches.test.ts index 8598a2c6e..3217541a0 100644 --- a/packages/api/test/resolvers/recent_searches.test.ts +++ b/packages/api/test/resolvers/recent_searches.test.ts @@ -5,7 +5,7 @@ import { PageContext } from '../../src/elastic/types' import { createTestUser, deleteTestUser } from '../db' import { graphqlRequest, request } from '../util' import { createPubSubClient } from '../../src/datalayer/pubsub' -import { getRepository } from '../../src/entity/utils' +import { getRepository } from '../../src/entity' import { SearchHistory } from '../../src/entity/search_history' describe('recent_searches resolver', () => { diff --git a/packages/api/test/resolvers/report.test.ts b/packages/api/test/resolvers/report.test.ts index f67b95ae6..d39c67c3f 100644 --- a/packages/api/test/resolvers/report.test.ts +++ b/packages/api/test/resolvers/report.test.ts @@ -5,7 +5,7 @@ import { createTestElasticPage, graphqlRequest, request } from '../util' import { ReportType } from '../../src/generated/graphql' import { ContentDisplayReport } from '../../src/entity/reports/content_display_report' import { expect } from 'chai' -import { getRepository } from '../../src/entity/utils' +import { getRepository } from '../../src/entity' describe('Report API', () => { let user: User diff --git a/packages/api/test/resolvers/rules.test.ts b/packages/api/test/resolvers/rules.test.ts index 5a55195bb..dc6bbcf9b 100644 --- a/packages/api/test/resolvers/rules.test.ts +++ b/packages/api/test/resolvers/rules.test.ts @@ -3,7 +3,7 @@ import { expect } from 'chai' import { graphqlRequest, request } from '../util' import { User } from '../../src/entity/user' import { createTestUser, deleteTestUser } from '../db' -import { getRepository } from '../../src/entity/utils' +import { getRepository } from '../../src/entity' import { Rule, RuleAction, RuleActionType } from '../../src/entity/rule' describe('Rules Resolver', () => { diff --git a/packages/api/test/resolvers/subscriptions.test.ts b/packages/api/test/resolvers/subscriptions.test.ts index 5ef6ff8b2..3ddb24bed 100644 --- a/packages/api/test/resolvers/subscriptions.test.ts +++ b/packages/api/test/resolvers/subscriptions.test.ts @@ -5,7 +5,7 @@ import sinonChai from 'sinon-chai' import { NewsletterEmail } from '../../src/entity/newsletter_email' import { Subscription } from '../../src/entity/subscription' import { User } from '../../src/entity/user' -import { getRepository } from '../../src/entity/utils' +import { getRepository } from '../../src/entity' import { SubscriptionStatus, SubscriptionType, diff --git a/packages/api/test/resolvers/user_device_tokens.test.ts b/packages/api/test/resolvers/user_device_tokens.test.ts index 869969a8c..11e9730dd 100644 --- a/packages/api/test/resolvers/user_device_tokens.test.ts +++ b/packages/api/test/resolvers/user_device_tokens.test.ts @@ -11,7 +11,7 @@ import { UserDeviceToken } from '../../src/entity/user_device_tokens' import { SetDeviceTokenErrorCode } from '../../src/generated/graphql' import 'mocha' import { User } from '../../src/entity/user' -import { getRepository } from '../../src/entity/utils' +import { getRepository } from '../../src/entity' describe('Device tokens API', () => { let authToken: string diff --git a/packages/api/test/resolvers/user_feed_article.test.ts b/packages/api/test/resolvers/user_feed_article.test.ts index 192a6c9c4..84fafcf4e 100644 --- a/packages/api/test/resolvers/user_feed_article.test.ts +++ b/packages/api/test/resolvers/user_feed_article.test.ts @@ -11,7 +11,7 @@ import { Page } from '../../src/entity/page' import { Link } from '../../src/entity/link' import { Highlight } from '../../src/entity/highlight' import 'mocha' -import { getRepository } from '../../src/entity/utils' +import { getRepository } from '../../src/entity' import { User } from '../../src/entity/user' describe('User feed article API', () => { diff --git a/packages/api/test/resolvers/webhooks.test.ts b/packages/api/test/resolvers/webhooks.test.ts index a22ca3d8d..d07aa544b 100644 --- a/packages/api/test/resolvers/webhooks.test.ts +++ b/packages/api/test/resolvers/webhooks.test.ts @@ -5,7 +5,7 @@ import 'mocha' import { User } from '../../src/entity/user' import { WebhookEvent } from '../../src/generated/graphql' import { Webhook } from '../../src/entity/webhook' -import { getRepository } from '../../src/entity/utils' +import { getRepository } from '../../src/entity' describe('Webhooks API', () => { let user: User diff --git a/packages/api/test/routers/auth.test.ts b/packages/api/test/routers/auth.test.ts index 48dcd8df7..a4d2f3b5e 100644 --- a/packages/api/test/routers/auth.test.ts +++ b/packages/api/test/routers/auth.test.ts @@ -5,8 +5,8 @@ import sinonChai from 'sinon-chai' import supertest from 'supertest' import { StatusType } from '../../src/datalayer/user/model' import { searchPages } from '../../src/elastic/pages' +import { getRepository } from '../../src/entity' import { User } from '../../src/entity/user' -import { getRepository } from '../../src/entity/utils' import { AuthProvider } from '../../src/routers/auth/auth_types' import { createPendingUserToken } from '../../src/routers/auth/jwt_helpers' import { diff --git a/packages/api/test/routers/emails.test.ts b/packages/api/test/routers/emails.test.ts index a625fe800..0757ed759 100644 --- a/packages/api/test/routers/emails.test.ts +++ b/packages/api/test/routers/emails.test.ts @@ -11,7 +11,7 @@ import { request } from '../util' import * as parser from '../../src/utils/parser' import * as sendNotification from '../../src/utils/sendNotification' import * as sendEmail from '../../src/utils/sendEmail' -import { getRepository } from '../../src/entity/utils' +import { getRepository } from '../../src/entity' import { ReceivedEmail } from '../../src/entity/received_email' import * as jwt from 'jsonwebtoken' diff --git a/packages/api/test/routers/integrations.test.ts b/packages/api/test/routers/integrations.test.ts index f05a7e669..f8a1b1e82 100644 --- a/packages/api/test/routers/integrations.test.ts +++ b/packages/api/test/routers/integrations.test.ts @@ -16,9 +16,9 @@ import { Page, PageContext, } from '../../src/elastic/types' +import { getRepository } from '../../src/entity' import { Integration, IntegrationType } from '../../src/entity/integration' import { User } from '../../src/entity/user' -import { getRepository } from '../../src/entity/utils' import { env } from '../../src/env' import { getHighlightUrl } from '../../src/services/highlights' import { READWISE_API_URL } from '../../src/services/integrations/readwise' diff --git a/packages/api/test/routers/webhooks.test.ts b/packages/api/test/routers/webhooks.test.ts index 20e76606f..0dc3bbc56 100644 --- a/packages/api/test/routers/webhooks.test.ts +++ b/packages/api/test/routers/webhooks.test.ts @@ -2,7 +2,7 @@ import { createTestUser, deleteTestUser } from '../db' import { request } from '../util' import { User } from '../../src/entity/user' import 'mocha' -import { getRepository } from '../../src/entity/utils' +import { getRepository } from '../../src/entity' import { Webhook } from '../../src/entity/webhook' import { expect } from 'chai' import nock from 'nock' diff --git a/packages/api/test/services/create_content_display_report.test.ts b/packages/api/test/services/create_content_display_report.test.ts index 6bf30d488..e9998a6f3 100644 --- a/packages/api/test/services/create_content_display_report.test.ts +++ b/packages/api/test/services/create_content_display_report.test.ts @@ -2,11 +2,8 @@ import 'mocha' import chai, { expect } from 'chai' import { createTestUser, deleteTestUser } from '../db' import sinonChai from 'sinon-chai' -import sinon from 'sinon' -import * as util from '../../src/utils/sendEmail' -import { MailDataRequired } from '@sendgrid/helpers/classes/mail' import { User } from '../../src/entity/user' -import { getRepository } from '../../src/entity/utils' +import { getRepository } from '../../src/entity' import { ContentDisplayReport } from '../../src/entity/reports/content_display_report' import { saveContentDisplayReport } from '../../src/services/reports' import { ReportType } from '../../src/generated/graphql' diff --git a/packages/api/test/services/create_user.test.ts b/packages/api/test/services/create_user.test.ts index 42cde34ce..514b3ab90 100644 --- a/packages/api/test/services/create_user.test.ts +++ b/packages/api/test/services/create_user.test.ts @@ -11,7 +11,6 @@ import { getUserFollowers, getUserFollowing, } from '../../src/services/followers' -import { StatusType } from '../../src/datalayer/user/model' import sinonChai from 'sinon-chai' import sinon from 'sinon' import * as util from '../../src/utils/sendEmail' @@ -19,6 +18,7 @@ import { MailDataRequired } from '@sendgrid/helpers/classes/mail' import { User } from '../../src/entity/user' import { getRepository } from '../../src/entity/utils' import { Filter } from "../../src/entity/filter" +import { getRepository } from '../../src/entity' chai.use(sinonChai) diff --git a/packages/api/test/services/labels.test.ts b/packages/api/test/services/labels.test.ts index 89d9e92c6..436ab07bd 100644 --- a/packages/api/test/services/labels.test.ts +++ b/packages/api/test/services/labels.test.ts @@ -11,7 +11,7 @@ import { LinkLabel } from '../../src/entity/link_label' import { Label } from '../../src/entity/label' import { Link } from '../../src/entity/link' import { labelsLoader } from '../../src/services/labels' -import { getRepository } from '../../src/entity/utils' +import { getRepository } from '../../src/entity' import { User } from '../../src/entity/user' describe('batch get labels from linkIds', () => { diff --git a/packages/api/test/services/save_newsletter_email.test.ts b/packages/api/test/services/save_newsletter_email.test.ts index 1e369d289..6b9fcd8ff 100644 --- a/packages/api/test/services/save_newsletter_email.test.ts +++ b/packages/api/test/services/save_newsletter_email.test.ts @@ -3,11 +3,11 @@ import 'mocha' import nock from 'nock' import { createPubSubClient } from '../../src/datalayer/pubsub' import { getPageByParam } from '../../src/elastic/pages' +import { getRepository } from '../../src/entity' import { NewsletterEmail } from '../../src/entity/newsletter_email' import { ReceivedEmail } from '../../src/entity/received_email' import { Subscription } from '../../src/entity/subscription' import { User } from '../../src/entity/user' -import { getRepository } from '../../src/entity/utils' import { createNewsletterEmail } from '../../src/services/newsletters' import { SaveContext } from '../../src/services/save_email' import { saveNewsletterEmail } from '../../src/services/save_newsletter_email' From 937095189349c223caf029f108c0e16509f3d1fe Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Wed, 23 Aug 2023 12:15:50 +0800 Subject: [PATCH 009/144] create a repository directory --- packages/api/src/apollo.ts | 2 +- packages/api/src/data-source.ts | 22 ++++++++++ packages/api/src/elastic/types.ts | 2 +- .../api/src/entity/library_item_preview.ts | 2 +- .../api/src/{entity => repository}/index.ts | 8 ++-- packages/api/src/resolvers/api_key/index.ts | 9 ++-- packages/api/src/resolvers/article/index.ts | 2 +- .../resolvers/article_saving_request/index.ts | 2 +- packages/api/src/resolvers/filters/index.ts | 4 +- .../api/src/resolvers/function_resolvers.ts | 26 +++++------ .../importers/uploadImportFileResolver.ts | 2 +- packages/api/src/resolvers/index.ts | 8 ++-- .../api/src/resolvers/integrations/index.ts | 2 +- packages/api/src/resolvers/labels/index.ts | 4 +- .../api/src/resolvers/newsletters/index.ts | 2 +- .../api/src/resolvers/popular_reads/index.ts | 21 ++++----- .../api/src/resolvers/recent_emails/index.ts | 2 +- .../src/resolvers/recent_searches/index.ts | 2 +- .../src/resolvers/recommendations/index.ts | 2 +- packages/api/src/resolvers/rules/index.ts | 2 +- packages/api/src/resolvers/save/index.ts | 2 +- .../send_install_instructions/index.ts | 2 +- .../api/src/resolvers/subscriptions/index.ts | 4 +- packages/api/src/resolvers/update/index.ts | 21 ++++----- .../api/src/resolvers/upload_files/index.ts | 2 +- packages/api/src/resolvers/user/index.ts | 44 ++++++++++--------- .../resolvers/user_personalization/index.ts | 4 +- packages/api/src/resolvers/webhooks/index.ts | 2 +- packages/api/src/routers/article_router.ts | 2 +- packages/api/src/routers/auth/auth_router.ts | 4 +- packages/api/src/routers/page_router.ts | 2 +- packages/api/src/routers/svc/content.ts | 4 +- .../api/src/routers/svc/email_attachment.ts | 4 +- packages/api/src/routers/svc/integrations.ts | 2 +- packages/api/src/routers/svc/reminders.ts | 4 +- packages/api/src/routers/svc/rss_feed.ts | 2 +- packages/api/src/routers/svc/webhooks.ts | 2 +- packages/api/src/routers/text_to_speech.ts | 4 +- packages/api/src/routers/user_router.ts | 2 +- packages/api/src/server.ts | 27 +----------- packages/api/src/services/archive_link.ts | 4 +- .../src/services/create_page_save_request.ts | 2 +- packages/api/src/services/create_user.ts | 4 +- packages/api/src/services/features.ts | 4 +- packages/api/src/services/followers.ts | 2 +- packages/api/src/services/groups.ts | 4 +- .../api/src/services/integrations/readwise.ts | 2 +- packages/api/src/services/labels.ts | 2 +- packages/api/src/services/newsletters.ts | 2 +- packages/api/src/services/received_emails.ts | 2 +- packages/api/src/services/reminders.ts | 2 +- packages/api/src/services/reports.ts | 2 +- packages/api/src/services/rules.ts | 2 +- packages/api/src/services/save_file.ts | 4 +- packages/api/src/services/save_url.ts | 2 +- packages/api/src/services/search_history.ts | 2 +- packages/api/src/services/subscriptions.ts | 2 +- .../api/src/services/user_device_tokens.ts | 4 +- packages/api/src/utils/auth.ts | 2 +- packages/api/src/utils/helpers.ts | 1 - packages/api/src/utils/parser.ts | 2 +- packages/api/test/db.ts | 6 +-- packages/api/test/global-teardown.ts | 2 +- packages/api/test/resolvers/api_key.test.ts | 8 ++-- packages/api/test/resolvers/article.test.ts | 2 +- packages/api/test/resolvers/features.test.ts | 2 +- .../api/test/resolvers/integrations.test.ts | 14 +++--- packages/api/test/resolvers/labels.test.ts | 36 +++++++-------- .../api/test/resolvers/newsletters.test.ts | 18 ++++---- .../api/test/resolvers/recent_emails.test.ts | 12 ++--- .../test/resolvers/recent_searches.test.ts | 10 ++--- packages/api/test/resolvers/reminders.test.ts | 2 +- packages/api/test/resolvers/report.test.ts | 10 ++--- packages/api/test/resolvers/rules.test.ts | 10 ++--- .../api/test/resolvers/subscriptions.test.ts | 4 +- .../test/resolvers/user_device_tokens.test.ts | 12 ++--- .../test/resolvers/user_feed_article.test.ts | 18 ++++---- packages/api/test/resolvers/webhooks.test.ts | 8 ++-- packages/api/test/routers/auth.test.ts | 2 +- packages/api/test/routers/emails.test.ts | 20 ++++----- .../api/test/routers/integrations.test.ts | 2 +- packages/api/test/routers/reminders.test.ts | 2 +- packages/api/test/routers/webhooks.test.ts | 12 ++--- .../create_content_display_report.test.ts | 16 +++---- .../api/test/services/create_user.test.ts | 36 +++++++-------- packages/api/test/services/labels.test.ts | 14 +++--- .../services/save_newsletter_email.test.ts | 2 +- 87 files changed, 298 insertions(+), 301 deletions(-) create mode 100644 packages/api/src/data-source.ts rename packages/api/src/{entity => repository}/index.ts (76%) diff --git a/packages/api/src/apollo.ts b/packages/api/src/apollo.ts index ea6f75d7f..b4f8c1977 100644 --- a/packages/api/src/apollo.ts +++ b/packages/api/src/apollo.ts @@ -19,7 +19,7 @@ import { functionResolvers } from './resolvers/function_resolvers' import { ClaimsToSet, ResolverContext } from './resolvers/types' import ScalarResolvers from './scalars' import typeDefs from './schema' -import { AppDataSource } from './server' +import { AppDataSource } from './data-source' import { tracer } from './tracing' import { getClaimsByToken, setAuthInCookie } from './utils/auth' import { SetClaimsRole } from './utils/dictionary' diff --git a/packages/api/src/data-source.ts b/packages/api/src/data-source.ts new file mode 100644 index 000000000..88425c26f --- /dev/null +++ b/packages/api/src/data-source.ts @@ -0,0 +1,22 @@ +import { DataSource } from 'typeorm' +import { SnakeNamingStrategy } from 'typeorm-naming-strategies' +import { env } from './env' +import { CustomTypeOrmLogger } from './utils/logger' + +export const AppDataSource = new DataSource({ + type: 'postgres', + host: env.pg.host, + port: env.pg.port, + schema: 'omnivore', + username: env.pg.userName, + password: env.pg.password, + database: env.pg.dbName, + logging: ['query', 'info'], + entities: [__dirname + '/entity/**/*{.js,.ts}'], + subscribers: [__dirname + '/events/**/*{.js,.ts}'], + namingStrategy: new SnakeNamingStrategy(), + logger: new CustomTypeOrmLogger(), + // cache: true, + connectTimeoutMS: 60000, // 60 seconds + maxQueryExecutionTime: 60000, // 60 seconds +}) diff --git a/packages/api/src/elastic/types.ts b/packages/api/src/elastic/types.ts index 7ff19154a..6613775f3 100644 --- a/packages/api/src/elastic/types.ts +++ b/packages/api/src/elastic/types.ts @@ -114,7 +114,7 @@ export interface RecommendingUser { userId: string name: string username: string - profileImageURL: string | null + profileImageURL?: string | null } export interface Recommendation { diff --git a/packages/api/src/entity/library_item_preview.ts b/packages/api/src/entity/library_item_preview.ts index cf2a9b7dd..21e97d35a 100644 --- a/packages/api/src/entity/library_item_preview.ts +++ b/packages/api/src/entity/library_item_preview.ts @@ -19,7 +19,7 @@ export class LibraryItemPreview { @JoinColumn({ name: 'sender_id' }) sender!: User - @Column('array', { name: 'recipient_ids' }) + @Column('text', { name: 'recipient_ids', array: true }) recipientIds!: string[] @OneToOne(() => LibraryItem, { onDelete: 'CASCADE' }) diff --git a/packages/api/src/entity/index.ts b/packages/api/src/repository/index.ts similarity index 76% rename from packages/api/src/entity/index.ts rename to packages/api/src/repository/index.ts index 0c87ae2d2..d8b7f53d8 100644 --- a/packages/api/src/entity/index.ts +++ b/packages/api/src/repository/index.ts @@ -1,8 +1,8 @@ import { EntityManager, EntityTarget, Repository } from 'typeorm' -import { AppDataSource } from '../server' -import { Reminder } from './reminder' -import { UploadFile } from './upload_file' -import { User } from './user' +import { AppDataSource } from '../data-source' +import { Reminder } from '../entity/reminder' +import { UploadFile } from '../entity/upload_file' +import { User } from '../entity/user' export const setClaims = async ( t: EntityManager, diff --git a/packages/api/src/resolvers/api_key/index.ts b/packages/api/src/resolvers/api_key/index.ts index 8b5cc6a5b..5e9769386 100644 --- a/packages/api/src/resolvers/api_key/index.ts +++ b/packages/api/src/resolvers/api_key/index.ts @@ -1,6 +1,4 @@ -import { getRepository } from '../../entity' import { ApiKey } from '../../entity/api_key' -import { User } from '../../entity/user' import { env } from '../../env' import { ApiKeysError, @@ -15,6 +13,7 @@ import { RevokeApiKeyErrorCode, RevokeApiKeySuccess, } from '../../generated/graphql' +import { getRepository, userRepository } from '../../repository' import { analytics } from '../../utils/analytics' import { generateApiKey, hashApiKey } from '../../utils/auth' import { authorized } from '../../utils/helpers' @@ -24,7 +23,7 @@ export const apiKeysResolver = authorized( log.info('apiKeysResolver') try { - const user = await getRepository(User).findOneBy({ id: uid }) + const user = await userRepository.findOneBy({ id: uid }) if (!user) { return { errorCodes: [ApiKeysErrorCode.Unauthorized], @@ -60,7 +59,7 @@ export const generateApiKeyResolver = authorized< >(async (_, { input: { name, expiresAt } }, { claims: { uid }, log }) => { try { log.info('generateApiKeyResolver') - const user = await getRepository(User).findOneBy({ id: uid }) + const user = await userRepository.findOneBy({ id: uid }) if (!user) { return { errorCodes: [GenerateApiKeyErrorCode.Unauthorized], @@ -117,7 +116,7 @@ export const revokeApiKeyResolver = authorized< log.info('RevokeApiKeyResolver') try { - const user = await getRepository(User).findOneBy({ id: uid }) + const user = await userRepository.findOneBy({ id: uid }) if (!user) { return { errorCodes: [RevokeApiKeyErrorCode.Unauthorized], diff --git a/packages/api/src/resolvers/article/index.ts b/packages/api/src/resolvers/article/index.ts index d50532d44..054fe1b56 100644 --- a/packages/api/src/resolvers/article/index.ts +++ b/packages/api/src/resolvers/article/index.ts @@ -20,7 +20,7 @@ import { PageType, SearchItem as SearchItemData, } from '../../elastic/types' -import { getRepository } from '../../entity' +import { getRepository } from '../../repository' import { UploadFile } from '../../entity/upload_file' import { User } from '../../entity/user' import { env } from '../../env' diff --git a/packages/api/src/resolvers/article_saving_request/index.ts b/packages/api/src/resolvers/article_saving_request/index.ts index ee935665b..99ba41be0 100644 --- a/packages/api/src/resolvers/article_saving_request/index.ts +++ b/packages/api/src/resolvers/article_saving_request/index.ts @@ -1,6 +1,5 @@ /* eslint-disable prefer-const */ import { getPageByParam } from '../../elastic/pages' -import { getRepository } from '../../entity' import { User } from '../../entity/user' import { env } from '../../env' import { @@ -14,6 +13,7 @@ import { MutationCreateArticleSavingRequestArgs, QueryArticleSavingRequestArgs, } from '../../generated/graphql' +import { getRepository } from '../../repository' import { createPageSaveRequest } from '../../services/create_page_save_request' import { analytics } from '../../utils/analytics' import { diff --git a/packages/api/src/resolvers/filters/index.ts b/packages/api/src/resolvers/filters/index.ts index 483b86902..3c413d8d4 100644 --- a/packages/api/src/resolvers/filters/index.ts +++ b/packages/api/src/resolvers/filters/index.ts @@ -1,5 +1,5 @@ import { Between } from 'typeorm' -import { getRepository, setClaims } from '../../entity' +import { AppDataSource } from '../../data-source' import { Filter } from '../../entity/filter' import { User } from '../../entity/user' import { env } from '../../env' @@ -24,7 +24,7 @@ import { UpdateFilterSuccess, UpdateFilterErrorCode, } from '../../generated/graphql' -import { AppDataSource } from '../../server' +import { getRepository, setClaims } from '../../repository' import { analytics } from '../../utils/analytics' import { env } from '../../env' import { isNil, mergeWith } from 'lodash' diff --git a/packages/api/src/resolvers/function_resolvers.ts b/packages/api/src/resolvers/function_resolvers.ts index e635ba08a..5b2730766 100644 --- a/packages/api/src/resolvers/function_resolvers.ts +++ b/packages/api/src/resolvers/function_resolvers.ts @@ -4,7 +4,7 @@ /* eslint-disable @typescript-eslint/no-unsafe-member-access */ /* eslint-disable @typescript-eslint/explicit-module-boundary-types */ import { getPageByParam } from '../elastic/pages' -import { getRepository } from '../entity' +import { getRepository } from '../repository' import { Subscription } from '../entity/subscription' import { UploadFile } from '../entity/upload_file' import { User } from '../entity/user' @@ -95,7 +95,7 @@ import { setLinkArchivedResolver, setRuleResolver, // setShareArticleResolver, - setShareHighlightResolver, + // setShareHighlightResolver, setUserPersonalizationResolver, setWebhookResolver, subscribeResolver, @@ -163,7 +163,7 @@ export const functionResolvers = { setBookmarkArticle: setBookmarkArticleResolver, setUserPersonalization: setUserPersonalizationResolver, createArticleSavingRequest: createArticleSavingRequestResolver, - setShareHighlight: setShareHighlightResolver, + // setShareHighlight: setShareHighlightResolver, reportItem: reportItemResolver, // updateLinkShareInfo: updateLinkShareInfoResolver, setLinkArchived: setLinkArchivedResolver, @@ -602,31 +602,31 @@ export const functionResolvers = { ...resultResolveTypeResolver('UpdateUser'), ...resultResolveTypeResolver('UpdateUserProfile'), ...resultResolveTypeResolver('Article'), - ...resultResolveTypeResolver('SharedArticle'), + // ...resultResolveTypeResolver('SharedArticle'), ...resultResolveTypeResolver('Articles'), ...resultResolveTypeResolver('User'), ...resultResolveTypeResolver('Users'), ...resultResolveTypeResolver('SaveArticleReadingProgress'), - ...resultResolveTypeResolver('FeedArticles'), + // ...resultResolveTypeResolver('FeedArticles'), ...resultResolveTypeResolver('CreateArticle'), ...resultResolveTypeResolver('CreateHighlight'), - ...resultResolveTypeResolver('CreateReaction'), - ...resultResolveTypeResolver('DeleteReaction'), + // ...resultResolveTypeResolver('CreateReaction'), + // ...resultResolveTypeResolver('DeleteReaction'), ...resultResolveTypeResolver('MergeHighlight'), ...resultResolveTypeResolver('UpdateHighlight'), ...resultResolveTypeResolver('DeleteHighlight'), ...resultResolveTypeResolver('UploadFileRequest'), - ...resultResolveTypeResolver('SetShareArticle'), - ...resultResolveTypeResolver('UpdateSharedComment'), + // ...resultResolveTypeResolver('SetShareArticle'), + // ...resultResolveTypeResolver('UpdateSharedComment'), ...resultResolveTypeResolver('SetBookmarkArticle'), - ...resultResolveTypeResolver('SetFollow'), - ...resultResolveTypeResolver('GetFollowers'), - ...resultResolveTypeResolver('GetFollowing'), + // ...resultResolveTypeResolver('SetFollow'), + // ...resultResolveTypeResolver('GetFollowers'), + // ...resultResolveTypeResolver('GetFollowing'), ...resultResolveTypeResolver('GetUserPersonalization'), ...resultResolveTypeResolver('SetUserPersonalization'), ...resultResolveTypeResolver('ArticleSavingRequest'), ...resultResolveTypeResolver('CreateArticleSavingRequest'), - ...resultResolveTypeResolver('SetShareHighlight'), + // ...resultResolveTypeResolver('SetShareHighlight'), ...resultResolveTypeResolver('ArchiveLink'), ...resultResolveTypeResolver('CreateNewsletterEmail'), ...resultResolveTypeResolver('NewsletterEmails'), diff --git a/packages/api/src/resolvers/importers/uploadImportFileResolver.ts b/packages/api/src/resolvers/importers/uploadImportFileResolver.ts index 175942813..f73131c73 100644 --- a/packages/api/src/resolvers/importers/uploadImportFileResolver.ts +++ b/packages/api/src/resolvers/importers/uploadImportFileResolver.ts @@ -1,6 +1,5 @@ import { DateTime } from 'luxon' import { v4 as uuidv4 } from 'uuid' -import { getRepository } from '../../entity' import { User } from '../../entity/user' import { env } from '../../env' import { @@ -9,6 +8,7 @@ import { UploadImportFileErrorCode, UploadImportFileSuccess, } from '../../generated/graphql' +import { getRepository } from '../../repository' import { analytics } from '../../utils/analytics' import { authorized } from '../../utils/helpers' import { logger } from '../../utils/logger' diff --git a/packages/api/src/resolvers/index.ts b/packages/api/src/resolvers/index.ts index f1cf2d8e2..07b2a209f 100644 --- a/packages/api/src/resolvers/index.ts +++ b/packages/api/src/resolvers/index.ts @@ -1,18 +1,18 @@ export * from './user' export * from './article' -export * from './user_friends' -export * from './user_feed_article' +// export * from './user_friends' +// export * from './user_feed_article' export * from './user_personalization' export * from './article_saving_request' export * from './upload_files' export * from './highlight' -export * from './reaction' +// export * from './reaction' export * from './report' export * from './links' export * from './newsletters' export * from './save' export * from './send_install_instructions' -export * from './reminders' +// export * from './reminders' export * from './user_device_tokens' export * from './labels' export * from './subscriptions' diff --git a/packages/api/src/resolvers/integrations/index.ts b/packages/api/src/resolvers/integrations/index.ts index 1a480f020..1deacfcb0 100644 --- a/packages/api/src/resolvers/integrations/index.ts +++ b/packages/api/src/resolvers/integrations/index.ts @@ -1,4 +1,3 @@ -import { getRepository } from '../../entity' import { Integration, IntegrationType } from '../../entity/integration' import { User } from '../../entity/user' import { env } from '../../env' @@ -19,6 +18,7 @@ import { SetIntegrationErrorCode, SetIntegrationSuccess, } from '../../generated/graphql' +import { getRepository } from '../../repository' import { getIntegrationService } from '../../services/integrations' import { analytics } from '../../utils/analytics' import { diff --git a/packages/api/src/resolvers/labels/index.ts b/packages/api/src/resolvers/labels/index.ts index 530793914..473fb8d71 100644 --- a/packages/api/src/resolvers/labels/index.ts +++ b/packages/api/src/resolvers/labels/index.ts @@ -1,4 +1,5 @@ import { Between } from 'typeorm' +import { AppDataSource } from '../../data-source' import { createPubSubClient } from '../../datalayer/pubsub' import { getHighlightById } from '../../elastic/highlights' import { @@ -8,7 +9,6 @@ import { updateLabelsInPage, } from '../../elastic/labels' import { getPageById } from '../../elastic/pages' -import { getRepository, setClaims } from '../../entity' import { Label } from '../../entity/label' import { User } from '../../entity/user' import { env } from '../../env' @@ -38,7 +38,7 @@ import { UpdateLabelErrorCode, UpdateLabelSuccess, } from '../../generated/graphql' -import { AppDataSource } from '../../server' +import { getRepository, setClaims } from '../../repository' import { createLabel, createLabels, diff --git a/packages/api/src/resolvers/newsletters/index.ts b/packages/api/src/resolvers/newsletters/index.ts index af7d72dd1..7e461554f 100644 --- a/packages/api/src/resolvers/newsletters/index.ts +++ b/packages/api/src/resolvers/newsletters/index.ts @@ -1,4 +1,3 @@ -import { getRepository } from '../../entity' import { NewsletterEmail } from '../../entity/newsletter_email' import { User } from '../../entity/user' import { env } from '../../env' @@ -14,6 +13,7 @@ import { NewsletterEmailsErrorCode, NewsletterEmailsSuccess, } from '../../generated/graphql' +import { getRepository } from '../../repository' import { createNewsletterEmail, deleteNewsletterEmail, diff --git a/packages/api/src/resolvers/popular_reads/index.ts b/packages/api/src/resolvers/popular_reads/index.ts index 012be63cb..0e00a9898 100644 --- a/packages/api/src/resolvers/popular_reads/index.ts +++ b/packages/api/src/resolvers/popular_reads/index.ts @@ -1,24 +1,19 @@ +import { env } from '../../env' import { - MutationAddPopularReadArgs, AddPopularReadError, AddPopularReadErrorCode, AddPopularReadSuccess, + MutationAddPopularReadArgs, } from '../../generated/graphql' - -import { authorized, userDataToUser } from '../../utils/helpers' -import { analytics } from '../../utils/analytics' +import { userRepository } from '../../repository' import { addPopularRead } from '../../services/popular_reads' -import { env } from '../../env' - +import { analytics } from '../../utils/analytics' +import { authorized } from '../../utils/helpers' export const addPopularReadResolver = authorized< AddPopularReadSuccess, AddPopularReadError, MutationAddPopularReadArgs ->(async (_, { name }, ctx) => { - const { - models, - claims: { uid }, - } = ctx +>(async (_, { name }, { uid }) => { analytics.track({ userId: uid, event: 'popular_read_added', @@ -28,7 +23,9 @@ export const addPopularReadResolver = authorized< }, }) - const user = userDataToUser(await models.user.get(uid)) + const user = await userRepository.findOneBy({ + id: uid, + }) if (!user) { return { errorCodes: [AddPopularReadErrorCode.Unauthorized] } } diff --git a/packages/api/src/resolvers/recent_emails/index.ts b/packages/api/src/resolvers/recent_emails/index.ts index 0c0266a1e..906f43e44 100644 --- a/packages/api/src/resolvers/recent_emails/index.ts +++ b/packages/api/src/resolvers/recent_emails/index.ts @@ -1,5 +1,4 @@ import { ILike } from 'typeorm' -import { getRepository } from '../../entity' import { NewsletterEmail } from '../../entity/newsletter_email' import { ReceivedEmail } from '../../entity/received_email' import { env } from '../../env' @@ -12,6 +11,7 @@ import { RecentEmailsErrorCode, RecentEmailsSuccess, } from '../../generated/graphql' +import { getRepository } from '../../repository' import { updateReceivedEmail } from '../../services/received_emails' import { saveNewsletterEmail } from '../../services/save_newsletter_email' import { authorized } from '../../utils/helpers' diff --git a/packages/api/src/resolvers/recent_searches/index.ts b/packages/api/src/resolvers/recent_searches/index.ts index 5f109e5b6..9a73d8b56 100644 --- a/packages/api/src/resolvers/recent_searches/index.ts +++ b/packages/api/src/resolvers/recent_searches/index.ts @@ -1,4 +1,3 @@ -import { getRepository } from '../../entity' import { User } from '../../entity/user' import { env } from '../../env' import { @@ -6,6 +5,7 @@ import { RecentSearchesErrorCode, RecentSearchesSuccess, } from '../../generated/graphql' +import { getRepository } from '../../repository' import { getRecentSearches } from '../../services/search_history' import { analytics } from '../../utils/analytics' import { authorized } from '../../utils/helpers' diff --git a/packages/api/src/resolvers/recommendations/index.ts b/packages/api/src/resolvers/recommendations/index.ts index 09d43eb61..29e12b60e 100644 --- a/packages/api/src/resolvers/recommendations/index.ts +++ b/packages/api/src/resolvers/recommendations/index.ts @@ -1,6 +1,5 @@ import { In } from 'typeorm' import { getPageByParam } from '../../elastic/pages' -import { getRepository } from '../../entity' import { Group } from '../../entity/groups/group' import { User } from '../../entity/user' import { env } from '../../env' @@ -29,6 +28,7 @@ import { RecommendHighlightsSuccess, RecommendSuccess, } from '../../generated/graphql' +import { getRepository } from '../../repository' import { createGroup, createLabelAndRuleForGroup, diff --git a/packages/api/src/resolvers/rules/index.ts b/packages/api/src/resolvers/rules/index.ts index b26e4b569..b17830b4a 100644 --- a/packages/api/src/resolvers/rules/index.ts +++ b/packages/api/src/resolvers/rules/index.ts @@ -1,4 +1,3 @@ -import { getRepository } from '../../entity' import { Rule } from '../../entity/rule' import { User } from '../../entity/user' import { @@ -15,6 +14,7 @@ import { SetRuleErrorCode, SetRuleSuccess, } from '../../generated/graphql' +import { getRepository } from '../../repository' import { authorized } from '../../utils/helpers' export const setRuleResolver = authorized< diff --git a/packages/api/src/resolvers/save/index.ts b/packages/api/src/resolvers/save/index.ts index 3c737d493..f26dcf288 100644 --- a/packages/api/src/resolvers/save/index.ts +++ b/packages/api/src/resolvers/save/index.ts @@ -1,4 +1,3 @@ -import { getRepository } from '../../entity' import { User } from '../../entity/user' import { env } from '../../env' import { @@ -9,6 +8,7 @@ import { SaveErrorCode, SaveSuccess, } from '../../generated/graphql' +import { getRepository } from '../../repository' import { saveFile } from '../../services/save_file' import { savePage } from '../../services/save_page' import { saveUrl } from '../../services/save_url' diff --git a/packages/api/src/resolvers/send_install_instructions/index.ts b/packages/api/src/resolvers/send_install_instructions/index.ts index e42e92687..104b086d3 100644 --- a/packages/api/src/resolvers/send_install_instructions/index.ts +++ b/packages/api/src/resolvers/send_install_instructions/index.ts @@ -5,7 +5,7 @@ import { SendInstallInstructionsErrorCode, SendInstallInstructionsSuccess, } from '../../generated/graphql' -import { AppDataSource } from '../../server' +import { AppDataSource } from '../../data-source' import { authorized } from '../../utils/helpers' import { sendEmail } from '../../utils/sendEmail' diff --git a/packages/api/src/resolvers/subscriptions/index.ts b/packages/api/src/resolvers/subscriptions/index.ts index 9fe846258..3bef89ae5 100644 --- a/packages/api/src/resolvers/subscriptions/index.ts +++ b/packages/api/src/resolvers/subscriptions/index.ts @@ -1,6 +1,6 @@ import Parser from 'rss-parser' import { Brackets } from 'typeorm' -import { getRepository } from '../../entity' +import { AppDataSource } from '../../data-source' import { Subscription } from '../../entity/subscription' import { User } from '../../entity/user' import { env } from '../../env' @@ -26,7 +26,7 @@ import { UpdateSubscriptionErrorCode, UpdateSubscriptionSuccess, } from '../../generated/graphql' -import { AppDataSource } from '../../server' +import { getRepository } from '../../repository' import { getSubscribeHandler, unsubscribe } from '../../services/subscriptions' import { Merge } from '../../util' import { analytics } from '../../utils/analytics' diff --git a/packages/api/src/resolvers/update/index.ts b/packages/api/src/resolvers/update/index.ts index 8462ff832..43ea989e2 100644 --- a/packages/api/src/resolvers/update/index.ts +++ b/packages/api/src/resolvers/update/index.ts @@ -6,8 +6,9 @@ import { UpdatePageErrorCode, UpdatePageSuccess, } from '../../generated/graphql' +import { userRepository } from '../../repository' import { Merge } from '../../util' -import { authorized, userDataToUser } from '../../utils/helpers' +import { authorized } from '../../utils/helpers' export type UpdatePageSuccessPartial = Merge< UpdatePageSuccess, @@ -19,21 +20,21 @@ export const updatePageResolver = authorized< UpdatePageError, MutationUpdatePageArgs >(async (_, { input }, ctx) => { - const { - models, - claims: { uid }, - } = ctx - - const user = userDataToUser(await models.user.get(uid)) + const { pubsub, uid } = ctx + const user = await userRepository.findOneBy({ id: uid }) if (!user) { return { errorCodes: [UpdatePageErrorCode.Unauthorized] } } const page = await getPageById(input.pageId) - if (!page) return { errorCodes: [UpdatePageErrorCode.NotFound] } - else if (page.userId !== user.id) + if (!page) { + return { errorCodes: [UpdatePageErrorCode.NotFound] } + } + + if (page.userId !== user.id) { return { errorCodes: [UpdatePageErrorCode.Unauthorized] } + } const pageData = { id: input.pageId, @@ -47,7 +48,7 @@ export const updatePageResolver = authorized< } const updateResult = await updatePage(input.pageId, pageData, { - pubsub: ctx.pubsub, + pubsub: pubsub, uid, refresh: true, }) diff --git a/packages/api/src/resolvers/upload_files/index.ts b/packages/api/src/resolvers/upload_files/index.ts index 2fcd49980..e3b921201 100644 --- a/packages/api/src/resolvers/upload_files/index.ts +++ b/packages/api/src/resolvers/upload_files/index.ts @@ -3,7 +3,6 @@ import normalizeUrl from 'normalize-url' import path from 'path' import { createPage, getPageByParam, updatePage } from '../../elastic/pages' import { PageType } from '../../elastic/types' -import { uploadFileRepository } from '../../entity' import { UploadFile } from '../../entity/upload_file' import { env } from '../../env' import { @@ -14,6 +13,7 @@ import { UploadFileRequestSuccess, UploadFileStatus, } from '../../generated/graphql' +import { uploadFileRepository } from '../../repository' import { validateUrl } from '../../services/create_page_save_request' import { analytics } from '../../utils/analytics' import { authorized, generateSlug } from '../../utils/helpers' diff --git a/packages/api/src/resolvers/user/index.ts b/packages/api/src/resolvers/user/index.ts index 2591a6269..636f4815f 100644 --- a/packages/api/src/resolvers/user/index.ts +++ b/packages/api/src/resolvers/user/index.ts @@ -1,7 +1,7 @@ import * as jwt from 'jsonwebtoken' import { RegistrationType } from '../../datalayer/user/model' +import { AppDataSource } from '../../data-source' import { deletePagesByParam } from '../../elastic/pages' -import { setClaims, userRepository } from '../../entity' import { User as UserEntity } from '../../entity/user' import { env } from '../../env' import { @@ -38,7 +38,7 @@ import { UsersError, UsersSuccess, } from '../../generated/graphql' -import { AppDataSource } from '../../server' +import { setClaims, userRepository } from '../../repository' import { createUser, getTopUsers } from '../../services/create_user' import { sendVerificationEmail } from '../../services/send_emails' import { authorized, userDataToUser } from '../../utils/helpers' @@ -49,7 +49,7 @@ export const updateUserResolver = authorized< UpdateUserSuccess, UpdateUserError, MutationUpdateUserArgs ->(async (_, { input: { name, bio } }, { uid }) => { +>(async (_, { input: { name, bio } }, { uid, authTrx }) => { const user = await userRepository.findOneBy({ id: uid, }) @@ -71,15 +71,16 @@ export const updateUserResolver = authorized< return { errorCodes } } - const updatedUser = await userRepository.save({ - id: uid, - name, - source: user.source, - sourceUserId: user.sourceUserId, - profile: { - bio, - }, - }) + const updatedUser = await authTrx((tx) => + tx.getRepository(UserEntity).save({ + ...user, + name, + profile: { + ...user.profile, + bio, + }, + }) + ) return { user: userDataToUser(updatedUser) } }) @@ -88,7 +89,7 @@ export const updateUserProfileResolver = authorized< UpdateUserProfileSuccess, UpdateUserProfileError, MutationUpdateUserProfileArgs ->(async (_, { input: { userId, username, pictureUrl } }, { uid }) => { +>(async (_, { input: { userId, username, pictureUrl } }, { uid, authTrx }) => { const user = await userRepository.findOneBy({ id: userId, }) @@ -128,13 +129,16 @@ export const updateUserProfileResolver = authorized< } } - const updatedUser = await userRepository.save({ - id: uid, - profile: { - username: lowerCasedUsername, - pictureUrl, - }, - }) + const updatedUser = await authTrx((tx) => + tx.getRepository(UserEntity).save({ + ...user, + profile: { + ...user.profile, + username: lowerCasedUsername, + pictureUrl, + }, + }) + ) return { user: userDataToUser(updatedUser) } }) diff --git a/packages/api/src/resolvers/user_personalization/index.ts b/packages/api/src/resolvers/user_personalization/index.ts index 54c179bdf..4a85baaa0 100644 --- a/packages/api/src/resolvers/user_personalization/index.ts +++ b/packages/api/src/resolvers/user_personalization/index.ts @@ -1,4 +1,4 @@ -import { getRepository, setClaims } from '../../entity' +import { AppDataSource } from '../../data-source' import { UserPersonalization } from '../../entity/user_personalization' import { GetUserPersonalizationError, @@ -9,7 +9,7 @@ import { SetUserPersonalizationSuccess, SortOrder, } from '../../generated/graphql' -import { AppDataSource } from '../../server' +import { getRepository, setClaims } from '../../repository' import { authorized } from '../../utils/helpers' export const setUserPersonalizationResolver = authorized< diff --git a/packages/api/src/resolvers/webhooks/index.ts b/packages/api/src/resolvers/webhooks/index.ts index 0d2d6b0d4..cd0a3208d 100644 --- a/packages/api/src/resolvers/webhooks/index.ts +++ b/packages/api/src/resolvers/webhooks/index.ts @@ -1,4 +1,3 @@ -import { getRepository } from '../../entity' import { User } from '../../entity/user' import { Webhook } from '../../entity/webhook' import { env } from '../../env' @@ -21,6 +20,7 @@ import { WebhooksSuccess, WebhookSuccess, } from '../../generated/graphql' +import { getRepository } from '../../repository' import { analytics } from '../../utils/analytics' import { authorized } from '../../utils/helpers' diff --git a/packages/api/src/routers/article_router.ts b/packages/api/src/routers/article_router.ts index f87951765..e368772a8 100644 --- a/packages/api/src/routers/article_router.ts +++ b/packages/api/src/routers/article_router.ts @@ -6,7 +6,7 @@ import express from 'express' import * as jwt from 'jsonwebtoken' import { createPubSubClient } from '../datalayer/pubsub' import { getPageById, updatePage } from '../elastic/pages' -import { getRepository } from '../entity' +import { getRepository } from '../repository' import { Speech, SpeechState } from '../entity/speech' import { env } from '../env' import { CreateArticleErrorCode } from '../generated/graphql' diff --git a/packages/api/src/routers/auth/auth_router.ts b/packages/api/src/routers/auth/auth_router.ts index 60d02be9d..250f57a47 100644 --- a/packages/api/src/routers/auth/auth_router.ts +++ b/packages/api/src/routers/auth/auth_router.ts @@ -14,6 +14,7 @@ import express from 'express' import * as jwt from 'jsonwebtoken' import url from 'url' import { promisify } from 'util' +import { AppDataSource } from '../../data-source' import { kx } from '../../datalayer/knex_config' import UserModel from '../../datalayer/user' import { @@ -21,12 +22,11 @@ import { StatusType, UserData, } from '../../datalayer/user/model' -import { getRepository, setClaims } from '../../entity' import { User } from '../../entity/user' import { env } from '../../env' import { LoginErrorCode, SignupErrorCode } from '../../generated/graphql' +import { getRepository, setClaims } from '../../repository' import { isErrorWithCode } from '../../resolvers' -import { AppDataSource } from '../../server' import { createUser, getUserByEmail } from '../../services/create_user' import { sendConfirmationEmail, diff --git a/packages/api/src/routers/page_router.ts b/packages/api/src/routers/page_router.ts index b00091c61..d14429f11 100644 --- a/packages/api/src/routers/page_router.ts +++ b/packages/api/src/routers/page_router.ts @@ -9,7 +9,7 @@ import { createPubSubClient } from '../datalayer/pubsub' import { createPage, getPageByParam, updatePage } from '../elastic/pages' import { addRecommendation } from '../elastic/recommendation' import { Recommendation } from '../elastic/types' -import { uploadFileRepository } from '../entity' +import { uploadFileRepository } from '../repository' import { env } from '../env' import { ArticleSavingRequestStatus, diff --git a/packages/api/src/routers/svc/content.ts b/packages/api/src/routers/svc/content.ts index e52a89700..d649a86c6 100644 --- a/packages/api/src/routers/svc/content.ts +++ b/packages/api/src/routers/svc/content.ts @@ -2,15 +2,15 @@ /* eslint-disable @typescript-eslint/no-unsafe-assignment */ /* eslint-disable @typescript-eslint/explicit-module-boundary-types */ import express from 'express' +import { AppDataSource } from '../../data-source' import { createPubSubClient, readPushSubscription, } from '../../datalayer/pubsub' import { getPageByParam, updatePage } from '../../elastic/pages' import { Page } from '../../elastic/types' -import { setClaims } from '../../entity' import { ArticleSavingRequestStatus } from '../../generated/graphql' -import { AppDataSource } from '../../server' +import { setClaims } from '../../repository' import { setFileUploadComplete } from '../../services/save_file' import { logger } from '../../utils/logger' diff --git a/packages/api/src/routers/svc/email_attachment.ts b/packages/api/src/routers/svc/email_attachment.ts index 62da08e7d..972c0f623 100644 --- a/packages/api/src/routers/svc/email_attachment.ts +++ b/packages/api/src/routers/svc/email_attachment.ts @@ -1,11 +1,11 @@ import express from 'express' +import { AppDataSource } from '../../data-source' import { createPubSubClient } from '../../datalayer/pubsub' import { createPage } from '../../elastic/pages' import { ArticleSavingRequestStatus, Page } from '../../elastic/types' -import { setClaims, uploadFileRepository } from '../../entity' import { env } from '../../env' import { PageType, UploadFileStatus } from '../../generated/graphql' -import { AppDataSource } from '../../server' +import { setClaims, uploadFileRepository } from '../../repository' import { getNewsletterEmail } from '../../services/newsletters' import { updateReceivedEmail } from '../../services/received_emails' import { setFileUploadComplete } from '../../services/save_file' diff --git a/packages/api/src/routers/svc/integrations.ts b/packages/api/src/routers/svc/integrations.ts index 67b9840f1..c452a78ff 100644 --- a/packages/api/src/routers/svc/integrations.ts +++ b/packages/api/src/routers/svc/integrations.ts @@ -8,8 +8,8 @@ import { v4 as uuidv4 } from 'uuid' import { EntityType, readPushSubscription } from '../../datalayer/pubsub' import { getPageById, searchPages } from '../../elastic/pages' import { Page } from '../../elastic/types' -import { getRepository } from '../../entity' import { Integration, IntegrationType } from '../../entity/integration' +import { getRepository } from '../../repository' import { Claims } from '../../resolvers/types' import { getIntegrationService } from '../../services/integrations' import { getClaimsByToken } from '../../utils/auth' diff --git a/packages/api/src/routers/svc/reminders.ts b/packages/api/src/routers/svc/reminders.ts index 6212ff721..9bb416460 100644 --- a/packages/api/src/routers/svc/reminders.ts +++ b/packages/api/src/routers/svc/reminders.ts @@ -1,11 +1,11 @@ import { MulticastMessage } from 'firebase-admin/messaging' +import { AppDataSource } from '../../data-source' import { createPubSubClient } from '../../datalayer/pubsub' import { updatePage } from '../../elastic/pages' -import { setClaims } from '../../entity' import { UserDeviceToken } from '../../entity/user_device_tokens' import { homePageURL } from '../../env' import { ContentReader } from '../../generated/graphql' -import { AppDataSource } from '../../server' +import { setClaims } from '../../repository' import { PageReminder, setRemindersComplete } from '../../services/reminders' interface PageToNotify { diff --git a/packages/api/src/routers/svc/rss_feed.ts b/packages/api/src/routers/svc/rss_feed.ts index 78e30cc0e..26f2344e9 100644 --- a/packages/api/src/routers/svc/rss_feed.ts +++ b/packages/api/src/routers/svc/rss_feed.ts @@ -1,9 +1,9 @@ /* eslint-disable @typescript-eslint/no-misused-promises */ import express from 'express' import { readPushSubscription } from '../../datalayer/pubsub' -import { getRepository } from '../../entity' import { Subscription } from '../../entity/subscription' import { SubscriptionStatus, SubscriptionType } from '../../generated/graphql' +import { getRepository } from '../../repository' import { enqueueRssFeedFetch } from '../../utils/createTask' import { logger } from '../../utils/logger' diff --git a/packages/api/src/routers/svc/webhooks.ts b/packages/api/src/routers/svc/webhooks.ts index 00937eeea..cbe65a18a 100644 --- a/packages/api/src/routers/svc/webhooks.ts +++ b/packages/api/src/routers/svc/webhooks.ts @@ -4,8 +4,8 @@ import axios, { Method } from 'axios' import express from 'express' import { readPushSubscription } from '../../datalayer/pubsub' -import { getRepository } from '../../entity' import { Webhook } from '../../entity/webhook' +import { getRepository } from '../../repository' import { logger } from '../../utils/logger' export function webhooksServiceRouter() { diff --git a/packages/api/src/routers/text_to_speech.ts b/packages/api/src/routers/text_to_speech.ts index 1afd8e819..655f2de33 100644 --- a/packages/api/src/routers/text_to_speech.ts +++ b/packages/api/src/routers/text_to_speech.ts @@ -7,10 +7,10 @@ import express from 'express' import { readPushSubscription } from '../datalayer/pubsub' import { getPageById } from '../elastic/pages' import { ArticleSavingRequestStatus } from '../elastic/types' -import { getRepository, setClaims } from '../entity' +import { getRepository, setClaims } from '../repository' import { Speech, SpeechState } from '../entity/speech' import { UserPersonalization } from '../entity/user_personalization' -import { AppDataSource } from '../server' +import { AppDataSource } from '../data-source' import { FeatureName, getFeature } from '../services/features' import { shouldSynthesize } from '../services/speech' import { getClaimsByToken } from '../utils/auth' diff --git a/packages/api/src/routers/user_router.ts b/packages/api/src/routers/user_router.ts index b80ba0bb1..f7e4692fc 100644 --- a/packages/api/src/routers/user_router.ts +++ b/packages/api/src/routers/user_router.ts @@ -2,7 +2,7 @@ /* eslint-disable @typescript-eslint/no-unsafe-assignment */ import cors from 'cors' import express from 'express' -import { getRepository } from '../entity' +import { getRepository } from '../repository' import { User } from '../entity/user' import { env } from '../env' import { getClaimsByToken, getTokenByRequest } from '../utils/auth' diff --git a/packages/api/src/server.ts b/packages/api/src/server.ts index 77eeaa7df..696dee3f9 100755 --- a/packages/api/src/server.ts +++ b/packages/api/src/server.ts @@ -11,10 +11,9 @@ import express, { Express } from 'express' import * as httpContext from 'express-http-context2' import rateLimit from 'express-rate-limit' import { createServer, Server } from 'http' -import { DataSource } from 'typeorm' -import { SnakeNamingStrategy } from 'typeorm-naming-strategies' import { config, loggers } from 'winston' import { makeApolloServer } from './apollo' +import { AppDataSource } from './data-source' import { initElasticsearch } from './elastic' import { env } from './env' import { articleRouter } from './routers/article_router' @@ -39,32 +38,10 @@ import { userRouter } from './routers/user_router' import { sentryConfig } from './sentry' import { getClaimsByToken, getTokenByRequest } from './utils/auth' import { corsConfig } from './utils/corsConfig' -import { - buildLogger, - buildLoggerTransport, - CustomTypeOrmLogger, -} from './utils/logger' +import { buildLogger, buildLoggerTransport } from './utils/logger' const PORT = process.env.PORT || 4000 -export const AppDataSource = new DataSource({ - type: 'postgres', - host: env.pg.host, - port: env.pg.port, - schema: 'omnivore', - username: env.pg.userName, - password: env.pg.password, - database: env.pg.dbName, - logging: ['query', 'info'], - entities: [__dirname + '/entity/**/*{.js,.ts}'], - subscribers: [__dirname + '/events/**/*{.js,.ts}'], - namingStrategy: new SnakeNamingStrategy(), - logger: new CustomTypeOrmLogger(), - cache: true, - connectTimeoutMS: 60000, // 60 seconds - maxQueryExecutionTime: 60000, // 60 seconds -}) - export const createApp = (): { app: Express apollo: ApolloServer diff --git a/packages/api/src/services/archive_link.ts b/packages/api/src/services/archive_link.ts index 8d74de5e5..511c324a3 100644 --- a/packages/api/src/services/archive_link.ts +++ b/packages/api/src/services/archive_link.ts @@ -1,6 +1,6 @@ -import { setClaims } from '../entity' +import { AppDataSource } from '../data-source' import { Link } from '../entity/link' -import { AppDataSource } from '../server' +import { setClaims } from '../repository' export const setLinkArchived = async ( userId: string, diff --git a/packages/api/src/services/create_page_save_request.ts b/packages/api/src/services/create_page_save_request.ts index 6c7b74408..3e0425e6b 100644 --- a/packages/api/src/services/create_page_save_request.ts +++ b/packages/api/src/services/create_page_save_request.ts @@ -8,12 +8,12 @@ import { updatePage, } from '../elastic/pages' import { ArticleSavingRequestStatus, Label, PageType } from '../elastic/types' -import { getRepository } from '../entity' import { User } from '../entity/user' import { ArticleSavingRequest, CreateArticleSavingRequestErrorCode, } from '../generated/graphql' +import { getRepository } from '../repository' import { enqueueParseRequest } from '../utils/createTask' import { cleanUrl, diff --git a/packages/api/src/services/create_user.ts b/packages/api/src/services/create_user.ts index 7b49e0f40..650fc8672 100644 --- a/packages/api/src/services/create_user.ts +++ b/packages/api/src/services/create_user.ts @@ -1,13 +1,13 @@ import { EntityManager, In } from 'typeorm' +import { AppDataSource } from '../data-source' import { StatusType } from '../datalayer/user/model' -import { getRepository, userRepository } from '../entity' import { GroupMembership } from '../entity/groups/group_membership' import { Invite } from '../entity/groups/invite' import { Profile } from '../entity/profile' import { User } from '../entity/user' import { SignupErrorCode } from '../generated/graphql' +import { getRepository, userRepository } from '../repository' import { AuthProvider } from '../routers/auth/auth_types' -import { AppDataSource } from '../server' import { logger } from '../utils/logger' import { validateUsername } from '../utils/usernamePolicy' import { sendConfirmationEmail } from './send_emails' diff --git a/packages/api/src/services/features.ts b/packages/api/src/services/features.ts index 36495fc07..598c0262a 100644 --- a/packages/api/src/services/features.ts +++ b/packages/api/src/services/features.ts @@ -1,9 +1,9 @@ import * as jwt from 'jsonwebtoken' import { IsNull, Not } from 'typeorm' -import { getRepository } from '../entity' +import { AppDataSource } from '../data-source' import { Feature } from '../entity/feature' import { env } from '../env' -import { AppDataSource } from '../server' +import { getRepository } from '../repository' import { logger } from '../utils/logger' export enum FeatureName { diff --git a/packages/api/src/services/followers.ts b/packages/api/src/services/followers.ts index 84543a24a..95ee89dc6 100644 --- a/packages/api/src/services/followers.ts +++ b/packages/api/src/services/followers.ts @@ -1,6 +1,6 @@ -import { getRepository } from '../entity' import { Follower } from '../entity/follower' import { User } from '../entity/user' +import { getRepository } from '../repository' export const getUserFollowers = async ( user: User, diff --git a/packages/api/src/services/groups.ts b/packages/api/src/services/groups.ts index a7b49ff6e..c23409610 100644 --- a/packages/api/src/services/groups.ts +++ b/packages/api/src/services/groups.ts @@ -1,5 +1,5 @@ import { nanoid } from 'nanoid' -import { getRepository } from '../entity' +import { AppDataSource } from '../data-source' import { Group } from '../entity/groups/group' import { GroupMembership } from '../entity/groups/group_membership' import { Invite } from '../entity/groups/invite' @@ -7,7 +7,7 @@ import { RuleActionType } from '../entity/rule' import { User } from '../entity/user' import { homePageURL } from '../env' import { RecommendationGroup, User as GraphqlUser } from '../generated/graphql' -import { AppDataSource } from '../server' +import { getRepository } from '../repository' import { userDataToUser } from '../utils/helpers' import { createLabel, getLabelByName } from './labels' import { createRule } from './rules' diff --git a/packages/api/src/services/integrations/readwise.ts b/packages/api/src/services/integrations/readwise.ts index 70d7a3151..d51aa7554 100644 --- a/packages/api/src/services/integrations/readwise.ts +++ b/packages/api/src/services/integrations/readwise.ts @@ -1,6 +1,6 @@ import axios from 'axios' import { HighlightType, Page } from '../../elastic/types' -import { getRepository } from '../../entity' +import { getRepository } from '../../repository' import { Integration } from '../../entity/integration' import { env } from '../../env' import { wait } from '../../utils/helpers' diff --git a/packages/api/src/services/labels.ts b/packages/api/src/services/labels.ts index 135514e9c..4544125ef 100644 --- a/packages/api/src/services/labels.ts +++ b/packages/api/src/services/labels.ts @@ -2,11 +2,11 @@ import DataLoader from 'dataloader' import { In } from 'typeorm' import { addLabelInPage } from '../elastic/labels' import { PageContext } from '../elastic/types' -import { getRepository } from '../entity' import { Label } from '../entity/label' import { Link } from '../entity/link' import { User } from '../entity/user' import { CreateLabelInput } from '../generated/graphql' +import { getRepository } from '../repository' import { generateRandomColor } from '../utils/helpers' import { logger } from '../utils/logger' diff --git a/packages/api/src/services/newsletters.ts b/packages/api/src/services/newsletters.ts index 20345b1f3..3f8b8dd0a 100644 --- a/packages/api/src/services/newsletters.ts +++ b/packages/api/src/services/newsletters.ts @@ -1,5 +1,4 @@ import { nanoid } from 'nanoid' -import { getRepository } from '../entity' import { NewsletterEmail } from '../entity/newsletter_email' import { User } from '../entity/user' import { env } from '../env' @@ -7,6 +6,7 @@ import { CreateNewsletterEmailErrorCode, SubscriptionStatus, } from '../generated/graphql' +import { getRepository } from '../repository' import addressparser = require('nodemailer/lib/addressparser') const parsedAddress = (emailAddress: string): string | undefined => { diff --git a/packages/api/src/services/received_emails.ts b/packages/api/src/services/received_emails.ts index 2995f7346..a5c9df5c9 100644 --- a/packages/api/src/services/received_emails.ts +++ b/packages/api/src/services/received_emails.ts @@ -1,5 +1,5 @@ -import { getRepository } from '../entity' import { ReceivedEmail } from '../entity/received_email' +import { getRepository } from '../repository' export const saveReceivedEmail = async ( from: string, diff --git a/packages/api/src/services/reminders.ts b/packages/api/src/services/reminders.ts index 95168735c..47ab2d990 100644 --- a/packages/api/src/services/reminders.ts +++ b/packages/api/src/services/reminders.ts @@ -1,7 +1,7 @@ import { EntityManager, IsNull, Not } from 'typeorm' import { getPageById } from '../elastic/pages' -import { getRepository } from '../entity' import { Reminder } from '../entity/reminder' +import { getRepository } from '../repository' import { logger } from '../utils/logger' export interface PageReminder { diff --git a/packages/api/src/services/reports.ts b/packages/api/src/services/reports.ts index 96590e94e..783169615 100644 --- a/packages/api/src/services/reports.ts +++ b/packages/api/src/services/reports.ts @@ -1,8 +1,8 @@ import { getPageById } from '../elastic/pages' -import { getRepository } from '../entity' import { AbuseReport } from '../entity/reports/abuse_report' import { ContentDisplayReport } from '../entity/reports/content_display_report' import { ReportItemInput, ReportType } from '../generated/graphql' +import { getRepository } from '../repository' import { logger } from '../utils/logger' export const saveContentDisplayReport = async ( diff --git a/packages/api/src/services/rules.ts b/packages/api/src/services/rules.ts index 94fd3f4eb..fd0a783f5 100644 --- a/packages/api/src/services/rules.ts +++ b/packages/api/src/services/rules.ts @@ -1,6 +1,6 @@ import { ILike } from 'typeorm' -import { getRepository } from '../entity' import { Rule, RuleAction } from '../entity/rule' +import { getRepository } from '../repository' export const createRule = async ( userId: string, diff --git a/packages/api/src/services/save_file.ts b/packages/api/src/services/save_file.ts index 5fa56b106..a6bbeecf4 100644 --- a/packages/api/src/services/save_file.ts +++ b/packages/api/src/services/save_file.ts @@ -1,6 +1,6 @@ +import { AppDataSource } from '../data-source' import { UserData } from '../datalayer/user/model' import { updatePage } from '../elastic/pages' -import { getRepository } from '../entity' import { UploadFile } from '../entity/upload_file' import { homePageURL } from '../env' import { @@ -9,8 +9,8 @@ import { SaveFileInput, SaveResult, } from '../generated/graphql' +import { getRepository } from '../repository' import { WithDataSourcesContext } from '../resolvers/types' -import { AppDataSource } from '../server' import { logger } from '../utils/logger' import { getStorageFileDetails } from '../utils/uploads' import { createLabels } from './labels' diff --git a/packages/api/src/services/save_url.ts b/packages/api/src/services/save_url.ts index 7c9267867..520514b76 100644 --- a/packages/api/src/services/save_url.ts +++ b/packages/api/src/services/save_url.ts @@ -1,9 +1,9 @@ import { PubsubClient } from '../datalayer/pubsub' import { ArticleSavingRequestStatus } from '../elastic/types' -import { getRepository } from '../entity' import { User } from '../entity/user' import { homePageURL } from '../env' import { SaveErrorCode, SaveResult, SaveUrlInput } from '../generated/graphql' +import { getRepository } from '../repository' import { logger } from '../utils/logger' import { createPageSaveRequest } from './create_page_save_request' import { createLabels } from './labels' diff --git a/packages/api/src/services/search_history.ts b/packages/api/src/services/search_history.ts index 582048c3c..14ddf4ef1 100644 --- a/packages/api/src/services/search_history.ts +++ b/packages/api/src/services/search_history.ts @@ -1,5 +1,5 @@ -import { getRepository } from '../entity' import { SearchHistory } from '../entity/search_history' +import { getRepository } from '../repository' export const getRecentSearches = async ( userId: string diff --git a/packages/api/src/services/subscriptions.ts b/packages/api/src/services/subscriptions.ts index ecc0db916..c6771ff05 100644 --- a/packages/api/src/services/subscriptions.ts +++ b/packages/api/src/services/subscriptions.ts @@ -1,8 +1,8 @@ import axios from 'axios' -import { getRepository } from '../entity' import { NewsletterEmail } from '../entity/newsletter_email' import { Subscription } from '../entity/subscription' import { SubscriptionStatus, SubscriptionType } from '../generated/graphql' +import { getRepository } from '../repository' import { logger } from '../utils/logger' import { sendEmail } from '../utils/sendEmail' import { createNewsletterEmail } from './newsletters' diff --git a/packages/api/src/services/user_device_tokens.ts b/packages/api/src/services/user_device_tokens.ts index f050a5388..7724ac2f0 100644 --- a/packages/api/src/services/user_device_tokens.ts +++ b/packages/api/src/services/user_device_tokens.ts @@ -1,9 +1,9 @@ -import { getRepository, setClaims } from '../entity' +import { AppDataSource } from '../data-source' import { User } from '../entity/user' import { UserDeviceToken } from '../entity/user_device_tokens' import { env } from '../env' import { SetDeviceTokenErrorCode } from '../generated/graphql' -import { AppDataSource } from '../server' +import { getRepository, setClaims } from '../repository' import { analytics } from '../utils/analytics' export const getDeviceToken = async ( diff --git a/packages/api/src/utils/auth.ts b/packages/api/src/utils/auth.ts index 1f410a4a4..d4ef7aee6 100644 --- a/packages/api/src/utils/auth.ts +++ b/packages/api/src/utils/auth.ts @@ -4,7 +4,7 @@ import express from 'express' import * as jwt from 'jsonwebtoken' import { promisify } from 'util' import { v4 as uuidv4 } from 'uuid' -import { getRepository } from '../entity' +import { getRepository } from '../repository' import { ApiKey } from '../entity/api_key' import { env } from '../env' import { Claims, ClaimsToSet } from '../resolvers/types' diff --git a/packages/api/src/utils/helpers.ts b/packages/api/src/utils/helpers.ts index ea72d7ffc..faeb00a4a 100644 --- a/packages/api/src/utils/helpers.ts +++ b/packages/api/src/utils/helpers.ts @@ -148,7 +148,6 @@ export const userDataToUser = ( profile: Profile } => ({ ...user, - name: user.name, source: user.source as RegistrationType, createdAt: user.createdAt || new Date(), friendsCount: user.friendsCount || 0, diff --git a/packages/api/src/utils/parser.ts b/packages/api/src/utils/parser.ts index b280ed784..0dad18031 100644 --- a/packages/api/src/utils/parser.ts +++ b/packages/api/src/utils/parser.ts @@ -16,7 +16,7 @@ import { ILike } from 'typeorm' import { promisify } from 'util' import { v4 as uuid } from 'uuid' import { Highlight } from '../elastic/types' -import { getRepository } from '../entity' +import { getRepository } from '../repository' import { User } from '../entity/user' import { env } from '../env' import { PageType, PreparedDocumentInput } from '../generated/graphql' diff --git a/packages/api/test/db.ts b/packages/api/test/db.ts index 22da1a403..aafeef018 100644 --- a/packages/api/test/db.ts +++ b/packages/api/test/db.ts @@ -1,7 +1,7 @@ import Postgrator from 'postgrator' import { FindOptionsWhere } from 'typeorm' import { SnakeNamingStrategy } from 'typeorm-naming-strategies' -import { getRepository, setClaims } from '../src/entity' +import { getRepository, setClaims, userRepository } from '../src/repository' import { Integration } from '../src/entity/integration' import { Label } from '../src/entity/label' import { Link } from '../src/entity/link' @@ -13,7 +13,7 @@ import { Subscription } from '../src/entity/subscription' import { User } from '../src/entity/user' import { UserDeviceToken } from '../src/entity/user_device_tokens' import { SubscriptionStatus, SubscriptionType } from '../src/generated/graphql' -import { AppDataSource } from '../src/server' +import { AppDataSource } from '../src/data-source' import { createUser } from '../src/services/create_user' import { Filter } from "../src/entity/filter" @@ -189,7 +189,7 @@ export const getDeviceToken = async ( } export const getUser = async (id: string): Promise => { - return getRepository(User).findOneBy({ id }) + return userRepository.findOneBy({ id }) } export const getLink = async (id: string): Promise => { diff --git a/packages/api/test/global-teardown.ts b/packages/api/test/global-teardown.ts index ff2b9e6a0..80ae6be4c 100644 --- a/packages/api/test/global-teardown.ts +++ b/packages/api/test/global-teardown.ts @@ -1,4 +1,4 @@ -import { AppDataSource } from '../src/server' +import { AppDataSource } from '../src/data-source' import { stopApolloServer } from './util' import { kx } from '../src/datalayer/knex_config' diff --git a/packages/api/test/resolvers/api_key.test.ts b/packages/api/test/resolvers/api_key.test.ts index 2bdc840fb..f3040d8b2 100644 --- a/packages/api/test/resolvers/api_key.test.ts +++ b/packages/api/test/resolvers/api_key.test.ts @@ -1,10 +1,10 @@ -import { User } from '../../src/entity/user' -import { createTestUser, deleteTestUser } from '../db' -import { graphqlRequest, request } from '../util' import { expect } from 'chai' import supertest from 'supertest' -import { getRepository } from '../../src/entity' import { ApiKey } from '../../src/entity/api_key' +import { User } from '../../src/entity/user' +import { getRepository } from '../../src/repository' +import { createTestUser, deleteTestUser } from '../db' +import { graphqlRequest, request } from '../util' const testAPIKey = (apiKey: string): supertest.Test => { const query = ` diff --git a/packages/api/test/resolvers/article.test.ts b/packages/api/test/resolvers/article.test.ts index 43678497e..08b478a75 100644 --- a/packages/api/test/resolvers/article.test.ts +++ b/packages/api/test/resolvers/article.test.ts @@ -22,7 +22,6 @@ import { PageContext, PageType, } from '../../src/elastic/types' -import { getRepository } from '../../src/entity' import { UploadFile } from '../../src/entity/upload_file' import { User } from '../../src/entity/user' import { @@ -31,6 +30,7 @@ import { UpdateReason, UploadFileStatus, } from '../../src/generated/graphql' +import { getRepository } from '../../src/repository' import * as createTask from '../../src/utils/createTask' import * as uploads from '../../src/utils/uploads' import { createTestUser, deleteTestUser } from '../db' diff --git a/packages/api/test/resolvers/features.test.ts b/packages/api/test/resolvers/features.test.ts index 2988cf90c..0f0a2841d 100644 --- a/packages/api/test/resolvers/features.test.ts +++ b/packages/api/test/resolvers/features.test.ts @@ -2,10 +2,10 @@ import { expect } from 'chai' import * as jwt from 'jsonwebtoken' import 'mocha' import sinon, { SinonFakeTimers } from 'sinon' -import { getRepository } from '../../src/entity' import { Feature } from '../../src/entity/feature' import { User } from '../../src/entity/user' import { env } from '../../src/env' +import { getRepository } from '../../src/repository' import { createTestUser, deleteTestUser } from '../db' import { graphqlRequest, request } from '../util' diff --git a/packages/api/test/resolvers/integrations.test.ts b/packages/api/test/resolvers/integrations.test.ts index a802a7092..97baee4e1 100644 --- a/packages/api/test/resolvers/integrations.test.ts +++ b/packages/api/test/resolvers/integrations.test.ts @@ -1,14 +1,14 @@ +import chai, { expect } from 'chai' import 'mocha' +import nock from 'nock' +import sinonChai from 'sinon-chai' +import { Integration } from '../../src/entity/integration' import { User } from '../../src/entity/user' +import { SetIntegrationErrorCode } from '../../src/generated/graphql' +import { getRepository } from '../../src/repository' +import { READWISE_API_URL } from '../../src/services/integrations/readwise' import { createTestUser, deleteTestIntegrations, deleteTestUser } from '../db' import { generateFakeUuid, graphqlRequest, request } from '../util' -import { SetIntegrationErrorCode } from '../../src/generated/graphql' -import chai, { expect } from 'chai' -import { getRepository } from '../../src/entity' -import { Integration } from '../../src/entity/integration' -import nock from 'nock' -import { READWISE_API_URL } from '../../src/services/integrations/readwise' -import sinonChai from 'sinon-chai' chai.use(sinonChai) diff --git a/packages/api/test/resolvers/labels.test.ts b/packages/api/test/resolvers/labels.test.ts index 9868d6036..0d43de0b4 100644 --- a/packages/api/test/resolvers/labels.test.ts +++ b/packages/api/test/resolvers/labels.test.ts @@ -1,3 +1,21 @@ +import { expect } from 'chai' +import 'mocha' +import { createPubSubClient } from '../../src/datalayer/pubsub' +import { refreshIndex } from '../../src/elastic' +import { + addHighlightToPage, + getHighlightById, +} from '../../src/elastic/highlights' +import { deletePage, getPageById } from '../../src/elastic/pages' +import { + Highlight, + HighlightType, + Page, + PageContext, +} from '../../src/elastic/types' +import { Label } from '../../src/entity/label' +import { User } from '../../src/entity/user' +import { getRepository } from '../../src/repository' import { createTestLabel, createTestUser, @@ -10,24 +28,6 @@ import { graphqlRequest, request, } from '../util' -import { Label } from '../../src/entity/label' -import { expect } from 'chai' -import 'mocha' -import { User } from '../../src/entity/user' -import { - Highlight, - HighlightType, - Page, - PageContext, -} from '../../src/elastic/types' -import { getRepository } from '../../src/entity' -import { deletePage, getPageById } from '../../src/elastic/pages' -import { createPubSubClient } from '../../src/datalayer/pubsub' -import { - addHighlightToPage, - getHighlightById, -} from '../../src/elastic/highlights' -import { refreshIndex } from '../../src/elastic' describe('Labels API', () => { let user: User diff --git a/packages/api/test/resolvers/newsletters.test.ts b/packages/api/test/resolvers/newsletters.test.ts index 54690b8f0..bc6bc00de 100644 --- a/packages/api/test/resolvers/newsletters.test.ts +++ b/packages/api/test/resolvers/newsletters.test.ts @@ -1,3 +1,12 @@ +import { expect } from 'chai' +import 'mocha' +import { NewsletterEmail } from '../../src/entity/newsletter_email' +import { User } from '../../src/entity/user' +import { + DeleteNewsletterEmailErrorCode, + SubscriptionStatus, +} from '../../src/generated/graphql' +import { getRepository } from '../../src/repository' import { createTestNewsletterEmail, createTestSubscription, @@ -6,15 +15,6 @@ import { getNewsletterEmail, } from '../db' import { generateFakeUuid, graphqlRequest, request } from '../util' -import { NewsletterEmail } from '../../src/entity/newsletter_email' -import { User } from '../../src/entity/user' -import { expect } from 'chai' -import { - DeleteNewsletterEmailErrorCode, - SubscriptionStatus, -} from '../../src/generated/graphql' -import 'mocha' -import { getRepository } from '../../src/entity' describe('Newsletters API', () => { let user: User diff --git a/packages/api/test/resolvers/recent_emails.test.ts b/packages/api/test/resolvers/recent_emails.test.ts index 28c96d2d3..cfabb6f55 100644 --- a/packages/api/test/resolvers/recent_emails.test.ts +++ b/packages/api/test/resolvers/recent_emails.test.ts @@ -1,13 +1,13 @@ -import 'mocha' import { expect } from 'chai' +import 'mocha' +import sinon from 'sinon' +import { NewsletterEmail } from '../../src/entity/newsletter_email' +import { ReceivedEmail } from '../../src/entity/received_email' import { User } from '../../src/entity/user' +import { getRepository } from '../../src/repository' +import * as sendEmail from '../../src/utils/sendEmail' import { createTestUser, deleteTestUser } from '../db' import { graphqlRequest, request } from '../util' -import { getRepository } from '../../src/entity' -import { ReceivedEmail } from '../../src/entity/received_email' -import { NewsletterEmail } from '../../src/entity/newsletter_email' -import sinon from 'sinon' -import * as sendEmail from '../../src/utils/sendEmail' describe('Recent Emails Resolver', () => { const recentEmailsQuery = ` diff --git a/packages/api/test/resolvers/recent_searches.test.ts b/packages/api/test/resolvers/recent_searches.test.ts index 3217541a0..93358c4b2 100644 --- a/packages/api/test/resolvers/recent_searches.test.ts +++ b/packages/api/test/resolvers/recent_searches.test.ts @@ -1,12 +1,12 @@ -import 'mocha' import { expect } from 'chai' -import { User } from '../../src/entity/user' +import 'mocha' +import { createPubSubClient } from '../../src/datalayer/pubsub' import { PageContext } from '../../src/elastic/types' +import { SearchHistory } from '../../src/entity/search_history' +import { User } from '../../src/entity/user' +import { getRepository } from '../../src/repository' import { createTestUser, deleteTestUser } from '../db' import { graphqlRequest, request } from '../util' -import { createPubSubClient } from '../../src/datalayer/pubsub' -import { getRepository } from '../../src/entity' -import { SearchHistory } from '../../src/entity/search_history' describe('recent_searches resolver', () => { let user: User diff --git a/packages/api/test/resolvers/reminders.test.ts b/packages/api/test/resolvers/reminders.test.ts index 26645748d..8988d78f6 100644 --- a/packages/api/test/resolvers/reminders.test.ts +++ b/packages/api/test/resolvers/reminders.test.ts @@ -22,7 +22,7 @@ import { DateTime } from 'luxon' import 'mocha' import { Page } from '../../src/elastic/types' -describe('Reminders API', () => { +xdescribe('Reminders API', () => { let authToken: string let page: Page let reminder: Reminder diff --git a/packages/api/test/resolvers/report.test.ts b/packages/api/test/resolvers/report.test.ts index d39c67c3f..90d7cf18e 100644 --- a/packages/api/test/resolvers/report.test.ts +++ b/packages/api/test/resolvers/report.test.ts @@ -1,11 +1,11 @@ -import { User } from '../../src/entity/user' +import { expect } from 'chai' import { Page } from '../../src/elastic/types' +import { ContentDisplayReport } from '../../src/entity/reports/content_display_report' +import { User } from '../../src/entity/user' +import { ReportType } from '../../src/generated/graphql' +import { getRepository } from '../../src/repository' import { createTestUser, deleteTestUser } from '../db' import { createTestElasticPage, graphqlRequest, request } from '../util' -import { ReportType } from '../../src/generated/graphql' -import { ContentDisplayReport } from '../../src/entity/reports/content_display_report' -import { expect } from 'chai' -import { getRepository } from '../../src/entity' describe('Report API', () => { let user: User diff --git a/packages/api/test/resolvers/rules.test.ts b/packages/api/test/resolvers/rules.test.ts index dc6bbcf9b..7f10512a6 100644 --- a/packages/api/test/resolvers/rules.test.ts +++ b/packages/api/test/resolvers/rules.test.ts @@ -1,10 +1,10 @@ -import 'mocha' import { expect } from 'chai' -import { graphqlRequest, request } from '../util' -import { User } from '../../src/entity/user' -import { createTestUser, deleteTestUser } from '../db' -import { getRepository } from '../../src/entity' +import 'mocha' import { Rule, RuleAction, RuleActionType } from '../../src/entity/rule' +import { User } from '../../src/entity/user' +import { getRepository } from '../../src/repository' +import { createTestUser, deleteTestUser } from '../db' +import { graphqlRequest, request } from '../util' describe('Rules Resolver', () => { const username = 'fakeUser' diff --git a/packages/api/test/resolvers/subscriptions.test.ts b/packages/api/test/resolvers/subscriptions.test.ts index 3ddb24bed..447adcdef 100644 --- a/packages/api/test/resolvers/subscriptions.test.ts +++ b/packages/api/test/resolvers/subscriptions.test.ts @@ -5,15 +5,15 @@ import sinonChai from 'sinon-chai' import { NewsletterEmail } from '../../src/entity/newsletter_email' import { Subscription } from '../../src/entity/subscription' import { User } from '../../src/entity/user' -import { getRepository } from '../../src/entity' import { SubscriptionStatus, SubscriptionType, } from '../../src/generated/graphql' import { - UNSUBSCRIBE_EMAIL_TEXT, unsubscribe, } from '../../src/services/subscriptions' +import { getRepository } from '../../src/repository' +import { UNSUBSCRIBE_EMAIL_TEXT } from '../../src/services/subscriptions' import * as sendEmail from '../../src/utils/sendEmail' import { createTestSubscription, createTestUser, deleteTestUser } from '../db' import { graphqlRequest, request } from '../util' diff --git a/packages/api/test/resolvers/user_device_tokens.test.ts b/packages/api/test/resolvers/user_device_tokens.test.ts index 11e9730dd..230258046 100644 --- a/packages/api/test/resolvers/user_device_tokens.test.ts +++ b/packages/api/test/resolvers/user_device_tokens.test.ts @@ -1,3 +1,9 @@ +import { expect } from 'chai' +import 'mocha' +import { User } from '../../src/entity/user' +import { UserDeviceToken } from '../../src/entity/user_device_tokens' +import { SetDeviceTokenErrorCode } from '../../src/generated/graphql' +import { getRepository } from '../../src/repository' import { createTestDeviceToken, createTestUser, @@ -6,12 +12,6 @@ import { getDeviceToken, } from '../db' import { generateFakeUuid, graphqlRequest, request } from '../util' -import { expect } from 'chai' -import { UserDeviceToken } from '../../src/entity/user_device_tokens' -import { SetDeviceTokenErrorCode } from '../../src/generated/graphql' -import 'mocha' -import { User } from '../../src/entity/user' -import { getRepository } from '../../src/entity' describe('Device tokens API', () => { let authToken: string diff --git a/packages/api/test/resolvers/user_feed_article.test.ts b/packages/api/test/resolvers/user_feed_article.test.ts index 84fafcf4e..d317d496f 100644 --- a/packages/api/test/resolvers/user_feed_article.test.ts +++ b/packages/api/test/resolvers/user_feed_article.test.ts @@ -1,3 +1,11 @@ +import { expect } from 'chai' +import 'mocha' +import { Highlight } from '../../src/entity/highlight' +import { Link } from '../../src/entity/link' +import { Page } from '../../src/entity/page' +import { User } from '../../src/entity/user' +import { SharedArticleErrorCode } from '../../src/generated/graphql' +import { getRepository } from '../../src/repository' import { createTestLink, createTestPage, @@ -5,16 +13,8 @@ import { deleteTestUser, } from '../db' import { graphqlRequest, request } from '../util' -import { expect } from 'chai' -import { SharedArticleErrorCode } from '../../src/generated/graphql' -import { Page } from '../../src/entity/page' -import { Link } from '../../src/entity/link' -import { Highlight } from '../../src/entity/highlight' -import 'mocha' -import { getRepository } from '../../src/entity' -import { User } from '../../src/entity/user' -describe('User feed article API', () => { +xdescribe('User feed article API', () => { const existingUsername = 'fakeUser' let user: User let authToken: string diff --git a/packages/api/test/resolvers/webhooks.test.ts b/packages/api/test/resolvers/webhooks.test.ts index d07aa544b..e5e863bd6 100644 --- a/packages/api/test/resolvers/webhooks.test.ts +++ b/packages/api/test/resolvers/webhooks.test.ts @@ -1,11 +1,11 @@ -import { createTestUser, deleteTestUser } from '../db' -import { graphqlRequest, request } from '../util' import { expect } from 'chai' import 'mocha' import { User } from '../../src/entity/user' -import { WebhookEvent } from '../../src/generated/graphql' import { Webhook } from '../../src/entity/webhook' -import { getRepository } from '../../src/entity' +import { WebhookEvent } from '../../src/generated/graphql' +import { getRepository } from '../../src/repository' +import { createTestUser, deleteTestUser } from '../db' +import { graphqlRequest, request } from '../util' describe('Webhooks API', () => { let user: User diff --git a/packages/api/test/routers/auth.test.ts b/packages/api/test/routers/auth.test.ts index a4d2f3b5e..51c9fefd5 100644 --- a/packages/api/test/routers/auth.test.ts +++ b/packages/api/test/routers/auth.test.ts @@ -5,8 +5,8 @@ import sinonChai from 'sinon-chai' import supertest from 'supertest' import { StatusType } from '../../src/datalayer/user/model' import { searchPages } from '../../src/elastic/pages' -import { getRepository } from '../../src/entity' import { User } from '../../src/entity/user' +import { getRepository } from '../../src/repository' import { AuthProvider } from '../../src/routers/auth/auth_types' import { createPendingUserToken } from '../../src/routers/auth/jwt_helpers' import { diff --git a/packages/api/test/routers/emails.test.ts b/packages/api/test/routers/emails.test.ts index 0757ed759..e5ea6a3c5 100644 --- a/packages/api/test/routers/emails.test.ts +++ b/packages/api/test/routers/emails.test.ts @@ -1,19 +1,19 @@ +import { expect } from 'chai' +import * as jwt from 'jsonwebtoken' +import 'mocha' +import sinon from 'sinon' +import { ReceivedEmail } from '../../src/entity/received_email' +import { User } from '../../src/entity/user' +import { getRepository } from '../../src/repository' +import * as parser from '../../src/utils/parser' +import * as sendEmail from '../../src/utils/sendEmail' +import * as sendNotification from '../../src/utils/sendNotification' import { createTestNewsletterEmail, createTestUser, deleteTestUser, } from '../db' -import { User } from '../../src/entity/user' -import 'mocha' -import sinon from 'sinon' -import { expect } from 'chai' import { request } from '../util' -import * as parser from '../../src/utils/parser' -import * as sendNotification from '../../src/utils/sendNotification' -import * as sendEmail from '../../src/utils/sendEmail' -import { getRepository } from '../../src/entity' -import { ReceivedEmail } from '../../src/entity/received_email' -import * as jwt from 'jsonwebtoken' describe('Emails Router', () => { const newsletterEmail = 'fakeUser@omnivore.app' diff --git a/packages/api/test/routers/integrations.test.ts b/packages/api/test/routers/integrations.test.ts index f8a1b1e82..16834d329 100644 --- a/packages/api/test/routers/integrations.test.ts +++ b/packages/api/test/routers/integrations.test.ts @@ -16,10 +16,10 @@ import { Page, PageContext, } from '../../src/elastic/types' -import { getRepository } from '../../src/entity' import { Integration, IntegrationType } from '../../src/entity/integration' import { User } from '../../src/entity/user' import { env } from '../../src/env' +import { getRepository } from '../../src/repository' import { getHighlightUrl } from '../../src/services/highlights' import { READWISE_API_URL } from '../../src/services/integrations/readwise' import { createTestUser, deleteTestIntegrations, deleteTestUser } from '../db' diff --git a/packages/api/test/routers/reminders.test.ts b/packages/api/test/routers/reminders.test.ts index b7e8d6216..2282ede9d 100644 --- a/packages/api/test/routers/reminders.test.ts +++ b/packages/api/test/routers/reminders.test.ts @@ -12,7 +12,7 @@ import { Reminder } from '../../src/entity/reminder' import { expect } from 'chai' import 'mocha' -describe('Reminders Router', () => { +xdescribe('Reminders Router', () => { let authToken: string let user: User let reminder: Reminder diff --git a/packages/api/test/routers/webhooks.test.ts b/packages/api/test/routers/webhooks.test.ts index 0dc3bbc56..20feb6c0f 100644 --- a/packages/api/test/routers/webhooks.test.ts +++ b/packages/api/test/routers/webhooks.test.ts @@ -1,11 +1,11 @@ +import { expect } from 'chai' +import 'mocha' +import nock from 'nock' +import { User } from '../../src/entity/user' +import { Webhook } from '../../src/entity/webhook' +import { getRepository } from '../../src/repository' import { createTestUser, deleteTestUser } from '../db' import { request } from '../util' -import { User } from '../../src/entity/user' -import 'mocha' -import { getRepository } from '../../src/entity' -import { Webhook } from '../../src/entity/webhook' -import { expect } from 'chai' -import nock from 'nock' describe('Webhooks Router', () => { const token = process.env.PUBSUB_VERIFICATION_TOKEN || '' diff --git a/packages/api/test/services/create_content_display_report.test.ts b/packages/api/test/services/create_content_display_report.test.ts index e9998a6f3..e583a0695 100644 --- a/packages/api/test/services/create_content_display_report.test.ts +++ b/packages/api/test/services/create_content_display_report.test.ts @@ -1,14 +1,14 @@ -import 'mocha' import chai, { expect } from 'chai' -import { createTestUser, deleteTestUser } from '../db' +import 'mocha' import sinonChai from 'sinon-chai' -import { User } from '../../src/entity/user' -import { getRepository } from '../../src/entity' -import { ContentDisplayReport } from '../../src/entity/reports/content_display_report' -import { saveContentDisplayReport } from '../../src/services/reports' -import { ReportType } from '../../src/generated/graphql' -import { createTestElasticPage } from '../util' import { Page } from '../../src/elastic/types' +import { ContentDisplayReport } from '../../src/entity/reports/content_display_report' +import { User } from '../../src/entity/user' +import { ReportType } from '../../src/generated/graphql' +import { getRepository } from '../../src/repository' +import { saveContentDisplayReport } from '../../src/services/reports' +import { createTestUser, deleteTestUser } from '../db' +import { createTestElasticPage } from '../util' chai.use(sinonChai) diff --git a/packages/api/test/services/create_user.test.ts b/packages/api/test/services/create_user.test.ts index 514b3ab90..a202a8648 100644 --- a/packages/api/test/services/create_user.test.ts +++ b/packages/api/test/services/create_user.test.ts @@ -1,24 +1,22 @@ -import 'mocha' +import { MailDataRequired } from '@sendgrid/helpers/classes/mail' import chai, { expect } from 'chai' -import { - createTestUser, - createUserWithoutProfile, deleteFiltersFromUser, - deleteTestUser, - getProfile -} from "../db" -import { createGroup } from '../../src/services/groups' +import 'mocha' +import sinon from 'sinon' +import sinonChai from 'sinon-chai' +import { StatusType, User } from '../../src/entity/user' +import { getRepository } from '../../src/repository' import { getUserFollowers, getUserFollowing, } from '../../src/services/followers' -import sinonChai from 'sinon-chai' -import sinon from 'sinon' +import { createGroup } from '../../src/services/groups' import * as util from '../../src/utils/sendEmail' -import { MailDataRequired } from '@sendgrid/helpers/classes/mail' -import { User } from '../../src/entity/user' -import { getRepository } from '../../src/entity/utils' -import { Filter } from "../../src/entity/filter" -import { getRepository } from '../../src/entity' +import { + createTestUser, + createUserWithoutProfile, deleteFiltersFromUser, + deleteTestUser, + getProfile, +} from '../db' chai.use(sinonChai) @@ -70,10 +68,10 @@ describe('create user', () => { const userFollowing = await getUserFollowing(user) const adminUserFollowers = await getUserFollowers(adminUser) const adminUserFollowing = await getUserFollowing(adminUser) - expect(userFollowers.map(u => u.id)).to.eql(admninIds) - expect(userFollowing.map(u => u.id)).to.eql(admninIds) - expect(adminUserFollowers.map(u => u.id)).to.eql(userIds) - expect(adminUserFollowing.map(u => u.id)).to.eql(userIds) + expect(userFollowers.map((u) => u.id)).to.eql(admninIds) + expect(userFollowing.map((u) => u.id)).to.eql(admninIds) + expect(adminUserFollowers.map((u) => u.id)).to.eql(userIds) + expect(adminUserFollowing.map((u) => u.id)).to.eql(userIds) }) it('creates profile when user exists but profile not', async () => { diff --git a/packages/api/test/services/labels.test.ts b/packages/api/test/services/labels.test.ts index 436ab07bd..58015483d 100644 --- a/packages/api/test/services/labels.test.ts +++ b/packages/api/test/services/labels.test.ts @@ -1,5 +1,11 @@ -import 'mocha' import { expect } from 'chai' +import 'mocha' +import { Label } from '../../src/entity/label' +import { Link } from '../../src/entity/link' +import { LinkLabel } from '../../src/entity/link_label' +import { User } from '../../src/entity/user' +import { getRepository } from '../../src/repository' +import { labelsLoader } from '../../src/services/labels' import { createTestLabel, createTestLink, @@ -7,12 +13,6 @@ import { createTestUser, deleteTestUser, } from '../db' -import { LinkLabel } from '../../src/entity/link_label' -import { Label } from '../../src/entity/label' -import { Link } from '../../src/entity/link' -import { labelsLoader } from '../../src/services/labels' -import { getRepository } from '../../src/entity' -import { User } from '../../src/entity/user' describe('batch get labels from linkIds', () => { let user: User diff --git a/packages/api/test/services/save_newsletter_email.test.ts b/packages/api/test/services/save_newsletter_email.test.ts index 6b9fcd8ff..91fe7170c 100644 --- a/packages/api/test/services/save_newsletter_email.test.ts +++ b/packages/api/test/services/save_newsletter_email.test.ts @@ -3,11 +3,11 @@ import 'mocha' import nock from 'nock' import { createPubSubClient } from '../../src/datalayer/pubsub' import { getPageByParam } from '../../src/elastic/pages' -import { getRepository } from '../../src/entity' import { NewsletterEmail } from '../../src/entity/newsletter_email' import { ReceivedEmail } from '../../src/entity/received_email' import { Subscription } from '../../src/entity/subscription' import { User } from '../../src/entity/user' +import { getRepository } from '../../src/repository' import { createNewsletterEmail } from '../../src/services/newsletters' import { SaveContext } from '../../src/services/save_email' import { saveNewsletterEmail } from '../../src/services/save_newsletter_email' From fc52b522f23ece38940143e02e59d72ffefff24b Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Wed, 23 Aug 2023 12:41:02 +0800 Subject: [PATCH 010/144] fix test --- packages/api/src/apollo.ts | 4 ++-- packages/api/src/data-source.ts | 6 +++--- packages/api/src/elastic/highlights.ts | 2 +- packages/api/src/elastic/labels.ts | 2 +- packages/api/src/elastic/pages.ts | 2 +- packages/api/src/elastic/types.ts | 2 +- packages/api/src/events/user/user_created.ts | 3 ++- packages/api/src/{datalayer => }/pubsub.ts | 6 +++--- packages/api/src/resolvers/labels/index.ts | 2 +- packages/api/src/resolvers/types.ts | 2 +- packages/api/src/routers/article_router.ts | 4 ++-- packages/api/src/routers/auth/apple_auth.ts | 5 ++--- packages/api/src/routers/auth/auth_router.ts | 16 ++++------------ packages/api/src/routers/auth/google_auth.ts | 5 ++--- packages/api/src/routers/auth/mobile/sign_in.ts | 7 +++---- packages/api/src/routers/auth/mobile/sign_up.ts | 5 ++--- packages/api/src/routers/local_debug_router.ts | 11 ++++++----- packages/api/src/routers/page_router.ts | 4 ++-- packages/api/src/routers/svc/content.ts | 5 +---- packages/api/src/routers/svc/email_attachment.ts | 2 +- packages/api/src/routers/svc/emails.ts | 5 +---- packages/api/src/routers/svc/integrations.ts | 2 +- packages/api/src/routers/svc/links.ts | 2 +- packages/api/src/routers/svc/newsletters.ts | 5 +---- packages/api/src/routers/svc/reminders.ts | 2 +- packages/api/src/routers/svc/rss_feed.ts | 2 +- packages/api/src/routers/svc/upload.ts | 2 +- packages/api/src/routers/svc/webhooks.ts | 2 +- packages/api/src/routers/text_to_speech.ts | 6 +++--- .../api/src/services/create_page_save_request.ts | 2 +- packages/api/src/services/create_user.ts | 3 +-- packages/api/src/services/popular_reads.ts | 2 +- packages/api/src/services/save_email.ts | 2 +- packages/api/src/services/save_file.ts | 4 ++-- .../api/src/services/save_newsletter_email.ts | 2 +- packages/api/src/services/save_url.ts | 2 +- packages/api/test/elastic/highlights.test.ts | 2 +- packages/api/test/elastic/labels.test.ts | 2 +- packages/api/test/elastic/pages.test.ts | 2 +- packages/api/test/global-teardown.ts | 8 +++----- packages/api/test/resolvers/article.test.ts | 2 +- .../resolvers/article_saving_request.test.ts | 2 +- packages/api/test/resolvers/highlight.test.ts | 2 +- packages/api/test/resolvers/labels.test.ts | 2 +- .../api/test/resolvers/recent_searches.test.ts | 2 +- .../test/resolvers/upload_file_request.test.ts | 2 +- packages/api/test/routers/auth.test.ts | 3 +-- packages/api/test/routers/integrations.test.ts | 2 +- packages/api/test/services/save_email.test.ts | 2 +- .../test/services/save_newsletter_email.test.ts | 2 +- packages/api/test/util.ts | 8 ++++---- 51 files changed, 80 insertions(+), 103 deletions(-) rename packages/api/src/{datalayer => }/pubsub.ts (96%) diff --git a/packages/api/src/apollo.ts b/packages/api/src/apollo.ts index b4f8c1977..fc8923d72 100644 --- a/packages/api/src/apollo.ts +++ b/packages/api/src/apollo.ts @@ -12,14 +12,14 @@ import { ExpressContext } from 'apollo-server-express/dist/ApolloServer' import * as jwt from 'jsonwebtoken' import { EntityManager } from 'typeorm' import { promisify } from 'util' -import { createPubSubClient } from './datalayer/pubsub' +import { AppDataSource } from './data-source' import { sanitizeDirectiveTransformer } from './directives' import { env } from './env' +import { createPubSubClient } from './pubsub' import { functionResolvers } from './resolvers/function_resolvers' import { ClaimsToSet, ResolverContext } from './resolvers/types' import ScalarResolvers from './scalars' import typeDefs from './schema' -import { AppDataSource } from './data-source' import { tracer } from './tracing' import { getClaimsByToken, setAuthInCookie } from './utils/auth' import { SetClaimsRole } from './utils/dictionary' diff --git a/packages/api/src/data-source.ts b/packages/api/src/data-source.ts index 88425c26f..c944ff5a8 100644 --- a/packages/api/src/data-source.ts +++ b/packages/api/src/data-source.ts @@ -16,7 +16,7 @@ export const AppDataSource = new DataSource({ subscribers: [__dirname + '/events/**/*{.js,.ts}'], namingStrategy: new SnakeNamingStrategy(), logger: new CustomTypeOrmLogger(), - // cache: true, - connectTimeoutMS: 60000, // 60 seconds - maxQueryExecutionTime: 60000, // 60 seconds + cache: true, + connectTimeoutMS: 40000, // 40 seconds + maxQueryExecutionTime: 10000, // 10 seconds }) diff --git a/packages/api/src/elastic/highlights.ts b/packages/api/src/elastic/highlights.ts index 3d0808c04..431568c7d 100644 --- a/packages/api/src/elastic/highlights.ts +++ b/packages/api/src/elastic/highlights.ts @@ -1,5 +1,5 @@ import { errors } from '@elastic/elasticsearch' -import { EntityType } from '../datalayer/pubsub' +import { EntityType } from '../pubsub' import { SortBy, SortOrder, SortParams } from '../utils/search' import { client, INDEX_ALIAS, logger } from './index' import { diff --git a/packages/api/src/elastic/labels.ts b/packages/api/src/elastic/labels.ts index 5864a0dff..d66cdc85f 100644 --- a/packages/api/src/elastic/labels.ts +++ b/packages/api/src/elastic/labels.ts @@ -1,5 +1,5 @@ import { errors } from '@elastic/elasticsearch' -import { EntityType } from '../datalayer/pubsub' +import { EntityType } from '../pubsub' import { client, INDEX_ALIAS, logger } from './index' import { Label, PageContext } from './types' diff --git a/packages/api/src/elastic/pages.ts b/packages/api/src/elastic/pages.ts index bd6b59e32..30e069652 100644 --- a/packages/api/src/elastic/pages.ts +++ b/packages/api/src/elastic/pages.ts @@ -1,7 +1,7 @@ import { errors } from '@elastic/elasticsearch' import { BuiltQuery, ESBuilder, esBuilder } from 'elastic-ts' -import { EntityType } from '../datalayer/pubsub' import { BulkActionType } from '../generated/graphql' +import { EntityType } from '../pubsub' import { wordsCount } from '../utils/helpers' import { DateFilter, diff --git a/packages/api/src/elastic/types.ts b/packages/api/src/elastic/types.ts index 6613775f3..06f769bb6 100644 --- a/packages/api/src/elastic/types.ts +++ b/packages/api/src/elastic/types.ts @@ -1,5 +1,5 @@ // Define the type of the body for the Search request -import { PubsubClient } from '../datalayer/pubsub' +import { PubsubClient } from '../pubsub' import { PickTuple } from '../util' import { DateFilter, diff --git a/packages/api/src/events/user/user_created.ts b/packages/api/src/events/user/user_created.ts index d73383dae..c5efee193 100644 --- a/packages/api/src/events/user/user_created.ts +++ b/packages/api/src/events/user/user_created.ts @@ -3,8 +3,9 @@ import { EventSubscriber, InsertEvent, } from 'typeorm' -import { createPubSubClient } from '../../datalayer/pubsub' import { Profile } from '../../entity/profile' +import { env } from '../../env' +import { createPubSubClient } from '../../pubsub' import { addPopularReadsForNewUser } from '../../services/popular_reads' import { IntercomClient } from '../../utils/intercom' diff --git a/packages/api/src/datalayer/pubsub.ts b/packages/api/src/pubsub.ts similarity index 96% rename from packages/api/src/datalayer/pubsub.ts rename to packages/api/src/pubsub.ts index 82eff598b..6b432753b 100644 --- a/packages/api/src/datalayer/pubsub.ts +++ b/packages/api/src/pubsub.ts @@ -1,8 +1,8 @@ import { PubSub } from '@google-cloud/pubsub' -import { env } from '../env' -import { ReportType } from '../generated/graphql' import express from 'express' -import { buildLogger } from '../utils/logger' +import { env } from './env' +import { ReportType } from './generated/graphql' +import { buildLogger } from './utils/logger' const logger = buildLogger('pubsub') diff --git a/packages/api/src/resolvers/labels/index.ts b/packages/api/src/resolvers/labels/index.ts index 473fb8d71..1d23b9011 100644 --- a/packages/api/src/resolvers/labels/index.ts +++ b/packages/api/src/resolvers/labels/index.ts @@ -1,6 +1,5 @@ import { Between } from 'typeorm' import { AppDataSource } from '../../data-source' -import { createPubSubClient } from '../../datalayer/pubsub' import { getHighlightById } from '../../elastic/highlights' import { deleteLabel, @@ -38,6 +37,7 @@ import { UpdateLabelErrorCode, UpdateLabelSuccess, } from '../../generated/graphql' +import { createPubSubClient } from '../../pubsub' import { getRepository, setClaims } from '../../repository' import { createLabel, diff --git a/packages/api/src/resolvers/types.ts b/packages/api/src/resolvers/types.ts index 17cf31258..759ff718a 100644 --- a/packages/api/src/resolvers/types.ts +++ b/packages/api/src/resolvers/types.ts @@ -4,7 +4,7 @@ import { Context as ApolloContext } from 'apollo-server-core' import * as jwt from 'jsonwebtoken' import { EntityManager } from 'typeorm' import winston from 'winston' -import { PubsubClient } from '../datalayer/pubsub' +import { PubsubClient } from '../pubsub' export interface Claims { uid: string diff --git a/packages/api/src/routers/article_router.ts b/packages/api/src/routers/article_router.ts index e368772a8..6e7d7f4bc 100644 --- a/packages/api/src/routers/article_router.ts +++ b/packages/api/src/routers/article_router.ts @@ -4,12 +4,12 @@ import { htmlToSpeechFile } from '@omnivore/text-to-speech-handler' import cors from 'cors' import express from 'express' import * as jwt from 'jsonwebtoken' -import { createPubSubClient } from '../datalayer/pubsub' import { getPageById, updatePage } from '../elastic/pages' -import { getRepository } from '../repository' import { Speech, SpeechState } from '../entity/speech' import { env } from '../env' import { CreateArticleErrorCode } from '../generated/graphql' +import { createPubSubClient } from '../pubsub' +import { getRepository } from '../repository' import { Claims } from '../resolvers/types' import { createPageSaveRequest } from '../services/create_page_save_request' import { getClaimsByToken } from '../utils/auth' diff --git a/packages/api/src/routers/auth/apple_auth.ts b/packages/api/src/routers/auth/apple_auth.ts index 73cbf56a0..cd75b3324 100644 --- a/packages/api/src/routers/auth/apple_auth.ts +++ b/packages/api/src/routers/auth/apple_auth.ts @@ -3,9 +3,9 @@ /* eslint-disable @typescript-eslint/no-unsafe-assignment */ import * as jwt from 'jsonwebtoken' import jwksClient from 'jwks-rsa' -import UserModel from '../../datalayer/user' import { env, homePageURL } from '../../env' import { LoginErrorCode } from '../../generated/graphql' +import { userRepository } from '../../repository' import { logger } from '../../utils/logger' import { createSsoToken, ssoRedirectURL } from '../../utils/sso' import { DecodeTokenResult } from './auth_types' @@ -119,8 +119,7 @@ export async function handleAppleWebAuth( } try { - const model = new UserModel() - const user = await model.getWhere({ + const user = await userRepository.findOneBy({ sourceUserId: decodedTokenResult.sourceUserId, source: 'APPLE', }) diff --git a/packages/api/src/routers/auth/auth_router.ts b/packages/api/src/routers/auth/auth_router.ts index 250f57a47..9e3200ad0 100644 --- a/packages/api/src/routers/auth/auth_router.ts +++ b/packages/api/src/routers/auth/auth_router.ts @@ -15,17 +15,10 @@ import * as jwt from 'jsonwebtoken' import url from 'url' import { promisify } from 'util' import { AppDataSource } from '../../data-source' -import { kx } from '../../datalayer/knex_config' -import UserModel from '../../datalayer/user' -import { - RegistrationType, - StatusType, - UserData, -} from '../../datalayer/user/model' -import { User } from '../../entity/user' +import { RegistrationType, StatusType, User } from '../../entity/user' import { env } from '../../env' import { LoginErrorCode, SignupErrorCode } from '../../generated/graphql' -import { getRepository, setClaims } from '../../repository' +import { getRepository, setClaims, userRepository } from '../../repository' import { isErrorWithCode } from '../../resolvers' import { createUser, getUserByEmail } from '../../services/create_user' import { @@ -247,8 +240,7 @@ export function authRouter() { return { errorCodes: [SignupErrorCode.GoogleAuthError] } } - const model = new UserModel(kx) - const user = await model.getWhere({ email: userData.email }) + const user = await userRepository.findOneBy({ email: userData.email }) // eslint-disable-next @typescript-eslint/ban-ts-comment const secret = (await signToken( @@ -343,7 +335,7 @@ export function authRouter() { async function handleSuccessfulLogin( req: express.Request, res: express.Response, - user: UserData | User, + user: User, newUser: boolean ): Promise { try { diff --git a/packages/api/src/routers/auth/google_auth.ts b/packages/api/src/routers/auth/google_auth.ts index a7c148da7..ae500165b 100644 --- a/packages/api/src/routers/auth/google_auth.ts +++ b/packages/api/src/routers/auth/google_auth.ts @@ -1,9 +1,9 @@ import { google, oauth2_v2 as oauthV2 } from 'googleapis' import { OAuth2Client } from 'googleapis-common' import url from 'url' -import UserModel from '../../datalayer/user' import { env, homePageURL } from '../../env' import { LoginErrorCode } from '../../generated/graphql' +import { userRepository } from '../../repository' import { logger } from '../../utils/logger' import { createSsoToken, ssoRedirectURL } from '../../utils/sso' import { DecodeTokenResult } from './auth_types' @@ -127,8 +127,7 @@ export async function handleGoogleWebAuth( redirectURL: authFailedRedirect, }) } - const model = new UserModel() - const user = await model.getWhere({ + const user = await userRepository.findOneBy({ email, source: 'GOOGLE', }) diff --git a/packages/api/src/routers/auth/mobile/sign_in.ts b/packages/api/src/routers/auth/mobile/sign_in.ts index 93cf22ea8..1d0c8523a 100644 --- a/packages/api/src/routers/auth/mobile/sign_in.ts +++ b/packages/api/src/routers/auth/mobile/sign_in.ts @@ -1,6 +1,6 @@ /* eslint-disable @typescript-eslint/restrict-template-expressions */ -import UserModel from '../../../datalayer/user' -import { StatusType } from '../../../datalayer/user/model' +import { StatusType } from '../../../entity/user' +import { userRepository } from '../../../repository' import { getUserByEmail } from '../../../services/create_user' import { sendConfirmationEmail } from '../../../services/send_emails' import { comparePassword } from '../../../utils/auth' @@ -97,8 +97,7 @@ async function createAuthResponsePayload( } try { - const model = new UserModel() - const user = await model.getWhere({ + const user = await userRepository.findOneBy({ email: decodedTokenResult.email, source: authProvider, }) diff --git a/packages/api/src/routers/auth/mobile/sign_up.ts b/packages/api/src/routers/auth/mobile/sign_up.ts index 3af00ec93..7431b7c00 100644 --- a/packages/api/src/routers/auth/mobile/sign_up.ts +++ b/packages/api/src/routers/auth/mobile/sign_up.ts @@ -1,5 +1,5 @@ /* eslint-disable @typescript-eslint/restrict-template-expressions */ -import UserModel from '../../../datalayer/user' +import { userRepository } from '../../../repository' import { createUser } from '../../../services/create_user' import { hashPassword } from '../../../utils/auth' import { logger } from '../../../utils/logger' @@ -97,8 +97,7 @@ async function createSignUpResponsePayload( } // check if user exists - const userModel = new UserModel() - const existingUser = await userModel.getWhere({ email }) + const existingUser = await userRepository.findOneBy({ email }) if (existingUser) { return { diff --git a/packages/api/src/routers/local_debug_router.ts b/packages/api/src/routers/local_debug_router.ts index 031af949a..146e53931 100644 --- a/packages/api/src/routers/local_debug_router.ts +++ b/packages/api/src/routers/local_debug_router.ts @@ -1,11 +1,11 @@ +import cors from 'cors' import express, { Router } from 'express' import { env } from '../env' -import { createWebAuthToken } from './auth/jwt_helpers' -import UserModel from '../datalayer/user' import { LoginErrorCode } from '../generated/graphql' -import cors from 'cors' +import { userRepository } from '../repository' import { createUser } from '../services/create_user' import { corsConfig } from '../utils/corsConfig' +import { createWebAuthToken } from './auth/jwt_helpers' // For local development only export function localDebugRouter(): Router { @@ -20,8 +20,9 @@ export function localDebugRouter(): Router { try { let userId: string | undefined = undefined - const userModel = new UserModel() - const existingUser = await userModel.getWhere({ email: fakeEmail }) + const existingUser = await userRepository.findOneBy({ + email: fakeEmail, + }) userId = existingUser?.id if (!userId) { diff --git a/packages/api/src/routers/page_router.ts b/packages/api/src/routers/page_router.ts index d14429f11..bfc2309bd 100644 --- a/packages/api/src/routers/page_router.ts +++ b/packages/api/src/routers/page_router.ts @@ -5,17 +5,17 @@ import cors from 'cors' import express from 'express' import * as jwt from 'jsonwebtoken' -import { createPubSubClient } from '../datalayer/pubsub' import { createPage, getPageByParam, updatePage } from '../elastic/pages' import { addRecommendation } from '../elastic/recommendation' import { Recommendation } from '../elastic/types' -import { uploadFileRepository } from '../repository' import { env } from '../env' import { ArticleSavingRequestStatus, PageType, UploadFileStatus, } from '../generated/graphql' +import { createPubSubClient } from '../pubsub' +import { uploadFileRepository } from '../repository' import { Claims } from '../resolvers/types' import { getTokenByRequest } from '../utils/auth' import { corsConfig } from '../utils/corsConfig' diff --git a/packages/api/src/routers/svc/content.ts b/packages/api/src/routers/svc/content.ts index d649a86c6..1a992ccee 100644 --- a/packages/api/src/routers/svc/content.ts +++ b/packages/api/src/routers/svc/content.ts @@ -3,13 +3,10 @@ /* eslint-disable @typescript-eslint/explicit-module-boundary-types */ import express from 'express' import { AppDataSource } from '../../data-source' -import { - createPubSubClient, - readPushSubscription, -} from '../../datalayer/pubsub' import { getPageByParam, updatePage } from '../../elastic/pages' import { Page } from '../../elastic/types' import { ArticleSavingRequestStatus } from '../../generated/graphql' +import { createPubSubClient, readPushSubscription } from '../../pubsub' import { setClaims } from '../../repository' import { setFileUploadComplete } from '../../services/save_file' import { logger } from '../../utils/logger' diff --git a/packages/api/src/routers/svc/email_attachment.ts b/packages/api/src/routers/svc/email_attachment.ts index 972c0f623..a6245dc28 100644 --- a/packages/api/src/routers/svc/email_attachment.ts +++ b/packages/api/src/routers/svc/email_attachment.ts @@ -1,10 +1,10 @@ import express from 'express' import { AppDataSource } from '../../data-source' -import { createPubSubClient } from '../../datalayer/pubsub' import { createPage } from '../../elastic/pages' import { ArticleSavingRequestStatus, Page } from '../../elastic/types' import { env } from '../../env' import { PageType, UploadFileStatus } from '../../generated/graphql' +import { createPubSubClient } from '../../pubsub' import { setClaims, uploadFileRepository } from '../../repository' import { getNewsletterEmail } from '../../services/newsletters' import { updateReceivedEmail } from '../../services/received_emails' diff --git a/packages/api/src/routers/svc/emails.ts b/packages/api/src/routers/svc/emails.ts index 9ceb2c913..2bc1698f4 100644 --- a/packages/api/src/routers/svc/emails.ts +++ b/packages/api/src/routers/svc/emails.ts @@ -1,10 +1,7 @@ import cors from 'cors' import express from 'express' -import { - createPubSubClient, - readPushSubscription, -} from '../../datalayer/pubsub' import { env } from '../../env' +import { createPubSubClient, readPushSubscription } from '../../pubsub' import { getNewsletterEmail } from '../../services/newsletters' import { saveReceivedEmail, diff --git a/packages/api/src/routers/svc/integrations.ts b/packages/api/src/routers/svc/integrations.ts index c452a78ff..349792c9c 100644 --- a/packages/api/src/routers/svc/integrations.ts +++ b/packages/api/src/routers/svc/integrations.ts @@ -5,10 +5,10 @@ import { stringify } from 'csv-stringify' import express from 'express' import { DateTime } from 'luxon' import { v4 as uuidv4 } from 'uuid' -import { EntityType, readPushSubscription } from '../../datalayer/pubsub' import { getPageById, searchPages } from '../../elastic/pages' import { Page } from '../../elastic/types' import { Integration, IntegrationType } from '../../entity/integration' +import { EntityType, readPushSubscription } from '../../pubsub' import { getRepository } from '../../repository' import { Claims } from '../../resolvers/types' import { getIntegrationService } from '../../services/integrations' diff --git a/packages/api/src/routers/svc/links.ts b/packages/api/src/routers/svc/links.ts index a769b6428..76903986e 100644 --- a/packages/api/src/routers/svc/links.ts +++ b/packages/api/src/routers/svc/links.ts @@ -2,7 +2,7 @@ /* eslint-disable @typescript-eslint/no-unsafe-assignment */ /* eslint-disable @typescript-eslint/explicit-module-boundary-types */ import express from 'express' -import { readPushSubscription } from '../../datalayer/pubsub' +import { readPushSubscription } from '../../pubsub' import { createPageSaveRequest } from '../../services/create_page_save_request' import { logger } from '../../utils/logger' diff --git a/packages/api/src/routers/svc/newsletters.ts b/packages/api/src/routers/svc/newsletters.ts index d39ea8280..2ae6397a3 100644 --- a/packages/api/src/routers/svc/newsletters.ts +++ b/packages/api/src/routers/svc/newsletters.ts @@ -1,9 +1,6 @@ import express from 'express' -import { - createPubSubClient, - readPushSubscription, -} from '../../datalayer/pubsub' import { SubscriptionStatus } from '../../generated/graphql' +import { createPubSubClient, readPushSubscription } from '../../pubsub' import { getNewsletterEmail, updateConfirmationCode, diff --git a/packages/api/src/routers/svc/reminders.ts b/packages/api/src/routers/svc/reminders.ts index 9bb416460..650c69e65 100644 --- a/packages/api/src/routers/svc/reminders.ts +++ b/packages/api/src/routers/svc/reminders.ts @@ -1,10 +1,10 @@ import { MulticastMessage } from 'firebase-admin/messaging' import { AppDataSource } from '../../data-source' -import { createPubSubClient } from '../../datalayer/pubsub' import { updatePage } from '../../elastic/pages' import { UserDeviceToken } from '../../entity/user_device_tokens' import { homePageURL } from '../../env' import { ContentReader } from '../../generated/graphql' +import { createPubSubClient } from '../../pubsub' import { setClaims } from '../../repository' import { PageReminder, setRemindersComplete } from '../../services/reminders' diff --git a/packages/api/src/routers/svc/rss_feed.ts b/packages/api/src/routers/svc/rss_feed.ts index 26f2344e9..dc8b4bc15 100644 --- a/packages/api/src/routers/svc/rss_feed.ts +++ b/packages/api/src/routers/svc/rss_feed.ts @@ -1,8 +1,8 @@ /* eslint-disable @typescript-eslint/no-misused-promises */ import express from 'express' -import { readPushSubscription } from '../../datalayer/pubsub' import { Subscription } from '../../entity/subscription' import { SubscriptionStatus, SubscriptionType } from '../../generated/graphql' +import { readPushSubscription } from '../../pubsub' import { getRepository } from '../../repository' import { enqueueRssFeedFetch } from '../../utils/createTask' import { logger } from '../../utils/logger' diff --git a/packages/api/src/routers/svc/upload.ts b/packages/api/src/routers/svc/upload.ts index 9d3167832..d93e07fb3 100644 --- a/packages/api/src/routers/svc/upload.ts +++ b/packages/api/src/routers/svc/upload.ts @@ -4,8 +4,8 @@ import express from 'express' import { DateTime } from 'luxon' import { v4 as uuidv4 } from 'uuid' -import { readPushSubscription } from '../../datalayer/pubsub' import { env } from '../../env' +import { readPushSubscription } from '../../pubsub' import { logger } from '../../utils/logger' import { uploadToBucket } from '../../utils/uploads' diff --git a/packages/api/src/routers/svc/webhooks.ts b/packages/api/src/routers/svc/webhooks.ts index cbe65a18a..97d17590c 100644 --- a/packages/api/src/routers/svc/webhooks.ts +++ b/packages/api/src/routers/svc/webhooks.ts @@ -3,8 +3,8 @@ /* eslint-disable @typescript-eslint/explicit-module-boundary-types */ import axios, { Method } from 'axios' import express from 'express' -import { readPushSubscription } from '../../datalayer/pubsub' import { Webhook } from '../../entity/webhook' +import { readPushSubscription } from '../../pubsub' import { getRepository } from '../../repository' import { logger } from '../../utils/logger' diff --git a/packages/api/src/routers/text_to_speech.ts b/packages/api/src/routers/text_to_speech.ts index 655f2de33..b5e3ee84e 100644 --- a/packages/api/src/routers/text_to_speech.ts +++ b/packages/api/src/routers/text_to_speech.ts @@ -4,13 +4,13 @@ import { htmlToSpeechFile } from '@omnivore/text-to-speech-handler' import cors from 'cors' import express from 'express' -import { readPushSubscription } from '../datalayer/pubsub' +import { AppDataSource } from '../data-source' import { getPageById } from '../elastic/pages' import { ArticleSavingRequestStatus } from '../elastic/types' -import { getRepository, setClaims } from '../repository' import { Speech, SpeechState } from '../entity/speech' import { UserPersonalization } from '../entity/user_personalization' -import { AppDataSource } from '../data-source' +import { readPushSubscription } from '../pubsub' +import { getRepository, setClaims } from '../repository' import { FeatureName, getFeature } from '../services/features' import { shouldSynthesize } from '../services/speech' import { getClaimsByToken } from '../utils/auth' diff --git a/packages/api/src/services/create_page_save_request.ts b/packages/api/src/services/create_page_save_request.ts index 3e0425e6b..ffc29e1ff 100644 --- a/packages/api/src/services/create_page_save_request.ts +++ b/packages/api/src/services/create_page_save_request.ts @@ -1,6 +1,5 @@ import * as privateIpLib from 'private-ip' import { v4 as uuidv4 } from 'uuid' -import { createPubSubClient, PubsubClient } from '../datalayer/pubsub' import { countByCreatedAt, createPage, @@ -13,6 +12,7 @@ import { ArticleSavingRequest, CreateArticleSavingRequestErrorCode, } from '../generated/graphql' +import { createPubSubClient, PubsubClient } from '../pubsub' import { getRepository } from '../repository' import { enqueueParseRequest } from '../utils/createTask' import { diff --git a/packages/api/src/services/create_user.ts b/packages/api/src/services/create_user.ts index 650fc8672..15d1be07c 100644 --- a/packages/api/src/services/create_user.ts +++ b/packages/api/src/services/create_user.ts @@ -1,10 +1,9 @@ import { EntityManager, In } from 'typeorm' import { AppDataSource } from '../data-source' -import { StatusType } from '../datalayer/user/model' import { GroupMembership } from '../entity/groups/group_membership' import { Invite } from '../entity/groups/invite' import { Profile } from '../entity/profile' -import { User } from '../entity/user' +import { StatusType, User } from '../entity/user' import { SignupErrorCode } from '../generated/graphql' import { getRepository, userRepository } from '../repository' import { AuthProvider } from '../routers/auth/auth_types' diff --git a/packages/api/src/services/popular_reads.ts b/packages/api/src/services/popular_reads.ts index 89d067c1b..42b002d4b 100644 --- a/packages/api/src/services/popular_reads.ts +++ b/packages/api/src/services/popular_reads.ts @@ -1,10 +1,10 @@ import * as httpContext from 'express-http-context2' import { readFileSync } from 'fs' import path from 'path' -import { createPubSubClient } from '../datalayer/pubsub' import { createPage } from '../elastic/pages' import { ArticleSavingRequestStatus, Page, PageContext } from '../elastic/types' import { PageType } from '../generated/graphql' +import { createPubSubClient } from '../pubsub' import { generateSlug, stringToHash } from '../utils/helpers' import { logger } from '../utils/logger' diff --git a/packages/api/src/services/save_email.ts b/packages/api/src/services/save_email.ts index 375b8d9ed..525ecc3f9 100644 --- a/packages/api/src/services/save_email.ts +++ b/packages/api/src/services/save_email.ts @@ -1,6 +1,6 @@ -import { PubsubClient } from '../datalayer/pubsub' import { createPage, getPageByParam, updatePage } from '../elastic/pages' import { ArticleSavingRequestStatus, Page } from '../elastic/types' +import { PubsubClient } from '../pubsub' import { enqueueThumbnailTask } from '../utils/createTask' import { cleanUrl, diff --git a/packages/api/src/services/save_file.ts b/packages/api/src/services/save_file.ts index a6bbeecf4..068e3e349 100644 --- a/packages/api/src/services/save_file.ts +++ b/packages/api/src/services/save_file.ts @@ -1,7 +1,7 @@ import { AppDataSource } from '../data-source' -import { UserData } from '../datalayer/user/model' import { updatePage } from '../elastic/pages' import { UploadFile } from '../entity/upload_file' +import { User } from '../entity/user' import { homePageURL } from '../env' import { ArticleSavingRequestStatus, @@ -24,7 +24,7 @@ export const setFileUploadComplete = async ( export const saveFile = async ( ctx: WithDataSourcesContext, - user: UserData, + user: User, input: SaveFileInput ): Promise => { logger.info('saving file with input', input) diff --git a/packages/api/src/services/save_newsletter_email.ts b/packages/api/src/services/save_newsletter_email.ts index 6bb0e4f73..1bae6d5d2 100644 --- a/packages/api/src/services/save_newsletter_email.ts +++ b/packages/api/src/services/save_newsletter_email.ts @@ -1,11 +1,11 @@ import { MulticastMessage } from 'firebase-admin/messaging' -import { createPubSubClient } from '../datalayer/pubsub' import { updatePage } from '../elastic/pages' import { Page } from '../elastic/types' import { NewsletterEmail } from '../entity/newsletter_email' import { UserDeviceToken } from '../entity/user_device_tokens' import { env } from '../env' import { ContentReader } from '../generated/graphql' +import { createPubSubClient } from '../pubsub' import { analytics } from '../utils/analytics' import { isBase64Image } from '../utils/helpers' import { logger } from '../utils/logger' diff --git a/packages/api/src/services/save_url.ts b/packages/api/src/services/save_url.ts index 520514b76..82ebf7736 100644 --- a/packages/api/src/services/save_url.ts +++ b/packages/api/src/services/save_url.ts @@ -1,8 +1,8 @@ -import { PubsubClient } from '../datalayer/pubsub' import { ArticleSavingRequestStatus } from '../elastic/types' import { User } from '../entity/user' import { homePageURL } from '../env' import { SaveErrorCode, SaveResult, SaveUrlInput } from '../generated/graphql' +import { PubsubClient } from '../pubsub' import { getRepository } from '../repository' import { logger } from '../utils/logger' import { createPageSaveRequest } from './create_page_save_request' diff --git a/packages/api/test/elastic/highlights.test.ts b/packages/api/test/elastic/highlights.test.ts index cbc497df5..7c66407c0 100644 --- a/packages/api/test/elastic/highlights.test.ts +++ b/packages/api/test/elastic/highlights.test.ts @@ -6,7 +6,7 @@ import { Page, PageContext, } from '../../src/elastic/types' -import { createPubSubClient } from '../../src/datalayer/pubsub' +import { createPubSubClient } from '../../src/pubsub' import { deletePage } from '../../src/elastic/pages' import { addHighlightToPage, diff --git a/packages/api/test/elastic/labels.test.ts b/packages/api/test/elastic/labels.test.ts index 3a1ce37b5..50ae8b952 100644 --- a/packages/api/test/elastic/labels.test.ts +++ b/packages/api/test/elastic/labels.test.ts @@ -9,7 +9,7 @@ import { PageContext, PageType, } from '../../src/elastic/types' -import { createPubSubClient } from '../../src/datalayer/pubsub' +import { createPubSubClient } from '../../src/pubsub' import { createPage, deletePage, getPageById } from '../../src/elastic/pages' import { addLabelInPage, setLabelsForHighlight } from '../../src/elastic/labels' import { addHighlightToPage } from '../../src/elastic/highlights' diff --git a/packages/api/test/elastic/pages.test.ts b/packages/api/test/elastic/pages.test.ts index 3bc9e2ab0..be6db6591 100644 --- a/packages/api/test/elastic/pages.test.ts +++ b/packages/api/test/elastic/pages.test.ts @@ -7,7 +7,7 @@ import { PageContext, PageType, } from '../../src/elastic/types' -import { createPubSubClient } from '../../src/datalayer/pubsub' +import { createPubSubClient } from '../../src/pubsub' import { countByCreatedAt, createPage, diff --git a/packages/api/test/global-teardown.ts b/packages/api/test/global-teardown.ts index 80ae6be4c..a10787d50 100644 --- a/packages/api/test/global-teardown.ts +++ b/packages/api/test/global-teardown.ts @@ -1,12 +1,10 @@ import { AppDataSource } from '../src/data-source' import { stopApolloServer } from './util' -import { kx } from '../src/datalayer/knex_config' export const mochaGlobalTeardown = async () => { - await AppDataSource.destroy() - await kx.destroy() - console.log('db connection closed') - await stopApolloServer() console.log('apollo server stopped') + + await AppDataSource.destroy() + console.log('db connection closed') } diff --git a/packages/api/test/resolvers/article.test.ts b/packages/api/test/resolvers/article.test.ts index 08b478a75..6b663f2a5 100644 --- a/packages/api/test/resolvers/article.test.ts +++ b/packages/api/test/resolvers/article.test.ts @@ -3,7 +3,6 @@ import { expect } from 'chai' import chaiString from 'chai-string' import 'mocha' import sinon from 'sinon' -import { createPubSubClient } from '../../src/datalayer/pubsub' import { refreshIndex } from '../../src/elastic' import { addHighlightToPage } from '../../src/elastic/highlights' import { @@ -30,6 +29,7 @@ import { UpdateReason, UploadFileStatus, } from '../../src/generated/graphql' +import { createPubSubClient } from '../../src/pubsub' import { getRepository } from '../../src/repository' import * as createTask from '../../src/utils/createTask' import * as uploads from '../../src/utils/uploads' diff --git a/packages/api/test/resolvers/article_saving_request.test.ts b/packages/api/test/resolvers/article_saving_request.test.ts index 2d38bf026..657f9b961 100644 --- a/packages/api/test/resolvers/article_saving_request.test.ts +++ b/packages/api/test/resolvers/article_saving_request.test.ts @@ -1,7 +1,7 @@ import { expect } from 'chai' import 'mocha' import sinon from 'sinon' -import { createPubSubClient } from '../../src/datalayer/pubsub' +import { createPubSubClient } from '../../src/pubsub' import { deletePagesByParam, getPageByParam } from '../../src/elastic/pages' import { ArticleSavingRequestStatus, diff --git a/packages/api/test/resolvers/highlight.test.ts b/packages/api/test/resolvers/highlight.test.ts index d4b2a9537..0eefc4952 100644 --- a/packages/api/test/resolvers/highlight.test.ts +++ b/packages/api/test/resolvers/highlight.test.ts @@ -10,7 +10,7 @@ import { expect } from 'chai' import 'mocha' import { User } from '../../src/entity/user' import chaiString from 'chai-string' -import { createPubSubClient } from '../../src/datalayer/pubsub' +import { createPubSubClient } from '../../src/pubsub' import { HighlightType, PageContext } from '../../src/elastic/types' import { deletePage, updatePage } from '../../src/elastic/pages' diff --git a/packages/api/test/resolvers/labels.test.ts b/packages/api/test/resolvers/labels.test.ts index 0d43de0b4..74b350805 100644 --- a/packages/api/test/resolvers/labels.test.ts +++ b/packages/api/test/resolvers/labels.test.ts @@ -1,6 +1,6 @@ import { expect } from 'chai' import 'mocha' -import { createPubSubClient } from '../../src/datalayer/pubsub' +import { createPubSubClient } from '../../src/pubsub' import { refreshIndex } from '../../src/elastic' import { addHighlightToPage, diff --git a/packages/api/test/resolvers/recent_searches.test.ts b/packages/api/test/resolvers/recent_searches.test.ts index 93358c4b2..ff46df46c 100644 --- a/packages/api/test/resolvers/recent_searches.test.ts +++ b/packages/api/test/resolvers/recent_searches.test.ts @@ -1,6 +1,6 @@ import { expect } from 'chai' import 'mocha' -import { createPubSubClient } from '../../src/datalayer/pubsub' +import { createPubSubClient } from '../../src/pubsub' import { PageContext } from '../../src/elastic/types' import { SearchHistory } from '../../src/entity/search_history' import { User } from '../../src/entity/user' diff --git a/packages/api/test/resolvers/upload_file_request.test.ts b/packages/api/test/resolvers/upload_file_request.test.ts index 50f57956c..98119ac94 100644 --- a/packages/api/test/resolvers/upload_file_request.test.ts +++ b/packages/api/test/resolvers/upload_file_request.test.ts @@ -6,7 +6,7 @@ import 'mocha' import { User } from '../../src/entity/user' import chaiString from 'chai-string' import { PageContext } from '../../src/elastic/types' -import { createPubSubClient } from '../../src/datalayer/pubsub' +import { createPubSubClient } from '../../src/pubsub' import { deletePage, getPageById } from '../../src/elastic/pages' chai.use(chaiString) diff --git a/packages/api/test/routers/auth.test.ts b/packages/api/test/routers/auth.test.ts index 51c9fefd5..4a17d58ba 100644 --- a/packages/api/test/routers/auth.test.ts +++ b/packages/api/test/routers/auth.test.ts @@ -3,9 +3,8 @@ import chai, { expect } from 'chai' import sinon from 'sinon' import sinonChai from 'sinon-chai' import supertest from 'supertest' -import { StatusType } from '../../src/datalayer/user/model' import { searchPages } from '../../src/elastic/pages' -import { User } from '../../src/entity/user' +import { StatusType, User } from '../../src/entity/user' import { getRepository } from '../../src/repository' import { AuthProvider } from '../../src/routers/auth/auth_types' import { createPendingUserToken } from '../../src/routers/auth/jwt_helpers' diff --git a/packages/api/test/routers/integrations.test.ts b/packages/api/test/routers/integrations.test.ts index 16834d329..1b841d0e2 100644 --- a/packages/api/test/routers/integrations.test.ts +++ b/packages/api/test/routers/integrations.test.ts @@ -7,7 +7,7 @@ import sinon from 'sinon' import { createPubSubClient, PubSubRequestBody, -} from '../../src/datalayer/pubsub' +} from '../../src/pubsub' import { addHighlightToPage } from '../../src/elastic/highlights' import { deletePage } from '../../src/elastic/pages' import { diff --git a/packages/api/test/services/save_email.test.ts b/packages/api/test/services/save_email.test.ts index e58425d80..353baab98 100644 --- a/packages/api/test/services/save_email.test.ts +++ b/packages/api/test/services/save_email.test.ts @@ -2,7 +2,7 @@ import 'mocha' import { expect } from 'chai' import { createTestUser, deleteTestUser } from '../db' import { SaveContext, saveEmail } from '../../src/services/save_email' -import { createPubSubClient } from '../../src/datalayer/pubsub' +import { createPubSubClient } from '../../src/pubsub' import { getPageByParam } from '../../src/elastic/pages' import nock from 'nock' import { User } from '../../src/entity/user' diff --git a/packages/api/test/services/save_newsletter_email.test.ts b/packages/api/test/services/save_newsletter_email.test.ts index 91fe7170c..8a290aa4c 100644 --- a/packages/api/test/services/save_newsletter_email.test.ts +++ b/packages/api/test/services/save_newsletter_email.test.ts @@ -1,7 +1,7 @@ import { expect } from 'chai' import 'mocha' import nock from 'nock' -import { createPubSubClient } from '../../src/datalayer/pubsub' +import { createPubSubClient } from '../../src/pubsub' import { getPageByParam } from '../../src/elastic/pages' import { NewsletterEmail } from '../../src/entity/newsletter_email' import { ReceivedEmail } from '../../src/entity/received_email' diff --git a/packages/api/test/util.ts b/packages/api/test/util.ts index 0b200506b..0fec43c71 100644 --- a/packages/api/test/util.ts +++ b/packages/api/test/util.ts @@ -1,11 +1,11 @@ -import { createApp } from '../src/server' import supertest from 'supertest' import { v4 } from 'uuid' -import { corsConfig } from '../src/utils/corsConfig' +import { createPage } from '../src/elastic/pages' import { ArticleSavingRequestStatus, Label, Page } from '../src/elastic/types' import { PageType } from '../src/generated/graphql' -import { createPubSubClient } from '../src/datalayer/pubsub' -import { createPage } from '../src/elastic/pages' +import { createPubSubClient } from '../src/pubsub' +import { createApp } from '../src/server' +import { corsConfig } from '../src/utils/corsConfig' const { app, apollo } = createApp() export const request = supertest(app) From 06000264d2b81f0cd54c28ded2201471bf98be14 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Thu, 24 Aug 2023 10:07:44 +0800 Subject: [PATCH 011/144] replace saveEmail --- .../api/src/datalayer/links/share_info.ts | 4 +- packages/api/src/entity/library_item.ts | 34 +++-- packages/api/src/entity/subscription.ts | 2 +- packages/api/src/repository/index.ts | 6 +- .../api/src/resolvers/recent_emails/index.ts | 4 +- packages/api/src/routers/svc/emails.ts | 34 +++-- packages/api/src/routers/svc/newsletters.ts | 4 +- packages/api/src/services/labels.ts | 21 ++- packages/api/src/services/library_item.ts | 32 ++++ packages/api/src/services/received_emails.ts | 7 +- packages/api/src/services/save_email.ts | 142 +++++++++++------- .../api/src/services/save_newsletter_email.ts | 62 ++------ packages/api/test/services/save_email.test.ts | 22 ++- .../services/save_newsletter_email.test.ts | 25 +-- .../db/migrations/0118.do.library_item.sql | 10 +- 15 files changed, 217 insertions(+), 192 deletions(-) create mode 100644 packages/api/src/services/library_item.ts diff --git a/packages/api/src/datalayer/links/share_info.ts b/packages/api/src/datalayer/links/share_info.ts index cf1126eaa..13d72451b 100644 --- a/packages/api/src/datalayer/links/share_info.ts +++ b/packages/api/src/datalayer/links/share_info.ts @@ -2,7 +2,6 @@ /* eslint-disable @typescript-eslint/no-unsafe-assignment */ import { Knex } from 'knex' import { LinkShareInfo } from '../../generated/graphql' -import { DataModels } from '../../resolvers/types' import { getPageByParam } from '../../elastic/pages' // once we have links setup properly in the API we will remove this method @@ -10,8 +9,7 @@ import { getPageByParam } from '../../elastic/pages' export const getShareInfoForArticle = async ( kx: Knex, userId: string, - articleId: string, - models: DataModels + articleId: string ): Promise => { // TEMP: because the old API uses articles instead of Links, we are actually // getting an article ID here and need to map it to a link ID. When the API diff --git a/packages/api/src/entity/library_item.ts b/packages/api/src/entity/library_item.ts index 6a8325a9b..bcb0d9ca4 100644 --- a/packages/api/src/entity/library_item.ts +++ b/packages/api/src/entity/library_item.ts @@ -7,6 +7,7 @@ import { ManyToMany, OneToOne, PrimaryGeneratedColumn, + Unique, UpdateDateColumn, } from 'typeorm' import { Label } from './label' @@ -45,10 +46,11 @@ export enum DirectionalityType { RTL = 'RTL', } +@Unique('library_item_user_original_url', ['user', 'originalUrl']) @Entity({ name: 'library_item' }) export class LibraryItem { @PrimaryGeneratedColumn('uuid') - id?: string + id!: string @OneToOne(() => User, { onDelete: 'CASCADE' }) @JoinColumn({ name: 'user_id' }) @@ -58,7 +60,7 @@ export class LibraryItem { enum: LibraryItemState, default: LibraryItemState.Succeeded, }) - state?: LibraryItemState + state!: LibraryItemState @Column('text') originalUrl!: string @@ -114,36 +116,36 @@ export class LibraryItem { @Column('json', { nullable: true }) metadata?: Record | null - @Column('integer', { nullable: true }) - readingProgressLastReadAnchor?: number | null + @Column('integer') + readingProgressLastReadAnchor!: number - @Column('integer', { nullable: true }) - readingProgressHighestReadAnchor?: number | null + @Column('integer') + readingProgressHighestReadAnchor!: number - @Column('real', { nullable: true }) - readingProgressTopPercent?: number | null + @Column('real') + readingProgressTopPercent!: number - @Column('real', { nullable: true }) - readingProgressBottomPercent?: number | null + @Column('real') + readingProgressBottomPercent!: number @Column('text', { nullable: true }) thumbnail?: string | null @Column('enum', { enum: LibraryItemType, default: LibraryItemType.Unknown }) - itemType?: LibraryItemType + itemType!: LibraryItemType @OneToOne(() => UploadFile, { onDelete: 'CASCADE' }) @JoinColumn({ name: 'upload_file_id' }) uploadFile?: UploadFile @Column('enum', { enum: ContentReaderType, default: ContentReaderType.WEB }) - contentReader?: ContentReaderType + contentReader!: ContentReaderType @Column('text', { nullable: true }) originalContent?: string | null - @Column('text', { nullable: true }) - readableContent?: string | null + @Column('text') + readableContent!: string @Column('text', { nullable: true }) modelName?: string | null @@ -158,11 +160,11 @@ export class LibraryItem { @Column('text', { nullable: true }) gcsArchiveId?: string | null - @OneToOne(() => Subscription, { onDelete: 'CASCADE' }) + @OneToOne(() => Subscription, { onDelete: 'CASCADE', eager: true }) @JoinColumn({ name: 'subscription_id' }) subscription?: Subscription - @ManyToMany(() => Label) + @ManyToMany(() => Label, { eager: true }) @JoinTable() labels?: Label[] diff --git a/packages/api/src/entity/subscription.ts b/packages/api/src/entity/subscription.ts index 3fa5a600c..90f7d593b 100644 --- a/packages/api/src/entity/subscription.ts +++ b/packages/api/src/entity/subscription.ts @@ -46,7 +46,7 @@ export class Subscription { unsubscribeHttpUrl?: string @Column('text', { nullable: true }) - icon?: string + icon?: string | null @Column('enum', { enum: SubscriptionType, diff --git a/packages/api/src/repository/index.ts b/packages/api/src/repository/index.ts index d8b7f53d8..94183df78 100644 --- a/packages/api/src/repository/index.ts +++ b/packages/api/src/repository/index.ts @@ -1,5 +1,6 @@ import { EntityManager, EntityTarget, Repository } from 'typeorm' import { AppDataSource } from '../data-source' +import { LibraryItem } from '../entity/library_item' import { Reminder } from '../entity/reminder' import { UploadFile } from '../entity/upload_file' import { User } from '../entity/user' @@ -15,9 +16,12 @@ export const setClaims = async ( } export const getRepository = (entity: EntityTarget): Repository => { - return AppDataSource.getRepository(entity) + return entityManager.getRepository(entity) } +export const entityManager = AppDataSource.createEntityManager() + export const userRepository = getRepository(User) export const uploadFileRepository = getRepository(UploadFile) export const reminderRepository = getRepository(Reminder) +export const libraryItemRepository = getRepository(LibraryItem) diff --git a/packages/api/src/resolvers/recent_emails/index.ts b/packages/api/src/resolvers/recent_emails/index.ts index 906f43e44..203eebdbf 100644 --- a/packages/api/src/resolvers/recent_emails/index.ts +++ b/packages/api/src/resolvers/recent_emails/index.ts @@ -13,7 +13,7 @@ import { } from '../../generated/graphql' import { getRepository } from '../../repository' import { updateReceivedEmail } from '../../services/received_emails' -import { saveNewsletterEmail } from '../../services/save_newsletter_email' +import { saveNewsletter } from '../../services/save_newsletter_email' import { authorized } from '../../utils/helpers' import { generateUniqueUrl, parseEmailAddress } from '../../utils/parser' import { sendEmail } from '../../utils/sendEmail' @@ -103,7 +103,7 @@ export const markEmailAsItemResolver = authorized< } } - const success = await saveNewsletterEmail( + const success = await saveNewsletter( { from: recentEmail.from, email: recentEmail.to, diff --git a/packages/api/src/routers/svc/emails.ts b/packages/api/src/routers/svc/emails.ts index 2bc1698f4..fab0bb213 100644 --- a/packages/api/src/routers/svc/emails.ts +++ b/packages/api/src/routers/svc/emails.ts @@ -1,13 +1,10 @@ import cors from 'cors' import express from 'express' import { env } from '../../env' -import { createPubSubClient, readPushSubscription } from '../../pubsub' +import { readPushSubscription } from '../../pubsub' import { getNewsletterEmail } from '../../services/newsletters' -import { - saveReceivedEmail, - updateReceivedEmail, -} from '../../services/received_emails' -import { saveEmail } from '../../services/save_email' +import { saveReceivedEmail } from '../../services/received_emails' +import { saveNewsletter } from '../../services/save_newsletter_email' import { analytics } from '../../utils/analytics' import { getClaimsByToken } from '../../utils/auth' import { corsConfig } from '../../utils/corsConfig' @@ -73,7 +70,6 @@ export function emailsServiceRouter() { return } const user = newsletterEmail.user - const ctx = { pubsub: createPubSubClient(), uid: user.id } const parsedFrom = parseEmailAddress(data.from) if ( @@ -83,15 +79,21 @@ export function emailsServiceRouter() { ) ) { logger.info('handling as article') - await saveEmail(ctx, { - title: getTitleFromEmailSubject(data.subject), - author: parsedFrom.name, - url: generateUniqueUrl(), - originalContent: data.html || data.text, - }) - - // update received email type - await updateReceivedEmail(data.receivedEmailId, 'article') + const savedNewsletter = await saveNewsletter( + { + title: getTitleFromEmailSubject(data.subject), + author: parsedFrom.name, + url: generateUniqueUrl(), + content: data.html || data.text, + receivedEmailId: data.receivedEmailId, + email: newsletterEmail.address, + }, + newsletterEmail + ) + if (!savedNewsletter) { + logger.info('Failed to save email') + return res.status(500).send('Failed to save email') + } res.status(200).send('Article') return diff --git a/packages/api/src/routers/svc/newsletters.ts b/packages/api/src/routers/svc/newsletters.ts index 2ae6397a3..48038d21a 100644 --- a/packages/api/src/routers/svc/newsletters.ts +++ b/packages/api/src/routers/svc/newsletters.ts @@ -8,7 +8,7 @@ import { import { updateReceivedEmail } from '../../services/received_emails' import { NewsletterMessage, - saveNewsletterEmail, + saveNewsletter, } from '../../services/save_newsletter_email' import { saveUrlFromEmail } from '../../services/save_url' import { getSubscriptionByNameAndUserId } from '../../services/subscriptions' @@ -137,7 +137,7 @@ export function newsletterServiceRouter() { } // save newsletter instead - const result = await saveNewsletterEmail(data, newsletterEmail, saveCtx) + const result = await saveNewsletter(data, newsletterEmail) if (!result) { logger.info('Error creating newsletter link from data', data) diff --git a/packages/api/src/services/labels.ts b/packages/api/src/services/labels.ts index 4544125ef..0b31730c1 100644 --- a/packages/api/src/services/labels.ts +++ b/packages/api/src/services/labels.ts @@ -10,15 +10,22 @@ import { getRepository } from '../repository' import { generateRandomColor } from '../utils/helpers' import { logger } from '../utils/logger' -const INTERNAL_LABELS_IN_LOWERCASE = [ - 'newsletters', - 'favorites', - 'rss', - 'library', -] +const INTERNAL_LABELS_WITH_COLOR = new Map< + string, + { name: string; color: string } +>([ + ['favorites', { name: 'Favorites', color: '#FFD700' }], + ['library', { name: 'Library', color: '#584C42' }], + ['rss', { name: 'RSS', color: '#F26522' }], + ['newsletter', { name: 'Newsletter', color: '#07D2D1' }], +]) + +export const getInternalLabelWithColor = (name: string) => { + return INTERNAL_LABELS_WITH_COLOR.get(name.toLowerCase()) +} const isLabelInternal = (name: string): boolean => { - return INTERNAL_LABELS_IN_LOWERCASE.includes(name.toLowerCase()) + return INTERNAL_LABELS_WITH_COLOR.has(name.toLowerCase()) } const batchGetLabelsFromLinkIds = async ( diff --git a/packages/api/src/services/library_item.ts b/packages/api/src/services/library_item.ts new file mode 100644 index 000000000..db36eb35f --- /dev/null +++ b/packages/api/src/services/library_item.ts @@ -0,0 +1,32 @@ +import { DeepPartial } from 'typeorm' +import { LibraryItem } from '../entity/library_item' +import { entityManager } from '../repository' +import { wordsCount } from '../utils/helpers' +import { logger } from '../utils/logger' + +const MAX_CONTENT_LENGTH = 10 * 1024 * 1024 // 10MB for readable content +const CONTENT_LENGTH_ERROR = 'Your page content is too large to be saved.' + +export const createLibraryItem = async ( + libraryItem: DeepPartial, + em = entityManager +): Promise => { + if ( + libraryItem.readableContent && + libraryItem.readableContent.length > MAX_CONTENT_LENGTH + ) { + logger.info('page content is too large', { + url: libraryItem.originalUrl, + contentLength: libraryItem.readableContent.length, + }) + + libraryItem.readableContent = CONTENT_LENGTH_ERROR + } + + return em.getRepository(LibraryItem).save({ + ...libraryItem, + savedAt: libraryItem.savedAt || new Date(), + wordCount: + libraryItem.wordCount ?? wordsCount(libraryItem.readableContent ?? ''), + }) +} diff --git a/packages/api/src/services/received_emails.ts b/packages/api/src/services/received_emails.ts index a5c9df5c9..ae3db8258 100644 --- a/packages/api/src/services/received_emails.ts +++ b/packages/api/src/services/received_emails.ts @@ -1,5 +1,5 @@ import { ReceivedEmail } from '../entity/received_email' -import { getRepository } from '../repository' +import { entityManager, getRepository } from '../repository' export const saveReceivedEmail = async ( from: string, @@ -23,7 +23,8 @@ export const saveReceivedEmail = async ( export const updateReceivedEmail = async ( id: string, - type: 'article' | 'non-article' + type: 'article' | 'non-article', + em = entityManager ) => { - await getRepository(ReceivedEmail).update(id, { type }) + await em.getRepository(ReceivedEmail).update(id, { type }) } diff --git a/packages/api/src/services/save_email.ts b/packages/api/src/services/save_email.ts index 525ecc3f9..02ccdf12c 100644 --- a/packages/api/src/services/save_email.ts +++ b/packages/api/src/services/save_email.ts @@ -1,10 +1,14 @@ -import { createPage, getPageByParam, updatePage } from '../elastic/pages' -import { ArticleSavingRequestStatus, Page } from '../elastic/types' -import { PubsubClient } from '../pubsub' +import { + LibraryItem, + LibraryItemState, + LibraryItemType, +} from '../entity/library_item' +import { entityManager, libraryItemRepository } from '../repository' import { enqueueThumbnailTask } from '../utils/createTask' import { cleanUrl, generateSlug, + isBase64Image, stringToHash, validatedDate, wordsCount, @@ -12,23 +16,24 @@ import { import { logger } from '../utils/logger' import { FAKE_URL_PREFIX, + fetchFavicon, parsePreparedContent, parseUrlMetadata, } from '../utils/parser' - -export type SaveContext = { - pubsub: PubsubClient - uid: string - refresh?: boolean -} +import { getInternalLabelWithColor } from './labels' +import { createLibraryItem } from './library_item' +import { updateReceivedEmail } from './received_emails' export type SaveEmailInput = { + userId: string originalContent: string url: string title: string author: string unsubMailTo?: string unsubHttpUrl?: string + newsletterEmailId?: string + receivedEmailId: string } const isStubUrl = (url: string): boolean => { @@ -36,9 +41,8 @@ const isStubUrl = (url: string): boolean => { } export const saveEmail = async ( - ctx: SaveContext, input: SaveEmailInput -): Promise => { +): Promise => { const url = input.url const parseResult = await parsePreparedContent( url, @@ -51,68 +55,92 @@ export const saveEmail = async ( null, true ) + const content = parseResult.parsedContent?.content || input.originalContent const slug = generateSlug(input.title) const metadata = isStubUrl(url) ? undefined : await parseUrlMetadata(url) - - const articleToSave: Page = { - id: '', - userId: ctx.uid, - slug, - content, - originalHtml: input.originalContent, - description: metadata?.description || parseResult.parsedContent?.excerpt, - title: input.title, - author: input.author, - url: cleanUrl(parseResult.canonicalUrl || url), - pageType: parseResult.pageType, - hash: stringToHash(content), - image: - metadata?.previewImage || - parseResult.parsedContent?.previewImage || - undefined, - publishedAt: validatedDate( - parseResult.parsedContent?.publishedDate ?? undefined - ), - createdAt: new Date(), - savedAt: new Date(), - readingProgressAnchorIndex: 0, - readingProgressPercent: 0, - subscription: input.author, - state: ArticleSavingRequestStatus.Succeeded, - siteIcon: parseResult.parsedContent?.siteIcon ?? undefined, - siteName: parseResult.parsedContent?.siteName ?? undefined, - wordsCount: wordsCount(content), + const cleanedUrl = cleanUrl(parseResult.canonicalUrl || url) + let siteIcon = parseResult.parsedContent?.siteIcon + if (!siteIcon || isBase64Image(siteIcon)) { + // fetch favicon if not already set or is a base64 image + siteIcon = await fetchFavicon(url) } - const page = await getPageByParam({ - userId: ctx.uid, - url: articleToSave.url, - state: ArticleSavingRequestStatus.Succeeded, + const existingLibraryItem = await libraryItemRepository.findOneBy({ + user: { id: input.userId }, + originalUrl: cleanedUrl, + state: LibraryItemState.Succeeded, }) - if (page) { - const result = await updatePage(page.id, { archivedAt: null }, ctx) - logger.info('updated page from email', result) + if (existingLibraryItem) { + const updatedLibraryItem = await libraryItemRepository.save({ + ...existingLibraryItem, + archivedAt: null, + }) + logger.info('updated page from email', updatedLibraryItem) - return page + return updatedLibraryItem } - const pageId = await createPage(articleToSave, ctx) - if (!pageId) { - logger.info('failed to create new page') + const newsletterLabel = getInternalLabelWithColor('newsletter') - return undefined - } + // start a transaction to create the library item and update the received email + const newLibraryItem = await entityManager.transaction(async (tx) => { + const newLibraryItem = await createLibraryItem( + { + user: { id: input.userId }, + slug, + readableContent: content, + originalContent: input.originalContent, + description: + metadata?.description || parseResult.parsedContent?.excerpt, + title: input.title, + author: input.author, + originalUrl: cleanedUrl, + itemType: parseResult.pageType as unknown as LibraryItemType, + textContentHash: stringToHash(content), + thumbnail: + metadata?.previewImage || + parseResult.parsedContent?.previewImage || + undefined, + publishedAt: validatedDate( + parseResult.parsedContent?.publishedDate ?? undefined + ), + subscription: { + name: input.author, + unsubscribeMailTo: input.unsubMailTo, + unsubscribeHttpUrl: input.unsubHttpUrl, + user: { id: input.userId }, + newsletterEmail: { id: input.newsletterEmailId }, + icon: siteIcon, + lastFetchedAt: new Date(), + }, + state: LibraryItemState.Succeeded, + siteIcon, + siteName: parseResult.parsedContent?.siteName ?? undefined, + wordCount: wordsCount(content), + labels: [ + { + ...newsletterLabel, + internal: true, + user: { id: input.userId }, + }, + ], + }, + tx + ) + + await updateReceivedEmail(input.receivedEmailId, 'article', tx) + + return newLibraryItem + }) // create a task to update thumbnail and pre-cache all images try { - const taskId = await enqueueThumbnailTask(ctx.uid, slug) + const taskId = await enqueueThumbnailTask(input.userId, slug) logger.info('Created thumbnail task', { taskId }) } catch (e) { logger.error('Failed to create thumbnail task', e) } - articleToSave.id = pageId - - return articleToSave + return newLibraryItem } diff --git a/packages/api/src/services/save_newsletter_email.ts b/packages/api/src/services/save_newsletter_email.ts index 1bae6d5d2..4768ee1dd 100644 --- a/packages/api/src/services/save_newsletter_email.ts +++ b/packages/api/src/services/save_newsletter_email.ts @@ -1,19 +1,12 @@ import { MulticastMessage } from 'firebase-admin/messaging' -import { updatePage } from '../elastic/pages' import { Page } from '../elastic/types' import { NewsletterEmail } from '../entity/newsletter_email' import { UserDeviceToken } from '../entity/user_device_tokens' import { env } from '../env' import { ContentReader } from '../generated/graphql' -import { createPubSubClient } from '../pubsub' import { analytics } from '../utils/analytics' -import { isBase64Image } from '../utils/helpers' import { logger } from '../utils/logger' -import { fetchFavicon } from '../utils/parser' -import { addLabelToPage } from './labels' -import { SaveContext, saveEmail, SaveEmailInput } from './save_email' -import { saveSubscription } from './subscriptions' -import { getDeviceTokensByUserId } from './user_device_tokens' +import { saveEmail, SaveEmailInput } from './save_email' export interface NewsletterMessage { email: string @@ -29,10 +22,9 @@ export interface NewsletterMessage { // Returns true if the link was created successfully. Can still fail to // send the push but that is ok and we wont retry in that case. -export const saveNewsletterEmail = async ( +export const saveNewsletter = async ( data: NewsletterMessage, - newsletterEmail: NewsletterEmail, - ctx?: SaveContext + newsletterEmail: NewsletterEmail ): Promise => { analytics.track({ userId: newsletterEmail.user.id, @@ -50,58 +42,30 @@ export const saveNewsletterEmail = async ( return false } - const saveCtx = ctx || { - pubsub: createPubSubClient(), - uid: newsletterEmail.user.id, - } const input: SaveEmailInput = { + userId: newsletterEmail.user.id, url: data.url, originalContent: data.content, title: data.title, author: data.author, unsubMailTo: data.unsubMailTo, unsubHttpUrl: data.unsubHttpUrl, + newsletterEmailId: newsletterEmail.id, + receivedEmailId: data.receivedEmailId, } - const page = await saveEmail(saveCtx, input) - if (!page) { + const savedLibraryItem = await saveEmail(input) + if (!savedLibraryItem) { logger.info('newsletter not created:', input.title) return false } - let icon = page.siteIcon - if (!icon || isBase64Image(icon)) { - // fetch favicon if not already set or is a base64 image - icon = await fetchFavicon(page.url) - if (icon) { - await updatePage(page.id, { siteIcon: icon }, saveCtx) - } - } - - const subscriptionId = await saveSubscription({ - userId: newsletterEmail.user.id, - name: data.author, - newsletterEmail, - unsubscribeMailTo: data.unsubMailTo, - unsubscribeHttpUrl: data.unsubHttpUrl, - icon, - }) - logger.info('subscription saved', subscriptionId) - - // adds newsletters label to page - const result = await addLabelToPage(saveCtx, page.id, { - name: 'Newsletter', - color: '#07D2D1', - }) - - logger.info('newsletter label added', { result }) - // sends push notification - const deviceTokens = await getDeviceTokensByUserId(newsletterEmail.user.id) - if (!deviceTokens) { - logger.info('Device tokens not set:', newsletterEmail.user.id) - return true - } + // const deviceTokens = await getDeviceTokensByUserId(newsletterEmail.user.id) + // if (!deviceTokens) { + // logger.info('Device tokens not set:', newsletterEmail.user.id) + // return true + // } // const multicastMessage = messageForLink(page, deviceTokens) // await sendMulticastPushNotifications( diff --git a/packages/api/test/services/save_email.test.ts b/packages/api/test/services/save_email.test.ts index 353baab98..7f9689d8e 100644 --- a/packages/api/test/services/save_email.test.ts +++ b/packages/api/test/services/save_email.test.ts @@ -1,11 +1,10 @@ -import 'mocha' import { expect } from 'chai' -import { createTestUser, deleteTestUser } from '../db' -import { SaveContext, saveEmail } from '../../src/services/save_email' -import { createPubSubClient } from '../../src/pubsub' -import { getPageByParam } from '../../src/elastic/pages' +import 'mocha' import nock from 'nock' +import { getPageByParam } from '../../src/elastic/pages' import { User } from '../../src/entity/user' +import { saveEmail } from '../../src/services/save_email' +import { createTestUser, deleteTestUser } from '../db' describe('saveEmail', () => { const fakeContent = 'fake content' @@ -30,26 +29,25 @@ describe('saveEmail', () => { const url = 'https://blog.omnivore.app/fake-url' const title = 'fake title' const author = 'fake author' - const ctx: SaveContext = { - pubsub: createPubSubClient(), - uid: user.id, - refresh: true, - } - await saveEmail(ctx, { + await saveEmail({ originalContent: `${fakeContent}`, url, title, author, + userId: user.id, + receivedEmailId: 'fakeId', }) // This ensures row level security doesnt prevent // saving the same URL - const secondResult = await saveEmail(ctx, { + const secondResult = await saveEmail({ originalContent: `${fakeContent}`, url, title, author, + userId: user.id, + receivedEmailId: 'fakeId', }) expect(secondResult).to.not.be.undefined diff --git a/packages/api/test/services/save_newsletter_email.test.ts b/packages/api/test/services/save_newsletter_email.test.ts index 8a290aa4c..f2a0f36f2 100644 --- a/packages/api/test/services/save_newsletter_email.test.ts +++ b/packages/api/test/services/save_newsletter_email.test.ts @@ -1,7 +1,6 @@ import { expect } from 'chai' import 'mocha' import nock from 'nock' -import { createPubSubClient } from '../../src/pubsub' import { getPageByParam } from '../../src/elastic/pages' import { NewsletterEmail } from '../../src/entity/newsletter_email' import { ReceivedEmail } from '../../src/entity/received_email' @@ -9,8 +8,7 @@ import { Subscription } from '../../src/entity/subscription' import { User } from '../../src/entity/user' import { getRepository } from '../../src/repository' import { createNewsletterEmail } from '../../src/services/newsletters' -import { SaveContext } from '../../src/services/save_email' -import { saveNewsletterEmail } from '../../src/services/save_newsletter_email' +import { saveNewsletter } from '../../src/services/save_newsletter_email' import { createTestUser, deleteTestUser } from '../db' describe('saveNewsletterEmail', () => { @@ -22,17 +20,11 @@ describe('saveNewsletterEmail', () => { let user: User let newsletterEmail: NewsletterEmail - let ctx: SaveContext let receivedEmail: ReceivedEmail before(async () => { user = await createTestUser('fakeUser') newsletterEmail = await createNewsletterEmail(user.id) - ctx = { - pubsub: createPubSubClient(), - refresh: true, - uid: user.id, - } receivedEmail = await getRepository(ReceivedEmail).save({ user: { id: user.id }, from, @@ -53,7 +45,7 @@ describe('saveNewsletterEmail', () => { nock('https://blog.omnivore.app').head('/fake-url').reply(200) const url = 'https://blog.omnivore.app/fake-url' - await saveNewsletterEmail( + await saveNewsletter( { from, email: newsletterEmail.address, @@ -64,8 +56,7 @@ describe('saveNewsletterEmail', () => { receivedEmailId: receivedEmail.id, unsubHttpUrl: 'https://blog.omnivore.app/unsubscribe', }, - newsletterEmail, - ctx + newsletterEmail ) const page = await getPageByParam({ userId: user.id, url }) @@ -90,7 +81,7 @@ describe('saveNewsletterEmail', () => { color: '#07D2D1', } - await saveNewsletterEmail( + await saveNewsletter( { email: newsletterEmail.address, content: `fake content 2`, @@ -100,8 +91,7 @@ describe('saveNewsletterEmail', () => { from, receivedEmailId: receivedEmail.id, }, - newsletterEmail, - ctx + newsletterEmail ) const page = await getPageByParam({ userId: user.id, url }) @@ -112,7 +102,7 @@ describe('saveNewsletterEmail', () => { const url = 'https://omnivore.app/no_url?q=no-unsubscribe' nock('https://omnivore.app').get('/no_url?q=no-unsubscribe').reply(404) - await saveNewsletterEmail( + await saveNewsletter( { email: newsletterEmail.address, content: `fake content 2`, @@ -122,8 +112,7 @@ describe('saveNewsletterEmail', () => { from, receivedEmailId: receivedEmail.id, }, - newsletterEmail, - ctx + newsletterEmail ) const subscriptions = await getRepository(Subscription).findBy({ diff --git a/packages/db/migrations/0118.do.library_item.sql b/packages/db/migrations/0118.do.library_item.sql index 5d8e6a459..3b2f6e6df 100755 --- a/packages/db/migrations/0118.do.library_item.sql +++ b/packages/db/migrations/0118.do.library_item.sql @@ -33,16 +33,16 @@ CREATE TABLE omnivore.library_item ( site_name text, site_icon text, metadata JSON, - reading_progress_last_read_anchor integer, - reading_progress_highest_read_anchor integer, - reading_progress_top_percent real, - reading_progress_bottom_percent real, + reading_progress_last_read_anchor integer NOT NULL DEFAULT 0, + reading_progress_highest_read_anchor integer NOT NULL DEFAULT 0, + reading_progress_top_percent real NOT NULL DEFAULT 0, + reading_progress_bottom_percent real NOT NULL DEFAULT 0, thumbnail text, item_type library_item_type NOT NULL DEFAULT 'UNKNOWN', upload_file_id uuid REFERENCES omnivore.upload_files ON DELETE CASCADE, content_reader content_reader_type NOT NULL DEFAULT 'WEB', original_content text, - readable_content text, + readable_content text NOT NULL DEFAULT '', content_tsv tsvector, site_tsv tsvector, title_tsv tsvector, From 5f0e1af28578e476b7268f242be7c1692e5f3367 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Thu, 24 Aug 2023 11:35:35 +0800 Subject: [PATCH 012/144] update highlight entity --- packages/api/src/entity/highlight.ts | 37 ++++++++++++++++++- packages/api/src/entity/library_item.ts | 6 ++- .../db/migrations/0119.do.entity_labels.sql | 14 +++++++ .../0119.do.library_item_labels.sql | 13 ------- .../db/migrations/0119.undo.entity_labels.sql | 9 +++++ .../0119.undo.library_item_labels.sql | 9 ----- .../migrations/0121.do.update_highlight.sql | 20 ++++++++++ .../migrations/0121.undo.update_highlight.sql | 16 ++++++++ 8 files changed, 100 insertions(+), 24 deletions(-) create mode 100755 packages/db/migrations/0119.do.entity_labels.sql delete mode 100755 packages/db/migrations/0119.do.library_item_labels.sql create mode 100755 packages/db/migrations/0119.undo.entity_labels.sql delete mode 100755 packages/db/migrations/0119.undo.library_item_labels.sql create mode 100755 packages/db/migrations/0121.do.update_highlight.sql create mode 100755 packages/db/migrations/0121.undo.update_highlight.sql diff --git a/packages/api/src/entity/highlight.ts b/packages/api/src/entity/highlight.ts index 6ef44d9af..dc9282190 100644 --- a/packages/api/src/entity/highlight.ts +++ b/packages/api/src/entity/highlight.ts @@ -3,12 +3,21 @@ import { CreateDateColumn, Entity, JoinColumn, + JoinTable, + ManyToMany, OneToOne, PrimaryGeneratedColumn, UpdateDateColumn, } from 'typeorm' -import { User } from './user' +import { Label } from './label' import { Page } from './page' +import { User } from './user' + +export enum HighlightType { + Highlight = 'HIGHLIGHT', + Redaction = 'REDACTION', // allowing people to remove text from the page + Note = 'NOTE', // allowing people to add a note at the document level +} @Entity({ name: 'highlight' }) export class Highlight { @@ -52,4 +61,30 @@ export class Highlight { @Column('timestamp') sharedAt?: Date + + @Column('real') + highlightPositionPercent!: number + + @Column('integer') + highlightPositionAnchorIndex!: number + + @Column('enum', { + enum: HighlightType, + default: HighlightType.Highlight, + }) + highlightType!: HighlightType + + @Column('text', { nullable: true }) + html?: string | null + + @Column('text', { nullable: true }) + color?: string | null + + @ManyToMany(() => Label, { eager: true }) + @JoinTable({ + name: 'entity_labels', + joinColumn: { name: 'highlight_id' }, + inverseJoinColumn: { name: 'label_id' }, + }) + labels?: Label[] } diff --git a/packages/api/src/entity/library_item.ts b/packages/api/src/entity/library_item.ts index bcb0d9ca4..406e48945 100644 --- a/packages/api/src/entity/library_item.ts +++ b/packages/api/src/entity/library_item.ts @@ -165,7 +165,11 @@ export class LibraryItem { subscription?: Subscription @ManyToMany(() => Label, { eager: true }) - @JoinTable() + @JoinTable({ + name: 'entity_labels', + joinColumn: { name: 'library_item_id' }, + inverseJoinColumn: { name: 'label_id' }, + }) labels?: Label[] @Column('enum', { enum: DirectionalityType, default: DirectionalityType.LTR }) diff --git a/packages/db/migrations/0119.do.entity_labels.sql b/packages/db/migrations/0119.do.entity_labels.sql new file mode 100755 index 000000000..cae9aa63a --- /dev/null +++ b/packages/db/migrations/0119.do.entity_labels.sql @@ -0,0 +1,14 @@ +-- Type: DO +-- Name: entity_labels +-- Description: Create table entity_labels + +BEGIN; + +CREATE TABLE omnivore.entity_labels ( + library_item_id uuid REFERENCES omnivore.library_item(id) ON DELETE CASCADE, + highlight_id uuid REFERENCES omnivore.highlight(id) ON DELETE CASCADE, + label_id uuid NOT NULL REFERENCES omnivore.labels(id) ON DELETE CASCADE, + PRIMARY KEY (library_item_id, highlight_id, label_id) +); + +COMMIT; diff --git a/packages/db/migrations/0119.do.library_item_labels.sql b/packages/db/migrations/0119.do.library_item_labels.sql deleted file mode 100755 index f9bcba00a..000000000 --- a/packages/db/migrations/0119.do.library_item_labels.sql +++ /dev/null @@ -1,13 +0,0 @@ --- Type: DO --- Name: library_item_labels --- Description: Create table library_item_labels - -BEGIN; - -CREATE TABLE omnivore.library_item_labels ( - library_item_id uuid NOT NULL REFERENCES omnivore.library_item(id) ON DELETE CASCADE, - label_id uuid NOT NULL REFERENCES omnivore.labels(id) ON DELETE CASCADE, - PRIMARY KEY (library_item_id, label_id) -); - -COMMIT; diff --git a/packages/db/migrations/0119.undo.entity_labels.sql b/packages/db/migrations/0119.undo.entity_labels.sql new file mode 100755 index 000000000..35d1f1576 --- /dev/null +++ b/packages/db/migrations/0119.undo.entity_labels.sql @@ -0,0 +1,9 @@ +-- Type: UNDO +-- Name: entity_labels +-- Description: Create table entity_labels + +BEGIN; + +DROP TABLE omnivore.entity_labels; + +COMMIT; diff --git a/packages/db/migrations/0119.undo.library_item_labels.sql b/packages/db/migrations/0119.undo.library_item_labels.sql deleted file mode 100755 index ec7ac7cb7..000000000 --- a/packages/db/migrations/0119.undo.library_item_labels.sql +++ /dev/null @@ -1,9 +0,0 @@ --- Type: UNDO --- Name: library_item_labels --- Description: Create table library_item_labels - -BEGIN; - -DROP TABLE omnivore.library_item_labels; - -COMMIT; diff --git a/packages/db/migrations/0121.do.update_highlight.sql b/packages/db/migrations/0121.do.update_highlight.sql new file mode 100755 index 000000000..111b9eb41 --- /dev/null +++ b/packages/db/migrations/0121.do.update_highlight.sql @@ -0,0 +1,20 @@ +-- Type: DO +-- Name: update_highlight +-- Description: Add fields to highlight table + +BEGIN; + +CREATE TYPE highlight_type AS ENUM ( + 'HIGHLIGHT', + 'REDACTION', + 'NOTE' +); + +ALTER TABLE omnivore.highlight + ADD COLUMN highlight_position_percent real NOT NULL DEFAULT 0, + ADD COLUMN highlight_position_anchor_index integer NOT NULL DEFAULT 0, + ADD COLUMN highlight_type highlight_type NOT NULL DEFAULT 'HIGHLIGHT', + ADD COLUMN color text, + ADD COLUMN html text; + +COMMIT; diff --git a/packages/db/migrations/0121.undo.update_highlight.sql b/packages/db/migrations/0121.undo.update_highlight.sql new file mode 100755 index 000000000..a223f8680 --- /dev/null +++ b/packages/db/migrations/0121.undo.update_highlight.sql @@ -0,0 +1,16 @@ +-- Type: UNDO +-- Name: update_highlight +-- Description: Add fields to highlight table + +BEGIN; + +ALTER TABLE omnivore.highlight + DROP COLUMN html, + DROP COLUMN color, + DROP COLUMN highlight_type, + DROP COLUMN highlight_position_anchor_index, + DROP COLUMN highlight_position_percent; + +DROP TYPE highlight_type; + +COMMIT; From 0b309b7a455961f852be9902344901f718d2744b Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Thu, 24 Aug 2023 13:13:12 +0800 Subject: [PATCH 013/144] add recommendation db table and entity --- packages/api/src/entity/highlight.ts | 14 ++-- packages/api/src/entity/library_item.ts | 22 +++++- packages/api/src/entity/recommendation.ts | 30 ++++++++ packages/api/src/services/library_item.ts | 74 ++++++++++++++++++- .../migrations/0121.do.update_highlight.sql | 5 +- .../migrations/0121.undo.update_highlight.sql | 3 + .../db/migrations/0122.do.recommendation.sql | 15 ++++ .../migrations/0122.undo.recommendation.sql | 9 +++ 8 files changed, 159 insertions(+), 13 deletions(-) create mode 100644 packages/api/src/entity/recommendation.ts create mode 100755 packages/db/migrations/0122.do.recommendation.sql create mode 100755 packages/db/migrations/0122.undo.recommendation.sql diff --git a/packages/api/src/entity/highlight.ts b/packages/api/src/entity/highlight.ts index dc9282190..22572e7fe 100644 --- a/packages/api/src/entity/highlight.ts +++ b/packages/api/src/entity/highlight.ts @@ -5,12 +5,12 @@ import { JoinColumn, JoinTable, ManyToMany, - OneToOne, + ManyToOne, PrimaryGeneratedColumn, UpdateDateColumn, } from 'typeorm' import { Label } from './label' -import { Page } from './page' +import { LibraryItem } from './library_item' import { User } from './user' export enum HighlightType { @@ -27,13 +27,13 @@ export class Highlight { @Column({ type: 'varchar', length: 14 }) shortId!: string - @OneToOne(() => User) + @ManyToOne(() => User, { onDelete: 'CASCADE' }) @JoinColumn({ name: 'user_id' }) user!: User - @OneToOne(() => Page) - @JoinColumn({ name: 'article_id' }) - page!: Page + @ManyToOne(() => LibraryItem, { onDelete: 'CASCADE' }) + @JoinColumn({ name: 'library_item_id' }) + libraryItem!: LibraryItem @Column('text') quote!: string @@ -80,7 +80,7 @@ export class Highlight { @Column('text', { nullable: true }) color?: string | null - @ManyToMany(() => Label, { eager: true }) + @ManyToMany(() => Label) @JoinTable({ name: 'entity_labels', joinColumn: { name: 'highlight_id' }, diff --git a/packages/api/src/entity/library_item.ts b/packages/api/src/entity/library_item.ts index 406e48945..e5001a0ec 100644 --- a/packages/api/src/entity/library_item.ts +++ b/packages/api/src/entity/library_item.ts @@ -5,12 +5,15 @@ import { JoinColumn, JoinTable, ManyToMany, + ManyToOne, + OneToMany, OneToOne, PrimaryGeneratedColumn, Unique, UpdateDateColumn, } from 'typeorm' import { Label } from './label' +import { Recommendation } from './recommendation' import { Subscription } from './subscription' import { UploadFile } from './upload_file' import { User } from './user' @@ -52,7 +55,7 @@ export class LibraryItem { @PrimaryGeneratedColumn('uuid') id!: string - @OneToOne(() => User, { onDelete: 'CASCADE' }) + @ManyToOne(() => User, { onDelete: 'CASCADE' }) @JoinColumn({ name: 'user_id' }) user!: User @@ -160,11 +163,11 @@ export class LibraryItem { @Column('text', { nullable: true }) gcsArchiveId?: string | null - @OneToOne(() => Subscription, { onDelete: 'CASCADE', eager: true }) + @OneToOne(() => Subscription, { cascade: true }) @JoinColumn({ name: 'subscription_id' }) subscription?: Subscription - @ManyToMany(() => Label, { eager: true }) + @ManyToMany(() => Label, { cascade: true }) @JoinTable({ name: 'entity_labels', joinColumn: { name: 'library_item_id' }, @@ -172,6 +175,17 @@ export class LibraryItem { }) labels?: Label[] + @OneToMany( + () => Recommendation, + (recommendation) => recommendation.libraryItem + ) + @JoinTable({ + name: 'recommendation', + joinColumn: { name: 'library_item_id' }, + inverseJoinColumn: { name: 'id' }, + }) + recommendations?: Recommendation[] + @Column('enum', { enum: DirectionalityType, default: DirectionalityType.LTR }) - directionality?: DirectionalityType + directionality!: DirectionalityType } diff --git a/packages/api/src/entity/recommendation.ts b/packages/api/src/entity/recommendation.ts new file mode 100644 index 000000000..d42c84cc5 --- /dev/null +++ b/packages/api/src/entity/recommendation.ts @@ -0,0 +1,30 @@ +import { + Column, + CreateDateColumn, + Entity, + JoinColumn, + ManyToOne, + PrimaryGeneratedColumn, +} from 'typeorm' +import { LibraryItem } from './library_item' +import { User } from './user' + +@Entity() +export class Recommendation { + @PrimaryGeneratedColumn('uuid') + id!: string + + @ManyToOne(() => User, { onDelete: 'CASCADE' }) + @JoinColumn({ name: 'recommender_id' }) + recommender!: User + + @ManyToOne(() => LibraryItem, { onDelete: 'CASCADE' }) + @JoinColumn({ name: 'library_item_id' }) + libraryItem!: LibraryItem + + @Column('text', { nullable: true }) + note?: string | null + + @CreateDateColumn() + createdAt!: Date +} diff --git a/packages/api/src/services/library_item.ts b/packages/api/src/services/library_item.ts index db36eb35f..cb5268ca8 100644 --- a/packages/api/src/services/library_item.ts +++ b/packages/api/src/services/library_item.ts @@ -1,12 +1,84 @@ import { DeepPartial } from 'typeorm' -import { LibraryItem } from '../entity/library_item' +import { Highlight } from '../entity/highlight' +import { Label } from '../entity/label' +import { + LibraryItem, + LibraryItemState, + LibraryItemType, +} from '../entity/library_item' import { entityManager } from '../repository' import { wordsCount } from '../utils/helpers' import { logger } from '../utils/logger' +import { + DateFilter, + FieldFilter, + HasFilter, + InFilter, + LabelFilter, + NoFilter, + ReadFilter, + SortParams, +} from '../utils/search' const MAX_CONTENT_LENGTH = 10 * 1024 * 1024 // 10MB for readable content const CONTENT_LENGTH_ERROR = 'Your page content is too large to be saved.' +export interface PageSearchArgs { + from?: number + size?: number + sort?: SortParams + query?: string + inFilter?: InFilter + readFilter?: ReadFilter + typeFilter?: LibraryItemType + labelFilters?: LabelFilter[] + hasFilters?: HasFilter[] + dateFilters?: DateFilter[] + termFilters?: FieldFilter[] + matchFilters?: FieldFilter[] + includePending?: boolean | null + includeDeleted?: boolean + ids?: string[] + recommendedBy?: string + includeContent?: boolean + noFilters?: NoFilter[] + siteName?: string +} + +export interface SearchItem { + annotation?: string | null + author?: string | null + createdAt: Date + description?: string | null + id: string + image?: string | null + pageId?: string + pageType: LibraryItemType + publishedAt?: Date + quote?: string | null + shortId?: string | null + slug: string + title: string + uploadFileId?: string | null + url: string + archivedAt?: Date | null + readingProgressTopPercent?: number + readingProgressPercent: number + readingProgressAnchorIndex: number + userId: string + state?: LibraryItemState + language?: string + readAt?: Date + savedAt: Date + updatedAt?: Date + labels?: Label[] + highlights?: Highlight[] + wordsCount?: number + siteName?: string + siteIcon?: string + content?: string +} + export const createLibraryItem = async ( libraryItem: DeepPartial, em = entityManager diff --git a/packages/db/migrations/0121.do.update_highlight.sql b/packages/db/migrations/0121.do.update_highlight.sql index 111b9eb41..aaad3f80a 100755 --- a/packages/db/migrations/0121.do.update_highlight.sql +++ b/packages/db/migrations/0121.do.update_highlight.sql @@ -11,10 +11,13 @@ CREATE TYPE highlight_type AS ENUM ( ); ALTER TABLE omnivore.highlight + ADD COLUMN library_item_id uuid NOT NULL REFERENCES omnivore.library_item ON DELETE CASCADE, ADD COLUMN highlight_position_percent real NOT NULL DEFAULT 0, ADD COLUMN highlight_position_anchor_index integer NOT NULL DEFAULT 0, ADD COLUMN highlight_type highlight_type NOT NULL DEFAULT 'HIGHLIGHT', ADD COLUMN color text, - ADD COLUMN html text; + ADD COLUMN html text, + DROP COLUMN article_id, + DROP COLUMN elastic_page_id; COMMIT; diff --git a/packages/db/migrations/0121.undo.update_highlight.sql b/packages/db/migrations/0121.undo.update_highlight.sql index a223f8680..8f8d435d7 100755 --- a/packages/db/migrations/0121.undo.update_highlight.sql +++ b/packages/db/migrations/0121.undo.update_highlight.sql @@ -5,6 +5,9 @@ BEGIN; ALTER TABLE omnivore.highlight + ADD COLUMN article_id uuid, + ADD COLUMN elastic_page_id uuid, + DROP COLUMN library_item_id, DROP COLUMN html, DROP COLUMN color, DROP COLUMN highlight_type, diff --git a/packages/db/migrations/0122.do.recommendation.sql b/packages/db/migrations/0122.do.recommendation.sql new file mode 100755 index 000000000..6b480aae9 --- /dev/null +++ b/packages/db/migrations/0122.do.recommendation.sql @@ -0,0 +1,15 @@ +-- Type: DO +-- Name: recommendation +-- Description: Create recommendation table + +BEGIN; + +CREATE TABLE omnivore.recommendation ( + id uuid PRIMARY KEY DEFAULT uuid_generate_v1mc(), + library_item_id uuid NOT NULL REFERENCES omnivore.library_item ON DELETE CASCADE, + recommender_id uuid NOT NULL REFERENCES omnivore.user ON DELETE CASCADE, + note text, + created_at timestamptz NOT NULL DEFAULT current_timestamp +); + +COMMIT; diff --git a/packages/db/migrations/0122.undo.recommendation.sql b/packages/db/migrations/0122.undo.recommendation.sql new file mode 100755 index 000000000..83f9dfd66 --- /dev/null +++ b/packages/db/migrations/0122.undo.recommendation.sql @@ -0,0 +1,9 @@ +-- Type: UNDO +-- Name: recommendation +-- Description: Create recommendation table + +BEGIN; + +DROP TABLE omnivore.recommendation; + +COMMIT; From 751df68175411584efee8f6d268c674b0a149a51 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Thu, 24 Aug 2023 21:47:13 +0800 Subject: [PATCH 014/144] implement search function --- packages/api/src/services/library_item.ts | 142 +++++++++++++++++++++- packages/api/src/utils/search.ts | 16 +-- 2 files changed, 149 insertions(+), 9 deletions(-) diff --git a/packages/api/src/services/library_item.ts b/packages/api/src/services/library_item.ts index cb5268ca8..f098332f7 100644 --- a/packages/api/src/services/library_item.ts +++ b/packages/api/src/services/library_item.ts @@ -1,4 +1,4 @@ -import { DeepPartial } from 'typeorm' +import { DeepPartial, SelectQueryBuilder } from 'typeorm' import { Highlight } from '../entity/highlight' import { Label } from '../entity/label' import { @@ -15,8 +15,11 @@ import { HasFilter, InFilter, LabelFilter, + LabelFilterType, NoFilter, ReadFilter, + SortBy, + SortOrder, SortParams, } from '../utils/search' @@ -102,3 +105,140 @@ export const createLibraryItem = async ( libraryItem.wordCount ?? wordsCount(libraryItem.readableContent ?? ''), }) } + +const buildWhereClause = ( + queryBuilder: SelectQueryBuilder, + userId: string, + args: PageSearchArgs +) => { + if (args.query) { + queryBuilder.andWhere(`tsv @@ websearch_to_tsquery(:query)`, { + query: args.query, + }) + } + + if (args.typeFilter) { + queryBuilder.andWhere(`library_item.item_type = :typeFilter`, { + typeFilter: args.typeFilter, + }) + } + + if (args.inFilter !== InFilter.ALL) { + switch (args.inFilter) { + case InFilter.INBOX: + queryBuilder.andWhere(`library_item.archived_at IS NULL`) + break + case InFilter.ARCHIVE: + queryBuilder.andWhere(`library_item.archived_at IS NOT NULL`) + break + case InFilter.TRASH: + // return only deleted pages within 14 days + queryBuilder.andWhere( + `library_item.state = :state AND deleted_at >= now() - interval '14 days'`, + { + state: LibraryItemState.Deleted, + } + ) + break + } + } + + if (args.readFilter !== ReadFilter.ALL) { + switch (args.readFilter) { + case ReadFilter.READ: + queryBuilder.andWhere('library_item.reading_progress_top_percent >= 98') + break + case ReadFilter.UNREAD: + queryBuilder.andWhere('library_item.reading_progress_top_percent < 98') + break + } + } + + if (args.hasFilters && args.hasFilters.length > 0) { + args.hasFilters.forEach((filter) => { + switch (filter) { + case HasFilter.HIGHLIGHTS: + queryBuilder.andWhere(`library_item.highlights IS NOT NULL`) + break + } + }) + } + + if (args.labelFilters && args.labelFilters.length > 0) { + const includeLabels = args.labelFilters?.filter( + (filter) => filter.type === LabelFilterType.INCLUDE + ) + const excludeLabels = args.labelFilters?.filter( + (filter) => filter.type === LabelFilterType.EXCLUDE + ) + + if (includeLabels && includeLabels.length > 0) { + queryBuilder.andWhere( + `library_item.id IN (SELECT library_item_id FROM library_item_label WHERE name IN (:...includeLabels))`, + { + includeLabels, + } + ) + } + + if (excludeLabels && excludeLabels.length > 0) { + queryBuilder.andWhere( + `library_item.id NOT IN (SELECT library_item_id FROM library_item_label WHERE name IN (:...excludeLabels))`, + { + excludeLabels, + } + ) + } + } + + if (args.dateFilters && args.dateFilters.length > 0) { + args.dateFilters.forEach((filter) => { + queryBuilder.andWhere( + `library_item.${filter.field} between :startDate and :endDate`, + { + startDate: filter.startDate ?? new Date(0), + endDate: filter.endDate ?? new Date(), + } + ) + }) + } + + if (args.termFilters && args.termFilters.length > 0) { + args.termFilters.forEach((filter) => { + queryBuilder.andWhere(`library_item.${filter.field} = :value`, { + value: filter.value, + }) + }) + } +} + +export const searchLibraryItems = async ( + args: PageSearchArgs, + userId: string +): Promise<[LibraryItem[], number] | null> => { + const { from = 0, size = 10, sort } = args + + // default order is descending + const sortOrder = sort?.order || SortOrder.DESCENDING + // default sort by saved_at + const sortField = sort?.by || SortBy.SAVED + + const queryBuilder = entityManager + .createQueryBuilder(LibraryItem, 'library_item') + .select('library_item.*') + .where('library_item.user_id = :userId', { userId }) + + // build the where clause + buildWhereClause(queryBuilder, userId, args) + + // add pagination and sorting + const libraryItems = await queryBuilder + .orderBy(`omnivore.library_item.${sortField}`, sortOrder) + .offset(from) + .limit(size) + .getMany() + + const count = await queryBuilder.getCount() + + return [libraryItems, count] +} diff --git a/packages/api/src/utils/search.ts b/packages/api/src/utils/search.ts index ccd488bb5..6087e1289 100644 --- a/packages/api/src/utils/search.ts +++ b/packages/api/src/utils/search.ts @@ -65,18 +65,18 @@ export interface DateFilter { } export enum SortBy { - SAVED = 'savedAt', - UPDATED = 'updatedAt', + SAVED = 'saved_at', + UPDATED = 'updated_at', SCORE = '_score', - PUBLISHED = 'publishedAt', - READ = 'readAt', - LISTENED = 'listenedAt', - WORDS_COUNT = 'wordsCount', + PUBLISHED = 'published_at', + READ = 'read_at', + LISTENED = 'listened_at', + WORDS_COUNT = 'word_count', } export enum SortOrder { - ASCENDING = 'asc', - DESCENDING = 'desc', + ASCENDING = 'ASC', + DESCENDING = 'DESC', } export interface SortParams { From 648a16615b8570d63fe7e53aa3f0cceb9529ae54 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Mon, 28 Aug 2023 18:20:44 +0800 Subject: [PATCH 015/144] update search query --- packages/api/src/elastic/pages.ts | 11 ++- packages/api/src/entity/library_item.ts | 9 ++ packages/api/src/resolvers/article/index.ts | 46 +++------- .../api/src/resolvers/upload_files/index.ts | 12 +-- packages/api/src/routers/svc/integrations.ts | 4 +- packages/api/src/services/library_item.ts | 89 ++++++++++++++----- packages/api/src/utils/search.ts | 26 +++--- packages/api/test/elastic/pages.test.ts | 24 ++--- packages/api/test/routers/auth.test.ts | 18 ++-- .../db/migrations/0118.do.library_item.sql | 13 ++- .../db/migrations/0118.undo.library_item.sql | 1 + 11 files changed, 155 insertions(+), 98 deletions(-) diff --git a/packages/api/src/elastic/pages.ts b/packages/api/src/elastic/pages.ts index 30e069652..7cb293075 100644 --- a/packages/api/src/elastic/pages.ts +++ b/packages/api/src/elastic/pages.ts @@ -226,8 +226,15 @@ const appendHasFilters = ( }, }) break - case HasFilter.SHARED_AT: - builder = builder.query('exists', { field: 'sharedAt' }) + case HasFilter.LABELS: + builder = builder.query('nested', { + path: 'labels', + query: { + exists: { + field: 'labels', + }, + }, + }) break } }) diff --git a/packages/api/src/entity/library_item.ts b/packages/api/src/entity/library_item.ts index e5001a0ec..7b1740481 100644 --- a/packages/api/src/entity/library_item.ts +++ b/packages/api/src/entity/library_item.ts @@ -12,6 +12,7 @@ import { Unique, UpdateDateColumn, } from 'typeorm' +import { Highlight } from './highlight' import { Label } from './label' import { Recommendation } from './recommendation' import { Subscription } from './subscription' @@ -188,4 +189,12 @@ export class LibraryItem { @Column('enum', { enum: DirectionalityType, default: DirectionalityType.LTR }) directionality!: DirectionalityType + + @OneToMany(() => Highlight, (highlight) => highlight.libraryItem) + @JoinTable({ + name: 'highlight', + joinColumn: { name: 'library_item_id' }, + inverseJoinColumn: { name: 'id' }, + }) + highlights?: Highlight[] } diff --git a/packages/api/src/resolvers/article/index.ts b/packages/api/src/resolvers/article/index.ts index 054fe1b56..df3392641 100644 --- a/packages/api/src/resolvers/article/index.ts +++ b/packages/api/src/resolvers/article/index.ts @@ -5,22 +5,7 @@ /* eslint-disable @typescript-eslint/no-floating-promises */ import { Readability } from '@omnivore/readability' import graphqlFields from 'graphql-fields' -import { searchHighlights } from '../../elastic/highlights' -import { - createPage, - getPageByParam, - searchAsYouType, - searchPages, - updatePage, - updatePages, -} from '../../elastic/pages' -import { - ArticleSavingRequestStatus, - Page, - PageType, - SearchItem as SearchItemData, -} from '../../elastic/types' -import { getRepository } from '../../repository' +import { LibraryItemType } from '../../entity/library_item' import { UploadFile } from '../../entity/upload_file' import { User } from '../../entity/user' import { env } from '../../env' @@ -28,6 +13,7 @@ import { Article, ArticleError, ArticleErrorCode, + ArticleSavingRequestStatus, ArticlesError, ArticleSuccess, BulkActionError, @@ -57,7 +43,6 @@ import { SaveArticleReadingProgressSuccess, SearchError, SearchErrorCode, - SearchItem, SearchSuccess, SetBookmarkArticleError, SetBookmarkArticleErrorCode, @@ -76,12 +61,14 @@ import { UpdatesSinceErrorCode, UpdatesSinceSuccess, } from '../../generated/graphql' +import { getRepository } from '../../repository' import { createPageSaveRequest } from '../../services/create_page_save_request' import { addLabelToPage, createLabels, getLabelsByIds, } from '../../services/labels' +import { searchLibraryItems } from '../../services/library_item' import { setFileUploadComplete } from '../../services/save_file' import { parsedContentToPage } from '../../services/save_page' import { traceAs } from '../../tracing' @@ -115,7 +102,7 @@ import { makeStorageFilePublic, } from '../../utils/uploads' import { WithDataSourcesContext } from '../types' -import { pageTypeForContentType } from '../upload_files' +import { itemTypeForContentType } from '../upload_files' export enum ArticleFormat { Markdown = 'markdown', @@ -220,7 +207,7 @@ export const createArticleResolver = authorized< let userArticleUrl: string | null = null let uploadFileHash = null let domContent = null - let pageType = PageType.Unknown + let pageType = LibraryItemType.Unknown const DUMMY_RESPONSE = { user, @@ -233,7 +220,7 @@ export const createArticleResolver = authorized< content: '', description: '', title: '', - pageType: PageType.Unknown, + pageType: LibraryItemType.Unknown, contentReader: ContentReader.Web, author: '', url, @@ -280,7 +267,7 @@ export const createArticleResolver = authorized< uploadFileHash = uploadFileDetails.md5Hash userArticleUrl = uploadFileDetails.fileUrl canonicalUrl = uploadFile.url - pageType = pageTypeForContentType(uploadFile.contentType) + pageType = itemTypeForContentType(uploadFile.contentType) title = titleForFilePath(uploadFile.url) } else if ( source !== 'puppeteer-parse' && @@ -298,7 +285,7 @@ export const createArticleResolver = authorized< parsedContent = parseResults.parsedContent canonicalUrl = parseResults.canonicalUrl domContent = parseResults.domContent - pageType = parseResults.pageType + pageType = parseResults.pageType as unknown as LibraryItemType } else if (!preparedDocument?.document) { // We have a URL but no document, so we try to send this to puppeteer // and return a dummy response. @@ -520,7 +507,7 @@ export const getArticlesResolver = authorized< }, }) - const [pages, totalCount] = (await searchPages( + const [pages, totalCount] = (await searchLibraryItems( { from: Number(startCursor), size: first + 1, // fetch one more item to get next cursor @@ -551,13 +538,9 @@ export const getArticlesResolver = authorized< pages.pop() } - const edges = pages.map((a) => { + const edges = pages.map((node) => { return { - node: { - ...a, - image: a.image && createImageProxyUrl(a.image, 320, 320), - isArchived: !!a.archivedAt, - }, + node, cursor: endCursor, } }) @@ -589,7 +572,6 @@ export type SetShareArticleSuccessPartial = Merge< } > -// TODO: not implemented yet // export const setShareArticleResolver = authorized< // SetShareArticleSuccessPartial, // SetShareArticleError, @@ -916,7 +898,7 @@ export const searchResolver = authorized< )) || [[], 0] } else { // otherwise, search pages - ;[results, totalCount] = (await searchPages( + ;[results, totalCount] = (await searchLibraryItems( { from: Number(startCursor), size: first + 1, // fetch one more item to get next cursor @@ -1031,7 +1013,7 @@ export const updatesSinceResolver = authorized< const startCursor = after || '' const size = first || 10 const startDate = new Date(since) - const [pages, totalCount] = (await searchPages( + const [pages, totalCount] = (await searchLibraryItems( { from: Number(startCursor), size: size + 1, // fetch one more item to get next cursor diff --git a/packages/api/src/resolvers/upload_files/index.ts b/packages/api/src/resolvers/upload_files/index.ts index e3b921201..8035480bc 100644 --- a/packages/api/src/resolvers/upload_files/index.ts +++ b/packages/api/src/resolvers/upload_files/index.ts @@ -2,7 +2,7 @@ import normalizeUrl from 'normalize-url' import path from 'path' import { createPage, getPageByParam, updatePage } from '../../elastic/pages' -import { PageType } from '../../elastic/types' +import { LibraryItemType } from '../../entity/library_item' import { UploadFile } from '../../entity/upload_file' import { env } from '../../env' import { @@ -28,11 +28,13 @@ const isFileUrl = (url: string): boolean => { return parsedUrl.protocol == 'file:' } -export const pageTypeForContentType = (contentType: string): PageType => { +export const itemTypeForContentType = ( + contentType: string +): LibraryItemType => { if (contentType == 'application/epub+zip') { - return PageType.Book + return LibraryItemType.Book } - return PageType.File + return LibraryItemType.File } export const uploadFileRequestResolver = authorized< @@ -151,7 +153,7 @@ export const uploadFileRequestResolver = authorized< title: title, hash: uploadFilePathName, content: '', - pageType: pageTypeForContentType(input.contentType), + pageType: itemTypeForContentType(input.contentType), uploadFileId: uploadFileData.id, slug: generateSlug(uploadFilePathName), createdAt: new Date(), diff --git a/packages/api/src/routers/svc/integrations.ts b/packages/api/src/routers/svc/integrations.ts index 349792c9c..0f5084e7a 100644 --- a/packages/api/src/routers/svc/integrations.ts +++ b/packages/api/src/routers/svc/integrations.ts @@ -5,7 +5,7 @@ import { stringify } from 'csv-stringify' import express from 'express' import { DateTime } from 'luxon' import { v4 as uuidv4 } from 'uuid' -import { getPageById, searchPages } from '../../elastic/pages' +import { getPageById, searchLibraryItems } from '../../elastic/pages' import { Page } from '../../elastic/types' import { Integration, IntegrationType } from '../../entity/integration' import { EntityType, readPushSubscription } from '../../pubsub' @@ -133,7 +133,7 @@ export function integrationsServiceRouter() { const dateFilters: DateFilter[] = [] syncedAt && dateFilters.push({ field: 'updatedAt', startDate: syncedAt }) - ;[pages, count] = (await searchPages( + ;[pages, count] = (await searchLibraryItems( { from: after, size, dateFilters }, userId )) as [Page[], number] diff --git a/packages/api/src/services/library_item.ts b/packages/api/src/services/library_item.ts index f098332f7..aaddab419 100644 --- a/packages/api/src/services/library_item.ts +++ b/packages/api/src/services/library_item.ts @@ -108,17 +108,18 @@ export const createLibraryItem = async ( const buildWhereClause = ( queryBuilder: SelectQueryBuilder, - userId: string, args: PageSearchArgs ) => { if (args.query) { - queryBuilder.andWhere(`tsv @@ websearch_to_tsquery(:query)`, { - query: args.query, - }) + queryBuilder + .addSelect('ts_rank_cd(library_item.search_tsv, query)', 'rank') + .addFrom("websearch_to_tsquery('english', ':query')", 'query') + .andWhere('query @@ library_item.search_tsv') + .setParameter('query', args.query) } if (args.typeFilter) { - queryBuilder.andWhere(`library_item.item_type = :typeFilter`, { + queryBuilder.andWhere('library_item.item_type = :typeFilter', { typeFilter: args.typeFilter, }) } @@ -126,18 +127,15 @@ const buildWhereClause = ( if (args.inFilter !== InFilter.ALL) { switch (args.inFilter) { case InFilter.INBOX: - queryBuilder.andWhere(`library_item.archived_at IS NULL`) + queryBuilder.andWhere('library_item.archived_at IS NULL') break case InFilter.ARCHIVE: - queryBuilder.andWhere(`library_item.archived_at IS NOT NULL`) + queryBuilder.andWhere('library_item.archived_at IS NOT NULL') break case InFilter.TRASH: // return only deleted pages within 14 days queryBuilder.andWhere( - `library_item.state = :state AND deleted_at >= now() - interval '14 days'`, - { - state: LibraryItemState.Deleted, - } + "library_item.state = 'DELETED' AND library_item.deleted_at >= now() - interval '14 days'" ) break } @@ -158,7 +156,12 @@ const buildWhereClause = ( args.hasFilters.forEach((filter) => { switch (filter) { case HasFilter.HIGHLIGHTS: - queryBuilder.andWhere(`library_item.highlights IS NOT NULL`) + queryBuilder.andWhere( + 'library_item.highlight_annotations IS NOT NULL' + ) + break + case HasFilter.LABELS: + queryBuilder.andWhere('library_item.label_names IS NOT NULL') break } }) @@ -173,17 +176,19 @@ const buildWhereClause = ( ) if (includeLabels && includeLabels.length > 0) { - queryBuilder.andWhere( - `library_item.id IN (SELECT library_item_id FROM library_item_label WHERE name IN (:...includeLabels))`, - { - includeLabels, - } - ) + includeLabels.forEach((includeLabel) => { + queryBuilder.andWhere( + 'library_item.label_names @> ARRAY[:...includeLabels]::text[] OR library_item.highlight_labels @> ARRAY[:...includeLabels]::text[]', + { + includeLabels: includeLabel.labels, + } + ) + }) } if (excludeLabels && excludeLabels.length > 0) { queryBuilder.andWhere( - `library_item.id NOT IN (SELECT library_item_id FROM library_item_label WHERE name IN (:...excludeLabels))`, + 'NOT library_item.label_names && ARRAY[:...excludeLabels]::text[] AND NOT library_item.highlight_labels && ARRAY[:...excludeLabels]::text[]', { excludeLabels, } @@ -210,6 +215,39 @@ const buildWhereClause = ( }) }) } + + if (args.matchFilters && args.matchFilters.length > 0) { + args.matchFilters.forEach((filter) => { + queryBuilder.andWhere( + `websearch_to_tsquery('english', ':query') @@ library_item.${filter.field}_tsv`, + { + query: filter.value, + } + ) + }) + } + + if (args.ids && args.ids.length > 0) { + queryBuilder.andWhere('library_item.id IN (:...ids)', { ids: args.ids }) + } + + if (!args.includePending) { + queryBuilder.andWhere('library_item.state != :state', { + state: LibraryItemState.Processing, + }) + } + + if (!args.includeDeleted && args.inFilter !== InFilter.TRASH) { + queryBuilder.andWhere('library_item.state != :state', { + state: LibraryItemState.Deleted, + }) + } + + if (args.noFilters) { + args.noFilters.forEach((filter) => { + queryBuilder.andWhere(`library_item.${filter.field} IS NULL`) + }) + } } export const searchLibraryItems = async ( @@ -225,15 +263,16 @@ export const searchLibraryItems = async ( const queryBuilder = entityManager .createQueryBuilder(LibraryItem, 'library_item') - .select('library_item.*') + .leftJoinAndSelect('library_item.labels', 'labels') + .leftJoinAndSelect('library_item.highlights', 'highlights') .where('library_item.user_id = :userId', { userId }) // build the where clause - buildWhereClause(queryBuilder, userId, args) + buildWhereClause(queryBuilder, args) // add pagination and sorting const libraryItems = await queryBuilder - .orderBy(`omnivore.library_item.${sortField}`, sortOrder) + .orderBy(`library_item.${sortField}`, sortOrder) .offset(from) .limit(size) .getMany() @@ -242,3 +281,9 @@ export const searchLibraryItems = async ( return [libraryItems, count] } + +export const libraryItemToSearchItem = ( + libraryItem: LibraryItem, + userId: string +): SearchItem => { + \ No newline at end of file diff --git a/packages/api/src/utils/search.ts b/packages/api/src/utils/search.ts index 6087e1289..741ecf4bb 100644 --- a/packages/api/src/utils/search.ts +++ b/packages/api/src/utils/search.ts @@ -9,7 +9,7 @@ import { SearchParserKeyWordOffset, SearchParserTextOffset, } from 'search-query-parser' -import { PageType } from '../generated/graphql' +import { LibraryItemType } from '../entity/library_item' export enum ReadFilter { ALL, @@ -30,7 +30,7 @@ export interface SearchFilter { query: string | undefined inFilter: InFilter readFilter: ReadFilter - typeFilter?: PageType + typeFilter?: LibraryItemType labelFilters: LabelFilter[] sortParams?: SortParams hasFilters: HasFilter[] @@ -55,7 +55,7 @@ export type LabelFilter = { export enum HasFilter { HIGHLIGHTS, - SHARED_AT, + LABELS, } export interface DateFilter { @@ -134,27 +134,27 @@ const parseInFilter = ( return query ? InFilter.ALL : InFilter.INBOX } -const parseTypeFilter = (str: string | undefined): PageType | undefined => { +const parseTypeFilter = ( + str: string | undefined +): LibraryItemType | undefined => { if (str === undefined) { return undefined } switch (str.toLowerCase()) { case 'article': - return PageType.Article + return LibraryItemType.Article case 'book': - return PageType.Book + return LibraryItemType.Book case 'pdf': case 'file': - return PageType.File + return LibraryItemType.File case 'profile': - return PageType.Profile + return LibraryItemType.Profile case 'website': - return PageType.Website + return LibraryItemType.Website case 'unknown': - return PageType.Unknown - case 'highlights': - return PageType.Highlights + return LibraryItemType.Unknown } return undefined } @@ -230,6 +230,8 @@ const parseHasFilter = (str?: string): HasFilter | undefined => { switch (str.toUpperCase()) { case 'HIGHLIGHTS': return HasFilter.HIGHLIGHTS + case 'LABELS': + return HasFilter.LABELS } } diff --git a/packages/api/test/elastic/pages.test.ts b/packages/api/test/elastic/pages.test.ts index be6db6591..baa01d616 100644 --- a/packages/api/test/elastic/pages.test.ts +++ b/packages/api/test/elastic/pages.test.ts @@ -1,13 +1,5 @@ -import 'mocha' import { expect } from 'chai' -import { InFilter, ReadFilter } from '../../src/utils/search' -import { - ArticleSavingRequestStatus, - Page, - PageContext, - PageType, -} from '../../src/elastic/types' -import { createPubSubClient } from '../../src/pubsub' +import 'mocha' import { countByCreatedAt, createPage, @@ -16,9 +8,17 @@ import { getPageById, getPageByParam, searchAsYouType, - searchPages, + searchLibraryItems, updatePage, } from '../../src/elastic/pages' +import { + ArticleSavingRequestStatus, + Page, + PageContext, + PageType, +} from '../../src/elastic/types' +import { createPubSubClient } from '../../src/pubsub' +import { InFilter, ReadFilter } from '../../src/utils/search' import { createTestElasticPage } from '../util' describe('pages in elastic', () => { @@ -126,7 +126,7 @@ describe('pages in elastic', () => { }) }) - describe('searchPages', () => { + describe('searchLibraryItems', () => { let page: Page before(async () => { @@ -140,7 +140,7 @@ describe('pages in elastic', () => { }) it('searches pages', async () => { - const searchResults = await searchPages( + const searchResults = await searchLibraryItems( { dateFilters: [], hasFilters: [], diff --git a/packages/api/test/routers/auth.test.ts b/packages/api/test/routers/auth.test.ts index 4a17d58ba..5559f1b5e 100644 --- a/packages/api/test/routers/auth.test.ts +++ b/packages/api/test/routers/auth.test.ts @@ -3,7 +3,7 @@ import chai, { expect } from 'chai' import sinon from 'sinon' import sinonChai from 'sinon-chai' import supertest from 'supertest' -import { searchPages } from '../../src/elastic/pages' +import { searchLibraryItems } from '../../src/elastic/pages' import { StatusType, User } from '../../src/entity/user' import { getRepository } from '../../src/repository' import { AuthProvider } from '../../src/routers/auth/auth_types' @@ -605,10 +605,10 @@ describe('auth router', () => { 'web' ).expect(200) const user = await getRepository(User).findOneBy({ name }) - const [popularReads, count] = (await searchPages({}, user?.id!)) || [ - [], - 0, - ] + const [popularReads, count] = (await searchLibraryItems( + {}, + user?.id! + )) || [[], 0] expect(count).to.eql(3) }) @@ -629,10 +629,10 @@ describe('auth router', () => { 'ios' ).expect(200) const user = await getRepository(User).findOneBy({ name }) - const [popularReads, count] = (await searchPages({}, user?.id!)) || [ - [], - 0, - ] + const [popularReads, count] = (await searchLibraryItems( + {}, + user?.id! + )) || [[], 0] expect(count).to.eql(4) }) diff --git a/packages/db/migrations/0118.do.library_item.sql b/packages/db/migrations/0118.do.library_item.sql index 3b2f6e6df..f65d2112a 100755 --- a/packages/db/migrations/0118.do.library_item.sql +++ b/packages/db/migrations/0118.do.library_item.sql @@ -55,6 +55,11 @@ CREATE TABLE omnivore.library_item ( gcs_archive_id text, directionality directionality_type NOT NULL DEFAULT 'LTR', subscription_id uuid REFERENCES omnivore.subscriptions ON DELETE CASCADE, + label_names text[], -- array of label names of the item + highlight_labels text[], -- array of label names of the item's highlights + highlight_annotations text[], -- array of highlight annotations of the item + note text, + note_tsv tsvector, UNIQUE (user_id, original_url) ); @@ -66,6 +71,7 @@ CREATE INDEX library_item_title_tsv_idx ON omnivore.library_item USING GIN (titl CREATE INDEX library_item_author_tsv_idx ON omnivore.library_item USING GIN (author_tsv); CREATE INDEX library_item_description_tsv_idx ON omnivore.library_item USING GIN (description_tsv); CREATE INDEX library_item_search_tsv_idx ON omnivore.library_item USING GIN (search_tsv); +CREATE INDEX library_item_note_tsv_idx ON omnivore.library_item USING GIN (note_tsv); CREATE OR REPLACE FUNCTION update_library_item_tsv() RETURNS trigger AS $$ begin @@ -74,15 +80,18 @@ begin new.title_tsv := to_tsvector('pg_catalog.english', coalesce(new.title, '')); new.author_tsv := to_tsvector('pg_catalog.english', coalesce(new.author, '')); new.description_tsv := to_tsvector('pg_catalog.english', coalesce(new.description, '')); + -- note_tsv is generated by both note and highlight_annotations + new.note_tsv := to_tsvector('pg_catalog.english', coalesce(new.note, '')) || to_tsvector('pg_catalog.english', string_agg(new.highlight_annotations, ' ')); new.search_tsv := setweight(new.title_tsv, 'A') || setweight(new.author_tsv, 'A') || setweight(new.site_tsv, 'A') || setweight(new.description_tsv, 'A') || -- full hostname (eg www.omnivore.app) - setweight(to_tsvector('pg_catalog.english', coalesce(regexp_replace(new.url, '^((http[s]?):\/)?\/?([^:\/\s]+)((\/\w+)*\/)([\w\-\.]+[^#?\s]+)(.*)?(#[\w\-]+)?$', '\3'), '')), 'A') || + setweight(to_tsvector('pg_catalog.english', coalesce(regexp_replace(new.original_url, '^((http[s]?):\/)?\/?([^:\/\s]+)((\/\w+)*\/)([\w\-\.]+[^#?\s]+)(.*)?(#[\w\-]+)?$', '\3'), '')), 'A') || -- secondary hostname (eg omnivore) - setweight(to_tsvector('pg_catalog.english', coalesce(regexp_replace(new.url, '^((http[s]?):\/)?\/?(.*\.)?([^:\/\s]+)(\..*)((\/+)*\/)?([\w\-\.]+[^#?\s]+)(.*)?(#[\w\-]+)?$', '\4'), '')), 'A') || + setweight(to_tsvector('pg_catalog.english', coalesce(regexp_replace(new.original_url, '^((http[s]?):\/)?\/?(.*\.)?([^:\/\s]+)(\..*)((\/+)*\/)?([\w\-\.]+[^#?\s]+)(.*)?(#[\w\-]+)?$', '\4'), '')), 'A') || + setweight(to_tsvector('pg_catalog.english', new.note_tsv), 'A') || setweight(new.content_tsv, 'B'); return new; end diff --git a/packages/db/migrations/0118.undo.library_item.sql b/packages/db/migrations/0118.undo.library_item.sql index 202b831dd..57e2a2e43 100755 --- a/packages/db/migrations/0118.undo.library_item.sql +++ b/packages/db/migrations/0118.undo.library_item.sql @@ -7,6 +7,7 @@ BEGIN; DROP TRIGGER library_item_tsv_update ON omnivore.library_item; DROP FUNCTION update_library_item_tsv(); +DROP INDEX omnivore.library_item_note_tsv_idx; DROP INDEX omnivore.library_item_search_tsv_idx; DROP INDEX omnivore.library_item_description_tsv_idx; DROP INDEX omnivore.library_item_author_tsv_idx; From 5dce78eedbb17a08adb100554a7cb522768018be Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Wed, 30 Aug 2023 10:25:26 +0800 Subject: [PATCH 016/144] update entity subscriber --- packages/api/src/entity/highlight.ts | 2 +- packages/api/src/entity/library_item.ts | 12 ++ packages/api/src/entity/link_label.ts | 26 ---- .../api/src/events/highlight_subscriber.ts | 103 ++++++++++++++++ .../api/src/events/library_item_subscriber.ts | 82 +++++++++++++ packages/api/src/repository/index.ts | 45 ++++++- packages/api/src/resolvers/article/index.ts | 4 +- packages/api/src/resolvers/save/index.ts | 8 +- packages/api/src/services/create_user.ts | 4 + packages/api/src/services/library_item.ts | 6 - packages/api/src/services/save_page.ts | 111 ++++++++---------- 11 files changed, 303 insertions(+), 100 deletions(-) delete mode 100644 packages/api/src/entity/link_label.ts create mode 100644 packages/api/src/events/highlight_subscriber.ts create mode 100644 packages/api/src/events/library_item_subscriber.ts diff --git a/packages/api/src/entity/highlight.ts b/packages/api/src/entity/highlight.ts index 22572e7fe..65cea59fb 100644 --- a/packages/api/src/entity/highlight.ts +++ b/packages/api/src/entity/highlight.ts @@ -80,7 +80,7 @@ export class Highlight { @Column('text', { nullable: true }) color?: string | null - @ManyToMany(() => Label) + @ManyToMany(() => Label, { cascade: true }) @JoinTable({ name: 'entity_labels', joinColumn: { name: 'highlight_id' }, diff --git a/packages/api/src/entity/library_item.ts b/packages/api/src/entity/library_item.ts index 7b1740481..87f02eff1 100644 --- a/packages/api/src/entity/library_item.ts +++ b/packages/api/src/entity/library_item.ts @@ -197,4 +197,16 @@ export class LibraryItem { inverseJoinColumn: { name: 'id' }, }) highlights?: Highlight[] + + @Column('text', { nullable: true }) + labelNames?: string[] | null + + @Column('text', { nullable: true }) + highlightLabels?: string[] | null + + @Column('text', { nullable: true }) + highlightAnnotations?: string[] | null + + @Column('text', { nullable: true }) + note?: string | null } diff --git a/packages/api/src/entity/link_label.ts b/packages/api/src/entity/link_label.ts deleted file mode 100644 index 744393c3c..000000000 --- a/packages/api/src/entity/link_label.ts +++ /dev/null @@ -1,26 +0,0 @@ -import { - CreateDateColumn, - Entity, - JoinColumn, - ManyToOne, - PrimaryGeneratedColumn, -} from 'typeorm' -import { Link } from './link' -import { Label } from './label' - -@Entity({ name: 'link_labels' }) -export class LinkLabel { - @PrimaryGeneratedColumn('uuid') - id!: string - - @ManyToOne(() => Link) - @JoinColumn({ name: 'link_id' }) - link!: Link - - @ManyToOne(() => Label) - @JoinColumn({ name: 'label_id' }) - label!: Label - - @CreateDateColumn() - createdAt!: Date -} diff --git a/packages/api/src/events/highlight_subscriber.ts b/packages/api/src/events/highlight_subscriber.ts new file mode 100644 index 000000000..5da980eda --- /dev/null +++ b/packages/api/src/events/highlight_subscriber.ts @@ -0,0 +1,103 @@ +import { + EntityManager, + EntitySubscriberInterface, + EventSubscriber, + InsertEvent, + ObjectLiteral, + RemoveEvent, + UpdateEvent, +} from 'typeorm' +import { Highlight } from '../entity/highlight' +import { Label } from '../entity/label' +import { LibraryItem } from '../entity/library_item' +import { createPubSubClient, EntityType } from '../pubsub' + +@EventSubscriber() +export class HighlightSubscriber + implements EntitySubscriberInterface +{ + private readonly pubsubClient = createPubSubClient() + + async updateLibraryItem(manager: EntityManager, libraryItemId: string) { + // get all the highlights belonging to the library_item + const highlights = await manager.getRepository(Highlight).find({ + where: { libraryItem: { id: libraryItemId } }, + relations: { + labels: true, + }, + }) + + const highlightLabels: string[] = [] + const highlightAnnotations: string[] = [] + + // for each highlight, add the lowercased label names to highlight_labels + // and the annotation to highlight_annotations + highlights.forEach((highlight) => { + highlight.labels && + highlightLabels.push( + ...highlight.labels.map((label) => label.name.toLowerCase()) + ) + highlightAnnotations.push(highlight.annotation || '') + }) + + // update highlight_labels and highlight_annotations on library_item + await manager.update(LibraryItem, libraryItemId, { + highlightAnnotations, + highlightLabels, + }) + } + + listenTo() { + return Highlight + } + + async afterInsert(event: InsertEvent): Promise { + await this.updateLibraryItem(event.manager, event.entity.libraryItem.id) + + await this.pubsubClient.entityCreated( + EntityType.HIGHLIGHT, + event.entity, + event.entity.libraryItem.user.id + ) + } + + async afterUpdate(event: UpdateEvent): Promise { + if (event.entity) { + await this.updateLibraryItem( + event.manager, + event.databaseEntity.libraryItem.id + ) + + // publish update event + await this.pubsubClient.entityUpdated( + EntityType.HIGHLIGHT, + { ...event.entity, libraryItem: event.databaseEntity.libraryItem }, + event.databaseEntity.libraryItem.user.id + ) + + // publish label added event if a label was added + if (event.entity.labels) { + await this.pubsubClient.entityCreated