From 1f4f68530f854361d96a5fc5451f9122eb59b344 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Wed, 3 Apr 2024 16:59:50 +0800 Subject: [PATCH 01/45] add tests for filters --- .../api/test/services/library_item.test.ts | 140 ++++++++++++++++++ 1 file changed, 140 insertions(+) create mode 100644 packages/api/test/services/library_item.test.ts diff --git a/packages/api/test/services/library_item.test.ts b/packages/api/test/services/library_item.test.ts new file mode 100644 index 000000000..c567db29d --- /dev/null +++ b/packages/api/test/services/library_item.test.ts @@ -0,0 +1,140 @@ +import { expect } from 'chai' +import 'mocha' +import { filterItemEvents } from '../../src/services/library_item' +import { parseSearchQuery } from '../../src/utils/search' + +describe('filterItemEvents', () => { + it('returns events if there are quotation marks in the subscription name', () => { + const query = 'subscription:"Best \\\"Omnivore\\\""' + const ast = parseSearchQuery(query) + const events = [ + { + subscription: 'Best "Omnivore"', + }, + ] + const result = filterItemEvents(ast, events) + expect(result).to.eql(events) + }) + + it('returns events if subscription name equals ignore case', () => { + const query = 'subscription:substack' + const ast = parseSearchQuery(query) + const events = [ + { + subscription: 'Substack', + }, + ] + const result = filterItemEvents(ast, events) + expect(result).to.eql(events) + }) + + it('returns events if site name equals ignore case', () => { + const query = 'site:youtube' + const ast = parseSearchQuery(query) + const events = [ + { + siteName: 'YouTube', + }, + ] + const result = filterItemEvents(ast, events) + expect(result).to.eql(events) + }) + + it('returns events if site name contains the search query', () => { + const query = 'site:standard' + const ast = parseSearchQuery(query) + const events = [ + { + siteName: 'Der Standard', + }, + ] + const result = filterItemEvents(ast, events) + expect(result).to.eql(events) + }) + + it('returns events if domain name contains the search query', () => { + const query = 'site:stackoverflow.com' + const ast = parseSearchQuery(query) + const events = [ + { + siteName: 'Stack Overflow', + originalUrl: 'https://stackoverflow.com/questions/123', + }, + ] + const result = filterItemEvents(ast, events) + expect(result).to.eql(events) + }) + + it('returns events if top level domain matches', () => { + const query = 'site:".com"' + const ast = parseSearchQuery(query) + const events = [ + { + siteName: 'Stack Overflow', + originalUrl: 'https://stackoverflow.com/questions/123', + }, + ] + const result = filterItemEvents(ast, events) + expect(result).to.eql(events) + }) + + it('returns events if labels match the search query', () => { + const query = 'label:foo' + const ast = parseSearchQuery(query) + const events = [ + { + labelNames: ['foo'], + }, + ] + const result = filterItemEvents(ast, events) + expect(result).to.eql(events) + }) + + it('returns events if labels contain quotation marks', () => { + const query = 'label:"foo \\\"bar\\\""' + const ast = parseSearchQuery(query) + const events = [ + { + labelNames: ['foo "bar"'], + }, + ] + const result = filterItemEvents(ast, events) + expect(result).to.eql(events) + }) + + it('returns events if labels contain space', () => { + const query = 'label:"foo bar"' + const ast = parseSearchQuery(query) + const events = [ + { + labelNames: ['foo bar'], + }, + ] + const result = filterItemEvents(ast, events) + expect(result).to.eql(events) + }) + + it('returns events if labels match the search query ignore case', () => { + const query = 'label:Foo' + const ast = parseSearchQuery(query) + const events = [ + { + labelNames: ['foo'], + }, + ] + const result = filterItemEvents(ast, events) + expect(result).to.eql(events) + }) + + it('returns events if labels match the search query with multiple labels', () => { + const query = 'label:foo,bar' + const ast = parseSearchQuery(query) + const events = [ + { + labelNames: ['foo', 'bar'], + }, + ] + const result = filterItemEvents(ast, events) + expect(result).to.eql(events) + }) +}) From 95556f659708fa65fe29fd0575ff75be219d4312 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Wed, 3 Apr 2024 17:00:21 +0800 Subject: [PATCH 02/45] escape quotes for subscription search on Web --- .../templates/navMenu/LibraryLegacyMenu.tsx | 33 ++++++----- .../templates/navMenu/LibraryMenu.tsx | 57 ++++++++++--------- packages/web/pages/settings/shortcuts.tsx | 44 ++++++-------- packages/web/utils/helper.ts | 1 + 4 files changed, 67 insertions(+), 68 deletions(-) create mode 100644 packages/web/utils/helper.ts diff --git a/packages/web/components/templates/navMenu/LibraryLegacyMenu.tsx b/packages/web/components/templates/navMenu/LibraryLegacyMenu.tsx index 7fa8dcf8a..20a31c0be 100644 --- a/packages/web/components/templates/navMenu/LibraryLegacyMenu.tsx +++ b/packages/web/components/templates/navMenu/LibraryLegacyMenu.tsx @@ -1,24 +1,25 @@ +import { useRegisterActions } from 'kbar' +import Link from 'next/link' +import { Circle, X } from 'phosphor-react' import { ReactNode, useEffect, useMemo, useRef } from 'react' -import { StyledText } from '../../elements/StyledText' -import { Box, HStack, SpanBox, VStack } from '../../elements/LayoutPrimitives' -import { Button } from '../../elements/Button' -import { Circle, NewspaperClipping, X } from 'phosphor-react' +import { usePersistedState } from '../../../lib/hooks/usePersistedState' +import { Label } from '../../../lib/networking/fragments/labelFragment' +import { SavedSearch } from '../../../lib/networking/fragments/savedSearchFragment' +import { useGetLabelsQuery } from '../../../lib/networking/queries/useGetLabelsQuery' +import { useGetSavedSearchQuery } from '../../../lib/networking/queries/useGetSavedSearchQuery' import { Subscription, SubscriptionType, useGetSubscriptionsQuery, } from '../../../lib/networking/queries/useGetSubscriptionsQuery' -import { useGetLabelsQuery } from '../../../lib/networking/queries/useGetLabelsQuery' -import { Label } from '../../../lib/networking/fragments/labelFragment' -import { theme } from '../../tokens/stitches.config' -import { useRegisterActions } from 'kbar' -import { LogoBox } from '../../elements/LogoBox' -import { usePersistedState } from '../../../lib/hooks/usePersistedState' -import { useGetSavedSearchQuery } from '../../../lib/networking/queries/useGetSavedSearchQuery' -import { SavedSearch } from '../../../lib/networking/fragments/savedSearchFragment' +import { escapeQuotes } from '../../../utils/helper' +import { Button } from '../../elements/Button' import { ToggleCaretDownIcon } from '../../elements/icons/ToggleCaretDownIcon' -import Link from 'next/link' import { ToggleCaretRightIcon } from '../../elements/icons/ToggleCaretRightIcon' +import { Box, HStack, SpanBox, VStack } from '../../elements/LayoutPrimitives' +import { LogoBox } from '../../elements/LogoBox' +import { StyledText } from '../../elements/StyledText' +import { theme } from '../../tokens/stitches.config' import { NavMenuFooter } from './Footer' export const LIBRARY_LEFT_MENU_WIDTH = '275px' @@ -255,7 +256,7 @@ function Subscriptions( name: name, keywords: '*' + name, perform: () => { - props.applySearchQuery(`subscription:\"${name}\"`) + props.applySearchQuery(`subscription:\"${escapeQuotes(name)}\"`) }, } }), @@ -291,7 +292,9 @@ function Subscriptions( return ( diff --git a/packages/web/components/templates/navMenu/LibraryMenu.tsx b/packages/web/components/templates/navMenu/LibraryMenu.tsx index 7a48e621b..0da615ab0 100644 --- a/packages/web/components/templates/navMenu/LibraryMenu.tsx +++ b/packages/web/components/templates/navMenu/LibraryMenu.tsx @@ -1,36 +1,37 @@ -import { ReactNode, useEffect, useMemo, useRef, useState } from "react" -import { StyledText } from '../../elements/StyledText' -import { Box, HStack, SpanBox, VStack } from '../../elements/LayoutPrimitives' -import { Button } from '../../elements/Button' +import { useRegisterActions } from 'kbar' +import Link from 'next/link' +import { useRouter } from 'next/router' import { Circle, DotsThree, MagnifyingGlass, X } from 'phosphor-react' +import { ReactNode, useEffect, useMemo, useRef, useState } from 'react' +import { usePersistedState } from '../../../lib/hooks/usePersistedState' +import { Label } from '../../../lib/networking/fragments/labelFragment' +import { SavedSearch } from '../../../lib/networking/fragments/savedSearchFragment' +import { useGetLabelsQuery } from '../../../lib/networking/queries/useGetLabelsQuery' +import { useGetSavedSearchQuery } from '../../../lib/networking/queries/useGetSavedSearchQuery' import { Subscription, SubscriptionType, useGetSubscriptionsQuery, } from '../../../lib/networking/queries/useGetSubscriptionsQuery' -import { useGetLabelsQuery } from '../../../lib/networking/queries/useGetLabelsQuery' -import { Label } from '../../../lib/networking/fragments/labelFragment' -import { theme } from '../../tokens/stitches.config' -import { useRegisterActions } from 'kbar' -import { LogoBox } from '../../elements/LogoBox' -import { usePersistedState } from '../../../lib/hooks/usePersistedState' -import { useGetSavedSearchQuery } from '../../../lib/networking/queries/useGetSavedSearchQuery' -import { SavedSearch } from '../../../lib/networking/fragments/savedSearchFragment' -import { ToggleCaretDownIcon } from '../../elements/icons/ToggleCaretDownIcon' -import Link from 'next/link' -import { ToggleCaretRightIcon } from '../../elements/icons/ToggleCaretRightIcon' -import { NavMenuFooter } from './Footer' +import { Shortcut } from '../../../pages/settings/shortcuts' +import { escapeQuotes } from '../../../utils/helper' +import { Button } from '../../elements/Button' +import { CoverImage } from '../../elements/CoverImage' +import { Dropdown, DropdownOption } from '../../elements/DropdownElements' +import { DiscoverIcon } from '../../elements/icons/DiscoverIcon' import { FollowingIcon } from '../../elements/icons/FollowingIcon' +import { HighlightsIcon } from '../../elements/icons/HighlightsIcon' import { HomeIcon } from '../../elements/icons/HomeIcon' import { LibraryIcon } from '../../elements/icons/LibraryIcon' -import { HighlightsIcon } from '../../elements/icons/HighlightsIcon' -import { CoverImage } from '../../elements/CoverImage' -import { Shortcut } from '../../../pages/settings/shortcuts' -import { OutlinedLabelChip } from '../../elements/OutlinedLabelChip' import { NewsletterIcon } from '../../elements/icons/NewsletterIcon' -import { Dropdown, DropdownOption } from '../../elements/DropdownElements' -import { useRouter } from 'next/router' -import { DiscoverIcon } from "../../elements/icons/DiscoverIcon" +import { ToggleCaretDownIcon } from '../../elements/icons/ToggleCaretDownIcon' +import { ToggleCaretRightIcon } from '../../elements/icons/ToggleCaretRightIcon' +import { Box, HStack, SpanBox, VStack } from '../../elements/LayoutPrimitives' +import { LogoBox } from '../../elements/LogoBox' +import { OutlinedLabelChip } from '../../elements/OutlinedLabelChip' +import { StyledText } from '../../elements/StyledText' +import { theme } from '../../tokens/stitches.config' +import { NavMenuFooter } from './Footer' export const LIBRARY_LEFT_MENU_WIDTH = '275px' @@ -221,7 +222,7 @@ const LibraryNav = (props: LibraryFilterMenuProps): JSX.Element => { } /> @@ -545,7 +546,7 @@ function Subscriptions( name: name, keywords: '*' + name, perform: () => { - props.applySearchQuery(`subscription:\"${name}\"`) + props.applySearchQuery(`subscription:\"${escapeQuotes(name)}\"`) }, } }), @@ -581,7 +582,9 @@ function Subscriptions( return ( @@ -735,7 +738,7 @@ type NavButtonRedirectProps = { } function NavRedirectButton(props: NavButtonRedirectProps): JSX.Element { - const [selected, setSelected] = useState(false); + const [selected, setSelected] = useState(false) const router = useRouter() useEffect(() => { diff --git a/packages/web/pages/settings/shortcuts.tsx b/packages/web/pages/settings/shortcuts.tsx index 1ee7d39f0..7d1879156 100644 --- a/packages/web/pages/settings/shortcuts.tsx +++ b/packages/web/pages/settings/shortcuts.tsx @@ -1,40 +1,32 @@ -import { - ReactNode, - useCallback, - useEffect, - useMemo, - useReducer, - useState, -} from 'react' -import { applyStoredTheme } from '../../lib/themeUpdater' - -import { useGetLabelsQuery } from '../../lib/networking/queries/useGetLabelsQuery' -import { useGetSavedSearchQuery } from '../../lib/networking/queries/useGetSavedSearchQuery' -import { SettingsLayout } from '../../components/templates/SettingsLayout' +import { styled } from '@stitches/react' +import { CheckSquare, Square } from 'phosphor-react' +import { ReactNode, useCallback, useEffect, useMemo, useReducer } from 'react' import { Toaster } from 'react-hot-toast' +import { Button } from '../../components/elements/Button' +import { CoverImage } from '../../components/elements/CoverImage' +import { DragIcon } from '../../components/elements/icons/DragIcon' +import { LabelChip } from '../../components/elements/LabelChip' import { Box, - VStack, HStack, - SpanBox, Separator, + SpanBox, + VStack, } from '../../components/elements/LayoutPrimitives' -import { LabelChip } from '../../components/elements/LabelChip' import { StyledText } from '../../components/elements/StyledText' +import { SettingsLayout } from '../../components/templates/SettingsLayout' +import { usePersistedState } from '../../lib/hooks/usePersistedState' +import { Label } from '../../lib/networking/fragments/labelFragment' +import { SavedSearch } from '../../lib/networking/fragments/savedSearchFragment' +import { useGetLabelsQuery } from '../../lib/networking/queries/useGetLabelsQuery' +import { useGetSavedSearchQuery } from '../../lib/networking/queries/useGetSavedSearchQuery' import { Subscription, SubscriptionType, useGetSubscriptionsQuery, } from '../../lib/networking/queries/useGetSubscriptionsQuery' -import { DragIcon } from '../../components/elements/icons/DragIcon' -import { CoverImage } from '../../components/elements/CoverImage' -import { Label } from '../../lib/networking/fragments/labelFragment' -import { usePersistedState } from '../../lib/hooks/usePersistedState' -import { CheckSquare, Square } from 'phosphor-react' -import { Button } from '../../components/elements/Button' -import { styled } from '@stitches/react' -import { SavedSearch } from '../../lib/networking/fragments/savedSearchFragment' - +import { applyStoredTheme } from '../../lib/themeUpdater' +import { escapeQuotes } from '../../utils/helper' type ListAction = 'RESET' | 'ADD_ITEM' | 'REMOVE_ITEM' const SHORTCUTS_KEY = 'library-shortcuts' @@ -416,7 +408,7 @@ const AvailableItems = (props: ListProps): JSX.Element => { : 'feed', filter: subscription.type == SubscriptionType.NEWSLETTER - ? `subscription:\"${subscription.name}\"` + ? `subscription:\"${escapeQuotes(subscription.name)}\"` : `rss:\"${subscription.url}\"`, } props.dispatchList({ diff --git a/packages/web/utils/helper.ts b/packages/web/utils/helper.ts new file mode 100644 index 000000000..831148eed --- /dev/null +++ b/packages/web/utils/helper.ts @@ -0,0 +1 @@ +export const escapeQuotes = (str: string) => str.replace(/"/g, '\\"') From e9a587bfdb9c8879e9b0e8461fbcae6ab72bf474 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Wed, 3 Apr 2024 17:07:36 +0800 Subject: [PATCH 03/45] escape quotes for label search on Web --- .../templates/navMenu/LibraryLegacyMenu.tsx | 12 +++++------- .../web/components/templates/navMenu/LibraryMenu.tsx | 9 +++++---- packages/web/pages/settings/pinned-searches.tsx | 3 ++- packages/web/pages/settings/shortcuts.tsx | 2 +- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/packages/web/components/templates/navMenu/LibraryLegacyMenu.tsx b/packages/web/components/templates/navMenu/LibraryLegacyMenu.tsx index 20a31c0be..58bf59f03 100644 --- a/packages/web/components/templates/navMenu/LibraryLegacyMenu.tsx +++ b/packages/web/components/templates/navMenu/LibraryLegacyMenu.tsx @@ -510,7 +510,7 @@ function LabelButton(props: LabelButtonProps): JSX.Element { const checkboxRef = useRef(null) const state = useMemo(() => { const term = props.searchTerm ?? '' - if (term.indexOf(`label:\"${props.label.name}\"`) >= 0) { + if (term.indexOf(`label:\"${escapeQuotes(props.label.name)}\"`) >= 0) { return 'on' } return 'off' @@ -560,7 +560,7 @@ function LabelButton(props: LabelButtonProps): JSX.Element { props.applySearchQuery(query.trim()) } else { props.applySearchQuery( - `${query.trim()} label:\"${props.label.name}\"` + `${query.trim()} label:\"${escapeQuotes(props.label.name)}\"` ) } }} @@ -579,16 +579,14 @@ function LabelButton(props: LabelButtonProps): JSX.Element { type="checkbox" checked={state === 'on'} onChange={(e) => { + const escapedName = escapeQuotes(props.label.name) if (e.target.checked) { props.applySearchQuery( - `${props.searchTerm ?? ''} label:\"${props.label.name}\"` + `${props.searchTerm ?? ''} label:\"${escapedName}\"` ) } else { const query = - props.searchTerm?.replace( - `label:\"${props.label.name}\"`, - '' - ) ?? '' + props.searchTerm?.replace(`label:\"${escapedName}\"`, '') ?? '' props.applySearchQuery(query) } }} diff --git a/packages/web/components/templates/navMenu/LibraryMenu.tsx b/packages/web/components/templates/navMenu/LibraryMenu.tsx index 0da615ab0..165a257ad 100644 --- a/packages/web/components/templates/navMenu/LibraryMenu.tsx +++ b/packages/web/components/templates/navMenu/LibraryMenu.tsx @@ -935,7 +935,7 @@ function LabelButton(props: LabelButtonProps): JSX.Element { const checkboxRef = useRef(null) const state = useMemo(() => { const term = props.searchTerm ?? '' - if (term.indexOf(`label:\"${props.label.name}\"`) >= 0) { + if (term.indexOf(`label:\"${escapeQuotes(props.label.name)}\"`) >= 0) { return 'on' } return 'off' @@ -985,7 +985,7 @@ function LabelButton(props: LabelButtonProps): JSX.Element { props.applySearchQuery(query.trim()) } else { props.applySearchQuery( - `${query.trim()} label:\"${props.label.name}\"` + `${query.trim()} label:\"${escapeQuotes(props.label.name)}\"` ) } }} @@ -1004,14 +1004,15 @@ function LabelButton(props: LabelButtonProps): JSX.Element { type="checkbox" checked={state === 'on'} onChange={(e) => { + const escapedLabelName = escapeQuotes(props.label.name) if (e.target.checked) { props.applySearchQuery( - `${props.searchTerm ?? ''} label:\"${props.label.name}\"` + `${props.searchTerm ?? ''} label:\"${escapedLabelName}\"` ) } else { const query = props.searchTerm?.replace( - `label:\"${props.label.name}\"`, + `label:\"${escapedLabelName}\"`, '' ) ?? '' props.applySearchQuery(query) diff --git a/packages/web/pages/settings/pinned-searches.tsx b/packages/web/pages/settings/pinned-searches.tsx index b06f8cce9..784062d14 100644 --- a/packages/web/pages/settings/pinned-searches.tsx +++ b/packages/web/pages/settings/pinned-searches.tsx @@ -21,6 +21,7 @@ import { Label } from '../../lib/networking/fragments/labelFragment' import { CheckSquare, Circle, Square } from 'phosphor-react' import { SavedSearch } from '../../lib/networking/fragments/savedSearchFragment' import { usePersistedState } from '../../lib/hooks/usePersistedState' +import { escapeQuotes } from '../../utils/helper' export type PinnedSearch = { type: 'saved-search' | 'label' @@ -282,7 +283,7 @@ function LabelButton(props: LabelButtonProps): JSX.Element { type: 'label', itemId: props.label.id, name: props.label.name, - search: `label:\"${props.label.name}\"`, + search: `label:\"${escapeQuotes(props.label.name)}\"`, }} listAction={props.listAction} > diff --git a/packages/web/pages/settings/shortcuts.tsx b/packages/web/pages/settings/shortcuts.tsx index 7d1879156..09355ee2b 100644 --- a/packages/web/pages/settings/shortcuts.tsx +++ b/packages/web/pages/settings/shortcuts.tsx @@ -357,7 +357,7 @@ const AvailableItems = (props: ListProps): JSX.Element => { type: 'label', label: label, name: label.name, - filter: `label:\"${label.name}\"`, + filter: `label:\"${escapeQuotes(label.name)}\"`, } props.dispatchList({ item, From dcfc9517974c8db41143221c9955ff7e621394fc Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Wed, 3 Apr 2024 17:16:37 +0800 Subject: [PATCH 04/45] revert import order change --- .../templates/navMenu/LibraryLegacyMenu.tsx | 30 ++++++------ .../templates/navMenu/LibraryMenu.tsx | 48 +++++++++---------- packages/web/pages/settings/shortcuts.tsx | 48 +++++++++++-------- 3 files changed, 67 insertions(+), 59 deletions(-) diff --git a/packages/web/components/templates/navMenu/LibraryLegacyMenu.tsx b/packages/web/components/templates/navMenu/LibraryLegacyMenu.tsx index 58bf59f03..8590c3578 100644 --- a/packages/web/components/templates/navMenu/LibraryLegacyMenu.tsx +++ b/packages/web/components/templates/navMenu/LibraryLegacyMenu.tsx @@ -1,26 +1,26 @@ -import { useRegisterActions } from 'kbar' -import Link from 'next/link' -import { Circle, X } from 'phosphor-react' import { ReactNode, useEffect, useMemo, useRef } from 'react' -import { usePersistedState } from '../../../lib/hooks/usePersistedState' -import { Label } from '../../../lib/networking/fragments/labelFragment' -import { SavedSearch } from '../../../lib/networking/fragments/savedSearchFragment' -import { useGetLabelsQuery } from '../../../lib/networking/queries/useGetLabelsQuery' -import { useGetSavedSearchQuery } from '../../../lib/networking/queries/useGetSavedSearchQuery' +import { StyledText } from '../../elements/StyledText' +import { Box, HStack, SpanBox, VStack } from '../../elements/LayoutPrimitives' +import { Button } from '../../elements/Button' +import { Circle, NewspaperClipping, X } from 'phosphor-react' import { Subscription, SubscriptionType, useGetSubscriptionsQuery, } from '../../../lib/networking/queries/useGetSubscriptionsQuery' -import { escapeQuotes } from '../../../utils/helper' -import { Button } from '../../elements/Button' -import { ToggleCaretDownIcon } from '../../elements/icons/ToggleCaretDownIcon' -import { ToggleCaretRightIcon } from '../../elements/icons/ToggleCaretRightIcon' -import { Box, HStack, SpanBox, VStack } from '../../elements/LayoutPrimitives' -import { LogoBox } from '../../elements/LogoBox' -import { StyledText } from '../../elements/StyledText' +import { useGetLabelsQuery } from '../../../lib/networking/queries/useGetLabelsQuery' +import { Label } from '../../../lib/networking/fragments/labelFragment' import { theme } from '../../tokens/stitches.config' +import { useRegisterActions } from 'kbar' +import { LogoBox } from '../../elements/LogoBox' +import { usePersistedState } from '../../../lib/hooks/usePersistedState' +import { useGetSavedSearchQuery } from '../../../lib/networking/queries/useGetSavedSearchQuery' +import { SavedSearch } from '../../../lib/networking/fragments/savedSearchFragment' +import { ToggleCaretDownIcon } from '../../elements/icons/ToggleCaretDownIcon' +import Link from 'next/link' +import { ToggleCaretRightIcon } from '../../elements/icons/ToggleCaretRightIcon' import { NavMenuFooter } from './Footer' +import { escapeQuotes } from '../../../utils/helper' export const LIBRARY_LEFT_MENU_WIDTH = '275px' diff --git a/packages/web/components/templates/navMenu/LibraryMenu.tsx b/packages/web/components/templates/navMenu/LibraryMenu.tsx index 165a257ad..a2c5cbb05 100644 --- a/packages/web/components/templates/navMenu/LibraryMenu.tsx +++ b/packages/web/components/templates/navMenu/LibraryMenu.tsx @@ -1,37 +1,37 @@ -import { useRegisterActions } from 'kbar' -import Link from 'next/link' -import { useRouter } from 'next/router' +import { ReactNode, useEffect, useMemo, useRef, useState } from "react" +import { StyledText } from '../../elements/StyledText' +import { Box, HStack, SpanBox, VStack } from '../../elements/LayoutPrimitives' +import { Button } from '../../elements/Button' import { Circle, DotsThree, MagnifyingGlass, X } from 'phosphor-react' -import { ReactNode, useEffect, useMemo, useRef, useState } from 'react' -import { usePersistedState } from '../../../lib/hooks/usePersistedState' -import { Label } from '../../../lib/networking/fragments/labelFragment' -import { SavedSearch } from '../../../lib/networking/fragments/savedSearchFragment' -import { useGetLabelsQuery } from '../../../lib/networking/queries/useGetLabelsQuery' -import { useGetSavedSearchQuery } from '../../../lib/networking/queries/useGetSavedSearchQuery' import { Subscription, SubscriptionType, useGetSubscriptionsQuery, } from '../../../lib/networking/queries/useGetSubscriptionsQuery' -import { Shortcut } from '../../../pages/settings/shortcuts' -import { escapeQuotes } from '../../../utils/helper' -import { Button } from '../../elements/Button' -import { CoverImage } from '../../elements/CoverImage' -import { Dropdown, DropdownOption } from '../../elements/DropdownElements' -import { DiscoverIcon } from '../../elements/icons/DiscoverIcon' +import { useGetLabelsQuery } from '../../../lib/networking/queries/useGetLabelsQuery' +import { Label } from '../../../lib/networking/fragments/labelFragment' +import { theme } from '../../tokens/stitches.config' +import { useRegisterActions } from 'kbar' +import { LogoBox } from '../../elements/LogoBox' +import { usePersistedState } from '../../../lib/hooks/usePersistedState' +import { useGetSavedSearchQuery } from '../../../lib/networking/queries/useGetSavedSearchQuery' +import { SavedSearch } from '../../../lib/networking/fragments/savedSearchFragment' +import { ToggleCaretDownIcon } from '../../elements/icons/ToggleCaretDownIcon' +import Link from 'next/link' +import { ToggleCaretRightIcon } from '../../elements/icons/ToggleCaretRightIcon' +import { NavMenuFooter } from './Footer' import { FollowingIcon } from '../../elements/icons/FollowingIcon' -import { HighlightsIcon } from '../../elements/icons/HighlightsIcon' import { HomeIcon } from '../../elements/icons/HomeIcon' import { LibraryIcon } from '../../elements/icons/LibraryIcon' -import { NewsletterIcon } from '../../elements/icons/NewsletterIcon' -import { ToggleCaretDownIcon } from '../../elements/icons/ToggleCaretDownIcon' -import { ToggleCaretRightIcon } from '../../elements/icons/ToggleCaretRightIcon' -import { Box, HStack, SpanBox, VStack } from '../../elements/LayoutPrimitives' -import { LogoBox } from '../../elements/LogoBox' +import { HighlightsIcon } from '../../elements/icons/HighlightsIcon' +import { CoverImage } from '../../elements/CoverImage' +import { Shortcut } from '../../../pages/settings/shortcuts' import { OutlinedLabelChip } from '../../elements/OutlinedLabelChip' -import { StyledText } from '../../elements/StyledText' -import { theme } from '../../tokens/stitches.config' -import { NavMenuFooter } from './Footer' +import { NewsletterIcon } from '../../elements/icons/NewsletterIcon' +import { Dropdown, DropdownOption } from '../../elements/DropdownElements' +import { useRouter } from 'next/router' +import { DiscoverIcon } from "../../elements/icons/DiscoverIcon" +import { escapeQuotes } from "../../../utils/helper" export const LIBRARY_LEFT_MENU_WIDTH = '275px' diff --git a/packages/web/pages/settings/shortcuts.tsx b/packages/web/pages/settings/shortcuts.tsx index 09355ee2b..f9fddfec2 100644 --- a/packages/web/pages/settings/shortcuts.tsx +++ b/packages/web/pages/settings/shortcuts.tsx @@ -1,31 +1,39 @@ -import { styled } from '@stitches/react' -import { CheckSquare, Square } from 'phosphor-react' -import { ReactNode, useCallback, useEffect, useMemo, useReducer } from 'react' -import { Toaster } from 'react-hot-toast' -import { Button } from '../../components/elements/Button' -import { CoverImage } from '../../components/elements/CoverImage' -import { DragIcon } from '../../components/elements/icons/DragIcon' -import { LabelChip } from '../../components/elements/LabelChip' import { - Box, - HStack, - Separator, - SpanBox, - VStack, -} from '../../components/elements/LayoutPrimitives' -import { StyledText } from '../../components/elements/StyledText' -import { SettingsLayout } from '../../components/templates/SettingsLayout' -import { usePersistedState } from '../../lib/hooks/usePersistedState' -import { Label } from '../../lib/networking/fragments/labelFragment' -import { SavedSearch } from '../../lib/networking/fragments/savedSearchFragment' + ReactNode, + useCallback, + useEffect, + useMemo, + useReducer, + useState, +} from 'react' +import { applyStoredTheme } from '../../lib/themeUpdater' + import { useGetLabelsQuery } from '../../lib/networking/queries/useGetLabelsQuery' import { useGetSavedSearchQuery } from '../../lib/networking/queries/useGetSavedSearchQuery' +import { SettingsLayout } from '../../components/templates/SettingsLayout' +import { Toaster } from 'react-hot-toast' +import { + Box, + VStack, + HStack, + SpanBox, + Separator, +} from '../../components/elements/LayoutPrimitives' +import { LabelChip } from '../../components/elements/LabelChip' +import { StyledText } from '../../components/elements/StyledText' import { Subscription, SubscriptionType, useGetSubscriptionsQuery, } from '../../lib/networking/queries/useGetSubscriptionsQuery' -import { applyStoredTheme } from '../../lib/themeUpdater' +import { DragIcon } from '../../components/elements/icons/DragIcon' +import { CoverImage } from '../../components/elements/CoverImage' +import { Label } from '../../lib/networking/fragments/labelFragment' +import { usePersistedState } from '../../lib/hooks/usePersistedState' +import { CheckSquare, Square } from 'phosphor-react' +import { Button } from '../../components/elements/Button' +import { styled } from '@stitches/react' +import { SavedSearch } from '../../lib/networking/fragments/savedSearchFragment' import { escapeQuotes } from '../../utils/helper' type ListAction = 'RESET' | 'ADD_ITEM' | 'REMOVE_ITEM' From 3c1abb964b819d84d012b0935c8c8813bcc905e6 Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Wed, 3 Apr 2024 22:07:28 +0800 Subject: [PATCH 05/45] Also use mailjet for reset password emails --- packages/api/src/services/send_emails.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/packages/api/src/services/send_emails.ts b/packages/api/src/services/send_emails.ts index 4d6a76bb2..540b289d0 100644 --- a/packages/api/src/services/send_emails.ts +++ b/packages/api/src/services/send_emails.ts @@ -104,6 +104,10 @@ export const sendPasswordResetEmail = async (user: { link, } + if (process.env.USE_MAILJET) { + return sendWithMailJet(user.email, link) + } + return sendEmail({ from: env.sender.message, to: user.email, From 491611b05f752dc966a55d83d04fa559cd9942ea Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Wed, 3 Apr 2024 22:25:19 +0800 Subject: [PATCH 06/45] sync items and highlights to user created database in notion --- .../api/src/resolvers/integrations/index.ts | 19 +- .../api/src/services/integrations/notion.ts | 171 +++++++++--------- 2 files changed, 100 insertions(+), 90 deletions(-) diff --git a/packages/api/src/resolvers/integrations/index.ts b/packages/api/src/resolvers/integrations/index.ts index fec1a9a82..b1fc0c03a 100644 --- a/packages/api/src/resolvers/integrations/index.ts +++ b/packages/api/src/resolvers/integrations/index.ts @@ -40,6 +40,7 @@ import { saveIntegration, updateIntegration, } from '../../services/integrations' +import { NotionClient } from '../../services/integrations/notion' import { analytics } from '../../utils/analytics' import { deleteTask, @@ -57,15 +58,14 @@ export const setIntegrationResolver = authorized< ...input, user: { id: uid }, id: input.id || undefined, - type: input.type || IntegrationType.Export, + type: input.type || undefined, syncedAt: input.syncedAt ? new Date(input.syncedAt) : undefined, importItemState: input.type === IntegrationType.Import ? input.importItemState || ImportItemState.Unarchived // default to unarchived : undefined, - // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment - settings: input.settings, } + if (input.id) { // Update const existingIntegration = await findIntegration({ id: input.id }, uid) @@ -96,6 +96,19 @@ export const setIntegrationResolver = authorized< if (integration.name.toLowerCase() === 'readwise') { // create a task to export all the items for readwise temporarily await enqueueExportToIntegration(integration.id, uid) + } else if (integration.name.toLowerCase() === 'notion') { + const settings = integration.settings as { parentDatabaseId?: string } + if (settings.parentDatabaseId) { + // update notion database properties + const notion = new NotionClient(integration.token, integration) + try { + await notion.updateDatabase(settings.parentDatabaseId) + } catch (error) { + return { + errorCodes: [SetIntegrationErrorCode.BadRequest], + } + } + } } analytics.capture({ diff --git a/packages/api/src/services/integrations/notion.ts b/packages/api/src/services/integrations/notion.ts index 7e891287d..31ffc2cff 100644 --- a/packages/api/src/services/integrations/notion.ts +++ b/packages/api/src/services/integrations/notion.ts @@ -1,6 +1,5 @@ import { Client } from '@notionhq/client' import axios from 'axios' -import { updateIntegration } from '.' import { Integration } from '../../entity/integration' import { LibraryItem } from '../../entity/library_item' import { env } from '../../env' @@ -111,7 +110,7 @@ type Property = 'highlights' interface Settings { parentPageId: string parentDatabaseId: string - properties: Property[] + properties?: Property[] } export class NotionClient implements IntegrationClient { @@ -244,7 +243,7 @@ export class NotionClient implements IntegrationClient { : undefined, }, children: - settings.properties.includes('highlights') && item.highlights + settings.properties?.includes('highlights') && item.highlights ? item.highlights .filter( (highlight) => !lastSync || highlight.updatedAt > lastSync // only new highlights @@ -315,103 +314,101 @@ export class NotionClient implements IntegrationClient { return false } - const pageId = settings.parentPageId - if (!pageId) { - logger.error('Notion parent page id not found') - return false - } - - let databaseId = settings.parentDatabaseId + const databaseId = settings.parentDatabaseId if (!databaseId) { - // create a database for the items - const database = await this.client.databases.create({ - parent: { - page_id: pageId, - }, - title: [ - { - text: { - content: 'Library', - }, - }, - ], - description: [ - { - text: { - content: 'Library of saved items from Omnivore', - }, - }, - ], - properties: { - Title: { - title: {}, - }, - Author: { - rich_text: {}, - }, - 'Original URL': { - url: {}, - }, - 'Omnivore URL': { - url: {}, - }, - 'Saved At': { - date: {}, - }, - 'Last Updated': { - date: {}, - }, - Tags: { - multi_select: {}, - }, - }, - }) - - // save the database id - databaseId = database.id - settings.parentDatabaseId = databaseId - await updateIntegration( - this.integrationData.id, - { - settings, - }, - this.integrationData.user.id - ) + logger.error('Notion database id not found') + return false } await Promise.all( items.map(async (item) => { - const notionPage = this.itemToNotionPage( - item, - settings, - this.integrationData?.syncedAt - ) - const url = notionPage.properties['Omnivore URL'].url + try { + const notionPage = this.itemToNotionPage( + item, + settings, + this.integrationData?.syncedAt + ) + const url = notionPage.properties['Omnivore URL'].url - const existingPage = await this.findPage(url, databaseId) - if (existingPage) { - // update the page - await this.client.pages.update({ - page_id: existingPage.id, - properties: notionPage.properties, - }) - - // append the children incrementally - if (notionPage.children && notionPage.children.length > 0) { - await this.client.blocks.children.append({ - block_id: existingPage.id, - children: notionPage.children, + const existingPage = await this.findPage(url, databaseId) + if (existingPage) { + // update the page + await this.client.pages.update({ + page_id: existingPage.id, + properties: notionPage.properties, }) + + // append the children incrementally + if (notionPage.children && notionPage.children.length > 0) { + await this.client.blocks.children.append({ + block_id: existingPage.id, + children: notionPage.children, + }) + } + + return } - return + // create the page + return this.createPage(notionPage) + } catch (error) { + logger.error(error) + return false } - - // create the page - return this.createPage(notionPage) }) ) return true } + + private findDatabase = async (databaseId: string) => { + return this.client.databases.retrieve({ + database_id: databaseId, + }) + } + + updateDatabase = async (databaseId: string) => { + const database = await this.findDatabase(databaseId) + + await this.client.databases.update({ + database_id: database.id, + title: [ + { + text: { + content: 'Library', + }, + }, + ], + description: [ + { + text: { + content: 'Library of saved items from Omnivore', + }, + }, + ], + properties: { + Title: { + title: {}, + }, + Author: { + rich_text: {}, + }, + 'Original URL': { + url: {}, + }, + 'Omnivore URL': { + url: {}, + }, + 'Saved At': { + date: {}, + }, + 'Last Updated': { + date: {}, + }, + Tags: { + multi_select: {}, + }, + }, + }) + } } From c45a9d365bb673bc969acc9254ce8844af000908 Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Wed, 3 Apr 2024 22:25:31 +0800 Subject: [PATCH 07/45] Also use mailjet for password changes --- packages/api/src/services/send_emails.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/packages/api/src/services/send_emails.ts b/packages/api/src/services/send_emails.ts index 540b289d0..886d22a96 100644 --- a/packages/api/src/services/send_emails.ts +++ b/packages/api/src/services/send_emails.ts @@ -82,6 +82,10 @@ export const sendVerificationEmail = async (user: { link, } + if (process.env.USE_MAILJET) { + return sendWithMailJet(user.email, link) + } + return sendEmail({ from: env.sender.message, to: user.email, From 40fccf52cd7637101eb1f2ca4da7f6d7257c0009 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Wed, 3 Apr 2024 23:00:21 +0800 Subject: [PATCH 08/45] Update web --- .../pages/settings/integrations/notion.tsx | 20 ++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/packages/web/pages/settings/integrations/notion.tsx b/packages/web/pages/settings/integrations/notion.tsx index f45693f74..a8760da8e 100644 --- a/packages/web/pages/settings/integrations/notion.tsx +++ b/packages/web/pages/settings/integrations/notion.tsx @@ -30,7 +30,7 @@ import { applyStoredTheme } from '../../../lib/themeUpdater' import { showSuccessToast } from '../../../lib/toastHelpers' type FieldType = { - parentPageId?: string + // parentPageId?: string parentDatabaseId?: string properties?: string[] } @@ -47,7 +47,7 @@ export default function Notion(): JSX.Element { useEffect(() => { form.setFieldsValue({ - parentPageId: notion.settings?.parentPageId, + // parentPageId: notion.settings?.parentPageId, parentDatabaseId: notion.settings?.parentDatabaseId, properties: notion.settings?.properties, }) @@ -167,7 +167,7 @@ export default function Notion(): JSX.Element { onFinish={onFinish} onFinishFailed={onFinishFailed} > - + {/* label="Notion Page Id" name="parentPageId" help="The id of the Notion page where the items will be exported to. You can find it in the URL of the page." @@ -179,14 +179,20 @@ export default function Notion(): JSX.Element { ]} > - + */} - label="Notion Database Id" + label="Notion Database ID" name="parentDatabaseId" - hidden + help="The ID of the Notion database where the items will be exported to. You can find it in the URL of the database." + rules={[ + { + required: true, + message: 'Please input your Notion Database ID!', + }, + ]} > - + From 10cca7f05d3442af1d9efc18d02c110f0f40e54a Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Thu, 4 Apr 2024 10:43:39 +0800 Subject: [PATCH 09/45] find the title property and update its name to Title --- .../api/src/resolvers/integrations/index.ts | 5 +++- .../api/src/services/integrations/notion.ts | 23 ++++++------------- 2 files changed, 11 insertions(+), 17 deletions(-) diff --git a/packages/api/src/resolvers/integrations/index.ts b/packages/api/src/resolvers/integrations/index.ts index b1fc0c03a..502b06880 100644 --- a/packages/api/src/resolvers/integrations/index.ts +++ b/packages/api/src/resolvers/integrations/index.ts @@ -96,7 +96,10 @@ export const setIntegrationResolver = authorized< if (integration.name.toLowerCase() === 'readwise') { // create a task to export all the items for readwise temporarily await enqueueExportToIntegration(integration.id, uid) - } else if (integration.name.toLowerCase() === 'notion') { + } else if ( + integration.name.toLowerCase() === 'notion' && + integration.settings + ) { const settings = integration.settings as { parentDatabaseId?: string } if (settings.parentDatabaseId) { // update notion database properties diff --git a/packages/api/src/services/integrations/notion.ts b/packages/api/src/services/integrations/notion.ts index 31ffc2cff..ef9b63319 100644 --- a/packages/api/src/services/integrations/notion.ts +++ b/packages/api/src/services/integrations/notion.ts @@ -369,26 +369,17 @@ export class NotionClient implements IntegrationClient { updateDatabase = async (databaseId: string) => { const database = await this.findDatabase(databaseId) + // find the title property and update it + const titleProperty = Object.entries(database.properties).find( + ([, property]) => property.type === 'title' + ) + const title = titleProperty ? titleProperty[0] : 'Name' await this.client.databases.update({ database_id: database.id, - title: [ - { - text: { - content: 'Library', - }, - }, - ], - description: [ - { - text: { - content: 'Library of saved items from Omnivore', - }, - }, - ], properties: { - Title: { - title: {}, + [title]: { + name: 'Title', }, Author: { rich_text: {}, From 5cfef63e079248d4501174ce48b2d4709dc17c56 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Thu, 4 Apr 2024 11:57:30 +0800 Subject: [PATCH 10/45] add database id validation and normalization --- .../pages/settings/integrations/notion.tsx | 60 +++++++++++++------ 1 file changed, 43 insertions(+), 17 deletions(-) diff --git a/packages/web/pages/settings/integrations/notion.tsx b/packages/web/pages/settings/integrations/notion.tsx index a8760da8e..6a402deaa 100644 --- a/packages/web/pages/settings/integrations/notion.tsx +++ b/packages/web/pages/settings/integrations/notion.tsx @@ -30,8 +30,7 @@ import { applyStoredTheme } from '../../../lib/themeUpdater' import { showSuccessToast } from '../../../lib/toastHelpers' type FieldType = { - // parentPageId?: string - parentDatabaseId?: string + parentDatabaseId: string properties?: string[] } @@ -47,7 +46,6 @@ export default function Notion(): JSX.Element { useEffect(() => { form.setFieldsValue({ - // parentPageId: notion.settings?.parentPageId, parentDatabaseId: notion.settings?.parentDatabaseId, properties: notion.settings?.properties, }) @@ -72,6 +70,28 @@ export default function Notion(): JSX.Element { }) } + const normalizeDatabaseId = useCallback( + (value: string) => { + // check if database id is in UUIDv4 format + const uuidRegex = + /^[0-9a-fA-F]{8}[0-9a-fA-F]{4}[0-9a-fA-F]{4}[0-9a-fA-F]{4}[0-9a-fA-F]{12}$/ + if (uuidRegex.test(value)) { + return value + } + + // extract the database id from the URL + // https://www.notion.so/ec460c235baa4da5bb412971a12e9dbe?v=8f4e324c0b584b67b8b7cfe9a2f996d7 -> ec460c235baa4da5bb412971a12e9dbe + const urlRegex = /https:\/\/www.notion.so\/([a-f0-9]{32})\?*/ + const match = value.match(urlRegex) + if (!match || match.length < 2) { + messageApi.error('Invalid Notion Database ID.') + return value + } + return match[1] + }, + [messageApi] + ) + const onFinish: FormProps['onFinish'] = async (values) => { try { await updateNotion(values) @@ -167,29 +187,35 @@ export default function Notion(): JSX.Element { onFinish={onFinish} onFinishFailed={onFinishFailed} > - {/* - label="Notion Page Id" - name="parentPageId" - help="The id of the Notion page where the items will be exported to. You can find it in the URL of the page." - rules={[ - { - required: true, - message: 'Please input your Notion Page Id!', - }, - ]} - > - - */} - label="Notion Database ID" name="parentDatabaseId" help="The ID of the Notion database where the items will be exported to. You can find it in the URL of the database." + normalize={normalizeDatabaseId} rules={[ { required: true, message: 'Please input your Notion Database ID!', }, + { + validator: (_, value) => { + // check if database id is in UUIDv4 format + const uuidRegex = /^[0-9a-fA-F]{8}[0-9a-fA-F]{4}[0-9a-fA-F]{4}[0-9a-fA-F]{4}[0-9a-fA-F]{12}$/ + if (uuidRegex.test(value)) { + return Promise.resolve() + } + // extract the database id from the URL + const urlRegex = + /https:\/\/www.notion.so\/([a-f0-9]{32})\?*/ + const match = value.match(urlRegex) + if (match && match.length >= 2) { + return Promise.resolve() + } + return Promise.reject( + new Error('Invalid Notion Database ID.') + ) + }, + }, ]} > From 07cee98ff9a36558e78ac34c349a2ff8b553eb84 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Thu, 21 Mar 2024 17:24:24 +0800 Subject: [PATCH 11/45] allow label and highlight events to trigger rules --- packages/api/src/entity/rule.ts | 3 +++ packages/api/src/jobs/trigger_rule.ts | 12 +++++++++++- packages/api/src/pubsub.ts | 14 ++++++-------- packages/api/src/services/highlights.ts | 13 +++++++------ packages/api/src/services/labels.ts | 10 +++++----- 5 files changed, 32 insertions(+), 20 deletions(-) diff --git a/packages/api/src/entity/rule.ts b/packages/api/src/entity/rule.ts index 51c63b972..18538971c 100644 --- a/packages/api/src/entity/rule.ts +++ b/packages/api/src/entity/rule.ts @@ -15,11 +15,14 @@ export enum RuleActionType { Delete = 'DELETE', MarkAsRead = 'MARK_AS_READ', SendNotification = 'SEND_NOTIFICATION', + Webhook = 'WEBHOOK', } export enum RuleEventType { PageCreated = 'PAGE_CREATED', PageUpdated = 'PAGE_UPDATED', + LabelCreated = 'PAGE_CREATED', + HighlightCreated = 'HIGHLIGHT_CREATED', } export interface RuleAction { diff --git a/packages/api/src/jobs/trigger_rule.ts b/packages/api/src/jobs/trigger_rule.ts index 596760295..3a72dac11 100644 --- a/packages/api/src/jobs/trigger_rule.ts +++ b/packages/api/src/jobs/trigger_rule.ts @@ -86,7 +86,9 @@ const sendNotification = async (obj: RuleActionObj) => { return sendPushNotifications(obj.userId, message, 'rule', data) } -const getRuleAction = (actionType: RuleActionType): RuleActionFunc => { +const getRuleAction = ( + actionType: RuleActionType +): RuleActionFunc | undefined => { switch (actionType) { case RuleActionType.AddLabel: return addLabels @@ -98,6 +100,9 @@ const getRuleAction = (actionType: RuleActionType): RuleActionFunc => { return markPageAsRead case RuleActionType.SendNotification: return sendNotification + default: + logger.error('Unknown rule action type', actionType) + return undefined } } @@ -150,6 +155,11 @@ const triggerActions = async ( for (const action of rule.actions) { const actionFunc = getRuleAction(action.type) + if (!actionFunc) { + logger.error('No action function found for action', action.type) + continue + } + const actionObj: RuleActionObj = { libraryItemId, userId, diff --git a/packages/api/src/pubsub.ts b/packages/api/src/pubsub.ts index 4401cfad9..d8c6bba4d 100644 --- a/packages/api/src/pubsub.ts +++ b/packages/api/src/pubsub.ts @@ -53,14 +53,12 @@ export const createPubSubClient = (): PubsubClient => { libraryItemId: string ): Promise => { // queue trigger rule job - if (type === EntityType.PAGE) { - await enqueueTriggerRuleJob({ - userId, - ruleEventType: RuleEventType.PageCreated, - libraryItemId, - data, - }) - } + await enqueueTriggerRuleJob({ + userId, + ruleEventType: `${type.toUpperCase()}_CREATED` as RuleEventType, + libraryItemId, + data, + }) // queue export item job await enqueueExportItem({ userId, diff --git a/packages/api/src/services/highlights.ts b/packages/api/src/services/highlights.ts index babf4d4a4..91249626b 100644 --- a/packages/api/src/services/highlights.ts +++ b/packages/api/src/services/highlights.ts @@ -9,6 +9,7 @@ import { createPubSubClient, EntityType } from '../pubsub' import { authTrx } from '../repository' import { highlightRepository } from '../repository/highlight' import { enqueueUpdateHighlight } from '../utils/createTask' +import { UpdateItemEvent } from './library_item' type HighlightEvent = { id: string; pageId: string } type CreateHighlightEvent = DeepPartial & HighlightEvent @@ -56,9 +57,9 @@ export const createHighlight = async ( userId ) - await pubsub.entityCreated( + await pubsub.entityCreated( EntityType.HIGHLIGHT, - { ...newHighlight, pageId: libraryItemId }, + { id: libraryItemId, highlights: [newHighlight] }, userId, libraryItemId ) @@ -104,9 +105,9 @@ export const mergeHighlights = async ( }) }) - await pubsub.entityCreated( + await pubsub.entityCreated( EntityType.HIGHLIGHT, - { ...newHighlight, pageId: libraryItemId }, + { id: libraryItemId, highlights: [newHighlight] }, userId, libraryItemId ) @@ -139,9 +140,9 @@ export const updateHighlight = async ( }) const libraryItemId = updatedHighlight.libraryItem.id - await pubsub.entityUpdated( + await pubsub.entityUpdated( EntityType.HIGHLIGHT, - { ...highlight, id: highlightId, pageId: libraryItemId }, + { id: libraryItemId, highlights: [highlight] }, userId, libraryItemId ) diff --git a/packages/api/src/services/labels.ts b/packages/api/src/services/labels.ts index f4bae13b7..ca0e02d64 100644 --- a/packages/api/src/services/labels.ts +++ b/packages/api/src/services/labels.ts @@ -8,7 +8,7 @@ import { CreateLabelInput, labelRepository } from '../repository/label' import { bulkEnqueueUpdateLabels } from '../utils/createTask' import { logger } from '../utils/logger' import { findHighlightById } from './highlights' -import { findLibraryItemIdsByLabelId } from './library_item' +import { findLibraryItemIdsByLabelId, UpdateItemEvent } from './library_item' type AddLabelsToLibraryItemEvent = { pageId: string @@ -144,9 +144,9 @@ export const saveLabelsInLibraryItem = async ( if (source === 'user') { // create pubsub event - await pubsub.entityCreated( + await pubsub.entityCreated( EntityType.LABEL, - { pageId: libraryItemId, labels, source }, + { id: libraryItemId, labels }, userId, libraryItemId ) @@ -215,9 +215,9 @@ export const saveLabelsInHighlight = async ( const libraryItemId = highlight.libraryItemId // create pubsub event - await pubsub.entityCreated( + await pubsub.entityCreated( EntityType.LABEL, - { highlightId, labels }, + { id: libraryItemId, highlights: [{ id: highlightId, labels }] }, userId, libraryItemId ) From c551d9fe6bdd9dad026b370ebfcff017ac2747b0 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Thu, 21 Mar 2024 18:13:42 +0800 Subject: [PATCH 12/45] allow send data to webhook with rules --- packages/api/src/entity/rule.ts | 2 +- packages/api/src/generated/graphql.ts | 5 ++- packages/api/src/generated/schema.graphql | 3 ++ packages/api/src/jobs/trigger_rule.ts | 33 +++++++++++++++- packages/api/src/pubsub.ts | 31 +++++++-------- packages/api/src/schema.ts | 3 ++ packages/api/src/services/highlights.ts | 6 +-- packages/api/src/services/labels.ts | 4 +- packages/api/src/services/library_item.ts | 46 +++++++++-------------- 9 files changed, 80 insertions(+), 53 deletions(-) diff --git a/packages/api/src/entity/rule.ts b/packages/api/src/entity/rule.ts index 18538971c..9ef6d797e 100644 --- a/packages/api/src/entity/rule.ts +++ b/packages/api/src/entity/rule.ts @@ -21,7 +21,7 @@ export enum RuleActionType { export enum RuleEventType { PageCreated = 'PAGE_CREATED', PageUpdated = 'PAGE_UPDATED', - LabelCreated = 'PAGE_CREATED', + LabelCreated = 'LABEL_CREATED', HighlightCreated = 'HIGHLIGHT_CREATED', } diff --git a/packages/api/src/generated/graphql.ts b/packages/api/src/generated/graphql.ts index ae5bcdbd8..3e3119db6 100644 --- a/packages/api/src/generated/graphql.ts +++ b/packages/api/src/generated/graphql.ts @@ -2502,10 +2502,13 @@ export enum RuleActionType { Archive = 'ARCHIVE', Delete = 'DELETE', MarkAsRead = 'MARK_AS_READ', - SendNotification = 'SEND_NOTIFICATION' + SendNotification = 'SEND_NOTIFICATION', + Webhook = 'WEBHOOK' } export enum RuleEventType { + HighlightCreated = 'HIGHLIGHT_CREATED', + LabelCreated = 'LABEL_CREATED', PageCreated = 'PAGE_CREATED', PageUpdated = 'PAGE_UPDATED' } diff --git a/packages/api/src/generated/schema.graphql b/packages/api/src/generated/schema.graphql index 94695e350..1782328b8 100644 --- a/packages/api/src/generated/schema.graphql +++ b/packages/api/src/generated/schema.graphql @@ -1879,9 +1879,12 @@ enum RuleActionType { DELETE MARK_AS_READ SEND_NOTIFICATION + WEBHOOK } enum RuleEventType { + HIGHLIGHT_CREATED + LABEL_CREATED PAGE_CREATED PAGE_UPDATED } diff --git a/packages/api/src/jobs/trigger_rule.ts b/packages/api/src/jobs/trigger_rule.ts index 3a72dac11..132218ebb 100644 --- a/packages/api/src/jobs/trigger_rule.ts +++ b/packages/api/src/jobs/trigger_rule.ts @@ -1,4 +1,5 @@ import { LiqeQuery } from '@omnivore/liqe' +import axios, { Method } from 'axios' import { ReadingProgressDataSource } from '../datasources/reading_progress_data_source' import { LibraryItem, LibraryItemState } from '../entity/library_item' import { Rule, RuleAction, RuleActionType, RuleEventType } from '../entity/rule' @@ -28,6 +29,7 @@ interface RuleActionObj { userId: string action: RuleAction data: ItemEvent | LibraryItem + ruleEventType: RuleEventType } type RuleActionFunc = (obj: RuleActionObj) => Promise @@ -86,6 +88,29 @@ const sendNotification = async (obj: RuleActionObj) => { return sendPushNotifications(obj.userId, message, 'rule', data) } +const sendToWebhook = async (obj: RuleActionObj) => { + const [url, method, contentType] = obj.action.params + const [type, action] = obj.ruleEventType.split('_') + + const body = { + action, + userId: obj.userId, + [type]: obj.data, + } + + logger.info('triggering webhook', { url, method }) + + return axios.request({ + url, + method: method as Method, + headers: { + 'Content-Type': contentType, + }, + data: body, + timeout: 5000, // 5s + }) +} + const getRuleAction = ( actionType: RuleActionType ): RuleActionFunc | undefined => { @@ -100,6 +125,8 @@ const getRuleAction = ( return markPageAsRead case RuleActionType.SendNotification: return sendNotification + case RuleActionType.Webhook: + return sendToWebhook default: logger.error('Unknown rule action type', actionType) return undefined @@ -110,7 +137,8 @@ const triggerActions = async ( libraryItemId: string, userId: string, rules: Rule[], - data: ItemEvent + data: ItemEvent, + ruleEventType: RuleEventType ) => { const actionPromises: Promise[] = [] @@ -165,6 +193,7 @@ const triggerActions = async ( userId, action, data: results[0], + ruleEventType, } actionPromises.push(actionFunc(actionObj)) @@ -188,7 +217,7 @@ export const triggerRule = async (jobData: TriggerRuleJobData) => { return false } - await triggerActions(libraryItemId, userId, rules, data) + await triggerActions(libraryItemId, userId, rules, data, ruleEventType) return true } diff --git a/packages/api/src/pubsub.ts b/packages/api/src/pubsub.ts index d8c6bba4d..d340f520b 100644 --- a/packages/api/src/pubsub.ts +++ b/packages/api/src/pubsub.ts @@ -12,6 +12,8 @@ import { import { buildLogger } from './utils/logger' import { isYouTubeVideoURL } from './utils/youtube' +export type BaseEntityEvent = { id: string; userId: string } + const logger = buildLogger('pubsub') const client = new PubSub() @@ -46,7 +48,7 @@ export const createPubSubClient = (): PubsubClient => { Buffer.from(JSON.stringify({ userId, email, name, username })) ) }, - entityCreated: async >( + entityCreated: async ( type: EntityType, data: T, userId: string, @@ -54,10 +56,10 @@ export const createPubSubClient = (): PubsubClient => { ): Promise => { // queue trigger rule job await enqueueTriggerRuleJob({ - userId, ruleEventType: `${type.toUpperCase()}_CREATED` as RuleEventType, - libraryItemId, data, + userId, + libraryItemId, }) // queue export item job await enqueueExportItem({ @@ -92,21 +94,20 @@ export const createPubSubClient = (): PubsubClient => { } } }, - entityUpdated: async >( + entityUpdated: async ( type: EntityType, data: T, userId: string, libraryItemId: string ): Promise => { // queue trigger rule job - if (type === EntityType.PAGE) { - await enqueueTriggerRuleJob({ - userId, - ruleEventType: RuleEventType.PageUpdated, - libraryItemId, - data, - }) - } + await enqueueTriggerRuleJob({ + userId, + ruleEventType: RuleEventType.PageUpdated, + libraryItemId, + data, + }) + // queue export item job await enqueueExportItem({ userId, @@ -145,7 +146,7 @@ export const createPubSubClient = (): PubsubClient => { } export enum EntityType { - PAGE = 'page', + ITEM = 'page', HIGHLIGHT = 'highlight', LABEL = 'label', RSS_FEED = 'feed', @@ -158,13 +159,13 @@ export interface PubsubClient { name: string, username: string ) => Promise - entityCreated: >( + entityCreated: ( type: EntityType, data: T, userId: string, libraryItemId: string ) => Promise - entityUpdated: >( + entityUpdated: ( type: EntityType, data: T, userId: string, diff --git a/packages/api/src/schema.ts b/packages/api/src/schema.ts index e106514e4..d2279407e 100755 --- a/packages/api/src/schema.ts +++ b/packages/api/src/schema.ts @@ -2167,6 +2167,7 @@ const schema = gql` DELETE MARK_AS_READ SEND_NOTIFICATION + WEBHOOK } type RulesError { @@ -2181,6 +2182,8 @@ const schema = gql` enum RuleEventType { PAGE_CREATED PAGE_UPDATED + LABEL_CREATED + HIGHLIGHT_CREATED } input SetRuleInput { diff --git a/packages/api/src/services/highlights.ts b/packages/api/src/services/highlights.ts index 91249626b..8cc10ad0b 100644 --- a/packages/api/src/services/highlights.ts +++ b/packages/api/src/services/highlights.ts @@ -59,7 +59,7 @@ export const createHighlight = async ( await pubsub.entityCreated( EntityType.HIGHLIGHT, - { id: libraryItemId, highlights: [newHighlight] }, + { id: libraryItemId, highlights: [newHighlight], userId }, userId, libraryItemId ) @@ -107,7 +107,7 @@ export const mergeHighlights = async ( await pubsub.entityCreated( EntityType.HIGHLIGHT, - { id: libraryItemId, highlights: [newHighlight] }, + { id: libraryItemId, highlights: [newHighlight], userId }, userId, libraryItemId ) @@ -142,7 +142,7 @@ export const updateHighlight = async ( const libraryItemId = updatedHighlight.libraryItem.id await pubsub.entityUpdated( EntityType.HIGHLIGHT, - { id: libraryItemId, highlights: [highlight] }, + { id: libraryItemId, highlights: [highlight], userId }, userId, libraryItemId ) diff --git a/packages/api/src/services/labels.ts b/packages/api/src/services/labels.ts index ca0e02d64..1f05d14c1 100644 --- a/packages/api/src/services/labels.ts +++ b/packages/api/src/services/labels.ts @@ -146,7 +146,7 @@ export const saveLabelsInLibraryItem = async ( // create pubsub event await pubsub.entityCreated( EntityType.LABEL, - { id: libraryItemId, labels }, + { id: libraryItemId, labels, userId }, userId, libraryItemId ) @@ -217,7 +217,7 @@ export const saveLabelsInHighlight = async ( // create pubsub event await pubsub.entityCreated( EntityType.LABEL, - { id: libraryItemId, highlights: [{ id: highlightId, labels }] }, + { id: libraryItemId, highlights: [{ id: highlightId, labels }], userId }, userId, libraryItemId ) diff --git a/packages/api/src/services/library_item.ts b/packages/api/src/services/library_item.ts index 83473104c..4ac000900 100644 --- a/packages/api/src/services/library_item.ts +++ b/packages/api/src/services/library_item.ts @@ -15,7 +15,7 @@ import { Highlight } from '../entity/highlight' import { Label } from '../entity/label' import { LibraryItem, LibraryItemState } from '../entity/library_item' import { BulkActionType, InputMaybe, SortParams } from '../generated/graphql' -import { createPubSubClient, EntityType } from '../pubsub' +import { BaseEntityEvent, createPubSubClient, EntityType } from '../pubsub' import { redisDataSource } from '../redis_data_source' import { authTrx, @@ -37,10 +37,13 @@ type IgnoredFields = | 'links' | 'textContentHash' export type ItemEvent = CreateItemEvent | UpdateItemEvent -export type CreateItemEvent = Omit, IgnoredFields> -export type UpdateItemEvent = Omit< - QueryDeepPartialEntity, - IgnoredFields +export type CreateItemEvent = Merge< + Omit, IgnoredFields>, + BaseEntityEvent +> +export type UpdateItemEvent = Merge< + Omit, IgnoredFields>, + BaseEntityEvent > export class RequiresSearchQueryError extends Error { @@ -841,7 +844,7 @@ export const softDeleteLibraryItem = async ( userId ) - await pubsub.entityDeleted(EntityType.PAGE, id, userId) + await pubsub.entityDeleted(EntityType.ITEM, id, userId) return deletedLibraryItem } @@ -883,13 +886,8 @@ export const updateLibraryItem = async ( if (libraryItem.state === LibraryItemState.Succeeded) { // send create event if the item was created await pubsub.entityCreated( - EntityType.PAGE, - { - ...updatedLibraryItem, - originalContent: undefined, - readableContent: undefined, - feedContent: undefined, - }, + EntityType.ITEM, + { ...updatedLibraryItem, userId }, userId, id ) @@ -898,13 +896,8 @@ export const updateLibraryItem = async ( } await pubsub.entityUpdated( - EntityType.PAGE, - { - ...libraryItem, - originalContent: undefined, - readableContent: undefined, - feedContent: undefined, - }, + EntityType.ITEM, + { ...libraryItem, id, userId }, userId, id ) @@ -966,8 +959,8 @@ export const updateLibraryItemReadingProgress = async ( const updatedItem = result[0][0] await pubsub.entityUpdated( - EntityType.PAGE, - updatedItem, + EntityType.ITEM, + { ...updatedItem, id, userId }, userId, id ) @@ -1067,13 +1060,8 @@ export const createOrUpdateLibraryItem = async ( } await pubsub.entityCreated( - EntityType.PAGE, - { - ...newLibraryItem, - originalContent: undefined, - readableContent: undefined, - feedContent: undefined, - }, + EntityType.ITEM, + { ...newLibraryItem, userId }, userId, newLibraryItem.id ) From 88d6455222963e8819eef303702f2dc575a8b522 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Thu, 21 Mar 2024 19:51:31 +0800 Subject: [PATCH 13/45] default filter is in:all --- packages/api/src/jobs/trigger_rule.ts | 21 +++++------- packages/api/src/pubsub.ts | 17 +--------- packages/api/src/services/highlights.ts | 17 +++++----- packages/api/src/services/labels.ts | 16 +++------ packages/api/src/services/library_item.ts | 34 +++++++++---------- .../networking/queries/useGetRulesQuery.tsx | 3 ++ packages/web/pages/settings/rules.tsx | 34 +++++++++++++------ 7 files changed, 66 insertions(+), 76 deletions(-) diff --git a/packages/api/src/jobs/trigger_rule.ts b/packages/api/src/jobs/trigger_rule.ts index 132218ebb..88cc65144 100644 --- a/packages/api/src/jobs/trigger_rule.ts +++ b/packages/api/src/jobs/trigger_rule.ts @@ -1,5 +1,5 @@ import { LiqeQuery } from '@omnivore/liqe' -import axios, { Method } from 'axios' +import axios from 'axios' import { ReadingProgressDataSource } from '../datasources/reading_progress_data_source' import { LibraryItem, LibraryItemState } from '../entity/library_item' import { Rule, RuleAction, RuleActionType, RuleEventType } from '../entity/rule' @@ -89,24 +89,19 @@ const sendNotification = async (obj: RuleActionObj) => { } const sendToWebhook = async (obj: RuleActionObj) => { - const [url, method, contentType] = obj.action.params - const [type, action] = obj.ruleEventType.split('_') + const [url] = obj.action.params - const body = { - action, - userId: obj.userId, - [type]: obj.data, + const data = { + event: obj.ruleEventType, + data: obj.data, } - logger.info('triggering webhook', { url, method }) + logger.info(`triggering webhook: ${url}`) - return axios.request({ - url, - method: method as Method, + return axios.post(url, data, { headers: { - 'Content-Type': contentType, + 'Content-Type': 'application/json', }, - data: body, timeout: 5000, // 5s }) } diff --git a/packages/api/src/pubsub.ts b/packages/api/src/pubsub.ts index d340f520b..55da4e7c9 100644 --- a/packages/api/src/pubsub.ts +++ b/packages/api/src/pubsub.ts @@ -7,7 +7,6 @@ import { enqueueExportItem, enqueueProcessYouTubeVideo, enqueueTriggerRuleJob, - enqueueWebhookJob, } from './utils/createTask' import { buildLogger } from './utils/logger' import { isYouTubeVideoURL } from './utils/youtube' @@ -67,14 +66,7 @@ export const createPubSubClient = (): PubsubClient => { libraryItemIds: [libraryItemId], }) - await enqueueWebhookJob({ - userId, - type, - action: 'created', - data, - }) - - if (type === EntityType.PAGE) { + if (type === EntityType.ITEM) { // if (await findGrantedFeatureByName(FeatureName.AISummaries, userId)) { // await enqueueAISummarizeJob({ // userId, @@ -113,13 +105,6 @@ export const createPubSubClient = (): PubsubClient => { userId, libraryItemIds: [libraryItemId], }) - - await enqueueWebhookJob({ - userId, - type, - action: 'updated', - data, - }) }, entityDeleted: async ( type: EntityType, diff --git a/packages/api/src/services/highlights.ts b/packages/api/src/services/highlights.ts index 8cc10ad0b..624c5b2da 100644 --- a/packages/api/src/services/highlights.ts +++ b/packages/api/src/services/highlights.ts @@ -9,11 +9,12 @@ import { createPubSubClient, EntityType } from '../pubsub' import { authTrx } from '../repository' import { highlightRepository } from '../repository/highlight' import { enqueueUpdateHighlight } from '../utils/createTask' -import { UpdateItemEvent } from './library_item' +import { ItemEvent } from './library_item' -type HighlightEvent = { id: string; pageId: string } -type CreateHighlightEvent = DeepPartial & HighlightEvent -type UpdateHighlightEvent = QueryDeepPartialEntity & HighlightEvent +export type HighlightEvent = Omit< + DeepPartial, + 'user' | 'userId' | 'sharedAt' +> export const getHighlightLocation = (patch: string): number | undefined => { const dmp = new diff_match_patch() @@ -57,7 +58,7 @@ export const createHighlight = async ( userId ) - await pubsub.entityCreated( + await pubsub.entityCreated( EntityType.HIGHLIGHT, { id: libraryItemId, highlights: [newHighlight], userId }, userId, @@ -105,7 +106,7 @@ export const mergeHighlights = async ( }) }) - await pubsub.entityCreated( + await pubsub.entityCreated( EntityType.HIGHLIGHT, { id: libraryItemId, highlights: [newHighlight], userId }, userId, @@ -140,9 +141,9 @@ export const updateHighlight = async ( }) const libraryItemId = updatedHighlight.libraryItem.id - await pubsub.entityUpdated( + await pubsub.entityUpdated( EntityType.HIGHLIGHT, - { id: libraryItemId, highlights: [highlight], userId }, + { id: libraryItemId, highlights: [highlight], userId } as ItemEvent, userId, libraryItemId ) diff --git a/packages/api/src/services/labels.ts b/packages/api/src/services/labels.ts index 1f05d14c1..4a23f30c0 100644 --- a/packages/api/src/services/labels.ts +++ b/packages/api/src/services/labels.ts @@ -8,17 +8,9 @@ import { CreateLabelInput, labelRepository } from '../repository/label' import { bulkEnqueueUpdateLabels } from '../utils/createTask' import { logger } from '../utils/logger' import { findHighlightById } from './highlights' -import { findLibraryItemIdsByLabelId, UpdateItemEvent } from './library_item' +import { findLibraryItemIdsByLabelId, ItemEvent } from './library_item' -type AddLabelsToLibraryItemEvent = { - pageId: string - labels: DeepPartial