From 6531dd30e9fcd372c7de3c367d17c2b291ebddba Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Fri, 14 Jul 2023 12:08:08 +0800 Subject: [PATCH 1/3] skip rss items which are published before 48 hours --- packages/rss-handler/src/index.ts | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/packages/rss-handler/src/index.ts b/packages/rss-handler/src/index.ts index 99c3555aa..1ee659a8a 100644 --- a/packages/rss-handler/src/index.ts +++ b/packages/rss-handler/src/index.ts @@ -114,15 +114,20 @@ export const rssHandler = Sentry.GCPFunction.wrapHttpFunction( // save each item in the feed for await (const item of feed.items) { - const publishedAt = item.pubDate || item.isoDate - console.log('Processing feed item', item.link, publishedAt) + const publishedAtString = item.pubDate || item.isoDate + console.log('Processing feed item', item.link, publishedAtString) - if (!item.link || !publishedAt) { + if (!item.link || !publishedAtString) { console.log('Invalid feed item', item) continue } - if (new Date(publishedAt) <= new Date(lastFetchedAt)) { + // skip old items and items that were published before 48h + const publishedAt = new Date(publishedAtString) + if ( + publishedAt < new Date(Date.now() - 48 * 60 * 60 * 1000) || + publishedAt < new Date(lastFetchedAt) + ) { console.log('Skipping old feed item', item.link) continue } From 153662ce6a8535003cafceb9212a79bd8f5dc5f2 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Fri, 14 Jul 2023 12:21:34 +0800 Subject: [PATCH 2/3] set savedAt and publishedAt timestamp for rss item --- packages/api/src/generated/graphql.ts | 2 ++ packages/api/src/generated/schema.graphql | 2 ++ packages/api/src/schema.ts | 2 ++ packages/api/src/services/save_page.ts | 12 +++++++++--- packages/puppeteer-parse/index.js | 8 +++++++- packages/rss-handler/src/index.ts | 2 ++ 6 files changed, 24 insertions(+), 4 deletions(-) diff --git a/packages/api/src/generated/graphql.ts b/packages/api/src/generated/graphql.ts index 0124ea373..ad9738e17 100644 --- a/packages/api/src/generated/graphql.ts +++ b/packages/api/src/generated/graphql.ts @@ -2257,7 +2257,9 @@ export type SavePageInput = { labels?: InputMaybe>; originalContent: Scalars['String']; parseResult?: InputMaybe; + publishedAt?: InputMaybe; rssFeedUrl?: InputMaybe; + savedAt?: InputMaybe; source: Scalars['String']; state?: InputMaybe; title?: InputMaybe; diff --git a/packages/api/src/generated/schema.graphql b/packages/api/src/generated/schema.graphql index 6fe0e53b5..d069781f2 100644 --- a/packages/api/src/generated/schema.graphql +++ b/packages/api/src/generated/schema.graphql @@ -1637,7 +1637,9 @@ input SavePageInput { labels: [CreateLabelInput!] originalContent: String! parseResult: ParseResult + publishedAt: Date rssFeedUrl: String + savedAt: Date source: String! state: ArticleSavingRequestStatus title: String diff --git a/packages/api/src/schema.ts b/packages/api/src/schema.ts index bde299fd6..907c1558e 100755 --- a/packages/api/src/schema.ts +++ b/packages/api/src/schema.ts @@ -562,6 +562,8 @@ const schema = gql` state: ArticleSavingRequestStatus labels: [CreateLabelInput!] rssFeedUrl: String + savedAt: Date + publishedAt: Date } input SaveUrlInput { diff --git a/packages/api/src/services/save_page.ts b/packages/api/src/services/save_page.ts index 158e9eb10..cbc025ba1 100644 --- a/packages/api/src/services/save_page.ts +++ b/packages/api/src/services/save_page.ts @@ -104,6 +104,8 @@ export const savePage = async ( originalHtml: parseResult.domContent, canonicalUrl: parseResult.canonicalUrl, rssFeedUrl: input.rssFeedUrl, + saveTime: input.savedAt ? new Date(input.savedAt) : undefined, + publishedAt: input.publishedAt ? new Date(input.publishedAt) : undefined, }) // save state @@ -223,6 +225,7 @@ export const parsedContentToPage = ({ uploadFileId, saveTime, rssFeedUrl, + publishedAt, }: { url: string userId: string @@ -239,6 +242,7 @@ export const parsedContentToPage = ({ uploadFileId?: string | null saveTime?: Date rssFeedUrl?: string | null + publishedAt?: Date | null }): Page => { return { id: pageId || '', @@ -259,13 +263,15 @@ export const parsedContentToPage = ({ pageType, hash: uploadFileHash || stringToHash(parsedContent?.content || url), image: parsedContent?.previewImage ?? undefined, - publishedAt: validatedDate(parsedContent?.publishedDate ?? undefined), + publishedAt: validatedDate( + publishedAt || parsedContent?.publishedDate || undefined + ), uploadFileId, readingProgressPercent: 0, readingProgressAnchorIndex: 0, state: ArticleSavingRequestStatus.Succeeded, - createdAt: saveTime || new Date(), - savedAt: saveTime || new Date(), + createdAt: validatedDate(saveTime) || new Date(), + savedAt: validatedDate(saveTime) || new Date(), siteName: parsedContent?.siteName ?? undefined, language: parsedContent?.language ?? undefined, siteIcon: parsedContent?.siteIcon ?? undefined, diff --git a/packages/puppeteer-parse/index.js b/packages/puppeteer-parse/index.js index 355d3f907..6cac26b65 100644 --- a/packages/puppeteer-parse/index.js +++ b/packages/puppeteer-parse/index.js @@ -335,7 +335,9 @@ async function fetchContent(req, res) { const urlStr = (req.query ? req.query.url : undefined) || (req.body ? req.body.url : undefined); const locale = (req.query ? req.query.locale : undefined) || (req.body ? req.body.locale : undefined); const timezone = (req.query ? req.query.timezone : undefined) || (req.body ? req.body.timezone : undefined); - const rssFeedUrl = (req.query ? req.query.rssFeedUrl : undefined) || (req.body ? req.body.rssFeedUrl : undefined); + const rssFeedUrl = req.body.rssFeedUrl; + const savedAt = req.body.savedAt; + const publishedAt = req.body.publishedAt; let logRecord = { url: urlStr, @@ -350,6 +352,8 @@ async function fetchContent(req, res) { locale, timezone, rssFeedUrl, + savedAt, + publishedAt, }; console.info(`Article parsing request`, logRecord); @@ -456,6 +460,8 @@ async function fetchContent(req, res) { state, labels, rssFeedUrl, + savedAt, + publishedAt, }); if (!apiResponse) { logRecord.error = 'error while saving page'; diff --git a/packages/rss-handler/src/index.ts b/packages/rss-handler/src/index.ts index 1ee659a8a..07ed4b49c 100644 --- a/packages/rss-handler/src/index.ts +++ b/packages/rss-handler/src/index.ts @@ -139,6 +139,8 @@ export const rssHandler = Sentry.GCPFunction.wrapHttpFunction( saveRequestId: '', labels: [{ name: 'RSS', color: '#f26522' }], rssFeedUrl: feedUrl, + savedAt: publishedAt, + publishedAt, } try { From 387174c8e99ef2df618cc9714d2adcb9fdf35f40 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Fri, 14 Jul 2023 13:54:55 +0800 Subject: [PATCH 3/3] use isoDate only --- packages/rss-handler/src/index.ts | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/packages/rss-handler/src/index.ts b/packages/rss-handler/src/index.ts index 07ed4b49c..cd69a63bf 100644 --- a/packages/rss-handler/src/index.ts +++ b/packages/rss-handler/src/index.ts @@ -114,16 +114,15 @@ export const rssHandler = Sentry.GCPFunction.wrapHttpFunction( // save each item in the feed for await (const item of feed.items) { - const publishedAtString = item.pubDate || item.isoDate - console.log('Processing feed item', item.link, publishedAtString) + console.log('Processing feed item', item.link, item.isoDate) - if (!item.link || !publishedAtString) { + if (!item.link || !item.isoDate) { console.log('Invalid feed item', item) continue } // skip old items and items that were published before 48h - const publishedAt = new Date(publishedAtString) + const publishedAt = new Date(item.isoDate) if ( publishedAt < new Date(Date.now() - 48 * 60 * 60 * 1000) || publishedAt < new Date(lastFetchedAt)