Merge pull request #2488 from omnivore-app/fix/rss

fix/rss

packages/api/src/generated/graphql.ts

@@ -2257,7 +2257,9 @@ export type SavePageInput = {
   labels?: InputMaybe<Array<CreateLabelInput>>;
   originalContent: Scalars['String'];
   parseResult?: InputMaybe<ParseResult>;
+  publishedAt?: InputMaybe<Scalars['Date']>;
   rssFeedUrl?: InputMaybe<Scalars['String']>;
+  savedAt?: InputMaybe<Scalars['Date']>;
   source: Scalars['String'];
   state?: InputMaybe<ArticleSavingRequestStatus>;
   title?: InputMaybe<Scalars['String']>;

packages/api/src/generated/schema.graphql

@@ -1637,7 +1637,9 @@ input SavePageInput {
   labels: [CreateLabelInput!]
   originalContent: String!
   parseResult: ParseResult
+  publishedAt: Date
   rssFeedUrl: String
+  savedAt: Date
   source: String!
   state: ArticleSavingRequestStatus
   title: String

packages/api/src/schema.ts

@@ -562,6 +562,8 @@ const schema = gql`
     state: ArticleSavingRequestStatus
     labels: [CreateLabelInput!]
     rssFeedUrl: String
+    savedAt: Date
+    publishedAt: Date
   }
 
   input SaveUrlInput {

packages/api/src/services/save_page.ts

@@ -104,6 +104,8 @@ export const savePage = async (
     originalHtml: parseResult.domContent,
     canonicalUrl: parseResult.canonicalUrl,
     rssFeedUrl: input.rssFeedUrl,
+    saveTime: input.savedAt ? new Date(input.savedAt) : undefined,
+    publishedAt: input.publishedAt ? new Date(input.publishedAt) : undefined,
   })
 
   // save state
@@ -223,6 +225,7 @@ export const parsedContentToPage = ({
   uploadFileId,
   saveTime,
   rssFeedUrl,
+  publishedAt,
 }: {
   url: string
   userId: string
@@ -239,6 +242,7 @@ export const parsedContentToPage = ({
   uploadFileId?: string | null
   saveTime?: Date
   rssFeedUrl?: string | null
+  publishedAt?: Date | null
 }): Page => {
   return {
     id: pageId || '',
@@ -259,13 +263,15 @@ export const parsedContentToPage = ({
     pageType,
     hash: uploadFileHash || stringToHash(parsedContent?.content || url),
     image: parsedContent?.previewImage ?? undefined,
-    publishedAt: validatedDate(parsedContent?.publishedDate ?? undefined),
+    publishedAt: validatedDate(
+      publishedAt || parsedContent?.publishedDate || undefined
+    ),
     uploadFileId,
     readingProgressPercent: 0,
     readingProgressAnchorIndex: 0,
     state: ArticleSavingRequestStatus.Succeeded,
-    createdAt: saveTime || new Date(),
-    savedAt: saveTime || new Date(),
+    createdAt: validatedDate(saveTime) || new Date(),
+    savedAt: validatedDate(saveTime) || new Date(),
     siteName: parsedContent?.siteName ?? undefined,
     language: parsedContent?.language ?? undefined,
     siteIcon: parsedContent?.siteIcon ?? undefined,

packages/puppeteer-parse/index.js

@@ -335,7 +335,9 @@ async function fetchContent(req, res) {
   const urlStr = (req.query ? req.query.url : undefined) || (req.body ? req.body.url : undefined);
   const locale = (req.query ? req.query.locale : undefined) || (req.body ? req.body.locale : undefined);
   const timezone = (req.query ? req.query.timezone : undefined) || (req.body ? req.body.timezone : undefined);
-  const rssFeedUrl = (req.query ? req.query.rssFeedUrl : undefined) || (req.body ? req.body.rssFeedUrl : undefined);
+  const rssFeedUrl = req.body.rssFeedUrl;
+  const savedAt = req.body.savedAt;
+  const publishedAt = req.body.publishedAt;
 
   let logRecord = {
     url: urlStr,
@@ -350,6 +352,8 @@ async function fetchContent(req, res) {
     locale,
     timezone,
     rssFeedUrl,
+    savedAt,
+    publishedAt,
   };
 
   console.info(`Article parsing request`, logRecord);
@@ -456,6 +460,8 @@ async function fetchContent(req, res) {
         state,
         labels,
         rssFeedUrl,
+        savedAt,
+        publishedAt,
       });
       if (!apiResponse) {
         logRecord.error = 'error while saving page';

packages/rss-handler/src/index.ts

@@ -114,15 +114,19 @@ export const rssHandler = Sentry.GCPFunction.wrapHttpFunction(
 
       // save each item in the feed
       for await (const item of feed.items) {
-        const publishedAt = item.pubDate || item.isoDate
-        console.log('Processing feed item', item.link, publishedAt)
+        console.log('Processing feed item', item.link, item.isoDate)
 
-        if (!item.link || !publishedAt) {
+        if (!item.link || !item.isoDate) {
           console.log('Invalid feed item', item)
           continue
         }
 
-        if (new Date(publishedAt) <= new Date(lastFetchedAt)) {
+        // skip old items and items that were published before 48h
+        const publishedAt = new Date(item.isoDate)
+        if (
+          publishedAt < new Date(Date.now() - 48 * 60 * 60 * 1000) ||
+          publishedAt < new Date(lastFetchedAt)
+        ) {
           console.log('Skipping old feed item', item.link)
           continue
         }
@@ -134,6 +138,8 @@ export const rssHandler = Sentry.GCPFunction.wrapHttpFunction(
           saveRequestId: '',
           labels: [{ name: 'RSS', color: '#f26522' }],
           rssFeedUrl: feedUrl,
+          savedAt: publishedAt,
+          publishedAt,
         }
 
         try {