diff --git a/packages/api/src/resolvers/article/index.ts b/packages/api/src/resolvers/article/index.ts index b7865a5c6..1eee8e0ae 100644 --- a/packages/api/src/resolvers/article/index.ts +++ b/packages/api/src/resolvers/article/index.ts @@ -54,7 +54,6 @@ import { } from '../../utils/helpers' import { ParsedContentPuppeteer, - parseOriginalContent, parsePreparedContent, } from '../../utils/parser' import { isSiteBlockedForParse } from '../../utils/blocked' @@ -236,8 +235,7 @@ export const createArticleResolver = authorized< parsedContent = parseResults.parsedContent canonicalUrl = parseResults.canonicalUrl domContent = parseResults.domContent - - pageType = parseOriginalContent(url, domContent) + pageType = parseResults.pageType } else if (!preparedDocument?.document) { // We have a URL but no document, so we try to send this to puppeteer // and return a dummy response. diff --git a/packages/api/src/services/save_email.ts b/packages/api/src/services/save_email.ts index e62175c47..95ee5c576 100644 --- a/packages/api/src/services/save_email.ts +++ b/packages/api/src/services/save_email.ts @@ -1,9 +1,5 @@ import { generateSlug, stringToHash, validatedDate } from '../utils/helpers' -import { - parseOriginalContent, - parsePreparedContent, - parseUrlMetadata, -} from '../utils/parser' +import { parsePreparedContent, parseUrlMetadata } from '../utils/parser' import normalizeUrl from 'normalize-url' import { PubsubClient } from '../datalayer/pubsub' import { ArticleSavingRequestStatus, Page } from '../elastic/types' @@ -44,7 +40,6 @@ export const saveEmail = async ( const content = parseResult.parsedContent?.content || input.originalContent const slug = generateSlug(title) - const pageType = parseOriginalContent(url, input.originalContent) const metadata = await parseUrlMetadata(url) const articleToSave: Page = { @@ -60,7 +55,7 @@ export const saveEmail = async ( stripHash: true, stripWWW: false, }), - pageType: pageType, + pageType: parseResult.pageType, hash: stringToHash(content), image: metadata?.previewImage || parseResult.parsedContent?.previewImage, publishedAt: validatedDate(parseResult.parsedContent?.publishedDate), diff --git a/packages/api/src/services/save_page.ts b/packages/api/src/services/save_page.ts index e6fd79110..f56f95884 100644 --- a/packages/api/src/services/save_page.ts +++ b/packages/api/src/services/save_page.ts @@ -3,7 +3,7 @@ import { homePageURL } from '../env' import { Maybe, SavePageInput, SaveResult } from '../generated/graphql' import { DataModels } from '../resolvers/types' import { generateSlug, stringToHash, validatedDate } from '../utils/helpers' -import { parseOriginalContent, parsePreparedContent } from '../utils/parser' +import { parsePreparedContent } from '../utils/parser' import normalizeUrl from 'normalize-url' import { createPageSaveRequest } from './create_page_save_request' @@ -72,8 +72,6 @@ export const savePage = async ( }, }) - const pageType = parseOriginalContent(input.url, input.originalContent) - const articleToSave: Page = { id: input.clientRequestId, slug, @@ -87,7 +85,7 @@ export const savePage = async ( stripHash: true, stripWWW: false, }), - pageType: pageType, + pageType: parseResult.pageType, hash: stringToHash(parseResult.parsedContent?.content || input.url), image: parseResult.parsedContent?.previewImage, publishedAt: validatedDate(parseResult.parsedContent?.publishedDate), diff --git a/packages/api/src/utils/parser.ts b/packages/api/src/utils/parser.ts index 3e7de931f..94e8502cb 100644 --- a/packages/api/src/utils/parser.ts +++ b/packages/api/src/utils/parser.ts @@ -80,6 +80,7 @@ export type ParsedContentPuppeteer = { domContent: string parsedContent: Readability.ParseResult | null canonicalUrl?: string | null + pageType: PageType } /* eslint-disable @typescript-eslint/no-explicit-any */ @@ -101,9 +102,8 @@ type ArticleParseLogRecord = LogRecord & { const DEBUG_MODE = process.env.DEBUG === 'true' || false -export const parseOriginalContent = (url: string, html: string): PageType => { +const parseOriginalContent = (window: DOMWindow): PageType => { try { - const { window } = new JSDOM(html, { url }) const e = window.document.querySelector("head meta[property='og:type']") const content = e?.getAttribute('content') if (!content) { @@ -121,7 +121,7 @@ export const parseOriginalContent = (url: string, html: string): PageType => { return PageType.Website } } catch (error) { - logger.error('Error extracting og:type from content for url', url, error) + logger.error('Error extracting og:type from content', error) } return PageType.Unknown @@ -232,6 +232,7 @@ export const parsePreparedContent = async ( canonicalUrl: url, parsedContent: null, domContent: preparedDocument.document, + pageType: PageType.Unknown, } } @@ -310,6 +311,7 @@ export const parsePreparedContent = async ( domContent: preparedDocument.document, parsedContent: article, canonicalUrl, + pageType: parseOriginalContent(window), } }