diff --git a/packages/api/src/elastic/types.ts b/packages/api/src/elastic/types.ts index 6ea001506..eb38976c2 100644 --- a/packages/api/src/elastic/types.ts +++ b/packages/api/src/elastic/types.ts @@ -62,6 +62,9 @@ export enum PageType { Unknown = 'UNKNOWN', Website = 'WEBSITE', Highlights = 'HIGHLIGHTS', + Tweet = 'TWEET', + Video = 'VIDEO', + Image = 'IMAGE', } export enum ArticleSavingRequestStatus { diff --git a/packages/api/src/generated/graphql.ts b/packages/api/src/generated/graphql.ts index 222de229a..22d51b8b4 100644 --- a/packages/api/src/generated/graphql.ts +++ b/packages/api/src/generated/graphql.ts @@ -1683,8 +1683,11 @@ export enum PageType { Book = 'BOOK', File = 'FILE', Highlights = 'HIGHLIGHTS', + Image = 'IMAGE', Profile = 'PROFILE', + Tweet = 'TWEET', Unknown = 'UNKNOWN', + Video = 'VIDEO', Website = 'WEBSITE' } diff --git a/packages/api/src/generated/schema.graphql b/packages/api/src/generated/schema.graphql index bee6ae46c..247ff2eda 100644 --- a/packages/api/src/generated/schema.graphql +++ b/packages/api/src/generated/schema.graphql @@ -1219,8 +1219,11 @@ enum PageType { BOOK FILE HIGHLIGHTS + IMAGE PROFILE + TWEET UNKNOWN + VIDEO WEBSITE } diff --git a/packages/api/src/resolvers/article/index.ts b/packages/api/src/resolvers/article/index.ts index 041d930c1..abe36814b 100644 --- a/packages/api/src/resolvers/article/index.ts +++ b/packages/api/src/resolvers/article/index.ts @@ -409,7 +409,7 @@ export const getArticleResolver: ResolverFn< Record, WithDataSourcesContext, QueryArticleArgs -> = async (_obj, { slug, format }, { claims, pubsub }, info) => { +> = async (_obj, { slug, format }, { claims }, info) => { try { if (!claims?.uid) { return { errorCodes: [ArticleErrorCode.Unauthorized] } @@ -950,7 +950,6 @@ export const searchResolver = authorized< originalArticleUrl: r.url, publishedAt: validatedDate(r.publishedAt), ownedByViewer: r.userId === claims.uid, - pageType: r.pageType || PageType.Unknown, siteIcon, } as SearchItem, cursor: endCursor, @@ -1054,7 +1053,6 @@ export const updatesSinceResolver = authorized< p.pageType === PageType.File ? ContentReader.Pdf : ContentReader.Web, - pageType: p.pageType || PageType.Unknown, } as SearchItem, cursor: endCursor, itemID: p.id, diff --git a/packages/api/src/resolvers/function_resolvers.ts b/packages/api/src/resolvers/function_resolvers.ts index 9d06dac42..4e2228ccb 100644 --- a/packages/api/src/resolvers/function_resolvers.ts +++ b/packages/api/src/resolvers/function_resolvers.ts @@ -559,6 +559,9 @@ export const functionResolvers = { } return item.url }, + pageType(item: SearchItem) { + return item.pageType || PageType.Unknown + }, }, ...resultResolveTypeResolver('Login'), ...resultResolveTypeResolver('LogOut'), diff --git a/packages/api/src/schema.ts b/packages/api/src/schema.ts index d749e97d6..3beeaff93 100755 --- a/packages/api/src/schema.ts +++ b/packages/api/src/schema.ts @@ -309,6 +309,9 @@ const schema = gql` WEBSITE HIGHLIGHTS UNKNOWN + TWEET + VIDEO + IMAGE } type Page { diff --git a/packages/api/src/utils/parser.ts b/packages/api/src/utils/parser.ts index e940e98d3..8316a4619 100644 --- a/packages/api/src/utils/parser.ts +++ b/packages/api/src/utils/parser.ts @@ -1,29 +1,29 @@ /* eslint-disable @typescript-eslint/no-unsafe-member-access */ /* eslint-disable @typescript-eslint/no-unsafe-assignment */ /* eslint-disable @typescript-eslint/no-unused-vars */ +import { preParseContent } from '@omnivore/content-handler' import { Readability } from '@omnivore/readability' -import createDOMPurify, { SanitizeElementHookEvent } from 'dompurify' -import { PageType, PreparedDocumentInput } from '../generated/graphql' -import { buildLogger, LogRecord } from './logger' -import { createImageProxyUrl } from './imageproxy' +import addressparser from 'addressparser' import axios from 'axios' +import createDOMPurify, { SanitizeElementHookEvent } from 'dompurify' import * as hljs from 'highlightjs' import { decode } from 'html-entities' +import * as jwt from 'jsonwebtoken' import { parseHTML } from 'linkedom' -import { getRepository } from '../entity/utils' -import { User } from '../entity/user' +import { NodeHtmlMarkdown } from 'node-html-markdown' import { ILike } from 'typeorm' +import { promisify } from 'util' import { v4 as uuid } from 'uuid' -import addressparser from 'addressparser' -import { preParseContent } from '@omnivore/content-handler' +import { User } from '../entity/user' +import { getRepository } from '../entity/utils' +import { env } from '../env' +import { PageType, PreparedDocumentInput } from '../generated/graphql' import { EmbeddedHighlightData, findEmbeddedHighlight, } from './highlightGenerator' -import { NodeHtmlMarkdown } from 'node-html-markdown' -import { promisify } from 'util' -import * as jwt from 'jsonwebtoken' -import { env } from '../env' +import { createImageProxyUrl } from './imageproxy' +import { buildLogger, LogRecord } from './logger' const logger = buildLogger('utils.parse') const signToken = promisify(jwt.sign) @@ -118,12 +118,20 @@ const parseOriginalContent = (document: Document): PageType => { return PageType.Profile case 'website': return PageType.Website + case 'tweet': + return PageType.Tweet + case 'image': + return PageType.Image + default: + if (content.toLowerCase().startsWith('video')) { + return PageType.Video + } + return PageType.Unknown } } catch (error) { logger.error('Error extracting og:type from content', error) + return PageType.Unknown } - - return PageType.Unknown } const getPurifiedContent = (html: string): Document => { diff --git a/packages/content-handler/src/websites/image-handler.ts b/packages/content-handler/src/websites/image-handler.ts index 2ccfcd610..9872f4c21 100644 --- a/packages/content-handler/src/websites/image-handler.ts +++ b/packages/content-handler/src/websites/image-handler.ts @@ -19,6 +19,7 @@ export class ImageHandler extends ContentHandler { ${title} +
diff --git a/packages/content-handler/src/websites/piped-video-handler.ts b/packages/content-handler/src/websites/piped-video-handler.ts index ffa8c050b..bae7fa3b1 100644 --- a/packages/content-handler/src/websites/piped-video-handler.ts +++ b/packages/content-handler/src/websites/piped-video-handler.ts @@ -69,6 +69,7 @@ export class PipedVideoHandler extends ContentHandler { + diff --git a/packages/content-handler/src/websites/twitter-handler.ts b/packages/content-handler/src/websites/twitter-handler.ts index 1116d5b7f..c196ec987 100644 --- a/packages/content-handler/src/websites/twitter-handler.ts +++ b/packages/content-handler/src/websites/twitter-handler.ts @@ -376,6 +376,7 @@ export class TwitterHandler extends ContentHandler { +
diff --git a/packages/content-handler/src/websites/youtube-handler.ts b/packages/content-handler/src/websites/youtube-handler.ts index 58bc12dd4..e86eda113 100644 --- a/packages/content-handler/src/websites/youtube-handler.ts +++ b/packages/content-handler/src/websites/youtube-handler.ts @@ -83,6 +83,7 @@ export class YoutubeHandler extends ContentHandler { +