Update savePage API to accept parsed content in the param

This commit is contained in:
Hongbo Wu 2022-12-27 21:59:31 +08:00
parent 6a49689d1e
commit e9b31e375f
5 changed files with 72 additions and 9 deletions

View file

@ -1612,6 +1612,21 @@ export enum PageType {
Website = 'WEBSITE'
}
export type ParseResult = {
byline: Scalars['String'];
content: Scalars['String'];
dir: Scalars['String'];
excerpt: Scalars['String'];
language?: InputMaybe<Scalars['String']>;
length: Scalars['Int'];
previewImage?: InputMaybe<Scalars['String']>;
publishedDate?: InputMaybe<Scalars['Date']>;
siteIcon?: InputMaybe<Scalars['String']>;
siteName?: InputMaybe<Scalars['String']>;
textContent: Scalars['String'];
title: Scalars['String'];
};
export type PreparedDocumentInput = {
document: Scalars['String'];
pageInfo: PageInfoInput;
@ -2071,6 +2086,7 @@ export type SaveFilterSuccess = {
export type SavePageInput = {
clientRequestId: Scalars['ID'];
originalContent: Scalars['String'];
parseResult?: InputMaybe<ParseResult>;
source: Scalars['String'];
title?: InputMaybe<Scalars['String']>;
url: Scalars['String'];
@ -3291,6 +3307,7 @@ export type ResolversTypes = {
PageInfo: ResolverTypeWrapper<PageInfo>;
PageInfoInput: PageInfoInput;
PageType: PageType;
ParseResult: ParseResult;
PreparedDocumentInput: PreparedDocumentInput;
Profile: ResolverTypeWrapper<Profile>;
Query: ResolverTypeWrapper<{}>;
@ -3704,6 +3721,7 @@ export type ResolversParentTypes = {
Page: Page;
PageInfo: PageInfo;
PageInfoInput: PageInfoInput;
ParseResult: ParseResult;
PreparedDocumentInput: PreparedDocumentInput;
Profile: Profile;
Query: {};

View file

@ -1169,6 +1169,21 @@ enum PageType {
WEBSITE
}
input ParseResult {
byline: String!
content: String!
dir: String!
excerpt: String!
language: String
length: Int!
previewImage: String
publishedDate: Date
siteIcon: String
siteName: String
textContent: String!
title: String!
}
input PreparedDocumentInput {
document: String!
pageInfo: PageInfoInput!
@ -1501,6 +1516,7 @@ type SaveFilterSuccess {
input SavePageInput {
clientRequestId: ID!
originalContent: String!
parseResult: ParseResult
source: String!
title: String
url: String!

View file

@ -526,12 +526,28 @@ const schema = gql`
uploadFileId: ID!
}
input ParseResult {
title: String!
byline: String!
dir: String!
content: String!
textContent: String!
length: Int!
excerpt: String!
siteName: String
siteIcon: String
previewImage: String
publishedDate: Date
language: String
}
input SavePageInput {
url: String!
source: String!
clientRequestId: ID!
title: String
originalContent: String!
parseResult: ParseResult
}
input SaveUrlInput {

View file

@ -77,13 +77,17 @@ export const savePage = async (
input: SavePageInput
): Promise<SaveResult> => {
const [slug, croppedPathname] = createSlug(input.url, input.title)
const parseResult = await parsePreparedContent(input.url, {
document: input.originalContent,
pageInfo: {
title: input.title,
canonicalUrl: input.url,
const parseResult = await parsePreparedContent(
input.url,
{
document: input.originalContent,
pageInfo: {
title: input.title,
canonicalUrl: input.url,
},
},
})
input.parseResult
)
const articleToSave = parsedContentToPage({
url: input.url,

View file

@ -17,8 +17,8 @@ import { v4 as uuid } from 'uuid'
import addressparser from 'addressparser'
import { preParseContent } from '@omnivore/content-handler'
import {
findEmbeddedHighlight,
EmbeddedHighlightData,
findEmbeddedHighlight,
} from './highlightGenerator'
const logger = buildLogger('utils.parse')
@ -174,6 +174,7 @@ const getReadabilityResult = async (
export const parsePreparedContent = async (
url: string,
preparedDocument: PreparedDocumentInput,
parseResult?: Readability.ParseResult | null,
isNewsletter?: boolean,
allowRetry = true
): Promise<ParsedContentPuppeteer> => {
@ -208,13 +209,21 @@ export const parsePreparedContent = async (
preParsedDom && (dom = preParsedDom)
try {
article = await getReadabilityResult(url, document, dom, isNewsletter)
article =
parseResult ||
(await getReadabilityResult(url, document, dom, isNewsletter))
if (!article?.textContent && allowRetry) {
const newDocument = {
...preparedDocument,
document: '<html>' + preparedDocument.document + '</html>',
}
return parsePreparedContent(url, newDocument, isNewsletter, false)
return parsePreparedContent(
url,
newDocument,
parseResult,
isNewsletter,
false
)
}
// Format code blocks