Merge pull request #1616 from omnivore-app/html-to-markdown

html to markdown

packages/api/package.json

@@ -74,6 +74,7 @@
     "luxon": "^2.3.1",
     "microsoft-cognitiveservices-speech-sdk": "^1.22.0",
     "nanoid": "^3.1.25",
+    "node-html-markdown": "^1.3.0",
     "nodemailer": "^6.7.3",
     "normalize-url": "^6.1.0",
     "oauth": "^0.9.15",

packages/api/src/elastic/pages.ts

@@ -388,6 +388,7 @@ export const searchPages = async (
       termFilters,
       matchFilters,
       ids,
+      includeContent,
     } = args
     // default order is descending
     const sortOrder = sort?.order || SortOrder.DESCENDING
@@ -424,7 +425,7 @@ export const searchPages = async (
       from,
       size,
       _source: {
-        excludes: ['originalHtml', 'content'],
+        excludes: includeContent ? [] : ['originalHtml', 'content'],
       },
     }
 
@@ -497,7 +498,7 @@ export const searchPages = async (
     return [
       response.body.hits.hits.map((hit: { _source: Page; _id: string }) => ({
         ...hit._source,
-        content: '',
+        content: includeContent ? hit._source.content : '',
         id: hit._id,
       })),
       response.body.hits.total.value,

packages/api/src/elastic/types.ts

@@ -286,6 +286,7 @@ export interface SearchItem {
   siteName?: string
   siteIcon?: string
   recommendations?: Recommendation[]
+  content?: string
 }
 
 const keys = ['_id', 'url', 'slug', 'userId', 'uploadFileId', 'state'] as const
@@ -315,4 +316,5 @@ export interface PageSearchArgs {
   includeDeleted?: boolean
   ids?: string[]
   recommendedBy?: string
+  includeContent?: boolean
 }

packages/api/src/generated/graphql.ts

@@ -1685,6 +1685,7 @@ export type Query = {
 
 
 export type QueryArticleArgs = {
+  format?: InputMaybe<Scalars['String']>;
   slug: Scalars['String'];
   username: Scalars['String'];
 };
@@ -1736,6 +1737,8 @@ export type QueryRulesArgs = {
 export type QuerySearchArgs = {
   after?: InputMaybe<Scalars['String']>;
   first?: InputMaybe<Scalars['Int']>;
+  format?: InputMaybe<Scalars['String']>;
+  includeContent?: InputMaybe<Scalars['Boolean']>;
   query?: InputMaybe<Scalars['String']>;
 };
 
@@ -2127,6 +2130,7 @@ export type SearchItem = {
   __typename?: 'SearchItem';
   annotation?: Maybe<Scalars['String']>;
   author?: Maybe<Scalars['String']>;
+  content?: Maybe<Scalars['String']>;
   contentReader: ContentReader;
   createdAt: Scalars['Date'];
   description?: Maybe<Scalars['String']>;
@@ -5173,6 +5177,7 @@ export type SearchErrorResolvers<ContextType = ResolverContext, ParentType exten
 export type SearchItemResolvers<ContextType = ResolverContext, ParentType extends ResolversParentTypes['SearchItem'] = ResolversParentTypes['SearchItem']> = {
   annotation?: Resolver<Maybe<ResolversTypes['String']>, ParentType, ContextType>;
   author?: Resolver<Maybe<ResolversTypes['String']>, ParentType, ContextType>;
+  content?: Resolver<Maybe<ResolversTypes['String']>, ParentType, ContextType>;
   contentReader?: Resolver<ResolversTypes['ContentReader'], ParentType, ContextType>;
   createdAt?: Resolver<ResolversTypes['Date'], ParentType, ContextType>;
   description?: Resolver<Maybe<ResolversTypes['String']>, ParentType, ContextType>;

packages/api/src/generated/schema.graphql

@@ -1201,7 +1201,7 @@ type Profile {
 
 type Query {
   apiKeys: ApiKeysResult!
-  article(slug: String!, username: String!): ArticleResult!
+  article(format: String, slug: String!, username: String!): ArticleResult!
   articleSavingRequest(id: ID!): ArticleSavingRequestResult!
   articles(after: String, first: Int, includePending: Boolean, query: String, sharedOnly: Boolean, sort: SortParams): ArticlesResult!
   deviceTokens: DeviceTokensResult!
@@ -1219,7 +1219,7 @@ type Query {
   recentSearches: RecentSearchesResult!
   reminder(linkId: ID!): ReminderResult!
   rules(enabled: Boolean): RulesResult!
-  search(after: String, first: Int, query: String): SearchResult!
+  search(after: String, first: Int, format: String, includeContent: Boolean, query: String): SearchResult!
   sendInstallInstructions: SendInstallInstructionsResult!
   sharedArticle(selectedHighlightId: String, slug: String!, username: String!): SharedArticleResult!
   subscriptions(sort: SortParams): SubscriptionsResult!
@@ -1548,6 +1548,7 @@ enum SearchErrorCode {
 type SearchItem {
   annotation: String
   author: String
+  content: String
   contentReader: ContentReader!
   createdAt: Date!
   description: String

packages/api/src/resolvers/article/index.ts

@@ -64,6 +64,7 @@ import {
   validatedDate,
 } from '../../utils/helpers'
 import {
+  htmlToMarkdown,
   ParsedContentPuppeteer,
   parsePreparedContent,
 } from '../../utils/parser'
@@ -401,7 +402,7 @@ export const getArticleResolver: ResolverFn<
   Record<string, unknown>,
   WithDataSourcesContext,
   QueryArticleArgs
-> = async (_obj, { slug }, { claims, pubsub }, info) => {
+> = async (_obj, { slug, format }, { claims, pubsub }, info) => {
   try {
     if (!claims?.uid) {
       return { errorCodes: [ArticleErrorCode.Unauthorized] }
@@ -443,6 +444,10 @@ export const getArticleResolver: ResolverFn<
       page.content = UNPARSEABLE_CONTENT
     }
 
+    if (format === 'markdown') {
+      page.content = htmlToMarkdown(page.content)
+    }
+
     return {
       article: { ...page, isArchived: !!page.archivedAt, linkId: page.id },
     }
@@ -869,6 +874,7 @@ export const searchResolver = authorized<
         size: first + 1, // fetch one more item to get next cursor
         sort: searchQuery.sortParams,
         includePending: true,
+        includeContent: params.includeContent ?? false,
         ...searchQuery,
       },
       claims.uid
@@ -890,6 +896,11 @@ export const searchResolver = authorized<
     if (siteIcon && !isBase64Image(siteIcon)) {
       siteIcon = createImageProxyUrl(siteIcon, 128, 128)
     }
+
+    if (params.includeContent && params.format === 'markdown' && r.content) {
+      r.content = htmlToMarkdown(r.content)
+    }
+
     return {
       node: {
         ...r,

packages/api/src/schema.ts

@@ -1550,6 +1550,7 @@ const schema = gql`
     siteIcon: String
     recommendations: [Recommendation!]
     wordsCount: Int
+    content: String
   }
 
   type SearchItemEdge {
@@ -2432,7 +2433,7 @@ const schema = gql`
       query: String
       includePending: Boolean
     ): ArticlesResult!
-    article(username: String!, slug: String!): ArticleResult!
+    article(username: String!, slug: String!, format: String): ArticleResult!
     sharedArticle(
       username: String!
       slug: String!
@@ -2453,7 +2454,13 @@ const schema = gql`
     newsletterEmails: NewsletterEmailsResult!
     reminder(linkId: ID!): ReminderResult!
     labels: LabelsResult!
-    search(after: String, first: Int, query: String): SearchResult!
+    search(
+      after: String
+      first: Int
+      query: String
+      includeContent: Boolean
+      format: String
+    ): SearchResult!
     subscriptions(sort: SortParams): SubscriptionsResult!
     sendInstallInstructions: SendInstallInstructionsResult!
     webhooks: WebhooksResult!

packages/api/src/utils/parser.ts

@@ -20,6 +20,7 @@ import {
   EmbeddedHighlightData,
   findEmbeddedHighlight,
 } from './highlightGenerator'
+import { NodeHtmlMarkdown } from 'node-html-markdown'
 
 const logger = buildLogger('utils.parse')
 
@@ -467,3 +468,17 @@ export const fetchFavicon = async (
     return undefined
   }
 }
+
+/* ********************************************************* *
+ * Re-use
+ * If using it several times, creating an instance saves time
+ * ********************************************************* */
+const nhm = new NodeHtmlMarkdown(
+  /* options (optional) */ {},
+  /* customTransformers (optional) */ undefined,
+  /* customCodeBlockTranslators (optional) */ undefined
+)
+
+export const htmlToMarkdown = (html: string) => {
+  return nhm.translate(/* html */ html)
+}

yarn.lock

@@ -20088,6 +20088,21 @@ node-gyp@^7.1.0:
     tar "^6.0.2"
     which "^2.0.2"
 
+node-html-markdown@^1.3.0:
+  version "1.3.0"
+  resolved "https://registry.yarnpkg.com/node-html-markdown/-/node-html-markdown-1.3.0.tgz#ef0b19a3bbfc0f1a880abb9ff2a0c9aa6bbff2a9"
+  integrity sha512-OeFi3QwC/cPjvVKZ114tzzu+YoR+v9UXW5RwSXGUqGb0qCl0DvP406tzdL7SFn8pZrMyzXoisfG2zcuF9+zw4g==
+  dependencies:
+    node-html-parser "^6.1.1"
+
+node-html-parser@^6.1.1:
+  version "6.1.4"
+  resolved "https://registry.yarnpkg.com/node-html-parser/-/node-html-parser-6.1.4.tgz#763cd362497e427d51fc73dda3dcd8ac52ba85a3"
+  integrity sha512-3muP9Uy/Pz7bQa9TNYVQzWJhNZMqyCx7xJle8kz2/y1UgzAUyXXShc1IcPaJy6u07CE3K5rQcRwlvHzmlySRjg==
+  dependencies:
+    css-select "^5.1.0"
+    he "1.2.0"
+
 node-int64@^0.4.0:
   version "0.4.0"
   resolved "https://registry.yarnpkg.com/node-int64/-/node-int64-0.4.0.tgz#87a9065cdb355d3182d8f94ce11188b825c68a3b"