Merge pull request #689 from omnivore-app/save-language

Save and filter by language

packages/api/src/elastic/pages.ts

@@ -17,7 +17,7 @@ import {
   SortBy,
   SortOrder,
   SortParams,
-  SubscriptionFilter,
+  TermFilter,
 } from '../utils/search'
 import { client, INDEX_ALIAS } from './index'
 import { EntityType } from '../datalayer/pubsub'
@@ -158,14 +158,13 @@ const appendDateFilters = (body: SearchBody, filters: DateFilter[]): void => {
   })
 }
 
-const appendSubscriptionFilter = (
-  body: SearchBody,
-  filter: SubscriptionFilter
-): void => {
-  body.query.bool.filter.push({
-    term: {
-      subscription: filter.name,
-    },
+const appendTermFilters = (body: SearchBody, filters: TermFilter[]): void => {
+  filters.forEach((filter) => {
+    body.query.bool.filter.push({
+      term: {
+        [filter.field]: filter.value,
+      },
+    })
   })
 }
 
@@ -338,7 +337,7 @@ export const searchPages = async (
     labelFilters: LabelFilter[]
     hasFilters: HasFilter[]
     dateFilters: DateFilter[]
-    subscriptionFilter?: SubscriptionFilter
+    termFilters?: TermFilter[]
     includePending?: boolean | null
   },
   userId: string
@@ -355,7 +354,7 @@ export const searchPages = async (
       inFilter = InFilter.ALL,
       hasFilters,
       dateFilters,
-      subscriptionFilter,
+      termFilters,
     } = args
     // default order is descending
     const sortOrder = sort?.order || SortOrder.DESCENDING
@@ -421,8 +420,8 @@ export const searchPages = async (
     if (dateFilters.length > 0) {
       appendDateFilters(body, dateFilters)
     }
-    if (subscriptionFilter) {
-      appendSubscriptionFilter(body, subscriptionFilter)
+    if (termFilters) {
+      appendTermFilters(body, termFilters)
     }
 
     if (!args.includePending) {

packages/api/src/elastic/types.ts

@@ -8,15 +8,9 @@ export interface SearchBody {
       filter: (
         | {
             term: {
-              userId: string
+              [K: string]: string
             }
           }
-        | {
-            term: {
-              subscription: string
-            }
-          }
-        | { term: { pageType: string } }
         | { exists: { field: string } }
         | {
             range: {
@@ -203,6 +197,7 @@ export interface Page {
   unsubHttpUrl?: string
   state: ArticleSavingRequestStatus
   taskName?: string
+  language?: string
 }
 
 export interface SearchItem {
@@ -226,6 +221,7 @@ export interface SearchItem {
   readingProgressAnchorIndex?: number
   userId: string
   state?: ArticleSavingRequestStatus
+  language?: string
 }
 
 const keys = ['_id', 'url', 'slug', 'userId', 'uploadFileId', 'state'] as const

packages/api/src/generated/graphql.ts

@@ -72,6 +72,7 @@ export type Article = {
   image?: Maybe<Scalars['String']>;
   isArchived: Scalars['Boolean'];
   labels?: Maybe<Array<Label>>;
+  language?: Maybe<Scalars['String']>;
   linkId?: Maybe<Scalars['ID']>;
   originalArticleUrl?: Maybe<Scalars['String']>;
   originalHtml?: Maybe<Scalars['String']>;
@@ -1411,6 +1412,7 @@ export type SearchItem = {
   image?: Maybe<Scalars['String']>;
   isArchived: Scalars['Boolean'];
   labels?: Maybe<Array<Label>>;
+  language?: Maybe<Scalars['String']>;
   originalArticleUrl?: Maybe<Scalars['String']>;
   ownedByViewer?: Maybe<Scalars['Boolean']>;
   pageId?: Maybe<Scalars['ID']>;
@@ -2759,6 +2761,7 @@ export type ArticleResolvers<ContextType = ResolverContext, ParentType extends R
   image?: Resolver<Maybe<ResolversTypes['String']>, ParentType, ContextType>;
   isArchived?: Resolver<ResolversTypes['Boolean'], ParentType, ContextType>;
   labels?: Resolver<Maybe<Array<ResolversTypes['Label']>>, ParentType, ContextType>;
+  language?: Resolver<Maybe<ResolversTypes['String']>, ParentType, ContextType>;
   linkId?: Resolver<Maybe<ResolversTypes['ID']>, ParentType, ContextType>;
   originalArticleUrl?: Resolver<Maybe<ResolversTypes['String']>, ParentType, ContextType>;
   originalHtml?: Resolver<Maybe<ResolversTypes['String']>, ParentType, ContextType>;
@@ -3498,6 +3501,7 @@ export type SearchItemResolvers<ContextType = ResolverContext, ParentType extend
   image?: Resolver<Maybe<ResolversTypes['String']>, ParentType, ContextType>;
   isArchived?: Resolver<ResolversTypes['Boolean'], ParentType, ContextType>;
   labels?: Resolver<Maybe<Array<ResolversTypes['Label']>>, ParentType, ContextType>;
+  language?: Resolver<Maybe<ResolversTypes['String']>, ParentType, ContextType>;
   originalArticleUrl?: Resolver<Maybe<ResolversTypes['String']>, ParentType, ContextType>;
   ownedByViewer?: Resolver<Maybe<ResolversTypes['Boolean']>, ParentType, ContextType>;
   pageId?: Resolver<Maybe<ResolversTypes['ID']>, ParentType, ContextType>;

packages/api/src/generated/schema.graphql

@@ -51,6 +51,7 @@ type Article {
   image: String
   isArchived: Boolean!
   labels: [Label!]
+  language: String
   linkId: ID
   originalArticleUrl: String
   originalHtml: String
@@ -1001,6 +1002,7 @@ type SearchItem {
   image: String
   isArchived: Boolean!
   labels: [Label!]
+  language: String
   originalArticleUrl: String
   ownedByViewer: Boolean
   pageId: ID

packages/api/src/readability.d.ts

@@ -165,6 +165,7 @@ declare module '@omnivore/readability' {
       /** Article published date */
       publishedDate?: Date
       dom?: Element
+      language?: string
     }
   }
 

packages/api/src/resolvers/article/index.ts

@@ -273,6 +273,7 @@ export const createArticleResolver = authorized<
         readingProgressPercent: 0,
         readingProgressAnchorIndex: 0,
         state: ArticleSavingRequestStatus.Succeeded,
+        language: parsedContent?.language,
       }
 
       let archive = false

packages/api/src/schema.ts

@@ -347,6 +347,7 @@ const schema = gql`
     unsubMailTo: String
     unsubHttpUrl: String
     state: ArticleSavingRequestStatus
+    language: String
   }
 
   # Query: article
@@ -1464,6 +1465,7 @@ const schema = gql`
     unsubHttpUrl: String
     state: ArticleSavingRequestStatus
     siteName: String
+    language: String
   }
 
   type SearchItemEdge {

packages/api/src/utils/parser.ts

@@ -287,6 +287,7 @@ export const parsePreparedContent = async (
       siteName: article?.siteName || (await jsonLdLinkMetadata).siteName,
       siteIcon: article?.siteIcon,
       byline: article?.byline || (await jsonLdLinkMetadata).byline,
+      language: article?.language,
     })
     logRecord.parseSuccess = true
   } catch (error) {

packages/api/src/utils/search.ts

@@ -23,7 +23,7 @@ export enum InFilter {
   ARCHIVE,
 }
 
-export type SearchFilter = {
+export interface SearchFilter {
   query: string | undefined
   inFilter: InFilter
   readFilter: ReadFilter
@@ -32,7 +32,7 @@ export type SearchFilter = {
   sortParams?: SortParams
   hasFilters: HasFilter[]
   dateFilters: DateFilter[]
-  subscriptionFilter?: SubscriptionFilter
+  termFilters: TermFilter[]
 }
 
 export enum LabelFilterType {
@@ -73,8 +73,9 @@ export interface SortParams {
   order?: SortOrder
 }
 
-export type SubscriptionFilter = {
-  name: string
+export interface TermFilter {
+  field: string
+  value: string
 }
 
 const parseIsFilter = (str: string | undefined): ReadFilter => {
@@ -218,15 +219,18 @@ const parseDateFilter = (
   }
 }
 
-const parseSubscriptionFilter = (
+const parseTermFilter = (
+  field: string,
   str?: string
-): SubscriptionFilter | undefined => {
+): TermFilter | undefined => {
   if (str === undefined) {
     return undefined
   }
 
   return {
-    name: str.toLowerCase(),
+    field,
+    // normalize the term to lower case
+    value: str.toLowerCase(),
   }
 }
 
@@ -239,6 +243,7 @@ export const parseSearchQuery = (query: string | undefined): SearchFilter => {
     labelFilters: [],
     hasFilters: [],
     dateFilters: [],
+    termFilters: [],
   }
 
   if (!searchQuery) {
@@ -249,6 +254,7 @@ export const parseSearchQuery = (query: string | undefined): SearchFilter => {
       labelFilters: [],
       hasFilters: [],
       dateFilters: [],
+      termFilters: [],
     }
   }
 
@@ -263,6 +269,7 @@ export const parseSearchQuery = (query: string | undefined): SearchFilter => {
       'saved',
       'published',
       'subscription',
+      'language',
     ],
     tokenize: true,
   })
@@ -321,9 +328,13 @@ export const parseSearchQuery = (query: string | undefined): SearchFilter => {
           dateFilter && result.dateFilters.push(dateFilter)
           break
         }
+        // term filters
         case 'subscription':
-          result.subscriptionFilter = parseSubscriptionFilter(keyword.value)
+        case 'language': {
+          const termFilter = parseTermFilter(keyword.keyword, keyword.value)
+          termFilter && result.termFilters.push(termFilter)
           break
+        }
       }
     }
   }

packages/db/elastic_migrations/index_settings.json

@@ -120,6 +120,10 @@
       },
       "taskName": {
         "type": "keyword"
+      },
+      "language": {
+        "type": "keyword",
+        "normalizer": "lowercase_normalizer"
       }
     }
   }

packages/readabilityjs/Readability.js

@@ -85,7 +85,7 @@ function Readability(doc, options) {
   this._articleByline = null;
   this._articlePublishedDate = null;
   this._articleDir = null;
-  this._articleSiteName = null;
+  this._languageCode = null;
   this._attempts = [];
 
   // Configurable options
@@ -1894,13 +1894,13 @@ Readability.prototype = {
 
     // get site name
     metadata.siteName = jsonld.siteName ||
-      values["og:site_name"];
+      values["og:site_name"] || null;
 
     // get website icon
     const iconLink = this._doc.querySelector(
       "link[rel='apple-touch-icon'], link[rel='shortcut icon'], link[rel='icon']"
     );
-    metadata.siteIcon = iconLink ? iconLink.href : '';
+    metadata.siteIcon = iconLink?.href;
 
     // get published date
     metadata.publishedDate = jsonld.publishedDate ||
@@ -1921,6 +1921,8 @@ Readability.prototype = {
       values["weibo:article:image"] ||
       values["weibo:webpage:image"];
 
+    metadata.locale = values["og:locale"];
+
     // TODO: Add canonical ULR search here as well
 
     // in many sites the meta value is escaped with HTML entities,
@@ -2833,6 +2835,16 @@ Readability.prototype = {
     return false;
   },
 
+  _getLanguage: function(code) {
+    if (!code) {
+      // Default to English
+      return 'English';
+    }
+
+    let lang = new Intl.DisplayNames(['en'], {type: 'language'});
+    return lang.of(code.split('-')[0]);
+  },
+
   /**
    * Runs readability.
    *
@@ -2860,6 +2872,8 @@ Readability.prototype = {
     // Extract JSON-LD metadata before removing scripts
     var jsonLd = this._disableJSONLD ? {} : this._getJSONLD(this._doc);
 
+    this._languageCode = this._doc.documentElement.lang
+
     // Remove script tags from the document.
     this._removeScripts(this._doc);
 
@@ -2898,11 +2912,12 @@ Readability.prototype = {
       textContent: textContent,
       length: textContent.length,
       excerpt: metadata.excerpt,
-      siteName: metadata.siteName || this._articleSiteName,
+      siteName: metadata.siteName,
       siteIcon: metadata.siteIcon,
       previewImage: metadata.previewImage,
       publishedDate: metadata.publishedDate || publishedAt || this._articlePublishedDate,
       dom: articleContent,
+      language: this._getLanguage(metadata.locale || this._languageCode),
     };
   }
 };