diff --git a/packages/api/src/elastic/pages.ts b/packages/api/src/elastic/pages.ts index 5d6f41697..4640c9747 100644 --- a/packages/api/src/elastic/pages.ts +++ b/packages/api/src/elastic/pages.ts @@ -17,7 +17,7 @@ import { SortBy, SortOrder, SortParams, - SubscriptionFilter, + TermFilter, } from '../utils/search' import { client, INDEX_ALIAS } from './index' import { EntityType } from '../datalayer/pubsub' @@ -158,14 +158,13 @@ const appendDateFilters = (body: SearchBody, filters: DateFilter[]): void => { }) } -const appendSubscriptionFilter = ( - body: SearchBody, - filter: SubscriptionFilter -): void => { - body.query.bool.filter.push({ - term: { - subscription: filter.name, - }, +const appendTermFilters = (body: SearchBody, filters: TermFilter[]): void => { + filters.forEach((filter) => { + body.query.bool.filter.push({ + term: { + [filter.field]: filter.value, + }, + }) }) } @@ -338,7 +337,7 @@ export const searchPages = async ( labelFilters: LabelFilter[] hasFilters: HasFilter[] dateFilters: DateFilter[] - subscriptionFilter?: SubscriptionFilter + termFilters?: TermFilter[] includePending?: boolean | null }, userId: string @@ -355,7 +354,7 @@ export const searchPages = async ( inFilter = InFilter.ALL, hasFilters, dateFilters, - subscriptionFilter, + termFilters, } = args // default order is descending const sortOrder = sort?.order || SortOrder.DESCENDING @@ -421,8 +420,8 @@ export const searchPages = async ( if (dateFilters.length > 0) { appendDateFilters(body, dateFilters) } - if (subscriptionFilter) { - appendSubscriptionFilter(body, subscriptionFilter) + if (termFilters) { + appendTermFilters(body, termFilters) } if (!args.includePending) { diff --git a/packages/api/src/elastic/types.ts b/packages/api/src/elastic/types.ts index 19e07391c..030c2a98e 100644 --- a/packages/api/src/elastic/types.ts +++ b/packages/api/src/elastic/types.ts @@ -8,15 +8,9 @@ export interface SearchBody { filter: ( | { term: { - userId: string + [K: string]: string } } - | { - term: { - subscription: string - } - } - | { term: { pageType: string } } | { exists: { field: string } } | { range: { @@ -203,6 +197,7 @@ export interface Page { unsubHttpUrl?: string state: ArticleSavingRequestStatus taskName?: string + language?: string } export interface SearchItem { @@ -226,6 +221,7 @@ export interface SearchItem { readingProgressAnchorIndex?: number userId: string state?: ArticleSavingRequestStatus + language?: string } const keys = ['_id', 'url', 'slug', 'userId', 'uploadFileId', 'state'] as const diff --git a/packages/api/src/generated/graphql.ts b/packages/api/src/generated/graphql.ts index ee80163a3..201e76621 100644 --- a/packages/api/src/generated/graphql.ts +++ b/packages/api/src/generated/graphql.ts @@ -72,6 +72,7 @@ export type Article = { image?: Maybe; isArchived: Scalars['Boolean']; labels?: Maybe>; + language?: Maybe; linkId?: Maybe; originalArticleUrl?: Maybe; originalHtml?: Maybe; @@ -1411,6 +1412,7 @@ export type SearchItem = { image?: Maybe; isArchived: Scalars['Boolean']; labels?: Maybe>; + language?: Maybe; originalArticleUrl?: Maybe; ownedByViewer?: Maybe; pageId?: Maybe; @@ -2759,6 +2761,7 @@ export type ArticleResolvers, ParentType, ContextType>; isArchived?: Resolver; labels?: Resolver>, ParentType, ContextType>; + language?: Resolver, ParentType, ContextType>; linkId?: Resolver, ParentType, ContextType>; originalArticleUrl?: Resolver, ParentType, ContextType>; originalHtml?: Resolver, ParentType, ContextType>; @@ -3498,6 +3501,7 @@ export type SearchItemResolvers, ParentType, ContextType>; isArchived?: Resolver; labels?: Resolver>, ParentType, ContextType>; + language?: Resolver, ParentType, ContextType>; originalArticleUrl?: Resolver, ParentType, ContextType>; ownedByViewer?: Resolver, ParentType, ContextType>; pageId?: Resolver, ParentType, ContextType>; diff --git a/packages/api/src/generated/schema.graphql b/packages/api/src/generated/schema.graphql index dee2426fd..d6c81bf3e 100644 --- a/packages/api/src/generated/schema.graphql +++ b/packages/api/src/generated/schema.graphql @@ -51,6 +51,7 @@ type Article { image: String isArchived: Boolean! labels: [Label!] + language: String linkId: ID originalArticleUrl: String originalHtml: String @@ -1001,6 +1002,7 @@ type SearchItem { image: String isArchived: Boolean! labels: [Label!] + language: String originalArticleUrl: String ownedByViewer: Boolean pageId: ID diff --git a/packages/api/src/readability.d.ts b/packages/api/src/readability.d.ts index 33fbe0578..ba8278b14 100644 --- a/packages/api/src/readability.d.ts +++ b/packages/api/src/readability.d.ts @@ -165,6 +165,7 @@ declare module '@omnivore/readability' { /** Article published date */ publishedDate?: Date dom?: Element + language?: string } } diff --git a/packages/api/src/resolvers/article/index.ts b/packages/api/src/resolvers/article/index.ts index 69ce55743..1250a9a01 100644 --- a/packages/api/src/resolvers/article/index.ts +++ b/packages/api/src/resolvers/article/index.ts @@ -273,6 +273,7 @@ export const createArticleResolver = authorized< readingProgressPercent: 0, readingProgressAnchorIndex: 0, state: ArticleSavingRequestStatus.Succeeded, + language: parsedContent?.language, } let archive = false diff --git a/packages/api/src/schema.ts b/packages/api/src/schema.ts index 7fa14f1ec..2a0e8959a 100755 --- a/packages/api/src/schema.ts +++ b/packages/api/src/schema.ts @@ -347,6 +347,7 @@ const schema = gql` unsubMailTo: String unsubHttpUrl: String state: ArticleSavingRequestStatus + language: String } # Query: article @@ -1464,6 +1465,7 @@ const schema = gql` unsubHttpUrl: String state: ArticleSavingRequestStatus siteName: String + language: String } type SearchItemEdge { diff --git a/packages/api/src/utils/parser.ts b/packages/api/src/utils/parser.ts index 33c238ff0..b397365f9 100644 --- a/packages/api/src/utils/parser.ts +++ b/packages/api/src/utils/parser.ts @@ -287,6 +287,7 @@ export const parsePreparedContent = async ( siteName: article?.siteName || (await jsonLdLinkMetadata).siteName, siteIcon: article?.siteIcon, byline: article?.byline || (await jsonLdLinkMetadata).byline, + language: article?.language, }) logRecord.parseSuccess = true } catch (error) { diff --git a/packages/api/src/utils/search.ts b/packages/api/src/utils/search.ts index c1fa1c7f4..90ba0ba5b 100644 --- a/packages/api/src/utils/search.ts +++ b/packages/api/src/utils/search.ts @@ -23,7 +23,7 @@ export enum InFilter { ARCHIVE, } -export type SearchFilter = { +export interface SearchFilter { query: string | undefined inFilter: InFilter readFilter: ReadFilter @@ -32,7 +32,7 @@ export type SearchFilter = { sortParams?: SortParams hasFilters: HasFilter[] dateFilters: DateFilter[] - subscriptionFilter?: SubscriptionFilter + termFilters: TermFilter[] } export enum LabelFilterType { @@ -73,8 +73,9 @@ export interface SortParams { order?: SortOrder } -export type SubscriptionFilter = { - name: string +export interface TermFilter { + field: string + value: string } const parseIsFilter = (str: string | undefined): ReadFilter => { @@ -218,15 +219,18 @@ const parseDateFilter = ( } } -const parseSubscriptionFilter = ( +const parseTermFilter = ( + field: string, str?: string -): SubscriptionFilter | undefined => { +): TermFilter | undefined => { if (str === undefined) { return undefined } return { - name: str.toLowerCase(), + field, + // normalize the term to lower case + value: str.toLowerCase(), } } @@ -239,6 +243,7 @@ export const parseSearchQuery = (query: string | undefined): SearchFilter => { labelFilters: [], hasFilters: [], dateFilters: [], + termFilters: [], } if (!searchQuery) { @@ -249,6 +254,7 @@ export const parseSearchQuery = (query: string | undefined): SearchFilter => { labelFilters: [], hasFilters: [], dateFilters: [], + termFilters: [], } } @@ -263,6 +269,7 @@ export const parseSearchQuery = (query: string | undefined): SearchFilter => { 'saved', 'published', 'subscription', + 'language', ], tokenize: true, }) @@ -321,9 +328,13 @@ export const parseSearchQuery = (query: string | undefined): SearchFilter => { dateFilter && result.dateFilters.push(dateFilter) break } + // term filters case 'subscription': - result.subscriptionFilter = parseSubscriptionFilter(keyword.value) + case 'language': { + const termFilter = parseTermFilter(keyword.keyword, keyword.value) + termFilter && result.termFilters.push(termFilter) break + } } } } diff --git a/packages/db/elastic_migrations/index_settings.json b/packages/db/elastic_migrations/index_settings.json index a4aece588..d5dca4841 100644 --- a/packages/db/elastic_migrations/index_settings.json +++ b/packages/db/elastic_migrations/index_settings.json @@ -120,6 +120,10 @@ }, "taskName": { "type": "keyword" + }, + "language": { + "type": "keyword", + "normalizer": "lowercase_normalizer" } } } diff --git a/packages/readabilityjs/Readability.js b/packages/readabilityjs/Readability.js index 47c1d9386..98ffbefbb 100644 --- a/packages/readabilityjs/Readability.js +++ b/packages/readabilityjs/Readability.js @@ -85,7 +85,7 @@ function Readability(doc, options) { this._articleByline = null; this._articlePublishedDate = null; this._articleDir = null; - this._articleSiteName = null; + this._languageCode = null; this._attempts = []; // Configurable options @@ -1894,13 +1894,13 @@ Readability.prototype = { // get site name metadata.siteName = jsonld.siteName || - values["og:site_name"]; + values["og:site_name"] || null; // get website icon const iconLink = this._doc.querySelector( "link[rel='apple-touch-icon'], link[rel='shortcut icon'], link[rel='icon']" ); - metadata.siteIcon = iconLink ? iconLink.href : ''; + metadata.siteIcon = iconLink?.href; // get published date metadata.publishedDate = jsonld.publishedDate || @@ -1921,6 +1921,8 @@ Readability.prototype = { values["weibo:article:image"] || values["weibo:webpage:image"]; + metadata.locale = values["og:locale"]; + // TODO: Add canonical ULR search here as well // in many sites the meta value is escaped with HTML entities, @@ -2833,6 +2835,16 @@ Readability.prototype = { return false; }, + _getLanguage: function(code) { + if (!code) { + // Default to English + return 'English'; + } + + let lang = new Intl.DisplayNames(['en'], {type: 'language'}); + return lang.of(code.split('-')[0]); + }, + /** * Runs readability. * @@ -2860,6 +2872,8 @@ Readability.prototype = { // Extract JSON-LD metadata before removing scripts var jsonLd = this._disableJSONLD ? {} : this._getJSONLD(this._doc); + this._languageCode = this._doc.documentElement.lang + // Remove script tags from the document. this._removeScripts(this._doc); @@ -2898,11 +2912,12 @@ Readability.prototype = { textContent: textContent, length: textContent.length, excerpt: metadata.excerpt, - siteName: metadata.siteName || this._articleSiteName, + siteName: metadata.siteName, siteIcon: metadata.siteIcon, previewImage: metadata.previewImage, publishedDate: metadata.publishedDate || publishedAt || this._articlePublishedDate, dom: articleContent, + language: this._getLanguage(metadata.locale || this._languageCode), }; } };