From 037520a1c3bc0796e54449632f813f581944dca7 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Mon, 23 May 2022 15:13:37 +0800 Subject: [PATCH 01/12] Add language in mappings --- packages/db/elastic_migrations/index_settings.json | 3 +++ 1 file changed, 3 insertions(+) diff --git a/packages/db/elastic_migrations/index_settings.json b/packages/db/elastic_migrations/index_settings.json index a4aece588..ade2a9958 100644 --- a/packages/db/elastic_migrations/index_settings.json +++ b/packages/db/elastic_migrations/index_settings.json @@ -120,6 +120,9 @@ }, "taskName": { "type": "keyword" + }, + "language": { + "type": "keyword" } } } From be91562ca1a09336d7a3481250d6f9a72ffec20b Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Mon, 23 May 2022 15:19:48 +0800 Subject: [PATCH 02/12] Add language in page --- packages/api/src/elastic/types.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/api/src/elastic/types.ts b/packages/api/src/elastic/types.ts index 19e07391c..47f3895bf 100644 --- a/packages/api/src/elastic/types.ts +++ b/packages/api/src/elastic/types.ts @@ -226,6 +226,7 @@ export interface SearchItem { readingProgressAnchorIndex?: number userId: string state?: ArticleSavingRequestStatus + language?: string } const keys = ['_id', 'url', 'slug', 'userId', 'uploadFileId', 'state'] as const From 3e0e8b486feae2f3093ecbe382988a965a3d1cbb Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Mon, 23 May 2022 15:21:22 +0800 Subject: [PATCH 03/12] Add language in searchitem --- packages/api/src/elastic/types.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/api/src/elastic/types.ts b/packages/api/src/elastic/types.ts index 47f3895bf..84da35d07 100644 --- a/packages/api/src/elastic/types.ts +++ b/packages/api/src/elastic/types.ts @@ -203,6 +203,7 @@ export interface Page { unsubHttpUrl?: string state: ArticleSavingRequestStatus taskName?: string + language?: string } export interface SearchItem { From b0ed10760e0e43ca31c40e7ffe8a907b5c2351c4 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Mon, 23 May 2022 15:25:21 +0800 Subject: [PATCH 04/12] Normalize language --- packages/db/elastic_migrations/index_settings.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/db/elastic_migrations/index_settings.json b/packages/db/elastic_migrations/index_settings.json index ade2a9958..d5dca4841 100644 --- a/packages/db/elastic_migrations/index_settings.json +++ b/packages/db/elastic_migrations/index_settings.json @@ -122,7 +122,8 @@ "type": "keyword" }, "language": { - "type": "keyword" + "type": "keyword", + "normalizer": "lowercase_normalizer" } } } From a702993485ee0c45d24a15537c15cc9050f54cfb Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Mon, 23 May 2022 15:26:49 +0800 Subject: [PATCH 05/12] Add search by language query --- packages/api/src/elastic/types.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/api/src/elastic/types.ts b/packages/api/src/elastic/types.ts index 84da35d07..132741558 100644 --- a/packages/api/src/elastic/types.ts +++ b/packages/api/src/elastic/types.ts @@ -17,6 +17,7 @@ export interface SearchBody { } } | { term: { pageType: string } } + | { term: { language: string } } | { exists: { field: string } } | { range: { From dd394144cda4ba75ebf46cd402de0ff28fc356f4 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Mon, 23 May 2022 15:31:11 +0800 Subject: [PATCH 06/12] Add language in schema --- packages/api/src/generated/graphql.ts | 4 ++++ packages/api/src/generated/schema.graphql | 2 ++ packages/api/src/schema.ts | 2 ++ 3 files changed, 8 insertions(+) diff --git a/packages/api/src/generated/graphql.ts b/packages/api/src/generated/graphql.ts index ee80163a3..201e76621 100644 --- a/packages/api/src/generated/graphql.ts +++ b/packages/api/src/generated/graphql.ts @@ -72,6 +72,7 @@ export type Article = { image?: Maybe; isArchived: Scalars['Boolean']; labels?: Maybe>; + language?: Maybe; linkId?: Maybe; originalArticleUrl?: Maybe; originalHtml?: Maybe; @@ -1411,6 +1412,7 @@ export type SearchItem = { image?: Maybe; isArchived: Scalars['Boolean']; labels?: Maybe>; + language?: Maybe; originalArticleUrl?: Maybe; ownedByViewer?: Maybe; pageId?: Maybe; @@ -2759,6 +2761,7 @@ export type ArticleResolvers, ParentType, ContextType>; isArchived?: Resolver; labels?: Resolver>, ParentType, ContextType>; + language?: Resolver, ParentType, ContextType>; linkId?: Resolver, ParentType, ContextType>; originalArticleUrl?: Resolver, ParentType, ContextType>; originalHtml?: Resolver, ParentType, ContextType>; @@ -3498,6 +3501,7 @@ export type SearchItemResolvers, ParentType, ContextType>; isArchived?: Resolver; labels?: Resolver>, ParentType, ContextType>; + language?: Resolver, ParentType, ContextType>; originalArticleUrl?: Resolver, ParentType, ContextType>; ownedByViewer?: Resolver, ParentType, ContextType>; pageId?: Resolver, ParentType, ContextType>; diff --git a/packages/api/src/generated/schema.graphql b/packages/api/src/generated/schema.graphql index dee2426fd..d6c81bf3e 100644 --- a/packages/api/src/generated/schema.graphql +++ b/packages/api/src/generated/schema.graphql @@ -51,6 +51,7 @@ type Article { image: String isArchived: Boolean! labels: [Label!] + language: String linkId: ID originalArticleUrl: String originalHtml: String @@ -1001,6 +1002,7 @@ type SearchItem { image: String isArchived: Boolean! labels: [Label!] + language: String originalArticleUrl: String ownedByViewer: Boolean pageId: ID diff --git a/packages/api/src/schema.ts b/packages/api/src/schema.ts index 7fa14f1ec..2a0e8959a 100755 --- a/packages/api/src/schema.ts +++ b/packages/api/src/schema.ts @@ -347,6 +347,7 @@ const schema = gql` unsubMailTo: String unsubHttpUrl: String state: ArticleSavingRequestStatus + language: String } # Query: article @@ -1464,6 +1465,7 @@ const schema = gql` unsubHttpUrl: String state: ArticleSavingRequestStatus siteName: String + language: String } type SearchItemEdge { From 1eeb5617a5b52dc0d70191b940772c31d3a25cf1 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Mon, 23 May 2022 21:47:36 +0800 Subject: [PATCH 07/12] Search by language --- packages/api/src/resolvers/article/index.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/api/src/resolvers/article/index.ts b/packages/api/src/resolvers/article/index.ts index 69ce55743..1250a9a01 100644 --- a/packages/api/src/resolvers/article/index.ts +++ b/packages/api/src/resolvers/article/index.ts @@ -273,6 +273,7 @@ export const createArticleResolver = authorized< readingProgressPercent: 0, readingProgressAnchorIndex: 0, state: ArticleSavingRequestStatus.Succeeded, + language: parsedContent?.language, } let archive = false From 6e06429636c09513236706cb3fa73f3ce13153e0 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Mon, 23 May 2022 21:48:09 +0800 Subject: [PATCH 08/12] Add language keyword in search query --- packages/api/src/utils/search.ts | 27 +++++++++++++++++++-------- 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/packages/api/src/utils/search.ts b/packages/api/src/utils/search.ts index c1fa1c7f4..90ba0ba5b 100644 --- a/packages/api/src/utils/search.ts +++ b/packages/api/src/utils/search.ts @@ -23,7 +23,7 @@ export enum InFilter { ARCHIVE, } -export type SearchFilter = { +export interface SearchFilter { query: string | undefined inFilter: InFilter readFilter: ReadFilter @@ -32,7 +32,7 @@ export type SearchFilter = { sortParams?: SortParams hasFilters: HasFilter[] dateFilters: DateFilter[] - subscriptionFilter?: SubscriptionFilter + termFilters: TermFilter[] } export enum LabelFilterType { @@ -73,8 +73,9 @@ export interface SortParams { order?: SortOrder } -export type SubscriptionFilter = { - name: string +export interface TermFilter { + field: string + value: string } const parseIsFilter = (str: string | undefined): ReadFilter => { @@ -218,15 +219,18 @@ const parseDateFilter = ( } } -const parseSubscriptionFilter = ( +const parseTermFilter = ( + field: string, str?: string -): SubscriptionFilter | undefined => { +): TermFilter | undefined => { if (str === undefined) { return undefined } return { - name: str.toLowerCase(), + field, + // normalize the term to lower case + value: str.toLowerCase(), } } @@ -239,6 +243,7 @@ export const parseSearchQuery = (query: string | undefined): SearchFilter => { labelFilters: [], hasFilters: [], dateFilters: [], + termFilters: [], } if (!searchQuery) { @@ -249,6 +254,7 @@ export const parseSearchQuery = (query: string | undefined): SearchFilter => { labelFilters: [], hasFilters: [], dateFilters: [], + termFilters: [], } } @@ -263,6 +269,7 @@ export const parseSearchQuery = (query: string | undefined): SearchFilter => { 'saved', 'published', 'subscription', + 'language', ], tokenize: true, }) @@ -321,9 +328,13 @@ export const parseSearchQuery = (query: string | undefined): SearchFilter => { dateFilter && result.dateFilters.push(dateFilter) break } + // term filters case 'subscription': - result.subscriptionFilter = parseSubscriptionFilter(keyword.value) + case 'language': { + const termFilter = parseTermFilter(keyword.keyword, keyword.value) + termFilter && result.termFilters.push(termFilter) break + } } } } From 87b0f683e1b8ca7db80aa14f374f08c51882de36 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Mon, 23 May 2022 21:48:35 +0800 Subject: [PATCH 09/12] Add language term in search body --- packages/api/src/elastic/types.ts | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/packages/api/src/elastic/types.ts b/packages/api/src/elastic/types.ts index 132741558..030c2a98e 100644 --- a/packages/api/src/elastic/types.ts +++ b/packages/api/src/elastic/types.ts @@ -8,16 +8,9 @@ export interface SearchBody { filter: ( | { term: { - userId: string + [K: string]: string } } - | { - term: { - subscription: string - } - } - | { term: { pageType: string } } - | { term: { language: string } } | { exists: { field: string } } | { range: { From 0ef9bb3944d04a2c967399d54e999ae626418d82 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Mon, 23 May 2022 21:49:05 +0800 Subject: [PATCH 10/12] Allow search by language in elastic --- packages/api/src/elastic/pages.ts | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/packages/api/src/elastic/pages.ts b/packages/api/src/elastic/pages.ts index 5d6f41697..4640c9747 100644 --- a/packages/api/src/elastic/pages.ts +++ b/packages/api/src/elastic/pages.ts @@ -17,7 +17,7 @@ import { SortBy, SortOrder, SortParams, - SubscriptionFilter, + TermFilter, } from '../utils/search' import { client, INDEX_ALIAS } from './index' import { EntityType } from '../datalayer/pubsub' @@ -158,14 +158,13 @@ const appendDateFilters = (body: SearchBody, filters: DateFilter[]): void => { }) } -const appendSubscriptionFilter = ( - body: SearchBody, - filter: SubscriptionFilter -): void => { - body.query.bool.filter.push({ - term: { - subscription: filter.name, - }, +const appendTermFilters = (body: SearchBody, filters: TermFilter[]): void => { + filters.forEach((filter) => { + body.query.bool.filter.push({ + term: { + [filter.field]: filter.value, + }, + }) }) } @@ -338,7 +337,7 @@ export const searchPages = async ( labelFilters: LabelFilter[] hasFilters: HasFilter[] dateFilters: DateFilter[] - subscriptionFilter?: SubscriptionFilter + termFilters?: TermFilter[] includePending?: boolean | null }, userId: string @@ -355,7 +354,7 @@ export const searchPages = async ( inFilter = InFilter.ALL, hasFilters, dateFilters, - subscriptionFilter, + termFilters, } = args // default order is descending const sortOrder = sort?.order || SortOrder.DESCENDING @@ -421,8 +420,8 @@ export const searchPages = async ( if (dateFilters.length > 0) { appendDateFilters(body, dateFilters) } - if (subscriptionFilter) { - appendSubscriptionFilter(body, subscriptionFilter) + if (termFilters) { + appendTermFilters(body, termFilters) } if (!args.includePending) { From 7d4d1d7b67b7a230da50bdc6580a71a2bc30ff33 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Mon, 23 May 2022 21:51:49 +0800 Subject: [PATCH 11/12] Parse language in readability --- packages/api/src/readability.d.ts | 1 + packages/api/src/utils/parser.ts | 1 + packages/readabilityjs/Readability.js | 14 ++++++++++++-- 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/packages/api/src/readability.d.ts b/packages/api/src/readability.d.ts index 33fbe0578..ba8278b14 100644 --- a/packages/api/src/readability.d.ts +++ b/packages/api/src/readability.d.ts @@ -165,6 +165,7 @@ declare module '@omnivore/readability' { /** Article published date */ publishedDate?: Date dom?: Element + language?: string } } diff --git a/packages/api/src/utils/parser.ts b/packages/api/src/utils/parser.ts index 33c238ff0..b397365f9 100644 --- a/packages/api/src/utils/parser.ts +++ b/packages/api/src/utils/parser.ts @@ -287,6 +287,7 @@ export const parsePreparedContent = async ( siteName: article?.siteName || (await jsonLdLinkMetadata).siteName, siteIcon: article?.siteIcon, byline: article?.byline || (await jsonLdLinkMetadata).byline, + language: article?.language, }) logRecord.parseSuccess = true } catch (error) { diff --git a/packages/readabilityjs/Readability.js b/packages/readabilityjs/Readability.js index 47c1d9386..3b77ef871 100644 --- a/packages/readabilityjs/Readability.js +++ b/packages/readabilityjs/Readability.js @@ -85,7 +85,7 @@ function Readability(doc, options) { this._articleByline = null; this._articlePublishedDate = null; this._articleDir = null; - this._articleSiteName = null; + this._languageCode = null; this._attempts = []; // Configurable options @@ -1921,6 +1921,8 @@ Readability.prototype = { values["weibo:article:image"] || values["weibo:webpage:image"]; + metadata.locale = values["og:locale"]; + // TODO: Add canonical ULR search here as well // in many sites the meta value is escaped with HTML entities, @@ -2833,6 +2835,11 @@ Readability.prototype = { return false; }, + _getLanguage: function(code) { + let lang = new Intl.DisplayNames(['en'], {type: 'language'}); + return lang.of(code.split('-')[0]); + }, + /** * Runs readability. * @@ -2860,6 +2867,8 @@ Readability.prototype = { // Extract JSON-LD metadata before removing scripts var jsonLd = this._disableJSONLD ? {} : this._getJSONLD(this._doc); + this._languageCode = this._doc.documentElement.lang + // Remove script tags from the document. this._removeScripts(this._doc); @@ -2898,11 +2907,12 @@ Readability.prototype = { textContent: textContent, length: textContent.length, excerpt: metadata.excerpt, - siteName: metadata.siteName || this._articleSiteName, + siteName: metadata.siteName, siteIcon: metadata.siteIcon, previewImage: metadata.previewImage, publishedDate: metadata.publishedDate || publishedAt || this._articlePublishedDate, dom: articleContent, + language: this._getLanguage(metadata.locale || this._languageCode), }; } }; From a34806a782ab40b5c223542a1fb13af5a5f485af Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Tue, 24 May 2022 13:55:04 +0800 Subject: [PATCH 12/12] Fix tests --- packages/readabilityjs/Readability.js | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/packages/readabilityjs/Readability.js b/packages/readabilityjs/Readability.js index 3b77ef871..98ffbefbb 100644 --- a/packages/readabilityjs/Readability.js +++ b/packages/readabilityjs/Readability.js @@ -1894,13 +1894,13 @@ Readability.prototype = { // get site name metadata.siteName = jsonld.siteName || - values["og:site_name"]; + values["og:site_name"] || null; // get website icon const iconLink = this._doc.querySelector( "link[rel='apple-touch-icon'], link[rel='shortcut icon'], link[rel='icon']" ); - metadata.siteIcon = iconLink ? iconLink.href : ''; + metadata.siteIcon = iconLink?.href; // get published date metadata.publishedDate = jsonld.publishedDate || @@ -2836,6 +2836,11 @@ Readability.prototype = { }, _getLanguage: function(code) { + if (!code) { + // Default to English + return 'English'; + } + let lang = new Intl.DisplayNames(['en'], {type: 'language'}); return lang.of(code.split('-')[0]); },