From e5f1f25a4da008f015adb942348ce25c48126f70 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Tue, 14 Mar 2023 18:26:45 +0800 Subject: [PATCH 01/12] Allow scoping searches to site: --- packages/api/src/elastic/pages.ts | 49 ++++++++++------ packages/api/src/elastic/types.ts | 9 ++- packages/api/src/utils/search.ts | 5 ++ packages/api/test/resolvers/article.test.ts | 65 ++++++++++++--------- 4 files changed, 82 insertions(+), 46 deletions(-) diff --git a/packages/api/src/elastic/pages.ts b/packages/api/src/elastic/pages.ts index 49d1bda5e..929737e19 100644 --- a/packages/api/src/elastic/pages.ts +++ b/packages/api/src/elastic/pages.ts @@ -1,13 +1,6 @@ -import { - ArticleSavingRequestStatus, - Page, - PageContext, - PageSearchArgs, - PageType, - ParamSet, - SearchBody, - SearchResponse, -} from './types' +import { ResponseError } from '@elastic/elasticsearch/lib/errors' +import { EntityType } from '../datalayer/pubsub' +import { BulkActionType } from '../generated/graphql' import { DateFilter, FieldFilter, @@ -21,9 +14,16 @@ import { SortOrder, } from '../utils/search' import { client, INDEX_ALIAS } from './index' -import { EntityType } from '../datalayer/pubsub' -import { ResponseError } from '@elastic/elasticsearch/lib/errors' -import { BulkActionType } from '../generated/graphql' +import { + ArticleSavingRequestStatus, + Page, + PageContext, + PageSearchArgs, + PageType, + ParamSet, + SearchBody, + SearchResponse, +} from './types' const appendQuery = (body: SearchBody, query: string): void => { body.query.bool.should.push({ @@ -224,6 +224,17 @@ const appendNoFilters = (body: SearchBody, noFilters: NoFilter[]): void => { }) } +const appendSiteNameFilter = (body: SearchBody, siteName: string): void => { + body.query.bool.should.push({ + multi_match: { + query: siteName, + fields: ['siteName', 'url'], + analyzer: 'simple', + }, + }) + body.query.bool.minimum_should_match = 1 +} + export const createPage = async ( page: Page, ctx: PageContext @@ -404,6 +415,7 @@ export const searchPages = async ( ids, includeContent, noFilters, + siteName, } = args // default order is descending const sortOrder = sort?.order || SortOrder.DESCENDING @@ -499,11 +511,10 @@ export const searchPages = async ( }) } - if (noFilters) { - appendNoFilters(body, noFilters) - } + noFilters && appendNoFilters(body, noFilters) + siteName && appendSiteNameFilter(body, siteName) - console.log('searching pages in elastic', JSON.stringify(body)) + console.debug('searching pages in elastic', JSON.stringify(body)) const response = await client.search, SearchBody>({ index: INDEX_ALIAS, @@ -523,6 +534,10 @@ export const searchPages = async ( response.body.hits.total.value, ] } catch (e) { + if (e instanceof ResponseError) { + console.error('failed to search pages in elastic', e.meta.body.error) + return undefined + } console.error('failed to search pages in elastic', e) return undefined } diff --git a/packages/api/src/elastic/types.ts b/packages/api/src/elastic/types.ts index 1ec528e54..70ee52beb 100644 --- a/packages/api/src/elastic/types.ts +++ b/packages/api/src/elastic/types.ts @@ -1,6 +1,6 @@ // Define the type of the body for the Search request -import { PickTuple } from '../util' import { PubsubClient } from '../datalayer/pubsub' +import { PickTuple } from '../util' import { DateFilter, FieldFilter, @@ -80,13 +80,15 @@ export interface SearchBody { multi_match: { query: string fields: string[] - operator: 'and' | 'or' - type: + operator?: 'and' | 'or' + type?: | 'best_fields' | 'most_fields' | 'cross_fields' | 'phrase' | 'phrase_prefix' + | 'bool_prefix' + analyzer?: string } }[] minimum_should_match?: number @@ -331,4 +333,5 @@ export interface PageSearchArgs { recommendedBy?: string includeContent?: boolean noFilters?: NoFilter[] + siteName?: string } diff --git a/packages/api/src/utils/search.ts b/packages/api/src/utils/search.ts index 15fd1f92c..9b87f7f72 100644 --- a/packages/api/src/utils/search.ts +++ b/packages/api/src/utils/search.ts @@ -37,6 +37,7 @@ export interface SearchFilter { ids: string[] recommendedBy?: string noFilters: NoFilter[] + siteName?: string } export enum LabelFilterType { @@ -335,6 +336,7 @@ export const parseSearchQuery = (query: string | undefined): SearchFilter => { 'recommendedBy', 'no', 'mode', + 'site', ], tokenize: true, }) @@ -428,6 +430,9 @@ export const parseSearchQuery = (query: string | undefined): SearchFilter => { case 'mode': // mode is ignored and used only by the frontend break + case 'site': + result.siteName = keyword.value + break } } } diff --git a/packages/api/test/resolvers/article.test.ts b/packages/api/test/resolvers/article.test.ts index 30ee73e67..3ab9829cd 100644 --- a/packages/api/test/resolvers/article.test.ts +++ b/packages/api/test/resolvers/article.test.ts @@ -1,31 +1,10 @@ -import { createTestUser, deleteTestUser } from '../db' -import { - createTestElasticPage, - generateFakeUuid, - graphqlRequest, - request, -} from '../util' import * as chai from 'chai' import { expect } from 'chai' -import 'mocha' -import { User } from '../../src/entity/user' import chaiString from 'chai-string' -import { - BulkActionType, - SyncUpdatedItemEdge, - UpdateReason, - UploadFileStatus, -} from '../../src/generated/graphql' -import { - ArticleSavingRequestStatus, - Highlight, - Page, - PageContext, - PageType, -} from '../../src/elastic/types' -import { UploadFile } from '../../src/entity/upload_file' +import 'mocha' import { createPubSubClient } from '../../src/datalayer/pubsub' -import { getRepository } from '../../src/entity/utils' +import { refreshIndex } from '../../src/elastic' +import { addHighlightToPage } from '../../src/elastic/highlights' import { createPage, deletePage, @@ -33,9 +12,30 @@ import { getPageById, updatePage, } from '../../src/elastic/pages' -import { addHighlightToPage } from '../../src/elastic/highlights' -import { refreshIndex } from '../../src/elastic' +import { + ArticleSavingRequestStatus, + Highlight, + Page, + PageContext, + PageType, +} from '../../src/elastic/types' import { SearchHistory } from '../../src/entity/search_history' +import { UploadFile } from '../../src/entity/upload_file' +import { User } from '../../src/entity/user' +import { getRepository } from '../../src/entity/utils' +import { + BulkActionType, + SyncUpdatedItemEdge, + UpdateReason, + UploadFileStatus, +} from '../../src/generated/graphql' +import { createTestUser, deleteTestUser } from '../db' +import { + createTestElasticPage, + generateFakeUuid, + graphqlRequest, + request, +} from '../util' chai.use(chaiString) @@ -847,6 +847,7 @@ describe('Article API', () => { url: url, savedAt: new Date(), state: ArticleSavingRequestStatus.Succeeded, + siteName: 'Example', } page.id = (await createPage(page, ctx))! pages.push(page) @@ -984,6 +985,18 @@ describe('Article API', () => { expect(res.body.data.search.pageInfo.totalCount).to.eq(0) }) }) + + context('when site:${site_name} is in the query', () => { + before(async () => { + keyword = "'search api' site:example" + }) + + it('returns items from the site', async () => { + const res = await graphqlRequest(query, authToken).expect(200) + + expect(res.body.data.search.pageInfo.totalCount).to.eq(5) + }) + }) }) describe('TypeaheadSearch API', () => { From 7d2cb07af04821aa4971fa095fe3abc776c2dc72 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Tue, 14 Mar 2023 22:24:17 +0800 Subject: [PATCH 02/12] eslint --- packages/api/src/elastic/pages.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/api/src/elastic/pages.ts b/packages/api/src/elastic/pages.ts index 929737e19..f908d45b1 100644 --- a/packages/api/src/elastic/pages.ts +++ b/packages/api/src/elastic/pages.ts @@ -535,6 +535,7 @@ export const searchPages = async ( ] } catch (e) { if (e instanceof ResponseError) { + // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access console.error('failed to search pages in elastic', e.meta.body.error) return undefined } From 66481c7aa604f5568337a4afd583ac9d04d8c9a7 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Wed, 15 Mar 2023 13:14:26 +0800 Subject: [PATCH 03/12] Add apollo config --- packages/api/.eslintignore | 1 + packages/api/apollo.config.js | 5 +++++ 2 files changed, 6 insertions(+) create mode 100644 packages/api/apollo.config.js diff --git a/packages/api/.eslintignore b/packages/api/.eslintignore index 4a2dcf8e2..69faa25dd 100644 --- a/packages/api/.eslintignore +++ b/packages/api/.eslintignore @@ -3,3 +3,4 @@ dist/ readabilityjs/ src/generated/ test/resolvers/ +apollo.config.js diff --git a/packages/api/apollo.config.js b/packages/api/apollo.config.js new file mode 100644 index 000000000..1bf9ef3b6 --- /dev/null +++ b/packages/api/apollo.config.js @@ -0,0 +1,5 @@ +module.exports = { + service: { + localSchemaFile: './src/generated/schema.graphql', + }, +} From 114f0405db68336d8d995b5c8b6936fd78ad3862 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Wed, 15 Mar 2023 13:14:40 +0800 Subject: [PATCH 04/12] update tsconfig --- packages/api/tsconfig.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/api/tsconfig.json b/packages/api/tsconfig.json index 7c8caecfd..ee1fb782a 100644 --- a/packages/api/tsconfig.json +++ b/packages/api/tsconfig.json @@ -7,5 +7,5 @@ "outDir": "dist" }, "include": ["src", "test"], - "exclude": ["./src/generated", "./test"] + "exclude": ["./src/generated"] } From 9c96a4eab202869b166be510dc2162b57fe9fe04 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Wed, 15 Mar 2023 13:15:21 +0800 Subject: [PATCH 05/12] fix puppeteer throwing error when create_time is not defined --- packages/puppeteer-parse/index.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/puppeteer-parse/index.js b/packages/puppeteer-parse/index.js index b39ec4e12..1ac32eb14 100644 --- a/packages/puppeteer-parse/index.js +++ b/packages/puppeteer-parse/index.js @@ -652,7 +652,8 @@ async function retrieveHtml(page, logRecord) { document.getElementById('px-block-form-wrapper')) { return 'IS_BLOCKED' } - if (create_time) { + // check if create_time is defined + if (typeof create_time !== 'undefined' && create_time) { // create_time is a global variable set by WeChat when rendering the page const date = new Date(create_time * 1000); const dateNode = document.createElement('div'); From aeb09539cccb3c31ca8f4aa9150b04d733fa3bb7 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Wed, 15 Mar 2023 13:24:17 +0800 Subject: [PATCH 06/12] Fallback to hostname --- packages/readabilityjs/Readability.js | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/packages/readabilityjs/Readability.js b/packages/readabilityjs/Readability.js index 832f66327..5f5fb96bd 100644 --- a/packages/readabilityjs/Readability.js +++ b/packages/readabilityjs/Readability.js @@ -1935,7 +1935,10 @@ Readability.prototype = { // get site name metadata.siteName = jsonld.siteName || - values["og:site_name"] || null; + values["og:site_name"] || + values["twitter:site"] || + values["site_name"] || + values["twitter:domain"]; // get website icon const siteIcon = this._doc.querySelector( @@ -3016,7 +3019,7 @@ Readability.prototype = { textContent: textContent, length: textContent.length, excerpt: metadata.excerpt, - siteName: metadata.siteName, + siteName: metadata.siteName || new URL(this._baseURI).hostname, // Fallback to hostname siteIcon: metadata.siteIcon, previewImage: metadata.previewImage, publishedDate: metadata.publishedDate || publishedAt || this._articlePublishedDate, From 45eed5c40dc9ffe8d7f2c19854690480a5d412bf Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Wed, 15 Mar 2023 13:32:04 +0800 Subject: [PATCH 07/12] Must match siteName --- packages/api/src/elastic/pages.ts | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/packages/api/src/elastic/pages.ts b/packages/api/src/elastic/pages.ts index f908d45b1..0c4ebb047 100644 --- a/packages/api/src/elastic/pages.ts +++ b/packages/api/src/elastic/pages.ts @@ -225,14 +225,11 @@ const appendNoFilters = (body: SearchBody, noFilters: NoFilter[]): void => { } const appendSiteNameFilter = (body: SearchBody, siteName: string): void => { - body.query.bool.should.push({ - multi_match: { - query: siteName, - fields: ['siteName', 'url'], - analyzer: 'simple', + body.query.bool.must.push({ + match: { + siteName, }, }) - body.query.bool.minimum_should_match = 1 } export const createPage = async ( @@ -682,8 +679,7 @@ export const searchAsYouType = async ( export const updatePagesAsync = async ( userId: string, - action: BulkActionType, - args?: PageSearchArgs + action: BulkActionType ): Promise => { // default action is archive let must_not = [ @@ -693,7 +689,7 @@ export const updatePagesAsync = async ( }, }, ] - let params: Record = { archivedAt: new Date() } + let params: Record = { archivedAt: new Date() } if (action === BulkActionType.Delete) { must_not = [] params = { state: ArticleSavingRequestStatus.Deleted } From adae20e7e16be4b19ae33cce7d45d738a7d2e624 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Wed, 15 Mar 2023 19:15:40 +0800 Subject: [PATCH 08/12] Use elasticsearch body builder --- packages/api/package.json | 1 + packages/api/src/elastic/pages.ts | 311 ++++++++++++++---------------- packages/api/src/elastic/types.ts | 120 +----------- yarn.lock | 12 ++ 4 files changed, 158 insertions(+), 286 deletions(-) diff --git a/packages/api/package.json b/packages/api/package.json index 983d263ee..8697ac80f 100644 --- a/packages/api/package.json +++ b/packages/api/package.json @@ -54,6 +54,7 @@ "dompurify": "^2.0.17", "dot-case": "^3.0.4", "dotenv": "^8.2.0", + "elastic-ts": "^0.9.0", "express": "^4.17.1", "express-http-context": "^1.2.4", "express-rate-limit": "^6.3.0", diff --git a/packages/api/src/elastic/pages.ts b/packages/api/src/elastic/pages.ts index 0c4ebb047..2ff88cfe3 100644 --- a/packages/api/src/elastic/pages.ts +++ b/packages/api/src/elastic/pages.ts @@ -1,4 +1,5 @@ import { ResponseError } from '@elastic/elasticsearch/lib/errors' +import { BuiltQuery, ESBuilder, esBuilder } from 'elastic-ts' import { EntityType } from '../datalayer/pubsub' import { BulkActionType } from '../generated/graphql' import { @@ -21,173 +22,160 @@ import { PageSearchArgs, PageType, ParamSet, - SearchBody, SearchResponse, } from './types' -const appendQuery = (body: SearchBody, query: string): void => { - body.query.bool.should.push({ - multi_match: { +const appendQuery = (builder: ESBuilder, query: string): ESBuilder => { + return builder + .orQuery('multi_match', { query, fields: ['title', 'content', 'author', 'description', 'siteName'], operator: 'and', type: 'cross_fields', - }, - }) - body.query.bool.minimum_should_match = 1 + }) + .queryMinimumShouldMatch(1) } -const appendTypeFilter = (body: SearchBody, filter: PageType): void => { - body.query.bool.must.push({ - term: { - pageType: filter, - }, - }) +const appendTypeFilter = (builder: ESBuilder, filter: PageType): ESBuilder => { + return builder.query('term', { pageType: filter }) } -const appendReadFilter = (body: SearchBody, filter: ReadFilter): void => { +const appendReadFilter = ( + builder: ESBuilder, + filter: ReadFilter +): ESBuilder => { switch (filter) { case ReadFilter.UNREAD: - body.query.bool.must.push({ - range: { - readingProgressPercent: { - lt: 98, - }, + return builder.query('range', { + readingProgressPercent: { + lt: 98, }, }) - break case ReadFilter.READ: - body.query.bool.must.push({ - range: { - readingProgressPercent: { - gte: 98, - }, + return builder.query('range', { + readingProgressPercent: { + gte: 98, }, }) } + return builder } -const appendInFilter = (body: SearchBody, filter: InFilter): void => { +const appendInFilter = (builder: ESBuilder, filter: InFilter): ESBuilder => { switch (filter) { case InFilter.ARCHIVE: - body.query.bool.must.push({ - exists: { - field: 'archivedAt', - }, - }) - break + return builder.query('exists', { field: 'archivedAt' }) case InFilter.INBOX: - body.query.bool.must_not.push({ - exists: { - field: 'archivedAt', - }, - }) + return builder.notQuery('exists', { field: 'archivedAt' }) } + return builder } -const appendHasFilters = (body: SearchBody, filters: HasFilter[]): void => { +const appendHasFilters = ( + builder: ESBuilder, + filters: HasFilter[] +): ESBuilder => { filters.forEach((filter) => { switch (filter) { case HasFilter.HIGHLIGHTS: - body.query.bool.must.push({ - nested: { - path: 'highlights', - query: { - exists: { - field: 'highlights', - }, + builder = builder.query('nested', { + path: 'highlights', + query: { + exists: { + field: 'highlights', }, }, }) break case HasFilter.SHARED_AT: - body.query.bool.must.push({ - exists: { - field: 'sharedAt', - }, - }) + builder = builder.query('exists', { field: 'sharedAt' }) break } }) + return builder } const appendExcludeLabelFilter = ( - body: SearchBody, + builder: ESBuilder, filters: LabelFilter[] -): void => { +): ESBuilder => { const labels = filters.map((filter) => filter.labels).flat() - body.query.bool.must_not.push({ - nested: { - path: 'labels', - query: { - terms: { - 'labels.name': labels, - }, + return builder.notQuery('nested', { + path: 'labels', + query: { + terms: { + 'labels.name': labels, }, }, }) } const appendIncludeLabelFilter = ( - body: SearchBody, + builder: ESBuilder, filters: LabelFilter[] -): void => { +): ESBuilder => { filters.forEach((filter) => { - body.query.bool.must.push({ - nested: { - path: 'labels', - query: { - terms: { - 'labels.name': filter.labels, - }, + builder = builder.query('nested', { + path: 'labels', + query: { + terms: { + 'labels.name': filter.labels, }, }, }) }) + return builder } -const appendDateFilters = (body: SearchBody, filters: DateFilter[]): void => { +const appendDateFilters = ( + builder: ESBuilder, + filters: DateFilter[] +): ESBuilder => { filters.forEach((filter) => { - body.query.bool.must.push({ - range: { - [filter.field]: { - gt: filter.startDate, - lt: filter.endDate, - }, + builder = builder.query('range', { + [filter.field]: { + gt: filter.startDate?.toISOString(), + lt: filter.endDate?.toISOString(), }, }) }) + return builder } -const appendTermFilters = (body: SearchBody, filters: FieldFilter[]): void => { +const appendTermFilters = ( + builder: ESBuilder, + filters: FieldFilter[] +): ESBuilder => { filters.forEach((filter) => { - body.query.bool.must.push({ - term: { - [filter.field]: filter.value, - }, + builder = builder.query('term', { + [filter.field]: filter.value, }) }) + return builder } -const appendMatchFilters = (body: SearchBody, filters: FieldFilter[]): void => { +const appendMatchFilters = ( + builder: ESBuilder, + filters: FieldFilter[] +): ESBuilder => { filters.forEach((filter) => { - body.query.bool.must.push({ - match: { - [filter.field]: filter.value, - }, + builder = builder.query('match', { + [filter.field]: filter.value, }) }) + return builder } -const appendIdsFilter = (body: SearchBody, ids: string[]): void => { - body.query.bool.must.push({ - terms: { - _id: ids, - }, +const appendIdsFilter = (builder: ESBuilder, ids: string[]): ESBuilder => { + return builder.query('terms', { + _id: ids, }) } -const appendRecommendedBy = (body: SearchBody, recommendedBy: string): void => { +const appendRecommendedBy = ( + builder: ESBuilder, + recommendedBy: string +): ESBuilder => { const query = recommendedBy === '*' ? { @@ -200,35 +188,48 @@ const appendRecommendedBy = (body: SearchBody, recommendedBy: string): void => { 'recommendations.name': recommendedBy, }, } - - body.query.bool.must.push({ - nested: { - path: 'recommendations', - query, - }, + return builder.query('nested', { + path: 'recommendations', + query, }) } -const appendNoFilters = (body: SearchBody, noFilters: NoFilter[]): void => { +const appendNoFilters = ( + builder: ESBuilder, + noFilters: NoFilter[] +): ESBuilder => { noFilters.forEach((filter) => { - body.query.bool.must_not.push({ - nested: { - path: filter.field, - query: { - exists: { - field: filter.field, - }, + builder = builder.notQuery('nested', { + path: filter.field, + query: { + exists: { + field: filter.field, }, }, }) }) + return builder } -const appendSiteNameFilter = (body: SearchBody, siteName: string): void => { - body.query.bool.must.push({ - match: { - siteName, - }, +const appendSiteNameFilter = ( + builder: ESBuilder, + siteName: string +): ESBuilder => { + return builder.query('bool', { + should: [ + { + match: { + siteName, + }, + }, + { + wildcard: { + // siteName is a domain name, so we need to wildcard the end + url: `*${siteName}*`, + }, + }, + ], + minimum_should_match: 1, }) } @@ -424,100 +425,76 @@ export const searchPages = async ( const excludeLabels = labelFilters?.filter( (filter) => filter.type === LabelFilterType.EXCLUDE ) - - const body: SearchBody = { - query: { - bool: { - must: [ - { - term: { - userId, - }, - }, - ], - should: [], - must_not: [], - }, - }, - sort: [ - { - [sortField]: { - order: sortOrder, - }, - }, - ], - from, - size, - _source: { + // start building the query + let builder = esBuilder() + .query('term', { userId }) + .sort(sortField, sortOrder) + .from(from) + .size(size) + .rawOption('_source', { excludes: includeContent ? [] : ['originalHtml', 'content'], - }, - } - + }) // append filters if (query) { - appendQuery(body, query) + builder = appendQuery(builder, query) } if (typeFilter) { - appendTypeFilter(body, typeFilter) + builder = appendTypeFilter(builder, typeFilter) } if (inFilter !== InFilter.ALL) { - appendInFilter(body, inFilter) + builder = appendInFilter(builder, inFilter) } if (readFilter !== ReadFilter.ALL) { - appendReadFilter(body, readFilter) + builder = appendReadFilter(builder, readFilter) } if (hasFilters && hasFilters.length > 0) { - appendHasFilters(body, hasFilters) + builder = appendHasFilters(builder, hasFilters) } if (includeLabels && includeLabels.length > 0) { - appendIncludeLabelFilter(body, includeLabels) + builder = appendIncludeLabelFilter(builder, includeLabels) } if (excludeLabels && excludeLabels.length > 0) { - appendExcludeLabelFilter(body, excludeLabels) + builder = appendExcludeLabelFilter(builder, excludeLabels) } if (dateFilters && dateFilters.length > 0) { - appendDateFilters(body, dateFilters) + builder = appendDateFilters(builder, dateFilters) } if (termFilters) { - appendTermFilters(body, termFilters) + builder = appendTermFilters(builder, termFilters) } if (matchFilters) { - appendMatchFilters(body, matchFilters) + builder = appendMatchFilters(builder, matchFilters) } if (ids && ids.length > 0) { - appendIdsFilter(body, ids) + builder = appendIdsFilter(builder, ids) } - if (args.recommendedBy) { - appendRecommendedBy(body, args.recommendedBy) + builder = appendRecommendedBy(builder, args.recommendedBy) } - if (!args.includePending) { - body.query.bool.must_not.push({ - term: { - state: ArticleSavingRequestStatus.Processing, - }, + builder = builder.notQuery('term', { + state: ArticleSavingRequestStatus.Processing, }) } - if (!args.includeDeleted) { - body.query.bool.must_not.push({ - term: { - state: ArticleSavingRequestStatus.Deleted, - }, + builder = builder.notQuery('term', { + state: ArticleSavingRequestStatus.Deleted, }) } - - noFilters && appendNoFilters(body, noFilters) - siteName && appendSiteNameFilter(body, siteName) + if (noFilters) { + builder = appendNoFilters(builder, noFilters) + } + if (siteName) { + builder = appendSiteNameFilter(builder, siteName) + } + // build the query + const body = builder.build() console.debug('searching pages in elastic', JSON.stringify(body)) - - const response = await client.search, SearchBody>({ + const response = await client.search, BuiltQuery>({ index: INDEX_ALIAS, body, }) - if (response.body.hits.total.value === 0) { return [[], 0] } diff --git a/packages/api/src/elastic/types.ts b/packages/api/src/elastic/types.ts index 70ee52beb..52527ade6 100644 --- a/packages/api/src/elastic/types.ts +++ b/packages/api/src/elastic/types.ts @@ -12,124 +12,6 @@ import { SortParams, } from '../utils/search' -export interface SearchBody { - query: { - bool: { - must: ( - | { - term: { - [K: string]: string - } - } - | { exists: { field: string } } - | { - range: { - readingProgressPercent: { gte: number } | { lt: number } - } - } - | { - range: { - [K: string]: { gt: Date | undefined } | { lt: Date | undefined } - } - } - | { - nested: { - path: 'labels' - query: { - terms: { - 'labels.name': string[] - } - } - } - } - | { - nested: { - path: 'highlights' - query: { - exists: { - field: 'highlights' - } - } - } - } - | { - nested: { - path: 'recommendations' - query: { - exists?: { - field: string - } - term?: { - 'recommendations.name': string - } - } - } - } - | { - match: { - [K: string]: string - } - } - | { - terms: { - [K: string]: string[] - } - } - )[] - should: { - multi_match: { - query: string - fields: string[] - operator?: 'and' | 'or' - type?: - | 'best_fields' - | 'most_fields' - | 'cross_fields' - | 'phrase' - | 'phrase_prefix' - | 'bool_prefix' - analyzer?: string - } - }[] - minimum_should_match?: number - must_not: ( - | { term: { state: ArticleSavingRequestStatus } } - | { - exists: { - field: string - } - } - | { - nested: { - path: 'labels' - query: { - terms: { - 'labels.name': string[] - } - } - } - } - | { - nested: { - path: string - query: { - exists: { - field: string - } - } - } - } - )[] - } - } - sort: [Record] - from: number - size: number - _source: { - excludes: string[] - } -} - // Complete definition of the Search response export interface ShardsResponse { total: number @@ -164,7 +46,7 @@ export interface SearchResponse { _explanation?: Explanation fields?: never highlight?: never - inner_hits?: any + inner_hits?: unknown matched_queries?: string[] sort?: string[] }> diff --git a/yarn.lock b/yarn.lock index aacec739b..a961e1803 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5996,6 +5996,11 @@ resolved "https://registry.yarnpkg.com/@sindresorhus/is/-/is-0.14.0.tgz#9fb3a3cf3132328151f353de4632e01e52102bea" integrity sha512-9NET910DNaIPngYnLLPeg+Ogzqsi9uM4mSboU5y6p8S5DzMTVEsJZrawi+BoDNUVBa2DhJqQYUFvMDfgU062LQ== +"@sindresorhus/is@^4.0.0": + version "4.6.0" + resolved "https://registry.yarnpkg.com/@sindresorhus/is/-/is-4.6.0.tgz#3c7c9c46e678feefe7a2e5bb609d3dbd665ffb3f" + integrity sha512-t09vSN3MdfsyCHoFcTRCH/iUtG7OJ0CsjzB8cjAmKc/va/kIgeDI/TxsigdncE/4be734m0cvIYwNaV4i2XqAw== + "@sinonjs/commons@^1", "@sinonjs/commons@^1.3.0", "@sinonjs/commons@^1.4.0", "@sinonjs/commons@^1.6.0", "@sinonjs/commons@^1.7.0", "@sinonjs/commons@^1.8.3": version "1.8.3" resolved "https://registry.yarnpkg.com/@sinonjs/commons/-/commons-1.8.3.tgz#3802ddd21a50a949b6721ddd72da36e67e7f1b2d" @@ -13507,6 +13512,13 @@ ee-first@1.1.1: resolved "https://registry.yarnpkg.com/ee-first/-/ee-first-1.1.1.tgz#590c61156b0ae2f4f0255732a158b266bc56b21d" integrity sha1-WQxhFWsK4vTwJVcyoViyZrxWsh0= +elastic-ts@^0.9.0: + version "0.9.0" + resolved "https://registry.yarnpkg.com/elastic-ts/-/elastic-ts-0.9.0.tgz#919b7646cd31d753235f3f9a336f7f26c5e3654d" + integrity sha512-w9Xj/67ygllZG1RS1uMnmWLMfvTEE8zUMyn9zYiEOIgt8RFLKu5X5qCT1N2eYo6IQNTCX5DUdlIw87ii36Udtg== + dependencies: + "@sindresorhus/is" "^4.0.0" + electron-to-chromium@^1.3.811: version "1.3.816" resolved "https://registry.yarnpkg.com/electron-to-chromium/-/electron-to-chromium-1.3.816.tgz#ab6488b126de92670a6459fe3e746050e0c6276f" From 895e50201a3a14d105a4f47bcef65aaa92e1e33b Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Wed, 15 Mar 2023 19:36:53 +0800 Subject: [PATCH 09/12] Fix tests --- packages/readabilityjs/Readability.js | 11 ++++++++++- .../test/test-pages/berthub-2/expected-metadata.json | 2 +- .../bookofhook.blogspot.com/expected-metadata.json | 2 +- .../community.musictribe.com/expected-metadata.json | 2 +- .../test-pages/computer.rip/expected-metadata.json | 2 +- .../test/test-pages/danluu/expected-metadata.json | 2 +- .../test/test-pages/gdcvault/expected-metadata.json | 2 +- .../test/test-pages/gflownet/expected-metadata.json | 2 +- .../test/test-pages/infoproc/expected-metadata.json | 2 +- .../johnhcochrane.blogspot/expected-metadata.json | 2 +- .../test/test-pages/milkroad/expected-metadata.json | 2 +- .../nytimes-podcasts/expected-metadata.json | 2 +- .../test-pages/nytimes.com/expected-metadata.json | 2 +- .../test/test-pages/nytimes/expected-metadata.json | 2 +- .../test-pages/robinwieruch.de/expected-metadata.json | 2 +- .../test-pages/sciencedirect/expected-metadata.json | 2 +- .../test-pages/substack-email/expected-metadata.json | 2 +- .../expected-metadata.json | 2 +- .../test/test-pages/sydney.com/expected-metadata.json | 2 +- .../test-pages/youtube-embed/expected-metadata.json | 2 +- .../test/test-pages/yuyue.com/expected-metadata.json | 2 +- .../test/test-pages/zhihu/expected-metadata.json | 2 +- 22 files changed, 31 insertions(+), 22 deletions(-) diff --git a/packages/readabilityjs/Readability.js b/packages/readabilityjs/Readability.js index 5f5fb96bd..5902ae973 100644 --- a/packages/readabilityjs/Readability.js +++ b/packages/readabilityjs/Readability.js @@ -3009,6 +3009,15 @@ Readability.prototype = { metadata.excerpt = paragraphs[0].textContent.trim(); } } + if (!metadata.siteName) { + // Fallback to hostname + try { + const host = new URL(this._baseURI).hostname; + metadata.siteName = host.replace(/^www\./, ""); + } catch (e) { + // Ignore + } + } var textContent = articleContent.textContent; return { @@ -3019,7 +3028,7 @@ Readability.prototype = { textContent: textContent, length: textContent.length, excerpt: metadata.excerpt, - siteName: metadata.siteName || new URL(this._baseURI).hostname, // Fallback to hostname + siteName: metadata.siteName, siteIcon: metadata.siteIcon, previewImage: metadata.previewImage, publishedDate: metadata.publishedDate || publishedAt || this._articlePublishedDate, diff --git a/packages/readabilityjs/test/test-pages/berthub-2/expected-metadata.json b/packages/readabilityjs/test/test-pages/berthub-2/expected-metadata.json index ade1d4433..3fa4cff99 100644 --- a/packages/readabilityjs/test/test-pages/berthub-2/expected-metadata.json +++ b/packages/readabilityjs/test/test-pages/berthub-2/expected-metadata.json @@ -3,7 +3,7 @@ "byline": null, "dir": null, "excerpt": "All BNT162b2 vaccine data on this page is sourced from this World Health Organization document.", - "siteName": null, + "siteName": "fakehost", "publishedDate": null, "readerable": true } diff --git a/packages/readabilityjs/test/test-pages/bookofhook.blogspot.com/expected-metadata.json b/packages/readabilityjs/test/test-pages/bookofhook.blogspot.com/expected-metadata.json index 42cbfd57e..a114dfad1 100644 --- a/packages/readabilityjs/test/test-pages/bookofhook.blogspot.com/expected-metadata.json +++ b/packages/readabilityjs/test/test-pages/bookofhook.blogspot.com/expected-metadata.json @@ -3,7 +3,7 @@ "byline": null, "dir": "ltr", "excerpt": "Productivity is one of my pet topics, because it's always dogged me a bit, especially early in my career.  I'd pull long days and nights and...", - "siteName": null, + "siteName": "fakehost", "previewImage": "https://lh3.googleusercontent.com/blogger_img_proxy/ABLy4EzWkihPRCY9pIfp0Kyte3jlABLfSMcK5dlFeOQ7OGHfoy_CEMNzvNgmyBoyd1ahGWWJ0UXGSSvUXxouVeOK5l9Fxqlpe5YQQfgFg7-ikDZ0EaToXHZFrA_EmV8V5tKR4bd1HProtJ8=w1200-h630-p-k-no-nu", "publishedDate": "2001-03-01T08:00:00.000Z", "readerable": true diff --git a/packages/readabilityjs/test/test-pages/community.musictribe.com/expected-metadata.json b/packages/readabilityjs/test/test-pages/community.musictribe.com/expected-metadata.json index f673d577b..72e4ea217 100644 --- a/packages/readabilityjs/test/test-pages/community.musictribe.com/expected-metadata.json +++ b/packages/readabilityjs/test/test-pages/community.musictribe.com/expected-metadata.json @@ -3,7 +3,7 @@ "byline": null, "dir": null, "excerpt": "This page displays a discussion entry.", - "siteName": null, + "siteName": "fakehost", "siteIcon": "http://fakehost/test/public/core_file/ec/5d/02/f6baf292a4e76c51e82e87d5cb98ad61.ico?c=5097", "previewImage": "https://community.musictribe.com/", "publishedDate": null, diff --git a/packages/readabilityjs/test/test-pages/computer.rip/expected-metadata.json b/packages/readabilityjs/test/test-pages/computer.rip/expected-metadata.json index 16d576194..efa23a06c 100644 --- a/packages/readabilityjs/test/test-pages/computer.rip/expected-metadata.json +++ b/packages/readabilityjs/test/test-pages/computer.rip/expected-metadata.json @@ -3,7 +3,7 @@ "byline": null, "dir": null, "excerpt": "To start: yes, long time no see. Well, COVID-19 has been like that. Some days I feel accomplished if I successfully check my email. I finally managed to clear out a backlog of an entire handfull of things that needed thoughtful responses, though, and so here I am, screaming into the void instead of at anyone in particular.", - "siteName": null, + "siteName": "fakehost", "publishedDate": "2020-11-28T00:00:00.000Z", "readerable": true } diff --git a/packages/readabilityjs/test/test-pages/danluu/expected-metadata.json b/packages/readabilityjs/test/test-pages/danluu/expected-metadata.json index 1ce25e1c0..51e3c276e 100644 --- a/packages/readabilityjs/test/test-pages/danluu/expected-metadata.json +++ b/packages/readabilityjs/test/test-pages/danluu/expected-metadata.json @@ -3,7 +3,7 @@ "byline": null, "dir": null, "excerpt": "this is an archive of an old article by John Carmack which seems to\n have disappeared off of the internet", - "siteName": null, + "siteName": "fakehost", "siteIcon": "data:;base64,=", "publishedDate": null, "language": "English", diff --git a/packages/readabilityjs/test/test-pages/gdcvault/expected-metadata.json b/packages/readabilityjs/test/test-pages/gdcvault/expected-metadata.json index 5e5525d51..fba5b7d87 100644 --- a/packages/readabilityjs/test/test-pages/gdcvault/expected-metadata.json +++ b/packages/readabilityjs/test/test-pages/gdcvault/expected-metadata.json @@ -3,7 +3,7 @@ "byline": null, "dir": null, "excerpt": "This talk is a detailed walkthrough of the game engine modifications needed to make The Last of Us Remastered run at 60 fps on PlayStation 4. Topics covered will include the fiber-based job system Naughty Dog adopted for the game, the overall...", - "siteName": null, + "siteName": "fakehost", "siteIcon": "http://fakehost/img/favicon.ico", "previewImage": "https://ubm-twvideo01.s3.amazonaws.com/o1/vault/gdc2015/Images/GDC15_Vault-thumb_v1.png", "publishedDate": null, diff --git a/packages/readabilityjs/test/test-pages/gflownet/expected-metadata.json b/packages/readabilityjs/test/test-pages/gflownet/expected-metadata.json index 470e252e0..358ad6299 100644 --- a/packages/readabilityjs/test/test-pages/gflownet/expected-metadata.json +++ b/packages/readabilityjs/test/test-pages/gflownet/expected-metadata.json @@ -3,7 +3,7 @@ "byline": null, "dir": null, "excerpt": "What follows is a high-level overview of this work, for more details refer to our paper. Given a reward \n \n \n \n R\n \n \n (\n \n \n x\n \n \n )\n \n \n \n R(x)\n \n and a deterministic episodic environment where episodes end with a ``generate \n \n \n \n x\n \n \n \n x\n \n '' action, how do we generate diverse and high-reward \n \n \n \n x\n \n \n \n x\n \n s?\n We propose to use Flow Networks to model discrete \n \n \n \n p\n \n \n (\n \n \n x\n \n \n )\n \n \n ∝\n \n \n R\n \n \n (\n \n \n x\n \n \n )\n \n \n \n p(x) \\propto R(x)\n \n from which we can sample sequentially (like episodic RL, rather than iteratively as MCMC methods would). We show that our method, GFlowNet, is very useful on a combinatorial domain, drug molecule synthesis, because unlike RL methods it generates diverse \n \n \n \n x\n \n \n \n x\n \n s by design.", - "siteName": null, + "siteName": "fakehost", "siteIcon": "", "publishedDate": null, "readerable": true diff --git a/packages/readabilityjs/test/test-pages/infoproc/expected-metadata.json b/packages/readabilityjs/test/test-pages/infoproc/expected-metadata.json index c46de6b7f..dcb68175c 100644 --- a/packages/readabilityjs/test/test-pages/infoproc/expected-metadata.json +++ b/packages/readabilityjs/test/test-pages/infoproc/expected-metadata.json @@ -3,7 +3,7 @@ "byline": "View my complete profile", "dir": null, "excerpt": "Theodore A. Postol is professor emeritus of Science, Technology, and International Security at the Massachusetts Institute of Technolog...", - "siteName": null, + "siteName": "fakehost", "siteIcon": "https://infoproc.blogspot.com/favicon.ico", "previewImage": "https://lh3.googleusercontent.com/blogger_img_proxy/ABLy4EwrOqWiqDKUTjOKgPfAJkAH9M0VlZ8ystq9wP0nHvpdfqFpWeootDAtR3o6ZhVbEHf76IEUuztQuEVauUdXxB_jbiK4qOsTNVH3qqz0TtrglbYoOw=w1200-h630-n-k-no-nu", "publishedDate": "2022-05-18T16:00:00.000Z", diff --git a/packages/readabilityjs/test/test-pages/johnhcochrane.blogspot/expected-metadata.json b/packages/readabilityjs/test/test-pages/johnhcochrane.blogspot/expected-metadata.json index e9b9b326b..c43c040a9 100644 --- a/packages/readabilityjs/test/test-pages/johnhcochrane.blogspot/expected-metadata.json +++ b/packages/readabilityjs/test/test-pages/johnhcochrane.blogspot/expected-metadata.json @@ -3,7 +3,7 @@ "byline": null, "dir": null, "excerpt": "Part 1: Who should get the vaccine first? Sell to the highest bidder. The disease and recession go away faster.", - "siteName": null, + "siteName": "fakehost", "previewImage": "https://lh6.googleusercontent.com/proxy/QfszXavmpOM6v0Er22-RrJg7cyNBPk6alnXmZ3MQnqDQ7GWeiytIaO_eIQTeh75iNlsbjFqSoqOytExYaUdlwJyJaj7fb5CEdu2DVI6OsMJvWLGyhA=w1200-h630-p-k-no-nu", "publishedDate": "2020-12-07T08:00:00.000Z", "readerable": true diff --git a/packages/readabilityjs/test/test-pages/milkroad/expected-metadata.json b/packages/readabilityjs/test/test-pages/milkroad/expected-metadata.json index 16ef00bd5..13d56b5f8 100644 --- a/packages/readabilityjs/test/test-pages/milkroad/expected-metadata.json +++ b/packages/readabilityjs/test/test-pages/milkroad/expected-metadata.json @@ -3,7 +3,7 @@ "byline": null, "dir": null, "excerpt": "The newsletter that makes you smarter about web3", - "siteName": null, + "siteName": "fakehost", "siteIcon": "https://media.beehiiv.net/uploads/publication/logo/654e9594-184c-4884-8e02-e6e58a3a6871/thumb_Untitled__1000_x_1000_px___2_.png", "previewImage": "https://media.beehiiv.net/uploads/asset/file/30564/Screenshot_2022-04-08_115750.png", "publishedDate": null, diff --git a/packages/readabilityjs/test/test-pages/nytimes-podcasts/expected-metadata.json b/packages/readabilityjs/test/test-pages/nytimes-podcasts/expected-metadata.json index 9f428f123..61fe09886 100644 --- a/packages/readabilityjs/test/test-pages/nytimes-podcasts/expected-metadata.json +++ b/packages/readabilityjs/test/test-pages/nytimes-podcasts/expected-metadata.json @@ -3,7 +3,7 @@ "byline": "The New York Times", "dir": null, "excerpt": "The Sept. 27, 2022 episode of “The Ezra Klein Show”", - "siteName": null, + "siteName": "fakehost", "siteIcon": "/vi-assets/static-assets/favicon-d2483f10ef688e6f89e23806b9700298.ico", "previewImage": "https://static01.nyt.com/newsgraphics/images/icons/defaultPromoCrop.png", "publishedDate": "2022-09-27T16:25:17.221Z", diff --git a/packages/readabilityjs/test/test-pages/nytimes.com/expected-metadata.json b/packages/readabilityjs/test/test-pages/nytimes.com/expected-metadata.json index c76bb2d55..dfcfb56e8 100644 --- a/packages/readabilityjs/test/test-pages/nytimes.com/expected-metadata.json +++ b/packages/readabilityjs/test/test-pages/nytimes.com/expected-metadata.json @@ -3,7 +3,7 @@ "byline": "Kate Conger", "dir": null, "excerpt": "The social media company went public in 2013. But Elon Musk is taking it private as part of his acquisition of the firm. Here’s what that means.", - "siteName": null, + "siteName": "fakehost", "siteIcon": "http://fakehost/vi-assets/static-assets/favicon-d2483f10ef688e6f89e23806b9700298.ico", "previewImage": "https://static01.nyt.com/images/2022/10/29/business/00jpTWITTER-PRIVATE2-print/00MUSK-TWITTER-facebookJumbo.jpg", "publishedDate": "2022-10-28T09:00:25.000Z", diff --git a/packages/readabilityjs/test/test-pages/nytimes/expected-metadata.json b/packages/readabilityjs/test/test-pages/nytimes/expected-metadata.json index 1bbcdfdb9..882499fc2 100644 --- a/packages/readabilityjs/test/test-pages/nytimes/expected-metadata.json +++ b/packages/readabilityjs/test/test-pages/nytimes/expected-metadata.json @@ -3,7 +3,7 @@ "byline": "Ernesto Londoño, Letícia Casado", "dir": null, "excerpt": "Critics see the recent behavior of Brazil’s president — polarizing in the best of times — as an unnerving sign of a flailing leader. His strategy, if there is one, is difficult to discern.", - "siteName": null, + "siteName": "fakehost", "previewImage": "https://static01.nyt.com/images/2021/03/31/world/31brazil/31brazil-facebookJumbo.jpg", "publishedDate": "2021-03-31T23:26:15.000Z", "readerable": true diff --git a/packages/readabilityjs/test/test-pages/robinwieruch.de/expected-metadata.json b/packages/readabilityjs/test/test-pages/robinwieruch.de/expected-metadata.json index 0054cf92a..5c0d038ab 100644 --- a/packages/readabilityjs/test/test-pages/robinwieruch.de/expected-metadata.json +++ b/packages/readabilityjs/test/test-pages/robinwieruch.de/expected-metadata.json @@ -3,7 +3,7 @@ "byline": "Robin Wieruch", "dir": null, "excerpt": "A tutorial on how to fetch data in React with Hooks from third-party APIs. You will use state and effect hooks for the data request from a real API ...", - "siteName": null, + "siteName": "fakehost", "siteIcon": "http://fakehost/favicon-32x32.png?v=9db82c76a9aaf54925ac42d41f3d384c", "previewImage": "https://www.robinwieruch.de/static/9b13b3546c675d6f1a1e565f5185cab6/9842e/banner.jpg", "publishedDate": null, diff --git a/packages/readabilityjs/test/test-pages/sciencedirect/expected-metadata.json b/packages/readabilityjs/test/test-pages/sciencedirect/expected-metadata.json index ffec318f4..6829c35f1 100644 --- a/packages/readabilityjs/test/test-pages/sciencedirect/expected-metadata.json +++ b/packages/readabilityjs/test/test-pages/sciencedirect/expected-metadata.json @@ -3,7 +3,7 @@ "byline": null, "dir": null, "excerpt": "The “Weak Garden of Eden” model for the origin and dispersal of modern humans (Harpendinget al., 1993) posits that modern humans spread into separate …", - "siteName": null, + "siteName": "fakehost", "siteIcon": "https://sdfestaticassets-eu-west-1.sciencedirectassets.com/shared-assets/13/images/favSD.ico", "previewImage": "https://ars.els-cdn.com/content/image/1-s2.0-S0047248420X00121-cov150h.gif", "publishedDate": null, diff --git a/packages/readabilityjs/test/test-pages/substack-email/expected-metadata.json b/packages/readabilityjs/test/test-pages/substack-email/expected-metadata.json index 469b85b83..6b0bf82df 100644 --- a/packages/readabilityjs/test/test-pages/substack-email/expected-metadata.json +++ b/packages/readabilityjs/test/test-pages/substack-email/expected-metadata.json @@ -3,7 +3,7 @@ "byline": "Andrew Bulkeley", "dir": null, "excerpt": "Hello 20\n Percent,", - "siteName": null, + "siteName": "fakehost", "publishedDate": "2001-01-13T16:00:00.000Z", "readerable": true } diff --git a/packages/readabilityjs/test/test-pages/substack-michaelshellenberger/expected-metadata.json b/packages/readabilityjs/test/test-pages/substack-michaelshellenberger/expected-metadata.json index f41a6b334..ae2fe8c8c 100644 --- a/packages/readabilityjs/test/test-pages/substack-michaelshellenberger/expected-metadata.json +++ b/packages/readabilityjs/test/test-pages/substack-michaelshellenberger/expected-metadata.json @@ -3,7 +3,7 @@ "byline": "Michael Shellenberger", "dir": null, "excerpt": "For decades, people have claimed that homelessness is just a\n housing problem. Sure, many also have substance use and mental\n illness issues. But if we just give homeless people their own\n own studio apartments, and decriminalize public camping,\n drugs, and shoplifting, the problem will go away, many\n claimed.", - "siteName": null, + "siteName": "fakehost", "publishedDate": "2001-05-25T16:00:00.000Z", "language": "English", "readerable": true diff --git a/packages/readabilityjs/test/test-pages/sydney.com/expected-metadata.json b/packages/readabilityjs/test/test-pages/sydney.com/expected-metadata.json index 80d5fd64f..422f596fa 100644 --- a/packages/readabilityjs/test/test-pages/sydney.com/expected-metadata.json +++ b/packages/readabilityjs/test/test-pages/sydney.com/expected-metadata.json @@ -3,7 +3,7 @@ "byline": "author-circle", "dir": null, "excerpt": "Explore our list of the best walks in Sydney where you’ll find stunning coastal views, historical sites, magnificent beaches and fascinating wildlife.", - "siteName": null, + "siteName": "fakehost", "siteIcon": "/sites/sydney/files/favicon-16x16.png", "previewImage": "https://www.sydney.com/sites/sydney/files/styles/open_graph/public/2020-06/Sydney%20Harbour%20from%20Bradleys%20Head%2C%20Mosman.jpg?itok=fJkitzF5", "publishedDate": null, diff --git a/packages/readabilityjs/test/test-pages/youtube-embed/expected-metadata.json b/packages/readabilityjs/test/test-pages/youtube-embed/expected-metadata.json index 1febfa04b..c66965fca 100644 --- a/packages/readabilityjs/test/test-pages/youtube-embed/expected-metadata.json +++ b/packages/readabilityjs/test/test-pages/youtube-embed/expected-metadata.json @@ -3,7 +3,7 @@ "byline": "Noclip - Video Game Documentaries", "dir": null, "excerpt": "The Story of Celeste's Development", - "siteName": null, + "siteName": "fakehost", "siteIcon": "", "previewImage": "https://i.ytimg.com/vi/c3mbELVqAmo/hqdefault.jpg", "publishedDate": null, diff --git a/packages/readabilityjs/test/test-pages/yuyue.com/expected-metadata.json b/packages/readabilityjs/test/test-pages/yuyue.com/expected-metadata.json index 4a1772b83..4e699688b 100644 --- a/packages/readabilityjs/test/test-pages/yuyue.com/expected-metadata.json +++ b/packages/readabilityjs/test/test-pages/yuyue.com/expected-metadata.json @@ -3,7 +3,7 @@ "byline": null, "dir": null, "excerpt": "一句话，用整体性学习，即把新旧知识链起来成网，而不是分...", - "siteName": null, + "siteName": "fakehost", "siteIcon": "https://mdn.alipayobjects.com/huamei_0prmtq/afts/img/A*sRUdR543RjcAAAAAAAAAAAAADvuFAQ/original", "previewImage": "https://cdn.nlark.com/yuque/0/2022/png/22724648/1671339142303-ce7c7caa-57b6-473b-8fb3-bfaf59a79d3b.png", "publishedDate": null, diff --git a/packages/readabilityjs/test/test-pages/zhihu/expected-metadata.json b/packages/readabilityjs/test/test-pages/zhihu/expected-metadata.json index 87e159d31..c9530839b 100644 --- a/packages/readabilityjs/test/test-pages/zhihu/expected-metadata.json +++ b/packages/readabilityjs/test/test-pages/zhihu/expected-metadata.json @@ -3,7 +3,7 @@ "byline": "匿名用户", "dir": null, "excerpt": "二手房，流程", - "siteName": null, + "siteName": "fakehost", "siteIcon": "https://static.zhihu.com/heifetz/assets/apple-touch-icon-152.81060cab.png", "publishedDate": "2016-06-28T07:03:00.000Z", "language": "English", From afa2721cf782492bf25f05d64aef121332df2280 Mon Sep 17 00:00:00 2001 From: sywhb Date: Wed, 15 Mar 2023 11:38:00 +0000 Subject: [PATCH 10/12] Update generated html --- packages/readabilityjs/test/index.html | 700 ++++++++++++------------- 1 file changed, 350 insertions(+), 350 deletions(-) diff --git a/packages/readabilityjs/test/index.html b/packages/readabilityjs/test/index.html index 50aea23d9..a1dcb7db1 100644 --- a/packages/readabilityjs/test/index.html +++ b/packages/readabilityjs/test/index.html @@ -14,172 +14,22 @@ From e4b983f7e824c85ce11655ebee72445f4740ab0a Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Thu, 16 Mar 2023 12:47:16 +0800 Subject: [PATCH 11/12] Update eslintignore --- packages/api/.eslintignore | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/api/.eslintignore b/packages/api/.eslintignore index 69faa25dd..4a2dcf8e2 100644 --- a/packages/api/.eslintignore +++ b/packages/api/.eslintignore @@ -3,4 +3,3 @@ dist/ readabilityjs/ src/generated/ test/resolvers/ -apollo.config.js From 1bd28ba43b11d76bf1c41bed98d70d70f1ce208a Mon Sep 17 00:00:00 2001 From: sywhb Date: Thu, 16 Mar 2023 04:48:15 +0000 Subject: [PATCH 12/12] Update generated html --- packages/readabilityjs/test/index.html | 872 ++++++++++++------------- 1 file changed, 436 insertions(+), 436 deletions(-) diff --git a/packages/readabilityjs/test/index.html b/packages/readabilityjs/test/index.html index a1dcb7db1..6e09bd447 100644 --- a/packages/readabilityjs/test/index.html +++ b/packages/readabilityjs/test/index.html @@ -14,454 +14,256 @@