From e815b46bf9ca8e040c7e32df7b6a1433ddae755e Mon Sep 17 00:00:00 2001 From: Jeff Greco Date: Mon, 22 Jan 2024 12:36:50 -0600 Subject: [PATCH 01/59] improve contrast on active table row background color --- packages/web/components/tokens/stitches.config.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/web/components/tokens/stitches.config.ts b/packages/web/components/tokens/stitches.config.ts index 4442c1083..75cd5e1dc 100644 --- a/packages/web/components/tokens/stitches.config.ts +++ b/packages/web/components/tokens/stitches.config.ts @@ -169,7 +169,7 @@ export const { styled, css, theme, getCssText, globalCss, keyframes, config } = thBackground3: '#FFFFFF', thBackground4: '#EBEBEB', thBackground5: '#F5F5F5', - thBackgroundActive: '#F9F9F9', + thBackgroundActive: 'rgb(255, 234, 159)', thBackgroundContrast: '#FFFFFF', thLeftMenuBackground: '#FCFCFC', thLibraryBackground: '#F3F3F3', From 8f5d4ae8a0bbb8578bfd1fc7c5fea9bd5bb2f96d Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Tue, 23 Jan 2024 18:45:46 +0800 Subject: [PATCH 02/59] fix: catch rejected promise if it is in a try-catch block --- packages/api/src/jobs/save_page.ts | 2 +- packages/api/src/utils/createTask.ts | 2 +- packages/rule-handler/src/label.ts | 2 +- packages/rule-handler/src/notification.ts | 2 +- packages/rule-handler/src/page.ts | 4 ++-- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/api/src/jobs/save_page.ts b/packages/api/src/jobs/save_page.ts index 9eccf5efb..38e76cf43 100644 --- a/packages/api/src/jobs/save_page.ts +++ b/packages/api/src/jobs/save_page.ts @@ -95,7 +95,7 @@ const uploadToSignedUrl = async ( responseType: 'stream', timeout: REQUEST_TIMEOUT, }) - return axios.put(uploadSignedUrl, stream.data, { + return await axios.put(uploadSignedUrl, stream.data, { headers: { 'Content-Type': contentType, }, diff --git a/packages/api/src/utils/createTask.ts b/packages/api/src/utils/createTask.ts index 410fe7fc7..21fb0ea8b 100644 --- a/packages/api/src/utils/createTask.ts +++ b/packages/api/src/utils/createTask.ts @@ -112,7 +112,7 @@ const createHttpTaskWithToken = async ({ } try { - return client.createTask({ parent, task }) + return await client.createTask({ parent, task }) } catch (error) { logError(error) return null diff --git a/packages/rule-handler/src/label.ts b/packages/rule-handler/src/label.ts index 9d3254ca6..16d9fe765 100644 --- a/packages/rule-handler/src/label.ts +++ b/packages/rule-handler/src/label.ts @@ -30,7 +30,7 @@ export const setLabels = async ( }) try { - return axios.post(`${apiEndpoint}/graphql`, data, { + return await axios.post(`${apiEndpoint}/graphql`, data, { headers: { Cookie: `auth=${auth};`, 'Content-Type': 'application/json', diff --git a/packages/rule-handler/src/notification.ts b/packages/rule-handler/src/notification.ts index 092f2440e..bddc1c28e 100644 --- a/packages/rule-handler/src/notification.ts +++ b/packages/rule-handler/src/notification.ts @@ -29,7 +29,7 @@ export const sendNotification = async ( } try { - return axios.post(`${apiEndpoint}/notification/send`, requestData, { + return await axios.post(`${apiEndpoint}/notification/send`, requestData, { headers: { Cookie: `auth=${auth};`, 'Content-Type': 'application/json', diff --git a/packages/rule-handler/src/page.ts b/packages/rule-handler/src/page.ts index 9070f7192..e91bc0173 100644 --- a/packages/rule-handler/src/page.ts +++ b/packages/rule-handler/src/page.ts @@ -27,7 +27,7 @@ export const archivePage = async ( }) try { - return axios.post(`${apiEndpoint}/graphql`, data, { + return await axios.post(`${apiEndpoint}/graphql`, data, { headers: { Cookie: `auth=${auth};`, 'Content-Type': 'application/json', @@ -66,7 +66,7 @@ export const markPageAsRead = async ( }) try { - return axios.post(`${apiEndpoint}/graphql`, data, { + return await axios.post(`${apiEndpoint}/graphql`, data, { headers: { Cookie: `auth=${auth};`, 'Content-Type': 'application/json', From 5495ad782e229e8919e7ff12a0cefb0c9973964e Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Tue, 23 Jan 2024 19:19:46 +0800 Subject: [PATCH 03/59] fix: do not throw error if title is undefined in cache --- packages/api/src/jobs/save_page.ts | 17 +---------------- 1 file changed, 1 insertion(+), 16 deletions(-) diff --git a/packages/api/src/jobs/save_page.ts b/packages/api/src/jobs/save_page.ts index 9eccf5efb..4252561ab 100644 --- a/packages/api/src/jobs/save_page.ts +++ b/packages/api/src/jobs/save_page.ts @@ -58,16 +58,6 @@ interface CreateArticleResponse { } } -interface SavePageResponse { - data: { - savePage: { - url: string - clientRequestId: string - errorCodes?: string[] - } - } -} - interface FetchResult { finalUrl: string title?: string @@ -77,12 +67,7 @@ interface FetchResult { } const isFetchResult = (obj: unknown): obj is FetchResult => { - return ( - typeof obj === 'object' && - obj !== null && - 'finalUrl' in obj && - 'title' in obj - ) + return typeof obj === 'object' && obj !== null && 'finalUrl' in obj } const uploadToSignedUrl = async ( From e96458192df2a9e09db03a770ed335df59e8bb2c Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Tue, 23 Jan 2024 19:30:38 +0800 Subject: [PATCH 04/59] fix redis client cert in nitter handler --- packages/content-handler/src/redis.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/content-handler/src/redis.ts b/packages/content-handler/src/redis.ts index e6e657e2b..5bc90decc 100644 --- a/packages/content-handler/src/redis.ts +++ b/packages/content-handler/src/redis.ts @@ -5,7 +5,7 @@ export const createRedisClient = (url?: string, cert?: string) => { connectTimeout: 10000, // 10 seconds tls: cert ? { - cert, + cert: cert.replace(/\\n/g, '\n'), // replace \n with new line rejectUnauthorized: false, // for self-signed certs } : undefined, From d4e487e181e090991782384eee544d1dbd37c5fa Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Tue, 23 Jan 2024 21:50:10 +0800 Subject: [PATCH 05/59] set the state to failed if content is not fetched --- packages/api/src/jobs/save_page.ts | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/packages/api/src/jobs/save_page.ts b/packages/api/src/jobs/save_page.ts index 07cbde2f3..191597873 100644 --- a/packages/api/src/jobs/save_page.ts +++ b/packages/api/src/jobs/save_page.ts @@ -279,7 +279,6 @@ export const savePageJob = async (data: Data, attemptsMade: number) => { const { userId, articleSavingRequestId, - state, labels, source, folder, @@ -289,14 +288,17 @@ export const savePageJob = async (data: Data, attemptsMade: number) => { taskId, url, } = data - let isImported, isSaved + let isImported, + isSaved, + state = data.state try { console.log(`savePageJob: ${userId} ${url}`) // get the fetch result from cache - const { title, content, contentType, readabilityResult } = - await getCachedFetchResult(url) + const fetchedResult = await getCachedFetchResult(url) + const { title, contentType, readabilityResult } = fetchedResult + let content = fetchedResult.content // for pdf content, we need to upload the pdf if (contentType === 'application/pdf') { @@ -329,9 +331,10 @@ export const savePageJob = async (data: Data, attemptsMade: number) => { } if (!content) { - throw new Error( - 'Invalid SavePage job, fetch result missing required data' - ) + console.log('content is not fetched', url) + // set the state to failed if we don't have content + content = 'Failed to fetch content' + state = ArticleSavingRequestStatus.Failed } const user = await userRepository.findById(userId) From c1d5ec722f68b0acfcd236934859841625c55e7b Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Tue, 23 Jan 2024 22:06:27 +0800 Subject: [PATCH 06/59] do not encode url when getting file data --- packages/api/src/jobs/save_page.ts | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/packages/api/src/jobs/save_page.ts b/packages/api/src/jobs/save_page.ts index 191597873..14500d959 100644 --- a/packages/api/src/jobs/save_page.ts +++ b/packages/api/src/jobs/save_page.ts @@ -114,7 +114,7 @@ const getUploadIdAndSignedUrl = async ( }`, variables: { input: { - url, + url: encodeURI(url), contentType: 'application/pdf', clientRequestId: articleSavingRequestId, }, @@ -304,11 +304,7 @@ export const savePageJob = async (data: Data, attemptsMade: number) => { if (contentType === 'application/pdf') { const encodedUrl = encodeURI(url) - const uploadFileId = await uploadPdf( - encodedUrl, - userId, - articleSavingRequestId - ) + const uploadFileId = await uploadPdf(url, userId, articleSavingRequestId) const uploadedPdf = await sendCreateArticleMutation(userId, { url: encodedUrl, articleSavingRequestId, From 4fce7e9287aed1078857f2a59e1cea449e6e97cf Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Mon, 22 Jan 2024 15:35:14 +0800 Subject: [PATCH 07/59] Add new job for handling upload content --- packages/pdf-handler/src/job.ts | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/packages/pdf-handler/src/job.ts b/packages/pdf-handler/src/job.ts index 3ba1c1849..563c92506 100644 --- a/packages/pdf-handler/src/job.ts +++ b/packages/pdf-handler/src/job.ts @@ -4,6 +4,13 @@ import { redisDataSource } from './redis_data_source' const QUEUE_NAME = 'omnivore-backend-queue' const JOB_NAME = 'update-pdf-content' +interface savePageJob { + userId: string + data: unknown + isRss: boolean + isImport: boolean +} + const queue = new Queue(QUEUE_NAME, { connection: redisDataSource.queueRedisClient, }) From f8264bfc10cd4f9f7e5b68d46cedcefd96548a69 Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Tue, 23 Jan 2024 23:45:28 +0800 Subject: [PATCH 08/59] Hard code tweet url to avoid import --- packages/api/src/services/save_page.ts | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/packages/api/src/services/save_page.ts b/packages/api/src/services/save_page.ts index bb887fd83..a91f90549 100644 --- a/packages/api/src/services/save_page.ts +++ b/packages/api/src/services/save_page.ts @@ -18,7 +18,6 @@ import { cleanUrl, generateSlug, stringToHash, - TWEET_URL_REGEX, validatedDate, wordsCount, } from '../utils/helpers' @@ -32,7 +31,7 @@ import { createLibraryItem, updateLibraryItem } from './library_item' // where we can use APIs to fetch their underlying content. const FORCE_PUPPETEER_URLS = [ - TWEET_URL_REGEX, + /twitter\.com\/(?:#!\/)?(\w+)\/status(?:es)?\/(\d+)(?:\/.*)?/, /^((?:https?:)?\/\/)?((?:www|m)\.)?((?:youtube\.com|youtu.be))(\/(?:[\w-]+\?v=|embed\/|v\/)?)([\w-]+)(\S+)?$/, ] const ALREADY_PARSED_SOURCES = [ @@ -58,7 +57,10 @@ const createSlug = (url: string, title?: string | null | undefined) => { const shouldParseInBackend = (input: SavePageInput): boolean => { return ( ALREADY_PARSED_SOURCES.indexOf(input.source) === -1 && - FORCE_PUPPETEER_URLS.some((regex) => regex.test(input.url)) + FORCE_PUPPETEER_URLS.some((regex) => { + console.log('REGEX: ', regex) + return regex.test(input.url) + }) ) } From c7cfb8693f95ec7875b9d64eb776e5458d722afa Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Tue, 23 Jan 2024 23:46:59 +0800 Subject: [PATCH 09/59] Remove unused change --- packages/pdf-handler/src/job.ts | 7 ------- 1 file changed, 7 deletions(-) diff --git a/packages/pdf-handler/src/job.ts b/packages/pdf-handler/src/job.ts index 563c92506..3ba1c1849 100644 --- a/packages/pdf-handler/src/job.ts +++ b/packages/pdf-handler/src/job.ts @@ -4,13 +4,6 @@ import { redisDataSource } from './redis_data_source' const QUEUE_NAME = 'omnivore-backend-queue' const JOB_NAME = 'update-pdf-content' -interface savePageJob { - userId: string - data: unknown - isRss: boolean - isImport: boolean -} - const queue = new Queue(QUEUE_NAME, { connection: redisDataSource.queueRedisClient, }) From 58d41951c239a696fda5cf3ead6063819d2ee862 Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Wed, 24 Jan 2024 02:11:37 +0800 Subject: [PATCH 10/59] Dont not fetch content for medium.com rss feeds All these are blocked --- packages/api/src/jobs/rss/refreshFeed.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/packages/api/src/jobs/rss/refreshFeed.ts b/packages/api/src/jobs/rss/refreshFeed.ts index f9339d6cc..b9c0052f9 100644 --- a/packages/api/src/jobs/rss/refreshFeed.ts +++ b/packages/api/src/jobs/rss/refreshFeed.ts @@ -123,6 +123,9 @@ export const isContentFetchBlocked = (feedUrl: string) => { if (feedUrl.startsWith('https://lwn.net/headlines/newrss')) { return true } + if (feedUrl.startsWith('https://medium.com')) { + return true + } return false } From 7a987c9a7cb6275912fa34c10c3587aeac7dfe81 Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Wed, 24 Jan 2024 09:42:18 +0800 Subject: [PATCH 11/59] Upgrade bullmq-admin --- pkg/bull-queue-admin/index.js | 4 +- pkg/bull-queue-admin/package.json | 2 +- pkg/bull-queue-admin/yarn.lock | 184 ++++-------------------------- 3 files changed, 24 insertions(+), 166 deletions(-) diff --git a/pkg/bull-queue-admin/index.js b/pkg/bull-queue-admin/index.js index 57fe2a4b9..fee71ed8f 100644 --- a/pkg/bull-queue-admin/index.js +++ b/pkg/bull-queue-admin/index.js @@ -105,8 +105,8 @@ const run = async () => { serverAdapter.getRouter() ) - app.listen(8080, () => { - console.log('Running on 8080...') + app.listen(process.env.PORT ?? 8080, () => { + console.log(`Running on ${process.env.PORT ?? 8080}...`) }) } diff --git a/pkg/bull-queue-admin/package.json b/pkg/bull-queue-admin/package.json index 29b7505e1..a04a8b00f 100644 --- a/pkg/bull-queue-admin/package.json +++ b/pkg/bull-queue-admin/package.json @@ -8,7 +8,7 @@ "test": "echo \"Error: no test specified\" && exit 1" }, "dependencies": { - "@bull-board/express": "^3.10.4", + "@bull-board/express": "^5.9.0", "body-parser": "^1.20.2", "bullmq": "^4.6.0", "connect-ensure-login": "^0.1.1", diff --git a/pkg/bull-queue-admin/yarn.lock b/pkg/bull-queue-admin/yarn.lock index 081e5971c..eb889b4de 100644 --- a/pkg/bull-queue-admin/yarn.lock +++ b/pkg/bull-queue-admin/yarn.lock @@ -2,29 +2,29 @@ # yarn lockfile v1 -"@bull-board/api@3.11.1": - version "3.11.1" - resolved "https://registry.npmjs.org/@bull-board/api/-/api-3.11.1.tgz" - integrity sha512-ElwX7sM+Ng4ZL9KUsbDubRE+r2hu/gss85OsROeE9bmyfkW14jOJkgr5MKUyjTTgPEeMs1Mw55TgQs2vxoWBiA== +"@bull-board/api@5.13.0": + version "5.13.0" + resolved "https://registry.yarnpkg.com/@bull-board/api/-/api-5.13.0.tgz#119dcb675555537b9a90d59294a3c093b02afac7" + integrity sha512-v5wvnGim1pmSK9PR1RYSx1dsCW6sOQYJnpvRgU6HZbVbW/5Z0luUBGpYYxM2Ry98uYKeU999mbS7xjunOkAQDw== dependencies: redis-info "^3.0.8" -"@bull-board/express@^3.10.4": - version "3.11.1" - resolved "https://registry.npmjs.org/@bull-board/express/-/express-3.11.1.tgz" - integrity sha512-+a5IQilu/CxvnGC0z6oJPANn7eF/Q61VU86JR9vRtmaY96mKKny0NuAyulJ2/Y9JznMCBOLUsXWL6fm9zD9kJw== +"@bull-board/express@^5.9.0": + version "5.13.0" + resolved "https://registry.yarnpkg.com/@bull-board/express/-/express-5.13.0.tgz#29f2c0a61d6206c13e8816b9d0f79bf06c92064f" + integrity sha512-KWI+nGLkZ1rLsOnjHvR7DdeP371l9Fr7narG0ygn78pybR5vAdcHXWde0Uusmm80JyM0nN1JOiqrhMp3mJH7NA== dependencies: - "@bull-board/api" "3.11.1" - "@bull-board/ui" "3.11.1" - ejs "3.1.7" - express "4.17.3" + "@bull-board/api" "5.13.0" + "@bull-board/ui" "5.13.0" + ejs "^3.1.7" + express "^4.17.3" -"@bull-board/ui@3.11.1": - version "3.11.1" - resolved "https://registry.npmjs.org/@bull-board/ui/-/ui-3.11.1.tgz" - integrity sha512-SRrfvxHF/WaBICiAFuWAoAlTvoBYUBmX94oRbSKzVILRFZMe3gs0hN071BFohrn4yOTFHAkWPN7cjMbaqHwCag== +"@bull-board/ui@5.13.0": + version "5.13.0" + resolved "https://registry.yarnpkg.com/@bull-board/ui/-/ui-5.13.0.tgz#b0f1a5a0e446c608b4c6c29bb5d866c05c9c8bd7" + integrity sha512-UZ88j0c/G/UI5+F3zh6If9LqgF4kMmvRmTgnNckm8HJbclUdqvrcHBpo/e0HfdCaWGGhI2Gc12hq+AECc6c4Sw== dependencies: - "@bull-board/api" "3.11.1" + "@bull-board/api" "5.13.0" "@ioredis/commands@^1.1.1": version "1.2.0" @@ -96,22 +96,6 @@ balanced-match@^1.0.0: resolved "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz" integrity sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw== -body-parser@1.19.2: - version "1.19.2" - resolved "https://registry.npmjs.org/body-parser/-/body-parser-1.19.2.tgz" - integrity sha512-SAAwOxgoCKMGs9uUAUFHygfLAyaniaoun6I8mFY9pRAJL9+Kec34aU+oIjDhTycub1jozEfEwx1W1IuOYxVSFw== - dependencies: - bytes "3.1.2" - content-type "~1.0.4" - debug "2.6.9" - depd "~1.1.2" - http-errors "1.8.1" - iconv-lite "0.4.24" - on-finished "~2.3.0" - qs "6.9.7" - raw-body "2.4.3" - type-is "~1.6.18" - body-parser@1.20.1: version "1.20.1" resolved "https://registry.npmjs.org/body-parser/-/body-parser-1.20.1.tgz" @@ -294,30 +278,20 @@ depd@2.0.0, depd@~2.0.0: resolved "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz" integrity sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw== -depd@~1.1.2: - version "1.1.2" - resolved "https://registry.npmjs.org/depd/-/depd-1.1.2.tgz" - integrity sha512-7emPTl6Dpo6JRXOXjLRxck+FlLRX5847cLKEn00PLAgc3g2hTZZgr+e4c2v6QpSmLeFP3n5yUo7ft6avBK/5jQ== - destroy@1.2.0: version "1.2.0" resolved "https://registry.npmjs.org/destroy/-/destroy-1.2.0.tgz" integrity sha512-2sJGJTaXIIaR1w4iJSNoN0hnMY7Gpc/n8D4qSCJw8QqFWXf7cuAgnEHxBpweaVcPevC2l3KpjYCx3NypQQgaJg== -destroy@~1.0.4: - version "1.0.4" - resolved "https://registry.npmjs.org/destroy/-/destroy-1.0.4.tgz" - integrity sha512-3NdhDuEXnfun/z7x9GOElY49LoqVHoGScmOKwmxhsS8N5Y+Z8KyPPDnaSzqWgYt/ji4mqwfTS34Htrk0zPIXVg== - ee-first@1.1.1: version "1.1.1" resolved "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz" integrity sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow== -ejs@3.1.7: - version "3.1.7" - resolved "https://registry.npmjs.org/ejs/-/ejs-3.1.7.tgz" - integrity sha512-BIar7R6abbUxDA3bfXrO4DSgwo8I+fB5/1zgujl3HLLjwd6+9iOnrT+t3grn2qbk9vOgBubXOFwX2m9axoFaGw== +ejs@^3.1.7: + version "3.1.9" + resolved "https://registry.yarnpkg.com/ejs/-/ejs-3.1.9.tgz#03c9e8777fe12686a9effcef22303ca3d8eeb361" + integrity sha512-rC+QVNMJWv+MtPgkt0y+0rVEIdbtxVADApW9JXrUVlzHetgcyczP/E7DJmWJ4fJCZF2cPcBk0laWO9ZHMG3DmQ== dependencies: jake "^10.8.5" @@ -350,42 +324,6 @@ express-session@^1.17.2: safe-buffer "5.2.1" uid-safe "~2.1.5" -express@4.17.3: - version "4.17.3" - resolved "https://registry.npmjs.org/express/-/express-4.17.3.tgz" - integrity sha512-yuSQpz5I+Ch7gFrPCk4/c+dIBKlQUxtgwqzph132bsT6qhuzss6I8cLJQz7B3rFblzd6wtcI0ZbGltH/C4LjUg== - dependencies: - accepts "~1.3.8" - array-flatten "1.1.1" - body-parser "1.19.2" - content-disposition "0.5.4" - content-type "~1.0.4" - cookie "0.4.2" - cookie-signature "1.0.6" - debug "2.6.9" - depd "~1.1.2" - encodeurl "~1.0.2" - escape-html "~1.0.3" - etag "~1.8.1" - finalhandler "~1.1.2" - fresh "0.5.2" - merge-descriptors "1.0.1" - methods "~1.1.2" - on-finished "~2.3.0" - parseurl "~1.3.3" - path-to-regexp "0.1.7" - proxy-addr "~2.0.7" - qs "6.9.7" - range-parser "~1.2.1" - safe-buffer "5.2.1" - send "0.17.2" - serve-static "1.14.2" - setprototypeof "1.2.0" - statuses "~1.5.0" - type-is "~1.6.18" - utils-merge "1.0.1" - vary "~1.1.2" - express@^4.17.3: version "4.18.2" resolved "https://registry.npmjs.org/express/-/express-4.18.2.tgz" @@ -443,19 +381,6 @@ finalhandler@1.2.0: statuses "2.0.1" unpipe "~1.0.0" -finalhandler@~1.1.2: - version "1.1.2" - resolved "https://registry.npmjs.org/finalhandler/-/finalhandler-1.1.2.tgz" - integrity sha512-aAWcW57uxVNrQZqFXjITpW3sIUQmHGG3qSb9mUah9MgMC4NeWhNOlNjXEYq3HjRAvL6arUviZGGJsBg6z0zsWA== - dependencies: - debug "2.6.9" - encodeurl "~1.0.2" - escape-html "~1.0.3" - on-finished "~2.3.0" - parseurl "~1.3.3" - statuses "~1.5.0" - unpipe "~1.0.0" - forwarded@0.2.0: version "0.2.0" resolved "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz" @@ -533,17 +458,6 @@ hasown@^2.0.0: dependencies: function-bind "^1.1.2" -http-errors@1.8.1: - version "1.8.1" - resolved "https://registry.npmjs.org/http-errors/-/http-errors-1.8.1.tgz" - integrity sha512-Kpk9Sm7NmI+RHhnj6OIWDI1d6fIoFAtFt9RLaTMRlg/8w49juAStsrBgp0Dp4OdxdVbRIeKhtCUvoi/RuAhO4g== - dependencies: - depd "~1.1.2" - inherits "2.0.4" - setprototypeof "1.2.0" - statuses ">= 1.5.0 < 2" - toidentifier "1.0.1" - http-errors@2.0.0: version "2.0.0" resolved "https://registry.npmjs.org/http-errors/-/http-errors-2.0.0.tgz" @@ -748,13 +662,6 @@ on-finished@2.4.1: dependencies: ee-first "1.1.1" -on-finished@~2.3.0: - version "2.3.0" - resolved "https://registry.npmjs.org/on-finished/-/on-finished-2.3.0.tgz" - integrity sha512-ikqdkGAAyf/X/gPhXGvfgAytDZtDbr+bkNUJ0N9h5MI/dmdgCs3l6hoHrcUv41sRKew3jIwrp4qQDXiK99Utww== - dependencies: - ee-first "1.1.1" - on-headers@~1.0.2: version "1.0.2" resolved "https://registry.npmjs.org/on-headers/-/on-headers-1.0.2.tgz" @@ -818,11 +725,6 @@ qs@6.11.0: dependencies: side-channel "^1.0.4" -qs@6.9.7: - version "6.9.7" - resolved "https://registry.npmjs.org/qs/-/qs-6.9.7.tgz" - integrity sha512-IhMFgUmuNpyRfxA90umL7ByLlgRXu6tIfKPpF5TmcfRLlLCckfP/g3IQmju6jjpu+Hh8rA+2p6A27ZSPOOHdKw== - random-bytes@~1.0.0: version "1.0.0" resolved "https://registry.npmjs.org/random-bytes/-/random-bytes-1.0.0.tgz" @@ -833,16 +735,6 @@ range-parser@~1.2.1: resolved "https://registry.npmjs.org/range-parser/-/range-parser-1.2.1.tgz" integrity sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg== -raw-body@2.4.3: - version "2.4.3" - resolved "https://registry.npmjs.org/raw-body/-/raw-body-2.4.3.tgz" - integrity sha512-UlTNLIcu0uzb4D2f4WltY6cVjLi+/jEN4lgEUj3E04tpMDpUlkBo/eSn6zou9hum2VMNpCCUone0O0WeJim07g== - dependencies: - bytes "3.1.2" - http-errors "1.8.1" - iconv-lite "0.4.24" - unpipe "1.0.0" - raw-body@2.5.1: version "2.5.1" resolved "https://registry.npmjs.org/raw-body/-/raw-body-2.5.1.tgz" @@ -907,25 +799,6 @@ semver@^7.5.4: dependencies: lru-cache "^6.0.0" -send@0.17.2: - version "0.17.2" - resolved "https://registry.npmjs.org/send/-/send-0.17.2.tgz" - integrity sha512-UJYB6wFSJE3G00nEivR5rgWp8c2xXvJ3OPWPhmuteU0IKj8nKbG3DrjiOmLwpnHGYWAVwA69zmTm++YG0Hmwww== - dependencies: - debug "2.6.9" - depd "~1.1.2" - destroy "~1.0.4" - encodeurl "~1.0.2" - escape-html "~1.0.3" - etag "~1.8.1" - fresh "0.5.2" - http-errors "1.8.1" - mime "1.6.0" - ms "2.1.3" - on-finished "~2.3.0" - range-parser "~1.2.1" - statuses "~1.5.0" - send@0.18.0: version "0.18.0" resolved "https://registry.npmjs.org/send/-/send-0.18.0.tgz" @@ -945,16 +818,6 @@ send@0.18.0: range-parser "~1.2.1" statuses "2.0.1" -serve-static@1.14.2: - version "1.14.2" - resolved "https://registry.npmjs.org/serve-static/-/serve-static-1.14.2.tgz" - integrity sha512-+TMNA9AFxUEGuC0z2mevogSnn9MXKb4fa7ngeRMJaaGv8vTwnIEkKi+QGvPt33HSnf8pRS+WGM0EbMtCJLKMBQ== - dependencies: - encodeurl "~1.0.2" - escape-html "~1.0.3" - parseurl "~1.3.3" - send "0.17.2" - serve-static@1.15.0: version "1.15.0" resolved "https://registry.npmjs.org/serve-static/-/serve-static-1.15.0.tgz" @@ -999,11 +862,6 @@ statuses@2.0.1: resolved "https://registry.npmjs.org/statuses/-/statuses-2.0.1.tgz" integrity sha512-RwNA9Z/7PrK06rYLIzFMlaF+l73iwpzsqRIFgbMLbTcLD6cOao82TaWefPXQvB2fOC4AjuYSEndS7N/mTCbkdQ== -"statuses@>= 1.5.0 < 2", statuses@~1.5.0: - version "1.5.0" - resolved "https://registry.npmjs.org/statuses/-/statuses-1.5.0.tgz" - integrity sha512-OpZ3zP+jT1PI7I8nemJX4AKmAX070ZkYPVWV/AaKTJl+tXCTGyVdC1a4SL8RUQYEwk/f34ZX8UTykN68FwrqAA== - strip-bom@^4.0.0: version "4.0.0" resolved "https://registry.npmjs.org/strip-bom/-/strip-bom-4.0.0.tgz" From 9e966bb2696569999219e3e0ffe07e8baa9fc41d Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Wed, 24 Jan 2024 11:26:35 +0800 Subject: [PATCH 12/59] add REFRESH_ERROR to the status of subscriptions and add a new column refreshed_at --- .../0159.do.alter_subscriptions.sql | 19 +++++++++++++++++++ .../0159.undo.alter_subscriptions.sql | 13 +++++++++++++ 2 files changed, 32 insertions(+) create mode 100755 packages/db/migrations/0159.do.alter_subscriptions.sql create mode 100755 packages/db/migrations/0159.undo.alter_subscriptions.sql diff --git a/packages/db/migrations/0159.do.alter_subscriptions.sql b/packages/db/migrations/0159.do.alter_subscriptions.sql new file mode 100755 index 000000000..4b15c50fa --- /dev/null +++ b/packages/db/migrations/0159.do.alter_subscriptions.sql @@ -0,0 +1,19 @@ +-- Type: DO +-- Name: alter_subscriptions +-- Description: Alter omnivore.subscriptions table to add a new state and date column + +BEGIN; + +ALTER TYPE subscription_status_type + ADD VALUE IF NOT EXISTS 'REFRESH_ERROR'; + +ALTER TABLE omnivore.subscriptions + ADD COLUMN refreshed_at timestamptz; +UPDATE omnivore.subscriptions + SET refreshed_at = last_fetched_at + WHERE last_fetched_at IS NOT NULL; + +ALTER TABLE omnivore.subscriptions + RENAME COLUMN last_fetched_at TO most_recent_item_date; + +COMMIT; diff --git a/packages/db/migrations/0159.undo.alter_subscriptions.sql b/packages/db/migrations/0159.undo.alter_subscriptions.sql new file mode 100755 index 000000000..7e4e59f4a --- /dev/null +++ b/packages/db/migrations/0159.undo.alter_subscriptions.sql @@ -0,0 +1,13 @@ +-- Type: UNDO +-- Name: alter_subscriptions +-- Description: Alter omnivore.subscriptions table to add a new state and date column + +BEGIN; + +ALTER TABLE omnivore.subscriptions + RENAME COLUMN most_recent_item_date TO last_fetched_at; + +ALTER TABLE omnivore.subscriptions + DROP COLUMN refreshed_at; + +COMMIT; From 68a4fb298d61c7dbc45693df46018f1d780fabce Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Wed, 24 Jan 2024 12:03:26 +0800 Subject: [PATCH 13/59] add refreshedAt to the gql schema --- packages/api/src/entity/subscription.ts | 18 +++++++++++++-- packages/api/src/generated/graphql.ts | 1 + packages/api/src/generated/schema.graphql | 1 + packages/api/src/jobs/rss/refreshAllFeeds.ts | 11 +++++----- packages/api/src/jobs/rss/refreshFeed.ts | 16 +++++++------- .../api/src/resolvers/subscriptions/index.ts | 22 ++++++++++--------- packages/api/src/schema.ts | 3 +++ packages/api/src/utils/createTask.ts | 13 +++++------ 8 files changed, 52 insertions(+), 33 deletions(-) diff --git a/packages/api/src/entity/subscription.ts b/packages/api/src/entity/subscription.ts index e1b8da23b..2c1f271b4 100644 --- a/packages/api/src/entity/subscription.ts +++ b/packages/api/src/entity/subscription.ts @@ -7,12 +7,23 @@ import { PrimaryGeneratedColumn, UpdateDateColumn, } from 'typeorm' -import { SubscriptionStatus, SubscriptionType } from '../generated/graphql' import { NewsletterEmail } from './newsletter_email' import { User } from './user' export const DEFAULT_SUBSCRIPTION_FOLDER = 'following' +export enum SubscriptionStatus { + Active = 'ACTIVE', + Deleted = 'DELETED', + Unsubscribed = 'UNSUBSCRIBED', + RefreshError = 'REFRESH_ERROR', +} + +export enum SubscriptionType { + Newsletter = 'NEWSLETTER', + Rss = 'RSS', +} + @Entity({ name: 'subscriptions' }) export class Subscription { @PrimaryGeneratedColumn('uuid') @@ -59,7 +70,7 @@ export class Subscription { count!: number @Column('timestamp', { nullable: true }) - lastFetchedAt?: Date | null + mostRecentItemDate?: Date | null @Column('text', { nullable: true }) lastFetchedChecksum?: string | null @@ -73,6 +84,9 @@ export class Subscription { @Column('timestamp', { nullable: true }) scheduledAt?: Date | null + @Column('timestamp', { nullable: true }) + refreshedAt?: Date | null + @Column('boolean') isPrivate?: boolean | null diff --git a/packages/api/src/generated/graphql.ts b/packages/api/src/generated/graphql.ts index c68e340ee..2c6f4fa11 100644 --- a/packages/api/src/generated/graphql.ts +++ b/packages/api/src/generated/graphql.ts @@ -2852,6 +2852,7 @@ export type Subscription = { export enum SubscriptionStatus { Active = 'ACTIVE', Deleted = 'DELETED', + RefreshError = 'REFRESH_ERROR', Unsubscribed = 'UNSUBSCRIBED' } diff --git a/packages/api/src/generated/schema.graphql b/packages/api/src/generated/schema.graphql index 5c2431d8a..a4ea6802d 100644 --- a/packages/api/src/generated/schema.graphql +++ b/packages/api/src/generated/schema.graphql @@ -2248,6 +2248,7 @@ type Subscription { enum SubscriptionStatus { ACTIVE DELETED + REFRESH_ERROR UNSUBSCRIBED } diff --git a/packages/api/src/jobs/rss/refreshAllFeeds.ts b/packages/api/src/jobs/rss/refreshAllFeeds.ts index 6a78d8596..765d34339 100644 --- a/packages/api/src/jobs/rss/refreshAllFeeds.ts +++ b/packages/api/src/jobs/rss/refreshAllFeeds.ts @@ -1,11 +1,10 @@ -import { Job, Queue } from 'bullmq' +import { Job } from 'bullmq' import { DataSource } from 'typeorm' -import { QUEUE_NAME, getBackendQueue } from '../../queue-processor' -import { redisDataSource } from '../../redis_data_source' +import { v4 as uuid } from 'uuid' +import { getBackendQueue } from '../../queue-processor' +import { validateUrl } from '../../services/create_page_save_request' import { RssSubscriptionGroup } from '../../utils/createTask' import { stringToHash } from '../../utils/helpers' -import { validateUrl } from '../../services/create_page_save_request' -import { v4 as uuid } from 'uuid' export type RSSRefreshContext = { type: 'all' | 'user-added' @@ -90,7 +89,7 @@ const updateSubscriptionGroup = async ( refreshContext, subscriptionIds: group.subscriptionIds, feedUrl: group.url, - lastFetchedTimestamps: group.fetchedDates.map( + lastFetchedTimestamps: group.mostRecentItemDates.map( (timestamp) => timestamp?.getTime() || 0 ), // unix timestamp in milliseconds lastFetchedChecksums: group.checksums, diff --git a/packages/api/src/jobs/rss/refreshFeed.ts b/packages/api/src/jobs/rss/refreshFeed.ts index b9c0052f9..86579476c 100644 --- a/packages/api/src/jobs/rss/refreshFeed.ts +++ b/packages/api/src/jobs/rss/refreshFeed.ts @@ -15,7 +15,7 @@ type FolderType = 'following' | 'inbox' interface RefreshFeedRequest { subscriptionIds: string[] feedUrl: string - lastFetchedTimestamps: number[] // unix timestamp in milliseconds + mostRecentItemDates: number[] // unix timestamp in milliseconds scheduledTimestamps: number[] // unix timestamp in milliseconds lastFetchedChecksums: string[] userIds: string[] @@ -28,7 +28,7 @@ export const isRefreshFeedRequest = (data: any): data is RefreshFeedRequest => { return ( 'subscriptionIds' in data && 'feedUrl' in data && - 'lastFetchedTimestamps' in data && + 'mostRecentItemDates' in data && 'scheduledTimestamps' in data && 'userIds' in data && 'lastFetchedChecksums' in data && @@ -422,7 +422,7 @@ const processSubscription = async ( userId: string, feedUrl: string, fetchResult: { content: string; checksum: string }, - lastFetchedAt: number, + mostRecentItemDate: number, scheduledAt: number, lastFetchedChecksum: string, fetchContent: boolean, @@ -445,7 +445,7 @@ const processSubscription = async ( console.log('Feed last build date', feedLastBuildDate) if ( feedLastBuildDate && - new Date(feedLastBuildDate) <= new Date(lastFetchedAt) + new Date(feedLastBuildDate) <= new Date(mostRecentItemDate) ) { console.log('Skipping old feed', feedLastBuildDate) return @@ -492,7 +492,7 @@ const processSubscription = async ( } // skip old items - if (isOldItem(feedItem, lastFetchedAt)) { + if (isOldItem(feedItem, mostRecentItemDate)) { console.log('Skipping old feed item', feedItem.link) continue } @@ -521,7 +521,7 @@ const processSubscription = async ( // no items saved if (!lastItemFetchedAt) { // the feed has been fetched before, no new valid items found - if (lastFetchedAt || !lastValidItem) { + if (mostRecentItemDate || !lastValidItem) { console.log('No new valid items found') return } @@ -571,7 +571,7 @@ export const _refreshFeed = async (request: RefreshFeedRequest) => { const { feedUrl, subscriptionIds, - lastFetchedTimestamps, + mostRecentItemDates, scheduledTimestamps, userIds, lastFetchedChecksums, @@ -618,7 +618,7 @@ export const _refreshFeed = async (request: RefreshFeedRequest) => { userIds[i], feedUrl, fetchResult, - lastFetchedTimestamps[i], + mostRecentItemDates[i], scheduledTimestamps[i], lastFetchedChecksums[i], fetchContents[i] && allowFetchContent, diff --git a/packages/api/src/resolvers/subscriptions/index.ts b/packages/api/src/resolvers/subscriptions/index.ts index b7e1507e4..06256b8bc 100644 --- a/packages/api/src/resolvers/subscriptions/index.ts +++ b/packages/api/src/resolvers/subscriptions/index.ts @@ -4,6 +4,8 @@ import { Brackets, In } from 'typeorm' import { DEFAULT_SUBSCRIPTION_FOLDER, Subscription, + SubscriptionStatus, + SubscriptionType, } from '../../entity/subscription' import { env } from '../../env' import { @@ -28,8 +30,6 @@ import { SubscriptionsError, SubscriptionsErrorCode, SubscriptionsSuccess, - SubscriptionStatus, - SubscriptionType, UnsubscribeError, UnsubscribeErrorCode, UnsubscribeSuccess, @@ -41,13 +41,13 @@ import { getRepository } from '../../repository' import { feedRepository } from '../../repository/feed' import { validateUrl } from '../../services/create_page_save_request' import { unsubscribe } from '../../services/subscriptions' +import { updateSubscription } from '../../services/update_subscription' import { Merge } from '../../util' import { analytics } from '../../utils/analytics' import { enqueueRssFeedFetch } from '../../utils/createTask' -import { getAbsoluteUrl, keysToCamelCase } from '../../utils/helpers' import { authorized } from '../../utils/gql-utils' +import { getAbsoluteUrl, keysToCamelCase } from '../../utils/helpers' import { parseFeed, parseOpml, RSS_PARSER_CONFIG } from '../../utils/parser' -import { updateSubscription } from '../../services/update_subscription' type PartialSubscription = Omit @@ -61,8 +61,7 @@ export const subscriptionsResolver = authorized< QuerySubscriptionsArgs >(async (_obj, { sort, type }, { uid, log }) => { try { - const sortBy = - sort?.by === SortBy.UpdatedTime ? 'lastFetchedAt' : 'createdAt' + const sortBy = sort?.by === SortBy.UpdatedTime ? 'refreshedAt' : 'createdAt' const sortOrder = sort?.order === SortOrder.Ascending ? 'ASC' : 'DESC' const queryBuilder = getRepository(Subscription) @@ -218,7 +217,10 @@ export const subscribeResolver = authorized< type: SubscriptionType.Rss, }) if (existingSubscription) { - if (existingSubscription.status === SubscriptionStatus.Active) { + if ( + existingSubscription.status === SubscriptionStatus.Active || + existingSubscription.status === SubscriptionStatus.RefreshError + ) { return { errorCodes: [SubscribeErrorCode.AlreadySubscribed], } @@ -239,7 +241,7 @@ export const subscribeResolver = authorized< url: feedUrl, subscriptionIds: [updatedSubscription.id], scheduledDates: [new Date()], // fetch immediately - fetchedDates: [updatedSubscription.lastFetchedAt || null], + mostRecentItemDates: [updatedSubscription.mostRecentItemDate || null], checksums: [updatedSubscription.lastFetchedChecksum || null], fetchContents: [updatedSubscription.fetchContent], folders: [updatedSubscription.folder || DEFAULT_SUBSCRIPTION_FOLDER], @@ -289,7 +291,7 @@ export const subscribeResolver = authorized< url: feedUrl, subscriptionIds: [newSubscription.id], scheduledDates: [new Date()], // fetch immediately - fetchedDates: [null], + mostRecentItemDates: [null], checksums: [null], fetchContents: [newSubscription.fetchContent], folders: [newSubscription.folder || DEFAULT_SUBSCRIPTION_FOLDER], @@ -319,7 +321,7 @@ export const updateSubscriptionResolver = authorized< UpdateSubscriptionSuccessPartial, UpdateSubscriptionError, MutationUpdateSubscriptionArgs ->(async (_, { input }, { authTrx, uid, log }) => { +>(async (_, { input }, { uid, log }) => { try { analytics.track({ userId: uid, diff --git a/packages/api/src/schema.ts b/packages/api/src/schema.ts index 47720b676..60e88f7bc 100755 --- a/packages/api/src/schema.ts +++ b/packages/api/src/schema.ts @@ -1686,12 +1686,15 @@ const schema = gql` autoAddToLibrary: Boolean fetchContent: Boolean! folder: String! + mostRecentItemDate: Date + refreshedAt: Date } enum SubscriptionStatus { ACTIVE UNSUBSCRIBED DELETED + REFRESH_ERROR } type SubscriptionsError { diff --git a/packages/api/src/utils/createTask.ts b/packages/api/src/utils/createTask.ts index 21fb0ea8b..09c84b62d 100644 --- a/packages/api/src/utils/createTask.ts +++ b/packages/api/src/utils/createTask.ts @@ -6,23 +6,22 @@ import { google } from '@google-cloud/tasks/build/protos/protos' import axios from 'axios' import { nanoid } from 'nanoid' import { DeepPartial } from 'typeorm' +import { v4 as uuid } from 'uuid' import { ImportItemState } from '../entity/integration' import { Recommendation } from '../entity/recommendation' -import { DEFAULT_SUBSCRIPTION_FOLDER } from '../entity/subscription' import { env } from '../env' import { ArticleSavingRequestStatus, CreateLabelInput, } from '../generated/graphql' +import { queueRSSRefreshFeedJob } from '../jobs/rss/refreshAllFeeds' +import { redisDataSource } from '../redis_data_source' import { signFeatureToken } from '../services/features' import { generateVerificationToken, OmnivoreAuthorizationHeader } from './auth' import { CreateTaskError } from './errors' +import { stringToHash } from './helpers' import { logger } from './logger' import View = google.cloud.tasks.v2.Task.View -import { stringToHash } from './helpers' -import { queueRSSRefreshFeedJob } from '../jobs/rss/refreshAllFeeds' -import { redisDataSource } from '../redis_data_source' -import { v4 as uuid } from 'uuid' // Instantiates a client. const client = new CloudTasksClient() @@ -630,7 +629,7 @@ export interface RssSubscriptionGroup { url: string subscriptionIds: string[] userIds: string[] - fetchedDates: (Date | null)[] + mostRecentItemDates: (Date | null)[] scheduledDates: Date[] checksums: (string | null)[] fetchContents: boolean[] @@ -648,7 +647,7 @@ export const enqueueRssFeedFetch = async ( }, subscriptionIds: subscriptionGroup.subscriptionIds, feedUrl: subscriptionGroup.url, - lastFetchedTimestamps: subscriptionGroup.fetchedDates.map( + mostRecentItemDates: subscriptionGroup.mostRecentItemDates.map( (timestamp) => timestamp?.getTime() || 0 ), // unix timestamp in milliseconds lastFetchedChecksums: subscriptionGroup.checksums, From 3cb7c38e7f09ab46632dcbeda4c1fab711560f98 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Wed, 24 Jan 2024 12:12:26 +0800 Subject: [PATCH 14/59] update refreshed_at when feed is processed --- packages/api/src/generated/graphql.ts | 7 ++++++- packages/api/src/generated/schema.graphql | 5 ++++- packages/api/src/jobs/rss/refreshFeed.ts | 9 ++++++--- packages/api/src/schema.ts | 3 ++- packages/api/src/services/update_subscription.ts | 14 +++++--------- 5 files changed, 23 insertions(+), 15 deletions(-) diff --git a/packages/api/src/generated/graphql.ts b/packages/api/src/generated/graphql.ts index 2c6f4fa11..fd24070f3 100644 --- a/packages/api/src/generated/graphql.ts +++ b/packages/api/src/generated/graphql.ts @@ -2839,8 +2839,10 @@ export type Subscription = { id: Scalars['ID']; isPrivate?: Maybe; lastFetchedAt?: Maybe; + mostRecentItemDate?: Maybe; name: Scalars['String']; newsletterEmail?: Maybe; + refreshedAt?: Maybe; status: SubscriptionStatus; type: SubscriptionType; unsubscribeHttpUrl?: Maybe; @@ -3210,9 +3212,10 @@ export type UpdateSubscriptionInput = { folder?: InputMaybe; id: Scalars['ID']; isPrivate?: InputMaybe; - lastFetchedAt?: InputMaybe; lastFetchedChecksum?: InputMaybe; + mostRecentItemDate?: InputMaybe; name?: InputMaybe; + refreshedAt?: InputMaybe; scheduledAt?: InputMaybe; status?: InputMaybe; }; @@ -6193,8 +6196,10 @@ export type SubscriptionResolvers; isPrivate?: SubscriptionResolver, "isPrivate", ParentType, ContextType>; lastFetchedAt?: SubscriptionResolver, "lastFetchedAt", ParentType, ContextType>; + mostRecentItemDate?: SubscriptionResolver, "mostRecentItemDate", ParentType, ContextType>; name?: SubscriptionResolver; newsletterEmail?: SubscriptionResolver, "newsletterEmail", ParentType, ContextType>; + refreshedAt?: SubscriptionResolver, "refreshedAt", ParentType, ContextType>; status?: SubscriptionResolver; type?: SubscriptionResolver; unsubscribeHttpUrl?: SubscriptionResolver, "unsubscribeHttpUrl", ParentType, ContextType>; diff --git a/packages/api/src/generated/schema.graphql b/packages/api/src/generated/schema.graphql index a4ea6802d..e81e604d9 100644 --- a/packages/api/src/generated/schema.graphql +++ b/packages/api/src/generated/schema.graphql @@ -2235,8 +2235,10 @@ type Subscription { id: ID! isPrivate: Boolean lastFetchedAt: Date + mostRecentItemDate: Date name: String! newsletterEmail: String + refreshedAt: Date status: SubscriptionStatus! type: SubscriptionType! unsubscribeHttpUrl: String @@ -2577,9 +2579,10 @@ input UpdateSubscriptionInput { folder: String id: ID! isPrivate: Boolean - lastFetchedAt: Date lastFetchedChecksum: String + mostRecentItemDate: Date name: String + refreshedAt: Date scheduledAt: Date status: SubscriptionStatus } diff --git a/packages/api/src/jobs/rss/refreshFeed.ts b/packages/api/src/jobs/rss/refreshFeed.ts index 86579476c..496d1e4c3 100644 --- a/packages/api/src/jobs/rss/refreshFeed.ts +++ b/packages/api/src/jobs/rss/refreshFeed.ts @@ -6,9 +6,9 @@ import Parser, { Item } from 'rss-parser' import { promisify } from 'util' import { env } from '../../env' import { redisDataSource } from '../../redis_data_source' +import { updateSubscription } from '../../services/update_subscription' import createHttpTaskWithToken from '../../utils/createTask' import { RSSRefreshContext } from './refreshAllFeeds' -import { updateSubscription } from '../../services/update_subscription' type FolderType = 'following' | 'inbox' @@ -429,6 +429,8 @@ const processSubscription = async ( folder: FolderType, feed: RssFeed ) => { + const refreshedAt = new Date() + let lastItemFetchedAt: Date | null = null let lastValidItem: RssFeedItem | null = null @@ -549,11 +551,12 @@ const processSubscription = async ( const updatePeriodInMs = getUpdatePeriodInHours(feed) * 60 * 60 * 1000 const nextScheduledAt = scheduledAt + updatePeriodInMs * updateFrequency - // update subscription lastFetchedAt + // update subscription mostRecentItemDate and refreshedAt const updatedSubscription = await updateSubscription(userId, subscriptionId, { - lastFetchedAt: lastItemFetchedAt, + mostRecentItemDate: lastItemFetchedAt, lastFetchedChecksum: updatedLastFetchedChecksum, scheduledAt: new Date(nextScheduledAt), + refreshedAt, }) console.log('Updated subscription', updatedSubscription) } diff --git a/packages/api/src/schema.ts b/packages/api/src/schema.ts index 60e88f7bc..298904d62 100755 --- a/packages/api/src/schema.ts +++ b/packages/api/src/schema.ts @@ -2600,7 +2600,6 @@ const schema = gql` id: ID! name: String description: String - lastFetchedAt: Date lastFetchedChecksum: String status: SubscriptionStatus scheduledAt: Date @@ -2608,6 +2607,8 @@ const schema = gql` autoAddToLibrary: Boolean fetchContent: Boolean folder: String + refreshedAt: Date + mostRecentItemDate: Date } union UpdateSubscriptionResult = diff --git a/packages/api/src/services/update_subscription.ts b/packages/api/src/services/update_subscription.ts index 1106e3043..c74b43f48 100644 --- a/packages/api/src/services/update_subscription.ts +++ b/packages/api/src/services/update_subscription.ts @@ -1,8 +1,4 @@ -import { Subscription } from '../entity/subscription' -import { - SubscriptionStatus, - UpdateSubscriptionInput, -} from '../generated/graphql' +import { Subscription, SubscriptionStatus } from '../entity/subscription' import { getRepository } from '../repository' const ensureOwns = async (userId: string, subscriptionId: string) => { @@ -23,11 +19,12 @@ type UpdateSubscriptionData = { fetchContent?: boolean | null folder?: string | null isPrivate?: boolean | null - lastFetchedAt?: Date | null + mostRecentItemDate?: Date | null lastFetchedChecksum?: string | null name?: string | null scheduledAt?: Date | null status?: SubscriptionStatus | null + refreshedAt?: Date | null } export const updateSubscription = async ( @@ -42,9 +39,8 @@ export const updateSubscription = async ( id: subscriptionId, name: newData.name || undefined, description: newData.description || undefined, - lastFetchedAt: newData.lastFetchedAt - ? new Date(newData.lastFetchedAt) - : undefined, + mostRecentItemDate: newData.mostRecentItemDate || undefined, + refreshedAt: newData.refreshedAt || undefined, lastFetchedChecksum: newData.lastFetchedChecksum || undefined, status: newData.status || undefined, scheduledAt: newData.scheduledAt From e4109a63f0d4c382fee089dbf274fc7d72188e2c Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Wed, 24 Jan 2024 12:15:38 +0800 Subject: [PATCH 15/59] Expose custom metrics for the queue processor --- packages/api/src/queue-processor.ts | 30 +++++++++++++++++++++++------ 1 file changed, 24 insertions(+), 6 deletions(-) diff --git a/packages/api/src/queue-processor.ts b/packages/api/src/queue-processor.ts index 7ccd5e272..b35cf7778 100644 --- a/packages/api/src/queue-processor.ts +++ b/packages/api/src/queue-processor.ts @@ -2,7 +2,7 @@ /* eslint-disable @typescript-eslint/restrict-template-expressions */ /* eslint-disable @typescript-eslint/require-await */ /* eslint-disable @typescript-eslint/no-misused-promises */ -import { Job, QueueEvents, Worker, Queue } from 'bullmq' +import { Job, QueueEvents, Worker, Queue, JobType } from 'bullmq' import express, { Express } from 'express' import { SnakeNamingStrategy } from 'typeorm-naming-strategies' import { appDataSource } from './data_source' @@ -63,6 +63,24 @@ const main = async () => { // respond healthy to auto-scaler. app.get('/_ah/health', (req, res) => res.sendStatus(200)) + app.get('/metrics', async (_, res) => { + const queue = await getBackendQueue() + if (!queue) { + res.sendStatus(400) + return + } + let output = '' + const metrics: JobType[] = ['active', 'failed', 'completed', 'prioritized'] + const counts = await queue.getJobCounts(...metrics) + console.log('counts: ', counts) + + metrics.forEach((metric, idx) => { + output += `omnivore_backend_queue_${metric}{} ${counts[metric]}\n` + }) + + res.status(200).setHeader('Content-Type', 'text/plain').send(output) + }) + const server = app.listen(port, () => { console.log(`[queue-processor]: started`) }) @@ -78,17 +96,17 @@ const main = async () => { throw '[queue-processor] error redis is not initialized' } - const queue = new Queue(QUEUE_NAME, { - connection: workerRedisClient, - }) + // Init the backend queue now + await getBackendQueue() const worker = new Worker( QUEUE_NAME, async (job: Job) => { switch (job.name) { case 'refresh-all-feeds': { - const counts = await queue.getJobCounts('wait') - if (counts.wait > 1000) { + const queue = await getBackendQueue() + const counts = await queue?.getJobCounts('prioritized') + if (counts && counts.wait > 1000) { return } return await refreshAllFeeds(appDataSource) From c29f31d618c01a2aa482a6c049f16d09d60af37c Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Wed, 24 Jan 2024 12:39:04 +0800 Subject: [PATCH 16/59] Add _count to prom metrics names --- packages/api/src/queue-processor.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/api/src/queue-processor.ts b/packages/api/src/queue-processor.ts index b35cf7778..9f82e26e1 100644 --- a/packages/api/src/queue-processor.ts +++ b/packages/api/src/queue-processor.ts @@ -75,7 +75,7 @@ const main = async () => { console.log('counts: ', counts) metrics.forEach((metric, idx) => { - output += `omnivore_backend_queue_${metric}{} ${counts[metric]}\n` + output += `omnivore_backend_queue_${metric}_count{} ${counts[metric]}\n` }) res.status(200).setHeader('Content-Type', 'text/plain').send(output) From d24db92e031c16ebfffa4b3b0b2f416f989416b9 Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Wed, 24 Jan 2024 12:40:00 +0800 Subject: [PATCH 17/59] Dont force init this queue --- packages/api/src/queue-processor.ts | 3 --- 1 file changed, 3 deletions(-) diff --git a/packages/api/src/queue-processor.ts b/packages/api/src/queue-processor.ts index 9f82e26e1..69b495ea8 100644 --- a/packages/api/src/queue-processor.ts +++ b/packages/api/src/queue-processor.ts @@ -96,9 +96,6 @@ const main = async () => { throw '[queue-processor] error redis is not initialized' } - // Init the backend queue now - await getBackendQueue() - const worker = new Worker( QUEUE_NAME, async (job: Job) => { From 6f3fccfcd084b2aa06f7606c35b7a98c3f88f2ab Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Wed, 24 Jan 2024 13:09:43 +0800 Subject: [PATCH 18/59] mark subscription as error if item was failed --- packages/api/src/jobs/rss/refreshFeed.ts | 215 ++++++++++-------- .../api/src/services/update_subscription.ts | 26 ++- 2 files changed, 141 insertions(+), 100 deletions(-) diff --git a/packages/api/src/jobs/rss/refreshFeed.ts b/packages/api/src/jobs/rss/refreshFeed.ts index 496d1e4c3..2c7f5d143 100644 --- a/packages/api/src/jobs/rss/refreshFeed.ts +++ b/packages/api/src/jobs/rss/refreshFeed.ts @@ -1,12 +1,14 @@ import axios from 'axios' import crypto from 'crypto' -import * as jwt from 'jsonwebtoken' import { parseHTML } from 'linkedom' import Parser, { Item } from 'rss-parser' -import { promisify } from 'util' +import { SubscriptionStatus } from '../../entity/subscription' import { env } from '../../env' import { redisDataSource } from '../../redis_data_source' -import { updateSubscription } from '../../services/update_subscription' +import { + updateSubscription, + updateSubscriptions, +} from '../../services/update_subscription' import createHttpTaskWithToken from '../../utils/createTask' import { RSSRefreshContext } from './refreshAllFeeds' @@ -333,7 +335,6 @@ const createItemWithPreviewContent = async ( } } -const signToken = promisify(jwt.sign) const parser = new Parser({ customFields: { item: [ @@ -441,7 +442,8 @@ const processSubscription = async ( const updatedLastFetchedChecksum = fetchResult.checksum // fetch feed - let itemCount = 0 + let itemCount = 0, + errorCount = 0 const feedLastBuildDate = feed.lastBuildDate console.log('Feed last build date', feedLastBuildDate) @@ -455,69 +457,71 @@ const processSubscription = async ( // save each item in the feed for (const item of feed.items) { - // use published or updated if isoDate is not available for atom feeds - const isoDate = - item.isoDate || item.published || item.updated || item.created - console.log('Processing feed item', item.links, item.isoDate, feedUrl) + try { + // use published or updated if isoDate is not available for atom feeds + const isoDate = + item.isoDate || item.published || item.updated || item.created + console.log('Processing feed item', item.links, item.isoDate, feedUrl) - if (!item.links || item.links.length === 0) { - console.log('Invalid feed item', item) - continue + if (!item.links || item.links.length === 0) { + throw new Error('Invalid feed item') + } + + const link = getLink(item.links) + if (!link) { + throw new Error('Invalid feed item link') + } + + const feedItem = { + ...item, + isoDate, + link, + } + + const publishedAt = feedItem.isoDate + ? new Date(feedItem.isoDate) + : new Date() + // remember the last valid item + if ( + !lastValidItem || + (lastValidItem.isoDate && publishedAt > new Date(lastValidItem.isoDate)) + ) { + lastValidItem = feedItem + } + + // Max limit per-feed update + if (itemCount > 99) { + continue + } + + // skip old items + if (isOldItem(feedItem, mostRecentItemDate)) { + console.log('Skipping old feed item', feedItem.link) + continue + } + + const created = await createTask( + fetchContentTasks, + userId, + feedUrl, + feedItem, + fetchContent, + folder + ) + if (!created) { + throw new Error('Failed to create task for feed item') + } + + // remember the last item fetched at + if (!lastItemFetchedAt || publishedAt > lastItemFetchedAt) { + lastItemFetchedAt = publishedAt + } + + itemCount = itemCount + 1 + } catch (error) { + console.error('Error while saving RSS feed item', error, item) + errorCount = errorCount + 1 } - - const link = getLink(item.links) - if (!link) { - console.log('Invalid feed item links', item.links) - continue - } - - const feedItem = { - ...item, - isoDate, - link, - } - - const publishedAt = feedItem.isoDate - ? new Date(feedItem.isoDate) - : new Date() - // remember the last valid item - if ( - !lastValidItem || - (lastValidItem.isoDate && publishedAt > new Date(lastValidItem.isoDate)) - ) { - lastValidItem = feedItem - } - - // Max limit per-feed update - if (itemCount > 99) { - continue - } - - // skip old items - if (isOldItem(feedItem, mostRecentItemDate)) { - console.log('Skipping old feed item', feedItem.link) - continue - } - - const created = await createTask( - fetchContentTasks, - userId, - feedUrl, - feedItem, - fetchContent, - folder - ) - if (!created) { - console.error('Failed to create task for feed item', feedItem.link) - continue - } - - // remember the last item fetched at - if (!lastItemFetchedAt || publishedAt > lastItemFetchedAt) { - lastItemFetchedAt = publishedAt - } - - itemCount = itemCount + 1 } // no items saved @@ -539,17 +543,18 @@ const processSubscription = async ( ) if (!created) { console.error('Failed to create task for feed item', lastValidItem.link) - throw new Error('Failed to create task for feed item') + errorCount = errorCount + 1 } lastItemFetchedAt = lastValidItem.isoDate ? new Date(lastValidItem.isoDate) - : new Date() + : refreshedAt } const updateFrequency = getUpdateFrequency(feed) const updatePeriodInMs = getUpdatePeriodInHours(feed) * 60 * 60 * 1000 const nextScheduledAt = scheduledAt + updatePeriodInMs * updateFrequency + const status = errorCount > 0 ? SubscriptionStatus.RefreshError : undefined // update subscription mostRecentItemDate and refreshedAt const updatedSubscription = await updateSubscription(userId, subscriptionId, { @@ -557,6 +562,7 @@ const processSubscription = async ( lastFetchedChecksum: updatedLastFetchedChecksum, scheduledAt: new Date(nextScheduledAt), refreshedAt, + status, }) console.log('Updated subscription', updatedSubscription) } @@ -570,38 +576,39 @@ export const refreshFeed = async (request: any) => { } export const _refreshFeed = async (request: RefreshFeedRequest) => { - try { - const { - feedUrl, - subscriptionIds, - mostRecentItemDates, - scheduledTimestamps, - userIds, - lastFetchedChecksums, - fetchContents, - folders, - refreshContext, - } = request - console.log('Processing feed', feedUrl, { refreshContext: refreshContext }) + const { + feedUrl, + subscriptionIds, + mostRecentItemDates, + scheduledTimestamps, + userIds, + lastFetchedChecksums, + fetchContents, + folders, + refreshContext, + } = request + console.log('Processing feed', feedUrl, { refreshContext: refreshContext }) + + try { const isBlocked = await isFeedBlocked(feedUrl) if (isBlocked) { console.log('feed is blocked: ', feedUrl) - return + throw new Error('feed is blocked') } const fetchResult = await fetchAndChecksum(feedUrl) if (!fetchResult) { console.error('Failed to fetch RSS feed', feedUrl) await incrementFeedFailure(feedUrl) - return + throw new Error('Failed to fetch RSS feed') } const feed = await parseFeed(feedUrl, fetchResult.content) if (!feed) { console.error('Failed to parse RSS feed', feedUrl) await incrementFeedFailure(feedUrl) - return + throw new Error('Failed to parse RSS feed') } let allowFetchContent = true @@ -615,19 +622,23 @@ export const _refreshFeed = async (request: RefreshFeedRequest) => { const fetchContentTasks = new Map() // url -> FetchContentTask // process each subscription sequentially for (let i = 0; i < subscriptionIds.length; i++) { - await processSubscription( - fetchContentTasks, - subscriptionIds[i], - userIds[i], - feedUrl, - fetchResult, - mostRecentItemDates[i], - scheduledTimestamps[i], - lastFetchedChecksums[i], - fetchContents[i] && allowFetchContent, - folders[i], - feed - ) + try { + await processSubscription( + fetchContentTasks, + subscriptionIds[i], + userIds[i], + feedUrl, + fetchResult, + mostRecentItemDates[i], + scheduledTimestamps[i], + lastFetchedChecksums[i], + fetchContents[i] && allowFetchContent, + folders[i], + feed + ) + } catch (error) { + console.error('Error while processing subscription', error) + } } // create fetch content tasks @@ -638,7 +649,17 @@ export const _refreshFeed = async (request: RefreshFeedRequest) => { task.item ) } + + return true } catch (e) { console.error('Error while saving RSS feeds', e) + + // mark subscriptions as error if we failed to get the feed + await updateSubscriptions(subscriptionIds, { + status: SubscriptionStatus.RefreshError, + refreshedAt: new Date(), + }) + + return false } } diff --git a/packages/api/src/services/update_subscription.ts b/packages/api/src/services/update_subscription.ts index c74b43f48..b6d1a51da 100644 --- a/packages/api/src/services/update_subscription.ts +++ b/packages/api/src/services/update_subscription.ts @@ -43,9 +43,7 @@ export const updateSubscription = async ( refreshedAt: newData.refreshedAt || undefined, lastFetchedChecksum: newData.lastFetchedChecksum || undefined, status: newData.status || undefined, - scheduledAt: newData.scheduledAt - ? new Date(newData.scheduledAt) - : undefined, + scheduledAt: newData.scheduledAt || undefined, autoAddToLibrary: newData.autoAddToLibrary ?? undefined, isPrivate: newData.isPrivate ?? undefined, fetchContent: newData.fetchContent ?? undefined, @@ -57,3 +55,25 @@ export const updateSubscription = async ( user: { id: userId }, }) } + +export const updateSubscriptions = async ( + subscriptionIds: string[], + newData: UpdateSubscriptionData +) => { + return getRepository(Subscription).save( + subscriptionIds.map((id) => ({ + id, + name: newData.name || undefined, + description: newData.description || undefined, + mostRecentItemDate: newData.mostRecentItemDate || undefined, + refreshedAt: newData.refreshedAt || undefined, + lastFetchedChecksum: newData.lastFetchedChecksum || undefined, + status: newData.status || undefined, + scheduledAt: newData.scheduledAt || undefined, + autoAddToLibrary: newData.autoAddToLibrary ?? undefined, + isPrivate: newData.isPrivate ?? undefined, + fetchContent: newData.fetchContent ?? undefined, + folder: newData.folder ?? undefined, + })) + ) +} From fd74ebffa23644fc7a122f7acd6230e2d551d786 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Wed, 24 Jan 2024 13:15:46 +0800 Subject: [PATCH 19/59] validate feed item url and convert relative url to absolute url --- packages/api/src/jobs/rss/refreshFeed.ts | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/packages/api/src/jobs/rss/refreshFeed.ts b/packages/api/src/jobs/rss/refreshFeed.ts index 2c7f5d143..7da104ad5 100644 --- a/packages/api/src/jobs/rss/refreshFeed.ts +++ b/packages/api/src/jobs/rss/refreshFeed.ts @@ -5,6 +5,7 @@ import Parser, { Item } from 'rss-parser' import { SubscriptionStatus } from '../../entity/subscription' import { env } from '../../env' import { redisDataSource } from '../../redis_data_source' +import { validateUrl } from '../../services/create_page_save_request' import { updateSubscription, updateSubscriptions, @@ -392,7 +393,10 @@ const getUpdatePeriodInHours = (feed: RssFeed) => { } // get link following the order of preference: via, alternate, self -const getLink = (links: RssFeedItemLink[]): string | undefined => { +const getLink = ( + links: RssFeedItemLink[], + feedUrl: string +): string | undefined => { // sort links by preference const sortedLinks: string[] = [] @@ -414,7 +418,18 @@ const getLink = (links: RssFeedItemLink[]): string | undefined => { }) // return the first link that is not undefined - return sortedLinks.find((link) => !!link) + const itemUrl = sortedLinks.find((link) => !!link) + if (!itemUrl) { + return undefined + } + + // convert relative url to absolute url + const url = new URL(itemUrl, feedUrl).href + if (!validateUrl(url)) { + return undefined + } + + return url } const processSubscription = async ( @@ -467,7 +482,7 @@ const processSubscription = async ( throw new Error('Invalid feed item') } - const link = getLink(item.links) + const link = getLink(item.links, feedUrl) if (!link) { throw new Error('Invalid feed item link') } From 9d378d3f46310f1142e73620cfc08cb39922b97f Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Wed, 24 Jan 2024 13:21:37 +0800 Subject: [PATCH 20/59] get active user subscriptions only --- packages/api/src/jobs/rss/refreshAllFeeds.ts | 27 +++++++++++--------- 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/packages/api/src/jobs/rss/refreshAllFeeds.ts b/packages/api/src/jobs/rss/refreshAllFeeds.ts index 765d34339..c4996a80e 100644 --- a/packages/api/src/jobs/rss/refreshAllFeeds.ts +++ b/packages/api/src/jobs/rss/refreshAllFeeds.ts @@ -22,21 +22,24 @@ export const refreshAllFeeds = async (db: DataSource): Promise => { ` SELECT url, - ARRAY_AGG(id) AS "subscriptionIds", - ARRAY_AGG(user_id) AS "userIds", - ARRAY_AGG(last_fetched_at) AS "fetchedDates", - ARRAY_AGG(coalesce(scheduled_at, NOW())) AS "scheduledDates", - ARRAY_AGG(last_fetched_checksum) AS checksums, - ARRAY_AGG(fetch_content) AS "fetchContents", - ARRAY_AGG(coalesce(folder, $3)) AS folders + ARRAY_AGG(s.id) AS "subscriptionIds", + ARRAY_AGG(s.user_id) AS "userIds", + ARRAY_AGG(s.most_recent_item_date) AS "mostRecentItemDates", + ARRAY_AGG(coalesce(s.scheduled_at, NOW())) AS "scheduledDates", + ARRAY_AGG(s.last_fetched_checksum) AS checksums, + ARRAY_AGG(s.fetch_content) AS "fetchContents", + ARRAY_AGG(coalesce(s.folder, $3)) AS folders FROM - omnivore.subscriptions + omnivore.subscriptions s + INNER JOIN + omnivore.user u ON u.id = s.user_id WHERE - type = $1 - AND status = $2 - AND (scheduled_at <= NOW() OR scheduled_at IS NULL) + s.type = $1 + AND s.status = $2 + AND u.status = $2 + AND (s.scheduled_at <= NOW() OR s.scheduled_at IS NULL) GROUP BY - url + s.url `, ['RSS', 'ACTIVE', 'following'] )) as RssSubscriptionGroup[] From 86c4bbabee5ac64ed1e35b88672c7cbc6f09b84b Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Wed, 24 Jan 2024 13:30:57 +0800 Subject: [PATCH 21/59] update logs --- packages/api/src/jobs/rss/refreshFeed.ts | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/packages/api/src/jobs/rss/refreshFeed.ts b/packages/api/src/jobs/rss/refreshFeed.ts index 7da104ad5..2f50d324e 100644 --- a/packages/api/src/jobs/rss/refreshFeed.ts +++ b/packages/api/src/jobs/rss/refreshFeed.ts @@ -637,10 +637,12 @@ export const _refreshFeed = async (request: RefreshFeedRequest) => { const fetchContentTasks = new Map() // url -> FetchContentTask // process each subscription sequentially for (let i = 0; i < subscriptionIds.length; i++) { + const subscriptionId = subscriptionIds[i] + try { await processSubscription( fetchContentTasks, - subscriptionIds[i], + subscriptionId, userIds[i], feedUrl, fetchResult, @@ -652,7 +654,10 @@ export const _refreshFeed = async (request: RefreshFeedRequest) => { feed ) } catch (error) { - console.error('Error while processing subscription', error) + console.error('Error while processing subscription', { + error, + subscriptionId, + }) } } @@ -666,8 +671,12 @@ export const _refreshFeed = async (request: RefreshFeedRequest) => { } return true - } catch (e) { - console.error('Error while saving RSS feeds', e) + } catch (error) { + console.error('Error while saving RSS feeds', { + feedUrl, + subscriptionIds, + error, + }) // mark subscriptions as error if we failed to get the feed await updateSubscriptions(subscriptionIds, { From d357d2b0afd00bcec4fb3b07d293dec13854704d Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Wed, 24 Jan 2024 13:41:49 +0800 Subject: [PATCH 22/59] fix tests --- packages/api/src/jobs/rss/refreshFeed.ts | 7 +++++-- packages/api/src/resolvers/function_resolvers.ts | 4 ++++ packages/api/src/services/subscriptions.ts | 4 ++-- packages/rss-handler/test/index.test.ts | 10 +++++----- 4 files changed, 16 insertions(+), 9 deletions(-) diff --git a/packages/api/src/jobs/rss/refreshFeed.ts b/packages/api/src/jobs/rss/refreshFeed.ts index 2f50d324e..57692d770 100644 --- a/packages/api/src/jobs/rss/refreshFeed.ts +++ b/packages/api/src/jobs/rss/refreshFeed.ts @@ -74,11 +74,14 @@ interface FetchContentTask { item: RssFeedItem } -export const isOldItem = (item: RssFeedItem, lastFetchedAt: number) => { +export const isOldItem = ( + item: RssFeedItem, + mostRecentItemTimestamp: number +) => { // existing items and items that were published before 24h const publishedAt = item.isoDate ? new Date(item.isoDate) : new Date() return ( - publishedAt <= new Date(lastFetchedAt) || + publishedAt <= new Date(mostRecentItemTimestamp) || publishedAt < new Date(Date.now() - 24 * 60 * 60 * 1000) ) } diff --git a/packages/api/src/resolvers/function_resolvers.ts b/packages/api/src/resolvers/function_resolvers.ts index 6cff177ca..6142a725d 100644 --- a/packages/api/src/resolvers/function_resolvers.ts +++ b/packages/api/src/resolvers/function_resolvers.ts @@ -477,6 +477,10 @@ export const functionResolvers = { DEFAULT_SUBSCRIPTION_FOLDER ) }, + // for campability with old clients + lastFetchedAt(subscription: Subscription) { + return subscription.mostRecentItemDate + }, }, NewsletterEmail: { subscriptionCount(newsletterEmail: NewsletterEmail) { diff --git a/packages/api/src/services/subscriptions.ts b/packages/api/src/services/subscriptions.ts index fcf25af01..22be8c2f4 100644 --- a/packages/api/src/services/subscriptions.ts +++ b/packages/api/src/services/subscriptions.ts @@ -103,7 +103,7 @@ export const saveSubscription = async ({ unsubscribeHttpUrl, unsubscribeMailTo, icon, - lastFetchedAt: new Date(), + refreshedAt: new Date(), } const existingSubscription = await getSubscriptionByName(name, userId) @@ -199,7 +199,7 @@ export const createSubscription = async ( newsletterEmail, status, unsubscribeMailTo, - lastFetchedAt: new Date(), + refreshedAt: new Date(), type: subscriptionType, url, }) diff --git a/packages/rss-handler/test/index.test.ts b/packages/rss-handler/test/index.test.ts index b43cf7973..f5c7b7b38 100644 --- a/packages/rss-handler/test/index.test.ts +++ b/packages/rss-handler/test/index.test.ts @@ -7,17 +7,17 @@ describe('isOldItem', () => { const item = { pubDate: '2020-01-01', } as RssFeedItem - const lastFetchedAt = Date.now() + const mostRecentItemTimestamp = Date.now() - expect(isOldItem(item, lastFetchedAt)).to.be.true + expect(isOldItem(item, mostRecentItemTimestamp)).to.be.true }) it('returns true if item was published at the last fetched time', () => { - const lastFetchedAt = Date.now() + const mostRecentItemTimestamp = Date.now() const item = { - pubDate: new Date(lastFetchedAt).toISOString(), + pubDate: new Date(mostRecentItemTimestamp).toISOString(), } as RssFeedItem - expect(isOldItem(item, lastFetchedAt)).to.be.true + expect(isOldItem(item, mostRecentItemTimestamp)).to.be.true }) }) From 9e55fe0d7b67269d12e1e3b5e4f74d4ff615d543 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Wed, 24 Jan 2024 14:12:34 +0800 Subject: [PATCH 23/59] do not retry if user is deleted --- packages/api/src/jobs/rss/refreshAllFeeds.ts | 50 ++++++++++---------- packages/api/src/jobs/save_page.ts | 3 +- 2 files changed, 28 insertions(+), 25 deletions(-) diff --git a/packages/api/src/jobs/rss/refreshAllFeeds.ts b/packages/api/src/jobs/rss/refreshAllFeeds.ts index c4996a80e..024ae0884 100644 --- a/packages/api/src/jobs/rss/refreshAllFeeds.ts +++ b/packages/api/src/jobs/rss/refreshAllFeeds.ts @@ -20,31 +20,33 @@ export const refreshAllFeeds = async (db: DataSource): Promise => { } as RSSRefreshContext const subscriptionGroups = (await db.createEntityManager().query( ` - SELECT - url, - ARRAY_AGG(s.id) AS "subscriptionIds", - ARRAY_AGG(s.user_id) AS "userIds", - ARRAY_AGG(s.most_recent_item_date) AS "mostRecentItemDates", - ARRAY_AGG(coalesce(s.scheduled_at, NOW())) AS "scheduledDates", - ARRAY_AGG(s.last_fetched_checksum) AS checksums, - ARRAY_AGG(s.fetch_content) AS "fetchContents", - ARRAY_AGG(coalesce(s.folder, $3)) AS folders - FROM - omnivore.subscriptions s - INNER JOIN - omnivore.user u ON u.id = s.user_id - WHERE - s.type = $1 - AND s.status = $2 - AND u.status = $2 - AND (s.scheduled_at <= NOW() OR s.scheduled_at IS NULL) - GROUP BY - s.url - `, - ['RSS', 'ACTIVE', 'following'] + SELECT + url, + ARRAY_AGG(s.id) AS "subscriptionIds", + ARRAY_AGG(s.user_id) AS "userIds", + ARRAY_AGG(s.most_recent_item_date) AS "mostRecentItemDates", + ARRAY_AGG(coalesce(s.scheduled_at, NOW())) AS "scheduledDates", + ARRAY_AGG(s.last_fetched_checksum) AS checksums, + ARRAY_AGG(s.fetch_content) AS "fetchContents", + ARRAY_AGG(coalesce(s.folder, $3)) AS folders + FROM + omnivore.subscriptions s + INNER JOIN + omnivore.user u ON u.id = s.user_id + WHERE + s.type = $1 + AND s.status = $2 + AND (s.scheduled_at <= NOW() OR s.scheduled_at IS NULL) + AND u.status = $4 + GROUP BY + s.url + `, + ['RSS', 'ACTIVE', 'following', 'ACTIVE'] )) as RssSubscriptionGroup[] - console.log(`rss: checking ${subscriptionGroups.length}`, { refreshContext }) + console.log(`rss: checking ${subscriptionGroups.length}`, { + refreshContext, + }) for (const group of subscriptionGroups) { try { @@ -92,7 +94,7 @@ const updateSubscriptionGroup = async ( refreshContext, subscriptionIds: group.subscriptionIds, feedUrl: group.url, - lastFetchedTimestamps: group.mostRecentItemDates.map( + mostRecentItemDates: group.mostRecentItemDates.map( (timestamp) => timestamp?.getTime() || 0 ), // unix timestamp in milliseconds lastFetchedChecksums: group.checksums, diff --git a/packages/api/src/jobs/save_page.ts b/packages/api/src/jobs/save_page.ts index 14500d959..08f650f38 100644 --- a/packages/api/src/jobs/save_page.ts +++ b/packages/api/src/jobs/save_page.ts @@ -339,7 +339,8 @@ export const savePageJob = async (data: Data, attemptsMade: number) => { userId, url, }) - throw new Error('Unable to save job, user can not be found.') + // if the user is not found, we do not retry + return false } // for non-pdf content, we need to save the page From 1563b44197632b10f22c1ad8e436abf7ca7bb69b Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Wed, 24 Jan 2024 14:54:39 +0800 Subject: [PATCH 24/59] save the timestamp when feed fails --- packages/api/src/entity/subscription.ts | 1 - packages/api/src/generated/graphql.ts | 4 +++- packages/api/src/generated/schema.graphql | 3 ++- packages/api/src/jobs/rss/refreshFeed.ts | 14 +++++++------- packages/api/src/resolvers/subscriptions/index.ts | 5 +---- packages/api/src/schema.ts | 3 ++- packages/api/src/services/update_subscription.ts | 5 ++++- .../db/migrations/0159.do.alter_subscriptions.sql | 4 +--- .../migrations/0159.undo.alter_subscriptions.sql | 1 + 9 files changed, 21 insertions(+), 19 deletions(-) diff --git a/packages/api/src/entity/subscription.ts b/packages/api/src/entity/subscription.ts index 2c1f271b4..32fa8e456 100644 --- a/packages/api/src/entity/subscription.ts +++ b/packages/api/src/entity/subscription.ts @@ -16,7 +16,6 @@ export enum SubscriptionStatus { Active = 'ACTIVE', Deleted = 'DELETED', Unsubscribed = 'UNSUBSCRIBED', - RefreshError = 'REFRESH_ERROR', } export enum SubscriptionType { diff --git a/packages/api/src/generated/graphql.ts b/packages/api/src/generated/graphql.ts index fd24070f3..33ecceb9b 100644 --- a/packages/api/src/generated/graphql.ts +++ b/packages/api/src/generated/graphql.ts @@ -2833,6 +2833,7 @@ export type Subscription = { count: Scalars['Int']; createdAt: Scalars['Date']; description?: Maybe; + failedAt?: Maybe; fetchContent: Scalars['Boolean']; folder: Scalars['String']; icon?: Maybe; @@ -2854,7 +2855,6 @@ export type Subscription = { export enum SubscriptionStatus { Active = 'ACTIVE', Deleted = 'DELETED', - RefreshError = 'REFRESH_ERROR', Unsubscribed = 'UNSUBSCRIBED' } @@ -3208,6 +3208,7 @@ export enum UpdateSubscriptionErrorCode { export type UpdateSubscriptionInput = { autoAddToLibrary?: InputMaybe; description?: InputMaybe; + failedAt?: InputMaybe; fetchContent?: InputMaybe; folder?: InputMaybe; id: Scalars['ID']; @@ -6190,6 +6191,7 @@ export type SubscriptionResolvers; createdAt?: SubscriptionResolver; description?: SubscriptionResolver, "description", ParentType, ContextType>; + failedAt?: SubscriptionResolver, "failedAt", ParentType, ContextType>; fetchContent?: SubscriptionResolver; folder?: SubscriptionResolver; icon?: SubscriptionResolver, "icon", ParentType, ContextType>; diff --git a/packages/api/src/generated/schema.graphql b/packages/api/src/generated/schema.graphql index e81e604d9..203d7b9bd 100644 --- a/packages/api/src/generated/schema.graphql +++ b/packages/api/src/generated/schema.graphql @@ -2229,6 +2229,7 @@ type Subscription { count: Int! createdAt: Date! description: String + failedAt: Date fetchContent: Boolean! folder: String! icon: String @@ -2250,7 +2251,6 @@ type Subscription { enum SubscriptionStatus { ACTIVE DELETED - REFRESH_ERROR UNSUBSCRIBED } @@ -2575,6 +2575,7 @@ enum UpdateSubscriptionErrorCode { input UpdateSubscriptionInput { autoAddToLibrary: Boolean description: String + failedAt: Date fetchContent: Boolean folder: String id: ID! diff --git a/packages/api/src/jobs/rss/refreshFeed.ts b/packages/api/src/jobs/rss/refreshFeed.ts index 57692d770..9a18bcce9 100644 --- a/packages/api/src/jobs/rss/refreshFeed.ts +++ b/packages/api/src/jobs/rss/refreshFeed.ts @@ -461,7 +461,7 @@ const processSubscription = async ( // fetch feed let itemCount = 0, - errorCount = 0 + failedAt: Date | undefined const feedLastBuildDate = feed.lastBuildDate console.log('Feed last build date', feedLastBuildDate) @@ -538,7 +538,7 @@ const processSubscription = async ( itemCount = itemCount + 1 } catch (error) { console.error('Error while saving RSS feed item', error, item) - errorCount = errorCount + 1 + failedAt = new Date() } } @@ -561,7 +561,7 @@ const processSubscription = async ( ) if (!created) { console.error('Failed to create task for feed item', lastValidItem.link) - errorCount = errorCount + 1 + failedAt = new Date() } lastItemFetchedAt = lastValidItem.isoDate @@ -572,7 +572,6 @@ const processSubscription = async ( const updateFrequency = getUpdateFrequency(feed) const updatePeriodInMs = getUpdatePeriodInHours(feed) * 60 * 60 * 1000 const nextScheduledAt = scheduledAt + updatePeriodInMs * updateFrequency - const status = errorCount > 0 ? SubscriptionStatus.RefreshError : undefined // update subscription mostRecentItemDate and refreshedAt const updatedSubscription = await updateSubscription(userId, subscriptionId, { @@ -580,7 +579,7 @@ const processSubscription = async ( lastFetchedChecksum: updatedLastFetchedChecksum, scheduledAt: new Date(nextScheduledAt), refreshedAt, - status, + failedAt, }) console.log('Updated subscription', updatedSubscription) } @@ -681,10 +680,11 @@ export const _refreshFeed = async (request: RefreshFeedRequest) => { error, }) + const now = new Date() // mark subscriptions as error if we failed to get the feed await updateSubscriptions(subscriptionIds, { - status: SubscriptionStatus.RefreshError, - refreshedAt: new Date(), + refreshedAt: now, + failedAt: now, }) return false diff --git a/packages/api/src/resolvers/subscriptions/index.ts b/packages/api/src/resolvers/subscriptions/index.ts index 06256b8bc..a13529325 100644 --- a/packages/api/src/resolvers/subscriptions/index.ts +++ b/packages/api/src/resolvers/subscriptions/index.ts @@ -217,10 +217,7 @@ export const subscribeResolver = authorized< type: SubscriptionType.Rss, }) if (existingSubscription) { - if ( - existingSubscription.status === SubscriptionStatus.Active || - existingSubscription.status === SubscriptionStatus.RefreshError - ) { + if (existingSubscription.status === SubscriptionStatus.Active) { return { errorCodes: [SubscribeErrorCode.AlreadySubscribed], } diff --git a/packages/api/src/schema.ts b/packages/api/src/schema.ts index 298904d62..877b4bfb4 100755 --- a/packages/api/src/schema.ts +++ b/packages/api/src/schema.ts @@ -1688,13 +1688,13 @@ const schema = gql` folder: String! mostRecentItemDate: Date refreshedAt: Date + failedAt: Date } enum SubscriptionStatus { ACTIVE UNSUBSCRIBED DELETED - REFRESH_ERROR } type SubscriptionsError { @@ -2609,6 +2609,7 @@ const schema = gql` folder: String refreshedAt: Date mostRecentItemDate: Date + failedAt: Date } union UpdateSubscriptionResult = diff --git a/packages/api/src/services/update_subscription.ts b/packages/api/src/services/update_subscription.ts index b6d1a51da..bcc077291 100644 --- a/packages/api/src/services/update_subscription.ts +++ b/packages/api/src/services/update_subscription.ts @@ -25,6 +25,7 @@ type UpdateSubscriptionData = { scheduledAt?: Date | null status?: SubscriptionStatus | null refreshedAt?: Date | null + failedAt?: Date | null } export const updateSubscription = async ( @@ -44,13 +45,14 @@ export const updateSubscription = async ( lastFetchedChecksum: newData.lastFetchedChecksum || undefined, status: newData.status || undefined, scheduledAt: newData.scheduledAt || undefined, + failedAt: newData.failedAt || undefined, autoAddToLibrary: newData.autoAddToLibrary ?? undefined, isPrivate: newData.isPrivate ?? undefined, fetchContent: newData.fetchContent ?? undefined, folder: newData.folder ?? undefined, }) - return await getRepository(Subscription).findOneByOrFail({ + return await repo.findOneByOrFail({ id: subscriptionId, user: { id: userId }, }) @@ -70,6 +72,7 @@ export const updateSubscriptions = async ( lastFetchedChecksum: newData.lastFetchedChecksum || undefined, status: newData.status || undefined, scheduledAt: newData.scheduledAt || undefined, + failedAt: newData.failedAt || undefined, autoAddToLibrary: newData.autoAddToLibrary ?? undefined, isPrivate: newData.isPrivate ?? undefined, fetchContent: newData.fetchContent ?? undefined, diff --git a/packages/db/migrations/0159.do.alter_subscriptions.sql b/packages/db/migrations/0159.do.alter_subscriptions.sql index 4b15c50fa..e4050eec5 100755 --- a/packages/db/migrations/0159.do.alter_subscriptions.sql +++ b/packages/db/migrations/0159.do.alter_subscriptions.sql @@ -4,10 +4,8 @@ BEGIN; -ALTER TYPE subscription_status_type - ADD VALUE IF NOT EXISTS 'REFRESH_ERROR'; - ALTER TABLE omnivore.subscriptions + ADD COLUMN failed_at timestamptz, ADD COLUMN refreshed_at timestamptz; UPDATE omnivore.subscriptions SET refreshed_at = last_fetched_at diff --git a/packages/db/migrations/0159.undo.alter_subscriptions.sql b/packages/db/migrations/0159.undo.alter_subscriptions.sql index 7e4e59f4a..6c9f8c3ca 100755 --- a/packages/db/migrations/0159.undo.alter_subscriptions.sql +++ b/packages/db/migrations/0159.undo.alter_subscriptions.sql @@ -8,6 +8,7 @@ ALTER TABLE omnivore.subscriptions RENAME COLUMN most_recent_item_date TO last_fetched_at; ALTER TABLE omnivore.subscriptions + DROP COLUMN failed_at, DROP COLUMN refreshed_at; COMMIT; From 3f3a96297fd813c13ee50c9e9942e629719218de Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Wed, 24 Jan 2024 14:55:50 +0800 Subject: [PATCH 25/59] update entity --- packages/api/src/entity/subscription.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/packages/api/src/entity/subscription.ts b/packages/api/src/entity/subscription.ts index 32fa8e456..fd17d8d73 100644 --- a/packages/api/src/entity/subscription.ts +++ b/packages/api/src/entity/subscription.ts @@ -86,6 +86,9 @@ export class Subscription { @Column('timestamp', { nullable: true }) refreshedAt?: Date | null + @Column('timestamp', { nullable: true }) + failedAt?: Date | null + @Column('boolean') isPrivate?: boolean | null From 7bb63b4ecd99ff12014122c81057b23aefa0615c Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Wed, 24 Jan 2024 15:32:43 +0800 Subject: [PATCH 26/59] for campability with old clients, use refreshedAt to replace lastFetchedAt --- packages/api/src/jobs/rss/refreshFeed.ts | 5 ++--- packages/api/src/resolvers/function_resolvers.ts | 2 +- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/packages/api/src/jobs/rss/refreshFeed.ts b/packages/api/src/jobs/rss/refreshFeed.ts index 9a18bcce9..ad09fa909 100644 --- a/packages/api/src/jobs/rss/refreshFeed.ts +++ b/packages/api/src/jobs/rss/refreshFeed.ts @@ -2,7 +2,6 @@ import axios from 'axios' import crypto from 'crypto' import { parseHTML } from 'linkedom' import Parser, { Item } from 'rss-parser' -import { SubscriptionStatus } from '../../entity/subscription' import { env } from '../../env' import { redisDataSource } from '../../redis_data_source' import { validateUrl } from '../../services/create_page_save_request' @@ -537,13 +536,13 @@ const processSubscription = async ( itemCount = itemCount + 1 } catch (error) { - console.error('Error while saving RSS feed item', error, item) + console.error('Error while saving RSS feed item', { error, item }) failedAt = new Date() } } // no items saved - if (!lastItemFetchedAt) { + if (!lastItemFetchedAt && !failedAt) { // the feed has been fetched before, no new valid items found if (mostRecentItemDate || !lastValidItem) { console.log('No new valid items found') diff --git a/packages/api/src/resolvers/function_resolvers.ts b/packages/api/src/resolvers/function_resolvers.ts index 6142a725d..ad3e3ac96 100644 --- a/packages/api/src/resolvers/function_resolvers.ts +++ b/packages/api/src/resolvers/function_resolvers.ts @@ -479,7 +479,7 @@ export const functionResolvers = { }, // for campability with old clients lastFetchedAt(subscription: Subscription) { - return subscription.mostRecentItemDate + return subscription.refreshedAt }, }, NewsletterEmail: { From c581372a93bf1b57313d6e2e7654dc21c84a36d2 Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Wed, 24 Jan 2024 21:13:13 +0800 Subject: [PATCH 27/59] Improve exported metrics naming convention --- packages/api/src/queue-processor.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/api/src/queue-processor.ts b/packages/api/src/queue-processor.ts index 69b495ea8..4cfe45d85 100644 --- a/packages/api/src/queue-processor.ts +++ b/packages/api/src/queue-processor.ts @@ -75,7 +75,7 @@ const main = async () => { console.log('counts: ', counts) metrics.forEach((metric, idx) => { - output += `omnivore_backend_queue_${metric}_count{} ${counts[metric]}\n` + output += `omnivore_queue_messages_${metric}\{queue="${QUEUE_NAME}"\} ${counts[metric]}\n` }) res.status(200).setHeader('Content-Type', 'text/plain').send(output) From 5f9be385b7be047bd75c6016c325123d6983b06c Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Wed, 24 Jan 2024 21:34:51 +0800 Subject: [PATCH 28/59] Remove unneeded escapes --- packages/api/src/queue-processor.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/api/src/queue-processor.ts b/packages/api/src/queue-processor.ts index 4cfe45d85..5aac7012d 100644 --- a/packages/api/src/queue-processor.ts +++ b/packages/api/src/queue-processor.ts @@ -75,7 +75,7 @@ const main = async () => { console.log('counts: ', counts) metrics.forEach((metric, idx) => { - output += `omnivore_queue_messages_${metric}\{queue="${QUEUE_NAME}"\} ${counts[metric]}\n` + output += `omnivore_queue_messages_${metric}{queue="${QUEUE_NAME}"} ${counts[metric]}\n` }) res.status(200).setHeader('Content-Type', 'text/plain').send(output) From 62df69fd26a4e6daa1ac862a9efa34c0a0e32e57 Mon Sep 17 00:00:00 2001 From: Satindar Dhillon Date: Wed, 24 Jan 2024 14:10:55 -0800 Subject: [PATCH 29/59] call helper function to locate root vc from main thread --- .../Views/Registration/RegistrationView.swift | 15 +++++++++++- .../Services/Authentication/GoogleAuth.swift | 24 +++---------------- 2 files changed, 17 insertions(+), 22 deletions(-) diff --git a/apple/OmnivoreKit/Sources/App/Views/Registration/RegistrationView.swift b/apple/OmnivoreKit/Sources/App/Views/Registration/RegistrationView.swift index db6250caf..c618d34b4 100644 --- a/apple/OmnivoreKit/Sources/App/Views/Registration/RegistrationView.swift +++ b/apple/OmnivoreKit/Sources/App/Views/Registration/RegistrationView.swift @@ -59,7 +59,8 @@ import Views } func handleGoogleAuth(authenticator: Authenticator) async { - let googleAuthResponse = await authenticator.handleGoogleAuth() + let presentingVC = presentingViewController() + let googleAuthResponse = await authenticator.handleGoogleAuth(presentingVC: presentingVC) switch googleAuthResponse { case let .loginError(error): @@ -71,3 +72,15 @@ import Views } } } + +@MainActor private func presentingViewController() -> PlatformViewController? { + #if os(iOS) + let scene = UIApplication.shared.connectedScenes.first as? UIWindowScene + return scene?.windows + .filter(\.isKeyWindow) + .first? + .rootViewController + #elseif os(macOS) + return NSApplication.shared.windows.first + #endif +} diff --git a/apple/OmnivoreKit/Sources/Services/Authentication/GoogleAuth.swift b/apple/OmnivoreKit/Sources/Services/Authentication/GoogleAuth.swift index 2fa15f0f2..89324e140 100644 --- a/apple/OmnivoreKit/Sources/Services/Authentication/GoogleAuth.swift +++ b/apple/OmnivoreKit/Sources/Services/Authentication/GoogleAuth.swift @@ -10,9 +10,9 @@ public enum GoogleAuthResponse { } extension Authenticator { - public func handleGoogleAuth() async -> GoogleAuthResponse { + public func handleGoogleAuth(presentingVC: PlatformViewController?) async -> GoogleAuthResponse { let idToken = await withCheckedContinuation { continuation in - googleSignIn { continuation.resume(returning: $0) } + googleSignIn(presenting: presentingVC) { continuation.resume(returning: $0) } } guard let idToken = idToken else { return .loginError(error: .unauthorized) } @@ -50,13 +50,7 @@ extension Authenticator { } } - func googleSignIn(completion: @escaping (String?) -> Void) { - #if os(iOS) - let presenting = presentingViewController() - #else - let presenting = NSApplication.shared.windows.first - #endif - + func googleSignIn(presenting: PlatformViewController?, completion: @escaping (String?) -> Void) { guard let presenting = presenting else { completion(nil) return @@ -82,15 +76,3 @@ extension Authenticator { } } } - -private func presentingViewController() -> PlatformViewController? { - #if os(iOS) - let scene = UIApplication.shared.connectedScenes.first as? UIWindowScene - return scene?.windows - .filter(\.isKeyWindow) - .first? - .rootViewController - #elseif os(macOS) - return nil - #endif -} From a639a2fa102642214f39b476c23a52dc1ed84881 Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Thu, 25 Jan 2024 12:37:02 +0800 Subject: [PATCH 30/59] Add types to exported metrics --- packages/api/src/queue-processor.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/api/src/queue-processor.ts b/packages/api/src/queue-processor.ts index 5aac7012d..233bf36e7 100644 --- a/packages/api/src/queue-processor.ts +++ b/packages/api/src/queue-processor.ts @@ -69,12 +69,14 @@ const main = async () => { res.sendStatus(400) return } + let output = '' const metrics: JobType[] = ['active', 'failed', 'completed', 'prioritized'] const counts = await queue.getJobCounts(...metrics) console.log('counts: ', counts) metrics.forEach((metric, idx) => { + output += `# TYPE omnivore_queue_messages_${metric} gauge\n` output += `omnivore_queue_messages_${metric}{queue="${QUEUE_NAME}"} ${counts[metric]}\n` }) From 5e239d2568f82d644cce301cc43c27dd40de61e4 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Wed, 24 Jan 2024 18:57:25 +0800 Subject: [PATCH 31/59] run readability in save-page instead of puppeteer --- packages/api/src/jobs/save_page.ts | 15 +- packages/api/src/services/save_email.ts | 1 - packages/api/src/services/save_page.ts | 16 +- packages/api/src/utils/parser.ts | 203 +++++++++--------- packages/content-fetch/src/request_handler.ts | 1 - packages/puppeteer-parse/src/index.ts | 142 ++---------- 6 files changed, 126 insertions(+), 252 deletions(-) diff --git a/packages/api/src/jobs/save_page.ts b/packages/api/src/jobs/save_page.ts index 08f650f38..3d143caf8 100644 --- a/packages/api/src/jobs/save_page.ts +++ b/packages/api/src/jobs/save_page.ts @@ -1,4 +1,3 @@ -import { Readability } from '@omnivore/readability' import axios from 'axios' import jwt from 'jsonwebtoken' import { promisify } from 'util' @@ -63,7 +62,6 @@ interface FetchResult { title?: string content?: string contentType?: string - readabilityResult?: Readability.ParseResult } const isFetchResult = (obj: unknown): obj is FetchResult => { @@ -297,7 +295,7 @@ export const savePageJob = async (data: Data, attemptsMade: number) => { // get the fetch result from cache const fetchedResult = await getCachedFetchResult(url) - const { title, contentType, readabilityResult } = fetchedResult + const { title, contentType } = fetchedResult let content = fetchedResult.content // for pdf content, we need to upload the pdf @@ -350,7 +348,6 @@ export const savePageJob = async (data: Data, attemptsMade: number) => { clientRequestId: articleSavingRequestId, title, originalContent: content, - parseResult: readabilityResult, state: state ? (state as ArticleSavingRequestStatus) : undefined, labels: labels, rssFeedUrl, @@ -362,13 +359,11 @@ export const savePageJob = async (data: Data, attemptsMade: number) => { user ) - // if (result.__typename == 'SaveError') { - // logger.error('Error saving page', { userId, url, result }) - // throw new Error('Error saving page') - // } + if (result.__typename == 'SaveError') { + throw new Error(result.message || 'Failed to save page') + } - // if the readability result is not parsed, the import is failed - isImported = !!readabilityResult + isImported = true isSaved = true } catch (e) { if (e instanceof Error) { diff --git a/packages/api/src/services/save_email.ts b/packages/api/src/services/save_email.ts index 092c2b98f..0c0f4f456 100644 --- a/packages/api/src/services/save_email.ts +++ b/packages/api/src/services/save_email.ts @@ -53,7 +53,6 @@ export const saveEmail = async ( // can leave this empty for now }, }, - null, true ) diff --git a/packages/api/src/services/save_page.ts b/packages/api/src/services/save_page.ts index a91f90549..9d55961e7 100644 --- a/packages/api/src/services/save_page.ts +++ b/packages/api/src/services/save_page.ts @@ -68,17 +68,13 @@ export const savePage = async ( input: SavePageInput, user: User ): Promise => { - const parseResult = await parsePreparedContent( - input.url, - { - document: input.originalContent, - pageInfo: { - title: input.title, - canonicalUrl: input.url, - }, + const parseResult = await parsePreparedContent(input.url, { + document: input.originalContent, + pageInfo: { + title: input.title, + canonicalUrl: input.url, }, - input.parseResult - ) + }) const [newSlug, croppedPathname] = createSlug(input.url, input.title) let slug = newSlug let clientRequestId = input.clientRequestId diff --git a/packages/api/src/utils/parser.ts b/packages/api/src/utils/parser.ts index 54373baea..ae57ee6d0 100644 --- a/packages/api/src/utils/parser.ts +++ b/packages/api/src/utils/parser.ts @@ -223,7 +223,6 @@ const getReadabilityResult = async ( export const parsePreparedContent = async ( url: string, preparedDocument: PreparedDocumentInput, - parseResult?: Readability.ParseResult | null, isNewsletter?: boolean, allowRetry = true ): Promise => { @@ -232,9 +231,6 @@ export const parsePreparedContent = async ( labels: { source: 'parsePreparedContent' }, } - // If we have a parse result, use it - let article = parseResult || null - let highlightData = undefined const { document, pageInfo } = preparedDocument if (!document) { @@ -262,137 +258,134 @@ export const parsePreparedContent = async ( } } - let dom: Document | null = null + const { title: pageInfoTitle, canonicalUrl } = pageInfo + + let parsedContent = null + let pageType = PageType.Unknown + let highlightData = undefined try { - dom = parseHTML(document).document + const dom = parseHTML(document).document + pageType = parseOriginalContent(dom) - if (!article) { - // Attempt to parse the article - // preParse content - dom = (await preParseContent(url, dom)) || dom + // Run readability + await preParseContent(url, dom) - article = await getReadabilityResult(url, document, dom, isNewsletter) + parsedContent = await getReadabilityResult(url, document, dom, isNewsletter) + + if (!parsedContent || !parsedContent.content) { + logger.info('No parsed content') + + if (allowRetry) { + logger.info('Retrying with content wrapped in html body') + + const newDocument = { + ...preparedDocument, + document: '' + document + '', // wrap in body + } + return parsePreparedContent(url, newDocument, isNewsletter, false) + } + + return { + canonicalUrl, + parsedContent, + domContent: document, + pageType, + } } - if (!article?.textContent && allowRetry) { - const newDocument = { - ...preparedDocument, - document: '' + document + '', // wrap in body - } - return parsePreparedContent( - url, - newDocument, - parseResult, - isNewsletter, - false - ) + // use title if not found after running readability + if (!parsedContent.title && pageInfoTitle) { + parsedContent.title = pageInfoTitle } // Format code blocks // TODO: we probably want to move this type of thing // to the handlers, and have some concept of postHandle - if (article?.content) { - const articleDom = parseHTML(article.content).document - const codeBlocks = articleDom.querySelectorAll( - 'code, pre[class^="prism-"], pre[class^="language-"]' - ) - if (codeBlocks.length > 0) { - codeBlocks.forEach((e) => { - if (e.textContent) { - const att = hljs.highlightAuto(e.textContent) - const code = articleDom.createElement('code') - const langClass = - `hljs language-${att.language}` + - (att.second_best?.language - ? ` language-${att.second_best?.language}` - : '') - code.setAttribute('class', langClass) - code.innerHTML = att.value - e.replaceWith(code) - } - }) - article.content = articleDom.documentElement.outerHTML - } - - highlightData = findEmbeddedHighlight(articleDom.documentElement) - - const ANCHOR_ELEMENTS_BLOCKED_ATTRIBUTES = [ - 'omnivore-highlight-id', - 'data-twitter-tweet-id', - 'data-instagram-id', - ] - - // Get the top level element? - const pageNode = articleDom.firstElementChild as HTMLElement - const nodesToVisitStack: [HTMLElement] = [pageNode] - const visitedNodeList = [] - - while (nodesToVisitStack.length > 0) { - const currentNode = nodesToVisitStack.pop() - if ( - currentNode?.nodeType !== 1 || - // Avoiding dynamic elements from being counted as anchor-allowed elements - ANCHOR_ELEMENTS_BLOCKED_ATTRIBUTES.some((attrib) => - currentNode.hasAttribute(attrib) - ) - ) { - continue + const newDom = parseHTML(parsedContent.content).document + const codeBlocks = newDom.querySelectorAll( + 'code, pre[class^="prism-"], pre[class^="language-"]' + ) + if (codeBlocks.length > 0) { + codeBlocks.forEach((e) => { + if (e.textContent) { + const att = hljs.highlightAuto(e.textContent) + const code = newDom.createElement('code') + const langClass = + `hljs language-${att.language}` + + (att.second_best?.language + ? ` language-${att.second_best?.language}` + : '') + code.setAttribute('class', langClass) + code.innerHTML = att.value + e.replaceWith(code) } - visitedNodeList.push(currentNode) - ;[].slice - .call(currentNode.childNodes) - .reverse() - .forEach(function (node) { - nodesToVisitStack.push(node) - }) - } - - visitedNodeList.shift() - visitedNodeList.forEach((node, index) => { - // start from index 1, index 0 reserved for anchor unknown. - node.setAttribute('data-omnivore-anchor-idx', (index + 1).toString()) }) - - article.content = articleDom.documentElement.outerHTML } + highlightData = findEmbeddedHighlight(newDom.documentElement) + + const ANCHOR_ELEMENTS_BLOCKED_ATTRIBUTES = [ + 'omnivore-highlight-id', + 'data-twitter-tweet-id', + 'data-instagram-id', + ] + + // Get the top level element? + const pageNode = newDom.firstElementChild as HTMLElement + const nodesToVisitStack: [HTMLElement] = [pageNode] + const visitedNodeList = [] + + while (nodesToVisitStack.length > 0) { + const currentNode = nodesToVisitStack.pop() + if ( + currentNode?.nodeType !== 1 || + // Avoiding dynamic elements from being counted as anchor-allowed elements + ANCHOR_ELEMENTS_BLOCKED_ATTRIBUTES.some((attrib) => + currentNode.hasAttribute(attrib) + ) + ) { + continue + } + visitedNodeList.push(currentNode) + ;[].slice + .call(currentNode.childNodes) + .reverse() + .forEach(function (node) { + nodesToVisitStack.push(node) + }) + } + + visitedNodeList.shift() + visitedNodeList.forEach((node, index) => { + // start from index 1, index 0 reserved for anchor unknown. + node.setAttribute('data-omnivore-anchor-idx', (index + 1).toString()) + }) + + const newHtml = newDom.documentElement.outerHTML const newWindow = parseHTML('') const DOMPurify = createDOMPurify(newWindow) DOMPurify.addHook('uponSanitizeElement', domPurifySanitizeHook) - const clean = DOMPurify.sanitize(article?.content || '', DOM_PURIFY_CONFIG) + const cleanHtml = DOMPurify.sanitize(newHtml, DOM_PURIFY_CONFIG) + parsedContent.content = cleanHtml - Object.assign(article || {}, { - content: clean, - title: article?.title, - previewImage: article?.previewImage, - siteName: article?.siteName, - siteIcon: article?.siteIcon, - byline: article?.byline, - language: article?.language, - }) logRecord.parseSuccess = true } catch (error) { - logger.info('Error parsing content', error) + logger.error('Error parsing content', error) + Object.assign(logRecord, { parseSuccess: false, parseError: error, }) } - const { title, canonicalUrl } = pageInfo - - Object.assign(article || {}, { - title: article?.title || title, - }) - - logger.info('parse-article completed') + logger.info('parse-article completed', logRecord) return { - domContent: document, - parsedContent: article, canonicalUrl, - pageType: dom ? parseOriginalContent(dom) : PageType.Unknown, + parsedContent, + domContent: document, + pageType, highlightData, } } diff --git a/packages/content-fetch/src/request_handler.ts b/packages/content-fetch/src/request_handler.ts index 0106d8058..56356ec1e 100644 --- a/packages/content-fetch/src/request_handler.ts +++ b/packages/content-fetch/src/request_handler.ts @@ -50,7 +50,6 @@ interface FetchResult { title?: string content?: string contentType?: string - readabilityResult?: unknown } export const cacheFetchResult = async (fetchResult: FetchResult) => { diff --git a/packages/puppeteer-parse/src/index.ts b/packages/puppeteer-parse/src/index.ts index 2629689c2..a02a3595b 100644 --- a/packages/puppeteer-parse/src/index.ts +++ b/packages/puppeteer-parse/src/index.ts @@ -1,10 +1,7 @@ /* eslint-disable @typescript-eslint/no-unsafe-member-access */ /* eslint-disable @typescript-eslint/no-unsafe-assignment */ -import { preHandleContent, preParseContent } from '@omnivore/content-handler' -import { Readability } from '@omnivore/readability' +import { preHandleContent } from '@omnivore/content-handler' import axios from 'axios' -import crypto from 'crypto' -import createDOMPurify, { SanitizeElementHookEvent } from 'dompurify' // const { Storage } = require('@google-cloud/storage'); import { parseHTML } from 'linkedom' import path from 'path' @@ -13,7 +10,6 @@ import puppeteer from 'puppeteer-extra' import AdblockerPlugin from 'puppeteer-extra-plugin-adblocker' import StealthPlugin from 'puppeteer-extra-plugin-stealth' import Url from 'url' -import { encode } from 'urlsafe-base64' // Add stealth plugin to hide puppeteer usage puppeteer.use(StealthPlugin()) @@ -28,12 +24,12 @@ puppeteer.use(AdblockerPlugin({ blockTrackers: true })) // const filePath = `${os.tmpdir()}/previewImage.png` -const MOBILE_USER_AGENT = - 'Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/93.0.4577.62 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)' +// const MOBILE_USER_AGENT = +// 'Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/93.0.4577.62 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)' const DESKTOP_USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 11_6_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4372.0 Safari/537.36' -const BOT_DESKTOP_USER_AGENT = - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 11_6_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4372.0 Safari/537.36' +// const BOT_DESKTOP_USER_AGENT = +// 'Mozilla/5.0 (Macintosh; Intel Mac OS X 11_6_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4372.0 Safari/537.36' const NON_BOT_DESKTOP_USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 11_6_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4372.0 Safari/537.36' const NON_BOT_HOSTS = ['bloomberg.com', 'forbes.com'] @@ -156,8 +152,8 @@ export const fetchContent = async ( page: Page | undefined, title: string | undefined, content: string | undefined, - contentType: string | undefined, - readabilityResult: Readability.ParseResult | null | undefined + contentType: string | undefined + try { url = getUrl(url) if (!url) { @@ -230,34 +226,26 @@ export const fetchContent = async ( console.info('fallback to scrapingbee', url) const sbResult = await fetchContentWithScrapingBee(url) - content = sbResult.domContent - title = sbResult.title - } else { - throw e + + return { + finalUrl: url, + title: sbResult.title, + content: sbResult.domContent, + contentType, + } } + + throw e } finally { // close browser context if it was opened if (context) { await context.close() } - // save non pdf content - if (url && contentType !== 'application/pdf') { - // parse content if it is not empty - if (content) { - let document = parseHTML(content).document - // preParse content - const preParsedDom = await preParseContent(url, document) - if (preParsedDom) { - document = preParsedDom - } - readabilityResult = await getReadabilityResult(url, document) - } - } console.info(`content-fetch result`, logRecord) } - return { finalUrl: url, title, content, readabilityResult, contentType } + return { finalUrl: url, title, content, contentType } } function validateUrlString(url: string) { @@ -741,99 +729,3 @@ async function retrieveHtml(page: Page, logRecord: Record) { // console.info(`preview-image`, logRecord); // return res.redirect(`${process.env.PREVIEW_IMAGE_CDN_ORIGIN}/${destination}`); // } - -const DOM_PURIFY_CONFIG = { - ADD_TAGS: ['iframe'], - ADD_ATTR: ['allow', 'allowfullscreen', 'frameborder', 'scrolling'], - FORBID_ATTR: [ - 'data-ml-dynamic', - 'data-ml-dynamic-type', - 'data-orig-url', - 'data-ml-id', - 'data-ml', - 'data-xid', - 'data-feature', - ], -} - -function domPurifySanitizeHook(node: Element, data: SanitizeElementHookEvent) { - if (data.tagName === 'iframe') { - const urlRegex = /^(https?:)?\/\/www\.youtube(-nocookie)?\.com\/embed\//i - const src = node.getAttribute('src') || '' - const dataSrc = node.getAttribute('data-src') || '' - - if (src && urlRegex.test(src)) { - return - } - - if (dataSrc && urlRegex.test(dataSrc)) { - node.setAttribute('src', dataSrc) - return - } - - node.parentNode?.removeChild(node) - } -} - -function getPurifiedContent(html: Document) { - const newWindow = parseHTML('') - const DOMPurify = createDOMPurify(newWindow) - DOMPurify.addHook('uponSanitizeElement', domPurifySanitizeHook) - const clean = DOMPurify.sanitize(html, DOM_PURIFY_CONFIG) - return parseHTML(clean).document -} - -function signImageProxyUrl(url: string) { - return encode( - crypto - .createHmac('sha256', process.env.IMAGE_PROXY_SECRET || '') - .update(url) - .digest() - ) -} - -function createImageProxyUrl(url: string, width = 0, height = 0) { - if (!process.env.IMAGE_PROXY_URL || !process.env.IMAGE_PROXY_SECRET) { - return url - } - - const urlWithOptions = `${url}#${width}x${height}` - const signature = signImageProxyUrl(urlWithOptions) - - return `${process.env.IMAGE_PROXY_URL}/${width}x${height},s${signature}/${url}` -} - -async function getReadabilityResult(url: string, document: Document) { - // First attempt to read the article as is. - // if that fails attempt to purify then read - const sources = [ - () => { - return document - }, - () => { - return getPurifiedContent(document) - }, - ] - - for (const source of sources) { - const document = source() - if (!document) { - continue - } - - try { - const article = await new Readability(document, { - createImageProxyUrl, - url, - }).parse() - - if (article) { - return article - } - } catch (error) { - console.log('parsing error for url', url, error) - } - } - - return null -} From 8834efabd46789e52434e57ab5c02092e5581f8a Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Wed, 24 Jan 2024 19:09:18 +0800 Subject: [PATCH 32/59] throw error if save-page job failed --- packages/api/src/jobs/save_page.ts | 2 +- packages/api/src/services/save_page.ts | 14 +++++++++----- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/packages/api/src/jobs/save_page.ts b/packages/api/src/jobs/save_page.ts index 3d143caf8..0e87eb30c 100644 --- a/packages/api/src/jobs/save_page.ts +++ b/packages/api/src/jobs/save_page.ts @@ -360,7 +360,7 @@ export const savePageJob = async (data: Data, attemptsMade: number) => { ) if (result.__typename == 'SaveError') { - throw new Error(result.message || 'Failed to save page') + throw new Error(result.message || result.errorCodes[0]) } isImported = true diff --git a/packages/api/src/services/save_page.ts b/packages/api/src/services/save_page.ts index 9d55961e7..7931fc48e 100644 --- a/packages/api/src/services/save_page.ts +++ b/packages/api/src/services/save_page.ts @@ -112,6 +112,7 @@ export const savePage = async ( }) } catch (e) { return { + __typename: 'SaveError', errorCodes: [SaveErrorCode.Unknown], message: 'Failed to create page save request', } @@ -183,11 +184,14 @@ export const savePage = async ( libraryItem: { id: clientRequestId }, } - if (!(await createHighlight(highlight, clientRequestId, user.id))) { - return { - errorCodes: [SaveErrorCode.EmbeddedHighlightFailed], - message: 'Failed to save highlight', - } + try { + await createHighlight(highlight, clientRequestId, user.id) + } catch (error) { + logger.error('Failed to create highlight', { + highlight, + clientRequestId, + userId: user.id, + }) } } From dae90aaffa650b3e37cd328c95c57c2b346f4f51 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Wed, 24 Jan 2024 21:36:21 +0800 Subject: [PATCH 33/59] fix code blocks not formatted correctly --- packages/api/package.json | 2 +- packages/api/src/readability.d.ts | 1 + packages/api/src/utils/parser.ts | 64 ++++++++++++++------------- packages/readabilityjs/Readability.js | 1 + 4 files changed, 36 insertions(+), 32 deletions(-) diff --git a/packages/api/package.json b/packages/api/package.json index 9060f0641..7d5420686 100644 --- a/packages/api/package.json +++ b/packages/api/package.json @@ -155,4 +155,4 @@ "volta": { "extends": "../../package.json" } -} \ No newline at end of file +} diff --git a/packages/api/src/readability.d.ts b/packages/api/src/readability.d.ts index 4722588cd..fcfcc1e55 100644 --- a/packages/api/src/readability.d.ts +++ b/packages/api/src/readability.d.ts @@ -166,6 +166,7 @@ declare module '@omnivore/readability' { /** Article published date */ publishedDate?: Date | null language?: string | null + documentElement: HTMLElement } } diff --git a/packages/api/src/utils/parser.ts b/packages/api/src/utils/parser.ts index ae57ee6d0..05fdcc3a2 100644 --- a/packages/api/src/utils/parser.ts +++ b/packages/api/src/utils/parser.ts @@ -231,9 +231,8 @@ export const parsePreparedContent = async ( labels: { source: 'parsePreparedContent' }, } - const { document, pageInfo } = preparedDocument - - if (!document) { + const { document: domContent, pageInfo } = preparedDocument + if (!domContent) { logger.info('No document') return { canonicalUrl: url, @@ -253,25 +252,30 @@ export const parsePreparedContent = async ( return { canonicalUrl: url, parsedContent: null, - domContent: document, + domContent, pageType: PageType.Unknown, } } const { title: pageInfoTitle, canonicalUrl } = pageInfo - let parsedContent = null + let parsedContent: Readability.ParseResult | null = null let pageType = PageType.Unknown let highlightData = undefined try { - const dom = parseHTML(document).document - pageType = parseOriginalContent(dom) + const document = parseHTML(domContent).document + pageType = parseOriginalContent(document) // Run readability - await preParseContent(url, dom) + await preParseContent(url, document) - parsedContent = await getReadabilityResult(url, document, dom, isNewsletter) + parsedContent = await getReadabilityResult( + url, + domContent, + document, + isNewsletter + ) if (!parsedContent || !parsedContent.content) { logger.info('No parsed content') @@ -281,7 +285,7 @@ export const parsePreparedContent = async ( const newDocument = { ...preparedDocument, - document: '' + document + '', // wrap in body + document: '' + domContent + '', // wrap in body } return parsePreparedContent(url, newDocument, isNewsletter, false) } @@ -289,7 +293,7 @@ export const parsePreparedContent = async ( return { canonicalUrl, parsedContent, - domContent: document, + domContent, pageType, } } @@ -299,31 +303,29 @@ export const parsePreparedContent = async ( parsedContent.title = pageInfoTitle } + const newDocumentElement = parsedContent.documentElement // Format code blocks // TODO: we probably want to move this type of thing // to the handlers, and have some concept of postHandle - const newDom = parseHTML(parsedContent.content).document - const codeBlocks = newDom.querySelectorAll( - 'code, pre[class^="prism-"], pre[class^="language-"]' + const codeBlocks = newDocumentElement.querySelectorAll( + 'pre[class^="prism-"], pre[class^="language-"], pre[class^="code-snippet"], code' ) if (codeBlocks.length > 0) { codeBlocks.forEach((e) => { - if (e.textContent) { - const att = hljs.highlightAuto(e.textContent) - const code = newDom.createElement('code') - const langClass = - `hljs language-${att.language}` + - (att.second_best?.language - ? ` language-${att.second_best?.language}` - : '') - code.setAttribute('class', langClass) - code.innerHTML = att.value - e.replaceWith(code) - } + const att = hljs.highlightAuto(e.innerText) + const code = document.createElement('code') + const langClass = + `hljs language-${att.language}` + + (att.second_best?.language + ? ` language-${att.second_best?.language}` + : '') + code.setAttribute('class', langClass) + code.innerHTML = att.value + e.replaceWith(code) }) } - highlightData = findEmbeddedHighlight(newDom.documentElement) + highlightData = findEmbeddedHighlight(newDocumentElement) const ANCHOR_ELEMENTS_BLOCKED_ATTRIBUTES = [ 'omnivore-highlight-id', @@ -332,8 +334,8 @@ export const parsePreparedContent = async ( ] // Get the top level element? - const pageNode = newDom.firstElementChild as HTMLElement - const nodesToVisitStack: [HTMLElement] = [pageNode] + // const pageNode = newDocumentElement.firstElementChild as HTMLElement + const nodesToVisitStack: [HTMLElement] = [newDocumentElement] const visitedNodeList = [] while (nodesToVisitStack.length > 0) { @@ -362,7 +364,7 @@ export const parsePreparedContent = async ( node.setAttribute('data-omnivore-anchor-idx', (index + 1).toString()) }) - const newHtml = newDom.documentElement.outerHTML + const newHtml = newDocumentElement.outerHTML const newWindow = parseHTML('') const DOMPurify = createDOMPurify(newWindow) DOMPurify.addHook('uponSanitizeElement', domPurifySanitizeHook) @@ -384,7 +386,7 @@ export const parsePreparedContent = async ( return { canonicalUrl, parsedContent, - domContent: document, + domContent, pageType, highlightData, } diff --git a/packages/readabilityjs/Readability.js b/packages/readabilityjs/Readability.js index 375b3a6ee..be51c4bed 100644 --- a/packages/readabilityjs/Readability.js +++ b/packages/readabilityjs/Readability.js @@ -3135,6 +3135,7 @@ Readability.prototype = { previewImage: metadata.previewImage, publishedDate, language, + documentElement: articleContent, }; } }; From 7ff327787bc9485e207d67360460aea48d7f4b75 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Wed, 24 Jan 2024 21:55:39 +0800 Subject: [PATCH 34/59] add test case --- .../realpython/expected-metadata.json | 6025 +++++++++++++++++ .../test/test-pages/realpython/expected.html | 540 ++ .../test/test-pages/realpython/source.html | 4150 ++++++++++++ .../test/test-pages/realpython/url.txt | 1 + 4 files changed, 10716 insertions(+) create mode 100644 packages/readabilityjs/test/test-pages/realpython/expected-metadata.json create mode 100644 packages/readabilityjs/test/test-pages/realpython/expected.html create mode 100644 packages/readabilityjs/test/test-pages/realpython/source.html create mode 100644 packages/readabilityjs/test/test-pages/realpython/url.txt diff --git a/packages/readabilityjs/test/test-pages/realpython/expected-metadata.json b/packages/readabilityjs/test/test-pages/realpython/expected-metadata.json new file mode 100644 index 000000000..c4b7bdc5b --- /dev/null +++ b/packages/readabilityjs/test/test-pages/realpython/expected-metadata.json @@ -0,0 +1,6025 @@ +{ + "title": "Python 3.10: Cool New Features for You to Try", + "byline": "Real Python", + "dir": null, + "excerpt": "In this tutorial, you'll explore some of the coolest and most useful features in Python 3.10. You'll appreciate more user-friendly error messages, learn about how you can handle complicated data structures with structural pattern matching, and explore new enhancements to Python's type system.", + "siteName": "Real Python", + "siteIcon": "http://fakehost/static/favicon.68cbf4197b0c.png", + "previewImage": "https://files.realpython.com/media/Python-3.10-Cool-New-Features-for-You-to-Try_Watermarked.e2782d8a16dc.jpg", + "publishedDate": "2021-10-04T14:00:00.000Z", + "language": "English", + "documentElement": [ + 1, + "DIV", + 2, + "id", + "readability-content", + 1, + "DIV", + 2, + "class", + "page", + 2, + "id", + "readability-page-1", + 1, + "div", + 1, + "figure", + 1, + "img", + 2, + "alt", + "Python 3.10: Cool New Features for You to Try", + 2, + "width", + "1920", + 2, + "height", + "1080", + 2, + "src", + "https://files.realpython.com/media/Python-3.10-Cool-New-Features-for-You-to-Try_Watermarked.e2782d8a16dc.jpg", + 2, + "srcset", + "/cdn-cgi/image/width=480,format=auto/https://files.realpython.com/media/Python-3.10-Cool-New-Features-for-You-to-Try_Watermarked.e2782d8a16dc.jpg 480w, /cdn-cgi/image/width=640,format=auto/https://files.realpython.com/media/Python-3.10-Cool-New-Features-for-You-to-Try_Watermarked.e2782d8a16dc.jpg 640w, /cdn-cgi/image/width=960,format=auto/https://files.realpython.com/media/Python-3.10-Cool-New-Features-for-You-to-Try_Watermarked.e2782d8a16dc.jpg 960w, /cdn-cgi/image/width=1920,format=auto/https://files.realpython.com/media/Python-3.10-Cool-New-Features-for-You-to-Try_Watermarked.e2782d8a16dc.jpg 1920w", + 2, + "sizes", + "(min-width: 1200px) 690px, (min-width: 780px) calc(-5vw + 669px), (min-width: 580px) 510px, calc(100vw - 30px)", + 2, + "fetchpriority", + "high", + -2, + 1, + "div", + 1, + "p", + 1, + "span", + 3, + " Watch Now", + -1, + 3, + " This tutorial has a related video course created by the Real Python team. Watch it together with the written tutorial to deepen your understanding: ", + 1, + "a", + 2, + "href", + "http://fakehost/courses/cool-new-features-python-310/", + 1, + "strong", + 3, + "Cool New Features in Python 3.10", + -3, + 1, + "p", + 1, + "a", + 2, + "href", + "https://www.python.org/downloads/release/python-3100/", + 3, + "Python 3.10 is out!", + -1, + 3, + " Volunteers have been working on the new version since May 2020 to bring you a better, faster, and more secure Python. As of ", + 1, + "a", + 2, + "href", + "https://www.python.org/dev/peps/pep-0619/", + 3, + "October 4, 2021", + -1, + 3, + ", the first official version is available.\n ", + -1, + 1, + "p", + 3, + "\n Each new version of Python brings a host of changes. You can read about all of them in the ", + 1, + "a", + 2, + "href", + "https://docs.python.org/3.10/whatsnew/3.10.html", + 3, + "documentation", + -1, + 3, + ". Here, you’ll get to learn about the coolest new features.\n ", + -1, + 1, + "p", + 1, + "strong", + 3, + "In this tutorial, you’ll learn about:", + -2, + 1, + "ul", + 1, + "li", + 3, + "Debugging with more helpful and precise ", + 1, + "strong", + 3, + "error messages", + -2, + 1, + "li", + 3, + "Using ", + 1, + "strong", + 3, + "structural pattern matching", + -1, + 3, + " to work with data structures\n ", + -1, + 1, + "li", + 3, + "Adding more readable and more specific ", + 1, + "strong", + 3, + "type hints", + -2, + 1, + "li", + 3, + "Checking the ", + 1, + "strong", + 3, + "length of sequences", + -1, + 3, + " when using ", + 1, + "code", + 3, + "zip()", + -2, + 1, + "li", + 3, + "Calculating ", + 1, + "strong", + 3, + "multivariable statistics", + -3, + 1, + "p", + 3, + "\n To try out the new features yourself, you need to run Python 3.10. You can get it from the ", + 1, + "a", + 2, + "href", + "https://www.python.org/downloads/", + 3, + "Python homepage", + -1, + 3, + ". Alternatively, you can ", + 1, + "a", + 2, + "href", + "https://realpython.com/python-versions-docker/", + 3, + "use Docker", + -1, + 3, + " with the ", + 1, + "a", + 2, + "href", + "https://hub.docker.com/_/python/", + 3, + "latest Python image", + -1, + 3, + ".\n ", + -1, + 1, + "section", + 1, + "h2", + 2, + "id", + "better-error-messages", + 3, + "\n Better Error Messages", + 1, + "a", + 2, + "href", + "#better-error-messages", + 2, + "title", + "Permanent link", + -2, + 1, + "p", + 3, + "\n Python is often lauded for being a user-friendly programming language. While this is true, there are certain parts of Python that could be friendlier. Python 3.10 comes with a host of more precise and constructive error messages. In this section, you’ll see some of the newest improvements. The full list is available in the ", + 1, + "a", + 2, + "href", + "https://docs.python.org/3.10/whatsnew/3.10.html#better-error-messages", + 3, + "documentation", + -1, + 3, + ".\n ", + -1, + 1, + "p", + 3, + "\n Think back to writing your first ", + 1, + "a", + 2, + "href", + "https://www.scriptol.com/programming/hello-world.php", + 3, + "Hello World", + -1, + 3, + " program in Python:\n ", + -1, + 1, + "p", + 3, + "\n Maybe you created a file, added the famous call to ", + 1, + "code", + 3, + "print()", + -1, + 3, + ", and saved it as ", + 1, + "code", + 3, + "hello.py", + -1, + 3, + ". You then ran the program, eager to call yourself a proper Pythonista. However, something went wrong:\n ", + -1, + 1, + "p", + 3, + "\n There was a ", + 1, + "code", + 3, + "SyntaxError", + -1, + 3, + " in the code. ", + 1, + "code", + 3, + "EOL", + -1, + 3, + ", what does that even mean? You went back to your code, and after a bit of staring and searching, you realized that there was a missing quotation mark at the end of your string.\n ", + -1, + 1, + "p", + 3, + "\n One of the more impactful improvements in Python 3.10 is better and more precise error messages for many common issues. If you run your buggy Hello World in Python 3.10, you’ll get a bit more help than in earlier versions of Python:\n ", + -1, + 1, + "p", + 3, + "\n The error message is still a bit technical, but gone is the mysterious ", + 1, + "code", + 3, + "EOL", + -1, + 3, + ". Instead, the message tells you that you need to terminate your string! There are similar improvements to many different error messages, as you’ll see below.\n ", + -1, + 1, + "p", + 3, + "\n A ", + 1, + "a", + 2, + "href", + "https://realpython.com/invalid-syntax-python/", + 1, + "code", + 3, + "SyntaxError", + -2, + 3, + " is an error raised when your code is parsed, before it even starts to execute. Syntax errors can be tricky to debug because the interpreter provides imprecise or sometimes even misleading error messages. The following code is missing a curly brace to terminate the dictionary:\n ", + -1, + 1, + "p", + 3, + "\n The missing closing curly brace that should have been on line 7 is an error. If you run this code with Python 3.9 or earlier, you’ll see the following error message:\n ", + -1, + 1, + "p", + 3, + "\n The error message highlights line 8, but there are no syntactical problems in line 8! If you’ve experienced your share of syntax errors in Python, you might already know that the trick is to look at the lines ", + 1, + "em", + 3, + "before", + -1, + 3, + " the one Python complains about. In this case, you’re looking for the missing closing brace on line 7.\n ", + -1, + 1, + "p", + 3, + "\n In Python 3.10, the same code shows a much more helpful and precise error message:\n ", + -1, + 1, + "p", + 3, + "\n This points you straight to the offending dictionary and allows you to fix the issue in no time.\n ", + -1, + 1, + "p", + 3, + "\n There are a few other ways to mess up dictionary syntax. A typical one is forgetting a comma after one of the items:\n ", + -1, + 1, + "p", + 3, + "\n In this code, a comma is missing at the end of line 4. Python 3.10 gives you a clear suggestion on how to fix your code:\n ", + -1, + 1, + "p", + 3, + "\n You can add the missing comma and have your code back up and running in no time.\n ", + -1, + 1, + "p", + 3, + "\n Another common mistake is using the ", + 1, + "a", + 2, + "href", + "https://realpython.com/python-assignment-operator/", + 3, + "assignment operator", + -1, + 3, + " (", + 1, + "code", + 3, + "=", + -1, + 3, + ") instead of the equality comparison operator (", + 1, + "code", + 3, + "==", + -1, + 3, + ") when you’re comparing values. Previously, this would just cause another ", + 1, + "code", + 3, + "invalid syntax", + -1, + 3, + " message. In the newest version of Python, you get some more advice:\n ", + -1, + 1, + "p", + 3, + "\n The parser suggests that you maybe meant to use a ", + 1, + "a", + 2, + "href", + "https://realpython.com/python-operators-expressions/#comparison-operators", + 3, + "comparison operator", + -1, + 3, + " or an ", + 1, + "a", + 2, + "href", + "https://realpython.com/python-walrus-operator/", + 3, + "assignment expression operator", + -1, + 3, + " instead.\n ", + -1, + 1, + "p", + 3, + "\n Take note of another nifty improvement in Python 3.10 error messages. The last two examples show how carets (", + 1, + "code", + 3, + "^^^", + -1, + 3, + ") highlight the whole offending expression. Previously, a single caret symbol (", + 1, + "code", + 3, + "^", + -1, + 3, + ") indicated just an approximate location.\n ", + -1, + 1, + "p", + 3, + "\n The final error message improvement that you’ll play with for now is that attribute and name errors can now offer suggestions if you misspell an attribute or a name:\n ", + -1, + 1, + "p", + 3, + "\n Note that the suggestions work for both built-in names and names that you define yourself, although they may ", + 1, + "a", + 2, + "href", + "https://docs.python.org/3.10/whatsnew/3.10.html#attributeerrors", + 3, + "not be available", + -1, + 3, + " in all environments. If you like these kinds of suggestions, check out ", + 1, + "a", + 2, + "href", + "https://github.com/SylvainDe/DidYouMean-Python", + 3, + "BetterErrorMessages", + -1, + 3, + ", which offers similar suggestions in even more contexts.\n ", + -1, + 1, + "p", + 3, + "\n The improvements you’ve seen in this section are just some of the ", + 1, + "a", + 2, + "href", + "https://docs.python.org/3.10/whatsnew/3.10.html#better-error-messages", + 3, + "many error messages", + -1, + 3, + " that have gotten a face-lift. The new Python will be even more user-friendly than before, and hopefully, the new error messages will save you both time and frustration going forward.\n ", + -2, + 1, + "section", + 1, + "h2", + 2, + "id", + "structural-pattern-matching", + 3, + "\n Structural Pattern Matching", + 1, + "a", + 2, + "href", + "#structural-pattern-matching", + 2, + "title", + "Permanent link", + -2, + 1, + "p", + 3, + "\n The biggest new feature in Python 3.10, probably both in terms of ", + 1, + "a", + 2, + "href", + "https://lwn.net/Articles/845480/", + 3, + "controversy", + -1, + 3, + " and ", + 1, + "a", + 2, + "href", + "https://en.wikipedia.org/wiki/Pattern_matching", + 3, + "potential impact", + -1, + 3, + ", is ", + 1, + "strong", + 3, + "structural pattern matching", + -1, + 3, + ". Its introduction has sometimes been referred to as ", + 1, + "code", + 3, + "switch ... case", + -1, + 3, + " coming to Python, but you’ll see that structural pattern matching is much more powerful than that.\n ", + -1, + 1, + "p", + 3, + "\n You’ll see three different examples that together highlight why this feature is called structural pattern matching and show you how you can use this new feature:\n ", + -1, + 1, + "ol", + 1, + "li", + 3, + "Detecting and deconstructing different ", + 1, + "strong", + 3, + "structures", + -1, + 3, + " in your data\n ", + -1, + 1, + "li", + 3, + "Using different kinds of ", + 1, + "strong", + 3, + "patterns", + -2, + 1, + "li", + 1, + "strong", + 3, + "Matching", + -1, + 3, + " literal patterns\n ", + -2, + 1, + "p", + 3, + "\n Structural pattern matching is a comprehensive addition to the Python language. To give you a taste of how you can take advantage of it in your own projects, the next three subsections will dive into some of the details. You’ll also see some links that can help you explore in even more depth if you want.\n ", + -1, + 1, + "section", + 1, + "h3", + 2, + "id", + "deconstructing-data-structures", + 3, + "\n Deconstructing Data Structures", + 1, + "a", + 2, + "href", + "#deconstructing-data-structures", + 2, + "title", + "Permanent link", + -2, + 1, + "p", + 3, + "\n At its core, structural pattern matching is about defining patterns to which your data structures can be matched. In this section, you’ll study a practical example where you’ll work with data that are structured differently, even though the meaning is the same. You’ll define several patterns, and depending on which pattern matches your data, you’ll process your data appropriately.\n ", + -1, + 1, + "p", + 3, + "\n This section will be a bit light on explanations of the possible patterns. Instead, it will try to give you an impression of the possibilities. The next section will step back and explain the patterns in more detail.\n ", + -1, + 1, + "p", + 3, + "\n Time to match your first pattern! The following example uses a ", + 1, + "code", + 3, + "match ... case", + -1, + 3, + " block to find the first name of a user by extracting it from a ", + 1, + "code", + 3, + "user", + -1, + 3, + " data structure:\n ", + -1, + 1, + "p", + 3, + "\n You can see structural pattern matching at work in the highlighted lines. ", + 1, + "code", + 3, + "user", + -1, + 3, + " is a small dictionary with user information. The ", + 1, + "code", + 3, + "case", + -1, + 3, + " line specifies a pattern that ", + 1, + "code", + 3, + "user", + -1, + 3, + " is matched against. In this case, you’re looking for a dictionary with a ", + 1, + "code", + 3, + "\"name\"", + -1, + 3, + " key whose value is a new dictionary. This nested dictionary has a key called ", + 1, + "code", + 3, + "\"first\"", + -1, + 3, + ". The corresponding value is bound to the variable ", + 1, + "code", + 3, + "first_name", + -1, + 3, + ".\n ", + -1, + 1, + "p", + 3, + "\n For a practical example, say that you’re processing user data where the underlying data model changes over time. Therefore, you need to be able to process different versions of the same data.\n ", + -1, + 1, + "p", + 3, + "\n In the next example, you’ll use data from ", + 1, + "a", + 2, + "href", + "https://randomuser.me/", + 3, + "randomuser.me", + -1, + 3, + ". This is a great API for generating random user data that you can use during testing and development. The API is also an example of an API that has changed over time. You can still access the ", + 1, + "a", + 2, + "href", + "https://randomuser.me/documentation#previous", + 3, + "old versions", + -1, + 3, + " of the API.\n ", + -1, + 1, + "p", + 3, + "\n You may expand the collapsed section below to see how you can use ", + 1, + "a", + 2, + "href", + "https://realpython.com/python-requests/", + 1, + "code", + 3, + "requests", + -2, + 3, + " to obtain different versions of the user data using the API:\n ", + -1, + 1, + "div", + 2, + "id", + "collapse_cardbb4757", + 2, + "data-parent", + "#collapse_cardbb4757", + 1, + "p", + 3, + "\n You can get a random user from the API using ", + 1, + "code", + 3, + "requests", + -1, + 3, + " as follows:\n ", + -1, + 1, + "p", + 1, + "code", + 3, + "get_user()", + -1, + 3, + " gets one random user in ", + 1, + "a", + 2, + "href", + "https://realpython.com/python-json/", + 3, + "JSON", + -1, + 3, + " format. Note the ", + 1, + "code", + 3, + "version", + -1, + 3, + " parameter. The structure of the returned data has changed quite a bit between earlier versions like ", + 1, + "code", + 3, + "\"1.1\"", + -1, + 3, + " and the current version ", + 1, + "code", + 3, + "\"1.3\"", + -1, + 3, + ", but in each case, the actual user data are contained in a list inside the ", + 1, + "code", + 3, + "\"results\"", + -1, + 3, + " array. The function returns the first—and only—user in this list.\n ", + -1, + 1, + "p", + 3, + "\n At the time of writing, the latest version of the API is 1.3 and the data has the following structure:\n ", + -1, + 1, + "p", + 3, + "\n One of the members that changed between different versions is ", + 1, + "code", + 3, + "\"dob\"", + -1, + 3, + ", the date of birth. Note that in version 1.3, this is a JSON object with two members, ", + 1, + "code", + 3, + "\"date\"", + -1, + 3, + " and ", + 1, + "code", + 3, + "\"age\"", + -1, + 3, + ".\n ", + -1, + 1, + "p", + 3, + "\n Compare the result above with a version 1.1 random user:\n ", + -1, + 1, + "p", + 3, + "\n Observe that in this older format, the value of the ", + 1, + "code", + 3, + "\"dob\"", + -1, + 3, + " member is a plain string.\n ", + -2, + 1, + "p", + 3, + "\n In this example, you’ll work with the information about the date of birth (", + 1, + "code", + 3, + "dob", + -1, + 3, + ") for each user. The structure of these data has changed between different versions of the Random User API:\n ", + -1, + 1, + "p", + 3, + "\n Note that in version 1.1, the date of birth is represented as a simple string, while in version 1.3, it’s a JSON object with two members: ", + 1, + "code", + 3, + "\"date\"", + -1, + 3, + " and ", + 1, + "code", + 3, + "\"age\"", + -1, + 3, + ". Say that you want to find the age of a user. Depending on the structure of your data, you’d either need to calculate the age based on the date of birth or look up the age if it’s already available.\n ", + -1, + 1, + "p", + 3, + "\n Traditionally, you would detect the structure of the data with an ", + 1, + "code", + 3, + "if", + -1, + 3, + " test, maybe based on the type of the ", + 1, + "code", + 3, + "\"dob\"", + -1, + 3, + " field. You can approach this differently in Python 3.10. Now, you can use structural pattern matching instead:\n ", + -1, + 1, + "p", + 3, + "\n The ", + 1, + "code", + 3, + "match ... case", + -1, + 3, + " construct is new in Python 3.10 and is how you perform structural pattern matching. You start with a ", + 1, + "code", + 3, + "match", + -1, + 3, + " statement that specifies what you want to match. In this example, that’s the ", + 1, + "code", + 3, + "user", + -1, + 3, + " data structure.\n ", + -1, + 1, + "p", + 3, + "\n One or several ", + 1, + "code", + 3, + "case", + -1, + 3, + " statements follow ", + 1, + "code", + 3, + "match", + -1, + 3, + ". Each ", + 1, + "code", + 3, + "case", + -1, + 3, + " describes one pattern, and the indented block beneath it says what should happen if there’s a match. In this example:\n ", + -1, + 1, + "ul", + 1, + "li", + 1, + "p", + 1, + "strong", + 3, + "Line 8", + -1, + 3, + " matches a dictionary with a ", + 1, + "code", + 3, + "\"dob\"", + -1, + 3, + " key whose value is another dictionary with an integer (", + 1, + "code", + 3, + "int", + -1, + 3, + ") item named ", + 1, + "code", + 3, + "\"age\"", + -1, + 3, + ". The name ", + 1, + "code", + 3, + "age", + -1, + 3, + " captures its value.\n ", + -2, + 1, + "li", + 1, + "p", + 1, + "strong", + 3, + "Line 10", + -1, + 3, + " matches any dictionary with a ", + 1, + "code", + 3, + "\"dob\"", + -1, + 3, + " key. The name ", + 1, + "code", + 3, + "dob", + -1, + 3, + " captures its value.\n ", + -3, + 1, + "p", + 3, + "\n One important feature of pattern matching is that at most one pattern will be matched. Since the pattern on line 10 matches any dictionary with ", + 1, + "code", + 3, + "\"dob\"", + -1, + 3, + ", it’s important that the more specific pattern on line 8 comes first.\n ", + -1, + 1, + "p", + 3, + "\n Before looking closer at the details of the patterns and how they work, try calling ", + 1, + "code", + 3, + "get_age()", + -1, + 3, + " with different data structures to see the result:\n ", + -1, + 1, + "p", + 3, + "\n Your code can calculate the age correctly for both versions of the user data, which have different dates of birth.\n ", + -1, + 1, + "p", + 3, + "\n Look closer at those patterns. The first pattern, ", + 1, + "code", + 3, + "{\"dob\": {\"age\": int(age)}}", + -1, + 3, + ", matches version 1.3 of the user data:\n ", + -1, + 1, + "p", + 3, + "\n The first pattern is a nested pattern. The outer curly braces say that a dictionary with the key ", + 1, + "code", + 3, + "\"dob\"", + -1, + 3, + " is required. The corresponding value should be a dictionary. This nested dictionary must match the subpattern ", + 1, + "code", + 3, + "{\"age\": int(age)}", + -1, + 3, + ". In other words, it needs to have an ", + 1, + "code", + 3, + "\"age\"", + -1, + 3, + " key with an integer value. That value is bound to the name ", + 1, + "code", + 3, + "age", + -1, + 3, + ".\n ", + -1, + 1, + "p", + 3, + "\n The second pattern, ", + 1, + "code", + 3, + "{\"dob\": dob}", + -1, + 3, + ", matches the older version 1.1 of the user data:\n ", + -1, + 1, + "p", + 3, + "\n This second pattern is a simpler pattern than the first one. Again, the curly braces indicate that it will match a dictionary. However, any dictionary with a ", + 1, + "code", + 3, + "\"dob\"", + -1, + 3, + " key is matched because there are no other restrictions specified. The value of that key is bound to the name ", + 1, + "code", + 3, + "dob", + -1, + 3, + ".\n ", + -1, + 1, + "p", + 3, + "\n The main takeaway is that you can describe the structure of your data using mostly familiar notation. One striking change, though, is that you can use names like ", + 1, + "code", + 3, + "dob", + -1, + 3, + " and ", + 1, + "code", + 3, + "age", + -1, + 3, + ", which aren’t yet defined. Instead, values from your data are ", + 1, + "strong", + 3, + "bound", + -1, + 3, + " to these names when a pattern matches.\n ", + -1, + 1, + "p", + 3, + "\n You’ve explored some of the power of structural pattern matching in this example. In the next section, you’ll dive a bit more into the details.\n ", + -2, + 1, + "section", + 1, + "h3", + 2, + "id", + "using-different-kinds-of-patterns", + 3, + "\n Using Different Kinds of Patterns", + 1, + "a", + 2, + "href", + "#using-different-kinds-of-patterns", + 2, + "title", + "Permanent link", + -2, + 1, + "p", + 3, + "\n You’ve seen an example of how you can use patterns to effectively unravel complicated data structures. Now, you’ll take a step back and look at the building blocks that make up this new feature. Many things come together to make it work. In fact, there are three ", + 1, + "a", + 2, + "href", + "https://www.python.org/dev/peps/pep-0001/#what-is-a-pep", + 3, + "Python Enhancement Proposals", + -1, + 3, + " (PEPs) that describe structural pattern matching:\n ", + -1, + 1, + "ol", + 1, + "li", + 1, + "strong", + 3, + "PEP 634:", + -1, + 1, + "a", + 2, + "href", + "https://www.python.org/dev/peps/pep-0634/", + 3, + "Specification", + -2, + 1, + "li", + 1, + "strong", + 3, + "PEP 635:", + -1, + 1, + "a", + 2, + "href", + "https://www.python.org/dev/peps/pep-0635/", + 3, + "Motivation and Rationale", + -2, + 1, + "li", + 1, + "strong", + 3, + "PEP 636:", + -1, + 1, + "a", + 2, + "href", + "https://www.python.org/dev/peps/pep-0636/", + 3, + "Tutorial", + -3, + 1, + "p", + 3, + "\n These documents give you a lot of background and detail if you’re interested in a deeper dive than what follows.\n ", + -1, + 1, + "p", + 3, + "\n Patterns are at the center of structural pattern matching. In this section, you’ll learn about some of the different kinds of patterns that exist:\n ", + -1, + 1, + "ul", + 1, + "li", + 1, + "strong", + 3, + "Mapping patterns", + -1, + 3, + " match mapping structures like dictionaries.\n ", + -1, + 1, + "li", + 1, + "strong", + 3, + "Sequence patterns", + -1, + 3, + " match sequence structures like tuples and lists.\n ", + -1, + 1, + "li", + 1, + "strong", + 3, + "Capture patterns", + -1, + 3, + " bind values to names.\n ", + -1, + 1, + "li", + 1, + "strong", + 3, + "AS patterns", + -1, + 3, + " bind the value of subpatterns to names.\n ", + -1, + 1, + "li", + 1, + "strong", + 3, + "OR patterns", + -1, + 3, + " match one of several different subpatterns.\n ", + -1, + 1, + "li", + 1, + "strong", + 3, + "Wildcard patterns", + -1, + 3, + " match anything.\n ", + -1, + 1, + "li", + 1, + "strong", + 3, + "Class patterns", + -1, + 3, + " match class structures.\n ", + -1, + 1, + "li", + 1, + "strong", + 3, + "Value patterns", + -1, + 3, + " match values stored in attributes.\n ", + -1, + 1, + "li", + 1, + "strong", + 3, + "Literal patterns", + -1, + 3, + " match literal values.\n ", + -2, + 1, + "p", + 3, + "\n You already used several of them in the example in the previous section. In particular, you used ", + 1, + "strong", + 3, + "mapping patterns", + -1, + 3, + " to unravel data stored in dictionaries. In this section, you’ll learn more about how some of these work. All the details are available in the PEPs mentioned above.\n ", + -1, + 1, + "p", + 3, + "\n A ", + 1, + "strong", + 3, + "capture pattern", + -1, + 3, + " is used to capture a match to a pattern and bind it to a name. Consider the following ", + 1, + "a", + 2, + "href", + "https://realpython.com/python-recursion/", + 3, + "recursive", + -1, + 3, + " function that sums a list of numbers:\n ", + -1, + 1, + "p", + 3, + "\n The first ", + 1, + "code", + 3, + "case", + -1, + 3, + " on line 3 matches the empty list and returns ", + 1, + "code", + 3, + "0", + -1, + 3, + " as its sum. The second ", + 1, + "code", + 3, + "case", + -1, + 3, + " on line 5 uses a ", + 1, + "strong", + 3, + "sequence pattern", + -1, + 3, + " with two capture patterns to match lists with one or more elements. The first element in the list is captured and bound to the name ", + 1, + "code", + 3, + "first", + -1, + 3, + ". The second capture pattern, ", + 1, + "code", + 3, + "*rest", + -1, + 3, + ", uses ", + 1, + "a", + 2, + "href", + "https://realpython.com/python-kwargs-and-args/#unpacking-with-the-asterisk-operators", + 3, + "unpacking syntax", + -1, + 3, + " to match any number of elements. ", + 1, + "code", + 3, + "rest", + -1, + 3, + " will bind to a list containing all elements of ", + 1, + "code", + 3, + "numbers", + -1, + 3, + " except the first one.\n ", + -1, + 1, + "p", + 1, + "code", + 3, + "sum_list()", + -1, + 3, + " calculates the sum of a list of numbers by recursively adding the first number in the list and the sum of the rest of the numbers. You can use it as follows:\n ", + -1, + 1, + "p", + 3, + "\n The sum of 4 + 5 + 9 + 4 is correctly calculated to be 22. As an exercise for yourself, you can try to trace the recursive calls to ", + 1, + "code", + 3, + "sum_list()", + -1, + 3, + " to make sure you understand how the code sums the whole list.\n ", + -1, + 1, + "p", + 1, + "code", + 3, + "sum_list()", + -1, + 3, + " handles summing up a list of numbers. Observe what happens if you try to sum anything that isn’t a list:\n ", + -1, + 1, + "p", + 3, + "\n Passing a string or a number to ", + 1, + "code", + 3, + "sum_list()", + -1, + 3, + " returns ", + 1, + "code", + 3, + "None", + -1, + 3, + ". This occurs because none of the patterns match, and the execution continues after the ", + 1, + "code", + 3, + "match", + -1, + 3, + " block. That happens to be the end of the function, so ", + 1, + "code", + 3, + "sum_list()", + -1, + 3, + " implicitly ", + 1, + "a", + 2, + "href", + "https://realpython.com/python-return-statement/#implicit-return-statements", + 3, + "returns ", + 1, + "code", + 3, + "None", + -2, + 3, + ".\n ", + -1, + 1, + "p", + 3, + "\n Often, though, you want to be alerted about failed matches. You can add a catchall pattern as the final case that handles this by raising an error, for example. You can use the underscore (", + 1, + "code", + 3, + "_", + -1, + 3, + ") as a ", + 1, + "strong", + 3, + "wildcard pattern", + -1, + 3, + " that matches anything without binding it to a name. You can add some error handling to ", + 1, + "code", + 3, + "sum_list()", + -1, + 3, + " as follows:\n ", + -1, + 1, + "p", + 3, + "\n The final ", + 1, + "code", + 3, + "case", + -1, + 3, + " will match anything that doesn’t match the first two patterns. This will raise a descriptive error, for instance, if you try to calculate ", + 1, + "code", + 3, + "sum_list(4594)", + -1, + 3, + ". This is useful when you need to alert your users that some input was not matched as expected.\n ", + -1, + 1, + "p", + 3, + "\n Your patterns are still not foolproof, though. Consider what happens if you try to sum a list of strings:\n ", + -1, + 1, + "p", + 3, + "\n The base case returns ", + 1, + "code", + 3, + "0", + -1, + 3, + ", so therefore the summing only works for types that you can add with numbers. Python doesn’t know how to add numbers and text strings together. You can restrict your pattern to only match integers using a ", + 1, + "strong", + 3, + "class pattern", + -1, + 3, + ":\n ", + -1, + 1, + "p", + 3, + "\n Adding ", + 1, + "code", + 3, + "int()", + -1, + 3, + " around ", + 1, + "code", + 3, + "first", + -1, + 3, + " makes sure that the pattern only matches if the value is an integer. This might be too restrictive, though. Your function should be able to sum both ", + 1, + "a", + 2, + "href", + "https://realpython.com/python-numbers/#integers", + 3, + "integers", + -1, + 3, + " and ", + 1, + "a", + 2, + "href", + "https://realpython.com/python-numbers/#floating-point-numbers", + 3, + "floating-point numbers", + -1, + 3, + ", so how can you allow this in your pattern?\n ", + -1, + 1, + "p", + 3, + "\n To check whether at least one out of several subpatterns match, you can use an ", + 1, + "strong", + 3, + "OR pattern", + -1, + 3, + ". OR patterns consist of two or more subpatterns, and the pattern matches if at least one of the subpatterns does. You can use this to match when the first element is either of type ", + 1, + "code", + 3, + "int", + -1, + 3, + " or type ", + 1, + "code", + 3, + "float", + -1, + 3, + ":\n ", + -1, + 1, + "p", + 3, + "\n You use the pipe symbol (", + 1, + "code", + 3, + "|", + -1, + 3, + ") to separate the subpatterns in an OR pattern. Your function now allows summing a list of floating-point numbers:\n ", + -1, + 1, + "p", + 3, + "\n There’s a lot of power and flexibility within structural pattern matching, even more than what you’ve seen so far. Some things that aren’t covered in this overview are:\n ", + -1, + 1, + "ul", + 1, + "li", + 3, + "Using ", + 1, + "a", + 2, + "href", + "https://www.python.org/dev/peps/pep-0635/#guards", + 3, + "guards", + -1, + 3, + " to restrict patterns\n ", + -1, + 1, + "li", + 3, + "Using ", + 1, + "a", + 2, + "href", + "https://www.python.org/dev/peps/pep-0635/#as-patterns", + 3, + "AS patterns", + -1, + 3, + " to capture the value of subpatterns\n ", + -1, + 1, + "li", + 3, + "Using ", + 1, + "a", + 2, + "href", + "https://www.python.org/dev/peps/pep-0636/#matching-positional-attributes", + 3, + "class patterns", + -1, + 3, + " to match custom ", + 1, + "a", + 2, + "href", + "https://docs.python.org/3/library/enum.html", + 3, + "enums", + -1, + 3, + " and ", + 1, + "a", + 2, + "href", + "https://realpython.com/python-data-classes/", + 3, + "data classes", + -3, + 1, + "p", + 3, + "\n If you’re interested, have a look in the documentation to learn more about these features as well. In the next section, you’ll learn about literal patterns and value patterns.\n ", + -2, + 1, + "section", + 1, + "h3", + 2, + "id", + "matching-literal-patterns", + 3, + "\n Matching Literal Patterns", + 1, + "a", + 2, + "href", + "#matching-literal-patterns", + 2, + "title", + "Permanent link", + -2, + 1, + "p", + 3, + "\n A ", + 1, + "strong", + 3, + "literal pattern", + -1, + 3, + " is a pattern that matches a literal object like an explicit string or number. In a sense, this is the most basic kind of pattern and allows you to emulate ", + 1, + "code", + 3, + "switch ... case", + -1, + 3, + " statements seen in other languages. The following example matches a specific name:\n ", + -1, + 1, + "p", + 3, + "\n The first ", + 1, + "code", + 3, + "case", + -1, + 3, + " matches the literal string ", + 1, + "code", + 3, + "\"Guido\"", + -1, + 3, + ". In this case, you use ", + 1, + "code", + 3, + "_", + -1, + 3, + " as a wildcard to print a generic greeting whenever ", + 1, + "code", + 3, + "name", + -1, + 3, + " is not ", + 1, + "code", + 3, + "\"Guido\"", + -1, + 3, + ". Such literal patterns can sometimes take the place of ", + 1, + "code", + 3, + "if ... elif ... else", + -1, + 3, + " constructs and can play the same role that ", + 1, + "code", + 3, + "switch ... case", + -1, + 3, + " does in some other languages.\n ", + -1, + 1, + "p", + 3, + "\n One limitation with structural pattern matching is that you can’t directly match values stored in variables. Say that you’ve defined ", + 1, + "code", + 3, + "bdfl = \"Guido\"", + -1, + 3, + ". A pattern like ", + 1, + "code", + 3, + "case bdfl:", + -1, + 3, + " will not match ", + 1, + "code", + 3, + "\"Guido\"", + -1, + 3, + ". Instead, this will be interpreted as a capture pattern that matches anything and binds that value to ", + 1, + "code", + 3, + "bdfl", + -1, + 3, + ", effectively overwriting the old value.\n ", + -1, + 1, + "p", + 3, + "\n You can, however, use a ", + 1, + "strong", + 3, + "value pattern", + -1, + 3, + " to match stored values. A value pattern looks a bit like a capture pattern but uses a previously defined dotted name that holds the value that will be matched against.\n ", + -1, + 1, + "p", + 3, + "\n You can, for example, use an ", + 1, + "a", + 2, + "href", + "https://docs.python.org/3/library/enum.html", + 3, + "enumeration", + -1, + 3, + " to create such dotted names:\n ", + -1, + 1, + "p", + 3, + "\n The first case now uses a value pattern to match ", + 1, + "code", + 3, + "Pythonista.BDFL", + -1, + 3, + ", which is ", + 1, + "code", + 3, + "\"Guido\"", + -1, + 3, + ". Note that you can use any dotted name in a value pattern. You could, for example, have used a regular class or a module instead of the enumeration.\n ", + -1, + 1, + "p", + 3, + "\n To see a bigger example of how to use literal patterns, consider the game of ", + 1, + "a", + 2, + "href", + "https://en.wikipedia.org/wiki/Fizz_buzz", + 3, + "FizzBuzz", + -1, + 3, + ". This is a counting game where you should replace some numbers with words according to the following rules:\n ", + -1, + 1, + "ul", + 1, + "li", + 3, + "You replace numbers divisible by ", + 1, + "strong", + 3, + "3", + -1, + 3, + " with ", + 1, + "strong", + 3, + "fizz", + -1, + 3, + ".\n ", + -1, + 1, + "li", + 3, + "You replace numbers divisible by ", + 1, + "strong", + 3, + "5", + -1, + 3, + " with ", + 1, + "strong", + 3, + "buzz", + -1, + 3, + ".\n ", + -1, + 1, + "li", + 3, + "You replace numbers divisible by both ", + 1, + "strong", + 3, + "3", + -1, + 3, + " and ", + 1, + "strong", + 3, + "5", + -1, + 3, + " with ", + 1, + "strong", + 3, + "fizzbuzz", + -1, + 3, + ".\n ", + -2, + 1, + "p", + 3, + "\n FizzBuzz is sometimes used to introduce conditionals in programming education and as a screening problem in interviews. Even though a solution is quite straightforward, ", + 1, + "a", + 2, + "href", + "https://twitter.com/joelgrus", + 3, + "Joel Grus", + -1, + 3, + " has written a full ", + 1, + "a", + 2, + "href", + "https://fizzbuzzbook.com/", + 3, + "book", + -1, + 3, + " about different ways to program the game.\n ", + -1, + 1, + "p", + 3, + "\n A typical solution in Python will use ", + 1, + "code", + 3, + "if ... elif ... else", + -1, + 3, + " as follows:\n ", + -1, + 1, + "p", + 3, + "\n The ", + 1, + "a", + 2, + "href", + "https://realpython.com/python-modulo-operator/", + 1, + "code", + 3, + "%", + -1, + 3, + " operator", + -1, + 3, + " calculates the modulus, which you can use to ", + 1, + "a", + 2, + "href", + "https://realpython.com/python-modulo-operator/#python-modulo-operator-in-practice", + 3, + "test divisibility", + -1, + 3, + ". Namely, if ", + 1, + "em", + 3, + "a", + -1, + 3, + " modulus ", + 1, + "em", + 3, + "b", + -1, + 3, + " is 0 for two numbers ", + 1, + "em", + 3, + "a", + -1, + 3, + " and ", + 1, + "em", + 3, + "b", + -1, + 3, + ", then ", + 1, + "em", + 3, + "a", + -1, + 3, + " is divisible by ", + 1, + "em", + 3, + "b", + -1, + 3, + ".\n ", + -1, + 1, + "p", + 3, + "\n In ", + 1, + "code", + 3, + "fizzbuzz()", + -1, + 3, + ", you calculate ", + 1, + "code", + 3, + "number % 3", + -1, + 3, + " and ", + 1, + "code", + 3, + "number % 5", + -1, + 3, + ", which you then use to test for divisibility with 3 and 5. Note that you must do the test for divisibility with both 3 and 5 first. If not, numbers that are divisible by both 3 and 5 will be covered by either the ", + 1, + "code", + 3, + "\"fizz\"", + -1, + 3, + " or the ", + 1, + "code", + 3, + "\"buzz\"", + -1, + 3, + " cases instead.\n ", + -1, + 1, + "p", + 3, + "\n You can check that your implementation gives the expected result:\n ", + -1, + 1, + "p", + 3, + "\n You can confirm for yourself that 3 is divisible by 3, 65 is divisible by 5, and 15 is divisible by both 3 and 5, while 14 and 92 aren’t divisible by either 3 or 5.\n ", + -1, + 1, + "p", + 3, + "\n An ", + 1, + "code", + 3, + "if ... elif ... else", + -1, + 3, + " structure where you’re comparing one or a few variables several times over is quite straightforward to rewrite using pattern matching instead. For example, you can do the following:\n ", + -1, + 1, + "p", + 3, + "\n You match on both ", + 1, + "code", + 3, + "mod_3", + -1, + 3, + " and ", + 1, + "code", + 3, + "mod_5", + -1, + 3, + ". Each ", + 1, + "code", + 3, + "case", + -1, + 3, + " pattern then matches either the literal number ", + 1, + "code", + 3, + "0", + -1, + 3, + " or the wildcard ", + 1, + "code", + 3, + "_", + -1, + 3, + " on the corresponding values.\n ", + -1, + 1, + "p", + 3, + "\n Compare and contrast this version with the previous one. Note how the pattern ", + 1, + "code", + 3, + "(0, 0)", + -1, + 3, + " corresponds to the test ", + 1, + "code", + 3, + "mod_3 == 0 and mod_5 == 0", + -1, + 3, + ", while ", + 1, + "code", + 3, + "(0, _)", + -1, + 3, + " corresponds to ", + 1, + "code", + 3, + "mod_3 == 0", + -1, + 3, + ".\n ", + -1, + 1, + "p", + 3, + "\n As you saw earlier, you can use an OR pattern to match on several different patterns. For example, since ", + 1, + "code", + 3, + "mod_3", + -1, + 3, + " can only take the values ", + 1, + "code", + 3, + "0", + -1, + 3, + ", ", + 1, + "code", + 3, + "1", + -1, + 3, + ", and ", + 1, + "code", + 3, + "2", + -1, + 3, + ", you can replace ", + 1, + "code", + 3, + "case (_, 0)", + -1, + 3, + " with ", + 1, + "code", + 3, + "case (1, 0) | (2, 0)", + -1, + 3, + ". Remember that ", + 1, + "code", + 3, + "(0, 0)", + -1, + 3, + " has already been covered.\n ", + -1, + 1, + "p", + 3, + "\n The Python core developers have ", + 1, + "a", + 2, + "href", + "https://www.python.org/dev/peps/pep-3103/", + 3, + "consciously chosen", + -1, + 3, + " not to include ", + 1, + "code", + 3, + "switch ... case", + -1, + 3, + " statements in the language earlier. However, there are some third-party packages that do, like ", + 1, + "a", + 2, + "href", + "https://pypi.org/project/switchlang/", + 3, + "switchlang", + -1, + 3, + ", which adds a ", + 1, + "code", + 3, + "switch", + -1, + 3, + " command that also works on earlier versions of Python.\n ", + -3, + 1, + "section", + 1, + "h2", + 2, + "id", + "type-unions-aliases-and-guards", + 3, + "\n Type Unions, Aliases, and Guards", + 1, + "a", + 2, + "href", + "#type-unions-aliases-and-guards", + 2, + "title", + "Permanent link", + -2, + 1, + "p", + 3, + "\n Reliably, each new Python release brings some improvements to the ", + 1, + "a", + 2, + "href", + "https://realpython.com/python-type-checking/", + 3, + "static typing", + -1, + 3, + " system. Python 3.10 is no exception. In fact, four different PEPs about typing accompany this new release:\n ", + -1, + 1, + "ol", + 1, + "li", + 1, + "strong", + 3, + "PEP 604:", + -1, + 1, + "a", + 2, + "href", + "https://www.python.org/dev/peps/pep-0604", + 3, + "Allow writing union types as ", + 1, + "code", + 3, + "X | Y", + -3, + 1, + "li", + 1, + "strong", + 3, + "PEP 613:", + -1, + 1, + "a", + 2, + "href", + "https://www.python.org/dev/peps/pep-0613", + 3, + "Explicit Type Aliases", + -2, + 1, + "li", + 1, + "strong", + 3, + "PEP 647:", + -1, + 1, + "a", + 2, + "href", + "https://www.python.org/dev/peps/pep-0647/", + 3, + "User-Defined Type Guards", + -2, + 1, + "li", + 1, + "strong", + 3, + "PEP 612:", + -1, + 1, + "a", + 2, + "href", + "https://www.python.org/dev/peps/pep-0612/", + 3, + "Parameter Specification Variables", + -3, + 1, + "p", + 3, + "\n PEP 604 will probably be the most widely used of these changes going forward, but you’ll get a brief overview of each of the features in this section.\n ", + -1, + 1, + "p", + 3, + "\n You can use ", + 1, + "strong", + 3, + "union types", + -1, + 3, + " to declare that a variable can have one of several different types. For example, you’ve been able to type hint a function calculating the mean of a list of numbers, floats, or integers as follows:\n ", + -1, + 1, + "p", + 3, + "\n The annotation ", + 1, + "code", + 3, + "List[Union[float, int]]", + -1, + 3, + " means that ", + 1, + "code", + 3, + "numbers", + -1, + 3, + " should be a list where each element is either a floating-point number or an integer. This works well, but the notation is a bit verbose. Also, you need to import both ", + 1, + "code", + 3, + "List", + -1, + 3, + " and ", + 1, + "code", + 3, + "Union", + -1, + 3, + " from ", + 1, + "code", + 3, + "typing", + -1, + 3, + ".\n ", + -1, + 1, + "p", + 3, + "\n In Python 3.10, you can replace ", + 1, + "code", + 3, + "Union[float, int]", + -1, + 3, + " with the more succinct ", + 1, + "code", + 3, + "float | int", + -1, + 3, + ". Combine this with the ability to use ", + 1, + "a", + 2, + "href", + "https://realpython.com/python-list/", + 1, + "code", + 3, + "list", + -2, + 3, + " instead of ", + 1, + "code", + 3, + "typing.List", + -1, + 3, + " in type hints, which ", + 1, + "a", + 2, + "href", + "https://realpython.com/python39-new-features/#type-hint-lists-and-dictionaries-directly", + 3, + "Python 3.9", + -1, + 3, + " introduced. You can then simplify your code while keeping all the type information:\n ", + -1, + 1, + "p", + 3, + "\n The annotation of ", + 1, + "code", + 3, + "numbers", + -1, + 3, + " is easier to read now, and as an added bonus, you didn’t need to import anything from ", + 1, + "code", + 3, + "typing", + -1, + 3, + ".\n ", + -1, + 1, + "p", + 3, + "\n A special case of union types is when a variable can have either a specific type or be ", + 1, + "code", + 3, + "None", + -1, + 3, + ". You can annotate such ", + 1, + "strong", + 3, + "optional types", + -1, + 3, + " either as ", + 1, + "code", + 3, + "Union[None, T]", + -1, + 3, + " or, equivalently, ", + 1, + "a", + 2, + "href", + "https://realpython.com/python-type-checking/#the-optional-type", + 1, + "code", + 3, + "Optional[T]", + -2, + 3, + " for some type ", + 1, + "code", + 3, + "T", + -1, + 3, + ". There is no new, special syntax for optional types, but you can use the new union syntax to avoid importing ", + 1, + "code", + 3, + "typing.Optional", + -1, + 3, + ":\n ", + -1, + 1, + "p", + 3, + "\n In this example, ", + 1, + "code", + 3, + "address", + -1, + 3, + " is allowed to be either ", + 1, + "code", + 3, + "None", + -1, + 3, + " or a string.\n ", + -1, + 1, + "p", + 3, + "\n You can also use the new union syntax at runtime in ", + 1, + "code", + 3, + "isinstance()", + -1, + 3, + " or ", + 1, + "code", + 3, + "issubclass()", + -1, + 3, + " tests:\n ", + -1, + 1, + "p", + 3, + "\n Traditionally, you’ve used tuples to test for several types at once—for example, ", + 1, + "code", + 3, + "(str, int)", + -1, + 3, + " instead of ", + 1, + "code", + 3, + "str | int", + -1, + 3, + ". This old syntax will still work.\n ", + -1, + 1, + "p", + 1, + "strong", + 3, + "Type aliases", + -1, + 3, + " allow you to quickly ", + 1, + "a", + 2, + "href", + "https://realpython.com/python-type-checking/#type-aliases", + 3, + "define new aliases", + -1, + 3, + " that can stand in for more complicated type declarations. For example, say that you’re ", + 1, + "a", + 2, + "href", + "https://realpython.com/python-type-checking/#example-a-deck-of-cards", + 3, + "representing a playing card", + -1, + 3, + " using a tuple of suit and rank strings and a deck of cards by a list of such playing card tuples. A deck of cards is then type hinted as ", + 1, + "code", + 3, + "list[tuple[str, str]]", + -1, + 3, + ".\n ", + -1, + 1, + "p", + 3, + "\n To simplify type annotation, you define type aliases as follows:\n ", + -1, + 1, + "p", + 3, + "\n This usually works okay. However, it’s often not possible for the type checker to know whether such a statement is a type alias or just the definition of a regular global variable. To help the type checker—or really, help the type checker help you—you can now explicitly annotate type aliases:\n ", + -1, + 1, + "p", + 3, + "\n Adding the ", + 1, + "code", + 3, + "TypeAlias", + -1, + 3, + " annotation clarifies the intention, both to a type checker and to anyone reading your code.\n ", + -1, + 1, + "p", + 1, + "strong", + 3, + "Type guards", + -1, + 3, + " are used to narrow down union types. The following function takes in either a string or ", + 1, + "code", + 3, + "None", + -1, + 3, + " but always returns a tuple of strings representing a playing card:\n ", + -1, + 1, + "p", + 3, + "\n The highlighted line works as a type guard, and static type checkers are able to realize that ", + 1, + "code", + 3, + "suit", + -1, + 3, + " is necessarily a string when it’s returned.\n ", + -1, + 1, + "p", + 3, + "\n Currently, the type checkers can only use a ", + 1, + "a", + 2, + "href", + "https://www.python.org/dev/peps/pep-0647/#motivation", + 3, + "few different constructs", + -1, + 3, + " to narrow down union types in this way. With the new ", + 1, + "a", + 2, + "href", + "https://docs.python.org/3.10/library/typing.html#typing.TypeGuard", + 1, + "code", + 3, + "typing.TypeGuard", + -2, + 3, + ", you can annotate custom functions that can be used to narrow down union types:\n ", + -1, + 1, + "p", + 1, + "code", + 3, + "is_deck_of_cards()", + -1, + 3, + " should return ", + 1, + "code", + 3, + "True", + -1, + 3, + " or ", + 1, + "code", + 3, + "False", + -1, + 3, + " depending on whether ", + 1, + "code", + 3, + "obj", + -1, + 3, + " represents a ", + 1, + "code", + 3, + "Deck", + -1, + 3, + " object or not. You can then use your guard function, and the type checker will be able to narrow down the types correctly:\n ", + -1, + 1, + "p", + 3, + "\n Inside of the ", + 1, + "code", + 3, + "if", + -1, + 3, + " block, the type checker knows that ", + 1, + "code", + 3, + "card_or_deck", + -1, + 3, + " is, in fact, of the type ", + 1, + "code", + 3, + "Deck", + -1, + 3, + ". See ", + 1, + "a", + 2, + "href", + "https://www.python.org/dev/peps/pep-0647/", + 3, + "PEP 647", + -1, + 3, + " for more details.\n ", + -1, + 1, + "p", + 3, + "\n The final new typing feature is ", + 1, + "strong", + 3, + "Parameter Specification Variables", + -1, + 3, + ", which is related to ", + 1, + "a", + 2, + "href", + "https://realpython.com/python-type-checking/#type-variables", + 3, + "type variables", + -1, + 3, + ". Consider the definition of a ", + 1, + "a", + 2, + "href", + "https://realpython.com/primer-on-python-decorators/", + 3, + "decorator", + -1, + 3, + ". In general, it looks something like the following:\n ", + -1, + 1, + "p", + 3, + "\n The annotations mean that the function returned by the decorator is a callable with some parameters and the same return type, ", + 1, + "code", + 3, + "R", + -1, + 3, + ", as the function passed into the decorator. The ", + 1, + "a", + 2, + "href", + "https://realpython.com/python-ellipsis/", + 3, + "ellipsis", + -1, + 3, + " (", + 1, + "code", + 3, + "...", + -1, + 3, + ") in the function header correctly allows any number of parameters, and each of those parameters can be of any type. However, there’s no validation that the returned callable has the same parameters as the function that was passed in. In practice, this means that type checkers aren’t able to check decorated functions properly.\n ", + -1, + 1, + "p", + 3, + "\n Unfortunately, you can’t use ", + 1, + "code", + 3, + "TypeVar", + -1, + 3, + " for the parameters because you don’t know how many parameters the function will have. In Python 3.10, you’ll have access to ", + 1, + "a", + 2, + "href", + "https://docs.python.org/3.10/library/typing.html#typing.ParamSpec", + 1, + "code", + 3, + "ParamSpec", + -2, + 3, + " in order to type hint these kinds of callables properly. ", + 1, + "code", + 3, + "ParamSpec", + -1, + 3, + " works similarly to ", + 1, + "code", + 3, + "TypeVar", + -1, + 3, + " but stands in for several parameters at once. You can rewrite your decorator as follows to take advantage of ", + 1, + "code", + 3, + "ParamSpec", + -1, + 3, + ":\n ", + -1, + 1, + "p", + 3, + "\n Note that you also use ", + 1, + "code", + 3, + "P", + -1, + 3, + " when you annotate ", + 1, + "code", + 3, + "wrapper()", + -1, + 3, + ". You can also use the new ", + 1, + "a", + 2, + "href", + "https://docs.python.org/3.10/library/typing.html#typing.Concatenate", + 1, + "code", + 3, + "typing.Concatenate", + -2, + 3, + " to add types to ", + 1, + "code", + 3, + "ParamSpec", + -1, + 3, + ". See the ", + 1, + "a", + 2, + "href", + "https://docs.python.org/3.10/library/typing.html", + 3, + "documentation", + -1, + 3, + " and ", + 1, + "a", + 2, + "href", + "https://www.python.org/dev/peps/pep-0612/", + 3, + "PEP 612", + -1, + 3, + " for details and examples.\n ", + -2, + 1, + "section", + 1, + "h2", + 2, + "id", + "stricter-zipping-of-sequences", + 3, + "\n Stricter Zipping of Sequences", + 1, + "a", + 2, + "href", + "#stricter-zipping-of-sequences", + 2, + "title", + "Permanent link", + -2, + 1, + "p", + 1, + "a", + 2, + "href", + "https://realpython.com/python-zip-function/", + 1, + "code", + 3, + "zip()", + -2, + 3, + " is a ", + 1, + "a", + 2, + "href", + "https://docs.python.org/3/library/functions.html#built-in-functions", + 3, + "built-in function", + -1, + 3, + " in Python that can combine elements from several sequences. Python 3.10 introduces the new ", + 1, + "code", + 3, + "strict", + -1, + 3, + " parameter, which adds a runtime test to check that all sequences being zipped have the same length.\n ", + -1, + 1, + "p", + 3, + "\n As an example, consider the following table of ", + 1, + "a", + 2, + "href", + "https://www.lego.com/", + 3, + "Lego", + -1, + 3, + " sets:\n ", + -1, + 1, + "p", + 3, + "\n One way to represent these data in plain Python would be with each column as a list. It could look something like this:\n ", + -1, + 1, + "p", + 3, + "\n Note that you have three independent lists, but there’s an implicit correspondence between their elements. The first name (", + 1, + "code", + 3, + "\"Louvre\"", + -1, + 3, + "), the first set number (", + 1, + "code", + 3, + "\"21024\"", + -1, + 3, + "), and the first number of pieces (", + 1, + "code", + 3, + "695", + -1, + 3, + ") all describe the first Lego set.\n ", + -1, + 1, + "p", + 1, + "code", + 3, + "zip()", + -1, + 3, + " can be used to iterate over these three lists in parallel:\n ", + -1, + 1, + "p", + 3, + "\n Note how each line collects information from all three lists and shows information about one particular set. This is a very common pattern that’s used in a lot of different Python code, including ", + 1, + "a", + 2, + "href", + "https://www.python.org/dev/peps/pep-0618/#examples", + 3, + "in the standard library", + -1, + 3, + ".\n ", + -1, + 1, + "p", + 3, + "\n You can also add ", + 1, + "code", + 3, + "list()", + -1, + 3, + " to collect the contents of all three lists in a single, nested list of tuples:\n ", + -1, + 1, + "p", + 3, + "\n Note how the nested list closely resembles the original table.\n ", + -1, + 1, + "p", + 3, + "\n The dark side of using ", + 1, + "code", + 3, + "zip()", + -1, + 3, + " is that it’s quite easy to introduce a subtle bug that can be hard to discover. Note what happens if there’s a missing item in one of your lists:\n ", + -1, + 1, + "p", + 3, + "\n All the information about the New York City set disappeared! Additionally, the set numbers for Saturn V and Millennium Falcon are wrong. If your datasets are bigger, these kinds of errors can be very hard to discover. And even when you observe that something’s wrong, it’s not always easy to diagnose and fix.\n ", + -1, + 1, + "p", + 3, + "\n The issue is that you assumed that the three lists have the same number of elements and that the information is in the same order in each list. After ", + 1, + "code", + 3, + "set_numbers", + -1, + 3, + " gets corrupted, this assumption is no longer true.\n ", + -1, + 1, + "p", + 1, + "a", + 2, + "href", + "https://www.python.org/dev/peps/pep-0618/", + 3, + "PEP 618", + -1, + 3, + " introduces a new ", + 1, + "code", + 3, + "strict", + -1, + 3, + " keyword parameter to ", + 1, + "code", + 3, + "zip()", + -1, + 3, + " that you can use to confirm all sequences have the same length. In your example, it would raise an error alerting you to the corrupted list:\n ", + -1, + 1, + "p", + 3, + "\n When the iteration reaches the New York City Lego set, the second argument ", + 1, + "code", + 3, + "set_numbers", + -1, + 3, + " is already exhausted, while there are still elements left in the first argument ", + 1, + "code", + 3, + "names", + -1, + 3, + ". Instead of silently giving the wrong result, your code fails with an error, and you can take action to find and fix the mistake.\n ", + -1, + 1, + "p", + 3, + "\n There are use cases when you want to combine sequences of unequal length. Expand the box below to see how ", + 1, + "code", + 3, + "zip()", + -1, + 3, + " and ", + 1, + "code", + 3, + "itertools.zip_longest()", + -1, + 3, + " handle these:\n ", + -1, + 1, + "div", + 2, + "id", + "collapse_cardc07d76", + 2, + "data-parent", + "#collapse_cardc07d76", + 1, + "p", + 3, + "\n The ", + 1, + "a", + 2, + "href", + "https://realpython.com/python-itertools/#what-is-itertools-and-why-should-you-use-it", + 3, + "following idiom", + -1, + 3, + " divides the Lego sets into pairs:\n ", + -1, + 1, + "p", + 3, + "\n There are five sets, a number that doesn’t divide evenly into pairs. In this case, the default behavior of ", + 1, + "code", + 3, + "zip()", + -1, + 3, + ", where the last element is dropped, might make sense. You could use ", + 1, + "code", + 3, + "strict=True", + -1, + 3, + " here as well, but that would raise an error when your list can’t be split into pairs. A third option, which could be the best in this case, is to use ", + 1, + "a", + 2, + "href", + "https://docs.python.org/3/library/itertools.html#itertools.zip_longest", + 1, + "code", + 3, + "zip_longest()", + -2, + 3, + " from the ", + 1, + "a", + 2, + "href", + "https://realpython.com/python-itertools/", + 1, + "code", + 3, + "itertools", + -2, + 3, + " standard library.\n ", + -1, + 1, + "p", + 3, + "\n As the name suggests, ", + 1, + "code", + 3, + "zip_longest()", + -1, + 3, + " combines sequences until the longest sequence is exhausted. If you use ", + 1, + "code", + 3, + "zip_longest()", + -1, + 3, + " to divide the Lego sets, it becomes more explicit that New York City doesn’t have any pairing:\n ", + -1, + 1, + "p", + 3, + "\n Note that ", + 1, + "code", + 3, + "'NYC'", + -1, + 3, + " shows up in the last tuple together with an empty string. You can control what’s filled in for missing values with the ", + 1, + "code", + 3, + "fillvalue", + -1, + 3, + " parameter.\n ", + -2, + 1, + "p", + 3, + "\n While ", + 1, + "code", + 3, + "strict", + -1, + 3, + " is not really adding any new functionality to ", + 1, + "code", + 3, + "zip()", + -1, + 3, + ", it can help you avoid those hard-to-find bugs.\n ", + -2, + 1, + "section", + 1, + "h2", + 2, + "id", + "new-functions-in-the-statistics-module", + 3, + "\n New Functions in the ", + 1, + "code", + 3, + "statistics", + -1, + 3, + " Module", + 1, + "a", + 2, + "href", + "#new-functions-in-the-statistics-module", + 2, + "title", + "Permanent link", + -2, + 1, + "p", + 3, + "\n The ", + 1, + "a", + 2, + "href", + "https://docs.python.org/3/library/statistics.html", + 1, + "code", + 3, + "statistics", + -2, + 3, + " module was added to the standard library all the way back in 2014 with the release of ", + 1, + "a", + 2, + "href", + "https://www.python.org/downloads/release/python-340/", + 3, + "Python 3.4", + -1, + 3, + ". The intent of ", + 1, + "code", + 3, + "statistics", + -1, + 3, + " is to make ", + 1, + "a", + 2, + "href", + "https://realpython.com/python-statistics/", + 3, + "statistical calculations", + -1, + 3, + " at the ", + 1, + "a", + 2, + "href", + "https://www.python.org/dev/peps/pep-0450/", + 3, + "level of graphing calculators", + -1, + 3, + " available in Python.\n ", + -1, + 1, + "p", + 3, + "\n Python 3.10 adds a few multivariable functions to ", + 1, + "code", + 3, + "statistics", + -1, + 3, + ":\n ", + -1, + 1, + "ul", + 1, + "li", + 1, + "strong", + 1, + "code", + 3, + "correlation()", + -2, + 3, + " to calculate Pearson’s ", + 1, + "a", + 2, + "href", + "https://realpython.com/numpy-scipy-pandas-correlation-python/", + 3, + "correlation", + -1, + 3, + " coefficient for two variables\n ", + -1, + 1, + "li", + 1, + "strong", + 1, + "code", + 3, + "covariance()", + -2, + 3, + " to calculate sample ", + 1, + "a", + 2, + "href", + "https://en.wikipedia.org/wiki/Covariance", + 3, + "covariance", + -1, + 3, + " for two variables\n ", + -1, + 1, + "li", + 1, + "strong", + 1, + "code", + 3, + "linear_regression()", + -2, + 3, + " to calculate the slope and intercept in a ", + 1, + "a", + 2, + "href", + "https://realpython.com/linear-regression-in-python/", + 3, + "linear regression", + -3, + 1, + "p", + 3, + "\n You can use each function to describe a certain aspect of the relationship between two variables. As an example, say that you have data from a set of blog posts—the number of words in each blog post and the number of views each post has had over some time period:\n ", + -1, + 1, + "p", + 3, + "\n You now want to investigate whether there’s any (linear) relationship between the number of words and number of views. In Python 3.10, you can calculate the ", + 1, + "strong", + 3, + "correlation", + -1, + 3, + " between ", + 1, + "code", + 3, + "words", + -1, + 3, + " and ", + 1, + "code", + 3, + "views", + -1, + 3, + " with the new ", + 1, + "a", + 2, + "href", + "https://docs.python.org/3.10/library/statistics.html#statistics.correlation", + 1, + "code", + 3, + "correlation()", + -2, + 3, + " function:\n ", + -1, + 1, + "p", + 3, + "\n The correlation between two variables is always a number between -1 and 1. If it’s close to 0, then there’s little correspondence between them, while a correlation close to -1 or 1 indicates that the behaviors of the two variables tend to follow each other. In this example, a correlation of 0.45 indicates that there’s a tendency for posts with more words to have more views, although it’s not a strong connection.\n ", + -1, + 1, + "p", + 3, + "\n You can also calculate the ", + 1, + "strong", + 3, + "covariance", + -1, + 3, + " between ", + 1, + "code", + 3, + "words", + -1, + 3, + " and ", + 1, + "code", + 3, + "views", + -1, + 3, + ". The covariance is another measure of the joint variability between two variables. You can calculate it with ", + 1, + "a", + 2, + "href", + "https://docs.python.org/3.10/library/statistics.html#statistics.covariance", + 1, + "code", + 3, + "covariance()", + -2, + 3, + ":\n ", + -1, + 1, + "p", + 3, + "\n In contrast to correlation, covariance is an absolute measure. It should be interpreted in the context of the variability within the variables themselves. In fact, you can normalize the covariance by the ", + 1, + "a", + 2, + "href", + "https://en.wikipedia.org/wiki/Standard_deviation", + 3, + "standard deviation", + -1, + 3, + " of each variable to recover ", + 1, + "a", + 2, + "href", + "https://en.wikipedia.org/wiki/Pearson_correlation_coefficient", + 3, + "Pearson’s correlation coefficient", + -1, + 3, + ":\n ", + -1, + 1, + "p", + 3, + "\n Note that this matches your earlier correlation coefficient exactly.\n ", + -1, + 1, + "p", + 3, + "\n A third way of looking at the linear correspondence between the two variables is through ", + 1, + "strong", + 3, + "simple linear regression", + -1, + 3, + ". You do the ", + 1, + "a", + 2, + "href", + "https://en.wikipedia.org/wiki/Simple_linear_regression", + 3, + "linear regression", + -1, + 3, + " by calculating two numbers, ", + 1, + "em", + 3, + "slope", + -1, + 3, + " and ", + 1, + "em", + 3, + "intercept", + -1, + 3, + ", so that the (squared) error is minimized in the approximation ", + 1, + "em", + 3, + "number of views", + -1, + 3, + " = ", + 1, + "em", + 3, + "slope", + -1, + 3, + " × ", + 1, + "em", + 3, + "number of words", + -1, + 3, + " + ", + 1, + "em", + 3, + "intercept", + -1, + 3, + ".\n ", + -1, + 1, + "p", + 3, + "\n In Python 3.10, you can use ", + 1, + "a", + 2, + "href", + "https://docs.python.org/3.10/library/statistics.html#statistics.linear_regression", + 1, + "code", + 3, + "linear_regression()", + -2, + 3, + ":\n ", + -1, + 1, + "p", + 3, + "\n Based on this regression, a post with 10,074 words could expect about 0.2424 × 10074 + 1104 = 3546 views. However, as you saw earlier, the correlation between the number of words and the number of views is quite weak. Therefore, you shouldn’t expect this prediction to be very accurate.\n ", + -1, + 1, + "p", + 3, + "\n The ", + 1, + "code", + 3, + "LinearRegression", + -1, + 3, + " object is a ", + 1, + "a", + 2, + "href", + "https://realpython.com/python-namedtuple/", + 3, + "named tuple", + -1, + 3, + ". This means that you can unpack the slope and intercept directly:\n ", + -1, + 1, + "p", + 3, + "\n Here, you use ", + 1, + "code", + 3, + "slope", + -1, + 3, + " and ", + 1, + "code", + 3, + "intercept", + -1, + 3, + " to predict the number of views on a blog post with 10,074 words.\n ", + -1, + 1, + "p", + 3, + "\n You still want to use some of the more advanced packages like pandas and statsmodels if you do a lot of statistical analysis. With the new additions to ", + 1, + "code", + 3, + "statistics", + -1, + 3, + " in Python 3.10, however, you have the chance to do basic analysis more easily without bringing in third-party dependencies.\n ", + -2, + 1, + "section", + 1, + "h2", + 2, + "id", + "other-pretty-cool-features", + 3, + "\n Other Pretty Cool Features", + 1, + "a", + 2, + "href", + "#other-pretty-cool-features", + 2, + "title", + "Permanent link", + -2, + 1, + "p", + 3, + "\n So far, you’ve seen the biggest and most impactful new features in Python 3.10. In this section, you’ll get a glimpse of a few of the other changes that the new version brings along. If you’re curious about all the changes made for this new version, check out the ", + 1, + "a", + 2, + "href", + "https://docs.python.org/3.10/whatsnew/3.10.html", + 3, + "documentation", + -1, + 3, + ".\n ", + -1, + 1, + "section", + 1, + "h3", + 2, + "id", + "default-text-encodings", + 3, + "\n Default Text Encodings", + 1, + "a", + 2, + "href", + "#default-text-encodings", + 2, + "title", + "Permanent link", + -2, + 1, + "p", + 3, + "\n When you open a text file, the default encoding used to interpret the characters is system dependent. In particular, ", + 1, + "a", + 2, + "href", + "https://docs.python.org/3.10/library/locale.html#locale.getpreferredencoding", + 1, + "code", + 3, + "locale.getpreferredencoding()", + -2, + 3, + " is used. On Mac and Linux, this usually returns ", + 1, + "code", + 3, + "\"UTF-8\"", + -1, + 3, + ", while the result on Windows is more varied.\n ", + -1, + 1, + "p", + 3, + "\n You should therefore always specify an encoding when you attempt to open a text file:\n ", + -1, + 1, + "p", + 3, + "\n If you don’t explicitly specify an encoding, the preferred locale encoding is used, and you could experience that a file that can be read on one computer fails to open on another.\n ", + -1, + 1, + "p", + 3, + "\n Python 3.7 introduced ", + 1, + "a", + 2, + "href", + "https://docs.python.org/3.10/library/os.html#utf8-mode", + 3, + "UTF-8 mode", + -1, + 3, + ", which allows you to force your programs to use UTF-8 encoding independent of the locale encoding. You can enable UTF-8 mode by giving the ", + 1, + "code", + 3, + "-X utf8", + -1, + 3, + " command-line option to the ", + 1, + "code", + 3, + "python", + -1, + 3, + " executable or by setting the ", + 1, + "code", + 3, + "PYTHONUTF8", + -1, + 3, + " environment variable.\n ", + -1, + 1, + "p", + 3, + "\n In Python 3.10, you can activate a warning that will tell you when a text file is opened without a specified encoding. Consider the following script, which doesn’t specify an encoding:\n ", + -1, + 1, + "p", + 3, + "\n The program will echo one or more text files back to the console, but with each line reversed. Run the program on itself with the ", + 1, + "a", + 2, + "href", + "https://docs.python.org/3.10/library/io.html#io-encoding-warning", + 3, + "encoding warning", + -1, + 3, + " enabled:\n ", + -1, + 1, + "p", + 3, + "\n Note the ", + 1, + "code", + 3, + "EncodingWarning", + -1, + 3, + " printed to the console. The command-line option ", + 1, + "code", + 3, + "-X warn_default_encoding", + -1, + 3, + " activates it. The warning will disappear if you specify an encoding—for example, ", + 1, + "code", + 3, + "encoding=\"utf-8\"", + -1, + 3, + "—when you open the file.\n ", + -1, + 1, + "p", + 3, + "\n There are times when you want to use the user-defined local encoding. You can still do so by explicitly using ", + 1, + "code", + 3, + "encoding=\"locale\"", + -1, + 3, + ". However, it’s recommended to use UTF-8 whenever possible. You can check out ", + 1, + "a", + 2, + "href", + "https://www.python.org/dev/peps/pep-0597/", + 3, + "PEP 597", + -1, + 3, + " for more information.\n ", + -2, + 1, + "section", + 1, + "h3", + 2, + "id", + "asynchronous-iteration", + 3, + "\n Asynchronous Iteration", + 1, + "a", + 2, + "href", + "#asynchronous-iteration", + 2, + "title", + "Permanent link", + -2, + 1, + "p", + 1, + "a", + 2, + "href", + "https://realpython.com/python-async-features/", + 3, + "Asynchronous programming", + -1, + 3, + " is a powerful programming paradigm that’s been available in Python ", + 1, + "a", + 2, + "href", + "https://www.python.org/dev/peps/pep-0492/", + 3, + "since version 3.5", + -1, + 3, + ". You can recognize an asynchronous program by its use of the ", + 1, + "code", + 3, + "async", + -1, + 3, + " keyword or ", + 1, + "a", + 2, + "href", + "https://realpython.com/python-classes/#special-methods-and-protocols", + 3, + "special methods", + -1, + 3, + " that ", + 1, + "a", + 2, + "href", + "https://www.python.org/dev/peps/pep-0492/#why-magic-methods-start-with-a", + 3, + "start with ", + 1, + "code", + 3, + ".__a", + -2, + 3, + " like ", + 1, + "a", + 2, + "href", + "https://docs.python.org/3/reference/datamodel.html#object.__aiter__", + 1, + "code", + 3, + ".__aiter__()", + -2, + 3, + " or ", + 1, + "a", + 2, + "href", + "https://docs.python.org/3/reference/datamodel.html#object.__aenter__", + 1, + "code", + 3, + ".__aenter__()", + -2, + 3, + ".\n ", + -1, + 1, + "p", + 3, + "\n In Python 3.10, two new asynchronous ", + 1, + "a", + 2, + "href", + "https://docs.python.org/3/library/functions.html#built-in-functions", + 3, + "built-in functions", + -1, + 3, + " are added: ", + 1, + "a", + 2, + "href", + "https://docs.python.org/3.10/library/functions.html#aiter", + 1, + "code", + 3, + "aiter()", + -2, + 3, + " and ", + 1, + "a", + 2, + "href", + "https://docs.python.org/3.10/library/functions.html#anext", + 1, + "code", + 3, + "anext()", + -2, + 3, + ". In practice, these functions call the ", + 1, + "code", + 3, + ".__aiter__()", + -1, + 3, + " and ", + 1, + "code", + 3, + ".__anext__()", + -1, + 3, + " special methods—analogous to the regular ", + 1, + "code", + 3, + "iter()", + -1, + 3, + " and ", + 1, + "code", + 3, + "next()", + -1, + 3, + "—so no new functionality is added. These are convenience functions that make your code more readable.\n ", + -1, + 1, + "p", + 3, + "\n In other words, in the newest version of Python, the following statements—where ", + 1, + "code", + 3, + "things", + -1, + 3, + " is an ", + 1, + "a", + 2, + "href", + "https://www.python.org/dev/peps/pep-0492/#asynchronous-iterators-and-async-for", + 3, + "asynchronous iterable", + -1, + 3, + "—are equivalent:\n ", + -1, + 1, + "p", + 3, + "\n In either case, ", + 1, + "code", + 3, + "it", + -1, + 3, + " ends up as an asynchronous iterator. Expand the following box to see a complete example using ", + 1, + "code", + 3, + "aiter()", + -1, + 3, + " and ", + 1, + "code", + 3, + "anext()", + -1, + 3, + ":\n ", + -1, + 1, + "div", + 2, + "id", + "collapse_card7ba5eb", + 2, + "data-parent", + "#collapse_card7ba5eb", + 1, + "p", + 3, + "\n The following program counts the number of lines in several files. In practice, you use Python’s ability to iterate over files to count the number of lines. The script uses asynchronous iteration in order to handle several files concurrently.\n ", + -1, + 1, + "p", + 3, + "\n Note that you need to install the third-party ", + 1, + "a", + 2, + "href", + "https://pypi.org/project/aiofiles/", + 1, + "code", + 3, + "aiofiles", + -2, + 3, + " package with ", + 1, + "a", + 2, + "href", + "https://realpython.com/what-is-pip/", + 1, + "code", + 3, + "pip", + -2, + 3, + " before running this code:\n ", + -1, + 1, + "p", + 1, + "code", + 3, + "asyncio", + -1, + 3, + " is used to create and run one asynchronous task per filename. ", + 1, + "code", + 3, + "count_lines()", + -1, + 3, + " opens one file asynchronously and iterates through it using ", + 1, + "code", + 3, + "aiter()", + -1, + 3, + " and ", + 1, + "code", + 3, + "anext()", + -1, + 3, + " in order to count the number of lines.\n ", + -2, + 1, + "p", + 3, + "\n See ", + 1, + "a", + 2, + "href", + "https://www.python.org/dev/peps/pep-0525/", + 3, + "PEP 525", + -1, + 3, + " to learn more about asynchronous iteration.\n ", + -2, + 1, + "section", + 1, + "h3", + 2, + "id", + "context-manager-syntax", + 3, + "\n Context Manager Syntax", + 1, + "a", + 2, + "href", + "#context-manager-syntax", + 2, + "title", + "Permanent link", + -2, + 1, + "p", + 1, + "a", + 2, + "href", + "https://realpython.com/python-with-statement/", + 3, + "Context managers", + -1, + 3, + " are great for managing resources in your programs. Until recently, though, their syntax has included an uncommon wart. You ", + 1, + "a", + 2, + "href", + "https://www.python.org/dev/peps/pep-0617/#some-rules-are-not-actually-ll-1", + 3, + "haven’t been allowed", + -1, + 3, + " to use parentheses to break long ", + 1, + "code", + 3, + "with", + -1, + 3, + " statements like this:\n ", + -1, + 1, + "p", + 3, + "\n In earlier versions of Python, this causes an ", + 1, + "code", + 3, + "invalid syntax", + -1, + 3, + " error message. Instead, you need to use a backslash (", + 1, + "code", + 3, + "\\", + -1, + 3, + ") if you want to control where you break your lines:\n ", + -1, + 1, + "p", + 3, + "\n While ", + 1, + "a", + 2, + "href", + "https://realpython.com/python-program-structure/#explicit-line-continuation", + 3, + "explicit line continuation", + -1, + 3, + " with backslashes is possible in Python, PEP 8 ", + 1, + "a", + 2, + "href", + "https://www.python.org/dev/peps/pep-0008/#maximum-line-length", + 3, + "discourages it", + -1, + 3, + ". The ", + 1, + "a", + 2, + "href", + "https://black.readthedocs.io/", + 3, + "Black", + -1, + 3, + " formatting tool ", + 1, + "a", + 2, + "href", + "https://black.readthedocs.io/en/stable/the_black_code_style/current_style.html", + 3, + "avoids", + -1, + 3, + " backslashes completely.\n ", + -1, + 1, + "p", + 3, + "\n In Python 3.10, you’re now allowed to add parentheses around ", + 1, + "code", + 3, + "with", + -1, + 3, + " statements to your heart’s content. Especially if you’re employing several context managers at once, like in the example above, this can help improve the readability of your code. Python’s ", + 1, + "a", + 2, + "href", + "https://docs.python.org/3.10/whatsnew/3.10.html#parenthesized-context-managers", + 3, + "documentation", + -1, + 3, + " shows a few other possibilities with this new syntax.\n ", + -1, + 1, + "p", + 3, + "\n One small ", + 1, + "strong", + 3, + "fun fact", + -1, + 3, + ": parenthesized ", + 1, + "code", + 3, + "with", + -1, + 3, + " statements actually work in version 3.9 of ", + 1, + "a", + 2, + "href", + "https://realpython.com/cpython-source-code-guide/", + 3, + "CPython", + -1, + 3, + ". Their implementation came almost for free with the introduction of the ", + 1, + "a", + 2, + "href", + "https://realpython.com/python39-new-features/#a-more-powerful-python-parser", + 3, + "PEG parser", + -1, + 3, + " in ", + 1, + "a", + 2, + "href", + "https://realpython.com/python39-new-features/", + 3, + "Python 3.9", + -1, + 3, + ". The reason that this is called a Python 3.10 feature is that using the PEG parser is voluntary in Python 3.9, while Python 3.9, with the old LL(1) parser, doesn’t support parenthesized ", + 1, + "code", + 3, + "with", + -1, + 3, + " statements.\n ", + -2, + 1, + "section", + 1, + "h3", + 2, + "id", + "modern-and-secure-ssl", + 3, + "\n Modern and Secure SSL", + 1, + "a", + 2, + "href", + "#modern-and-secure-ssl", + 2, + "title", + "Permanent link", + -2, + 1, + "p", + 3, + "\n Security can be challenging! A good rule of thumb is to avoid rolling your own security algorithms and instead rely on established packages.\n ", + -1, + 1, + "p", + 3, + "\n Python uses ", + 1, + "a", + 2, + "href", + "https://www.openssl.org/", + 3, + "OpenSSL", + -1, + 3, + " for different cryptographic features that are exposed in the ", + 1, + "a", + 2, + "href", + "https://docs.python.org/3/library/hashlib.html", + 1, + "code", + 3, + "hashlib", + -2, + 3, + ", ", + 1, + "a", + 2, + "href", + "https://docs.python.org/3/library/hmac.html", + 1, + "code", + 3, + "hmac", + -2, + 3, + ", and ", + 1, + "a", + 2, + "href", + "https://docs.python.org/3/library/ssl.html", + 1, + "code", + 3, + "ssl", + -2, + 3, + " standard library modules. Your system can manage OpenSSL, or a Python installer can include OpenSSL.\n ", + -1, + 1, + "p", + 3, + "\n Python 3.9 supports using any of the ", + 1, + "a", + 2, + "href", + "https://en.wikipedia.org/wiki/OpenSSL#Major_version_releases", + 3, + "OpenSSL versions", + -1, + 3, + " 1.0.2 LTS, 1.1.0, and 1.1.1 LTS. Both OpenSSL 1.0.2 LTS and OpenSSL 1.1.0 are past their lifetime, so Python 3.10 will only support OpenSSL 1.1.1 LTS, as described in the following table:\n ", + -1, + 1, + "div", + 1, + "table", + 1, + "thead", + 1, + "tr", + 1, + "th", + 3, + "\n Open SSL version\n ", + -1, + 1, + "th", + 3, + "\n Python 3.9\n ", + -1, + 1, + "th", + 3, + "\n Python 3.10\n ", + -1, + 1, + "th", + 3, + "\n End-of-life\n ", + -3, + 1, + "tbody", + 1, + "tr", + 1, + "td", + 3, + "\n 1.0.2 LTS\n ", + -1, + 1, + "td", + 3, + "\n ✔\n ", + -1, + 1, + "td", + 3, + "\n ✖\n ", + -2, + 1, + "tr", + 1, + "td", + 3, + "\n 1.1.0\n ", + -1, + 1, + "td", + 3, + "\n ✔\n ", + -1, + 1, + "td", + 3, + "\n ✖\n ", + -2, + 1, + "tr", + 1, + "td", + 3, + "\n 1.1.1 LTS\n ", + -1, + 1, + "td", + 3, + "\n ✔\n ", + -1, + 1, + "td", + 3, + "\n ✔\n ", + -5, + 1, + "p", + 3, + "\n This end of support for older versions will only affect you if you need to upgrade the system Python on an older operating system. If you use macOS or Windows, or if you install Python from ", + 1, + "a", + 2, + "href", + "https://www.python.org/", + 3, + "python.org", + -1, + 3, + " or use ", + 1, + "a", + 2, + "href", + "https://docs.conda.io/projects/conda/en/latest/user-guide/install/index.html", + 3, + "(Ana)Conda", + -1, + 3, + ", you’ll see no change.\n ", + -1, + 1, + "p", + 3, + "\n However, ", + 1, + "a", + 2, + "href", + "https://ubuntu.com/", + 3, + "Ubuntu", + -1, + 3, + " 18.04 LTS uses OpenSSL 1.1.0, while ", + 1, + "a", + 2, + "href", + "https://www.redhat.com/en/technologies/linux-platforms/enterprise-linux", + 3, + "Red Hat Enterprise Linux", + -1, + 3, + " (RHEL) 7 and ", + 1, + "a", + 2, + "href", + "https://www.centos.org/", + 3, + "CentOS", + -1, + 3, + " 7 both use OpenSSL 1.0.2 LTS. If you need to run Python 3.10 on these systems, you should look at installing it yourself using either the ", + 1, + "a", + 2, + "href", + "https://www.python.org/", + 3, + "python.org", + -1, + 3, + " or Conda installer.\n ", + -1, + 1, + "p", + 3, + "\n Dropping support for older versions of OpenSSL will make Python more secure. It’ll also help the Python developers in that code will be easier to maintain. Ultimately, this helps you because your Python experience will be more robust. See ", + 1, + "a", + 2, + "href", + "https://www.python.org/dev/peps/pep-0644/", + 3, + "PEP 644", + -1, + 3, + " for more details.\n ", + -2, + 1, + "section", + 1, + "h3", + 2, + "id", + "more-information-about-your-python-interpreter", + 3, + "\n More Information About Your Python Interpreter", + 1, + "a", + 2, + "href", + "#more-information-about-your-python-interpreter", + 2, + "title", + "Permanent link", + -2, + 1, + "p", + 3, + "\n The ", + 1, + "a", + 2, + "href", + "https://docs.python.org/3/library/sys.html", + 1, + "code", + 3, + "sys", + -2, + 3, + " module contains a lot of information about your system, the current Python runtime, and the script currently being executed. You can, for example, inquire about the paths where ", + 1, + "a", + 2, + "href", + "https://realpython.com/python-import/#pythons-import-path", + 3, + "Python looks for modules", + -1, + 3, + " with ", + 1, + "a", + 2, + "href", + "https://docs.python.org/3/library/sys.html#sys.path", + 1, + "code", + 3, + "sys.path", + -2, + 3, + " and see all modules that ", + 1, + "a", + 2, + "href", + "https://realpython.com/python-import/#import-internals", + 3, + "have been imported", + -1, + 3, + " in the current session with ", + 1, + "a", + 2, + "href", + "https://docs.python.org/3/library/sys.html#sys.modules", + 1, + "code", + 3, + "sys.modules", + -2, + 3, + ".\n ", + -1, + 1, + "p", + 3, + "\n In Python 3.10, ", + 1, + "code", + 3, + "sys", + -1, + 3, + " has two new attributes. First, you can now get a list of the names of all modules in the standard library:\n ", + -1, + 1, + "p", + 3, + "\n Here, you can see that there are around 300 modules in the standard library, several of which start with the letter ", + 1, + "code", + 3, + "z", + -1, + 3, + ". Note that only top-level modules and packages are listed. Subpackages like ", + 1, + "a", + 2, + "href", + "https://realpython.com/python38-new-features/#importlibmetadata", + 1, + "code", + 3, + "importlib.metadata", + -2, + 3, + " don’t get a separate entry.\n ", + -1, + 1, + "p", + 3, + "\n You will probably not be using ", + 1, + "a", + 2, + "href", + "https://docs.python.org/3.10/library/sys.html#sys.stdlib_module_names", + 1, + "code", + 3, + "sys.stdlib_module_names", + -2, + 3, + " all that often. Still, the list ties in nicely with similar introspection features like ", + 1, + "a", + 2, + "href", + "https://docs.python.org/3/library/keyword.html#keyword.kwlist", + 1, + "code", + 3, + "keyword.kwlist", + -2, + 3, + " and ", + 1, + "a", + 2, + "href", + "https://docs.python.org/3/library/sys.html#sys.builtin_module_names", + 1, + "code", + 3, + "sys.builtin_module_names", + -2, + 3, + ".\n ", + -1, + 1, + "p", + 3, + "\n One possible use case for the new attribute is to identify which of the currently imported modules are third-party dependencies:\n ", + -1, + 1, + "p", + 3, + "\n You find the imported top-level modules by looking at names in ", + 1, + "code", + 3, + "sys.modules", + -1, + 3, + " that don’t have a dot in their name. By comparing them to the standard library module names, you find that ", + 1, + "a", + 2, + "href", + "https://realpython.com/numpy-array-programming/", + 1, + "code", + 3, + "numpy", + -2, + 3, + ", ", + 1, + "a", + 2, + "href", + "https://realpython.com/python-packages/#dateutil-for-working-with-dates-and-times", + 1, + "code", + 3, + "dateutil", + -2, + 3, + ", and ", + 1, + "a", + 2, + "href", + "https://realpython.com/python-pandas-tricks/", + 1, + "code", + 3, + "pandas", + -2, + 3, + " are some of the imported third-party modules in this example.\n ", + -1, + 1, + "p", + 3, + "\n The other new attribute is ", + 1, + "a", + 2, + "href", + "https://docs.python.org/3.10/library/sys.html#sys.orig_argv", + 1, + "code", + 3, + "sys.orig_argv", + -2, + 3, + ". This is related to ", + 1, + "a", + 2, + "href", + "https://docs.python.org/3/library/sys.html#sys.argv", + 1, + "code", + 3, + "sys.argv", + -2, + 3, + ", which holds the ", + 1, + "a", + 2, + "href", + "https://realpython.com/python-command-line-arguments/#the-sysargv-array", + 3, + "command-line arguments", + -1, + 3, + " given to your program when it was started. In contrast, ", + 1, + "code", + 3, + "sys.orig_argv", + -1, + 3, + " lists the command-line arguments passed to the ", + 1, + "code", + 3, + "python", + -1, + 3, + " executable itself. Consider the following example:\n ", + -1, + 1, + "p", + 3, + "\n This script echoes back the ", + 1, + "code", + 3, + "orig_argv", + -1, + 3, + " and ", + 1, + "code", + 3, + "argv", + -1, + 3, + " lists. Run it to see how the information is captured:\n ", + -1, + 1, + "p", + 3, + "\n Essentially, all arguments—including the name of the Python executable—end up in ", + 1, + "code", + 3, + "orig_argv", + -1, + 3, + ". This is in contrast to ", + 1, + "code", + 3, + "argv", + -1, + 3, + ", which only contains the arguments that aren’t handled by ", + 1, + "code", + 3, + "python", + -1, + 3, + " itself.\n ", + -1, + 1, + "p", + 3, + "\n Again, this is not a feature that you’ll use a lot. If your program needs to concern itself with how it’s being run, you’re usually better off relying on information that’s already exposed instead of trying to parse this list. For example, you can choose to use the ", + 1, + "a", + 2, + "href", + "#stricter-zipping-of-sequences", + 3, + "strict ", + 1, + "code", + 3, + "zip()", + -1, + 3, + " mode", + -1, + 3, + " only when your script is not running with the optimized flag, ", + 1, + "code", + 3, + "-O", + -1, + 3, + ", like this:\n ", + -1, + 1, + "p", + 3, + "\n The ", + 1, + "a", + 2, + "href", + "https://docs.python.org/3/library/constants.html#__debug__", + 1, + "code", + 3, + "__debug__", + -2, + 3, + " flag is set when the interpreter starts. It’ll be ", + 1, + "code", + 3, + "False", + -1, + 3, + " if you’re running ", + 1, + "code", + 3, + "python", + -1, + 3, + " with ", + 1, + "a", + 2, + "href", + "https://docs.python.org/3/using/cmdline.html#cmdoption-o", + 1, + "code", + 3, + "-O", + -2, + 3, + " or ", + 1, + "a", + 2, + "href", + "https://docs.python.org/3/using/cmdline.html#cmdoption-oo", + 1, + "code", + 3, + "-OO", + -2, + 3, + " specified, and ", + 1, + "code", + 3, + "True", + -1, + 3, + " otherwise. Using ", + 1, + "code", + 3, + "__debug__", + -1, + 3, + " is usually preferable to ", + 1, + "code", + 3, + "\"-O\" not in sys.orig_argv", + -1, + 3, + " or some similar construct.\n ", + -1, + 1, + "p", + 3, + "\n One of the ", + 1, + "a", + 2, + "href", + "https://bugs.python.org/issue23427#msg371028", + 3, + "motivating use cases", + -1, + 3, + " for ", + 1, + "code", + 3, + "sys.orig_argv", + -1, + 3, + " is that you can use it to spawn a new Python process with the same or modified command-line arguments as your current process.\n ", + -2, + 1, + "section", + 1, + "h3", + 2, + "id", + "future-annotations", + 3, + "\n Future Annotations", + 1, + "a", + 2, + "href", + "#future-annotations", + 2, + "title", + "Permanent link", + -2, + 1, + "p", + 1, + "a", + 2, + "href", + "https://realpython.com/python-type-checking/#annotations", + 3, + "Annotations", + -1, + 3, + " were introduced in Python 3 to give you a way to attach metadata to variables, function parameters, and return values. They are most commonly used to add type hints to your code.\n ", + -1, + 1, + "p", + 3, + "\n One challenge with annotations is that they must be valid Python code. For one thing, this makes it ", + 1, + "a", + 2, + "href", + "https://realpython.com/python37-new-features/#typing-enhancements", + 3, + "hard to type hint", + -1, + 3, + " recursive classes. ", + 1, + "a", + 2, + "href", + "https://www.python.org/dev/peps/pep-0563/", + 3, + "PEP 563", + -1, + 3, + " introduced ", + 1, + "a", + 2, + "href", + "https://realpython.com/python-news-april-2021/#pep-563-pep-649-and-the-future-of-python-type-annotations", + 3, + "postponed evaluation of annotations", + -1, + 3, + ", making it possible to annotate with names that haven’t yet been defined. Since Python 3.7, you can activate postponed evaluation of annotations with a ", + 1, + "a", + 2, + "href", + "https://docs.python.org/3/library/__future__.html", + 1, + "code", + 3, + "__future__", + -2, + 3, + " import:\n ", + -1, + 1, + "p", + 3, + "\n The intention was that postponed evaluation would become the default at some point in the future. After the ", + 1, + "a", + 2, + "href", + "https://pyfound.blogspot.com/2020/04/the-2020-python-language-summit.html", + 3, + "2020 Python Language Summit", + -1, + 3, + ", it was decided to make this happen in Python 3.10.\n ", + -1, + 1, + "p", + 3, + "\n However, after more testing, it became clear that postponed evaluation didn’t work well for projects that use annotations at runtime. Key people in the ", + 1, + "a", + 2, + "href", + "https://realpython.com/fastapi-python-web-apis/", + 3, + "FastAPI", + -1, + 3, + " and the ", + 1, + "a", + 2, + "href", + "https://pydantic-docs.helpmanual.io/", + 3, + "Pydantic", + -1, + 3, + " projects ", + 1, + "a", + 2, + "href", + "https://dev.to/tiangolo/the-future-of-fastapi-and-pydantic-is-bright-3pbm", + 3, + "voiced their concerns", + -1, + 3, + ". At the last minute, it was decided to reschedule these changes for Python 3.11.\n ", + -1, + 1, + "p", + 3, + "\n To ease the transition into future behavior, a few changes have been made in Python 3.10 as well. Most importantly, a new ", + 1, + "a", + 2, + "href", + "https://docs.python.org/3.10/library/inspect.html#inspect.get_annotations", + 1, + "code", + 3, + "inspect.get_annotations()", + -2, + 3, + " function has been added. You should call this to access annotations at runtime:\n ", + -1, + 1, + "p", + 3, + "\n Check out ", + 1, + "a", + 2, + "href", + "https://docs.python.org/3.10/howto/annotations.html", + 3, + "Annotations Best Practices", + -1, + 3, + " for details.\n ", + -3, + 1, + "section", + 1, + "h2", + 2, + "id", + "how-to-detect-python-310-at-runtime", + 3, + "\n How to Detect Python 3.10 at Runtime", + 1, + "a", + 2, + "href", + "#how-to-detect-python-310-at-runtime", + 2, + "title", + "Permanent link", + -2, + 1, + "p", + 3, + "\n Python 3.10 is the first version of Python with a two-digit minor version number. While this is mostly an interesting fun fact and an indication that Python 3 has been around for quite some time, it does also have some practical consequences.\n ", + -1, + 1, + "p", + 3, + "\n When your code needs to do something specific based on the version of Python at runtime, you’ve gotten away with doing a ", + 1, + "a", + 2, + "href", + "https://docs.python.org/3/reference/expressions.html#value-comparisons", + 3, + "lexicographical", + -1, + 3, + " comparison of version strings until now. While it’s never been good practice, it’s been possible to do the following:\n ", + -1, + 1, + "p", + 3, + "\n In Python 3.10, this code will raise ", + 1, + "code", + 3, + "SystemExit", + -1, + 3, + " and stop your program. This happens because, as strings, ", + 1, + "code", + 3, + "\"3.10\"", + -1, + 3, + " is less than ", + 1, + "code", + 3, + "\"3.6\"", + -1, + 3, + ".\n ", + -1, + 1, + "p", + 3, + "\n The correct way to compare version numbers is to use tuples of numbers:\n ", + -1, + 1, + "p", + 1, + "a", + 2, + "href", + "https://docs.python.org/3/library/sys.html#sys.version_info", + 1, + "code", + 3, + "sys.version_info", + -2, + 3, + " is a tuple object you can use for comparisons.\n ", + -1, + 1, + "p", + 3, + "\n If you’re doing these kinds of comparisons in your code, you should check your code with ", + 1, + "a", + 2, + "href", + "https://pypi.org/project/flake8-2020/", + 3, + "flake8-2020", + -1, + 3, + " to make sure you’re handling versions correctly:\n ", + -1, + 1, + "p", + 3, + "\n With the ", + 1, + "code", + 3, + "flake8-2020", + -1, + 3, + " extension activated, you’ll get a recommendation about replacing ", + 1, + "code", + 3, + "sys.version", + -1, + 3, + " with ", + 1, + "code", + 3, + "sys.version_info", + -1, + 3, + ".\n ", + -2, + 1, + "section", + 1, + "h2", + 2, + "id", + "so-should-you-upgrade-to-python-310", + 3, + "\n So, Should You Upgrade to Python 3.10?", + 1, + "a", + 2, + "href", + "#so-should-you-upgrade-to-python-310", + 2, + "title", + "Permanent link", + -2, + 1, + "p", + 3, + "\n You’ve now seen the coolest features of the newest and latest version of Python. The question now is whether you should upgrade to Python 3.10, and if yes, when you should do so. There are two different aspects to consider when thinking about upgrading to Python 3.10:\n ", + -1, + 1, + "ol", + 1, + "li", + 3, + "Should you upgrade your environment so that you ", + 1, + "strong", + 3, + "run your code", + -1, + 3, + " with the Python 3.10 interpreter?\n ", + -1, + 1, + "li", + 3, + "Should you ", + 1, + "strong", + 3, + "write your code", + -1, + 3, + " using the new Python 3.10 features?\n ", + -2, + 1, + "p", + 3, + "\n Clearly, if you want to test out structural pattern matching or any of the other cool new features you’ve read about here, you need Python 3.10. It’s possible to install the latest version side by side with your current Python version. A straightforward way to do this is to use an environment manager like ", + 1, + "a", + 2, + "href", + "https://realpython.com/intro-to-pyenv/", + 3, + "pyenv", + -1, + 3, + " or ", + 1, + "a", + 2, + "href", + "https://realpython.com/python-windows-machine-learning-setup/", + 3, + "Conda", + -1, + 3, + ". You can also ", + 1, + "a", + 2, + "href", + "https://realpython.com/python-versions-docker/", + 3, + "use Docker", + -1, + 3, + " to run Python 3.10 without installing it locally.\n ", + -1, + 1, + "p", + 3, + "\n Python 3.10 has been through about five months of beta testing, so there shouldn’t be any big issues with starting to use it for your own development. You may find that some of your dependencies don’t immediately have ", + 1, + "a", + 2, + "href", + "https://realpython.com/python-wheels/", + 3, + "wheels for Python 3.10", + -1, + 3, + " available, which makes them more cumbersome to install. But in general, using the newest Python for local development is fairly safe.\n ", + -1, + 1, + "p", + 3, + "\n As always, you should be careful before upgrading your production environment. Be vigilant about testing that your code runs well on the new version. In particular, you want to be on the lookout for features that are ", + 1, + "a", + 2, + "href", + "https://docs.python.org/3.10/whatsnew/3.10.html#deprecated", + 3, + "deprecated", + -1, + 3, + " or ", + 1, + "a", + 2, + "href", + "https://docs.python.org/3.10/whatsnew/3.10.html#removed", + 3, + "removed", + -1, + 3, + ".\n ", + -1, + 1, + "p", + 3, + "\n Whether you can start using the new features in your code or not depends on your user base and the environment where your code is running. If you can guarantee that Python 3.10 is available, then there’s no danger in using the new union type syntax or any other new feature.\n ", + -1, + 1, + "p", + 3, + "\n If you’re distributing an app or a library that’s used by others instead, you may want to be a bit more conservative. Currently, ", + 1, + "a", + 2, + "href", + "https://www.python.org/dev/peps/pep-0494", + 3, + "Python 3.6", + -1, + 3, + " is the oldest officially supported Python version. It reaches end-of-life in December 2021, after which ", + 1, + "a", + 2, + "href", + "https://www.python.org/dev/peps/pep-0537", + 3, + "Python 3.7", + -1, + 3, + " will be the minimum supported version.\n ", + -1, + 1, + "p", + 3, + "\n The documentation includes a useful guide about ", + 1, + "a", + 2, + "href", + "https://docs.python.org/3.10/whatsnew/3.10.html#porting-to-python-3-10", + 3, + "porting your code to Python 3.10", + -1, + 3, + ". Check it out for more details!\n ", + -2, + 1, + "section", + 1, + "h2", + 2, + "id", + "conclusion", + 3, + "\n Conclusion", + 1, + "a", + 2, + "href", + "#conclusion", + 2, + "title", + "Permanent link", + -2, + 1, + "p", + 3, + "\n The release of a new Python version is always worth celebrating. Even if you can’t start using the new features right away, they’ll become broadly available and part of your daily life within a few years.\n ", + -1, + 1, + "p", + 1, + "strong", + 3, + "In this tutorial, you’ve seen new features like:", + -2, + 1, + "ul", + 1, + "li", + 3, + "Friendlier ", + 1, + "strong", + 3, + "error messages", + -2, + 1, + "li", + 3, + "Powerful ", + 1, + "strong", + 3, + "structural pattern matching", + -2, + 1, + "li", + 1, + "strong", + 3, + "Type hint", + -1, + 3, + " improvements\n ", + -1, + 1, + "li", + 3, + "Safer ", + 1, + "strong", + 3, + "combination of sequences", + -2, + 1, + "li", + 3, + "New ", + 1, + "strong", + 3, + "statistics functions", + -3, + 1, + "p", + 3, + "\n For more Python 3.10 tips and a discussion with members of the ", + 1, + "em", + 3, + "Real Python", + -1, + 3, + " team, check out ", + 1, + "a", + 2, + "href", + "https://realpython.com/podcasts/rpp/81/", + 3, + "Real Python Podcast Episode #81", + -1, + 3, + ".\n ", + -1, + 1, + "p", + 3, + "\n Have fun trying out the new features! Share your experiences in the comments below.\n ", + -2, + 1, + "p", + 1, + "span", + 3, + " Watch Now", + -1, + 3, + " This tutorial has a related video course created by the Real Python team. Watch it together with the written tutorial to deepen your understanding: ", + 1, + "a", + 2, + "href", + "http://fakehost/courses/cool-new-features-python-310/", + 1, + "strong", + 3, + "Cool New Features in Python 3.10", + -4, + 1, + "div", + 1, + "p", + 3, + "\n Get a short ", + 3, + "&", + 3, + " sweet ", + 1, + "strong", + 3, + "Python Trick", + -1, + 3, + " delivered to your inbox every couple of days. No spam ever. Unsubscribe any time. Curated by the Real Python team.\n ", + -1, + 1, + "p", + 1, + "img", + 2, + "loading", + "lazy", + 2, + "src", + "http://fakehost/static/pytrick-dict-merge.4201a0125a5e.png", + 2, + "width", + "738", + 2, + "height", + "490", + 2, + "alt", + "Python Tricks Dictionary Merge", + -3, + 1, + "div", + 2, + "id", + "author", + 1, + "div", + 1, + "p", + 1, + "a", + 2, + "href", + "http://fakehost/team/gahjelle/", + 1, + "img", + 2, + "loading", + "lazy", + 2, + "src", + "https://robocrop.realpython.net/?url=https%3A//files.realpython.com/media/gahjelle.470149ee709e.jpg&w=800&h=800&mode=crop&sig=e9b761c6cf1359953014dba05554f5424eb116e1", + 2, + "srcset", + "https://robocrop.realpython.net/?url=https%3A//files.realpython.com/media/gahjelle.470149ee709e.jpg&w=200&h=200&mode=crop&sig=c6390201e73d3e09429d73da5bb29c17ab10403a 200w, https://robocrop.realpython.net/?url=https%3A//files.realpython.com/media/gahjelle.470149ee709e.jpg&w=266&h=266&mode=crop&sig=7df7c39b123c3f9d8a4597311d09c3bd947f5fac 266w, https://robocrop.realpython.net/?url=https%3A//files.realpython.com/media/gahjelle.470149ee709e.jpg&w=400&h=400&mode=crop&sig=fcea459ee24a7b320573cadee324cf75509dc1d6 400w, https://robocrop.realpython.net/?url=https%3A//files.realpython.com/media/gahjelle.470149ee709e.jpg&w=800&h=800&mode=crop&sig=e9b761c6cf1359953014dba05554f5424eb116e1 800w", + 2, + "sizes", + "(min-width: 580px) 154px, calc(33.08vw - 24px)", + 2, + "width", + "800", + 2, + "height", + "800", + 2, + "alt", + "Geir Arne Hjelle", + -2, + 1, + "a", + 2, + "href", + "http://fakehost/team/gahjelle/", + 1, + "img", + 2, + "loading", + "lazy", + 2, + "src", + "https://robocrop.realpython.net/?url=https%3A//files.realpython.com/media/gahjelle.470149ee709e.jpg&w=800&h=800&mode=crop&sig=e9b761c6cf1359953014dba05554f5424eb116e1", + 2, + "srcset", + "https://robocrop.realpython.net/?url=https%3A//files.realpython.com/media/gahjelle.470149ee709e.jpg&w=200&h=200&mode=crop&sig=c6390201e73d3e09429d73da5bb29c17ab10403a 200w, https://robocrop.realpython.net/?url=https%3A//files.realpython.com/media/gahjelle.470149ee709e.jpg&w=266&h=266&mode=crop&sig=7df7c39b123c3f9d8a4597311d09c3bd947f5fac 266w, https://robocrop.realpython.net/?url=https%3A//files.realpython.com/media/gahjelle.470149ee709e.jpg&w=400&h=400&mode=crop&sig=fcea459ee24a7b320573cadee324cf75509dc1d6 400w, https://robocrop.realpython.net/?url=https%3A//files.realpython.com/media/gahjelle.470149ee709e.jpg&w=800&h=800&mode=crop&sig=e9b761c6cf1359953014dba05554f5424eb116e1 800w", + 2, + "sizes", + "(min-width: 1200px) 140px, calc(-1.5vw + 137px)", + 2, + "width", + "800", + 2, + "height", + "800", + 2, + "alt", + "Geir Arne Hjelle", + -4, + 1, + "hr", + -1, + 1, + "div", + 1, + "p", + 1, + "em", + 3, + "Each tutorial at Real Python is created by a team of developers so that it meets our high quality standards. The team members who worked on this tutorial are:", + -2, + 1, + "div", + 1, + "p", + 1, + "a", + 2, + "href", + "http://fakehost/team/dbader/", + 1, + "img", + 2, + "loading", + "lazy", + 2, + "src", + "https://robocrop.realpython.net/?url=https%3A//files.realpython.com/media/daniel-square.d58bf4388750.jpg&w=1000&h=1000&mode=crop&sig=304f5f568993310d5e87b2ca3c504260c018effa", + 2, + "srcset", + "https://robocrop.realpython.net/?url=https%3A//files.realpython.com/media/daniel-square.d58bf4388750.jpg&w=250&h=250&mode=crop&sig=981634b4528584f2e9c7ee663477173599e1781a 250w, https://robocrop.realpython.net/?url=https%3A//files.realpython.com/media/daniel-square.d58bf4388750.jpg&w=333&h=333&mode=crop&sig=96005b865a7753e3bd47fd88dafab70a686b1224 333w, https://robocrop.realpython.net/?url=https%3A//files.realpython.com/media/daniel-square.d58bf4388750.jpg&w=500&h=500&mode=crop&sig=841025b97d25f05c1b90802032e477020462fe01 500w, https://robocrop.realpython.net/?url=https%3A//files.realpython.com/media/daniel-square.d58bf4388750.jpg&w=1000&h=1000&mode=crop&sig=304f5f568993310d5e87b2ca3c504260c018effa 1000w", + 2, + "sizes", + "(min-width: 1200px) 73px, (min-width: 780px) calc(-0.75vw + 69px), (min-width: 580px) 43px, calc(33.46vw - 64px)", + 2, + "width", + "1000", + 2, + "height", + "1000", + 2, + "alt", + "Dan Bader", + -3, + 1, + "p", + 1, + "a", + 2, + "href", + "http://fakehost/team/sparker/", + 1, + "img", + 2, + "loading", + "lazy", + 2, + "src", + "https://robocrop.realpython.net/?url=https%3A//files.realpython.com/media/profpic_sp.a008488b6af0.jpeg&w=800&h=800&mode=crop&sig=ec9f8b060e53c9b35339584491f286fd52c11452", + 2, + "srcset", + "https://robocrop.realpython.net/?url=https%3A//files.realpython.com/media/profpic_sp.a008488b6af0.jpeg&w=200&h=200&mode=crop&sig=ce5407d34a490d4daf29ad57e74fd3870dd28774 200w, https://robocrop.realpython.net/?url=https%3A//files.realpython.com/media/profpic_sp.a008488b6af0.jpeg&w=266&h=266&mode=crop&sig=f9684a52e8fabcc7dd3f88e96321df221ac1723a 266w, https://robocrop.realpython.net/?url=https%3A//files.realpython.com/media/profpic_sp.a008488b6af0.jpeg&w=400&h=400&mode=crop&sig=dfd70d0dde444f036864e5cc145df9b18fbae649 400w, https://robocrop.realpython.net/?url=https%3A//files.realpython.com/media/profpic_sp.a008488b6af0.jpeg&w=800&h=800&mode=crop&sig=ec9f8b060e53c9b35339584491f286fd52c11452 800w", + 2, + "sizes", + "(min-width: 1200px) 73px, (min-width: 780px) calc(-0.75vw + 69px), (min-width: 580px) 43px, calc(33.46vw - 64px)", + 2, + "width", + "800", + 2, + "height", + "800", + 2, + "alt", + "Sadie Parker", + -6, + 1, + "div", + 1, + "div", + 1, + "p", + 3, + "\n Master ", + 1, + "u", + 1, + "span", + 3, + "Real-World Python Skills", + -2, + 3, + " With Unlimited Access to Real", + 3, + "Python\n ", + -1, + 1, + "p", + 1, + "img", + 2, + "loading", + "lazy", + 2, + "src", + "http://fakehost/static/videos/lesson-locked.f5105cfd26db.svg", + 2, + "width", + "510", + 2, + "height", + "260", + -2, + 1, + "p", + 1, + "strong", + 3, + "Join us and get access to thousands of tutorials, hands-on video courses, and a community of expert", + 3, + "Pythonistas:", + -2, + 1, + "p", + 1, + "a", + 2, + "href", + "http://fakehost/account/join/?utm_source=rp_article_footer&utm_content=python310-new-features", + 3, + "Level Up Your Python Skills »", + -3, + 1, + "div", + 1, + "p", + 3, + "\n Master ", + 1, + "u", + 1, + "span", + 3, + "Real-World Python Skills", + -2, + 1, + "br", + -1, + 3, + "\n With Unlimited Access to Real", + 3, + "Python\n ", + -1, + 1, + "p", + 1, + "img", + 2, + "loading", + "lazy", + 2, + "src", + "http://fakehost/static/videos/lesson-locked.f5105cfd26db.svg", + 2, + "width", + "510", + 2, + "height", + "260", + -2, + 1, + "p", + 1, + "strong", + 3, + "Join us and get access to thousands of tutorials, hands-on video courses, and a community of expert Pythonistas:", + -2, + 1, + "p", + 1, + "a", + 2, + "href", + "http://fakehost/account/join/?utm_source=rp_article_footer&utm_content=python310-new-features", + 3, + "Level Up Your Python Skills »", + -7 + ], + "readerable": true +} diff --git a/packages/readabilityjs/test/test-pages/realpython/expected.html b/packages/readabilityjs/test/test-pages/realpython/expected.html new file mode 100644 index 000000000..0fe34c658 --- /dev/null +++ b/packages/readabilityjs/test/test-pages/realpython/expected.html @@ -0,0 +1,540 @@ +
+
+
+ Python 3.10: Cool New Features for You to Try +
+
+

+ Watch Now This tutorial has a related video course created by the Real Python team. Watch it together with the written tutorial to deepen your understanding: Cool New Features in Python 3.10 +

+

+ Python 3.10 is out! Volunteers have been working on the new version since May 2020 to bring you a better, faster, and more secure Python. As of October 4, 2021, the first official version is available. +

+

Each new version of Python brings a host of changes. You can read about all of them in the documentation. Here, you’ll get to learn about the coolest new features.

+

+ In this tutorial, you’ll learn about: +

+
    +
  • Debugging with more helpful and precise error messages +
    • +
  • Using structural pattern matching to work with data structures
    • +
  • Adding more readable and more specific type hints +
    • +
  • Checking the length of sequences when using zip() +
    • +
  • Calculating multivariable statistics +
    • +
+

To try out the new features yourself, you need to run Python 3.10. You can get it from the Python homepage. Alternatively, you can use Docker with the latest Python image.

+
+

Better Error Messages +

+

Python is often lauded for being a user-friendly programming language. While this is true, there are certain parts of Python that could be friendlier. Python 3.10 comes with a host of more precise and constructive error messages. In this section, you’ll see some of the newest improvements. The full list is available in the documentation.

+

Think back to writing your first Hello World program in Python:

+

Maybe you created a file, added the famous call to print(), and saved it as hello.py. You then ran the program, eager to call yourself a proper Pythonista. However, something went wrong:

+

There was a SyntaxError in the code. EOL, what does that even mean? You went back to your code, and after a bit of staring and searching, you realized that there was a missing quotation mark at the end of your string.

+

One of the more impactful improvements in Python 3.10 is better and more precise error messages for many common issues. If you run your buggy Hello World in Python 3.10, you’ll get a bit more help than in earlier versions of Python:

+

The error message is still a bit technical, but gone is the mysterious EOL. Instead, the message tells you that you need to terminate your string! There are similar improvements to many different error messages, as you’ll see below.

+

A SyntaxError is an error raised when your code is parsed, before it even starts to execute. Syntax errors can be tricky to debug because the interpreter provides imprecise or sometimes even misleading error messages. The following code is missing a curly brace to terminate the dictionary:

+

The missing closing curly brace that should have been on line 7 is an error. If you run this code with Python 3.9 or earlier, you’ll see the following error message:

+

The error message highlights line 8, but there are no syntactical problems in line 8! If you’ve experienced your share of syntax errors in Python, you might already know that the trick is to look at the lines before the one Python complains about. In this case, you’re looking for the missing closing brace on line 7.

+

In Python 3.10, the same code shows a much more helpful and precise error message:

+

This points you straight to the offending dictionary and allows you to fix the issue in no time.

+

There are a few other ways to mess up dictionary syntax. A typical one is forgetting a comma after one of the items:

+

In this code, a comma is missing at the end of line 4. Python 3.10 gives you a clear suggestion on how to fix your code:

+

You can add the missing comma and have your code back up and running in no time.

+

Another common mistake is using the assignment operator (=) instead of the equality comparison operator (==) when you’re comparing values. Previously, this would just cause another invalid syntax message. In the newest version of Python, you get some more advice:

+

The parser suggests that you maybe meant to use a comparison operator or an assignment expression operator instead.

+

Take note of another nifty improvement in Python 3.10 error messages. The last two examples show how carets (^^^) highlight the whole offending expression. Previously, a single caret symbol (^) indicated just an approximate location.

+

The final error message improvement that you’ll play with for now is that attribute and name errors can now offer suggestions if you misspell an attribute or a name:

+

Note that the suggestions work for both built-in names and names that you define yourself, although they may not be available in all environments. If you like these kinds of suggestions, check out BetterErrorMessages, which offers similar suggestions in even more contexts.

+

The improvements you’ve seen in this section are just some of the many error messages that have gotten a face-lift. The new Python will be even more user-friendly than before, and hopefully, the new error messages will save you both time and frustration going forward.

+
+
+

Structural Pattern Matching +

+

The biggest new feature in Python 3.10, probably both in terms of controversy and potential impact, is structural pattern matching. Its introduction has sometimes been referred to as switch ... case coming to Python, but you’ll see that structural pattern matching is much more powerful than that.

+

You’ll see three different examples that together highlight why this feature is called structural pattern matching and show you how you can use this new feature:

+
    +
  1. Detecting and deconstructing different structures in your data
    2. +
  2. Using different kinds of patterns +
    3. +
  3. + Matching literal patterns +
    4. +
+

Structural pattern matching is a comprehensive addition to the Python language. To give you a taste of how you can take advantage of it in your own projects, the next three subsections will dive into some of the details. You’ll also see some links that can help you explore in even more depth if you want.

+
+

Deconstructing Data Structures +

+

At its core, structural pattern matching is about defining patterns to which your data structures can be matched. In this section, you’ll study a practical example where you’ll work with data that are structured differently, even though the meaning is the same. You’ll define several patterns, and depending on which pattern matches your data, you’ll process your data appropriately.

+

This section will be a bit light on explanations of the possible patterns. Instead, it will try to give you an impression of the possibilities. The next section will step back and explain the patterns in more detail.

+

Time to match your first pattern! The following example uses a match ... case block to find the first name of a user by extracting it from a user data structure:

+

You can see structural pattern matching at work in the highlighted lines. user is a small dictionary with user information. The case line specifies a pattern that user is matched against. In this case, you’re looking for a dictionary with a "name" key whose value is a new dictionary. This nested dictionary has a key called "first". The corresponding value is bound to the variable first_name.

+

For a practical example, say that you’re processing user data where the underlying data model changes over time. Therefore, you need to be able to process different versions of the same data.

+

In the next example, you’ll use data from randomuser.me. This is a great API for generating random user data that you can use during testing and development. The API is also an example of an API that has changed over time. You can still access the old versions of the API.

+

You may expand the collapsed section below to see how you can use requests to obtain different versions of the user data using the API:

+
+

You can get a random user from the API using requests as follows:

+

+ get_user() gets one random user in JSON format. Note the version parameter. The structure of the returned data has changed quite a bit between earlier versions like "1.1" and the current version "1.3", but in each case, the actual user data are contained in a list inside the "results" array. The function returns the first—and only—user in this list. +

+

At the time of writing, the latest version of the API is 1.3 and the data has the following structure:

+

One of the members that changed between different versions is "dob", the date of birth. Note that in version 1.3, this is a JSON object with two members, "date" and "age".

+

Compare the result above with a version 1.1 random user:

+

Observe that in this older format, the value of the "dob" member is a plain string.

+
+

In this example, you’ll work with the information about the date of birth (dob) for each user. The structure of these data has changed between different versions of the Random User API:

+

Note that in version 1.1, the date of birth is represented as a simple string, while in version 1.3, it’s a JSON object with two members: "date" and "age". Say that you want to find the age of a user. Depending on the structure of your data, you’d either need to calculate the age based on the date of birth or look up the age if it’s already available.

+

Traditionally, you would detect the structure of the data with an if test, maybe based on the type of the "dob" field. You can approach this differently in Python 3.10. Now, you can use structural pattern matching instead:

+

The match ... case construct is new in Python 3.10 and is how you perform structural pattern matching. You start with a match statement that specifies what you want to match. In this example, that’s the user data structure.

+

One or several case statements follow match. Each case describes one pattern, and the indented block beneath it says what should happen if there’s a match. In this example:

+
    +
  • +

    + Line 8 matches a dictionary with a "dob" key whose value is another dictionary with an integer (int) item named "age". The name age captures its value. +

    +
    • +
  • +

    + Line 10 matches any dictionary with a "dob" key. The name dob captures its value. +

    +
    • +
+

One important feature of pattern matching is that at most one pattern will be matched. Since the pattern on line 10 matches any dictionary with "dob", it’s important that the more specific pattern on line 8 comes first.

+

Before looking closer at the details of the patterns and how they work, try calling get_age() with different data structures to see the result:

+

Your code can calculate the age correctly for both versions of the user data, which have different dates of birth.

+

Look closer at those patterns. The first pattern, {"dob": {"age": int(age)}}, matches version 1.3 of the user data:

+

The first pattern is a nested pattern. The outer curly braces say that a dictionary with the key "dob" is required. The corresponding value should be a dictionary. This nested dictionary must match the subpattern {"age": int(age)}. In other words, it needs to have an "age" key with an integer value. That value is bound to the name age.

+

The second pattern, {"dob": dob}, matches the older version 1.1 of the user data:

+

This second pattern is a simpler pattern than the first one. Again, the curly braces indicate that it will match a dictionary. However, any dictionary with a "dob" key is matched because there are no other restrictions specified. The value of that key is bound to the name dob.

+

The main takeaway is that you can describe the structure of your data using mostly familiar notation. One striking change, though, is that you can use names like dob and age, which aren’t yet defined. Instead, values from your data are bound to these names when a pattern matches.

+

You’ve explored some of the power of structural pattern matching in this example. In the next section, you’ll dive a bit more into the details.

+
+
+

Using Different Kinds of Patterns +

+

You’ve seen an example of how you can use patterns to effectively unravel complicated data structures. Now, you’ll take a step back and look at the building blocks that make up this new feature. Many things come together to make it work. In fact, there are three Python Enhancement Proposals (PEPs) that describe structural pattern matching:

+
    +
  1. + PEP 634: Specification +
    2. +
  2. + PEP 635: Motivation and Rationale +
    3. +
  3. + PEP 636: Tutorial +
    4. +
+

These documents give you a lot of background and detail if you’re interested in a deeper dive than what follows.

+

Patterns are at the center of structural pattern matching. In this section, you’ll learn about some of the different kinds of patterns that exist:

+
    +
  • + Mapping patterns match mapping structures like dictionaries. +
    • +
  • + Sequence patterns match sequence structures like tuples and lists. +
    • +
  • + Capture patterns bind values to names. +
    • +
  • + AS patterns bind the value of subpatterns to names. +
    • +
  • + OR patterns match one of several different subpatterns. +
    • +
  • + Wildcard patterns match anything. +
    • +
  • + Class patterns match class structures. +
    • +
  • + Value patterns match values stored in attributes. +
    • +
  • + Literal patterns match literal values. +
    • +
+

You already used several of them in the example in the previous section. In particular, you used mapping patterns to unravel data stored in dictionaries. In this section, you’ll learn more about how some of these work. All the details are available in the PEPs mentioned above.

+

A capture pattern is used to capture a match to a pattern and bind it to a name. Consider the following recursive function that sums a list of numbers:

+

The first case on line 3 matches the empty list and returns 0 as its sum. The second case on line 5 uses a sequence pattern with two capture patterns to match lists with one or more elements. The first element in the list is captured and bound to the name first. The second capture pattern, *rest, uses unpacking syntax to match any number of elements. rest will bind to a list containing all elements of numbers except the first one.

+

+ sum_list() calculates the sum of a list of numbers by recursively adding the first number in the list and the sum of the rest of the numbers. You can use it as follows: +

+

The sum of 4 + 5 + 9 + 4 is correctly calculated to be 22. As an exercise for yourself, you can try to trace the recursive calls to sum_list() to make sure you understand how the code sums the whole list.

+

+ sum_list() handles summing up a list of numbers. Observe what happens if you try to sum anything that isn’t a list: +

+

Passing a string or a number to sum_list() returns None. This occurs because none of the patterns match, and the execution continues after the match block. That happens to be the end of the function, so sum_list() implicitly returns None.

+

Often, though, you want to be alerted about failed matches. You can add a catchall pattern as the final case that handles this by raising an error, for example. You can use the underscore (_) as a wildcard pattern that matches anything without binding it to a name. You can add some error handling to sum_list() as follows:

+

The final case will match anything that doesn’t match the first two patterns. This will raise a descriptive error, for instance, if you try to calculate sum_list(4594). This is useful when you need to alert your users that some input was not matched as expected.

+

Your patterns are still not foolproof, though. Consider what happens if you try to sum a list of strings:

+

The base case returns 0, so therefore the summing only works for types that you can add with numbers. Python doesn’t know how to add numbers and text strings together. You can restrict your pattern to only match integers using a class pattern:

+

Adding int() around first makes sure that the pattern only matches if the value is an integer. This might be too restrictive, though. Your function should be able to sum both integers and floating-point numbers, so how can you allow this in your pattern?

+

To check whether at least one out of several subpatterns match, you can use an OR pattern. OR patterns consist of two or more subpatterns, and the pattern matches if at least one of the subpatterns does. You can use this to match when the first element is either of type int or type float:

+

You use the pipe symbol (|) to separate the subpatterns in an OR pattern. Your function now allows summing a list of floating-point numbers:

+

There’s a lot of power and flexibility within structural pattern matching, even more than what you’ve seen so far. Some things that aren’t covered in this overview are:

+ +

If you’re interested, have a look in the documentation to learn more about these features as well. In the next section, you’ll learn about literal patterns and value patterns.

+
+
+

Matching Literal Patterns +

+

A literal pattern is a pattern that matches a literal object like an explicit string or number. In a sense, this is the most basic kind of pattern and allows you to emulate switch ... case statements seen in other languages. The following example matches a specific name:

+

The first case matches the literal string "Guido". In this case, you use _ as a wildcard to print a generic greeting whenever name is not "Guido". Such literal patterns can sometimes take the place of if ... elif ... else constructs and can play the same role that switch ... case does in some other languages.

+

One limitation with structural pattern matching is that you can’t directly match values stored in variables. Say that you’ve defined bdfl = "Guido". A pattern like case bdfl: will not match "Guido". Instead, this will be interpreted as a capture pattern that matches anything and binds that value to bdfl, effectively overwriting the old value.

+

You can, however, use a value pattern to match stored values. A value pattern looks a bit like a capture pattern but uses a previously defined dotted name that holds the value that will be matched against.

+

You can, for example, use an enumeration to create such dotted names:

+

The first case now uses a value pattern to match Pythonista.BDFL, which is "Guido". Note that you can use any dotted name in a value pattern. You could, for example, have used a regular class or a module instead of the enumeration.

+

To see a bigger example of how to use literal patterns, consider the game of FizzBuzz. This is a counting game where you should replace some numbers with words according to the following rules:

+
    +
  • You replace numbers divisible by 3 with fizz.
    • +
  • You replace numbers divisible by 5 with buzz.
    • +
  • You replace numbers divisible by both 3 and 5 with fizzbuzz.
    • +
+

FizzBuzz is sometimes used to introduce conditionals in programming education and as a screening problem in interviews. Even though a solution is quite straightforward, Joel Grus has written a full book about different ways to program the game.

+

A typical solution in Python will use if ... elif ... else as follows:

+

The % operator calculates the modulus, which you can use to test divisibility. Namely, if a modulus b is 0 for two numbers a and b, then a is divisible by b.

+

In fizzbuzz(), you calculate number % 3 and number % 5, which you then use to test for divisibility with 3 and 5. Note that you must do the test for divisibility with both 3 and 5 first. If not, numbers that are divisible by both 3 and 5 will be covered by either the "fizz" or the "buzz" cases instead.

+

You can check that your implementation gives the expected result:

+

You can confirm for yourself that 3 is divisible by 3, 65 is divisible by 5, and 15 is divisible by both 3 and 5, while 14 and 92 aren’t divisible by either 3 or 5.

+

An if ... elif ... else structure where you’re comparing one or a few variables several times over is quite straightforward to rewrite using pattern matching instead. For example, you can do the following:

+

You match on both mod_3 and mod_5. Each case pattern then matches either the literal number 0 or the wildcard _ on the corresponding values.

+

Compare and contrast this version with the previous one. Note how the pattern (0, 0) corresponds to the test mod_3 == 0 and mod_5 == 0, while (0, _) corresponds to mod_3 == 0.

+

As you saw earlier, you can use an OR pattern to match on several different patterns. For example, since mod_3 can only take the values 0, 1, and 2, you can replace case (_, 0) with case (1, 0) | (2, 0). Remember that (0, 0) has already been covered.

+

The Python core developers have consciously chosen not to include switch ... case statements in the language earlier. However, there are some third-party packages that do, like switchlang, which adds a switch command that also works on earlier versions of Python.

+
+
+
+

Type Unions, Aliases, and Guards +

+

Reliably, each new Python release brings some improvements to the static typing system. Python 3.10 is no exception. In fact, four different PEPs about typing accompany this new release:

+
    +
  1. + PEP 604: Allow writing union types as X | Y +
    2. +
  2. + PEP 613: Explicit Type Aliases +
    3. +
  3. + PEP 647: User-Defined Type Guards +
    4. +
  4. + PEP 612: Parameter Specification Variables +
    5. +
+

PEP 604 will probably be the most widely used of these changes going forward, but you’ll get a brief overview of each of the features in this section.

+

You can use union types to declare that a variable can have one of several different types. For example, you’ve been able to type hint a function calculating the mean of a list of numbers, floats, or integers as follows:

+

The annotation List[Union[float, int]] means that numbers should be a list where each element is either a floating-point number or an integer. This works well, but the notation is a bit verbose. Also, you need to import both List and Union from typing.

+

In Python 3.10, you can replace Union[float, int] with the more succinct float | int. Combine this with the ability to use list instead of typing.List in type hints, which Python 3.9 introduced. You can then simplify your code while keeping all the type information:

+

The annotation of numbers is easier to read now, and as an added bonus, you didn’t need to import anything from typing.

+

A special case of union types is when a variable can have either a specific type or be None. You can annotate such optional types either as Union[None, T] or, equivalently, Optional[T] for some type T. There is no new, special syntax for optional types, but you can use the new union syntax to avoid importing typing.Optional:

+

In this example, address is allowed to be either None or a string.

+

You can also use the new union syntax at runtime in isinstance() or issubclass() tests:

+

Traditionally, you’ve used tuples to test for several types at once—for example, (str, int) instead of str | int. This old syntax will still work.

+

+ Type aliases allow you to quickly define new aliases that can stand in for more complicated type declarations. For example, say that you’re representing a playing card using a tuple of suit and rank strings and a deck of cards by a list of such playing card tuples. A deck of cards is then type hinted as list[tuple[str, str]]. +

+

To simplify type annotation, you define type aliases as follows:

+

This usually works okay. However, it’s often not possible for the type checker to know whether such a statement is a type alias or just the definition of a regular global variable. To help the type checker—or really, help the type checker help you—you can now explicitly annotate type aliases:

+

Adding the TypeAlias annotation clarifies the intention, both to a type checker and to anyone reading your code.

+

+ Type guards are used to narrow down union types. The following function takes in either a string or None but always returns a tuple of strings representing a playing card: +

+

The highlighted line works as a type guard, and static type checkers are able to realize that suit is necessarily a string when it’s returned.

+

Currently, the type checkers can only use a few different constructs to narrow down union types in this way. With the new typing.TypeGuard, you can annotate custom functions that can be used to narrow down union types:

+

+ is_deck_of_cards() should return True or False depending on whether obj represents a Deck object or not. You can then use your guard function, and the type checker will be able to narrow down the types correctly: +

+

Inside of the if block, the type checker knows that card_or_deck is, in fact, of the type Deck. See PEP 647 for more details.

+

The final new typing feature is Parameter Specification Variables, which is related to type variables. Consider the definition of a decorator. In general, it looks something like the following:

+

The annotations mean that the function returned by the decorator is a callable with some parameters and the same return type, R, as the function passed into the decorator. The ellipsis (...) in the function header correctly allows any number of parameters, and each of those parameters can be of any type. However, there’s no validation that the returned callable has the same parameters as the function that was passed in. In practice, this means that type checkers aren’t able to check decorated functions properly.

+

Unfortunately, you can’t use TypeVar for the parameters because you don’t know how many parameters the function will have. In Python 3.10, you’ll have access to ParamSpec in order to type hint these kinds of callables properly. ParamSpec works similarly to TypeVar but stands in for several parameters at once. You can rewrite your decorator as follows to take advantage of ParamSpec:

+

Note that you also use P when you annotate wrapper(). You can also use the new typing.Concatenate to add types to ParamSpec. See the documentation and PEP 612 for details and examples.

+
+
+

Stricter Zipping of Sequences +

+

+ zip() is a built-in function in Python that can combine elements from several sequences. Python 3.10 introduces the new strict parameter, which adds a runtime test to check that all sequences being zipped have the same length. +

+

As an example, consider the following table of Lego sets:

+

One way to represent these data in plain Python would be with each column as a list. It could look something like this:

+

Note that you have three independent lists, but there’s an implicit correspondence between their elements. The first name ("Louvre"), the first set number ("21024"), and the first number of pieces (695) all describe the first Lego set.

+

+ zip() can be used to iterate over these three lists in parallel: +

+

Note how each line collects information from all three lists and shows information about one particular set. This is a very common pattern that’s used in a lot of different Python code, including in the standard library.

+

You can also add list() to collect the contents of all three lists in a single, nested list of tuples:

+

Note how the nested list closely resembles the original table.

+

The dark side of using zip() is that it’s quite easy to introduce a subtle bug that can be hard to discover. Note what happens if there’s a missing item in one of your lists:

+

All the information about the New York City set disappeared! Additionally, the set numbers for Saturn V and Millennium Falcon are wrong. If your datasets are bigger, these kinds of errors can be very hard to discover. And even when you observe that something’s wrong, it’s not always easy to diagnose and fix.

+

The issue is that you assumed that the three lists have the same number of elements and that the information is in the same order in each list. After set_numbers gets corrupted, this assumption is no longer true.

+

+ PEP 618 introduces a new strict keyword parameter to zip() that you can use to confirm all sequences have the same length. In your example, it would raise an error alerting you to the corrupted list: +

+

When the iteration reaches the New York City Lego set, the second argument set_numbers is already exhausted, while there are still elements left in the first argument names. Instead of silently giving the wrong result, your code fails with an error, and you can take action to find and fix the mistake.

+

There are use cases when you want to combine sequences of unequal length. Expand the box below to see how zip() and itertools.zip_longest() handle these:

+
+

The following idiom divides the Lego sets into pairs:

+

There are five sets, a number that doesn’t divide evenly into pairs. In this case, the default behavior of zip(), where the last element is dropped, might make sense. You could use strict=True here as well, but that would raise an error when your list can’t be split into pairs. A third option, which could be the best in this case, is to use zip_longest() from the itertools standard library.

+

As the name suggests, zip_longest() combines sequences until the longest sequence is exhausted. If you use zip_longest() to divide the Lego sets, it becomes more explicit that New York City doesn’t have any pairing:

+

Note that 'NYC' shows up in the last tuple together with an empty string. You can control what’s filled in for missing values with the fillvalue parameter.

+
+

While strict is not really adding any new functionality to zip(), it can help you avoid those hard-to-find bugs.

+
+
+

New Functions in the statistics Module +

+

The statistics module was added to the standard library all the way back in 2014 with the release of Python 3.4. The intent of statistics is to make statistical calculations at the level of graphing calculators available in Python.

+

Python 3.10 adds a few multivariable functions to statistics:

+
    +
  • + correlation() to calculate Pearson’s correlation coefficient for two variables +
    • +
  • + covariance() to calculate sample covariance for two variables +
    • +
  • + linear_regression() to calculate the slope and intercept in a linear regression +
    • +
+

You can use each function to describe a certain aspect of the relationship between two variables. As an example, say that you have data from a set of blog posts—the number of words in each blog post and the number of views each post has had over some time period:

+

You now want to investigate whether there’s any (linear) relationship between the number of words and number of views. In Python 3.10, you can calculate the correlation between words and views with the new correlation() function:

+

The correlation between two variables is always a number between -1 and 1. If it’s close to 0, then there’s little correspondence between them, while a correlation close to -1 or 1 indicates that the behaviors of the two variables tend to follow each other. In this example, a correlation of 0.45 indicates that there’s a tendency for posts with more words to have more views, although it’s not a strong connection.

+

You can also calculate the covariance between words and views. The covariance is another measure of the joint variability between two variables. You can calculate it with covariance():

+

In contrast to correlation, covariance is an absolute measure. It should be interpreted in the context of the variability within the variables themselves. In fact, you can normalize the covariance by the standard deviation of each variable to recover Pearson’s correlation coefficient:

+

Note that this matches your earlier correlation coefficient exactly.

+

A third way of looking at the linear correspondence between the two variables is through simple linear regression. You do the linear regression by calculating two numbers, slope and intercept, so that the (squared) error is minimized in the approximation number of views = slope × number of words + intercept.

+

In Python 3.10, you can use linear_regression():

+

Based on this regression, a post with 10,074 words could expect about 0.2424 × 10074 + 1104 = 3546 views. However, as you saw earlier, the correlation between the number of words and the number of views is quite weak. Therefore, you shouldn’t expect this prediction to be very accurate.

+

The LinearRegression object is a named tuple. This means that you can unpack the slope and intercept directly:

+

Here, you use slope and intercept to predict the number of views on a blog post with 10,074 words.

+

You still want to use some of the more advanced packages like pandas and statsmodels if you do a lot of statistical analysis. With the new additions to statistics in Python 3.10, however, you have the chance to do basic analysis more easily without bringing in third-party dependencies.

+
+
+

Other Pretty Cool Features +

+

So far, you’ve seen the biggest and most impactful new features in Python 3.10. In this section, you’ll get a glimpse of a few of the other changes that the new version brings along. If you’re curious about all the changes made for this new version, check out the documentation.

+
+

Default Text Encodings +

+

When you open a text file, the default encoding used to interpret the characters is system dependent. In particular, locale.getpreferredencoding() is used. On Mac and Linux, this usually returns "UTF-8", while the result on Windows is more varied.

+

You should therefore always specify an encoding when you attempt to open a text file:

+

If you don’t explicitly specify an encoding, the preferred locale encoding is used, and you could experience that a file that can be read on one computer fails to open on another.

+

Python 3.7 introduced UTF-8 mode, which allows you to force your programs to use UTF-8 encoding independent of the locale encoding. You can enable UTF-8 mode by giving the -X utf8 command-line option to the python executable or by setting the PYTHONUTF8 environment variable.

+

In Python 3.10, you can activate a warning that will tell you when a text file is opened without a specified encoding. Consider the following script, which doesn’t specify an encoding:

+

The program will echo one or more text files back to the console, but with each line reversed. Run the program on itself with the encoding warning enabled:

+

Note the EncodingWarning printed to the console. The command-line option -X warn_default_encoding activates it. The warning will disappear if you specify an encoding—for example, encoding="utf-8"—when you open the file.

+

There are times when you want to use the user-defined local encoding. You can still do so by explicitly using encoding="locale". However, it’s recommended to use UTF-8 whenever possible. You can check out PEP 597 for more information.

+
+
+

Asynchronous Iteration +

+

+ Asynchronous programming is a powerful programming paradigm that’s been available in Python since version 3.5. You can recognize an asynchronous program by its use of the async keyword or special methods that start with .__a like .__aiter__() or .__aenter__(). +

+

In Python 3.10, two new asynchronous built-in functions are added: aiter() and anext(). In practice, these functions call the .__aiter__() and .__anext__() special methods—analogous to the regular iter() and next()—so no new functionality is added. These are convenience functions that make your code more readable.

+

In other words, in the newest version of Python, the following statements—where things is an asynchronous iterable—are equivalent:

+

In either case, it ends up as an asynchronous iterator. Expand the following box to see a complete example using aiter() and anext():

+
+

The following program counts the number of lines in several files. In practice, you use Python’s ability to iterate over files to count the number of lines. The script uses asynchronous iteration in order to handle several files concurrently.

+

Note that you need to install the third-party aiofiles package with pip before running this code:

+

+ asyncio is used to create and run one asynchronous task per filename. count_lines() opens one file asynchronously and iterates through it using aiter() and anext() in order to count the number of lines. +

+
+

See PEP 525 to learn more about asynchronous iteration.

+
+
+

Context Manager Syntax +

+

+ Context managers are great for managing resources in your programs. Until recently, though, their syntax has included an uncommon wart. You haven’t been allowed to use parentheses to break long with statements like this: +

+

In earlier versions of Python, this causes an invalid syntax error message. Instead, you need to use a backslash (\) if you want to control where you break your lines:

+

While explicit line continuation with backslashes is possible in Python, PEP 8 discourages it. The Black formatting tool avoids backslashes completely.

+

In Python 3.10, you’re now allowed to add parentheses around with statements to your heart’s content. Especially if you’re employing several context managers at once, like in the example above, this can help improve the readability of your code. Python’s documentation shows a few other possibilities with this new syntax.

+

One small fun fact: parenthesized with statements actually work in version 3.9 of CPython. Their implementation came almost for free with the introduction of the PEG parser in Python 3.9. The reason that this is called a Python 3.10 feature is that using the PEG parser is voluntary in Python 3.9, while Python 3.9, with the old LL(1) parser, doesn’t support parenthesized with statements.

+
+
+

Modern and Secure SSL +

+

Security can be challenging! A good rule of thumb is to avoid rolling your own security algorithms and instead rely on established packages.

+

Python uses OpenSSL for different cryptographic features that are exposed in the hashlib, hmac, and ssl standard library modules. Your system can manage OpenSSL, or a Python installer can include OpenSSL.

+

Python 3.9 supports using any of the OpenSSL versions 1.0.2 LTS, 1.1.0, and 1.1.1 LTS. Both OpenSSL 1.0.2 LTS and OpenSSL 1.1.0 are past their lifetime, so Python 3.10 will only support OpenSSL 1.1.1 LTS, as described in the following table:

+
+ + + + + + + + + + + + + + + + + + + + + + + + + + +
Open SSL version Python 3.9 Python 3.10 End-of-life
1.0.2 LTS ✔ ✖
1.1.0 ✔ ✖
1.1.1 LTS ✔ ✔
+
+

This end of support for older versions will only affect you if you need to upgrade the system Python on an older operating system. If you use macOS or Windows, or if you install Python from python.org or use (Ana)Conda, you’ll see no change.

+

However, Ubuntu 18.04 LTS uses OpenSSL 1.1.0, while Red Hat Enterprise Linux (RHEL) 7 and CentOS 7 both use OpenSSL 1.0.2 LTS. If you need to run Python 3.10 on these systems, you should look at installing it yourself using either the python.org or Conda installer.

+

Dropping support for older versions of OpenSSL will make Python more secure. It’ll also help the Python developers in that code will be easier to maintain. Ultimately, this helps you because your Python experience will be more robust. See PEP 644 for more details.

+
+
+

More Information About Your Python Interpreter +

+

The sys module contains a lot of information about your system, the current Python runtime, and the script currently being executed. You can, for example, inquire about the paths where Python looks for modules with sys.path and see all modules that have been imported in the current session with sys.modules.

+

In Python 3.10, sys has two new attributes. First, you can now get a list of the names of all modules in the standard library:

+

Here, you can see that there are around 300 modules in the standard library, several of which start with the letter z. Note that only top-level modules and packages are listed. Subpackages like importlib.metadata don’t get a separate entry.

+

You will probably not be using sys.stdlib_module_names all that often. Still, the list ties in nicely with similar introspection features like keyword.kwlist and sys.builtin_module_names.

+

One possible use case for the new attribute is to identify which of the currently imported modules are third-party dependencies:

+

You find the imported top-level modules by looking at names in sys.modules that don’t have a dot in their name. By comparing them to the standard library module names, you find that numpy, dateutil, and pandas are some of the imported third-party modules in this example.

+

The other new attribute is sys.orig_argv. This is related to sys.argv, which holds the command-line arguments given to your program when it was started. In contrast, sys.orig_argv lists the command-line arguments passed to the python executable itself. Consider the following example:

+

This script echoes back the orig_argv and argv lists. Run it to see how the information is captured:

+

Essentially, all arguments—including the name of the Python executable—end up in orig_argv. This is in contrast to argv, which only contains the arguments that aren’t handled by python itself.

+

Again, this is not a feature that you’ll use a lot. If your program needs to concern itself with how it’s being run, you’re usually better off relying on information that’s already exposed instead of trying to parse this list. For example, you can choose to use the strict zip() mode only when your script is not running with the optimized flag, -O, like this:

+

The __debug__ flag is set when the interpreter starts. It’ll be False if you’re running python with -O or -OO specified, and True otherwise. Using __debug__ is usually preferable to "-O" not in sys.orig_argv or some similar construct.

+

One of the motivating use cases for sys.orig_argv is that you can use it to spawn a new Python process with the same or modified command-line arguments as your current process.

+
+
+

Future Annotations +

+

+ Annotations were introduced in Python 3 to give you a way to attach metadata to variables, function parameters, and return values. They are most commonly used to add type hints to your code. +

+

One challenge with annotations is that they must be valid Python code. For one thing, this makes it hard to type hint recursive classes. PEP 563 introduced postponed evaluation of annotations, making it possible to annotate with names that haven’t yet been defined. Since Python 3.7, you can activate postponed evaluation of annotations with a __future__ import:

+

The intention was that postponed evaluation would become the default at some point in the future. After the 2020 Python Language Summit, it was decided to make this happen in Python 3.10.

+

However, after more testing, it became clear that postponed evaluation didn’t work well for projects that use annotations at runtime. Key people in the FastAPI and the Pydantic projects voiced their concerns. At the last minute, it was decided to reschedule these changes for Python 3.11.

+

To ease the transition into future behavior, a few changes have been made in Python 3.10 as well. Most importantly, a new inspect.get_annotations() function has been added. You should call this to access annotations at runtime:

+

Check out Annotations Best Practices for details.

+
+
+
+

How to Detect Python 3.10 at Runtime +

+

Python 3.10 is the first version of Python with a two-digit minor version number. While this is mostly an interesting fun fact and an indication that Python 3 has been around for quite some time, it does also have some practical consequences.

+

When your code needs to do something specific based on the version of Python at runtime, you’ve gotten away with doing a lexicographical comparison of version strings until now. While it’s never been good practice, it’s been possible to do the following:

+

In Python 3.10, this code will raise SystemExit and stop your program. This happens because, as strings, "3.10" is less than "3.6".

+

The correct way to compare version numbers is to use tuples of numbers:

+

+ sys.version_info is a tuple object you can use for comparisons. +

+

If you’re doing these kinds of comparisons in your code, you should check your code with flake8-2020 to make sure you’re handling versions correctly:

+

With the flake8-2020 extension activated, you’ll get a recommendation about replacing sys.version with sys.version_info.

+
+
+

So, Should You Upgrade to Python 3.10? +

+

You’ve now seen the coolest features of the newest and latest version of Python. The question now is whether you should upgrade to Python 3.10, and if yes, when you should do so. There are two different aspects to consider when thinking about upgrading to Python 3.10:

+
    +
  1. Should you upgrade your environment so that you run your code with the Python 3.10 interpreter?
    2. +
  2. Should you write your code using the new Python 3.10 features?
    3. +
+

Clearly, if you want to test out structural pattern matching or any of the other cool new features you’ve read about here, you need Python 3.10. It’s possible to install the latest version side by side with your current Python version. A straightforward way to do this is to use an environment manager like pyenv or Conda. You can also use Docker to run Python 3.10 without installing it locally.

+

Python 3.10 has been through about five months of beta testing, so there shouldn’t be any big issues with starting to use it for your own development. You may find that some of your dependencies don’t immediately have wheels for Python 3.10 available, which makes them more cumbersome to install. But in general, using the newest Python for local development is fairly safe.

+

As always, you should be careful before upgrading your production environment. Be vigilant about testing that your code runs well on the new version. In particular, you want to be on the lookout for features that are deprecated or removed.

+

Whether you can start using the new features in your code or not depends on your user base and the environment where your code is running. If you can guarantee that Python 3.10 is available, then there’s no danger in using the new union type syntax or any other new feature.

+

If you’re distributing an app or a library that’s used by others instead, you may want to be a bit more conservative. Currently, Python 3.6 is the oldest officially supported Python version. It reaches end-of-life in December 2021, after which Python 3.7 will be the minimum supported version.

+

The documentation includes a useful guide about porting your code to Python 3.10. Check it out for more details!

+
+
+

Conclusion +

+

The release of a new Python version is always worth celebrating. Even if you can’t start using the new features right away, they’ll become broadly available and part of your daily life within a few years.

+

+ In this tutorial, you’ve seen new features like: +

+
    +
  • Friendlier error messages +
    • +
  • Powerful structural pattern matching +
    • +
  • + Type hint improvements +
    • +
  • Safer combination of sequences +
    • +
  • New statistics functions +
    • +
+

For more Python 3.10 tips and a discussion with members of the Real Python team, check out Real Python Podcast Episode #81.

+

Have fun trying out the new features! Share your experiences in the comments below.

+
+

+ Watch Now This tutorial has a related video course created by the Real Python team. Watch it together with the written tutorial to deepen your understanding: Cool New Features in Python 3.10 +

+
+
+

Get a short & sweet Python Trick delivered to your inbox every couple of days. No spam ever. Unsubscribe any time. Curated by the Real Python team.

+

Python Tricks Dictionary Merge +

+
+
+
+

Geir Arne Hjelle Geir Arne Hjelle +

+
+
+
+

+ Each tutorial at Real Python is created by a team of developers so that it meets our high quality standards. The team members who worked on this tutorial are: +

+
+

Dan Bader +

+

Sadie Parker +

+
+
+
+
+
+

Master Real-World Python Skills With Unlimited Access to Real Python

+

+ +

+

+ Join us and get access to thousands of tutorials, hands-on video courses, and a community of expert Pythonistas: +

+

+ Level Up Your Python Skills » +

+
+
+

Master Real-World Python Skills
With Unlimited Access to Real Python

+

+ +

+

+ Join us and get access to thousands of tutorials, hands-on video courses, and a community of expert Pythonistas: +

+

+ Level Up Your Python Skills » +

+
+
+
+
\ No newline at end of file diff --git a/packages/readabilityjs/test/test-pages/realpython/source.html b/packages/readabilityjs/test/test-pages/realpython/source.html new file mode 100644 index 000000000..b5ffc54f0 --- /dev/null +++ b/packages/readabilityjs/test/test-pages/realpython/source.html @@ -0,0 +1,4150 @@ + + + + + + + Python 3.10: Cool New Features for You to Try – Real Python + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+
+ +
+
+ Python 3.10: Cool New Features for You to Try +
+

+ Python 3.10: Cool New Features for You to Try +

+
+ by Geir Arne Hjelle + 2 Comments + intermediate python +
+
+ +
+
+ + Share + Share + Email +
+
+
+
+ +
+
+
+
+ +
+
+ Remove ads +
+
+

+ + Watch Now This tutorial has a related video course created by the Real Python team. Watch it together with the written tutorial to deepen your understanding: Cool New Features in Python 3.10 +

+
+

+ Python 3.10 is out! Volunteers have been working on the new version since May 2020 to bring you a better, faster, and more secure Python. As of October 4, 2021, the first official version is available. +

+

+ Each new version of Python brings a host of changes. You can read about all of them in the documentation. Here, you’ll get to learn about the coolest new features. +

+

+ In this tutorial, you’ll learn about: +

+
    +
  • Debugging with more helpful and precise error messages +
    • +
  • Using structural pattern matching to work with data structures +
    • +
  • Adding more readable and more specific type hints +
    • +
  • Checking the length of sequences when using zip() +
    • +
  • Calculating multivariable statistics +
    • +
+

+ To try out the new features yourself, you need to run Python 3.10. You can get it from the Python homepage. Alternatively, you can use Docker with the latest Python image. +

+ + +
+

+ Better Error Messages +

+

+ Python is often lauded for being a user-friendly programming language. While this is true, there are certain parts of Python that could be friendlier. Python 3.10 comes with a host of more precise and constructive error messages. In this section, you’ll see some of the newest improvements. The full list is available in the documentation. +

+

+ Think back to writing your first Hello World program in Python: +

+
+
+ Python +
+
+
+
+ 
# hello.py
+
+print("Hello, World!)
+
+
+ +
+
+

+ Maybe you created a file, added the famous call to print(), and saved it as hello.py. You then ran the program, eager to call yourself a proper Pythonista. However, something went wrong: +

+
+
+ Shell +
+ + +
+
+
+
+ 
$ python hello.py
+  File "/home/rp/hello.py", line 3
+    print("Hello, World!)
+                        ^
+SyntaxError: EOL while scanning string literal
+
+
+ +
+
+

+ There was a SyntaxError in the code. EOL, what does that even mean? You went back to your code, and after a bit of staring and searching, you realized that there was a missing quotation mark at the end of your string. +

+

+ One of the more impactful improvements in Python 3.10 is better and more precise error messages for many common issues. If you run your buggy Hello World in Python 3.10, you’ll get a bit more help than in earlier versions of Python: +

+
+
+ Shell +
+ + +
+
+
+
+ 
$ python hello.py
+  File "/home/rp/hello.py", line 3
+    print("Hello, World!)
+          ^
+SyntaxError: unterminated string literal (detected at line 3)
+
+
+ +
+
+

+ The error message is still a bit technical, but gone is the mysterious EOL. Instead, the message tells you that you need to terminate your string! There are similar improvements to many different error messages, as you’ll see below. +

+

+ A SyntaxError is an error raised when your code is parsed, before it even starts to execute. Syntax errors can be tricky to debug because the interpreter provides imprecise or sometimes even misleading error messages. The following code is missing a curly brace to terminate the dictionary: +

+
+
+ Python +
+
+
+
+ 
 1# unterminated_dict.py
+ 2
+ 3months = {
+ 4    10: "October",
+ 5    11: "November",
+ 6    12: "December"
+ 7
+ 8print(f"{months[10]} is the tenth month")
+
+
+ +
+
+

+ The missing closing curly brace that should have been on line 7 is an error. If you run this code with Python 3.9 or earlier, you’ll see the following error message: +

+
+
+ Python Traceback +
+
+
+
+ 
  File "/home/rp/unterminated_dict.py", line 8
+    print(f"{months[10]} is the tenth month")
+    ^
+SyntaxError: invalid syntax
+
+
+ +
+
+

+ The error message highlights line 8, but there are no syntactical problems in line 8! If you’ve experienced your share of syntax errors in Python, you might already know that the trick is to look at the lines before the one Python complains about. In this case, you’re looking for the missing closing brace on line 7. +

+

+ In Python 3.10, the same code shows a much more helpful and precise error message: +

+
+
+ Python Traceback +
+
+
+
+ 
  File "/home/rp/unterminated_dict.py", line 3
+    months = {
+             ^
+SyntaxError: '{' was never closed
+
+
+ +
+
+

+ This points you straight to the offending dictionary and allows you to fix the issue in no time. +

+

+ There are a few other ways to mess up dictionary syntax. A typical one is forgetting a comma after one of the items: +

+
+
+ Python +
+
+
+
+ 
 1# missing_comma.py
+ 2
+ 3months = {
+ 4    10: "October"
+ 5    11: "November",
+ 6    12: "December",
+ 7}
+
+
+ +
+
+

+ In this code, a comma is missing at the end of line 4. Python 3.10 gives you a clear suggestion on how to fix your code: +

+
+
+ Python Traceback +
+
+
+
+ 
  File "/home/real_python/missing_comma.py", line 4
+    10: "October"
+        ^^^^^^^^^
+SyntaxError: invalid syntax. Perhaps you forgot a comma?
+
+
+ +
+
+

+ You can add the missing comma and have your code back up and running in no time. +

+

+ Another common mistake is using the assignment operator (=) instead of the equality comparison operator (==) when you’re comparing values. Previously, this would just cause another invalid syntax message. In the newest version of Python, you get some more advice: +

+
+
+ Python +
+ + +
+
+
+
+ 
>>> if month = "October":
+  File "<stdin>", line 1
+    if month = "October":
+       ^^^^^^^^^^^^^^^^^
+SyntaxError: invalid syntax. Maybe you meant '==' or ':=' instead of '='?
+
+
+ +
+
+

+ The parser suggests that you maybe meant to use a comparison operator or an assignment expression operator instead. +

+

+ Take note of another nifty improvement in Python 3.10 error messages. The last two examples show how carets (^^^) highlight the whole offending expression. Previously, a single caret symbol (^) indicated just an approximate location. +

+

+ The final error message improvement that you’ll play with for now is that attribute and name errors can now offer suggestions if you misspell an attribute or a name: +

+
+
+ Python +
+ + +
+
+
+
+ 
>>> import math
+>>> math.py
+AttributeError: module 'math' has no attribute 'py'. Did you mean: 'pi'?
+
+>>> pint
+NameError: name 'pint' is not defined. Did you mean: 'print'?
+
+>>> release = "3.10"
+>>> relaese
+NameError: name 'relaese' is not defined. Did you mean: 'release'?
+
+
+ +
+
+

+ Note that the suggestions work for both built-in names and names that you define yourself, although they may not be available in all environments. If you like these kinds of suggestions, check out BetterErrorMessages, which offers similar suggestions in even more contexts. +

+

+ The improvements you’ve seen in this section are just some of the many error messages that have gotten a face-lift. The new Python will be even more user-friendly than before, and hopefully, the new error messages will save you both time and frustration going forward. +

+
+
+
+
+ +
+
+ Remove ads +
+
+
+

+ Structural Pattern Matching +

+

+ The biggest new feature in Python 3.10, probably both in terms of controversy and potential impact, is structural pattern matching. Its introduction has sometimes been referred to as switch ... case coming to Python, but you’ll see that structural pattern matching is much more powerful than that. +

+

+ You’ll see three different examples that together highlight why this feature is called structural pattern matching and show you how you can use this new feature: +

+
    +
  1. Detecting and deconstructing different structures in your data +
    2. +
  2. Using different kinds of patterns +
    3. +
  3. + Matching literal patterns +
    4. +
+

+ Structural pattern matching is a comprehensive addition to the Python language. To give you a taste of how you can take advantage of it in your own projects, the next three subsections will dive into some of the details. You’ll also see some links that can help you explore in even more depth if you want. +

+
+

+ Deconstructing Data Structures +

+

+ At its core, structural pattern matching is about defining patterns to which your data structures can be matched. In this section, you’ll study a practical example where you’ll work with data that are structured differently, even though the meaning is the same. You’ll define several patterns, and depending on which pattern matches your data, you’ll process your data appropriately. +

+

+ This section will be a bit light on explanations of the possible patterns. Instead, it will try to give you an impression of the possibilities. The next section will step back and explain the patterns in more detail. +

+

+ Time to match your first pattern! The following example uses a match ... case block to find the first name of a user by extracting it from a user data structure: +

+
+
+ Python +
+ + +
+
+
+
+ 
>>> user = {
+...     "name": {"first": "Pablo", "last": "Galindo Salgado"},
+...     "title": "Python 3.10 release manager",
+... }
+
+>>> match user:
+...     case {"name": {"first": first_name}}:
+...         pass
+...
+
+>>> first_name
+'Pablo'
+
+
+ +
+
+

+ You can see structural pattern matching at work in the highlighted lines. user is a small dictionary with user information. The case line specifies a pattern that user is matched against. In this case, you’re looking for a dictionary with a "name" key whose value is a new dictionary. This nested dictionary has a key called "first". The corresponding value is bound to the variable first_name. +

+

+ For a practical example, say that you’re processing user data where the underlying data model changes over time. Therefore, you need to be able to process different versions of the same data. +

+

+ In the next example, you’ll use data from randomuser.me. This is a great API for generating random user data that you can use during testing and development. The API is also an example of an API that has changed over time. You can still access the old versions of the API. +

+

+ You may expand the collapsed section below to see how you can use requests to obtain different versions of the user data using the API: +

+
+
+

+ +

+
+
+
+

+ You can get a random user from the API using requests as follows: +

+
+
+ Python +
+
+
+
+ 
# random_user.py
+
+import requests
+
+def get_user(version="1.3"):
+    """Get random users"""
+    url = f"https://randomuser.me/api/{version}/?results=1"
+    response = requests.get(url)
+    if response:
+        return response.json()["results"][0]
+
+
+ +
+
+

+ get_user() gets one random user in JSON format. Note the version parameter. The structure of the returned data has changed quite a bit between earlier versions like "1.1" and the current version "1.3", but in each case, the actual user data are contained in a list inside the "results" array. The function returns the first—and only—user in this list. +

+

+ At the time of writing, the latest version of the API is 1.3 and the data has the following structure: +

+
+
+ JSON +
+
+
+
+ 
{
+    "gender": "female",
+    "name": {
+        "title": "Miss",
+        "first": "Ilona",
+        "last": "Jokela"
+    },
+    "location": {
+        "street": {
+            "number": 4473,
+            "name": "Mannerheimintie"
+        },
+        "city": "Harjavalta",
+        "state": "Ostrobothnia",
+        "country": "Finland",
+        "postcode": 44879,
+        "coordinates": {
+            "latitude": "-6.0321",
+            "longitude": "123.2213"
+        },
+        "timezone": {
+            "offset": "+5:30",
+            "description": "Bombay, Calcutta, Madras, New Delhi"
+        }
+    },
+    "email": "ilona.jokela@example.com",
+    "login": {
+        "uuid": "632b7617-6312-4edf-9c24-d6334a6af52d",
+        "username": "brownsnake482",
+        "password": "biatch",
+        "salt": "ofk518ZW",
+        "md5": "6d589615ca44f6e583c85d45bf431c54",
+        "sha1": "cd87c931d579bdff77af96c09e0eea82d1edfc19",
+        "sha256": "6038ede83d4ce74116faa67fb3b1b2e6f6898e5749b57b5a0312bd46a539214a"
+    },
+    "dob": {
+        "date": "1957-05-20T08:36:09.083Z",
+        "age": 64
+    },
+    "registered": {
+        "date": "2006-07-30T18:39:20.050Z",
+        "age": 15
+    },
+    "phone": "07-369-318",
+    "cell": "048-284-01-59",
+    "id": {
+        "name": "HETU",
+        "value": "NaNNA204undefined"
+    },
+    "picture": {
+        "large": "https://randomuser.me/api/portraits/women/28.jpg",
+        "medium": "https://randomuser.me/api/portraits/med/women/28.jpg",
+        "thumbnail": "https://randomuser.me/api/portraits/thumb/women/28.jpg"
+    },
+    "nat": "FI"
+}
+
+
+ +
+
+

+ One of the members that changed between different versions is "dob", the date of birth. Note that in version 1.3, this is a JSON object with two members, "date" and "age". +

+ +

+ Compare the result above with a version 1.1 random user: +

+
+
+ JSON +
+
+
+
+ 
{
+    "gender": "female",
+    "name": {
+        "title": "miss",
+        "first": "ilona",
+        "last": "jokela"
+    },
+    "location": {
+        "street": "7336 myllypuronkatu",
+        "city": "kurikka",
+        "state": "central ostrobothnia",
+        "postcode": 53740
+    },
+    "email": "ilona.jokela@example.com",
+    "login": {
+        "username": "blackelephant837",
+        "password": "sand",
+        "salt": "yofk518Z",
+        "md5": "b26367ea967600d679ee3e0b9bda012f",
+        "sha1": "87d2910595acba5b8e8aa8b00a841bab08580e2f",
+        "sha256": "73bd0d205d0dc83ae184ae222ff2e9de5ea4039119a962c4f97fabd5bbfa7aca"
+    },
+    "dob": "1966-04-17 11:57:01",
+    "registered": "2005-08-10 10:15:01",
+    "phone": "04-636-931",
+    "cell": "048-828-40-15",
+    "id": {
+        "name": "HETU",
+        "value": "366-9204"
+    },
+    "picture": {
+        "large": "https://randomuser.me/api/portraits/women/24.jpg",
+        "medium": "https://randomuser.me/api/portraits/med/women/24.jpg",
+        "thumbnail": "https://randomuser.me/api/portraits/thumb/women/24.jpg"
+    },
+    "nat": "FI"
+}
+
+
+ +
+
+

+ Observe that in this older format, the value of the "dob" member is a plain string. +

+
+
+
+

+ In this example, you’ll work with the information about the date of birth (dob) for each user. The structure of these data has changed between different versions of the Random User API: +

+
+
+ JSON +
+
+
+
+ 
# Version 1.1
+"dob": "1966-04-17 11:57:01"
+
+# Version 1.3
+"dob": {"date": "1957-05-20T08:36:09.083Z", "age": 64}
+
+
+ +
+
+

+ Note that in version 1.1, the date of birth is represented as a simple string, while in version 1.3, it’s a JSON object with two members: "date" and "age". Say that you want to find the age of a user. Depending on the structure of your data, you’d either need to calculate the age based on the date of birth or look up the age if it’s already available. +

+ +

+ Traditionally, you would detect the structure of the data with an if test, maybe based on the type of the "dob" field. You can approach this differently in Python 3.10. Now, you can use structural pattern matching instead: +

+
+
+ Python +
+
+
+
+ 
 1# random_user.py (continued)
+ 2
+ 3from datetime import datetime
+ 4
+ 5def get_age(user):
+ 6    """Get the age of a user"""
+ 7    match user:
+ 8        case {"dob": {"age": int(age)}}:
+ 9            return age
+10        case {"dob": dob}:
+11            now = datetime.now()
+12            dob_date = datetime.strptime(dob, "%Y-%m-%d %H:%M:%S")
+13            return now.year - dob_date.year
+
+
+ +
+
+

+ The match ... case construct is new in Python 3.10 and is how you perform structural pattern matching. You start with a match statement that specifies what you want to match. In this example, that’s the user data structure. +

+

+ One or several case statements follow match. Each case describes one pattern, and the indented block beneath it says what should happen if there’s a match. In this example: +

+
    +
  • +

    + Line 8 matches a dictionary with a "dob" key whose value is another dictionary with an integer (int) item named "age". The name age captures its value. +

    +
    • +
  • +

    + Line 10 matches any dictionary with a "dob" key. The name dob captures its value. +

    +
    • +
+

+ One important feature of pattern matching is that at most one pattern will be matched. Since the pattern on line 10 matches any dictionary with "dob", it’s important that the more specific pattern on line 8 comes first. +

+ +

+ Before looking closer at the details of the patterns and how they work, try calling get_age() with different data structures to see the result: +

+
+
+ Python +
+ + +
+
+
+
+ 
>>> import random_user
+
+>>> users11 = random_user.get_user(version="1.1")
+>>> random_user.get_age(users11)
+55
+
+>>> users13 = random_user.get_user(version="1.3")
+>>> random_user.get_age(users13)
+64
+
+
+ +
+
+

+ Your code can calculate the age correctly for both versions of the user data, which have different dates of birth. +

+

+ Look closer at those patterns. The first pattern, {"dob": {"age": int(age)}}, matches version 1.3 of the user data: +

+
+
+ Python +
+
+
+
+ 
{
+    ...
+    "dob": {"date": "1957-05-20T08:36:09.083Z", "age": 64},
+    ...
+}
+
+
+ +
+
+

+ The first pattern is a nested pattern. The outer curly braces say that a dictionary with the key "dob" is required. The corresponding value should be a dictionary. This nested dictionary must match the subpattern {"age": int(age)}. In other words, it needs to have an "age" key with an integer value. That value is bound to the name age. +

+

+ The second pattern, {"dob": dob}, matches the older version 1.1 of the user data: +

+
+
+ Python +
+
+
+
+ 
{
+    ...
+    "dob": "1966-04-17 11:57:01",
+    ...
+}
+
+
+ +
+
+

+ This second pattern is a simpler pattern than the first one. Again, the curly braces indicate that it will match a dictionary. However, any dictionary with a "dob" key is matched because there are no other restrictions specified. The value of that key is bound to the name dob. +

+

+ The main takeaway is that you can describe the structure of your data using mostly familiar notation. One striking change, though, is that you can use names like dob and age, which aren’t yet defined. Instead, values from your data are bound to these names when a pattern matches. +

+

+ You’ve explored some of the power of structural pattern matching in this example. In the next section, you’ll dive a bit more into the details. +

+
+
+
+
+ +
+
+ Remove ads +
+
+
+

+ Using Different Kinds of Patterns +

+

+ You’ve seen an example of how you can use patterns to effectively unravel complicated data structures. Now, you’ll take a step back and look at the building blocks that make up this new feature. Many things come together to make it work. In fact, there are three Python Enhancement Proposals (PEPs) that describe structural pattern matching: +

+
    +
  1. + PEP 634: Specification +
    2. +
  2. + PEP 635: Motivation and Rationale +
    3. +
  3. + PEP 636: Tutorial +
    4. +
+

+ These documents give you a lot of background and detail if you’re interested in a deeper dive than what follows. +

+

+ Patterns are at the center of structural pattern matching. In this section, you’ll learn about some of the different kinds of patterns that exist: +

+
    +
  • + Mapping patterns match mapping structures like dictionaries. +
    • +
  • + Sequence patterns match sequence structures like tuples and lists. +
    • +
  • + Capture patterns bind values to names. +
    • +
  • + AS patterns bind the value of subpatterns to names. +
    • +
  • + OR patterns match one of several different subpatterns. +
    • +
  • + Wildcard patterns match anything. +
    • +
  • + Class patterns match class structures. +
    • +
  • + Value patterns match values stored in attributes. +
    • +
  • + Literal patterns match literal values. +
    • +
+

+ You already used several of them in the example in the previous section. In particular, you used mapping patterns to unravel data stored in dictionaries. In this section, you’ll learn more about how some of these work. All the details are available in the PEPs mentioned above. +

+

+ A capture pattern is used to capture a match to a pattern and bind it to a name. Consider the following recursive function that sums a list of numbers: +

+
+
+ Python +
+
+
+
+ 
 1def sum_list(numbers):
+ 2    match numbers:
+ 3        case []:
+ 4            return 0
+ 5        case [first, *rest]:
+ 6            return first + sum_list(rest)
+
+
+ +
+
+

+ The first case on line 3 matches the empty list and returns 0 as its sum. The second case on line 5 uses a sequence pattern with two capture patterns to match lists with one or more elements. The first element in the list is captured and bound to the name first. The second capture pattern, *rest, uses unpacking syntax to match any number of elements. rest will bind to a list containing all elements of numbers except the first one. +

+

+ sum_list() calculates the sum of a list of numbers by recursively adding the first number in the list and the sum of the rest of the numbers. You can use it as follows: +

+
+
+ Python +
+ + +
+
+
+
+ 
>>> sum_list([4, 5, 9, 4])
+22
+
+
+ +
+
+

+ The sum of 4 + 5 + 9 + 4 is correctly calculated to be 22. As an exercise for yourself, you can try to trace the recursive calls to sum_list() to make sure you understand how the code sums the whole list. +

+ +

+ sum_list() handles summing up a list of numbers. Observe what happens if you try to sum anything that isn’t a list: +

+
+
+ Python +
+ + +
+
+
+
+ 
>>> print(sum_list("4594"))
+None
+
+>>> print(sum_list(4594))
+None
+
+
+ +
+
+

+ Passing a string or a number to sum_list() returns None. This occurs because none of the patterns match, and the execution continues after the match block. That happens to be the end of the function, so sum_list() implicitly returns None. +

+

+ Often, though, you want to be alerted about failed matches. You can add a catchall pattern as the final case that handles this by raising an error, for example. You can use the underscore (_) as a wildcard pattern that matches anything without binding it to a name. You can add some error handling to sum_list() as follows: +

+
+
+ Python +
+
+
+
+ 
def sum_list(numbers):
+    match numbers:
+        case []:
+            return 0
+        case [first, *rest]:
+            return first + sum_list(rest)
+        case _:
+            wrong_type = numbers.__class__.__name__
+            raise ValueError(f"Can only sum lists, not {wrong_type!r}")
+
+
+ +
+
+

+ The final case will match anything that doesn’t match the first two patterns. This will raise a descriptive error, for instance, if you try to calculate sum_list(4594). This is useful when you need to alert your users that some input was not matched as expected. +

+

+ Your patterns are still not foolproof, though. Consider what happens if you try to sum a list of strings: +

+
+
+ Python +
+ + +
+
+
+
+ 
>>> sum_list(["45", "94"])
+TypeError: can only concatenate str (not "int") to str
+
+
+ +
+
+

+ The base case returns 0, so therefore the summing only works for types that you can add with numbers. Python doesn’t know how to add numbers and text strings together. You can restrict your pattern to only match integers using a class pattern: +

+
+
+ Python +
+
+
+
+ 
def sum_list(numbers):
+    match numbers:
+        case []:
+            return 0
+        case [int(first), *rest]:
+            return first + sum_list(rest)
+        case _:
+            raise ValueError(f"Can only sum lists of numbers")
+
+
+ +
+
+

+ Adding int() around first makes sure that the pattern only matches if the value is an integer. This might be too restrictive, though. Your function should be able to sum both integers and floating-point numbers, so how can you allow this in your pattern? +

+

+ To check whether at least one out of several subpatterns match, you can use an OR pattern. OR patterns consist of two or more subpatterns, and the pattern matches if at least one of the subpatterns does. You can use this to match when the first element is either of type int or type float: +

+
+
+ Python +
+
+
+
+ 
def sum_list(numbers):
+    match numbers:
+        case []:
+            return 0
+        case [int(first) | float(first), *rest]:
+            return first + sum_list(rest)
+        case _:
+            raise ValueError(f"Can only sum lists of numbers")
+
+
+ +
+
+

+ You use the pipe symbol (|) to separate the subpatterns in an OR pattern. Your function now allows summing a list of floating-point numbers: +

+
+
+ Python +
+ + +
+
+
+
+ 
>>> sum_list([45.94, 46.17, 46.72])
+138.82999999999998
+
+
+ +
+
+

+ There’s a lot of power and flexibility within structural pattern matching, even more than what you’ve seen so far. Some things that aren’t covered in this overview are: +

+ +

+ If you’re interested, have a look in the documentation to learn more about these features as well. In the next section, you’ll learn about literal patterns and value patterns. +

+
+
+
+
+ +
+
+ Remove ads +
+
+
+

+ Matching Literal Patterns +

+

+ A literal pattern is a pattern that matches a literal object like an explicit string or number. In a sense, this is the most basic kind of pattern and allows you to emulate switch ... case statements seen in other languages. The following example matches a specific name: +

+
+
+ Python +
+
+
+
+ 
def greet(name):
+    match name:
+        case "Guido":
+            print("Hi, Guido!")
+        case _:
+            print("Howdy, stranger!")
+
+
+ +
+
+

+ The first case matches the literal string "Guido". In this case, you use _ as a wildcard to print a generic greeting whenever name is not "Guido". Such literal patterns can sometimes take the place of if ... elif ... else constructs and can play the same role that switch ... case does in some other languages. +

+

+ One limitation with structural pattern matching is that you can’t directly match values stored in variables. Say that you’ve defined bdfl = "Guido". A pattern like case bdfl: will not match "Guido". Instead, this will be interpreted as a capture pattern that matches anything and binds that value to bdfl, effectively overwriting the old value. +

+

+ You can, however, use a value pattern to match stored values. A value pattern looks a bit like a capture pattern but uses a previously defined dotted name that holds the value that will be matched against. +

+ +

+ You can, for example, use an enumeration to create such dotted names: +

+
+
+ Python +
+
+
+
+ 
import enum
+
+class Pythonista(str, enum.Enum):
+    BDFL = "Guido"
+    FLUFL = "Barry"
+
+def greet(name):
+    match name:
+        case Pythonista.BDFL:
+            print("Hi, Guido!")
+        case _:
+            print("Howdy, stranger!")
+
+
+ +
+
+

+ The first case now uses a value pattern to match Pythonista.BDFL, which is "Guido". Note that you can use any dotted name in a value pattern. You could, for example, have used a regular class or a module instead of the enumeration. +

+

+ To see a bigger example of how to use literal patterns, consider the game of FizzBuzz. This is a counting game where you should replace some numbers with words according to the following rules: +

+
    +
  • You replace numbers divisible by 3 with fizz. +
    • +
  • You replace numbers divisible by 5 with buzz. +
    • +
  • You replace numbers divisible by both 3 and 5 with fizzbuzz. +
    • +
+

+ FizzBuzz is sometimes used to introduce conditionals in programming education and as a screening problem in interviews. Even though a solution is quite straightforward, Joel Grus has written a full book about different ways to program the game. +

+

+ A typical solution in Python will use if ... elif ... else as follows: +

+
+
+ Python +
+
+
+
+ 
def fizzbuzz(number):
+    mod_3 = number % 3
+    mod_5 = number % 5
+
+    if mod_3 == 0 and mod_5 == 0:
+        return "fizzbuzz"
+    elif mod_3 == 0:
+        return "fizz"
+    elif mod_5 == 0:
+        return "buzz"
+    else:
+        return str(number)
+
+
+ +
+
+

+ The % operator calculates the modulus, which you can use to test divisibility. Namely, if a modulus b is 0 for two numbers a and b, then a is divisible by b. +

+

+ In fizzbuzz(), you calculate number % 3 and number % 5, which you then use to test for divisibility with 3 and 5. Note that you must do the test for divisibility with both 3 and 5 first. If not, numbers that are divisible by both 3 and 5 will be covered by either the "fizz" or the "buzz" cases instead. +

+

+ You can check that your implementation gives the expected result: +

+
+
+ Python +
+ + +
+
+
+
+ 
>>> fizzbuzz(3)
+fizz
+
+>>> fizzbuzz(14)
+14
+
+>>> fizzbuzz(15)
+fizzbuzz
+
+>>> fizzbuzz(92)
+92
+
+>>> fizzbuzz(65)
+buzz
+
+
+ +
+
+

+ You can confirm for yourself that 3 is divisible by 3, 65 is divisible by 5, and 15 is divisible by both 3 and 5, while 14 and 92 aren’t divisible by either 3 or 5. +

+

+ An if ... elif ... else structure where you’re comparing one or a few variables several times over is quite straightforward to rewrite using pattern matching instead. For example, you can do the following: +

+
+
+ Python +
+
+
+
+ 
def fizzbuzz(number):
+    mod_3 = number % 3
+    mod_5 = number % 5
+
+    match (mod_3, mod_5):
+        case (0, 0):
+            return "fizzbuzz"
+        case (0, _):
+            return "fizz"
+        case (_, 0):
+            return "buzz"
+        case _:
+            return str(number)
+
+
+ +
+
+

+ You match on both mod_3 and mod_5. Each case pattern then matches either the literal number 0 or the wildcard _ on the corresponding values. +

+

+ Compare and contrast this version with the previous one. Note how the pattern (0, 0) corresponds to the test mod_3 == 0 and mod_5 == 0, while (0, _) corresponds to mod_3 == 0. +

+

+ As you saw earlier, you can use an OR pattern to match on several different patterns. For example, since mod_3 can only take the values 0, 1, and 2, you can replace case (_, 0) with case (1, 0) | (2, 0). Remember that (0, 0) has already been covered. +

+ +

+ The Python core developers have consciously chosen not to include switch ... case statements in the language earlier. However, there are some third-party packages that do, like switchlang, which adds a switch command that also works on earlier versions of Python. +

+
+
+
+
+ +
+
+ Remove ads +
+
+
+
+

+ Type Unions, Aliases, and Guards +

+

+ Reliably, each new Python release brings some improvements to the static typing system. Python 3.10 is no exception. In fact, four different PEPs about typing accompany this new release: +

+
    +
  1. + PEP 604: Allow writing union types as X | Y +
    2. +
  2. + PEP 613: Explicit Type Aliases +
    3. +
  3. + PEP 647: User-Defined Type Guards +
    4. +
  4. + PEP 612: Parameter Specification Variables +
    5. +
+

+ PEP 604 will probably be the most widely used of these changes going forward, but you’ll get a brief overview of each of the features in this section. +

+

+ You can use union types to declare that a variable can have one of several different types. For example, you’ve been able to type hint a function calculating the mean of a list of numbers, floats, or integers as follows: +

+
+
+ Python +
+
+
+
+ 
from typing import List, Union
+
+def mean(numbers: List[Union[float, int]]) -> float:
+    return sum(numbers) / len(numbers)
+
+
+ +
+
+

+ The annotation List[Union[float, int]] means that numbers should be a list where each element is either a floating-point number or an integer. This works well, but the notation is a bit verbose. Also, you need to import both List and Union from typing. +

+ +

+ In Python 3.10, you can replace Union[float, int] with the more succinct float | int. Combine this with the ability to use list instead of typing.List in type hints, which Python 3.9 introduced. You can then simplify your code while keeping all the type information: +

+
+
+ Python +
+
+
+
+ 
def mean(numbers: list[float | int]) -> float:
+    return sum(numbers) / len(numbers)
+
+
+ +
+
+

+ The annotation of numbers is easier to read now, and as an added bonus, you didn’t need to import anything from typing. +

+

+ A special case of union types is when a variable can have either a specific type or be None. You can annotate such optional types either as Union[None, T] or, equivalently, Optional[T] for some type T. There is no new, special syntax for optional types, but you can use the new union syntax to avoid importing typing.Optional: +

+
+
+ Python +
+
+
+
+ 
address: str | None
+
+
+ +
+
+

+ In this example, address is allowed to be either None or a string. +

+

+ You can also use the new union syntax at runtime in isinstance() or issubclass() tests: +

+
+
+ Python +
+ + +
+
+
+
+ 
>>> isinstance("mypy", str | int)
+True
+
+>>> issubclass(str, int | float | bytes)
+False
+
+
+ +
+
+

+ Traditionally, you’ve used tuples to test for several types at once—for example, (str, int) instead of str | int. This old syntax will still work. +

+

+ Type aliases allow you to quickly define new aliases that can stand in for more complicated type declarations. For example, say that you’re representing a playing card using a tuple of suit and rank strings and a deck of cards by a list of such playing card tuples. A deck of cards is then type hinted as list[tuple[str, str]]. +

+

+ To simplify type annotation, you define type aliases as follows: +

+
+
+ Python +
+
+
+
+ 
Card = tuple[str, str]
+Deck = list[Card]
+
+
+ +
+
+

+ This usually works okay. However, it’s often not possible for the type checker to know whether such a statement is a type alias or just the definition of a regular global variable. To help the type checker—or really, help the type checker help you—you can now explicitly annotate type aliases: +

+
+
+ Python +
+
+
+
+ 
from typing import TypeAlias
+
+Card: TypeAlias = tuple[str, str]
+Deck: TypeAlias = list[Card]
+
+
+ +
+
+

+ Adding the TypeAlias annotation clarifies the intention, both to a type checker and to anyone reading your code. +

+

+ Type guards are used to narrow down union types. The following function takes in either a string or None but always returns a tuple of strings representing a playing card: +

+
+
+ Python +
+
+
+
+ 
def get_ace(suit: str | None) -> tuple[str, str]:
+    if suit is None:
+        suit = "♠"
+    return (suit, "A")
+
+
+ +
+
+

+ The highlighted line works as a type guard, and static type checkers are able to realize that suit is necessarily a string when it’s returned. +

+

+ Currently, the type checkers can only use a few different constructs to narrow down union types in this way. With the new typing.TypeGuard, you can annotate custom functions that can be used to narrow down union types: +

+
+
+ Python +
+
+
+
+ 
from typing import Any, TypeAlias, TypeGuard
+
+Card: TypeAlias = tuple[str, str]
+Deck: TypeAlias = list[Card]
+
+def is_deck_of_cards(obj: Any) -> TypeGuard[Deck]:
+    # Return True if obj is a deck of cards, otherwise False
+
+
+ +
+
+

+ is_deck_of_cards() should return True or False depending on whether obj represents a Deck object or not. You can then use your guard function, and the type checker will be able to narrow down the types correctly: +

+
+
+ Python +
+
+
+
+ 
def get_score(card_or_deck: Card | Deck) -> int:
+    if is_deck_of_cards(card_or_deck):
+        # Calculate score of a deck of cards
+    ...
+
+
+ +
+
+

+ Inside of the if block, the type checker knows that card_or_deck is, in fact, of the type Deck. See PEP 647 for more details. +

+

+ The final new typing feature is Parameter Specification Variables, which is related to type variables. Consider the definition of a decorator. In general, it looks something like the following: +

+
+
+ Python +
+
+
+
+ 
import functools
+from typing import Any, Callable, TypeVar
+
+R = TypeVar("R")
+
+def decorator(func: Callable[..., R]) -> Callable[..., R]:
+    @functools.wraps(func)
+    def wrapper(*args: Any, **kwargs: Any) -> R:
+        ...
+    return wrapper
+
+
+ +
+
+

+ The annotations mean that the function returned by the decorator is a callable with some parameters and the same return type, R, as the function passed into the decorator. The ellipsis (...) in the function header correctly allows any number of parameters, and each of those parameters can be of any type. However, there’s no validation that the returned callable has the same parameters as the function that was passed in. In practice, this means that type checkers aren’t able to check decorated functions properly. +

+

+ Unfortunately, you can’t use TypeVar for the parameters because you don’t know how many parameters the function will have. In Python 3.10, you’ll have access to ParamSpec in order to type hint these kinds of callables properly. ParamSpec works similarly to TypeVar but stands in for several parameters at once. You can rewrite your decorator as follows to take advantage of ParamSpec: +

+
+
+ Python +
+
+
+
+ 
import functools
+from typing import Callable, ParamSpec, TypeVar
+
+P = ParamSpec("P")
+R = TypeVar("R")
+
+def decorator(func: Callable[P, R]) -> Callable[P, R]:
+    @functools.wraps(func)
+    def wrapper(*args: P.args, **kwargs: P.kwargs) -> R:
+        ...
+    return wrapper
+
+
+ +
+
+

+ Note that you also use P when you annotate wrapper(). You can also use the new typing.Concatenate to add types to ParamSpec. See the documentation and PEP 612 for details and examples. +

+
+
+
+
+ +
+
+ Remove ads +
+
+
+

+ Stricter Zipping of Sequences +

+

+ zip() is a built-in function in Python that can combine elements from several sequences. Python 3.10 introduces the new strict parameter, which adds a runtime test to check that all sequences being zipped have the same length. +

+

+ As an example, consider the following table of Lego sets: +

+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ Name + + Set Number + + Pieces +
+ Louvre + + 21024 + + 695 +
+ Diagon Alley + + 75978 + + 5544 +
+ NASA Apollo Saturn V + + 92176 + + 1969 +
+ Millennium Falcon + + 75192 + + 7541 +
+ New York City + + 21028 + + 598 +
+
+

+ One way to represent these data in plain Python would be with each column as a list. It could look something like this: +

+
+
+ Python +
+ + +
+
+
+
+ 
>>> names = ["Louvre", "Diagon Alley", "Saturn V", "Millennium Falcon", "NYC"]
+>>> set_numbers = ["21024", "75978", "92176", "75192", "21028"]
+>>> num_pieces = [695, 5544, 1969, 7541, 598]
+
+
+ +
+
+

+ Note that you have three independent lists, but there’s an implicit correspondence between their elements. The first name ("Louvre"), the first set number ("21024"), and the first number of pieces (695) all describe the first Lego set. +

+ +

+ zip() can be used to iterate over these three lists in parallel: +

+
+
+ Python +
+ + +
+
+
+
+ 
>>> for name, num, pieces in zip(names, set_numbers, num_pieces):
+...     print(f"{name} ({num}): {pieces} pieces")
+...
+Louvre (21024): 695 pieces
+Diagon Alley (75978): 5544 pieces
+Saturn V (92176): 1969 pieces
+Millennium Falcon (75192): 7541 pieces
+NYC (21028): 598 pieces
+
+
+ +
+
+

+ Note how each line collects information from all three lists and shows information about one particular set. This is a very common pattern that’s used in a lot of different Python code, including in the standard library. +

+

+ You can also add list() to collect the contents of all three lists in a single, nested list of tuples: +

+
+
+ Python +
+ + +
+
+
+
+ 
>>> list(zip(names, set_numbers, num_pieces))
+[('Louvre', '21024', 695),
+ ('Diagon Alley', '75978', 5544),
+ ('Saturn V', '92176', 1969),
+ ('Millennium Falcon', '75192', 7541),
+ ('NYC', '21028', 598)]
+
+
+ +
+
+

+ Note how the nested list closely resembles the original table. +

+

+ The dark side of using zip() is that it’s quite easy to introduce a subtle bug that can be hard to discover. Note what happens if there’s a missing item in one of your lists: +

+
+
+ Python +
+ + +
+
+
+
+ 
>>> set_numbers = ["21024", "75978", "75192", "21028"]  # Saturn V missing
+
+>>> list(zip(names, set_numbers, num_pieces))
+[('Louvre', '21024', 695),
+ ('Diagon Alley', '75978', 5544),
+ ('Saturn V', '75192', 1969),
+ ('Millennium Falcon', '21028', 7541)]
+
+
+ +
+
+

+ All the information about the New York City set disappeared! Additionally, the set numbers for Saturn V and Millennium Falcon are wrong. If your datasets are bigger, these kinds of errors can be very hard to discover. And even when you observe that something’s wrong, it’s not always easy to diagnose and fix. +

+

+ The issue is that you assumed that the three lists have the same number of elements and that the information is in the same order in each list. After set_numbers gets corrupted, this assumption is no longer true. +

+

+ PEP 618 introduces a new strict keyword parameter to zip() that you can use to confirm all sequences have the same length. In your example, it would raise an error alerting you to the corrupted list: +

+
+
+ Python +
+ + +
+
+
+
+ 
>>> list(zip(names, set_numbers, num_pieces, strict=True))
+Traceback (most recent call last):
+  File "<stdin>", line 1, in <module>
+ValueError: zip() argument 2 is shorter than argument 1
+
+
+ +
+
+

+ When the iteration reaches the New York City Lego set, the second argument set_numbers is already exhausted, while there are still elements left in the first argument names. Instead of silently giving the wrong result, your code fails with an error, and you can take action to find and fix the mistake. +

+

+ There are use cases when you want to combine sequences of unequal length. Expand the box below to see how zip() and itertools.zip_longest() handle these: +

+
+
+

+ +

+
+
+
+

+ The following idiom divides the Lego sets into pairs: +

+
+
+ Python +
+ + +
+
+
+
+ 
>>> num_per_group = 2
+>>> list(zip(*[iter(names)] * num_per_group))
+[('Louvre', 'Diagon Alley'), ('Saturn V', 'Millennium Falcon')]
+
+
+ +
+
+

+ There are five sets, a number that doesn’t divide evenly into pairs. In this case, the default behavior of zip(), where the last element is dropped, might make sense. You could use strict=True here as well, but that would raise an error when your list can’t be split into pairs. A third option, which could be the best in this case, is to use zip_longest() from the itertools standard library. +

+

+ As the name suggests, zip_longest() combines sequences until the longest sequence is exhausted. If you use zip_longest() to divide the Lego sets, it becomes more explicit that New York City doesn’t have any pairing: +

+
+
+ Python +
+ + +
+
+
+
+ 
>>> from itertools import zip_longest
+
+>>> list(zip_longest(*[iter(names)] * num_per_group, fillvalue=""))
+[('Louvre', 'Diagon Alley'),
+ ('Saturn V', 'Millennium Falcon'),
+ ('NYC', '')]
+
+
+ +
+
+

+ Note that 'NYC' shows up in the last tuple together with an empty string. You can control what’s filled in for missing values with the fillvalue parameter. +

+
+
+
+

+ While strict is not really adding any new functionality to zip(), it can help you avoid those hard-to-find bugs. +

+
+
+
+
+ +
+
+ Remove ads +
+
+
+

+ New Functions in the statistics Module +

+

+ The statistics module was added to the standard library all the way back in 2014 with the release of Python 3.4. The intent of statistics is to make statistical calculations at the level of graphing calculators available in Python. +

+ +

+ Python 3.10 adds a few multivariable functions to statistics: +

+
    +
  • + correlation() to calculate Pearson’s correlation coefficient for two variables +
    • +
  • + covariance() to calculate sample covariance for two variables +
    • +
  • + linear_regression() to calculate the slope and intercept in a linear regression +
    • +
+

+ You can use each function to describe a certain aspect of the relationship between two variables. As an example, say that you have data from a set of blog posts—the number of words in each blog post and the number of views each post has had over some time period: +

+
+
+ Python +
+ + +
+
+
+
+ 
>>> words = [7742, 11539, 16898, 13447, 4608, 6628, 2683, 6156, 2623, 6948]
+>>> views = [8368, 5901, 3978, 3329, 2611, 2096, 1515, 1177, 814, 467]
+
+
+ +
+
+

+ You now want to investigate whether there’s any (linear) relationship between the number of words and number of views. In Python 3.10, you can calculate the correlation between words and views with the new correlation() function: +

+
+
+ Python +
+ + +
+
+
+
+ 
>>> import statistics
+
+>>> statistics.correlation(words, views)
+0.454180067865917
+
+
+ +
+
+

+ The correlation between two variables is always a number between -1 and 1. If it’s close to 0, then there’s little correspondence between them, while a correlation close to -1 or 1 indicates that the behaviors of the two variables tend to follow each other. In this example, a correlation of 0.45 indicates that there’s a tendency for posts with more words to have more views, although it’s not a strong connection. +

+ +

+ You can also calculate the covariance between words and views. The covariance is another measure of the joint variability between two variables. You can calculate it with covariance(): +

+
+
+ Python +
+ + +
+
+
+
+ 
>>> import statistics
+
+>>> statistics.covariance(words, views)
+5292289.977777777
+
+
+ +
+
+

+ In contrast to correlation, covariance is an absolute measure. It should be interpreted in the context of the variability within the variables themselves. In fact, you can normalize the covariance by the standard deviation of each variable to recover Pearson’s correlation coefficient: +

+
+
+ Python +
+ + +
+
+
+
+ 
>>> import statistics
+
+>>> cov = statistics.covariance(words, views)
+>>> σ_words, σ_views = statistics.stdev(words), statistics.stdev(views)
+>>> cov / (σ_words * σ_views)
+0.454180067865917
+
+
+ +
+
+

+ Note that this matches your earlier correlation coefficient exactly. +

+

+ A third way of looking at the linear correspondence between the two variables is through simple linear regression. You do the linear regression by calculating two numbers, slope and intercept, so that the (squared) error is minimized in the approximation number of views = slope × number of words + intercept. +

+

+ In Python 3.10, you can use linear_regression(): +

+
+
+ Python +
+ + +
+
+
+
+ 
>>> import statistics
+
+>>> statistics.linear_regression(words, views)
+LinearRegression(slope=0.2424443064354672, intercept=1103.6954940247645)
+
+
+ +
+
+

+ Based on this regression, a post with 10,074 words could expect about 0.2424 × 10074 + 1104 = 3546 views. However, as you saw earlier, the correlation between the number of words and the number of views is quite weak. Therefore, you shouldn’t expect this prediction to be very accurate. +

+

+ The LinearRegression object is a named tuple. This means that you can unpack the slope and intercept directly: +

+
+
+ Python +
+ + +
+
+
+
+ 
>>> import statistics
+
+>>> slope, intercept = statistics.linear_regression(words, views)
+>>> slope * 10074 + intercept
+3546.0794370556605
+
+
+ +
+
+

+ Here, you use slope and intercept to predict the number of views on a blog post with 10,074 words. +

+

+ You still want to use some of the more advanced packages like pandas and statsmodels if you do a lot of statistical analysis. With the new additions to statistics in Python 3.10, however, you have the chance to do basic analysis more easily without bringing in third-party dependencies. +

+
+
+
+
+ +
+
+ Remove ads +
+
+
+

+ Other Pretty Cool Features +

+

+ So far, you’ve seen the biggest and most impactful new features in Python 3.10. In this section, you’ll get a glimpse of a few of the other changes that the new version brings along. If you’re curious about all the changes made for this new version, check out the documentation. +

+
+

+ Default Text Encodings +

+

+ When you open a text file, the default encoding used to interpret the characters is system dependent. In particular, locale.getpreferredencoding() is used. On Mac and Linux, this usually returns "UTF-8", while the result on Windows is more varied. +

+

+ You should therefore always specify an encoding when you attempt to open a text file: +

+
+
+ Python +
+
+
+
+ 
with open("some_file.txt", mode="r", encoding="utf-8") as file:
+    ...  # Do something with file
+
+
+ +
+
+

+ If you don’t explicitly specify an encoding, the preferred locale encoding is used, and you could experience that a file that can be read on one computer fails to open on another. +

+

+ Python 3.7 introduced UTF-8 mode, which allows you to force your programs to use UTF-8 encoding independent of the locale encoding. You can enable UTF-8 mode by giving the -X utf8 command-line option to the python executable or by setting the PYTHONUTF8 environment variable. +

+

+ In Python 3.10, you can activate a warning that will tell you when a text file is opened without a specified encoding. Consider the following script, which doesn’t specify an encoding: +

+
+
+ Python +
+
+
+
+ 
# mirror.py
+
+import pathlib
+import sys
+
+def mirror_file(filename):
+    for line in pathlib.Path(filename).open(mode="r"):
+        print(f"{line.rstrip()[::-1]:>72}")
+
+if __name__ == "__main__":
+    for filename in sys.argv[1:]:
+        mirror_file(filename)
+
+
+ +
+
+

+ The program will echo one or more text files back to the console, but with each line reversed. Run the program on itself with the encoding warning enabled: +

+
+
+ Shell +
+ + +
+
+
+
+ 
$ python -X warn_default_encoding mirror.py mirror.py
+/home/rp/mirror.py:7: EncodingWarning: 'encoding' argument not specified
+  for line in pathlib.Path(filename).open(mode="r"):
+                                                             yp.rorrim #
+
+                                                          bilhtap tropmi
+                                                              sys tropmi
+
+                                              :)emanelif(elif_rorrim fed
+                  :)"r"=edom(nepo.)emanelif(htaP.bilhtap ni enil rof
+                             )"}27>:]1-::[)(pirtsr.enil{"f(tnirp
+
+                                              :"__niam__" == __eman__ fi
+                                       :]:1[vgra.sys ni emanelif rof
+                                           )emanelif(elif_rorrim
+
+
+ +
+
+

+ Note the EncodingWarning printed to the console. The command-line option -X warn_default_encoding activates it. The warning will disappear if you specify an encoding—for example, encoding="utf-8"—when you open the file. +

+

+ There are times when you want to use the user-defined local encoding. You can still do so by explicitly using encoding="locale". However, it’s recommended to use UTF-8 whenever possible. You can check out PEP 597 for more information. +

+
+
+

+ Asynchronous Iteration +

+

+ Asynchronous programming is a powerful programming paradigm that’s been available in Python since version 3.5. You can recognize an asynchronous program by its use of the async keyword or special methods that start with .__a like .__aiter__() or .__aenter__(). +

+

+ In Python 3.10, two new asynchronous built-in functions are added: aiter() and anext(). In practice, these functions call the .__aiter__() and .__anext__() special methods—analogous to the regular iter() and next()—so no new functionality is added. These are convenience functions that make your code more readable. +

+

+ In other words, in the newest version of Python, the following statements—where things is an asynchronous iterable—are equivalent: +

+
+
+ Python +
+ + +
+
+
+
+ 
>>> it = things.__aiter__()
+>>> it = aiter(things)
+
+
+ +
+
+

+ In either case, it ends up as an asynchronous iterator. Expand the following box to see a complete example using aiter() and anext(): +

+
+
+

+ +

+
+
+
+

+ The following program counts the number of lines in several files. In practice, you use Python’s ability to iterate over files to count the number of lines. The script uses asynchronous iteration in order to handle several files concurrently. +

+

+ Note that you need to install the third-party aiofiles package with pip before running this code: +

+
+
+ Python +
+
+
+
+ 
# line_count.py
+
+import asyncio
+import sys
+import aiofiles
+
+async def count_lines(filename):
+    """Count the number of lines in the given file"""
+    num_lines = 0
+
+    async with aiofiles.open(filename, mode="r") as file:
+        lines = aiter(file)
+        while True:
+            try:
+                await anext(lines)
+                num_lines += 1
+            except StopAsyncIteration:
+                break
+
+    print(f"{filename}: {num_lines}")
+
+async def count_all_files(filenames):
+    """Asynchronously count lines in all files"""
+    tasks = [asyncio.create_task(count_lines(f)) for f in filenames]
+    await asyncio.gather(*tasks)
+
+if __name__ == "__main__":
+    asyncio.run(count_all_files(filenames=sys.argv[1:]))
+
+
+ +
+
+

+ asyncio is used to create and run one asynchronous task per filename. count_lines() opens one file asynchronously and iterates through it using aiter() and anext() in order to count the number of lines. +

+
+
+
+

+ See PEP 525 to learn more about asynchronous iteration. +

+
+
+
+
+ +
+
+ Remove ads +
+
+
+

+ Context Manager Syntax +

+

+ Context managers are great for managing resources in your programs. Until recently, though, their syntax has included an uncommon wart. You haven’t been allowed to use parentheses to break long with statements like this: +

+
+
+ Python +
+
+
+
+ 
with (
+    read_path.open(mode="r", encoding="utf-8") as read_file,
+    write_path.open(mode="w", encoding="utf-8") as write_file,
+):
+    ...
+
+
+ +
+
+

+ In earlier versions of Python, this causes an invalid syntax error message. Instead, you need to use a backslash (\) if you want to control where you break your lines: +

+
+
+ Python +
+
+
+
+ 
with read_path.open(mode="r", encoding="utf-8") as read_file, \
+     write_path.open(mode="w", encoding="utf-8") as write_file:
+    ...
+
+
+ +
+
+

+ While explicit line continuation with backslashes is possible in Python, PEP 8 discourages it. The Black formatting tool avoids backslashes completely. +

+

+ In Python 3.10, you’re now allowed to add parentheses around with statements to your heart’s content. Especially if you’re employing several context managers at once, like in the example above, this can help improve the readability of your code. Python’s documentation shows a few other possibilities with this new syntax. +

+

+ One small fun fact: parenthesized with statements actually work in version 3.9 of CPython. Their implementation came almost for free with the introduction of the PEG parser in Python 3.9. The reason that this is called a Python 3.10 feature is that using the PEG parser is voluntary in Python 3.9, while Python 3.9, with the old LL(1) parser, doesn’t support parenthesized with statements. +

+
+
+

+ Modern and Secure SSL +

+

+ Security can be challenging! A good rule of thumb is to avoid rolling your own security algorithms and instead rely on established packages. +

+

+ Python uses OpenSSL for different cryptographic features that are exposed in the hashlib, hmac, and ssl standard library modules. Your system can manage OpenSSL, or a Python installer can include OpenSSL. +

+

+ Python 3.9 supports using any of the OpenSSL versions 1.0.2 LTS, 1.1.0, and 1.1.1 LTS. Both OpenSSL 1.0.2 LTS and OpenSSL 1.1.0 are past their lifetime, so Python 3.10 will only support OpenSSL 1.1.1 LTS, as described in the following table: +

+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ Open SSL version + + Python 3.9 + + Python 3.10 + + End-of-life +
+ 1.0.2 LTS + + ✔ + + ✖ + + December 20, 2019 +
+ 1.1.0 + + ✔ + + ✖ + + September 10, 2019 +
+ 1.1.1 LTS + + ✔ + + ✔ + + September 11, 2023 +
+
+

+ This end of support for older versions will only affect you if you need to upgrade the system Python on an older operating system. If you use macOS or Windows, or if you install Python from python.org or use (Ana)Conda, you’ll see no change. +

+

+ However, Ubuntu 18.04 LTS uses OpenSSL 1.1.0, while Red Hat Enterprise Linux (RHEL) 7 and CentOS 7 both use OpenSSL 1.0.2 LTS. If you need to run Python 3.10 on these systems, you should look at installing it yourself using either the python.org or Conda installer. +

+

+ Dropping support for older versions of OpenSSL will make Python more secure. It’ll also help the Python developers in that code will be easier to maintain. Ultimately, this helps you because your Python experience will be more robust. See PEP 644 for more details. +

+
+
+

+ More Information About Your Python Interpreter +

+

+ The sys module contains a lot of information about your system, the current Python runtime, and the script currently being executed. You can, for example, inquire about the paths where Python looks for modules with sys.path and see all modules that have been imported in the current session with sys.modules. +

+

+ In Python 3.10, sys has two new attributes. First, you can now get a list of the names of all modules in the standard library: +

+
+
+ Python +
+ + +
+
+
+
+ 
>>> import sys
+
+>>> len(sys.stdlib_module_names)
+302
+
+>>> sorted(sys.stdlib_module_names)[-5:]
+['zipapp', 'zipfile', 'zipimport', 'zlib', 'zoneinfo']
+
+
+ +
+
+

+ Here, you can see that there are around 300 modules in the standard library, several of which start with the letter z. Note that only top-level modules and packages are listed. Subpackages like importlib.metadata don’t get a separate entry. +

+

+ You will probably not be using sys.stdlib_module_names all that often. Still, the list ties in nicely with similar introspection features like keyword.kwlist and sys.builtin_module_names. +

+

+ One possible use case for the new attribute is to identify which of the currently imported modules are third-party dependencies: +

+
+
+ Python +
+ + +
+
+
+
+ 
>>> import pandas as pd
+>>> import sys
+
+>>> {m for m in sys.modules if "." not in m} - sys.stdlib_module_names
+{'__main__', 'numpy', '_cython_0_29_24', 'dateutil', 'pytz',
+ 'six', 'pandas', 'cython_runtime'}
+
+
+ +
+
+

+ You find the imported top-level modules by looking at names in sys.modules that don’t have a dot in their name. By comparing them to the standard library module names, you find that numpy, dateutil, and pandas are some of the imported third-party modules in this example. +

+

+ The other new attribute is sys.orig_argv. This is related to sys.argv, which holds the command-line arguments given to your program when it was started. In contrast, sys.orig_argv lists the command-line arguments passed to the python executable itself. Consider the following example: +

+
+
+ Python +
+
+
+
+ 
# argvs.py
+
+import sys
+
+print(f"argv: {sys.argv}")
+print(f"orig_argv: {sys.orig_argv}")
+
+
+ +
+
+

+ This script echoes back the orig_argv and argv lists. Run it to see how the information is captured: +

+
+
+ Shell +
+ + +
+
+
+
+ 
$ python -X utf8 -O argvs.py 3.10 --upgrade
+argv: ['argvs.py', '3.10', '--upgrade']
+orig_argv: ['python', '-X', 'utf8', '-O', 'argvs.py', '3.10', '--upgrade']
+
+
+ +
+
+

+ Essentially, all arguments—including the name of the Python executable—end up in orig_argv. This is in contrast to argv, which only contains the arguments that aren’t handled by python itself. +

+

+ Again, this is not a feature that you’ll use a lot. If your program needs to concern itself with how it’s being run, you’re usually better off relying on information that’s already exposed instead of trying to parse this list. For example, you can choose to use the strict zip() mode only when your script is not running with the optimized flag, -O, like this: +

+
+
+ Python +
+
+
+
+ 
list(zip(names, set_numbers, num_pieces, strict=__debug__))
+
+
+ +
+
+

+ The __debug__ flag is set when the interpreter starts. It’ll be False if you’re running python with -O or -OO specified, and True otherwise. Using __debug__ is usually preferable to "-O" not in sys.orig_argv or some similar construct. +

+

+ One of the motivating use cases for sys.orig_argv is that you can use it to spawn a new Python process with the same or modified command-line arguments as your current process. +

+
+
+
+
+ +
+
+ Remove ads +
+
+
+

+ Future Annotations +

+

+ Annotations were introduced in Python 3 to give you a way to attach metadata to variables, function parameters, and return values. They are most commonly used to add type hints to your code. +

+

+ One challenge with annotations is that they must be valid Python code. For one thing, this makes it hard to type hint recursive classes. PEP 563 introduced postponed evaluation of annotations, making it possible to annotate with names that haven’t yet been defined. Since Python 3.7, you can activate postponed evaluation of annotations with a __future__ import: +

+
+
+ Python +
+
+
+
+ 
from __future__ import annotations
+
+
+ +
+
+

+ The intention was that postponed evaluation would become the default at some point in the future. After the 2020 Python Language Summit, it was decided to make this happen in Python 3.10. +

+

+ However, after more testing, it became clear that postponed evaluation didn’t work well for projects that use annotations at runtime. Key people in the FastAPI and the Pydantic projects voiced their concerns. At the last minute, it was decided to reschedule these changes for Python 3.11. +

+

+ To ease the transition into future behavior, a few changes have been made in Python 3.10 as well. Most importantly, a new inspect.get_annotations() function has been added. You should call this to access annotations at runtime: +

+
+
+ Python +
+ + +
+
+
+
+ 
>>> import inspect
+
+>>> def mean(numbers: list[int | float]) -> float:
+...     return sum(numbers) / len(numbers)
+...
+
+>>> inspect.get_annotations(mean)
+{'numbers': list[int | float], 'return': <class 'float'>}
+
+
+ +
+
+

+ Check out Annotations Best Practices for details. +

+
+
+
+

+ How to Detect Python 3.10 at Runtime +

+

+ Python 3.10 is the first version of Python with a two-digit minor version number. While this is mostly an interesting fun fact and an indication that Python 3 has been around for quite some time, it does also have some practical consequences. +

+

+ When your code needs to do something specific based on the version of Python at runtime, you’ve gotten away with doing a lexicographical comparison of version strings until now. While it’s never been good practice, it’s been possible to do the following: +

+
+
+ Python +
+
+
+
+ 
# bad_version_check.py
+
+import sys
+
+# Don't do the following
+if sys.version < "3.6":
+    raise SystemExit("Only Python 3.6 and above is supported")
+
+
+ +
+
+

+ In Python 3.10, this code will raise SystemExit and stop your program. This happens because, as strings, "3.10" is less than "3.6". +

+

+ The correct way to compare version numbers is to use tuples of numbers: +

+
+
+ Python +
+
+
+
+ 
# good_version_check.py
+
+import sys
+
+if sys.version_info < (3, 6):
+    raise SystemExit("Only Python 3.6 and above is supported")
+
+
+ +
+
+

+ sys.version_info is a tuple object you can use for comparisons. +

+

+ If you’re doing these kinds of comparisons in your code, you should check your code with flake8-2020 to make sure you’re handling versions correctly: +

+
+
+ Shell +
+ + +
+
+
+
+ 
$ python -m pip install flake8-2020
+
+$ flake8 bad_version_check.py good_version_check.py
+bad_version_check.py:3:4: YTT103 `sys.version` compared to string
+                          (python3.10), use `sys.version_info`
+
+
+ +
+
+

+ With the flake8-2020 extension activated, you’ll get a recommendation about replacing sys.version with sys.version_info. +

+
+
+
+
+ +
+
+ Remove ads +
+
+
+

+ So, Should You Upgrade to Python 3.10? +

+

+ You’ve now seen the coolest features of the newest and latest version of Python. The question now is whether you should upgrade to Python 3.10, and if yes, when you should do so. There are two different aspects to consider when thinking about upgrading to Python 3.10: +

+
    +
  1. Should you upgrade your environment so that you run your code with the Python 3.10 interpreter? +
    2. +
  2. Should you write your code using the new Python 3.10 features? +
    3. +
+

+ Clearly, if you want to test out structural pattern matching or any of the other cool new features you’ve read about here, you need Python 3.10. It’s possible to install the latest version side by side with your current Python version. A straightforward way to do this is to use an environment manager like pyenv or Conda. You can also use Docker to run Python 3.10 without installing it locally. +

+

+ Python 3.10 has been through about five months of beta testing, so there shouldn’t be any big issues with starting to use it for your own development. You may find that some of your dependencies don’t immediately have wheels for Python 3.10 available, which makes them more cumbersome to install. But in general, using the newest Python for local development is fairly safe. +

+

+ As always, you should be careful before upgrading your production environment. Be vigilant about testing that your code runs well on the new version. In particular, you want to be on the lookout for features that are deprecated or removed. +

+

+ Whether you can start using the new features in your code or not depends on your user base and the environment where your code is running. If you can guarantee that Python 3.10 is available, then there’s no danger in using the new union type syntax or any other new feature. +

+

+ If you’re distributing an app or a library that’s used by others instead, you may want to be a bit more conservative. Currently, Python 3.6 is the oldest officially supported Python version. It reaches end-of-life in December 2021, after which Python 3.7 will be the minimum supported version. +

+

+ The documentation includes a useful guide about porting your code to Python 3.10. Check it out for more details! +

+
+
+

+ Conclusion +

+

+ The release of a new Python version is always worth celebrating. Even if you can’t start using the new features right away, they’ll become broadly available and part of your daily life within a few years. +

+

+ In this tutorial, you’ve seen new features like: +

+
    +
  • Friendlier error messages +
    • +
  • Powerful structural pattern matching +
    • +
  • + Type hint improvements +
    • +
  • Safer combination of sequences +
    • +
  • New statistics functions +
    • +
+

+ For more Python 3.10 tips and a discussion with members of the Real Python team, check out Real Python Podcast Episode #81. +

+

+ Have fun trying out the new features! Share your experiences in the comments below. +

+
+
+
+ +
+
+ + + +
+
+
+

+ + Watch Now This tutorial has a related video course created by the Real Python team. Watch it together with the written tutorial to deepen your understanding: Cool New Features in Python 3.10 +

+
+
+
+

+ 🐍 Python Tricks 💌 +

+
+
+
+
+

+ Get a short & sweet Python Trick delivered to your inbox every couple of days. No spam ever. Unsubscribe any time. Curated by the Real Python team. +

+
+
+ Python Tricks Dictionary Merge +
+
+
+
+ +
+ +
+
+
+
+
+
+
+

+ About Geir Arne Hjelle +

+
+
+
+
+ Geir Arne Hjelle Geir Arne Hjelle +
+
+

+ Geir Arne is an avid Pythonista and a member of the Real Python tutorial team. +

» More about Geir Arne +
+
+
+
+
+
+
+
+

+ Each tutorial at Real Python is created by a team of developers so that it meets our high quality standards. The team members who worked on this tutorial are: +

+
+
+
+ Aldren Santos +
+
+ +

+ Aldren +

+
+
+ Christopher Trudeau +
+
+ +

+ Christopher +

+
+
+ David Amos +
+
+ +

+ David +

+
+
+
+
+ Dan Bader +
+
+ +

+ Dan +

+
+
+ Sadie Parker +
+
+ +

+ Sadie +

+
+
+
+
+
+
+
+
+
+

+ Master Real-World Python Skills With Unlimited Access to Real Python +

+

+ +

+

+ Join us and get access to thousands of tutorials, hands-on video courses, and a community of expert Pythonistas: +

+

+ Level Up Your Python Skills » +

+
+
+

+ Master Real-World Python Skills
+ With Unlimited Access to Real Python +

+

+ +

+

+ Join us and get access to thousands of tutorials, hands-on video courses, and a community of expert Pythonistas: +

+

+ Level Up Your Python Skills » +

+
+
+
+

+ What Do You Think? +

+
+
+ Rate this article: +
+ + + +
+
+
+ + LinkedIn + Twitter + Facebook + Email +
+
+
+

+ What’s your #1 takeaway or favorite thing you learned? How are you going to put your newfound skills to use? Leave a comment below and let us know. +

+
+

+ Commenting Tips: The most useful comments are those written with the goal of learning from or helping out other students. Get tips for asking good questions and get answers to common questions in our support portal. +

+
+ Looking for a real-time conversation? Visit the Real Python Community Chat or join the next “Office Hours” Live Q&A Session. Happy Pythoning! +
+
+
+
+
+

+ Keep Learning +

+
+

+ Related Tutorial Categories: intermediate python +

+

+ Recommended Video Course: Cool New Features in Python 3.10 +

+
+
+ + +
+
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + +
+
+
+ + + + +
+
+
+
+
+
+
+
+
+
+ + diff --git a/packages/readabilityjs/test/test-pages/realpython/url.txt b/packages/readabilityjs/test/test-pages/realpython/url.txt new file mode 100644 index 000000000..852b2800d --- /dev/null +++ b/packages/readabilityjs/test/test-pages/realpython/url.txt @@ -0,0 +1 @@ +https://realpython.com/python310-new-features/ \ No newline at end of file From 04fd8d2e5d44eb53080860c58f5ab4754c411e46 Mon Sep 17 00:00:00 2001 From: sywhb Date: Wed, 24 Jan 2024 13:56:40 +0000 Subject: [PATCH 35/59] Update generated html --- packages/readabilityjs/test/index.html | 6 + .../test/test-pages/realpython/distiller.html | 932 ++++++++++++++++++ 2 files changed, 938 insertions(+) create mode 100644 packages/readabilityjs/test/test-pages/realpython/distiller.html diff --git a/packages/readabilityjs/test/index.html b/packages/readabilityjs/test/index.html index 3b0a9e2c8..cfa8c53ef 100644 --- a/packages/readabilityjs/test/index.html +++ b/packages/readabilityjs/test/index.html @@ -80,6 +80,12 @@ [dom-distiller] +
  • realpython
    + [source] + [readability] + [dom-distiller] +
    • +
  • josephg
    [source] [readability] diff --git a/packages/readabilityjs/test/test-pages/realpython/distiller.html b/packages/readabilityjs/test/test-pages/realpython/distiller.html new file mode 100644 index 000000000..1c34bc3f0 --- /dev/null +++ b/packages/readabilityjs/test/test-pages/realpython/distiller.html @@ -0,0 +1,932 @@ +
    Python 3.10: Cool New Features for You to Try

    + Watch Now This tutorial has a related video course created by the Real Python team. Watch it together with the written tutorial to deepen your understanding: Cool New Features in Python 3.10 +

    + Python 3.10 is out! Volunteers have been working on the new version since May 2020 to bring you a better, faster, and more secure Python. As of October 4, 2021, the first official version is available. +

    + Each new version of Python brings a host of changes. You can read about all of them in the documentation. Here, you’ll get to learn about the coolest new features. +

    + In this tutorial, you’ll learn about: +

    • Debugging with more helpful and precise error messages
    • Using structural pattern matching to work with data structures
    • Adding more readable and more specific type hints
    • Checking the length of sequences when using zip()
    • Calculating multivariable statistics

    + To try out the new features yourself, you need to run Python 3.10. You can get it from the Python homepage. Alternatively, you can use Docker with the latest Python image. +

    + Better Error Messages +

    + Python is often lauded for being a user-friendly programming language. While this is true, there are certain parts of Python that could be friendlier. Python 3.10 comes with a host of more precise and constructive error messages. In this section, you’ll see some of the newest improvements. The full list is available in the documentation. +

    + Think back to writing your first Hello World program in Python: +

    + Python + +

    + Maybe you created a file, added the famous call to print(), and saved it as hello.py. You then ran the program, eager to call yourself a proper Pythonista. However, something went wrong: +

    + Shell +

    + There was a SyntaxError in the code. EOL, what does that even mean? You went back to your code, and after a bit of staring and searching, you realized that there was a missing quotation mark at the end of your string. +

    + One of the more impactful improvements in Python 3.10 is better and more precise error messages for many common issues. If you run your buggy Hello World in Python 3.10, you’ll get a bit more help than in earlier versions of Python: +

    + Shell +

    + The error message is still a bit technical, but gone is the mysterious EOL. Instead, the message tells you that you need to terminate your string! There are similar improvements to many different error messages, as you’ll see below. +

    + A SyntaxError is an error raised when your code is parsed, before it even starts to execute. Syntax errors can be tricky to debug because the interpreter provides imprecise or sometimes even misleading error messages. The following code is missing a curly brace to terminate the dictionary: +

    + Python + +

    + The missing closing curly brace that should have been on line 7 is an error. If you run this code with Python 3.9 or earlier, you’ll see the following error message: +

    + Python Traceback + +

    + The error message highlights line 8, but there are no syntactical problems in line 8! If you’ve experienced your share of syntax errors in Python, you might already know that the trick is to look at the lines before the one Python complains about. In this case, you’re looking for the missing closing brace on line 7. +

    + In Python 3.10, the same code shows a much more helpful and precise error message: +

    + Python Traceback + +

    + This points you straight to the offending dictionary and allows you to fix the issue in no time. +

    + There are a few other ways to mess up dictionary syntax. A typical one is forgetting a comma after one of the items: +

    + Python + +

    + In this code, a comma is missing at the end of line 4. Python 3.10 gives you a clear suggestion on how to fix your code: +

    + Python Traceback + +

    + You can add the missing comma and have your code back up and running in no time. +

    + Another common mistake is using the assignment operator (=) instead of the equality comparison operator (==) when you’re comparing values. Previously, this would just cause another invalid syntax message. In the newest version of Python, you get some more advice: +

    + Python +

    + The parser suggests that you maybe meant to use a comparison operator or an assignment expression operator instead. +

    + Take note of another nifty improvement in Python 3.10 error messages. The last two examples show how carets (^^^) highlight the whole offending expression. Previously, a single caret symbol (^) indicated just an approximate location. +

    + The final error message improvement that you’ll play with for now is that attribute and name errors can now offer suggestions if you misspell an attribute or a name: +

    + Python +

    + Note that the suggestions work for both built-in names and names that you define yourself, although they may not be available in all environments. If you like these kinds of suggestions, check out BetterErrorMessages, which offers similar suggestions in even more contexts. +

    + The improvements you’ve seen in this section are just some of the many error messages that have gotten a face-lift. The new Python will be even more user-friendly than before, and hopefully, the new error messages will save you both time and frustration going forward. +

    + Structural Pattern Matching +

    + The biggest new feature in Python 3.10, probably both in terms of controversy and potential impact, is structural pattern matching. Its introduction has sometimes been referred to as switch ... case coming to Python, but you’ll see that structural pattern matching is much more powerful than that. +

    + You’ll see three different examples that together highlight why this feature is called structural pattern matching and show you how you can use this new feature: +

    1. Detecting and deconstructing different structures in your data
    2. Using different kinds of patterns
    3. Matching literal patterns

    + Structural pattern matching is a comprehensive addition to the Python language. To give you a taste of how you can take advantage of it in your own projects, the next three subsections will dive into some of the details. You’ll also see some links that can help you explore in even more depth if you want. +

    + Deconstructing Data Structures +

    + At its core, structural pattern matching is about defining patterns to which your data structures can be matched. In this section, you’ll study a practical example where you’ll work with data that are structured differently, even though the meaning is the same. You’ll define several patterns, and depending on which pattern matches your data, you’ll process your data appropriately. +

    + This section will be a bit light on explanations of the possible patterns. Instead, it will try to give you an impression of the possibilities. The next section will step back and explain the patterns in more detail. +

    + Time to match your first pattern! The following example uses a match ... case block to find the first name of a user by extracting it from a user data structure: +

    + Python +

    + You can see structural pattern matching at work in the highlighted lines. user is a small dictionary with user information. The case line specifies a pattern that user is matched against. In this case, you’re looking for a dictionary with a "name" key whose value is a new dictionary. This nested dictionary has a key called "first". The corresponding value is bound to the variable first_name. +

    + For a practical example, say that you’re processing user data where the underlying data model changes over time. Therefore, you need to be able to process different versions of the same data. +

    + In the next example, you’ll use data from randomuser.me. This is a great API for generating random user data that you can use during testing and development. The API is also an example of an API that has changed over time. You can still access the old versions of the API. +

    + You may expand the collapsed section below to see how you can use requests to obtain different versions of the user data using the API: +

    + You can get a random user from the API using requests as follows: +

    + Python + +

    + get_user() gets one random user in JSON format. Note the version parameter. The structure of the returned data has changed quite a bit between earlier versions like "1.1" and the current version "1.3", but in each case, the actual user data are contained in a list inside the "results" array. The function returns the first—and only—user in this list. +

    + At the time of writing, the latest version of the API is 1.3 and the data has the following structure: +

    + JSON + +

    + One of the members that changed between different versions is "dob", the date of birth. Note that in version 1.3, this is a JSON object with two members, "date" and "age". +

    + Compare the result above with a version 1.1 random user: +

    + JSON + +

    + Observe that in this older format, the value of the "dob" member is a plain string. +

    + In this example, you’ll work with the information about the date of birth (dob) for each user. The structure of these data has changed between different versions of the Random User API: +

    + JSON + +

    + Note that in version 1.1, the date of birth is represented as a simple string, while in version 1.3, it’s a JSON object with two members: "date" and "age". Say that you want to find the age of a user. Depending on the structure of your data, you’d either need to calculate the age based on the date of birth or look up the age if it’s already available. +

    + Traditionally, you would detect the structure of the data with an if test, maybe based on the type of the "dob" field. You can approach this differently in Python 3.10. Now, you can use structural pattern matching instead: +

    + Python + +

    + The match ... case construct is new in Python 3.10 and is how you perform structural pattern matching. You start with a match statement that specifies what you want to match. In this example, that’s the user data structure. +

    + One or several case statements follow match. Each case describes one pattern, and the indented block beneath it says what should happen if there’s a match. In this example: +

    • + Line 8 matches a dictionary with a "dob" key whose value is another dictionary with an integer (int) item named "age". The name age captures its value. +

    • + Line 10 matches any dictionary with a "dob" key. The name dob captures its value. +

    + One important feature of pattern matching is that at most one pattern will be matched. Since the pattern on line 10 matches any dictionary with "dob", it’s important that the more specific pattern on line 8 comes first. +

    + Before looking closer at the details of the patterns and how they work, try calling get_age() with different data structures to see the result: +

    + Python +

    + Your code can calculate the age correctly for both versions of the user data, which have different dates of birth. +

    + Look closer at those patterns. The first pattern, {"dob": {"age": int(age)}}, matches version 1.3 of the user data: +

    + Python + +

    + The first pattern is a nested pattern. The outer curly braces say that a dictionary with the key "dob" is required. The corresponding value should be a dictionary. This nested dictionary must match the subpattern {"age": int(age)}. In other words, it needs to have an "age" key with an integer value. That value is bound to the name age. +

    + The second pattern, {"dob": dob}, matches the older version 1.1 of the user data: +

    + Python + +

    + This second pattern is a simpler pattern than the first one. Again, the curly braces indicate that it will match a dictionary. However, any dictionary with a "dob" key is matched because there are no other restrictions specified. The value of that key is bound to the name dob. +

    + The main takeaway is that you can describe the structure of your data using mostly familiar notation. One striking change, though, is that you can use names like dob and age, which aren’t yet defined. Instead, values from your data are bound to these names when a pattern matches. +

    + You’ve explored some of the power of structural pattern matching in this example. In the next section, you’ll dive a bit more into the details. +

    + These documents give you a lot of background and detail if you’re interested in a deeper dive than what follows. +

    + Patterns are at the center of structural pattern matching. In this section, you’ll learn about some of the different kinds of patterns that exist: +

    • Mapping patterns match mapping structures like dictionaries.
    • Sequence patterns match sequence structures like tuples and lists.
    • Capture patterns bind values to names.
    • AS patterns bind the value of subpatterns to names.
    • OR patterns match one of several different subpatterns.
    • Wildcard patterns match anything.
    • Class patterns match class structures.
    • Value patterns match values stored in attributes.
    • Literal patterns match literal values.

    + You already used several of them in the example in the previous section. In particular, you used mapping patterns to unravel data stored in dictionaries. In this section, you’ll learn more about how some of these work. All the details are available in the PEPs mentioned above. +

    + A capture pattern is used to capture a match to a pattern and bind it to a name. Consider the following recursive function that sums a list of numbers: +

    + Python + +

    + The first case on line 3 matches the empty list and returns 0 as its sum. The second case on line 5 uses a sequence pattern with two capture patterns to match lists with one or more elements. The first element in the list is captured and bound to the name first. The second capture pattern, *rest, uses unpacking syntax to match any number of elements. rest will bind to a list containing all elements of numbers except the first one. +

    + sum_list() calculates the sum of a list of numbers by recursively adding the first number in the list and the sum of the rest of the numbers. You can use it as follows: +

    + Python +

    + The sum of 4 + 5 + 9 + 4 is correctly calculated to be 22. As an exercise for yourself, you can try to trace the recursive calls to sum_list() to make sure you understand how the code sums the whole list. +

    + sum_list() handles summing up a list of numbers. Observe what happens if you try to sum anything that isn’t a list: +

    + Python +

    + Passing a string or a number to sum_list() returns None. This occurs because none of the patterns match, and the execution continues after the match block. That happens to be the end of the function, so sum_list() implicitly returns None. +

    + Often, though, you want to be alerted about failed matches. You can add a catchall pattern as the final case that handles this by raising an error, for example. You can use the underscore (_) as a wildcard pattern that matches anything without binding it to a name. You can add some error handling to sum_list() as follows: +

    + Python + +

    + The final case will match anything that doesn’t match the first two patterns. This will raise a descriptive error, for instance, if you try to calculate sum_list(4594). This is useful when you need to alert your users that some input was not matched as expected. +

    + Your patterns are still not foolproof, though. Consider what happens if you try to sum a list of strings: +

    + Python +

    + The base case returns 0, so therefore the summing only works for types that you can add with numbers. Python doesn’t know how to add numbers and text strings together. You can restrict your pattern to only match integers using a class pattern: +

    + Python + +

    + Adding int() around first makes sure that the pattern only matches if the value is an integer. This might be too restrictive, though. Your function should be able to sum both integers and floating-point numbers, so how can you allow this in your pattern? +

    + To check whether at least one out of several subpatterns match, you can use an OR pattern. OR patterns consist of two or more subpatterns, and the pattern matches if at least one of the subpatterns does. You can use this to match when the first element is either of type int or type float: +

    + Python + +

    + You use the pipe symbol (|) to separate the subpatterns in an OR pattern. Your function now allows summing a list of floating-point numbers: +

    + Python +

    + There’s a lot of power and flexibility within structural pattern matching, even more than what you’ve seen so far. Some things that aren’t covered in this overview are: +

    + If you’re interested, have a look in the documentation to learn more about these features as well. In the next section, you’ll learn about literal patterns and value patterns. +

    + Matching Literal Patterns +

    + A literal pattern is a pattern that matches a literal object like an explicit string or number. In a sense, this is the most basic kind of pattern and allows you to emulate switch ... case statements seen in other languages. The following example matches a specific name: +

    + Python + +

    + The first case matches the literal string "Guido". In this case, you use _ as a wildcard to print a generic greeting whenever name is not "Guido". Such literal patterns can sometimes take the place of if ... elif ... else constructs and can play the same role that switch ... case does in some other languages. +

    + One limitation with structural pattern matching is that you can’t directly match values stored in variables. Say that you’ve defined bdfl = "Guido". A pattern like case bdfl: will not match "Guido". Instead, this will be interpreted as a capture pattern that matches anything and binds that value to bdfl, effectively overwriting the old value. +

    + You can, however, use a value pattern to match stored values. A value pattern looks a bit like a capture pattern but uses a previously defined dotted name that holds the value that will be matched against. +

    + You can, for example, use an enumeration to create such dotted names: +

    + Python + +

    + The first case now uses a value pattern to match Pythonista.BDFL, which is "Guido". Note that you can use any dotted name in a value pattern. You could, for example, have used a regular class or a module instead of the enumeration. +

    + To see a bigger example of how to use literal patterns, consider the game of FizzBuzz. This is a counting game where you should replace some numbers with words according to the following rules: +

    • You replace numbers divisible by 3 with fizz.
    • You replace numbers divisible by 5 with buzz.
    • You replace numbers divisible by both 3 and 5 with fizzbuzz.

    + FizzBuzz is sometimes used to introduce conditionals in programming education and as a screening problem in interviews. Even though a solution is quite straightforward, Joel Grus has written a full book about different ways to program the game. +

    + A typical solution in Python will use if ... elif ... else as follows: +

    + Python + +

    + The % operator calculates the modulus, which you can use to test divisibility. Namely, if a modulus b is 0 for two numbers a and b, then a is divisible by b. +

    + In fizzbuzz(), you calculate number % 3 and number % 5, which you then use to test for divisibility with 3 and 5. Note that you must do the test for divisibility with both 3 and 5 first. If not, numbers that are divisible by both 3 and 5 will be covered by either the "fizz" or the "buzz" cases instead. +

    + You can check that your implementation gives the expected result: +

    + Python +

    + You can confirm for yourself that 3 is divisible by 3, 65 is divisible by 5, and 15 is divisible by both 3 and 5, while 14 and 92 aren’t divisible by either 3 or 5. +

    + An if ... elif ... else structure where you’re comparing one or a few variables several times over is quite straightforward to rewrite using pattern matching instead. For example, you can do the following: +

    + Python + +

    + You match on both mod_3 and mod_5. Each case pattern then matches either the literal number 0 or the wildcard _ on the corresponding values. +

    + Compare and contrast this version with the previous one. Note how the pattern (0, 0) corresponds to the test mod_3 == 0 and mod_5 == 0, while (0, _) corresponds to mod_3 == 0. +

    + As you saw earlier, you can use an OR pattern to match on several different patterns. For example, since mod_3 can only take the values 0, 1, and 2, you can replace case (_, 0) with case (1, 0) | (2, 0). Remember that (0, 0) has already been covered. +

    + The Python core developers have consciously chosen not to include switch ... case statements in the language earlier. However, there are some third-party packages that do, like switchlang, which adds a switch command that also works on earlier versions of Python. +

    + Type Unions, Aliases, and Guards +

    + Reliably, each new Python release brings some improvements to the static typing system. Python 3.10 is no exception. In fact, four different PEPs about typing accompany this new release: +

    + PEP 604 will probably be the most widely used of these changes going forward, but you’ll get a brief overview of each of the features in this section. +

    + You can use union types to declare that a variable can have one of several different types. For example, you’ve been able to type hint a function calculating the mean of a list of numbers, floats, or integers as follows: +

    + Python + +

    + The annotation List[Union[float, int]] means that numbers should be a list where each element is either a floating-point number or an integer. This works well, but the notation is a bit verbose. Also, you need to import both List and Union from typing. +

    + In Python 3.10, you can replace Union[float, int] with the more succinct float | int. Combine this with the ability to use list instead of typing.List in type hints, which Python 3.9 introduced. You can then simplify your code while keeping all the type information: +

    + Python + +

    + The annotation of numbers is easier to read now, and as an added bonus, you didn’t need to import anything from typing. +

    + A special case of union types is when a variable can have either a specific type or be None. You can annotate such optional types either as Union[None, T] or, equivalently, Optional[T] for some type T. There is no new, special syntax for optional types, but you can use the new union syntax to avoid importing typing.Optional: +

    + Python + +

    + In this example, address is allowed to be either None or a string. +

    + You can also use the new union syntax at runtime in isinstance() or issubclass() tests: +

    + Python +

    + Traditionally, you’ve used tuples to test for several types at once—for example, (str, int) instead of str | int. This old syntax will still work. +

    + Type aliases allow you to quickly define new aliases that can stand in for more complicated type declarations. For example, say that you’re representing a playing card using a tuple of suit and rank strings and a deck of cards by a list of such playing card tuples. A deck of cards is then type hinted as list[tuple[str, str]]. +

    + To simplify type annotation, you define type aliases as follows: +

    + Python + +

    + This usually works okay. However, it’s often not possible for the type checker to know whether such a statement is a type alias or just the definition of a regular global variable. To help the type checker—or really, help the type checker help you—you can now explicitly annotate type aliases: +

    + Python + +

    + Adding the TypeAlias annotation clarifies the intention, both to a type checker and to anyone reading your code. +

    + Type guards are used to narrow down union types. The following function takes in either a string or None but always returns a tuple of strings representing a playing card: +

    + Python + +

    + The highlighted line works as a type guard, and static type checkers are able to realize that suit is necessarily a string when it’s returned. +

    + Currently, the type checkers can only use a few different constructs to narrow down union types in this way. With the new typing.TypeGuard, you can annotate custom functions that can be used to narrow down union types: +

    + Python + +

    + is_deck_of_cards() should return True or False depending on whether obj represents a Deck object or not. You can then use your guard function, and the type checker will be able to narrow down the types correctly: +

    + Python + +

    + Inside of the if block, the type checker knows that card_or_deck is, in fact, of the type Deck. See PEP 647 for more details. +

    + The final new typing feature is Parameter Specification Variables, which is related to type variables. Consider the definition of a decorator. In general, it looks something like the following: +

    + Python + +

    + The annotations mean that the function returned by the decorator is a callable with some parameters and the same return type, R, as the function passed into the decorator. The ellipsis (...) in the function header correctly allows any number of parameters, and each of those parameters can be of any type. However, there’s no validation that the returned callable has the same parameters as the function that was passed in. In practice, this means that type checkers aren’t able to check decorated functions properly. +

    + Unfortunately, you can’t use TypeVar for the parameters because you don’t know how many parameters the function will have. In Python 3.10, you’ll have access to ParamSpec in order to type hint these kinds of callables properly. ParamSpec works similarly to TypeVar but stands in for several parameters at once. You can rewrite your decorator as follows to take advantage of ParamSpec: +

    + Python + +

    + Note that you also use P when you annotate wrapper(). You can also use the new typing.Concatenate to add types to ParamSpec. See the documentation and PEP 612 for details and examples. +

    + Stricter Zipping of Sequences +

    + zip() is a built-in function in Python that can combine elements from several sequences. Python 3.10 introduces the new strict parameter, which adds a runtime test to check that all sequences being zipped have the same length. +

    + As an example, consider the following table of Lego sets: +

    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    + Name + + Set Number + + Pieces +
    + Louvre + + 21024 + + 695 +
    + Diagon Alley + + 75978 + + 5544 +
    + NASA Apollo Saturn V + + 92176 + + 1969 +
    + Millennium Falcon + + 75192 + + 7541 +
    + New York City + + 21028 + + 598 +

    + One way to represent these data in plain Python would be with each column as a list. It could look something like this: +

    + Python +

    + Note that you have three independent lists, but there’s an implicit correspondence between their elements. The first name ("Louvre"), the first set number ("21024"), and the first number of pieces (695) all describe the first Lego set. +

    + zip() can be used to iterate over these three lists in parallel: +

    + Python +

    + Note how each line collects information from all three lists and shows information about one particular set. This is a very common pattern that’s used in a lot of different Python code, including in the standard library. +

    + You can also add list() to collect the contents of all three lists in a single, nested list of tuples: +

    + Python +

    + Note how the nested list closely resembles the original table. +

    + The dark side of using zip() is that it’s quite easy to introduce a subtle bug that can be hard to discover. Note what happens if there’s a missing item in one of your lists: +

    + Python +

    + All the information about the New York City set disappeared! Additionally, the set numbers for Saturn V and Millennium Falcon are wrong. If your datasets are bigger, these kinds of errors can be very hard to discover. And even when you observe that something’s wrong, it’s not always easy to diagnose and fix. +

    + The issue is that you assumed that the three lists have the same number of elements and that the information is in the same order in each list. After set_numbers gets corrupted, this assumption is no longer true. +

    + PEP 618 introduces a new strict keyword parameter to zip() that you can use to confirm all sequences have the same length. In your example, it would raise an error alerting you to the corrupted list: +

    + Python +

    + When the iteration reaches the New York City Lego set, the second argument set_numbers is already exhausted, while there are still elements left in the first argument names. Instead of silently giving the wrong result, your code fails with an error, and you can take action to find and fix the mistake. +

    + There are use cases when you want to combine sequences of unequal length. Expand the box below to see how zip() and itertools.zip_longest() handle these: +

    + The following idiom divides the Lego sets into pairs: +

    + Python +

    + There are five sets, a number that doesn’t divide evenly into pairs. In this case, the default behavior of zip(), where the last element is dropped, might make sense. You could use strict=True here as well, but that would raise an error when your list can’t be split into pairs. A third option, which could be the best in this case, is to use zip_longest() from the itertools standard library. +

    + As the name suggests, zip_longest() combines sequences until the longest sequence is exhausted. If you use zip_longest() to divide the Lego sets, it becomes more explicit that New York City doesn’t have any pairing: +

    + Python +

    + Note that 'NYC' shows up in the last tuple together with an empty string. You can control what’s filled in for missing values with the fillvalue parameter. +

    + While strict is not really adding any new functionality to zip(), it can help you avoid those hard-to-find bugs. +

    + New Functions in the statistics Module +

    + The statistics module was added to the standard library all the way back in 2014 with the release of Python 3.4. The intent of statistics is to make statistical calculations at the level of graphing calculators available in Python. +

    + Python 3.10 adds a few multivariable functions to statistics: +

    • correlation() to calculate Pearson’s correlation coefficient for two variables
    • covariance() to calculate sample covariance for two variables
    • linear_regression() to calculate the slope and intercept in a linear regression

    + You can use each function to describe a certain aspect of the relationship between two variables. As an example, say that you have data from a set of blog posts—the number of words in each blog post and the number of views each post has had over some time period: +

    + Python +

    + You now want to investigate whether there’s any (linear) relationship between the number of words and number of views. In Python 3.10, you can calculate the correlation between words and views with the new correlation() function: +

    + Python +

    + The correlation between two variables is always a number between -1 and 1. If it’s close to 0, then there’s little correspondence between them, while a correlation close to -1 or 1 indicates that the behaviors of the two variables tend to follow each other. In this example, a correlation of 0.45 indicates that there’s a tendency for posts with more words to have more views, although it’s not a strong connection. +

    + You can also calculate the covariance between words and views. The covariance is another measure of the joint variability between two variables. You can calculate it with covariance(): +

    + Python +

    + In contrast to correlation, covariance is an absolute measure. It should be interpreted in the context of the variability within the variables themselves. In fact, you can normalize the covariance by the standard deviation of each variable to recover Pearson’s correlation coefficient: +

    + Python +

    + Note that this matches your earlier correlation coefficient exactly. +

    + A third way of looking at the linear correspondence between the two variables is through simple linear regression. You do the linear regression by calculating two numbers, slope and intercept, so that the (squared) error is minimized in the approximation number of views = slope × number of words + intercept. +

    + In Python 3.10, you can use linear_regression(): +

    + Python +

    + Based on this regression, a post with 10,074 words could expect about 0.2424 × 10074 + 1104 = 3546 views. However, as you saw earlier, the correlation between the number of words and the number of views is quite weak. Therefore, you shouldn’t expect this prediction to be very accurate. +

    + The LinearRegression object is a named tuple. This means that you can unpack the slope and intercept directly: +

    + Python +

    + Here, you use slope and intercept to predict the number of views on a blog post with 10,074 words. +

    + You still want to use some of the more advanced packages like pandas and statsmodels if you do a lot of statistical analysis. With the new additions to statistics in Python 3.10, however, you have the chance to do basic analysis more easily without bringing in third-party dependencies. +

    + Other Pretty Cool Features +

    + So far, you’ve seen the biggest and most impactful new features in Python 3.10. In this section, you’ll get a glimpse of a few of the other changes that the new version brings along. If you’re curious about all the changes made for this new version, check out the documentation. +

    + Default Text Encodings +

    + When you open a text file, the default encoding used to interpret the characters is system dependent. In particular, locale.getpreferredencoding() is used. On Mac and Linux, this usually returns "UTF-8", while the result on Windows is more varied. +

    + You should therefore always specify an encoding when you attempt to open a text file: +

    + Python + +

    + If you don’t explicitly specify an encoding, the preferred locale encoding is used, and you could experience that a file that can be read on one computer fails to open on another. +

    + Python 3.7 introduced UTF-8 mode, which allows you to force your programs to use UTF-8 encoding independent of the locale encoding. You can enable UTF-8 mode by giving the -X utf8 command-line option to the python executable or by setting the PYTHONUTF8 environment variable. +

    + In Python 3.10, you can activate a warning that will tell you when a text file is opened without a specified encoding. Consider the following script, which doesn’t specify an encoding: +

    + Python + +

    + The program will echo one or more text files back to the console, but with each line reversed. Run the program on itself with the encoding warning enabled: +

    + Shell +

    + Note the EncodingWarning printed to the console. The command-line option -X warn_default_encoding activates it. The warning will disappear if you specify an encoding—for example, encoding="utf-8"—when you open the file. +

    + There are times when you want to use the user-defined local encoding. You can still do so by explicitly using encoding="locale". However, it’s recommended to use UTF-8 whenever possible. You can check out PEP 597 for more information. +

    + Asynchronous Iteration +

    + Asynchronous programming is a powerful programming paradigm that’s been available in Python since version 3.5. You can recognize an asynchronous program by its use of the async keyword or special methods that start with .__a like .__aiter__() or .__aenter__(). +

    + In Python 3.10, two new asynchronous built-in functions are added: aiter() and anext(). In practice, these functions call the .__aiter__() and .__anext__() special methods—analogous to the regular iter() and next()—so no new functionality is added. These are convenience functions that make your code more readable. +

    + In other words, in the newest version of Python, the following statements—where things is an asynchronous iterable—are equivalent: +

    + Python +

    + In either case, it ends up as an asynchronous iterator. Expand the following box to see a complete example using aiter() and anext(): +

    + The following program counts the number of lines in several files. In practice, you use Python’s ability to iterate over files to count the number of lines. The script uses asynchronous iteration in order to handle several files concurrently. +

    + Note that you need to install the third-party aiofiles package with pip before running this code: +

    + Python + +

    + asyncio is used to create and run one asynchronous task per filename. count_lines() opens one file asynchronously and iterates through it using aiter() and anext() in order to count the number of lines. +

    + See PEP 525 to learn more about asynchronous iteration. +

    + Context Manager Syntax +

    + Context managers are great for managing resources in your programs. Until recently, though, their syntax has included an uncommon wart. You haven’t been allowed to use parentheses to break long with statements like this: +

    + Python + +

    + In earlier versions of Python, this causes an invalid syntax error message. Instead, you need to use a backslash (\) if you want to control where you break your lines: +

    + Python + +

    + While explicit line continuation with backslashes is possible in Python, PEP 8 discourages it. The Black formatting tool avoids backslashes completely. +

    + In Python 3.10, you’re now allowed to add parentheses around with statements to your heart’s content. Especially if you’re employing several context managers at once, like in the example above, this can help improve the readability of your code. Python’s documentation shows a few other possibilities with this new syntax. +

    + One small fun fact: parenthesized with statements actually work in version 3.9 of CPython. Their implementation came almost for free with the introduction of the PEG parser in Python 3.9. The reason that this is called a Python 3.10 feature is that using the PEG parser is voluntary in Python 3.9, while Python 3.9, with the old LL(1) parser, doesn’t support parenthesized with statements. +

    + Modern and Secure SSL +

    + Security can be challenging! A good rule of thumb is to avoid rolling your own security algorithms and instead rely on established packages. +

    + Python uses OpenSSL for different cryptographic features that are exposed in the hashlib, hmac, and ssl standard library modules. Your system can manage OpenSSL, or a Python installer can include OpenSSL. +

    + Python 3.9 supports using any of the OpenSSL versions 1.0.2 LTS, 1.1.0, and 1.1.1 LTS. Both OpenSSL 1.0.2 LTS and OpenSSL 1.1.0 are past their lifetime, so Python 3.10 will only support OpenSSL 1.1.1 LTS, as described in the following table: +

    + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    + Open SSL version + + Python 3.9 + + Python 3.10 + + End-of-life +
    + 1.0.2 LTS + + ✔ + + ✖ + + December 20, 2019 +
    + 1.1.0 + + ✔ + + ✖ + + September 10, 2019 +
    + 1.1.1 LTS + + ✔ + + ✔ + + September 11, 2023 +

    + This end of support for older versions will only affect you if you need to upgrade the system Python on an older operating system. If you use macOS or Windows, or if you install Python from python.org or use (Ana)Conda, you’ll see no change. +

    + However, Ubuntu 18.04 LTS uses OpenSSL 1.1.0, while Red Hat Enterprise Linux (RHEL) 7 and CentOS 7 both use OpenSSL 1.0.2 LTS. If you need to run Python 3.10 on these systems, you should look at installing it yourself using either the python.org or Conda installer. +

    + Dropping support for older versions of OpenSSL will make Python more secure. It’ll also help the Python developers in that code will be easier to maintain. Ultimately, this helps you because your Python experience will be more robust. See PEP 644 for more details. +

    + More Information About Your Python Interpreter +

    + The sys module contains a lot of information about your system, the current Python runtime, and the script currently being executed. You can, for example, inquire about the paths where Python looks for modules with sys.path and see all modules that have been imported in the current session with sys.modules. +

    + In Python 3.10, sys has two new attributes. First, you can now get a list of the names of all modules in the standard library: +

    + Python +

    + Here, you can see that there are around 300 modules in the standard library, several of which start with the letter z. Note that only top-level modules and packages are listed. Subpackages like importlib.metadata don’t get a separate entry. +

    + You will probably not be using sys.stdlib_module_names all that often. Still, the list ties in nicely with similar introspection features like keyword.kwlist and sys.builtin_module_names. +

    + One possible use case for the new attribute is to identify which of the currently imported modules are third-party dependencies: +

    + Python +

    + You find the imported top-level modules by looking at names in sys.modules that don’t have a dot in their name. By comparing them to the standard library module names, you find that numpy, dateutil, and pandas are some of the imported third-party modules in this example. +

    + The other new attribute is sys.orig_argv. This is related to sys.argv, which holds the command-line arguments given to your program when it was started. In contrast, sys.orig_argv lists the command-line arguments passed to the python executable itself. Consider the following example: +

    + Python + +

    + This script echoes back the orig_argv and argv lists. Run it to see how the information is captured: +

    + Shell +

    + Essentially, all arguments—including the name of the Python executable—end up in orig_argv. This is in contrast to argv, which only contains the arguments that aren’t handled by python itself. +

    + Again, this is not a feature that you’ll use a lot. If your program needs to concern itself with how it’s being run, you’re usually better off relying on information that’s already exposed instead of trying to parse this list. For example, you can choose to use the strict zip() mode only when your script is not running with the optimized flag, -O, like this: +

    + Python + +

    + The __debug__ flag is set when the interpreter starts. It’ll be False if you’re running python with -O or -OO specified, and True otherwise. Using __debug__ is usually preferable to "-O" not in sys.orig_argv or some similar construct. +

    + One of the motivating use cases for sys.orig_argv is that you can use it to spawn a new Python process with the same or modified command-line arguments as your current process. +

    + Future Annotations +

    + Annotations were introduced in Python 3 to give you a way to attach metadata to variables, function parameters, and return values. They are most commonly used to add type hints to your code. +

    + One challenge with annotations is that they must be valid Python code. For one thing, this makes it hard to type hint recursive classes. PEP 563 introduced postponed evaluation of annotations, making it possible to annotate with names that haven’t yet been defined. Since Python 3.7, you can activate postponed evaluation of annotations with a __future__ import: +

    + Python + +

    + The intention was that postponed evaluation would become the default at some point in the future. After the 2020 Python Language Summit, it was decided to make this happen in Python 3.10. +

    + However, after more testing, it became clear that postponed evaluation didn’t work well for projects that use annotations at runtime. Key people in the FastAPI and the Pydantic projects voiced their concerns. At the last minute, it was decided to reschedule these changes for Python 3.11. +

    + To ease the transition into future behavior, a few changes have been made in Python 3.10 as well. Most importantly, a new inspect.get_annotations() function has been added. You should call this to access annotations at runtime: +

    + Python +

    + Check out Annotations Best Practices for details. +

    + How to Detect Python 3.10 at Runtime +

    + Python 3.10 is the first version of Python with a two-digit minor version number. While this is mostly an interesting fun fact and an indication that Python 3 has been around for quite some time, it does also have some practical consequences. +

    + When your code needs to do something specific based on the version of Python at runtime, you’ve gotten away with doing a lexicographical comparison of version strings until now. While it’s never been good practice, it’s been possible to do the following: +

    + Python + +

    + In Python 3.10, this code will raise SystemExit and stop your program. This happens because, as strings, "3.10" is less than "3.6". +

    + The correct way to compare version numbers is to use tuples of numbers: +

    + Python + +

    + sys.version_info is a tuple object you can use for comparisons. +

    + If you’re doing these kinds of comparisons in your code, you should check your code with flake8-2020 to make sure you’re handling versions correctly: +

    + Shell +

    + With the flake8-2020 extension activated, you’ll get a recommendation about replacing sys.version with sys.version_info. +

    + So, Should You Upgrade to Python 3.10? +

    + You’ve now seen the coolest features of the newest and latest version of Python. The question now is whether you should upgrade to Python 3.10, and if yes, when you should do so. There are two different aspects to consider when thinking about upgrading to Python 3.10: +

    1. Should you upgrade your environment so that you run your code with the Python 3.10 interpreter?
    2. Should you write your code using the new Python 3.10 features?

    + Clearly, if you want to test out structural pattern matching or any of the other cool new features you’ve read about here, you need Python 3.10. It’s possible to install the latest version side by side with your current Python version. A straightforward way to do this is to use an environment manager like pyenv or Conda. You can also use Docker to run Python 3.10 without installing it locally. +

    + Python 3.10 has been through about five months of beta testing, so there shouldn’t be any big issues with starting to use it for your own development. You may find that some of your dependencies don’t immediately have wheels for Python 3.10 available, which makes them more cumbersome to install. But in general, using the newest Python for local development is fairly safe. +

    + As always, you should be careful before upgrading your production environment. Be vigilant about testing that your code runs well on the new version. In particular, you want to be on the lookout for features that are deprecated or removed. +

    + Whether you can start using the new features in your code or not depends on your user base and the environment where your code is running. If you can guarantee that Python 3.10 is available, then there’s no danger in using the new union type syntax or any other new feature. +

    + If you’re distributing an app or a library that’s used by others instead, you may want to be a bit more conservative. Currently, Python 3.6 is the oldest officially supported Python version. It reaches end-of-life in December 2021, after which Python 3.7 will be the minimum supported version. +

    + The documentation includes a useful guide about porting your code to Python 3.10. Check it out for more details! +

    + Conclusion +

    + The release of a new Python version is always worth celebrating. Even if you can’t start using the new features right away, they’ll become broadly available and part of your daily life within a few years. +

    + In this tutorial, you’ve seen new features like: +

    • Friendlier error messages
    • Powerful structural pattern matching
    • Type hint improvements
    • Safer combination of sequences
    • New statistics functions

    + For more Python 3.10 tips and a discussion with members of the Real Python team, check out Real Python Podcast Episode #81. +

    + Have fun trying out the new features! Share your experiences in the comments below. +

    + Watch Now This tutorial has a related video course created by the Real Python team. Watch it together with the written tutorial to deepen your understanding: Cool New Features in Python 3.10 +

    + Get a short & sweet Python Trick delivered to your inbox every couple of days. No spam ever. Unsubscribe any time. Curated by the Real Python team. +

    Python Tricks Dictionary MergeGeir Arne HjelleGeir Arne Hjelle

    + Geir Arne is an avid Pythonista and a member of the Real Python tutorial team. +

    \ No newline at end of file From fd7c2ffb49848bddaae5f83b1ef52882bcd9417d Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Thu, 25 Jan 2024 16:17:33 +0800 Subject: [PATCH 36/59] fix code blocks not formatted correctly in articles from wechat official accounts --- packages/api/src/utils/parser.ts | 37 +- packages/readabilityjs/Readability.js | 5 +- .../realpython/expected-metadata.json | 13522 ++++++++++++++++ .../test/test-pages/realpython/expected.html | 1028 ++ 4 files changed, 14574 insertions(+), 18 deletions(-) diff --git a/packages/api/src/utils/parser.ts b/packages/api/src/utils/parser.ts index 05fdcc3a2..a52d5968e 100644 --- a/packages/api/src/utils/parser.ts +++ b/packages/api/src/utils/parser.ts @@ -307,23 +307,28 @@ export const parsePreparedContent = async ( // Format code blocks // TODO: we probably want to move this type of thing // to the handlers, and have some concept of postHandle - const codeBlocks = newDocumentElement.querySelectorAll( - 'pre[class^="prism-"], pre[class^="language-"], pre[class^="code-snippet"], code' + const codeBlocks = newDocumentElement.querySelectorAll( + 'pre[class^="prism-"], pre[class^="language-"], code' ) - if (codeBlocks.length > 0) { - codeBlocks.forEach((e) => { - const att = hljs.highlightAuto(e.innerText) - const code = document.createElement('code') - const langClass = - `hljs language-${att.language}` + - (att.second_best?.language - ? ` language-${att.second_best?.language}` - : '') - code.setAttribute('class', langClass) - code.innerHTML = att.value - e.replaceWith(code) - }) - } + codeBlocks.forEach((e) => { + if (!e.textContent) { + return e.parentNode?.removeChild(e) + } + + // replace
    or

    or

    with \n + e.innerHTML = e.innerHTML.replace(/<(br|p|\/p)>/g, '\n') + + const att = hljs.highlightAuto(e.textContent) + const code = document.createElement('code') + const langClass = + `hljs language-${att.language}` + + (att.second_best?.language + ? ` language-${att.second_best?.language}` + : '') + code.setAttribute('class', langClass) + code.innerHTML = att.value + e.replaceWith(code) + }) highlightData = findEmbeddedHighlight(newDocumentElement) diff --git a/packages/readabilityjs/Readability.js b/packages/readabilityjs/Readability.js index be51c4bed..b268c0f2e 100644 --- a/packages/readabilityjs/Readability.js +++ b/packages/readabilityjs/Readability.js @@ -206,7 +206,7 @@ Readability.prototype = { unlikelyCandidates: /\bad\b|ai2html|banner|breadcrumbs|breadcrumb|combx|comment|community|cover-wrap|disqus|extra|footer|gdpr|header|legends|menu|related|remark|replies|rss|shoutbox|sidebar|skyscraper|social|sponsor|supplemental|ad-break|agegate|pagination|pager(?!ow)|popup|yom-remote|copyright|keywords|outline|infinite-list|beta|recirculation|site-index|hide-for-print|post-end-share-cta|post-end-cta-full|post-footer|post-head|post-tag|li-date|main-navigation|programtic-ads|outstream_article|hfeed|comment-holder|back-to-top|show-up-next|onward-journey|topic-tracker|list-nav|block-ad-entity|adSpecs|gift-article-button|modal-title|in-story-masthead|share-tools|standard-dock|expanded-dock|margins-h|subscribe-dialog|icon|bumped|dvz-social-media-buttons|post-toc|mobile-menu|mobile-navbar|tl_article_header|mvp(-post)*-(add-story|soc(-mob)*-wrap)|w-condition-invisible|rich-text-block main w-richtext|rich-text-block_ataglance at-a-glance test w-richtext|PostsPage-commentsSection|hide-text/i, // okMaybeItsACandidate: /and|article(?!-breadcrumb)|body|column|content|main|shadow|post-header/i, get okMaybeItsACandidate() { - return new RegExp(`and|(?", + 3, + ">", + 3, + ">", + -1, + 1, + "span", + 3, + "if", + -1, + 1, + "span", + 3, + "month", + -1, + 1, + "span", + 3, + "=", + -1, + 1, + "span", + 3, + "\"October\"", + -1, + 1, + "span", + 3, + ":", + -1, + 3, + "\n File ", + 1, + "span", + 3, + "\"", + 3, + "<", + 3, + "stdin", + 3, + ">", + 3, + "\"", + -1, + 3, + ", line ", + 1, + "span", + 3, + "1", + -1, + 1, + "span", + -1, + 1, + "span", + 3, + "if", + -1, + 1, + "span", + 3, + "month", + -1, + 1, + "span", + 3, + "=", + -1, + 1, + "span", + 3, + "\"October\"", + -1, + 1, + "span", + 3, + ":", + -1, + 1, + "span", + -1, + 1, + "span", + 3, + "^^^^^^^^^^^^^^^^^", + -1, + 1, + "span", + 3, + "SyntaxError", + -1, + 3, + ": ", + 1, + "span", + 3, + "invalid syntax. Maybe you meant '==' or ':=' instead of '='?", + -4, + 1, + "template", + 1, + "span", + 3, + "Copied!", + -3, + 1, "p", 3, "\n The parser suggests that you maybe meant to use a ", @@ -456,6 +1335,146 @@ "\n The final error message improvement that you’ll play with for now is that attribute and name errors can now offer suggestions if you misspell an attribute or a name:\n ", -1, 1, + "div", + 2, + "data-is-repl", + "true", + 2, + "data-syntax-language", + "pycon", + 2, + "aria-label", + "Code block", + 1, + "div", + 1, + "pre", + 1, + "code", + 1, + "span", + 3, + ">", + 3, + ">", + 3, + ">", + -1, + 1, + "span", + 3, + "import", + -1, + 1, + "span", + 3, + "math", + -1, + 1, + "span", + 3, + ">", + 3, + ">", + 3, + ">", + -1, + 1, + "span", + 3, + "math", + -1, + 1, + "span", + 3, + ".", + -1, + 1, + "span", + 3, + "py", + -1, + 1, + "span", + 1, + "span", + 3, + "AttributeError: module 'math' has no attribute 'py'. Did you mean: 'pi'?", + -2, + 1, + "span", + 3, + ">", + 3, + ">", + 3, + ">", + -1, + 1, + "span", + 3, + "pint", + -1, + 1, + "span", + 1, + "span", + 3, + "NameError: name 'pint' is not defined. Did you mean: 'print'?", + -2, + 1, + "span", + 3, + ">", + 3, + ">", + 3, + ">", + -1, + 1, + "span", + 3, + "release", + -1, + 1, + "span", + 3, + "=", + -1, + 1, + "span", + 3, + "\"3.10\"", + -1, + 1, + "span", + 3, + ">", + 3, + ">", + 3, + ">", + -1, + 1, + "span", + 3, + "relaese", + -1, + 1, + "span", + 1, + "span", + 3, + "NameError: name 'relaese' is not defined. Did you mean: 'release'?", + -5, + 1, + "template", + 1, + "span", + 3, + "Copied!", + -3, + 1, "p", 3, "\n Note that the suggestions work for both built-in names and names that you define yourself, although they may ", @@ -643,6 +1662,261 @@ " data structure:\n ", -1, 1, + "div", + 2, + "data-is-repl", + "true", + 2, + "data-syntax-language", + "pycon", + 2, + "aria-label", + "Code block", + 1, + "div", + 1, + "pre", + 1, + "code", + 1, + "span", + 3, + ">", + 3, + ">", + 3, + ">", + -1, + 1, + "span", + 3, + "user", + -1, + 1, + "span", + 3, + "=", + -1, + 1, + "span", + 3, + "{", + -1, + 1, + "span", + 3, + "... ", + -1, + 1, + "span", + 3, + "\"name\"", + -1, + 1, + "span", + 3, + ":", + -1, + 1, + "span", + 3, + "{", + -1, + 1, + "span", + 3, + "\"first\"", + -1, + 1, + "span", + 3, + ":", + -1, + 1, + "span", + 3, + "\"Pablo\"", + -1, + 1, + "span", + 3, + ",", + -1, + 1, + "span", + 3, + "\"last\"", + -1, + 1, + "span", + 3, + ":", + -1, + 1, + "span", + 3, + "\"Galindo Salgado\"", + -1, + 1, + "span", + 3, + "},", + -1, + 1, + "span", + 3, + "... ", + -1, + 1, + "span", + 3, + "\"title\"", + -1, + 1, + "span", + 3, + ":", + -1, + 1, + "span", + 3, + "\"Python 3.10 release manager\"", + -1, + 1, + "span", + 3, + ",", + -1, + 1, + "span", + 3, + "... ", + -1, + 1, + "span", + 3, + "}", + -1, + 1, + "span", + 1, + "span", + 3, + ">", + 3, + ">", + 3, + ">", + -1, + 1, + "span", + 3, + "match", + -1, + 1, + "span", + 3, + "user", + -1, + 1, + "span", + 3, + ":", + -2, + 1, + "span", + 1, + "span", + 3, + "... ", + -1, + 1, + "span", + 3, + "case", + -1, + 1, + "span", + 3, + "{", + -1, + 1, + "span", + 3, + "\"name\"", + -1, + 1, + "span", + 3, + ":", + -1, + 1, + "span", + 3, + "{", + -1, + 1, + "span", + 3, + "\"first\"", + -1, + 1, + "span", + 3, + ":", + -1, + 1, + "span", + 3, + "first_name", + -1, + 1, + "span", + 3, + "}}:", + -2, + 1, + "span", + 3, + "... ", + -1, + 1, + "span", + 3, + "pass", + -1, + 1, + "span", + 3, + "...", + -1, + 1, + "span", + 3, + ">", + 3, + ">", + 3, + ">", + -1, + 1, + "span", + 3, + "first_name", + -1, + 1, + "span", + 3, + "'Pablo'", + -4, + 1, + "template", + 1, + "span", + 3, + "Copied!", + -3, + 1, "p", 3, "\n You can see structural pattern matching at work in the highlighted lines. ", @@ -757,6 +2031,225 @@ " as follows:\n ", -1, 1, + "div", + 2, + "data-syntax-language", + "python", + 2, + "aria-label", + "Code block", + 1, + "div", + 1, + "pre", + 1, + "code", + 1, + "span", + 3, + "# random_user.py", + -1, + 1, + "span", + 3, + "import", + -1, + 1, + "span", + 3, + "requests", + -1, + 1, + "span", + 3, + "def", + -1, + 1, + "span", + 3, + "get_user", + -1, + 1, + "span", + 3, + "(", + -1, + 1, + "span", + 3, + "version", + -1, + 1, + "span", + 3, + "=", + -1, + 1, + "span", + 3, + "\"1.3\"", + -1, + 1, + "span", + 3, + "):", + -1, + 1, + "span", + -1, + 1, + "span", + 3, + "\"\"\"Get random users\"\"\"", + -1, + 1, + "span", + 3, + "url", + -1, + 1, + "span", + 3, + "=", + -1, + 1, + "span", + 3, + "f", + -1, + 1, + "span", + 3, + "\"https://randomuser.me/api/", + -1, + 1, + "span", + 3, + "{", + -1, + 1, + "span", + 3, + "version", + -1, + 1, + "span", + 3, + "}", + -1, + 1, + "span", + 3, + "/?results=1\"", + -1, + 1, + "span", + 3, + "response", + -1, + 1, + "span", + 3, + "=", + -1, + 1, + "span", + 3, + "requests", + -1, + 1, + "span", + 3, + ".", + -1, + 1, + "span", + 3, + "get", + -1, + 1, + "span", + 3, + "(", + -1, + 1, + "span", + 3, + "url", + -1, + 1, + "span", + 3, + ")", + -1, + 1, + "span", + 3, + "if", + -1, + 1, + "span", + 3, + "response", + -1, + 1, + "span", + 3, + ":", + -1, + 1, + "span", + 3, + "return", + -1, + 1, + "span", + 3, + "response", + -1, + 1, + "span", + 3, + ".", + -1, + 1, + "span", + 3, + "json", + -1, + 1, + "span", + 3, + "()[", + -1, + 1, + "span", + 3, + "\"results\"", + -1, + 1, + "span", + 3, + "][", + -1, + 1, + "span", + 3, + "0", + -1, + 1, + "span", + 3, + "]", + -4, + 1, + "template", + 1, + "span", + 3, + "Copied!", + -3, + 1, "p", 1, "code", @@ -810,6 +2303,1136 @@ "\n At the time of writing, the latest version of the API is 1.3 and the data has the following structure:\n ", -1, 1, + "div", + 2, + "data-syntax-language", + "json", + 2, + "aria-label", + "Code block", + 1, + "div", + 1, + "pre", + 1, + "code", + 1, + "span", + 3, + "{", + -1, + 1, + "span", + -1, + 1, + "span", + 3, + "\"gender\"", + -1, + 1, + "span", + 3, + ":", + -1, + 1, + "span", + -1, + 1, + "span", + 3, + "\"female\"", + -1, + 1, + "span", + 3, + ",", + -1, + 1, + "span", + -1, + 1, + "span", + 3, + "\"name\"", + -1, + 1, + "span", + 3, + ":", + -1, + 1, + "span", + -1, + 1, + "span", + 3, + "{", + -1, + 1, + "span", + -1, + 1, + "span", + 3, + "\"title\"", + -1, + 1, + "span", + 3, + ":", + -1, + 1, + "span", + -1, + 1, + "span", + 3, + "\"Miss\"", + -1, + 1, + "span", + 3, + ",", + -1, + 1, + "span", + -1, + 1, + "span", + 3, + "\"first\"", + -1, + 1, + "span", + 3, + ":", + -1, + 1, + "span", + -1, + 1, + "span", + 3, + "\"Ilona\"", + -1, + 1, + "span", + 3, + ",", + -1, + 1, + "span", + -1, + 1, + "span", + 3, + "\"last\"", + -1, + 1, + "span", + 3, + ":", + -1, + 1, + "span", + -1, + 1, + "span", + 3, + "\"Jokela\"", + -1, + 1, + "span", + -1, + 1, + "span", + 3, + "},", + -1, + 1, + "span", + -1, + 1, + "span", + 3, + "\"location\"", + -1, + 1, + "span", + 3, + ":", + -1, + 1, + "span", + -1, + 1, + "span", + 3, + "{", + -1, + 1, + "span", + -1, + 1, + "span", + 3, + "\"street\"", + -1, + 1, + "span", + 3, + ":", + -1, + 1, + "span", + -1, + 1, + "span", + 3, + "{", + -1, + 1, + "span", + -1, + 1, + "span", + 3, + "\"number\"", + -1, + 1, + "span", + 3, + ":", + -1, + 1, + "span", + -1, + 1, + "span", + 3, + "4473", + -1, + 1, + "span", + 3, + ",", + -1, + 1, + "span", + -1, + 1, + "span", + 3, + "\"name\"", + -1, + 1, + "span", + 3, + ":", + -1, + 1, + "span", + -1, + 1, + "span", + 3, + "\"Mannerheimintie\"", + -1, + 1, + "span", + -1, + 1, + "span", + 3, + "},", + -1, + 1, + "span", + -1, + 1, + "span", + 3, + "\"city\"", + -1, + 1, + "span", + 3, + ":", + -1, + 1, + "span", + -1, + 1, + "span", + 3, + "\"Harjavalta\"", + -1, + 1, + "span", + 3, + ",", + -1, + 1, + "span", + -1, + 1, + "span", + 3, + "\"state\"", + -1, + 1, + "span", + 3, + ":", + -1, + 1, + "span", + -1, + 1, + "span", + 3, + "\"Ostrobothnia\"", + -1, + 1, + "span", + 3, + ",", + -1, + 1, + "span", + -1, + 1, + "span", + 3, + "\"country\"", + -1, + 1, + "span", + 3, + ":", + -1, + 1, + "span", + -1, + 1, + "span", + 3, + "\"Finland\"", + -1, + 1, + "span", + 3, + ",", + -1, + 1, + "span", + -1, + 1, + "span", + 3, + "\"postcode\"", + -1, + 1, + "span", + 3, + ":", + -1, + 1, + "span", + -1, + 1, + "span", + 3, + "44879", + -1, + 1, + "span", + 3, + ",", + -1, + 1, + "span", + -1, + 1, + "span", + 3, + "\"coordinates\"", + -1, + 1, + "span", + 3, + ":", + -1, + 1, + "span", + -1, + 1, + "span", + 3, + "{", + -1, + 1, + "span", + -1, + 1, + "span", + 3, + "\"latitude\"", + -1, + 1, + "span", + 3, + ":", + -1, + 1, + "span", + -1, + 1, + "span", + 3, + "\"-6.0321\"", + -1, + 1, + "span", + 3, + ",", + -1, + 1, + "span", + -1, + 1, + "span", + 3, + "\"longitude\"", + -1, + 1, + "span", + 3, + ":", + -1, + 1, + "span", + -1, + 1, + "span", + 3, + "\"123.2213\"", + -1, + 1, + "span", + -1, + 1, + "span", + 3, + "},", + -1, + 1, + "span", + -1, + 1, + "span", + 3, + "\"timezone\"", + -1, + 1, + "span", + 3, + ":", + -1, + 1, + "span", + -1, + 1, + "span", + 3, + "{", + -1, + 1, + "span", + -1, + 1, + "span", + 3, + "\"offset\"", + -1, + 1, + "span", + 3, + ":", + -1, + 1, + "span", + -1, + 1, + "span", + 3, + "\"+5:30\"", + -1, + 1, + "span", + 3, + ",", + -1, + 1, + "span", + -1, + 1, + "span", + 3, + "\"description\"", + -1, + 1, + "span", + 3, + ":", + -1, + 1, + "span", + -1, + 1, + "span", + 3, + "\"Bombay, Calcutta, Madras, New Delhi\"", + -1, + 1, + "span", + -1, + 1, + "span", + 3, + "}", + -1, + 1, + "span", + -1, + 1, + "span", + 3, + "},", + -1, + 1, + "span", + -1, + 1, + "span", + 3, + "\"email\"", + -1, + 1, + "span", + 3, + ":", + -1, + 1, + "span", + -1, + 1, + "span", + 3, + "\"ilona.jokela@example.com\"", + -1, + 1, + "span", + 3, + ",", + -1, + 1, + "span", + -1, + 1, + "span", + 3, + "\"login\"", + -1, + 1, + "span", + 3, + ":", + -1, + 1, + "span", + -1, + 1, + "span", + 3, + "{", + -1, + 1, + "span", + -1, + 1, + "span", + 3, + "\"uuid\"", + -1, + 1, + "span", + 3, + ":", + -1, + 1, + "span", + -1, + 1, + "span", + 3, + "\"632b7617-6312-4edf-9c24-d6334a6af52d\"", + -1, + 1, + "span", + 3, + ",", + -1, + 1, + "span", + -1, + 1, + "span", + 3, + "\"username\"", + -1, + 1, + "span", + 3, + ":", + -1, + 1, + "span", + -1, + 1, + "span", + 3, + "\"brownsnake482\"", + -1, + 1, + "span", + 3, + ",", + -1, + 1, + "span", + -1, + 1, + "span", + 3, + "\"password\"", + -1, + 1, + "span", + 3, + ":", + -1, + 1, + "span", + -1, + 1, + "span", + 3, + "\"biatch\"", + -1, + 1, + "span", + 3, + ",", + -1, + 1, + "span", + -1, + 1, + "span", + 3, + "\"salt\"", + -1, + 1, + "span", + 3, + ":", + -1, + 1, + "span", + -1, + 1, + "span", + 3, + "\"ofk518ZW\"", + -1, + 1, + "span", + 3, + ",", + -1, + 1, + "span", + -1, + 1, + "span", + 3, + "\"md5\"", + -1, + 1, + "span", + 3, + ":", + -1, + 1, + "span", + -1, + 1, + "span", + 3, + "\"6d589615ca44f6e583c85d45bf431c54\"", + -1, + 1, + "span", + 3, + ",", + -1, + 1, + "span", + -1, + 1, + "span", + 3, + "\"sha1\"", + -1, + 1, + "span", + 3, + ":", + -1, + 1, + "span", + -1, + 1, + "span", + 3, + "\"cd87c931d579bdff77af96c09e0eea82d1edfc19\"", + -1, + 1, + "span", + 3, + ",", + -1, + 1, + "span", + -1, + 1, + "span", + 3, + "\"sha256\"", + -1, + 1, + "span", + 3, + ":", + -1, + 1, + "span", + -1, + 1, + "span", + 3, + "\"6038ede83d4ce74116faa67fb3b1b2e6f6898e5749b57b5a0312bd46a539214a\"", + -1, + 1, + "span", + -1, + 1, + "span", + 3, + "},", + -1, + 1, + "span", + 1, + "span", + -1, + 1, + "span", + 3, + "\"dob\"", + -1, + 1, + "span", + 3, + ":", + -1, + 1, + "span", + -1, + 1, + "span", + 3, + "{", + -2, + 1, + "span", + 1, + "span", + -1, + 1, + "span", + 3, + "\"age\"", + -1, + 1, + "span", + 3, + ":", + -1, + 1, + "span", + -1, + 1, + "span", + 3, + "64", + -2, + 1, + "span", + 1, + "span", + -1, + 1, + "span", + 3, + "},", + -2, + 1, + "span", + -1, + 1, + "span", + 3, + "\"registered\"", + -1, + 1, + "span", + 3, + ":", + -1, + 1, + "span", + -1, + 1, + "span", + 3, + "{", + -1, + 1, + "span", + -1, + 1, + "span", + 3, + "\"date\"", + -1, + 1, + "span", + 3, + ":", + -1, + 1, + "span", + -1, + 1, + "span", + 3, + ",", + -1, + 1, + "span", + -1, + 1, + "span", + 3, + "\"age\"", + -1, + 1, + "span", + 3, + ":", + -1, + 1, + "span", + -1, + 1, + "span", + 3, + "15", + -1, + 1, + "span", + -1, + 1, + "span", + 3, + "},", + -1, + 1, + "span", + -1, + 1, + "span", + 3, + "\"phone\"", + -1, + 1, + "span", + 3, + ":", + -1, + 1, + "span", + -1, + 1, + "span", + 3, + "\"07-369-318\"", + -1, + 1, + "span", + 3, + ",", + -1, + 1, + "span", + -1, + 1, + "span", + 3, + "\"cell\"", + -1, + 1, + "span", + 3, + ":", + -1, + 1, + "span", + -1, + 1, + "span", + 3, + "\"048-284-01-59\"", + -1, + 1, + "span", + 3, + ",", + -1, + 1, + "span", + -1, + 1, + "span", + 3, + "\"id\"", + -1, + 1, + "span", + 3, + ":", + -1, + 1, + "span", + -1, + 1, + "span", + 3, + "{", + -1, + 1, + "span", + -1, + 1, + "span", + 3, + "\"name\"", + -1, + 1, + "span", + 3, + ":", + -1, + 1, + "span", + -1, + 1, + "span", + 3, + "\"HETU\"", + -1, + 1, + "span", + 3, + ",", + -1, + 1, + "span", + -1, + 1, + "span", + 3, + "\"value\"", + -1, + 1, + "span", + 3, + ":", + -1, + 1, + "span", + -1, + 1, + "span", + 3, + "\"NaNNA204undefined\"", + -1, + 1, + "span", + -1, + 1, + "span", + 3, + "},", + -1, + 1, + "span", + -1, + 1, + "span", + 3, + "\"picture\"", + -1, + 1, + "span", + 3, + ":", + -1, + 1, + "span", + -1, + 1, + "span", + 3, + "{", + -1, + 1, + "span", + -1, + 1, + "span", + 3, + "\"large\"", + -1, + 1, + "span", + 3, + ":", + -1, + 1, + "span", + -1, + 1, + "span", + 3, + "\"https://randomuser.me/api/portraits/women/28.jpg\"", + -1, + 1, + "span", + 3, + ",", + -1, + 1, + "span", + -1, + 1, + "span", + 3, + "\"medium\"", + -1, + 1, + "span", + 3, + ":", + -1, + 1, + "span", + -1, + 1, + "span", + 3, + "\"https://randomuser.me/api/portraits/med/women/28.jpg\"", + -1, + 1, + "span", + 3, + ",", + -1, + 1, + "span", + -1, + 1, + "span", + 3, + "\"thumbnail\"", + -1, + 1, + "span", + 3, + ":", + -1, + 1, + "span", + -1, + 1, + "span", + 3, + "\"https://randomuser.me/api/portraits/thumb/women/28.jpg\"", + -1, + 1, + "span", + -1, + 1, + "span", + 3, + "},", + -1, + 1, + "span", + -1, + 1, + "span", + 3, + "\"nat\"", + -1, + 1, + "span", + 3, + ":", + -1, + 1, + "span", + -1, + 1, + "span", + 3, + "\"FI\"", + -1, + 1, + "span", + 3, + "}", + -4, + 1, + "template", + 1, + "span", + 3, + "Copied!", + -3, + 1, "p", 3, "\n One of the members that changed between different versions is ", @@ -841,6 +3464,771 @@ "\n Compare the result above with a version 1.1 random user:\n ", -1, 1, + "div", + 2, + "data-syntax-language", + "json", + 2, + "aria-label", + "Code block", + 1, + "div", + 1, + "pre", + 1, + "code", + 1, + "span", + 3, + "{", + -1, + 1, + "span", + -1, + 1, + "span", + 3, + "\"gender\"", + -1, + 1, + "span", + 3, + ":", + -1, + 1, + "span", + -1, + 1, + "span", + 3, + "\"female\"", + -1, + 1, + "span", + 3, + ",", + -1, + 1, + "span", + -1, + 1, + "span", + 3, + "\"name\"", + -1, + 1, + "span", + 3, + ":", + -1, + 1, + "span", + -1, + 1, + "span", + 3, + "{", + -1, + 1, + "span", + -1, + 1, + "span", + 3, + "\"title\"", + -1, + 1, + "span", + 3, + ":", + -1, + 1, + "span", + -1, + 1, + "span", + 3, + "\"miss\"", + -1, + 1, + "span", + 3, + ",", + -1, + 1, + "span", + -1, + 1, + "span", + 3, + "\"first\"", + -1, + 1, + "span", + 3, + ":", + -1, + 1, + "span", + -1, + 1, + "span", + 3, + "\"ilona\"", + -1, + 1, + "span", + 3, + ",", + -1, + 1, + "span", + -1, + 1, + "span", + 3, + "\"last\"", + -1, + 1, + "span", + 3, + ":", + -1, + 1, + "span", + -1, + 1, + "span", + 3, + "\"jokela\"", + -1, + 1, + "span", + -1, + 1, + "span", + 3, + "},", + -1, + 1, + "span", + -1, + 1, + "span", + 3, + "\"location\"", + -1, + 1, + "span", + 3, + ":", + -1, + 1, + "span", + -1, + 1, + "span", + 3, + "{", + -1, + 1, + "span", + -1, + 1, + "span", + 3, + "\"street\"", + -1, + 1, + "span", + 3, + ":", + -1, + 1, + "span", + -1, + 1, + "span", + 3, + "\"7336 myllypuronkatu\"", + -1, + 1, + "span", + 3, + ",", + -1, + 1, + "span", + -1, + 1, + "span", + 3, + "\"city\"", + -1, + 1, + "span", + 3, + ":", + -1, + 1, + "span", + -1, + 1, + "span", + 3, + "\"kurikka\"", + -1, + 1, + "span", + 3, + ",", + -1, + 1, + "span", + -1, + 1, + "span", + 3, + "\"state\"", + -1, + 1, + "span", + 3, + ":", + -1, + 1, + "span", + -1, + 1, + "span", + 3, + "\"central ostrobothnia\"", + -1, + 1, + "span", + 3, + ",", + -1, + 1, + "span", + -1, + 1, + "span", + 3, + "\"postcode\"", + -1, + 1, + "span", + 3, + ":", + -1, + 1, + "span", + -1, + 1, + "span", + 3, + "53740", + -1, + 1, + "span", + -1, + 1, + "span", + 3, + "},", + -1, + 1, + "span", + -1, + 1, + "span", + 3, + "\"email\"", + -1, + 1, + "span", + 3, + ":", + -1, + 1, + "span", + -1, + 1, + "span", + 3, + "\"ilona.jokela@example.com\"", + -1, + 1, + "span", + 3, + ",", + -1, + 1, + "span", + -1, + 1, + "span", + 3, + "\"login\"", + -1, + 1, + "span", + 3, + ":", + -1, + 1, + "span", + -1, + 1, + "span", + 3, + "{", + -1, + 1, + "span", + -1, + 1, + "span", + 3, + "\"username\"", + -1, + 1, + "span", + 3, + ":", + -1, + 1, + "span", + -1, + 1, + "span", + 3, + "\"blackelephant837\"", + -1, + 1, + "span", + 3, + ",", + -1, + 1, + "span", + -1, + 1, + "span", + 3, + "\"password\"", + -1, + 1, + "span", + 3, + ":", + -1, + 1, + "span", + -1, + 1, + "span", + 3, + "\"sand\"", + -1, + 1, + "span", + 3, + ",", + -1, + 1, + "span", + -1, + 1, + "span", + 3, + "\"salt\"", + -1, + 1, + "span", + 3, + ":", + -1, + 1, + "span", + -1, + 1, + "span", + 3, + "\"yofk518Z\"", + -1, + 1, + "span", + 3, + ",", + -1, + 1, + "span", + -1, + 1, + "span", + 3, + "\"md5\"", + -1, + 1, + "span", + 3, + ":", + -1, + 1, + "span", + -1, + 1, + "span", + 3, + "\"b26367ea967600d679ee3e0b9bda012f\"", + -1, + 1, + "span", + 3, + ",", + -1, + 1, + "span", + -1, + 1, + "span", + 3, + "\"sha1\"", + -1, + 1, + "span", + 3, + ":", + -1, + 1, + "span", + -1, + 1, + "span", + 3, + "\"87d2910595acba5b8e8aa8b00a841bab08580e2f\"", + -1, + 1, + "span", + 3, + ",", + -1, + 1, + "span", + -1, + 1, + "span", + 3, + "\"sha256\"", + -1, + 1, + "span", + 3, + ":", + -1, + 1, + "span", + -1, + 1, + "span", + 3, + "\"73bd0d205d0dc83ae184ae222ff2e9de5ea4039119a962c4f97fabd5bbfa7aca\"", + -1, + 1, + "span", + -1, + 1, + "span", + 3, + "},", + -1, + 1, + "span", + -1, + 1, + "span", + 3, + "\"registered\"", + -1, + 1, + "span", + 3, + ":", + -1, + 1, + "span", + -1, + 1, + "span", + 3, + ",", + -1, + 1, + "span", + -1, + 1, + "span", + 3, + "\"phone\"", + -1, + 1, + "span", + 3, + ":", + -1, + 1, + "span", + -1, + 1, + "span", + 3, + "\"04-636-931\"", + -1, + 1, + "span", + 3, + ",", + -1, + 1, + "span", + -1, + 1, + "span", + 3, + "\"cell\"", + -1, + 1, + "span", + 3, + ":", + -1, + 1, + "span", + -1, + 1, + "span", + 3, + "\"048-828-40-15\"", + -1, + 1, + "span", + 3, + ",", + -1, + 1, + "span", + -1, + 1, + "span", + 3, + "\"id\"", + -1, + 1, + "span", + 3, + ":", + -1, + 1, + "span", + -1, + 1, + "span", + 3, + "{", + -1, + 1, + "span", + -1, + 1, + "span", + 3, + "\"name\"", + -1, + 1, + "span", + 3, + ":", + -1, + 1, + "span", + -1, + 1, + "span", + 3, + "\"HETU\"", + -1, + 1, + "span", + 3, + ",", + -1, + 1, + "span", + -1, + 1, + "span", + 3, + "\"value\"", + -1, + 1, + "span", + 3, + ":", + -1, + 1, + "span", + -1, + 1, + "span", + 3, + "\"366-9204\"", + -1, + 1, + "span", + -1, + 1, + "span", + 3, + "},", + -1, + 1, + "span", + -1, + 1, + "span", + 3, + "\"picture\"", + -1, + 1, + "span", + 3, + ":", + -1, + 1, + "span", + -1, + 1, + "span", + 3, + "{", + -1, + 1, + "span", + -1, + 1, + "span", + 3, + "\"large\"", + -1, + 1, + "span", + 3, + ":", + -1, + 1, + "span", + -1, + 1, + "span", + 3, + "\"https://randomuser.me/api/portraits/women/24.jpg\"", + -1, + 1, + "span", + 3, + ",", + -1, + 1, + "span", + -1, + 1, + "span", + 3, + "\"medium\"", + -1, + 1, + "span", + 3, + ":", + -1, + 1, + "span", + -1, + 1, + "span", + 3, + "\"https://randomuser.me/api/portraits/med/women/24.jpg\"", + -1, + 1, + "span", + 3, + ",", + -1, + 1, + "span", + -1, + 1, + "span", + 3, + "\"thumbnail\"", + -1, + 1, + "span", + 3, + ":", + -1, + 1, + "span", + -1, + 1, + "span", + 3, + "\"https://randomuser.me/api/portraits/thumb/women/24.jpg\"", + -1, + 1, + "span", + -1, + 1, + "span", + 3, + "},", + -1, + 1, + "span", + -1, + 1, + "span", + 3, + "\"nat\"", + -1, + 1, + "span", + 3, + ":", + -1, + 1, + "span", + -1, + 1, + "span", + 3, + "\"FI\"", + -1, + 1, + "span", + 3, + "}", + -4, + 1, + "template", + 1, + "span", + 3, + "Copied!", + -3, + 1, "p", 3, "\n Observe that in this older format, the value of the ", @@ -865,6 +4253,154 @@ ") for each user. The structure of these data has changed between different versions of the Random User API:\n ", -1, 1, + "div", + 2, + "data-syntax-language", + "json", + 2, + "aria-label", + "Code block", + 1, + "div", + 1, + "pre", + 1, + "code", + 1, + "span", + 3, + "#", + -1, + 1, + "span", + -1, + 1, + "span", + 3, + "Versio", + -1, + 1, + "span", + 3, + "n", + -1, + 1, + "span", + -1, + 1, + "span", + 3, + "1.1", + -1, + 1, + "span", + 3, + "\"dob\"", + -1, + 1, + "span", + 3, + ":", + -1, + 1, + "span", + -1, + 1, + "span", + 3, + "#", + -1, + 1, + "span", + -1, + 1, + "span", + 3, + "Versio", + -1, + 1, + "span", + 3, + "n", + -1, + 1, + "span", + -1, + 1, + "span", + 3, + "1.3", + -1, + 1, + "span", + 3, + "\"dob\"", + -1, + 1, + "span", + 3, + ":", + -1, + 1, + "span", + -1, + 1, + "span", + 3, + "{", + -1, + 1, + "span", + 3, + "\"date\"", + -1, + 1, + "span", + 3, + ":", + -1, + 1, + "span", + -1, + 1, + "span", + 3, + ",", + -1, + 1, + "span", + -1, + 1, + "span", + 3, + "\"age\"", + -1, + 1, + "span", + 3, + ":", + -1, + 1, + "span", + -1, + 1, + "span", + 3, + "64", + -1, + 1, + "span", + 3, + "}", + -4, + 1, + "template", + 1, + "span", + 3, + "Copied!", + -3, + 1, "p", 3, "\n Note that in version 1.1, the date of birth is represented as a simple string, while in version 1.3, it’s a JSON object with two members: ", @@ -903,6 +4439,396 @@ " field. You can approach this differently in Python 3.10. Now, you can use structural pattern matching instead:\n ", -1, 1, + "div", + 2, + "data-syntax-language", + "python", + 2, + "aria-label", + "Code block", + 1, + "div", + 1, + "pre", + 1, + "code", + 1, + "span", + 3, + " 1", + -1, + 1, + "span", + 3, + "# random_user.py (continued)", + -1, + 1, + "span", + 3, + " 2", + -1, + 1, + "span", + 3, + " 3", + -1, + 1, + "span", + 3, + "from", + -1, + 1, + "span", + 3, + "datetime", + -1, + 1, + "span", + 3, + "import", + -1, + 1, + "span", + 3, + "datetime", + -1, + 1, + "span", + 3, + " 4", + -1, + 1, + "span", + 3, + " 5", + -1, + 1, + "span", + 3, + "def", + -1, + 1, + "span", + 3, + "get_age", + -1, + 1, + "span", + 3, + "(", + -1, + 1, + "span", + 3, + "user", + -1, + 1, + "span", + 3, + "):", + -1, + 1, + "span", + 3, + " 6", + -1, + 1, + "span", + -1, + 1, + "span", + 3, + "\"\"\"Get the age of a user\"\"\"", + -1, + 1, + "span", + 1, + "span", + 3, + " 7", + -1, + 1, + "span", + 3, + "match", + -1, + 1, + "span", + 3, + "user", + -1, + 1, + "span", + 3, + ":", + -2, + 1, + "span", + 1, + "span", + 3, + " 8", + -1, + 1, + "span", + 3, + "case", + -1, + 1, + "span", + 3, + "{", + -1, + 1, + "span", + 3, + "\"dob\"", + -1, + 1, + "span", + 3, + ":", + -1, + 1, + "span", + 3, + "{", + -1, + 1, + "span", + 3, + "\"age\"", + -1, + 1, + "span", + 3, + ":", + -1, + 1, + "span", + 3, + "int", + -1, + 1, + "span", + 3, + "(", + -1, + 1, + "span", + 3, + "age", + -1, + 1, + "span", + 3, + ")}}:", + -2, + 1, + "span", + 3, + " 9", + -1, + 1, + "span", + 3, + "return", + -1, + 1, + "span", + 3, + "age", + -1, + 1, + "span", + 1, + "span", + 3, + "10", + -1, + 1, + "span", + 3, + "case", + -1, + 1, + "span", + 3, + "{", + -1, + 1, + "span", + 3, + "\"dob\"", + -1, + 1, + "span", + 3, + ":", + -1, + 1, + "span", + 3, + "dob", + -1, + 1, + "span", + 3, + "}:", + -2, + 1, + "span", + 3, + "11", + -1, + 1, + "span", + 3, + "now", + -1, + 1, + "span", + 3, + "=", + -1, + 1, + "span", + 3, + "datetime", + -1, + 1, + "span", + 3, + ".", + -1, + 1, + "span", + 3, + "now", + -1, + 1, + "span", + 3, + "()", + -1, + 1, + "span", + 3, + "12", + -1, + 1, + "span", + 3, + "dob_date", + -1, + 1, + "span", + 3, + "=", + -1, + 1, + "span", + 3, + "datetime", + -1, + 1, + "span", + 3, + ".", + -1, + 1, + "span", + 3, + "strptime", + -1, + 1, + "span", + 3, + "(", + -1, + 1, + "span", + 3, + "dob", + -1, + 1, + "span", + 3, + ",", + -1, + 1, + "span", + 3, + "\"%Y-%m-", + -1, + 1, + "span", + 3, + "%d", + -1, + 1, + "span", + 3, + " %H:%M:%S\"", + -1, + 1, + "span", + 3, + ")", + -1, + 1, + "span", + 3, + "13", + -1, + 1, + "span", + 3, + "return", + -1, + 1, + "span", + 3, + "now", + -1, + 1, + "span", + 3, + ".", + -1, + 1, + "span", + 3, + "year", + -1, + 1, + "span", + 3, + "-", + -1, + 1, + "span", + 3, + "dob_date", + -1, + 1, + "span", + 3, + ".", + -1, + 1, + "span", + 3, + "year", + -4, + 1, + "template", + 1, + "span", + 3, + "Copied!", + -3, + 1, "p", 3, "\n The ", @@ -1047,6 +4973,255 @@ " with different data structures to see the result:\n ", -1, 1, + "div", + 2, + "data-is-repl", + "true", + 2, + "data-syntax-language", + "pycon", + 2, + "aria-label", + "Code block", + 1, + "div", + 1, + "pre", + 1, + "code", + 1, + "span", + 3, + ">", + 3, + ">", + 3, + ">", + -1, + 1, + "span", + 3, + "import", + -1, + 1, + "span", + 3, + "random_user", + -1, + 1, + "span", + 3, + ">", + 3, + ">", + 3, + ">", + -1, + 1, + "span", + 3, + "users11", + -1, + 1, + "span", + 3, + "=", + -1, + 1, + "span", + 3, + "random_user", + -1, + 1, + "span", + 3, + ".", + -1, + 1, + "span", + 3, + "get_user", + -1, + 1, + "span", + 3, + "(", + -1, + 1, + "span", + 3, + "version", + -1, + 1, + "span", + 3, + "=", + -1, + 1, + "span", + 3, + "\"1.1\"", + -1, + 1, + "span", + 3, + ")", + -1, + 1, + "span", + 3, + ">", + 3, + ">", + 3, + ">", + -1, + 1, + "span", + 3, + "random_user", + -1, + 1, + "span", + 3, + ".", + -1, + 1, + "span", + 3, + "get_age", + -1, + 1, + "span", + 3, + "(", + -1, + 1, + "span", + 3, + "users11", + -1, + 1, + "span", + 3, + ")", + -1, + 1, + "span", + 3, + "55", + -1, + 1, + "span", + 3, + ">", + 3, + ">", + 3, + ">", + -1, + 1, + "span", + 3, + "users13", + -1, + 1, + "span", + 3, + "=", + -1, + 1, + "span", + 3, + "random_user", + -1, + 1, + "span", + 3, + ".", + -1, + 1, + "span", + 3, + "get_user", + -1, + 1, + "span", + 3, + "(", + -1, + 1, + "span", + 3, + "version", + -1, + 1, + "span", + 3, + "=", + -1, + 1, + "span", + 3, + "\"1.3\"", + -1, + 1, + "span", + 3, + ")", + -1, + 1, + "span", + 3, + ">", + 3, + ">", + 3, + ">", + -1, + 1, + "span", + 3, + "random_user", + -1, + 1, + "span", + 3, + ".", + -1, + 1, + "span", + 3, + "get_age", + -1, + 1, + "span", + 3, + "(", + -1, + 1, + "span", + 3, + "users13", + -1, + 1, + "span", + 3, + ")", + -1, + 1, + "span", + 3, + "64", + -4, + 1, + "template", + 1, + "span", + 3, + "Copied!", + -3, + 1, "p", 3, "\n Your code can calculate the age correctly for both versions of the user data, which have different dates of birth.\n ", @@ -1064,6 +5239,97 @@ ", matches version 1.3 of the user data:\n ", -1, 1, + "div", + 2, + "data-syntax-language", + "python", + 2, + "aria-label", + "Code block", + 1, + "div", + 1, + "pre", + 1, + "code", + 1, + "span", + 3, + "{", + -1, + 1, + "span", + 3, + "...", + -1, + 1, + "span", + 3, + "\"dob\"", + -1, + 1, + "span", + 3, + ":", + -1, + 1, + "span", + 3, + "{", + -1, + 1, + "span", + 3, + "\"date\"", + -1, + 1, + "span", + 3, + ":", + -1, + 1, + "span", + 3, + ",", + -1, + 1, + "span", + 3, + "\"age\"", + -1, + 1, + "span", + 3, + ":", + -1, + 1, + "span", + 3, + "64", + -1, + 1, + "span", + 3, + "},", + -1, + 1, + "span", + 3, + "...", + -1, + 1, + "span", + 3, + "}", + -4, + 1, + "template", + 1, + "span", + 3, + "Copied!", + -3, + 1, "p", 3, "\n The first pattern is a nested pattern. The outer curly braces say that a dictionary with the key ", @@ -1375,6 +5641,189 @@ " function that sums a list of numbers:\n ", -1, 1, + "div", + 2, + "data-syntax-language", + "python", + 2, + "aria-label", + "Code block", + 1, + "div", + 1, + "pre", + 1, + "code", + 1, + "span", + 3, + " 1", + -1, + 1, + "span", + 3, + "def", + -1, + 1, + "span", + 3, + "sum_list", + -1, + 1, + "span", + 3, + "(", + -1, + 1, + "span", + 3, + "numbers", + -1, + 1, + "span", + 3, + "):", + -1, + 1, + "span", + 3, + " 2", + -1, + 1, + "span", + 3, + "match", + -1, + 1, + "span", + 3, + "numbers", + -1, + 1, + "span", + 3, + ":", + -1, + 1, + "span", + 3, + " 3", + -1, + 1, + "span", + 3, + "case", + -1, + 1, + "span", + 3, + "[]:", + -1, + 1, + "span", + 3, + " 4", + -1, + 1, + "span", + 3, + "return", + -1, + 1, + "span", + 3, + "0", + -1, + 1, + "span", + 1, + "span", + 3, + " 5", + -1, + 1, + "span", + 3, + "case", + -1, + 1, + "span", + 3, + "[", + -1, + 1, + "span", + 3, + "first", + -1, + 1, + "span", + 3, + ",", + -1, + 1, + "span", + 3, + "*", + -1, + 1, + "span", + 3, + "rest", + -1, + 1, + "span", + 3, + "]:", + -2, + 1, + "span", + 3, + " 6", + -1, + 1, + "span", + 3, + "return", + -1, + 1, + "span", + 3, + "first", + -1, + 1, + "span", + 3, + "+", + -1, + 1, + "span", + 3, + "sum_list", + -1, + 1, + "span", + 3, + "(", + -1, + 1, + "span", + 3, + "rest", + -1, + 1, + "span", + 3, + ")", + -4, + 1, + "template", + 1, + "span", + 3, + "Copied!", + -3, + 1, "p", 3, "\n The first ", @@ -1456,6 +5905,94 @@ " calculates the sum of a list of numbers by recursively adding the first number in the list and the sum of the rest of the numbers. You can use it as follows:\n ", -1, 1, + "div", + 2, + "data-is-repl", + "true", + 2, + "data-syntax-language", + "pycon", + 2, + "aria-label", + "Code block", + 1, + "div", + 1, + "pre", + 1, + "code", + 1, + "span", + 3, + ">", + 3, + ">", + 3, + ">", + -1, + 1, + "span", + 3, + "sum_list", + -1, + 1, + "span", + 3, + "([", + -1, + 1, + "span", + 3, + "4", + -1, + 1, + "span", + 3, + ",", + -1, + 1, + "span", + 3, + "5", + -1, + 1, + "span", + 3, + ",", + -1, + 1, + "span", + 3, + "9", + -1, + 1, + "span", + 3, + ",", + -1, + 1, + "span", + 3, + "4", + -1, + 1, + "span", + 3, + "])", + -1, + 1, + "span", + 3, + "22", + -4, + 1, + "template", + 1, + "span", + 3, + "Copied!", + -3, + 1, "p", 3, "\n The sum of 4 + 5 + 9 + 4 is correctly calculated to be 22. As an exercise for yourself, you can try to trace the recursive calls to ", @@ -1478,6 +6015,118 @@ " handles summing up a list of numbers. Observe what happens if you try to sum anything that isn’t a list:\n ", -1, 1, + "div", + 2, + "data-is-repl", + "true", + 2, + "data-syntax-language", + "pycon", + 2, + "aria-label", + "Code block", + 1, + "div", + 1, + "pre", + 1, + "code", + 1, + "span", + 3, + ">", + 3, + ">", + 3, + ">", + -1, + 1, + "span", + 3, + "print", + -1, + 1, + "span", + 3, + "(", + -1, + 1, + "span", + 3, + "sum_list", + -1, + 1, + "span", + 3, + "(", + -1, + 1, + "span", + 3, + "\"4594\"", + -1, + 1, + "span", + 3, + "))", + -1, + 1, + "span", + 3, + "None", + -1, + 1, + "span", + 3, + ">", + 3, + ">", + 3, + ">", + -1, + 1, + "span", + 3, + "print", + -1, + 1, + "span", + 3, + "(", + -1, + 1, + "span", + 3, + "sum_list", + -1, + 1, + "span", + 3, + "(", + -1, + 1, + "span", + 3, + "4594", + -1, + 1, + "span", + 3, + "))", + -1, + 1, + "span", + 3, + "None", + -4, + 1, + "template", + 1, + "span", + 3, + "Copied!", + -3, + 1, "p", 3, "\n Passing a string or a number to ", @@ -1551,6 +6200,262 @@ " as follows:\n ", -1, 1, + "div", + 2, + "data-syntax-language", + "python", + 2, + "aria-label", + "Code block", + 1, + "div", + 1, + "pre", + 1, + "code", + 1, + "span", + 3, + "def", + -1, + 1, + "span", + 3, + "sum_list", + -1, + 1, + "span", + 3, + "(", + -1, + 1, + "span", + 3, + "numbers", + -1, + 1, + "span", + 3, + "):", + -1, + 1, + "span", + 3, + "match", + -1, + 1, + "span", + 3, + "numbers", + -1, + 1, + "span", + 3, + ":", + -1, + 1, + "span", + 3, + "case", + -1, + 1, + "span", + 3, + "[]:", + -1, + 1, + "span", + 3, + "return", + -1, + 1, + "span", + 3, + "0", + -1, + 1, + "span", + 3, + "case", + -1, + 1, + "span", + 3, + "[", + -1, + 1, + "span", + 3, + "first", + -1, + 1, + "span", + 3, + ",", + -1, + 1, + "span", + 3, + "*", + -1, + 1, + "span", + 3, + "rest", + -1, + 1, + "span", + 3, + "]:", + -1, + 1, + "span", + 3, + "return", + -1, + 1, + "span", + 3, + "first", + -1, + 1, + "span", + 3, + "+", + -1, + 1, + "span", + 3, + "sum_list", + -1, + 1, + "span", + 3, + "(", + -1, + 1, + "span", + 3, + "rest", + -1, + 1, + "span", + 3, + ")", + -1, + 1, + "span", + 1, + "span", + 3, + "case", + -1, + 1, + "span", + -1, + 1, + "span", + 3, + "_", + -1, + 1, + "span", + 3, + ":", + -2, + 1, + "span", + 3, + "wrong_type", + -1, + 1, + "span", + 3, + "=", + -1, + 1, + "span", + 3, + "numbers", + -1, + 1, + "span", + 3, + ".", + -1, + 1, + "span", + 3, + "__class__", + -1, + 1, + "span", + 3, + ".", + -1, + 1, + "span", + 3, + "__name__", + -1, + 1, + "span", + 3, + "raise", + -1, + 1, + "span", + 3, + "ValueError", + -1, + 1, + "span", + 3, + "(", + -1, + 1, + "span", + 3, + "f", + -1, + 1, + "span", + 3, + "\"Can only sum lists, not ", + -1, + 1, + "span", + 3, + "{", + -1, + 1, + "span", + 3, + "wrong_type", + -1, + 1, + "span", + 3, + "!r}", + -1, + 1, + "span", + 3, + "\"", + -1, + 1, + "span", + 3, + ")", + -4, + 1, + "template", + 1, + "span", + 3, + "Copied!", + -3, + 1, "p", 3, "\n The final ", @@ -1575,6 +6480,74 @@ "\n Your patterns are still not foolproof, though. Consider what happens if you try to sum a list of strings:\n ", -1, 1, + "div", + 2, + "data-is-repl", + "true", + 2, + "data-syntax-language", + "pycon", + 2, + "aria-label", + "Code block", + 1, + "div", + 1, + "pre", + 1, + "code", + 1, + "span", + 3, + ">", + 3, + ">", + 3, + ">", + -1, + 1, + "span", + 3, + "sum_list", + -1, + 1, + "span", + 3, + "([", + -1, + 1, + "span", + 3, + "\"45\"", + -1, + 1, + "span", + 3, + ",", + -1, + 1, + "span", + 3, + "\"94\"", + -1, + 1, + "span", + 3, + "])", + -1, + 1, + "span", + 3, + "TypeError: can only concatenate str (not \"int\") to str", + -4, + 1, + "template", + 1, + "span", + 3, + "Copied!", + -3, + 1, "p", 3, "\n The base case returns ", @@ -1594,6 +6567,217 @@ ":\n ", -1, 1, + "div", + 2, + "data-syntax-language", + "python", + 2, + "aria-label", + "Code block", + 1, + "div", + 1, + "pre", + 1, + "code", + 1, + "span", + 3, + "def", + -1, + 1, + "span", + 3, + "sum_list", + -1, + 1, + "span", + 3, + "(", + -1, + 1, + "span", + 3, + "numbers", + -1, + 1, + "span", + 3, + "):", + -1, + 1, + "span", + 3, + "match", + -1, + 1, + "span", + 3, + "numbers", + -1, + 1, + "span", + 3, + ":", + -1, + 1, + "span", + 3, + "case", + -1, + 1, + "span", + 3, + "[]:", + -1, + 1, + "span", + 3, + "return", + -1, + 1, + "span", + 3, + "0", + -1, + 1, + "span", + 1, + "span", + 3, + "case", + -1, + 1, + "span", + 3, + "[", + -1, + 1, + "span", + 3, + "int", + -1, + 1, + "span", + 3, + "(", + -1, + 1, + "span", + 3, + "first", + -1, + 1, + "span", + 3, + "),", + -1, + 1, + "span", + 3, + "*", + -1, + 1, + "span", + 3, + "rest", + -1, + 1, + "span", + 3, + "]:", + -2, + 1, + "span", + 3, + "return", + -1, + 1, + "span", + 3, + "first", + -1, + 1, + "span", + 3, + "+", + -1, + 1, + "span", + 3, + "sum_list", + -1, + 1, + "span", + 3, + "(", + -1, + 1, + "span", + 3, + "rest", + -1, + 1, + "span", + 3, + ")", + -1, + 1, + "span", + 3, + "case", + -1, + 1, + "span", + -1, + 1, + "span", + 3, + "_", + -1, + 1, + "span", + 3, + ":", + -1, + 1, + "span", + 3, + "raise", + -1, + 1, + "span", + 3, + "ValueError", + -1, + 1, + "span", + 3, + "(", + -1, + 1, + "span", + 3, + "f", + -1, + 1, + "span", + 3, + "\"Can only sum lists of numbers\"", + -1, + 1, + "span", + 3, + ")", + -4, + 1, + "template", + 1, + "span", + 3, + "Copied!", + -3, + 1, "p", 3, "\n Adding ", @@ -1659,6 +6843,242 @@ ":\n ", -1, 1, + "div", + 2, + "data-syntax-language", + "python", + 2, + "aria-label", + "Code block", + 1, + "div", + 1, + "pre", + 1, + "code", + 1, + "span", + 3, + "def", + -1, + 1, + "span", + 3, + "sum_list", + -1, + 1, + "span", + 3, + "(", + -1, + 1, + "span", + 3, + "numbers", + -1, + 1, + "span", + 3, + "):", + -1, + 1, + "span", + 3, + "match", + -1, + 1, + "span", + 3, + "numbers", + -1, + 1, + "span", + 3, + ":", + -1, + 1, + "span", + 3, + "case", + -1, + 1, + "span", + 3, + "[]:", + -1, + 1, + "span", + 3, + "return", + -1, + 1, + "span", + 3, + "0", + -1, + 1, + "span", + 1, + "span", + 3, + "case", + -1, + 1, + "span", + 3, + "[", + -1, + 1, + "span", + 3, + "int", + -1, + 1, + "span", + 3, + "(", + -1, + 1, + "span", + 3, + "first", + -1, + 1, + "span", + 3, + ")", + -1, + 1, + "span", + 3, + "|", + -1, + 1, + "span", + 3, + "float", + -1, + 1, + "span", + 3, + "(", + -1, + 1, + "span", + 3, + "first", + -1, + 1, + "span", + 3, + "),", + -1, + 1, + "span", + 3, + "*", + -1, + 1, + "span", + 3, + "rest", + -1, + 1, + "span", + 3, + "]:", + -2, + 1, + "span", + 3, + "return", + -1, + 1, + "span", + 3, + "first", + -1, + 1, + "span", + 3, + "+", + -1, + 1, + "span", + 3, + "sum_list", + -1, + 1, + "span", + 3, + "(", + -1, + 1, + "span", + 3, + "rest", + -1, + 1, + "span", + 3, + ")", + -1, + 1, + "span", + 3, + "case", + -1, + 1, + "span", + -1, + 1, + "span", + 3, + "_", + -1, + 1, + "span", + 3, + ":", + -1, + 1, + "span", + 3, + "raise", + -1, + 1, + "span", + 3, + "ValueError", + -1, + 1, + "span", + 3, + "(", + -1, + 1, + "span", + 3, + "f", + -1, + 1, + "span", + 3, + "\"Can only sum lists of numbers\"", + -1, + 1, + "span", + 3, + ")", + -4, + 1, + "template", + 1, + "span", + 3, + "Copied!", + -3, + 1, "p", 3, "\n You use the pipe symbol (", @@ -1671,6 +7091,84 @@ ") to separate the subpatterns in an OR pattern. Your function now allows summing a list of floating-point numbers:\n ", -1, 1, + "div", + 2, + "data-is-repl", + "true", + 2, + "data-syntax-language", + "pycon", + 2, + "aria-label", + "Code block", + 1, + "div", + 1, + "pre", + 1, + "code", + 1, + "span", + 3, + ">", + 3, + ">", + 3, + ">", + -1, + 1, + "span", + 3, + "sum_list", + -1, + 1, + "span", + 3, + "([", + -1, + 1, + "span", + 3, + "45.94", + -1, + 1, + "span", + 3, + ",", + -1, + 1, + "span", + 3, + "46.17", + -1, + 1, + "span", + 3, + ",", + -1, + 1, + "span", + 3, + "46.72", + -1, + 1, + "span", + 3, + "])", + -1, + 1, + "span", + 3, + "138.82999999999998", + -4, + 1, + "template", + 1, + "span", + 3, + "Copied!", + -3, + 1, "p", 3, "\n There’s a lot of power and flexibility within structural pattern matching, even more than what you’ve seen so far. Some things that aren’t covered in this overview are:\n ", @@ -1782,6 +7280,142 @@ " statements seen in other languages. The following example matches a specific name:\n ", -1, 1, + "div", + 2, + "data-syntax-language", + "python", + 2, + "aria-label", + "Code block", + 1, + "div", + 1, + "pre", + 1, + "code", + 1, + "span", + 3, + "def", + -1, + 1, + "span", + 3, + "greet", + -1, + 1, + "span", + 3, + "(", + -1, + 1, + "span", + 3, + "name", + -1, + 1, + "span", + 3, + "):", + -1, + 1, + "span", + 3, + "match", + -1, + 1, + "span", + 3, + "name", + -1, + 1, + "span", + 3, + ":", + -1, + 1, + "span", + 1, + "span", + 3, + "case", + -1, + 1, + "span", + 3, + "\"Guido\"", + -1, + 1, + "span", + 3, + ":", + -2, + 1, + "span", + 3, + "print", + -1, + 1, + "span", + 3, + "(", + -1, + 1, + "span", + 3, + "\"Hi, Guido!\"", + -1, + 1, + "span", + 3, + ")", + -1, + 1, + "span", + 3, + "case", + -1, + 1, + "span", + -1, + 1, + "span", + 3, + "_", + -1, + 1, + "span", + 3, + ":", + -1, + 1, + "span", + 3, + "print", + -1, + 1, + "span", + 3, + "(", + -1, + 1, + "span", + 3, + "\"Howdy, stranger!\"", + -1, + 1, + "span", + 3, + ")", + -4, + 1, + "template", + 1, + "span", + 3, + "Copied!", + -3, + 1, "p", 3, "\n The first ", @@ -1896,6 +7530,237 @@ " to create such dotted names:\n ", -1, 1, + "div", + 2, + "data-syntax-language", + "python", + 2, + "aria-label", + "Code block", + 1, + "div", + 1, + "pre", + 1, + "code", + 1, + "span", + 3, + "import", + -1, + 1, + "span", + 3, + "enum", + -1, + 1, + "span", + 3, + "class", + -1, + 1, + "span", + 3, + "Pythonista", + -1, + 1, + "span", + 3, + "(", + -1, + 1, + "span", + 3, + "str", + -1, + 1, + "span", + 3, + ",", + -1, + 1, + "span", + 3, + "enum", + -1, + 1, + "span", + 3, + ".", + -1, + 1, + "span", + 3, + "Enum", + -1, + 1, + "span", + 3, + "):", + -1, + 1, + "span", + 3, + "BDFL", + -1, + 1, + "span", + 3, + "=", + -1, + 1, + "span", + 3, + "\"Guido\"", + -1, + 1, + "span", + 3, + "FLUFL", + -1, + 1, + "span", + 3, + "=", + -1, + 1, + "span", + 3, + "\"Barry\"", + -1, + 1, + "span", + 3, + "def", + -1, + 1, + "span", + 3, + "greet", + -1, + 1, + "span", + 3, + "(", + -1, + 1, + "span", + 3, + "name", + -1, + 1, + "span", + 3, + "):", + -1, + 1, + "span", + 3, + "match", + -1, + 1, + "span", + 3, + "name", + -1, + 1, + "span", + 3, + ":", + -1, + 1, + "span", + 1, + "span", + 3, + "case", + -1, + 1, + "span", + 3, + "Pythonista", + -1, + 1, + "span", + 3, + ".", + -1, + 1, + "span", + 3, + "BDFL", + -1, + 1, + "span", + 3, + ":", + -2, + 1, + "span", + 3, + "print", + -1, + 1, + "span", + 3, + "(", + -1, + 1, + "span", + 3, + "\"Hi, Guido!\"", + -1, + 1, + "span", + 3, + ")", + -1, + 1, + "span", + 3, + "case", + -1, + 1, + "span", + -1, + 1, + "span", + 3, + "_", + -1, + 1, + "span", + 3, + ":", + -1, + 1, + "span", + 3, + "print", + -1, + 1, + "span", + 3, + "(", + -1, + 1, + "span", + 3, + "\"Howdy, stranger!\"", + -1, + 1, + "span", + 3, + ")", + -4, + 1, + "template", + 1, + "span", + 3, + "Copied!", + -3, + 1, "p", 3, "\n The first case now uses a value pattern to match ", @@ -2033,6 +7898,262 @@ " as follows:\n ", -1, 1, + "div", + 2, + "data-syntax-language", + "python", + 2, + "aria-label", + "Code block", + 1, + "div", + 1, + "pre", + 1, + "code", + 1, + "span", + 3, + "def", + -1, + 1, + "span", + 3, + "fizzbuzz", + -1, + 1, + "span", + 3, + "(", + -1, + 1, + "span", + 3, + "number", + -1, + 1, + "span", + 3, + "):", + -1, + 1, + "span", + 3, + "mod_3", + -1, + 1, + "span", + 3, + "=", + -1, + 1, + "span", + 3, + "number", + -1, + 1, + "span", + 3, + "%", + -1, + 1, + "span", + 3, + "3", + -1, + 1, + "span", + 3, + "mod_5", + -1, + 1, + "span", + 3, + "=", + -1, + 1, + "span", + 3, + "number", + -1, + 1, + "span", + 3, + "%", + -1, + 1, + "span", + 3, + "5", + -1, + 1, + "span", + 3, + "if", + -1, + 1, + "span", + 3, + "mod_3", + -1, + 1, + "span", + 3, + "==", + -1, + 1, + "span", + 3, + "0", + -1, + 1, + "span", + 3, + "and", + -1, + 1, + "span", + 3, + "mod_5", + -1, + 1, + "span", + 3, + "==", + -1, + 1, + "span", + 3, + "0", + -1, + 1, + "span", + 3, + ":", + -1, + 1, + "span", + 3, + "return", + -1, + 1, + "span", + 3, + "\"fizzbuzz\"", + -1, + 1, + "span", + 3, + "elif", + -1, + 1, + "span", + 3, + "mod_3", + -1, + 1, + "span", + 3, + "==", + -1, + 1, + "span", + 3, + "0", + -1, + 1, + "span", + 3, + ":", + -1, + 1, + "span", + 3, + "return", + -1, + 1, + "span", + 3, + "\"fizz\"", + -1, + 1, + "span", + 3, + "elif", + -1, + 1, + "span", + 3, + "mod_5", + -1, + 1, + "span", + 3, + "==", + -1, + 1, + "span", + 3, + "0", + -1, + 1, + "span", + 3, + ":", + -1, + 1, + "span", + 3, + "return", + -1, + 1, + "span", + 3, + "\"buzz\"", + -1, + 1, + "span", + 3, + "else", + -1, + 1, + "span", + 3, + ":", + -1, + 1, + "span", + 3, + "return", + -1, + 1, + "span", + 3, + "str", + -1, + 1, + "span", + 3, + "(", + -1, + 1, + "span", + 3, + "number", + -1, + 1, + "span", + 3, + ")", + -4, + 1, + "template", + 1, + "span", + 3, + "Copied!", + -3, + 1, "p", 3, "\n The ", @@ -2150,6 +8271,200 @@ "\n You can check that your implementation gives the expected result:\n ", -1, 1, + "div", + 2, + "data-is-repl", + "true", + 2, + "data-syntax-language", + "pycon", + 2, + "aria-label", + "Code block", + 1, + "div", + 1, + "pre", + 1, + "code", + 1, + "span", + 3, + ">", + 3, + ">", + 3, + ">", + -1, + 1, + "span", + 3, + "fizzbuzz", + -1, + 1, + "span", + 3, + "(", + -1, + 1, + "span", + 3, + "3", + -1, + 1, + "span", + 3, + ")", + -1, + 1, + "span", + 3, + "fizz", + -1, + 1, + "span", + 3, + ">", + 3, + ">", + 3, + ">", + -1, + 1, + "span", + 3, + "fizzbuzz", + -1, + 1, + "span", + 3, + "(", + -1, + 1, + "span", + 3, + "14", + -1, + 1, + "span", + 3, + ")", + -1, + 1, + "span", + 3, + "14", + -1, + 1, + "span", + 3, + ">", + 3, + ">", + 3, + ">", + -1, + 1, + "span", + 3, + "fizzbuzz", + -1, + 1, + "span", + 3, + "(", + -1, + 1, + "span", + 3, + "15", + -1, + 1, + "span", + 3, + ")", + -1, + 1, + "span", + 3, + "fizzbuzz", + -1, + 1, + "span", + 3, + ">", + 3, + ">", + 3, + ">", + -1, + 1, + "span", + 3, + "fizzbuzz", + -1, + 1, + "span", + 3, + "(", + -1, + 1, + "span", + 3, + "92", + -1, + 1, + "span", + 3, + ")", + -1, + 1, + "span", + 3, + "92", + -1, + 1, + "span", + 3, + ">", + 3, + ">", + 3, + ">", + -1, + 1, + "span", + 3, + "fizzbuzz", + -1, + 1, + "span", + 3, + "(", + -1, + 1, + "span", + 3, + "65", + -1, + 1, + "span", + 3, + ")", + -1, + 1, + "span", + 3, + "buzz", + -4, + 1, + "template", + 1, + "span", + 3, + "Copied!", + -3, + 1, "p", 3, "\n You can confirm for yourself that 3 is divisible by 3, 65 is divisible by 5, and 15 is divisible by both 3 and 5, while 14 and 92 aren’t divisible by either 3 or 5.\n ", @@ -2167,6 +8482,301 @@ " structure where you’re comparing one or a few variables several times over is quite straightforward to rewrite using pattern matching instead. For example, you can do the following:\n ", -1, 1, + "div", + 2, + "data-syntax-language", + "python", + 2, + "aria-label", + "Code block", + 1, + "div", + 1, + "pre", + 1, + "code", + 1, + "span", + 3, + "def", + -1, + 1, + "span", + 3, + "fizzbuzz", + -1, + 1, + "span", + 3, + "(", + -1, + 1, + "span", + 3, + "number", + -1, + 1, + "span", + 3, + "):", + -1, + 1, + "span", + 3, + "mod_3", + -1, + 1, + "span", + 3, + "=", + -1, + 1, + "span", + 3, + "number", + -1, + 1, + "span", + 3, + "%", + -1, + 1, + "span", + 3, + "3", + -1, + 1, + "span", + 3, + "mod_5", + -1, + 1, + "span", + 3, + "=", + -1, + 1, + "span", + 3, + "number", + -1, + 1, + "span", + 3, + "%", + -1, + 1, + "span", + 3, + "5", + -1, + 1, + "span", + 3, + "match", + -1, + 1, + "span", + 3, + "(", + -1, + 1, + "span", + 3, + "mod_3", + -1, + 1, + "span", + 3, + ",", + -1, + 1, + "span", + 3, + "mod_5", + -1, + 1, + "span", + 3, + "):", + -1, + 1, + "span", + 3, + "case", + -1, + 1, + "span", + 3, + "(", + -1, + 1, + "span", + 3, + "0", + -1, + 1, + "span", + 3, + ",", + -1, + 1, + "span", + 3, + "0", + -1, + 1, + "span", + 3, + "):", + -1, + 1, + "span", + 3, + "return", + -1, + 1, + "span", + 3, + "\"fizzbuzz\"", + -1, + 1, + "span", + 3, + "case", + -1, + 1, + "span", + -1, + 1, + "span", + 3, + "(", + -1, + 1, + "span", + 3, + "0", + -1, + 1, + "span", + 3, + ",", + -1, + 1, + "span", + 3, + "_", + -1, + 1, + "span", + 3, + "):", + -1, + 1, + "span", + 3, + "return", + -1, + 1, + "span", + 3, + "\"fizz\"", + -1, + 1, + "span", + 3, + "case", + -1, + 1, + "span", + -1, + 1, + "span", + 3, + "(", + -1, + 1, + "span", + 3, + "_", + -1, + 1, + "span", + 3, + ",", + -1, + 1, + "span", + 3, + "0", + -1, + 1, + "span", + 3, + "):", + -1, + 1, + "span", + 3, + "return", + -1, + 1, + "span", + 3, + "\"buzz\"", + -1, + 1, + "span", + 3, + "case", + -1, + 1, + "span", + -1, + 1, + "span", + 3, + "_", + -1, + 1, + "span", + 3, + ":", + -1, + 1, + "span", + 3, + "return", + -1, + 1, + "span", + 3, + "str", + -1, + 1, + "span", + 3, + "(", + -1, + 1, + "span", + 3, + "number", + -1, + 1, + "span", + 3, + ")", + -4, + 1, + "template", + 1, + "span", + 3, + "Copied!", + -3, + 1, "p", 3, "\n You match on both ", @@ -2449,6 +9059,191 @@ " to declare that a variable can have one of several different types. For example, you’ve been able to type hint a function calculating the mean of a list of numbers, floats, or integers as follows:\n ", -1, 1, + "div", + 2, + "data-syntax-language", + "python", + 2, + "aria-label", + "Code block", + 1, + "div", + 1, + "pre", + 1, + "code", + 1, + "span", + 3, + "from", + -1, + 1, + "span", + 3, + "typing", + -1, + 1, + "span", + 3, + "import", + -1, + 1, + "span", + 3, + "List", + -1, + 1, + "span", + 3, + ",", + -1, + 1, + "span", + 3, + "Union", + -1, + 1, + "span", + 1, + "span", + 3, + "def", + -1, + 1, + "span", + 3, + "mean", + -1, + 1, + "span", + 3, + "(", + -1, + 1, + "span", + 3, + "numbers", + -1, + 1, + "span", + 3, + ":", + -1, + 1, + "span", + 3, + "List", + -1, + 1, + "span", + 3, + "[", + -1, + 1, + "span", + 3, + "Union", + -1, + 1, + "span", + 3, + "[", + -1, + 1, + "span", + 3, + "float", + -1, + 1, + "span", + 3, + ",", + -1, + 1, + "span", + 3, + "int", + -1, + 1, + "span", + 3, + "]])", + -1, + 1, + "span", + 3, + "-", + 3, + ">", + -1, + 1, + "span", + 3, + "float", + -1, + 1, + "span", + 3, + ":", + -2, + 1, + "span", + 3, + "return", + -1, + 1, + "span", + 3, + "sum", + -1, + 1, + "span", + 3, + "(", + -1, + 1, + "span", + 3, + "numbers", + -1, + 1, + "span", + 3, + ")", + -1, + 1, + "span", + 3, + "/", + -1, + 1, + "span", + 3, + "len", + -1, + 1, + "span", + 3, + "(", + -1, + 1, + "span", + 3, + "numbers", + -1, + 1, + "span", + 3, + ")", + -4, + 1, + "template", + 1, + "span", + 3, + "Copied!", + -3, + 1, "p", 3, "\n The annotation ", @@ -2537,6 +9332,151 @@ " introduced. You can then simplify your code while keeping all the type information:\n ", -1, 1, + "div", + 2, + "data-syntax-language", + "python", + 2, + "aria-label", + "Code block", + 1, + "div", + 1, + "pre", + 1, + "code", + 1, + "span", + 1, + "span", + 3, + "def", + -1, + 1, + "span", + 3, + "mean", + -1, + 1, + "span", + 3, + "(", + -1, + 1, + "span", + 3, + "numbers", + -1, + 1, + "span", + 3, + ":", + -1, + 1, + "span", + 3, + "list", + -1, + 1, + "span", + 3, + "[", + -1, + 1, + "span", + 3, + "float", + -1, + 1, + "span", + 3, + "|", + -1, + 1, + "span", + 3, + "int", + -1, + 1, + "span", + 3, + "])", + -1, + 1, + "span", + 3, + "-", + 3, + ">", + -1, + 1, + "span", + 3, + "float", + -1, + 1, + "span", + 3, + ":", + -2, + 1, + "span", + 3, + "return", + -1, + 1, + "span", + 3, + "sum", + -1, + 1, + "span", + 3, + "(", + -1, + 1, + "span", + 3, + "numbers", + -1, + 1, + "span", + 3, + ")", + -1, + 1, + "span", + 3, + "/", + -1, + 1, + "span", + 3, + "len", + -1, + 1, + "span", + 3, + "(", + -1, + 1, + "span", + 3, + "numbers", + -1, + 1, + "span", + 3, + ")", + -4, + 1, + "template", + 1, + "span", + 3, + "Copied!", + -3, + 1, "p", 3, "\n The annotation of ", @@ -2646,6 +9586,148 @@ " tests:\n ", -1, 1, + "div", + 2, + "data-is-repl", + "true", + 2, + "data-syntax-language", + "pycon", + 2, + "aria-label", + "Code block", + 1, + "div", + 1, + "pre", + 1, + "code", + 1, + "span", + 3, + ">", + 3, + ">", + 3, + ">", + -1, + 1, + "span", + 3, + "isinstance", + -1, + 1, + "span", + 3, + "(", + -1, + 1, + "span", + 3, + "\"mypy\"", + -1, + 1, + "span", + 3, + ",", + -1, + 1, + "span", + 3, + "str", + -1, + 1, + "span", + 3, + "|", + -1, + 1, + "span", + 3, + "int", + -1, + 1, + "span", + 3, + ")", + -1, + 1, + "span", + 3, + "True", + -1, + 1, + "span", + 3, + ">", + 3, + ">", + 3, + ">", + -1, + 1, + "span", + 3, + "issubclass", + -1, + 1, + "span", + 3, + "(", + -1, + 1, + "span", + 3, + "str", + -1, + 1, + "span", + 3, + ",", + -1, + 1, + "span", + 3, + "int", + -1, + 1, + "span", + 3, + "|", + -1, + 1, + "span", + 3, + "float", + -1, + 1, + "span", + 3, + "|", + -1, + 1, + "span", + 3, + "bytes", + -1, + 1, + "span", + 3, + ")", + -1, + 1, + "span", + 3, + "False", + -4, + 1, + "template", + 1, + "span", + 3, + "Copied!", + -3, + 1, "p", 3, "\n Traditionally, you’ve used tuples to test for several types at once—for example, ", @@ -2707,11 +9789,233 @@ "\n To simplify type annotation, you define type aliases as follows:\n ", -1, 1, + "div", + 2, + "data-syntax-language", + "python", + 2, + "aria-label", + "Code block", + 1, + "div", + 1, + "pre", + 1, + "code", + 1, + "span", + 3, + "Card", + -1, + 1, + "span", + 3, + "=", + -1, + 1, + "span", + 3, + "tuple", + -1, + 1, + "span", + 3, + "[", + -1, + 1, + "span", + 3, + "str", + -1, + 1, + "span", + 3, + ",", + -1, + 1, + "span", + 3, + "str", + -1, + 1, + "span", + 3, + "]", + -1, + 1, + "span", + 3, + "Deck", + -1, + 1, + "span", + 3, + "=", + -1, + 1, + "span", + 3, + "list", + -1, + 1, + "span", + 3, + "[", + -1, + 1, + "span", + 3, + "Card", + -1, + 1, + "span", + 3, + "]", + -4, + 1, + "template", + 1, + "span", + 3, + "Copied!", + -3, + 1, "p", 3, "\n This usually works okay. However, it’s often not possible for the type checker to know whether such a statement is a type alias or just the definition of a regular global variable. To help the type checker—or really, help the type checker help you—you can now explicitly annotate type aliases:\n ", -1, 1, + "div", + 2, + "data-syntax-language", + "python", + 2, + "aria-label", + "Code block", + 1, + "div", + 1, + "pre", + 1, + "code", + 1, + "span", + 3, + "from", + -1, + 1, + "span", + 3, + "typing", + -1, + 1, + "span", + 3, + "import", + -1, + 1, + "span", + 3, + "TypeAlias", + -1, + 1, + "span", + 3, + "Card", + -1, + 1, + "span", + 3, + ":", + -1, + 1, + "span", + 3, + "TypeAlias", + -1, + 1, + "span", + 3, + "=", + -1, + 1, + "span", + 3, + "tuple", + -1, + 1, + "span", + 3, + "[", + -1, + 1, + "span", + 3, + "str", + -1, + 1, + "span", + 3, + ",", + -1, + 1, + "span", + 3, + "str", + -1, + 1, + "span", + 3, + "]", + -1, + 1, + "span", + 3, + "Deck", + -1, + 1, + "span", + 3, + ":", + -1, + 1, + "span", + 3, + "TypeAlias", + -1, + 1, + "span", + 3, + "=", + -1, + 1, + "span", + 3, + "list", + -1, + 1, + "span", + 3, + "[", + -1, + 1, + "span", + 3, + "Card", + -1, + 1, + "span", + 3, + "]", + -4, + 1, + "template", + 1, + "span", + 3, + "Copied!", + -3, + 1, "p", 3, "\n Adding the ", @@ -2741,6 +10045,181 @@ " but always returns a tuple of strings representing a playing card:\n ", -1, 1, + "div", + 2, + "data-syntax-language", + "python", + 2, + "aria-label", + "Code block", + 1, + "div", + 1, + "pre", + 1, + "code", + 1, + "span", + 3, + "def", + -1, + 1, + "span", + 3, + "get_ace", + -1, + 1, + "span", + 3, + "(", + -1, + 1, + "span", + 3, + "suit", + -1, + 1, + "span", + 3, + ":", + -1, + 1, + "span", + 3, + "str", + -1, + 1, + "span", + 3, + "|", + -1, + 1, + "span", + 3, + "None", + -1, + 1, + "span", + 3, + ")", + -1, + 1, + "span", + 3, + "-", + 3, + ">", + -1, + 1, + "span", + 3, + "tuple", + -1, + 1, + "span", + 3, + "[", + -1, + 1, + "span", + 3, + "str", + -1, + 1, + "span", + 3, + ",", + -1, + 1, + "span", + 3, + "str", + -1, + 1, + "span", + 3, + "]:", + -1, + 1, + "span", + 1, + "span", + 3, + "if", + -1, + 1, + "span", + 3, + "suit", + -1, + 1, + "span", + 3, + "is", + -1, + 1, + "span", + 3, + "None", + -1, + 1, + "span", + 3, + ":", + -2, + 1, + "span", + 3, + "suit", + -1, + 1, + "span", + 3, + "=", + -1, + 1, + "span", + 3, + "\"♠\"", + -1, + 1, + "span", + 3, + "return", + -1, + 1, + "span", + 3, + "(", + -1, + 1, + "span", + 3, + "suit", + -1, + 1, + "span", + 3, + ",", + -1, + 1, + "span", + 3, + "\"A\"", + -1, + 1, + "span", + 3, + ")", + -4, + 1, + "template", + 1, + "span", + 3, + "Copied!", + -3, + 1, "p", 3, "\n The highlighted line works as a type guard, and static type checkers are able to realize that ", @@ -2780,6 +10259,226 @@ ", you can annotate custom functions that can be used to narrow down union types:\n ", -1, 1, + "div", + 2, + "data-syntax-language", + "python", + 2, + "aria-label", + "Code block", + 1, + "div", + 1, + "pre", + 1, + "code", + 1, + "span", + 3, + "from", + -1, + 1, + "span", + 3, + "typing", + -1, + 1, + "span", + 3, + "import", + -1, + 1, + "span", + 3, + "Any", + -1, + 1, + "span", + 3, + ",", + -1, + 1, + "span", + 3, + "TypeAlias", + -1, + 1, + "span", + 3, + ",", + -1, + 1, + "span", + 3, + "TypeGuard", + -1, + 1, + "span", + 3, + "Card", + -1, + 1, + "span", + 3, + ":", + -1, + 1, + "span", + 3, + "TypeAlias", + -1, + 1, + "span", + 3, + "=", + -1, + 1, + "span", + 3, + "tuple", + -1, + 1, + "span", + 3, + "[", + -1, + 1, + "span", + 3, + "str", + -1, + 1, + "span", + 3, + ",", + -1, + 1, + "span", + 3, + "str", + -1, + 1, + "span", + 3, + "]", + -1, + 1, + "span", + 3, + "Deck", + -1, + 1, + "span", + 3, + ":", + -1, + 1, + "span", + 3, + "TypeAlias", + -1, + 1, + "span", + 3, + "=", + -1, + 1, + "span", + 3, + "list", + -1, + 1, + "span", + 3, + "[", + -1, + 1, + "span", + 3, + "Card", + -1, + 1, + "span", + 3, + "]", + -1, + 1, + "span", + 1, + "span", + 3, + "def", + -1, + 1, + "span", + 3, + "is_deck_of_cards", + -1, + 1, + "span", + 3, + "(", + -1, + 1, + "span", + 3, + "obj", + -1, + 1, + "span", + 3, + ":", + -1, + 1, + "span", + 3, + "Any", + -1, + 1, + "span", + 3, + ")", + -1, + 1, + "span", + 3, + "-", + 3, + ">", + -1, + 1, + "span", + 3, + "TypeGuard", + -1, + 1, + "span", + 3, + "[", + -1, + 1, + "span", + 3, + "Deck", + -1, + 1, + "span", + 3, + "]:", + -2, + 1, + "span", + 3, + "# Return True if obj is a deck of cards, otherwise False", + -4, + 1, + "template", + 1, + "span", + 3, + "Copied!", + -3, + 1, "p", 1, "code", @@ -2818,6 +10517,126 @@ " object or not. You can then use your guard function, and the type checker will be able to narrow down the types correctly:\n ", -1, 1, + "div", + 2, + "data-syntax-language", + "python", + 2, + "aria-label", + "Code block", + 1, + "div", + 1, + "pre", + 1, + "code", + 1, + "span", + 3, + "def", + -1, + 1, + "span", + 3, + "get_score", + -1, + 1, + "span", + 3, + "(", + -1, + 1, + "span", + 3, + "card_or_deck", + -1, + 1, + "span", + 3, + ":", + -1, + 1, + "span", + 3, + "Card", + -1, + 1, + "span", + 3, + "|", + -1, + 1, + "span", + 3, + "Deck", + -1, + 1, + "span", + 3, + ")", + -1, + 1, + "span", + 3, + "-", + 3, + ">", + -1, + 1, + "span", + 3, + "int", + -1, + 1, + "span", + 3, + ":", + -1, + 1, + "span", + 1, + "span", + 3, + "if", + -1, + 1, + "span", + 3, + "is_deck_of_cards", + -1, + 1, + "span", + 3, + "(", + -1, + 1, + "span", + 3, + "card_or_deck", + -1, + 1, + "span", + 3, + "):", + -2, + 1, + "span", + 3, + "# Calculate score of a deck of cards", + -1, + 1, + "span", + 3, + "...", + -4, + 1, + "template", + 1, + "span", + 3, + "Copied!", + -3, + 1, "p", 3, "\n Inside of the ", @@ -2886,6 +10705,328 @@ ". In general, it looks something like the following:\n ", -1, 1, + "div", + 2, + "data-syntax-language", + "python", + 2, + "aria-label", + "Code block", + 1, + "div", + 1, + "pre", + 1, + "code", + 1, + "span", + 3, + "import", + -1, + 1, + "span", + 3, + "functools", + -1, + 1, + "span", + 3, + "from", + -1, + 1, + "span", + 3, + "typing", + -1, + 1, + "span", + 3, + "import", + -1, + 1, + "span", + 3, + "Any", + -1, + 1, + "span", + 3, + ",", + -1, + 1, + "span", + 3, + "Callable", + -1, + 1, + "span", + 3, + ",", + -1, + 1, + "span", + 3, + "TypeVar", + -1, + 1, + "span", + 3, + "R", + -1, + 1, + "span", + 3, + "=", + -1, + 1, + "span", + 3, + "TypeVar", + -1, + 1, + "span", + 3, + "(", + -1, + 1, + "span", + 3, + "\"R\"", + -1, + 1, + "span", + 3, + ")", + -1, + 1, + "span", + 1, + "span", + 3, + "def", + -1, + 1, + "span", + 3, + "decorator", + -1, + 1, + "span", + 3, + "(", + -1, + 1, + "span", + 3, + "func", + -1, + 1, + "span", + 3, + ":", + -1, + 1, + "span", + 3, + "Callable", + -1, + 1, + "span", + 3, + "[", + -1, + 1, + "span", + 3, + "...", + -1, + 1, + "span", + 3, + ",", + -1, + 1, + "span", + 3, + "R", + -1, + 1, + "span", + 3, + "])", + -1, + 1, + "span", + 3, + "-", + 3, + ">", + -1, + 1, + "span", + 3, + "Callable", + -1, + 1, + "span", + 3, + "[", + -1, + 1, + "span", + 3, + "...", + -1, + 1, + "span", + 3, + ",", + -1, + 1, + "span", + 3, + "R", + -1, + 1, + "span", + 3, + "]:", + -2, + 1, + "span", + 3, + "@functools", + -1, + 1, + "span", + 3, + ".", + -1, + 1, + "span", + 3, + "wraps", + -1, + 1, + "span", + 3, + "(", + -1, + 1, + "span", + 3, + "func", + -1, + 1, + "span", + 3, + ")", + -1, + 1, + "span", + 3, + "def", + -1, + 1, + "span", + 3, + "wrapper", + -1, + 1, + "span", + 3, + "(", + -1, + 1, + "span", + 3, + "*", + -1, + 1, + "span", + 3, + "args", + -1, + 1, + "span", + 3, + ":", + -1, + 1, + "span", + 3, + "Any", + -1, + 1, + "span", + 3, + ",", + -1, + 1, + "span", + 3, + "**", + -1, + 1, + "span", + 3, + "kwargs", + -1, + 1, + "span", + 3, + ":", + -1, + 1, + "span", + 3, + "Any", + -1, + 1, + "span", + 3, + ")", + -1, + 1, + "span", + 3, + "-", + 3, + ">", + -1, + 1, + "span", + 3, + "R", + -1, + 1, + "span", + 3, + ":", + -1, + 1, + "span", + 3, + "...", + -1, + 1, + "span", + 3, + "return", + -1, + 1, + "span", + 3, + "wrapper", + -4, + 1, + "template", + 1, + "span", + 3, + "Copied!", + -3, + 1, "p", 3, "\n The annotations mean that the function returned by the decorator is a callable with some parameters and the same return type, ", @@ -2960,6 +11101,376 @@ ":\n ", -1, 1, + "div", + 2, + "data-syntax-language", + "python", + 2, + "aria-label", + "Code block", + 1, + "div", + 1, + "pre", + 1, + "code", + 1, + "span", + 3, + "import", + -1, + 1, + "span", + 3, + "functools", + -1, + 1, + "span", + 3, + "from", + -1, + 1, + "span", + 3, + "typing", + -1, + 1, + "span", + 3, + "import", + -1, + 1, + "span", + 3, + "Callable", + -1, + 1, + "span", + 3, + ",", + -1, + 1, + "span", + 3, + "ParamSpec", + -1, + 1, + "span", + 3, + ",", + -1, + 1, + "span", + 3, + "TypeVar", + -1, + 1, + "span", + 3, + "P", + -1, + 1, + "span", + 3, + "=", + -1, + 1, + "span", + 3, + "ParamSpec", + -1, + 1, + "span", + 3, + "(", + -1, + 1, + "span", + 3, + "\"P\"", + -1, + 1, + "span", + 3, + ")", + -1, + 1, + "span", + 3, + "R", + -1, + 1, + "span", + 3, + "=", + -1, + 1, + "span", + 3, + "TypeVar", + -1, + 1, + "span", + 3, + "(", + -1, + 1, + "span", + 3, + "\"R\"", + -1, + 1, + "span", + 3, + ")", + -1, + 1, + "span", + 3, + "def", + -1, + 1, + "span", + 3, + "decorator", + -1, + 1, + "span", + 3, + "(", + -1, + 1, + "span", + 3, + "func", + -1, + 1, + "span", + 3, + ":", + -1, + 1, + "span", + 3, + "Callable", + -1, + 1, + "span", + 3, + "[", + -1, + 1, + "span", + 3, + "P", + -1, + 1, + "span", + 3, + ",", + -1, + 1, + "span", + 3, + "R", + -1, + 1, + "span", + 3, + "])", + -1, + 1, + "span", + 3, + "-", + 3, + ">", + -1, + 1, + "span", + 3, + "Callable", + -1, + 1, + "span", + 3, + "[", + -1, + 1, + "span", + 3, + "P", + -1, + 1, + "span", + 3, + ",", + -1, + 1, + "span", + 3, + "R", + -1, + 1, + "span", + 3, + "]:", + -1, + 1, + "span", + 3, + "@functools", + -1, + 1, + "span", + 3, + ".", + -1, + 1, + "span", + 3, + "wraps", + -1, + 1, + "span", + 3, + "(", + -1, + 1, + "span", + 3, + "func", + -1, + 1, + "span", + 3, + ")", + -1, + 1, + "span", + 3, + "def", + -1, + 1, + "span", + 3, + "wrapper", + -1, + 1, + "span", + 3, + "(", + -1, + 1, + "span", + 3, + "*", + -1, + 1, + "span", + 3, + "args", + -1, + 1, + "span", + 3, + ":", + -1, + 1, + "span", + 3, + "P", + -1, + 1, + "span", + 3, + ".", + -1, + 1, + "span", + 3, + "args", + -1, + 1, + "span", + 3, + ",", + -1, + 1, + "span", + 3, + "**", + -1, + 1, + "span", + 3, + "kwargs", + -1, + 1, + "span", + 3, + ":", + -1, + 1, + "span", + 3, + "P", + -1, + 1, + "span", + 3, + ".", + -1, + 1, + "span", + 3, + "kwargs", + -1, + 1, + "span", + 3, + ")", + -1, + 1, + "span", + 3, + "-", + 3, + ">", + -1, + 1, + "span", + 3, + "R", + -1, + 1, + "span", + 3, + ":", + -1, + 1, + "span", + 3, + "...", + -1, + 1, + "span", + 3, + "return", + -1, + 1, + "span", + 3, + "wrapper", + -4, + 1, + "template", + 1, + "span", + 3, + "Copied!", + -3, + 1, "p", 3, "\n Note that you also use ", @@ -3088,6 +11599,252 @@ "\n One way to represent these data in plain Python would be with each column as a list. It could look something like this:\n ", -1, 1, + "div", + 2, + "data-is-repl", + "true", + 2, + "data-syntax-language", + "pycon", + 2, + "aria-label", + "Code block", + 1, + "div", + 1, + "pre", + 1, + "code", + 1, + "span", + 3, + ">", + 3, + ">", + 3, + ">", + -1, + 1, + "span", + 3, + "names", + -1, + 1, + "span", + 3, + "=", + -1, + 1, + "span", + 3, + "[", + -1, + 1, + "span", + 3, + "\"Louvre\"", + -1, + 1, + "span", + 3, + ",", + -1, + 1, + "span", + 3, + "\"Diagon Alley\"", + -1, + 1, + "span", + 3, + ",", + -1, + 1, + "span", + 3, + "\"Saturn V\"", + -1, + 1, + "span", + 3, + ",", + -1, + 1, + "span", + 3, + "\"Millennium Falcon\"", + -1, + 1, + "span", + 3, + ",", + -1, + 1, + "span", + 3, + "\"NYC\"", + -1, + 1, + "span", + 3, + "]", + -1, + 1, + "span", + 3, + ">", + 3, + ">", + 3, + ">", + -1, + 1, + "span", + 3, + "set_numbers", + -1, + 1, + "span", + 3, + "=", + -1, + 1, + "span", + 3, + "[", + -1, + 1, + "span", + 3, + "\"21024\"", + -1, + 1, + "span", + 3, + ",", + -1, + 1, + "span", + 3, + "\"75978\"", + -1, + 1, + "span", + 3, + ",", + -1, + 1, + "span", + 3, + "\"92176\"", + -1, + 1, + "span", + 3, + ",", + -1, + 1, + "span", + 3, + "\"75192\"", + -1, + 1, + "span", + 3, + ",", + -1, + 1, + "span", + 3, + "\"21028\"", + -1, + 1, + "span", + 3, + "]", + -1, + 1, + "span", + 3, + ">", + 3, + ">", + 3, + ">", + -1, + 1, + "span", + 3, + "num_pieces", + -1, + 1, + "span", + 3, + "=", + -1, + 1, + "span", + 3, + "[", + -1, + 1, + "span", + 3, + "695", + -1, + 1, + "span", + 3, + ",", + -1, + 1, + "span", + 3, + "5544", + -1, + 1, + "span", + 3, + ",", + -1, + 1, + "span", + 3, + "1969", + -1, + 1, + "span", + 3, + ",", + -1, + 1, + "span", + 3, + "7541", + -1, + 1, + "span", + 3, + ",", + -1, + 1, + "span", + 3, + "598", + -1, + 1, + "span", + 3, + "]", + -4, + 1, + "template", + 1, + "span", + 3, + "Copied!", + -3, + 1, "p", 3, "\n Note that you have three independent lists, but there’s an implicit correspondence between their elements. The first name (", @@ -3124,6 +11881,234 @@ " can be used to iterate over these three lists in parallel:\n ", -1, 1, + "div", + 2, + "data-is-repl", + "true", + 2, + "data-syntax-language", + "pycon", + 2, + "aria-label", + "Code block", + 1, + "div", + 1, + "pre", + 1, + "code", + 1, + "span", + 3, + ">", + 3, + ">", + 3, + ">", + -1, + 1, + "span", + 3, + "for", + -1, + 1, + "span", + 3, + "name", + -1, + 1, + "span", + 3, + ",", + -1, + 1, + "span", + 3, + "num", + -1, + 1, + "span", + 3, + ",", + -1, + 1, + "span", + 3, + "pieces", + -1, + 1, + "span", + 3, + "in", + -1, + 1, + "span", + 3, + "zip", + -1, + 1, + "span", + 3, + "(", + -1, + 1, + "span", + 3, + "names", + -1, + 1, + "span", + 3, + ",", + -1, + 1, + "span", + 3, + "set_numbers", + -1, + 1, + "span", + 3, + ",", + -1, + 1, + "span", + 3, + "num_pieces", + -1, + 1, + "span", + 3, + "):", + -1, + 1, + "span", + 3, + "... ", + -1, + 1, + "span", + 3, + "print", + -1, + 1, + "span", + 3, + "(", + -1, + 1, + "span", + 3, + "f", + -1, + 1, + "span", + 3, + "\"", + -1, + 1, + "span", + 3, + "{", + -1, + 1, + "span", + 3, + "name", + -1, + 1, + "span", + 3, + "}", + -1, + 1, + "span", + 3, + " (", + -1, + 1, + "span", + 3, + "{", + -1, + 1, + "span", + 3, + "num", + -1, + 1, + "span", + 3, + "}", + -1, + 1, + "span", + 3, + "): ", + -1, + 1, + "span", + 3, + "{", + -1, + 1, + "span", + 3, + "pieces", + -1, + 1, + "span", + 3, + "}", + -1, + 1, + "span", + 3, + " pieces\"", + -1, + 1, + "span", + 3, + ")", + -1, + 1, + "span", + 3, + "...", + -1, + 1, + "span", + 3, + "Louvre (21024): 695 pieces", + -1, + 1, + "span", + 3, + "Diagon Alley (75978): 5544 pieces", + -1, + 1, + "span", + 3, + "Saturn V (92176): 1969 pieces", + -1, + 1, + "span", + 3, + "Millennium Falcon (75192): 7541 pieces", + -1, + 1, + "span", + 3, + "NYC (21028): 598 pieces", + -4, + 1, + "template", + 1, + "span", + 3, + "Copied!", + -3, + 1, "p", 3, "\n Note how each line collects information from all three lists and shows information about one particular set. This is a very common pattern that’s used in a lot of different Python code, including ", @@ -3151,6 +12136,114 @@ " to collect the contents of all three lists in a single, nested list of tuples:\n ", -1, 1, + "div", + 2, + "data-is-repl", + "true", + 2, + "data-syntax-language", + "pycon", + 2, + "aria-label", + "Code block", + 1, + "div", + 1, + "pre", + 1, + "code", + 1, + "span", + 3, + ">", + 3, + ">", + 3, + ">", + -1, + 1, + "span", + 3, + "list", + -1, + 1, + "span", + 3, + "(", + -1, + 1, + "span", + 3, + "zip", + -1, + 1, + "span", + 3, + "(", + -1, + 1, + "span", + 3, + "names", + -1, + 1, + "span", + 3, + ",", + -1, + 1, + "span", + 3, + "set_numbers", + -1, + 1, + "span", + 3, + ",", + -1, + 1, + "span", + 3, + "num_pieces", + -1, + 1, + "span", + 3, + "))", + -1, + 1, + "span", + 3, + "[('Louvre', '21024', 695),", + -1, + 1, + "span", + 3, + " ('Diagon Alley', '75978', 5544),", + -1, + 1, + "span", + 3, + " ('Saturn V', '92176', 1969),", + -1, + 1, + "span", + 3, + " ('Millennium Falcon', '75192', 7541),", + -1, + 1, + "span", + 3, + " ('NYC', '21028', 598)]", + -4, + 1, + "template", + 1, + "span", + 3, + "Copied!", + -3, + 1, "p", 3, "\n Note how the nested list closely resembles the original table.\n ", @@ -3168,6 +12261,178 @@ " is that it’s quite easy to introduce a subtle bug that can be hard to discover. Note what happens if there’s a missing item in one of your lists:\n ", -1, 1, + "div", + 2, + "data-is-repl", + "true", + 2, + "data-syntax-language", + "pycon", + 2, + "aria-label", + "Code block", + 1, + "div", + 1, + "pre", + 1, + "code", + 1, + "span", + 3, + ">", + 3, + ">", + 3, + ">", + -1, + 1, + "span", + 3, + "set_numbers", + -1, + 1, + "span", + 3, + "=", + -1, + 1, + "span", + 3, + "[", + -1, + 1, + "span", + 3, + "\"21024\"", + -1, + 1, + "span", + 3, + ",", + -1, + 1, + "span", + 3, + "\"75978\"", + -1, + 1, + "span", + 3, + ",", + -1, + 1, + "span", + 3, + "\"75192\"", + -1, + 1, + "span", + 3, + ",", + -1, + 1, + "span", + 3, + "\"21028\"", + -1, + 1, + "span", + 3, + "]", + -1, + 1, + "span", + 3, + "# Saturn V missing", + -1, + 1, + "span", + 3, + ">", + 3, + ">", + 3, + ">", + -1, + 1, + "span", + 3, + "list", + -1, + 1, + "span", + 3, + "(", + -1, + 1, + "span", + 3, + "zip", + -1, + 1, + "span", + 3, + "(", + -1, + 1, + "span", + 3, + "names", + -1, + 1, + "span", + 3, + ",", + -1, + 1, + "span", + 3, + "set_numbers", + -1, + 1, + "span", + 3, + ",", + -1, + 1, + "span", + 3, + "num_pieces", + -1, + 1, + "span", + 3, + "))", + -1, + 1, + "span", + 3, + "[('Louvre', '21024', 695),", + -1, + 1, + "span", + 3, + " ('Diagon Alley', '75978', 5544),", + -1, + 1, + "span", + 3, + " ('Saturn V', '75192', 1969),", + -1, + 1, + "span", + 3, + " ('Millennium Falcon', '21028', 7541)]", + -4, + 1, + "template", + 1, + "span", + 3, + "Copied!", + -3, + 1, "p", 3, "\n All the information about the New York City set disappeared! Additionally, the set numbers for Saturn V and Millennium Falcon are wrong. If your datasets are bigger, these kinds of errors can be very hard to discover. And even when you observe that something’s wrong, it’s not always easy to diagnose and fix.\n ", @@ -3212,6 +12477,161 @@ " that you can use to confirm all sequences have the same length. In your example, it would raise an error alerting you to the corrupted list:\n ", -1, 1, + "div", + 2, + "data-is-repl", + "true", + 2, + "data-syntax-language", + "pycon", + 2, + "aria-label", + "Code block", + 1, + "div", + 1, + "pre", + 1, + "code", + 1, + "span", + 1, + "span", + 3, + ">", + 3, + ">", + 3, + ">", + -1, + 1, + "span", + 3, + "list", + -1, + 1, + "span", + 3, + "(", + -1, + 1, + "span", + 3, + "zip", + -1, + 1, + "span", + 3, + "(", + -1, + 1, + "span", + 3, + "names", + -1, + 1, + "span", + 3, + ",", + -1, + 1, + "span", + 3, + "set_numbers", + -1, + 1, + "span", + 3, + ",", + -1, + 1, + "span", + 3, + "num_pieces", + -1, + 1, + "span", + 3, + ",", + -1, + 1, + "span", + 3, + "strict", + -1, + 1, + "span", + 3, + "=", + -1, + 1, + "span", + 3, + "True", + -1, + 1, + "span", + 3, + "))", + -2, + 1, + "span", + 3, + "Traceback (most recent call last):", + -1, + 3, + "\n File ", + 1, + "span", + 3, + "\"", + 3, + "<", + 3, + "stdin", + 3, + ">", + 3, + "\"", + -1, + 3, + ", line ", + 1, + "span", + 3, + "1", + -1, + 3, + ", in ", + 1, + "span", + 3, + "<", + 3, + "module", + 3, + ">", + -1, + 1, + "span", + 3, + "ValueError", + -1, + 3, + ": ", + 1, + "span", + 3, + "zip() argument 2 is shorter than argument 1", + -4, + 1, + "template", + 1, + "span", + 3, + "Copied!", + -3, + 1, "p", 3, "\n When the iteration reaches the New York City Lego set, the second argument ", @@ -3273,6 +12693,133 @@ " divides the Lego sets into pairs:\n ", -1, 1, + "div", + 2, + "data-is-repl", + "true", + 2, + "data-syntax-language", + "pycon", + 2, + "aria-label", + "Code block", + 1, + "div", + 1, + "pre", + 1, + "code", + 1, + "span", + 3, + ">", + 3, + ">", + 3, + ">", + -1, + 1, + "span", + 3, + "num_per_group", + -1, + 1, + "span", + 3, + "=", + -1, + 1, + "span", + 3, + "2", + -1, + 1, + "span", + 3, + ">", + 3, + ">", + 3, + ">", + -1, + 1, + "span", + 3, + "list", + -1, + 1, + "span", + 3, + "(", + -1, + 1, + "span", + 3, + "zip", + -1, + 1, + "span", + 3, + "(", + -1, + 1, + "span", + 3, + "*", + -1, + 1, + "span", + 3, + "[", + -1, + 1, + "span", + 3, + "iter", + -1, + 1, + "span", + 3, + "(", + -1, + 1, + "span", + 3, + "names", + -1, + 1, + "span", + 3, + ")]", + -1, + 1, + "span", + 3, + "*", + -1, + 1, + "span", + 3, + "num_per_group", + -1, + 1, + "span", + 3, + "))", + -1, + 1, + "span", + 3, + "[('Louvre', 'Diagon Alley'), ('Saturn V', 'Millennium Falcon')]", + -4, + 1, + "template", + 1, + "span", + 3, + "Copied!", + -3, + 1, "p", 3, "\n There are five sets, a number that doesn’t divide evenly into pairs. In this case, the default behavior of ", @@ -3335,6 +12882,168 @@ " to divide the Lego sets, it becomes more explicit that New York City doesn’t have any pairing:\n ", -1, 1, + "div", + 2, + "data-is-repl", + "true", + 2, + "data-syntax-language", + "pycon", + 2, + "aria-label", + "Code block", + 1, + "div", + 1, + "pre", + 1, + "code", + 1, + "span", + 3, + ">", + 3, + ">", + 3, + ">", + -1, + 1, + "span", + 3, + "from", + -1, + 1, + "span", + 3, + "itertools", + -1, + 1, + "span", + 3, + "import", + -1, + 1, + "span", + 3, + "zip_longest", + -1, + 1, + "span", + 3, + ">", + 3, + ">", + 3, + ">", + -1, + 1, + "span", + 3, + "list", + -1, + 1, + "span", + 3, + "(", + -1, + 1, + "span", + 3, + "zip_longest", + -1, + 1, + "span", + 3, + "(", + -1, + 1, + "span", + 3, + "*", + -1, + 1, + "span", + 3, + "[", + -1, + 1, + "span", + 3, + "iter", + -1, + 1, + "span", + 3, + "(", + -1, + 1, + "span", + 3, + "names", + -1, + 1, + "span", + 3, + ")]", + -1, + 1, + "span", + 3, + "*", + -1, + 1, + "span", + 3, + "num_per_group", + -1, + 1, + "span", + 3, + ",", + -1, + 1, + "span", + 3, + "fillvalue", + -1, + 1, + "span", + 3, + "=", + -1, + 1, + "span", + 3, + "\"\"", + -1, + 1, + "span", + 3, + "))", + -1, + 1, + "span", + 3, + "[('Louvre', 'Diagon Alley'),", + -1, + 1, + "span", + 3, + " ('Saturn V', 'Millennium Falcon'),", + -1, + 1, + "span", + 3, + " ('NYC', '')]", + -4, + 1, + "template", + 1, + "span", + 3, + "Copied!", + -3, + 1, "p", 3, "\n Note that ", @@ -3534,6 +13243,278 @@ "\n You can use each function to describe a certain aspect of the relationship between two variables. As an example, say that you have data from a set of blog posts—the number of words in each blog post and the number of views each post has had over some time period:\n ", -1, 1, + "div", + 2, + "data-is-repl", + "true", + 2, + "data-syntax-language", + "pycon", + 2, + "aria-label", + "Code block", + 1, + "div", + 1, + "pre", + 1, + "code", + 1, + "span", + 3, + ">", + 3, + ">", + 3, + ">", + -1, + 1, + "span", + 3, + "words", + -1, + 1, + "span", + 3, + "=", + -1, + 1, + "span", + 3, + "[", + -1, + 1, + "span", + 3, + "7742", + -1, + 1, + "span", + 3, + ",", + -1, + 1, + "span", + 3, + "11539", + -1, + 1, + "span", + 3, + ",", + -1, + 1, + "span", + 3, + "16898", + -1, + 1, + "span", + 3, + ",", + -1, + 1, + "span", + 3, + "13447", + -1, + 1, + "span", + 3, + ",", + -1, + 1, + "span", + 3, + "4608", + -1, + 1, + "span", + 3, + ",", + -1, + 1, + "span", + 3, + "6628", + -1, + 1, + "span", + 3, + ",", + -1, + 1, + "span", + 3, + "2683", + -1, + 1, + "span", + 3, + ",", + -1, + 1, + "span", + 3, + "6156", + -1, + 1, + "span", + 3, + ",", + -1, + 1, + "span", + 3, + "2623", + -1, + 1, + "span", + 3, + ",", + -1, + 1, + "span", + 3, + "6948", + -1, + 1, + "span", + 3, + "]", + -1, + 1, + "span", + 3, + ">", + 3, + ">", + 3, + ">", + -1, + 1, + "span", + 3, + "views", + -1, + 1, + "span", + 3, + "=", + -1, + 1, + "span", + 3, + "[", + -1, + 1, + "span", + 3, + "8368", + -1, + 1, + "span", + 3, + ",", + -1, + 1, + "span", + 3, + "5901", + -1, + 1, + "span", + 3, + ",", + -1, + 1, + "span", + 3, + "3978", + -1, + 1, + "span", + 3, + ",", + -1, + 1, + "span", + 3, + "3329", + -1, + 1, + "span", + 3, + ",", + -1, + 1, + "span", + 3, + "2611", + -1, + 1, + "span", + 3, + ",", + -1, + 1, + "span", + 3, + "2096", + -1, + 1, + "span", + 3, + ",", + -1, + 1, + "span", + 3, + "1515", + -1, + 1, + "span", + 3, + ",", + -1, + 1, + "span", + 3, + "1177", + -1, + 1, + "span", + 3, + ",", + -1, + 1, + "span", + 3, + "814", + -1, + 1, + "span", + 3, + ",", + -1, + 1, + "span", + 3, + "467", + -1, + 1, + "span", + 3, + "]", + -4, + 1, + "template", + 1, + "span", + 3, + "Copied!", + -3, + 1, "p", 3, "\n You now want to investigate whether there’s any (linear) relationship between the number of words and number of views. In Python 3.10, you can calculate the ", @@ -3572,6 +13553,103 @@ " function:\n ", -1, 1, + "div", + 2, + "data-is-repl", + "true", + 2, + "data-syntax-language", + "pycon", + 2, + "aria-label", + "Code block", + 1, + "div", + 1, + "pre", + 1, + "code", + 1, + "span", + 3, + ">", + 3, + ">", + 3, + ">", + -1, + 1, + "span", + 3, + "import", + -1, + 1, + "span", + 3, + "statistics", + -1, + 1, + "span", + 3, + ">", + 3, + ">", + 3, + ">", + -1, + 1, + "span", + 3, + "statistics", + -1, + 1, + "span", + 3, + ".", + -1, + 1, + "span", + 3, + "correlation", + -1, + 1, + "span", + 3, + "(", + -1, + 1, + "span", + 3, + "words", + -1, + 1, + "span", + 3, + ",", + -1, + 1, + "span", + 3, + "views", + -1, + 1, + "span", + 3, + ")", + -1, + 1, + "span", + 3, + "0.454180067865917", + -4, + 1, + "template", + 1, + "span", + 3, + "Copied!", + -3, + 1, "p", 3, "\n The correlation between two variables is always a number between -1 and 1. If it’s close to 0, then there’s little correspondence between them, while a correlation close to -1 or 1 indicates that the behaviors of the two variables tend to follow each other. In this example, a correlation of 0.45 indicates that there’s a tendency for posts with more words to have more views, although it’s not a strong connection.\n ", @@ -3615,6 +13693,103 @@ ":\n ", -1, 1, + "div", + 2, + "data-is-repl", + "true", + 2, + "data-syntax-language", + "pycon", + 2, + "aria-label", + "Code block", + 1, + "div", + 1, + "pre", + 1, + "code", + 1, + "span", + 3, + ">", + 3, + ">", + 3, + ">", + -1, + 1, + "span", + 3, + "import", + -1, + 1, + "span", + 3, + "statistics", + -1, + 1, + "span", + 3, + ">", + 3, + ">", + 3, + ">", + -1, + 1, + "span", + 3, + "statistics", + -1, + 1, + "span", + 3, + ".", + -1, + 1, + "span", + 3, + "covariance", + -1, + 1, + "span", + 3, + "(", + -1, + 1, + "span", + 3, + "words", + -1, + 1, + "span", + 3, + ",", + -1, + 1, + "span", + 3, + "views", + -1, + 1, + "span", + 3, + ")", + -1, + 1, + "span", + 3, + "5292289.977777777", + -4, + 1, + "template", + 1, + "span", + 3, + "Copied!", + -3, + 1, "p", 3, "\n In contrast to correlation, covariance is an absolute measure. It should be interpreted in the context of the variability within the variables themselves. In fact, you can normalize the covariance by the ", @@ -3640,6 +13815,246 @@ ":\n ", -1, 1, + "div", + 2, + "data-is-repl", + "true", + 2, + "data-syntax-language", + "pycon", + 2, + "aria-label", + "Code block", + 1, + "div", + 1, + "pre", + 1, + "code", + 1, + "span", + 3, + ">", + 3, + ">", + 3, + ">", + -1, + 1, + "span", + 3, + "import", + -1, + 1, + "span", + 3, + "statistics", + -1, + 1, + "span", + 3, + ">", + 3, + ">", + 3, + ">", + -1, + 1, + "span", + 3, + "cov", + -1, + 1, + "span", + 3, + "=", + -1, + 1, + "span", + 3, + "statistics", + -1, + 1, + "span", + 3, + ".", + -1, + 1, + "span", + 3, + "covariance", + -1, + 1, + "span", + 3, + "(", + -1, + 1, + "span", + 3, + "words", + -1, + 1, + "span", + 3, + ",", + -1, + 1, + "span", + 3, + "views", + -1, + 1, + "span", + 3, + ")", + -1, + 1, + "span", + 3, + ">", + 3, + ">", + 3, + ">", + -1, + 1, + "span", + 3, + "σ_words", + -1, + 1, + "span", + 3, + ",", + -1, + 1, + "span", + 3, + "σ_views", + -1, + 1, + "span", + 3, + "=", + -1, + 1, + "span", + 3, + "statistics", + -1, + 1, + "span", + 3, + ".", + -1, + 1, + "span", + 3, + "stdev", + -1, + 1, + "span", + 3, + "(", + -1, + 1, + "span", + 3, + "words", + -1, + 1, + "span", + 3, + "),", + -1, + 1, + "span", + 3, + "statistics", + -1, + 1, + "span", + 3, + ".", + -1, + 1, + "span", + 3, + "stdev", + -1, + 1, + "span", + 3, + "(", + -1, + 1, + "span", + 3, + "views", + -1, + 1, + "span", + 3, + ")", + -1, + 1, + "span", + 3, + ">", + 3, + ">", + 3, + ">", + -1, + 1, + "span", + 3, + "cov", + -1, + 1, + "span", + 3, + "/", + -1, + 1, + "span", + 3, + "(", + -1, + 1, + "span", + 3, + "σ_words", + -1, + 1, + "span", + 3, + "*", + -1, + 1, + "span", + 3, + "σ_views", + -1, + 1, + "span", + 3, + ")", + -1, + 1, + "span", + 3, + "0.454180067865917", + -4, + 1, + "template", + 1, + "span", + 3, + "Copied!", + -3, + 1, "p", 3, "\n Note that this matches your earlier correlation coefficient exactly.\n ", @@ -3726,6 +14141,103 @@ ":\n ", -1, 1, + "div", + 2, + "data-is-repl", + "true", + 2, + "data-syntax-language", + "pycon", + 2, + "aria-label", + "Code block", + 1, + "div", + 1, + "pre", + 1, + "code", + 1, + "span", + 3, + ">", + 3, + ">", + 3, + ">", + -1, + 1, + "span", + 3, + "import", + -1, + 1, + "span", + 3, + "statistics", + -1, + 1, + "span", + 3, + ">", + 3, + ">", + 3, + ">", + -1, + 1, + "span", + 3, + "statistics", + -1, + 1, + "span", + 3, + ".", + -1, + 1, + "span", + 3, + "linear_regression", + -1, + 1, + "span", + 3, + "(", + -1, + 1, + "span", + 3, + "words", + -1, + 1, + "span", + 3, + ",", + -1, + 1, + "span", + 3, + "views", + -1, + 1, + "span", + 3, + ")", + -1, + 1, + "span", + 3, + "LinearRegression(slope=0.2424443064354672, intercept=1103.6954940247645)", + -4, + 1, + "template", + 1, + "span", + 3, + "Copied!", + -3, + 1, "p", 3, "\n Based on this regression, a post with 10,074 words could expect about 0.2424 × 10074 + 1104 = 3546 views. However, as you saw earlier, the correlation between the number of words and the number of views is quite weak. Therefore, you shouldn’t expect this prediction to be very accurate.\n ", @@ -3753,6 +14265,157 @@ ". This means that you can unpack the slope and intercept directly:\n ", -1, 1, + "div", + 2, + "data-is-repl", + "true", + 2, + "data-syntax-language", + "pycon", + 2, + "aria-label", + "Code block", + 1, + "div", + 1, + "pre", + 1, + "code", + 1, + "span", + 3, + ">", + 3, + ">", + 3, + ">", + -1, + 1, + "span", + 3, + "import", + -1, + 1, + "span", + 3, + "statistics", + -1, + 1, + "span", + 3, + ">", + 3, + ">", + 3, + ">", + -1, + 1, + "span", + 3, + "slope", + -1, + 1, + "span", + 3, + ",", + -1, + 1, + "span", + 3, + "intercept", + -1, + 1, + "span", + 3, + "=", + -1, + 1, + "span", + 3, + "statistics", + -1, + 1, + "span", + 3, + ".", + -1, + 1, + "span", + 3, + "linear_regression", + -1, + 1, + "span", + 3, + "(", + -1, + 1, + "span", + 3, + "words", + -1, + 1, + "span", + 3, + ",", + -1, + 1, + "span", + 3, + "views", + -1, + 1, + "span", + 3, + ")", + -1, + 1, + "span", + 3, + ">", + 3, + ">", + 3, + ">", + -1, + 1, + "span", + 3, + "slope", + -1, + 1, + "span", + 3, + "*", + -1, + 1, + "span", + 3, + "10074", + -1, + 1, + "span", + 3, + "+", + -1, + 1, + "span", + 3, + "intercept", + -1, + 1, + "span", + 3, + "3546.0794370556605", + -4, + 1, + "template", + 1, + "span", + 3, + "Copied!", + -3, + 1, "p", 3, "\n Here, you use ", @@ -3864,6 +14527,117 @@ "\n You should therefore always specify an encoding when you attempt to open a text file:\n ", -1, 1, + "div", + 2, + "data-syntax-language", + "python", + 2, + "aria-label", + "Code block", + 1, + "div", + 1, + "pre", + 1, + "code", + 1, + "span", + 3, + "with", + -1, + 1, + "span", + 3, + "open", + -1, + 1, + "span", + 3, + "(", + -1, + 1, + "span", + 3, + "\"some_file.txt\"", + -1, + 1, + "span", + 3, + ",", + -1, + 1, + "span", + 3, + "mode", + -1, + 1, + "span", + 3, + "=", + -1, + 1, + "span", + 3, + "\"r\"", + -1, + 1, + "span", + 3, + ",", + -1, + 1, + "span", + 3, + "encoding", + -1, + 1, + "span", + 3, + "=", + -1, + 1, + "span", + 3, + "\"utf-8\"", + -1, + 1, + "span", + 3, + ")", + -1, + 1, + "span", + 3, + "as", + -1, + 1, + "span", + 3, + "file", + -1, + 1, + "span", + 3, + ":", + -1, + 1, + "span", + 3, + "...", + -1, + 1, + "span", + 3, + "# Do something with file", + -4, + 1, + "template", + 1, + "span", + 3, + "Copied!", + -3, + 1, "p", 3, "\n If you don’t explicitly specify an encoding, the preferred locale encoding is used, and you could experience that a file that can be read on one computer fails to open on another.\n ", @@ -3910,6 +14684,336 @@ "\n In Python 3.10, you can activate a warning that will tell you when a text file is opened without a specified encoding. Consider the following script, which doesn’t specify an encoding:\n ", -1, 1, + "div", + 2, + "data-syntax-language", + "python", + 2, + "aria-label", + "Code block", + 1, + "div", + 1, + "pre", + 1, + "code", + 1, + "span", + 3, + "# mirror.py", + -1, + 1, + "span", + 3, + "import", + -1, + 1, + "span", + 3, + "pathlib", + -1, + 1, + "span", + 3, + "import", + -1, + 1, + "span", + 3, + "sys", + -1, + 1, + "span", + 3, + "def", + -1, + 1, + "span", + 3, + "mirror_file", + -1, + 1, + "span", + 3, + "(", + -1, + 1, + "span", + 3, + "filename", + -1, + 1, + "span", + 3, + "):", + -1, + 1, + "span", + 1, + "span", + 3, + "for", + -1, + 1, + "span", + 3, + "line", + -1, + 1, + "span", + 3, + "in", + -1, + 1, + "span", + 3, + "pathlib", + -1, + 1, + "span", + 3, + ".", + -1, + 1, + "span", + 3, + "Path", + -1, + 1, + "span", + 3, + "(", + -1, + 1, + "span", + 3, + "filename", + -1, + 1, + "span", + 3, + ")", + -1, + 1, + "span", + 3, + ".", + -1, + 1, + "span", + 3, + "open", + -1, + 1, + "span", + 3, + "(", + -1, + 1, + "span", + 3, + "mode", + -1, + 1, + "span", + 3, + "=", + -1, + 1, + "span", + 3, + "\"r\"", + -1, + 1, + "span", + 3, + "):", + -2, + 1, + "span", + 3, + "print", + -1, + 1, + "span", + 3, + "(", + -1, + 1, + "span", + 3, + "f", + -1, + 1, + "span", + 3, + "\"", + -1, + 1, + "span", + 3, + "{", + -1, + 1, + "span", + 3, + "line", + -1, + 1, + "span", + 3, + ".", + -1, + 1, + "span", + 3, + "rstrip", + -1, + 1, + "span", + 3, + "()[::", + -1, + 1, + "span", + 3, + "-", + -1, + 1, + "span", + 3, + "1", + -1, + 1, + "span", + 3, + "]", + -1, + 1, + "span", + 3, + ":", + -1, + 1, + "span", + 3, + ">", + 3, + "72", + -1, + 1, + "span", + 3, + "}", + -1, + 1, + "span", + 3, + "\"", + -1, + 1, + "span", + 3, + ")", + -1, + 1, + "span", + 3, + "if", + -1, + 1, + "span", + 3, + "__name__", + -1, + 1, + "span", + 3, + "==", + -1, + 1, + "span", + 3, + "\"__main__\"", + -1, + 1, + "span", + 3, + ":", + -1, + 1, + "span", + 3, + "for", + -1, + 1, + "span", + 3, + "filename", + -1, + 1, + "span", + 3, + "in", + -1, + 1, + "span", + 3, + "sys", + -1, + 1, + "span", + 3, + ".", + -1, + 1, + "span", + 3, + "argv", + -1, + 1, + "span", + 3, + "[", + -1, + 1, + "span", + 3, + "1", + -1, + 1, + "span", + 3, + ":]:", + -1, + 1, + "span", + 3, + "mirror_file", + -1, + 1, + "span", + 3, + "(", + -1, + 1, + "span", + 3, + "filename", + -1, + 1, + "span", + 3, + ")", + -4, + 1, + "template", + 1, + "span", + 3, + "Copied!", + -3, + 1, "p", 3, "\n The program will echo one or more text files back to the console, but with each line reversed. Run the program on itself with the ", @@ -3925,6 +15029,120 @@ " enabled:\n ", -1, 1, + "div", + 2, + "data-is-repl", + "true", + 2, + "data-syntax-language", + "console", + 2, + "aria-label", + "Code block", + 1, + "div", + 1, + "pre", + 1, + "code", + 1, + "span", + 3, + "$ ", + -1, + 3, + "python", + 1, + "span", + -1, + 3, + "-X", + 1, + "span", + -1, + 3, + "warn_default_encoding", + 1, + "span", + -1, + 3, + "mirror.py", + 1, + "span", + -1, + 3, + "mirror.py\n", + 1, + "span", + 1, + "span", + 3, + "/home/rp/mirror.py:7: EncodingWarning: 'encoding' argument not specified", + -2, + 1, + "span", + 1, + "span", + 3, + " for line in pathlib.Path(filename).open(mode=\"r\"):", + -2, + 1, + "span", + 3, + " yp.rorrim #", + -1, + 1, + "span", + 3, + " bilhtap tropmi", + -1, + 1, + "span", + 3, + " sys tropmi", + -1, + 1, + "span", + 3, + " :)emanelif(elif_rorrim fed", + -1, + 1, + "span", + 3, + " :)\"r\"=edom(nepo.)emanelif(htaP.bilhtap ni enil rof", + -1, + 1, + "span", + 3, + " )\"}27", + 3, + ">", + 3, + ":]1-::[)(pirtsr.enil{\"f(tnirp", + -1, + 1, + "span", + 3, + " :\"__niam__\" == __eman__ fi", + -1, + 1, + "span", + 3, + " :]:1[vgra.sys ni emanelif rof", + -1, + 1, + "span", + 3, + " )emanelif(elif_rorrim", + -4, + 1, + "template", + 1, + "span", + 3, + "Copied!", + -3, + 1, "p", 3, "\n Note the ", @@ -4158,6 +15376,108 @@ "—are equivalent:\n ", -1, 1, + "div", + 2, + "data-is-repl", + "true", + 2, + "data-syntax-language", + "pycon", + 2, + "aria-label", + "Code block", + 1, + "div", + 1, + "pre", + 1, + "code", + 1, + "span", + 3, + ">", + 3, + ">", + 3, + ">", + -1, + 1, + "span", + 3, + "it", + -1, + 1, + "span", + 3, + "=", + -1, + 1, + "span", + 3, + "things", + -1, + 1, + "span", + 3, + ".", + -1, + 1, + "span", + 3, + "__aiter__", + -1, + 1, + "span", + 3, + "()", + -1, + 1, + "span", + 3, + ">", + 3, + ">", + 3, + ">", + -1, + 1, + "span", + 3, + "it", + -1, + 1, + "span", + 3, + "=", + -1, + 1, + "span", + 3, + "aiter", + -1, + 1, + "span", + 3, + "(", + -1, + 1, + "span", + 3, + "things", + -1, + 1, + "span", + 3, + ")", + -4, + 1, + "template", + 1, + "span", + 3, + "Copied!", + -3, + 1, "p", 3, "\n In either case, ", @@ -4226,6 +15546,627 @@ " before running this code:\n ", -1, 1, + "div", + 2, + "data-syntax-language", + "python", + 2, + "aria-label", + "Code block", + 1, + "div", + 1, + "pre", + 1, + "code", + 1, + "span", + 3, + "# line_count.py", + -1, + 1, + "span", + 3, + "import", + -1, + 1, + "span", + 3, + "asyncio", + -1, + 1, + "span", + 3, + "import", + -1, + 1, + "span", + 3, + "sys", + -1, + 1, + "span", + 3, + "import", + -1, + 1, + "span", + 3, + "aiofiles", + -1, + 1, + "span", + 3, + "async", + -1, + 1, + "span", + 3, + "def", + -1, + 1, + "span", + 3, + "count_lines", + -1, + 1, + "span", + 3, + "(", + -1, + 1, + "span", + 3, + "filename", + -1, + 1, + "span", + 3, + "):", + -1, + 1, + "span", + -1, + 1, + "span", + 3, + "\"\"\"Count the number of lines in the given file\"\"\"", + -1, + 1, + "span", + 3, + "num_lines", + -1, + 1, + "span", + 3, + "=", + -1, + 1, + "span", + 3, + "0", + -1, + 1, + "span", + 3, + "async", + -1, + 1, + "span", + 3, + "with", + -1, + 1, + "span", + 3, + "aiofiles", + -1, + 1, + "span", + 3, + ".", + -1, + 1, + "span", + 3, + "open", + -1, + 1, + "span", + 3, + "(", + -1, + 1, + "span", + 3, + "filename", + -1, + 1, + "span", + 3, + ",", + -1, + 1, + "span", + 3, + "mode", + -1, + 1, + "span", + 3, + "=", + -1, + 1, + "span", + 3, + "\"r\"", + -1, + 1, + "span", + 3, + ")", + -1, + 1, + "span", + 3, + "as", + -1, + 1, + "span", + 3, + "file", + -1, + 1, + "span", + 3, + ":", + -1, + 1, + "span", + 1, + "span", + 3, + "lines", + -1, + 1, + "span", + 3, + "=", + -1, + 1, + "span", + 3, + "aiter", + -1, + 1, + "span", + 3, + "(", + -1, + 1, + "span", + 3, + "file", + -1, + 1, + "span", + 3, + ")", + -2, + 1, + "span", + 3, + "while", + -1, + 1, + "span", + 3, + "True", + -1, + 1, + "span", + 3, + ":", + -1, + 1, + "span", + 3, + "try", + -1, + 1, + "span", + 3, + ":", + -1, + 1, + "span", + 1, + "span", + 3, + "await", + -1, + 1, + "span", + 3, + "anext", + -1, + 1, + "span", + 3, + "(", + -1, + 1, + "span", + 3, + "lines", + -1, + 1, + "span", + 3, + ")", + -2, + 1, + "span", + 3, + "num_lines", + -1, + 1, + "span", + 3, + "+=", + -1, + 1, + "span", + 3, + "1", + -1, + 1, + "span", + 3, + "except", + -1, + 1, + "span", + 3, + "StopAsyncIteration", + -1, + 1, + "span", + 3, + ":", + -1, + 1, + "span", + 3, + "break", + -1, + 1, + "span", + 3, + "print", + -1, + 1, + "span", + 3, + "(", + -1, + 1, + "span", + 3, + "f", + -1, + 1, + "span", + 3, + "\"", + -1, + 1, + "span", + 3, + "{", + -1, + 1, + "span", + 3, + "filename", + -1, + 1, + "span", + 3, + "}", + -1, + 1, + "span", + 3, + ": ", + -1, + 1, + "span", + 3, + "{", + -1, + 1, + "span", + 3, + "num_lines", + -1, + 1, + "span", + 3, + "}", + -1, + 1, + "span", + 3, + "\"", + -1, + 1, + "span", + 3, + ")", + -1, + 1, + "span", + 3, + "async", + -1, + 1, + "span", + 3, + "def", + -1, + 1, + "span", + 3, + "count_all_files", + -1, + 1, + "span", + 3, + "(", + -1, + 1, + "span", + 3, + "filenames", + -1, + 1, + "span", + 3, + "):", + -1, + 1, + "span", + -1, + 1, + "span", + 3, + "\"\"\"Asynchronously count lines in all files\"\"\"", + -1, + 1, + "span", + 3, + "tasks", + -1, + 1, + "span", + 3, + "=", + -1, + 1, + "span", + 3, + "[", + -1, + 1, + "span", + 3, + "asyncio", + -1, + 1, + "span", + 3, + ".", + -1, + 1, + "span", + 3, + "create_task", + -1, + 1, + "span", + 3, + "(", + -1, + 1, + "span", + 3, + "count_lines", + -1, + 1, + "span", + 3, + "(", + -1, + 1, + "span", + 3, + "f", + -1, + 1, + "span", + 3, + "))", + -1, + 1, + "span", + 3, + "for", + -1, + 1, + "span", + 3, + "f", + -1, + 1, + "span", + 3, + "in", + -1, + 1, + "span", + 3, + "filenames", + -1, + 1, + "span", + 3, + "]", + -1, + 1, + "span", + 3, + "await", + -1, + 1, + "span", + 3, + "asyncio", + -1, + 1, + "span", + 3, + ".", + -1, + 1, + "span", + 3, + "gather", + -1, + 1, + "span", + 3, + "(", + -1, + 1, + "span", + 3, + "*", + -1, + 1, + "span", + 3, + "tasks", + -1, + 1, + "span", + 3, + ")", + -1, + 1, + "span", + 3, + "if", + -1, + 1, + "span", + 3, + "__name__", + -1, + 1, + "span", + 3, + "==", + -1, + 1, + "span", + 3, + "\"__main__\"", + -1, + 1, + "span", + 3, + ":", + -1, + 1, + "span", + 3, + "asyncio", + -1, + 1, + "span", + 3, + ".", + -1, + 1, + "span", + 3, + "run", + -1, + 1, + "span", + 3, + "(", + -1, + 1, + "span", + 3, + "count_all_files", + -1, + 1, + "span", + 3, + "(", + -1, + 1, + "span", + 3, + "filenames", + -1, + 1, + "span", + 3, + "=", + -1, + 1, + "span", + 3, + "sys", + -1, + 1, + "span", + 3, + ".", + -1, + 1, + "span", + 3, + "argv", + -1, + 1, + "span", + 3, + "[", + -1, + 1, + "span", + 3, + "1", + -1, + 1, + "span", + 3, + ":]))", + -4, + 1, + "template", + 1, + "span", + 3, + "Copied!", + -3, + 1, "p", 1, "code", @@ -4320,6 +16261,197 @@ " statements like this:\n ", -1, 1, + "div", + 2, + "data-syntax-language", + "python", + 2, + "aria-label", + "Code block", + 1, + "div", + 1, + "pre", + 1, + "code", + 1, + "span", + 3, + "with", + -1, + 1, + "span", + 3, + "(", + -1, + 1, + "span", + 3, + "read_path", + -1, + 1, + "span", + 3, + ".", + -1, + 1, + "span", + 3, + "open", + -1, + 1, + "span", + 3, + "(", + -1, + 1, + "span", + 3, + "mode", + -1, + 1, + "span", + 3, + "=", + -1, + 1, + "span", + 3, + "\"r\"", + -1, + 1, + "span", + 3, + ",", + -1, + 1, + "span", + 3, + "encoding", + -1, + 1, + "span", + 3, + "=", + -1, + 1, + "span", + 3, + "\"utf-8\"", + -1, + 1, + "span", + 3, + ")", + -1, + 1, + "span", + 3, + "as", + -1, + 1, + "span", + 3, + "read_file", + -1, + 1, + "span", + 3, + ",", + -1, + 1, + "span", + 3, + "write_path", + -1, + 1, + "span", + 3, + ".", + -1, + 1, + "span", + 3, + "open", + -1, + 1, + "span", + 3, + "(", + -1, + 1, + "span", + 3, + "mode", + -1, + 1, + "span", + 3, + "=", + -1, + 1, + "span", + 3, + "\"w\"", + -1, + 1, + "span", + 3, + ",", + -1, + 1, + "span", + 3, + "encoding", + -1, + 1, + "span", + 3, + "=", + -1, + 1, + "span", + 3, + "\"utf-8\"", + -1, + 1, + "span", + 3, + ")", + -1, + 1, + "span", + 3, + "as", + -1, + 1, + "span", + 3, + "write_file", + -1, + 1, + "span", + 3, + ",", + -1, + 1, + "span", + 3, + "):", + -1, + 1, + "span", + 3, + "...", + -4, + 1, + "template", + 1, + "span", + 3, + "Copied!", + -3, + 1, "p", 3, "\n In earlier versions of Python, this causes an ", @@ -4339,6 +16471,189 @@ ") if you want to control where you break your lines:\n ", -1, 1, + "div", + 2, + "data-syntax-language", + "python", + 2, + "aria-label", + "Code block", + 1, + "div", + 1, + "pre", + 1, + "code", + 1, + "span", + 3, + "with", + -1, + 1, + "span", + 3, + "read_path", + -1, + 1, + "span", + 3, + ".", + -1, + 1, + "span", + 3, + "open", + -1, + 1, + "span", + 3, + "(", + -1, + 1, + "span", + 3, + "mode", + -1, + 1, + "span", + 3, + "=", + -1, + 1, + "span", + 3, + "\"r\"", + -1, + 1, + "span", + 3, + ",", + -1, + 1, + "span", + 3, + "encoding", + -1, + 1, + "span", + 3, + "=", + -1, + 1, + "span", + 3, + "\"utf-8\"", + -1, + 1, + "span", + 3, + ")", + -1, + 1, + "span", + 3, + "as", + -1, + 1, + "span", + 3, + "read_file", + -1, + 1, + "span", + 3, + ",", + -1, + 3, + " \\\n ", + 1, + "span", + 3, + "write_path", + -1, + 1, + "span", + 3, + ".", + -1, + 1, + "span", + 3, + "open", + -1, + 1, + "span", + 3, + "(", + -1, + 1, + "span", + 3, + "mode", + -1, + 1, + "span", + 3, + "=", + -1, + 1, + "span", + 3, + "\"w\"", + -1, + 1, + "span", + 3, + ",", + -1, + 1, + "span", + 3, + "encoding", + -1, + 1, + "span", + 3, + "=", + -1, + 1, + "span", + 3, + "\"utf-8\"", + -1, + 1, + "span", + 3, + ")", + -1, + 1, + "span", + 3, + "as", + -1, + 1, + "span", + 3, + "write_file", + -1, + 1, + "span", + 3, + ":", + -1, + 1, + "span", + 3, + "...", + -4, + 1, + "template", + 1, + "span", + 3, + "Copied!", + -3, + 1, "p", 3, "\n While ", @@ -4808,6 +17123,152 @@ " has two new attributes. First, you can now get a list of the names of all modules in the standard library:\n ", -1, 1, + "div", + 2, + "data-is-repl", + "true", + 2, + "data-syntax-language", + "pycon", + 2, + "aria-label", + "Code block", + 1, + "div", + 1, + "pre", + 1, + "code", + 1, + "span", + 3, + ">", + 3, + ">", + 3, + ">", + -1, + 1, + "span", + 3, + "import", + -1, + 1, + "span", + 3, + "sys", + -1, + 1, + "span", + 3, + ">", + 3, + ">", + 3, + ">", + -1, + 1, + "span", + 3, + "len", + -1, + 1, + "span", + 3, + "(", + -1, + 1, + "span", + 3, + "sys", + -1, + 1, + "span", + 3, + ".", + -1, + 1, + "span", + 3, + "stdlib_module_names", + -1, + 1, + "span", + 3, + ")", + -1, + 1, + "span", + 3, + "302", + -1, + 1, + "span", + 3, + ">", + 3, + ">", + 3, + ">", + -1, + 1, + "span", + 3, + "sorted", + -1, + 1, + "span", + 3, + "(", + -1, + 1, + "span", + 3, + "sys", + -1, + 1, + "span", + 3, + ".", + -1, + 1, + "span", + 3, + "stdlib_module_names", + -1, + 1, + "span", + 3, + ")[", + -1, + 1, + "span", + 3, + "-", + -1, + 1, + "span", + 3, + "5", + -1, + 1, + "span", + 3, + ":]", + -1, + 1, + "span", + 3, + "['zipapp', 'zipfile', 'zipimport', 'zlib', 'zoneinfo']", + -4, + 1, + "template", + 1, + "span", + 3, + "Copied!", + -3, + 1, "p", 3, "\n Here, you can see that there are around 300 modules in the standard library, several of which start with the letter ", @@ -4878,6 +17339,187 @@ "\n One possible use case for the new attribute is to identify which of the currently imported modules are third-party dependencies:\n ", -1, 1, + "div", + 2, + "data-is-repl", + "true", + 2, + "data-syntax-language", + "pycon", + 2, + "aria-label", + "Code block", + 1, + "div", + 1, + "pre", + 1, + "code", + 1, + "span", + 3, + ">", + 3, + ">", + 3, + ">", + -1, + 1, + "span", + 3, + "import", + -1, + 1, + "span", + 3, + "pandas", + -1, + 1, + "span", + 3, + "as", + -1, + 1, + "span", + 3, + "pd", + -1, + 1, + "span", + 3, + ">", + 3, + ">", + 3, + ">", + -1, + 1, + "span", + 3, + "import", + -1, + 1, + "span", + 3, + "sys", + -1, + 1, + "span", + 3, + ">", + 3, + ">", + 3, + ">", + -1, + 1, + "span", + 3, + "{", + -1, + 1, + "span", + 3, + "m", + -1, + 1, + "span", + 3, + "for", + -1, + 1, + "span", + 3, + "m", + -1, + 1, + "span", + 3, + "in", + -1, + 1, + "span", + 3, + "sys", + -1, + 1, + "span", + 3, + ".", + -1, + 1, + "span", + 3, + "modules", + -1, + 1, + "span", + 3, + "if", + -1, + 1, + "span", + 3, + "\".\"", + -1, + 1, + "span", + 3, + "not", + -1, + 1, + "span", + 3, + "in", + -1, + 1, + "span", + 3, + "m", + -1, + 1, + "span", + 3, + "}", + -1, + 1, + "span", + 3, + "-", + -1, + 1, + "span", + 3, + "sys", + -1, + 1, + "span", + 3, + ".", + -1, + 1, + "span", + 3, + "stdlib_module_names", + -1, + 1, + "span", + 3, + "{'__main__', 'numpy', '_cython_0_29_24', 'dateutil', 'pytz',", + -1, + 1, + "span", + 3, + " 'six', 'pandas', 'cython_runtime'}", + -4, + 1, + "template", + 1, + "span", + 3, + "Copied!", + -3, + 1, "p", 3, "\n You find the imported top-level modules by looking at names in ", @@ -4979,6 +17621,152 @@ " executable itself. Consider the following example:\n ", -1, 1, + "div", + 2, + "data-syntax-language", + "python", + 2, + "aria-label", + "Code block", + 1, + "div", + 1, + "pre", + 1, + "code", + 1, + "span", + 3, + "# argvs.py", + -1, + 1, + "span", + 3, + "import", + -1, + 1, + "span", + 3, + "sys", + -1, + 1, + "span", + 3, + "print", + -1, + 1, + "span", + 3, + "(", + -1, + 1, + "span", + 3, + "f", + -1, + 1, + "span", + 3, + "\"argv: ", + -1, + 1, + "span", + 3, + "{", + -1, + 1, + "span", + 3, + "sys", + -1, + 1, + "span", + 3, + ".", + -1, + 1, + "span", + 3, + "argv", + -1, + 1, + "span", + 3, + "}", + -1, + 1, + "span", + 3, + "\"", + -1, + 1, + "span", + 3, + ")", + -1, + 1, + "span", + 3, + "print", + -1, + 1, + "span", + 3, + "(", + -1, + 1, + "span", + 3, + "f", + -1, + 1, + "span", + 3, + "\"orig_argv: ", + -1, + 1, + "span", + 3, + "{", + -1, + 1, + "span", + 3, + "sys", + -1, + 1, + "span", + 3, + ".", + -1, + 1, + "span", + 3, + "orig_argv", + -1, + 1, + "span", + 3, + "}", + -1, + 1, + "span", + 3, + "\"", + -1, + 1, + "span", + 3, + ")", + -4, + 1, + "template", + 1, + "span", + 3, + "Copied!", + -3, + 1, "p", 3, "\n This script echoes back the ", @@ -4998,6 +17786,82 @@ " lists. Run it to see how the information is captured:\n ", -1, 1, + "div", + 2, + "data-is-repl", + "true", + 2, + "data-syntax-language", + "console", + 2, + "aria-label", + "Code block", + 1, + "div", + 1, + "pre", + 1, + "code", + 1, + "span", + 3, + "$ ", + -1, + 3, + "python", + 1, + "span", + -1, + 3, + "-X", + 1, + "span", + -1, + 3, + "utf8", + 1, + "span", + -1, + 3, + "-O", + 1, + "span", + -1, + 3, + "argvs.py", + 1, + "span", + -1, + 1, + "span", + 3, + "3", + -1, + 3, + ".10", + 1, + "span", + -1, + 3, + "--upgrade\n", + 1, + "span", + 3, + "argv: ['argvs.py', '3.10', '--upgrade']", + -1, + 1, + "span", + 3, + "orig_argv: ['python', '-X', 'utf8', '-O', 'argvs.py', '3.10', '--upgrade']", + -4, + 1, + "template", + 1, + "span", + 3, + "Copied!", + -3, + 1, "p", 3, "\n Essentially, all arguments—including the name of the Python executable—end up in ", @@ -5053,6 +17917,97 @@ ", like this:\n ", -1, 1, + "div", + 2, + "data-syntax-language", + "python", + 2, + "aria-label", + "Code block", + 1, + "div", + 1, + "pre", + 1, + "code", + 1, + "span", + 3, + "list", + -1, + 1, + "span", + 3, + "(", + -1, + 1, + "span", + 3, + "zip", + -1, + 1, + "span", + 3, + "(", + -1, + 1, + "span", + 3, + "names", + -1, + 1, + "span", + 3, + ",", + -1, + 1, + "span", + 3, + "set_numbers", + -1, + 1, + "span", + 3, + ",", + -1, + 1, + "span", + 3, + "num_pieces", + -1, + 1, + "span", + 3, + ",", + -1, + 1, + "span", + 3, + "strict", + -1, + 1, + "span", + 3, + "=", + -1, + 1, + "span", + 3, + "__debug__", + -1, + 1, + "span", + 3, + "))", + -4, + 1, + "template", + 1, + "span", + 3, + "Copied!", + -3, + 1, "p", 3, "\n The ", @@ -5229,6 +18184,47 @@ " import:\n ", -1, 1, + "div", + 2, + "data-syntax-language", + "python", + 2, + "aria-label", + "Code block", + 1, + "div", + 1, + "pre", + 1, + "code", + 1, + "span", + 3, + "from", + -1, + 1, + "span", + 3, + "__future__", + -1, + 1, + "span", + 3, + "import", + -1, + 1, + "span", + 3, + "annotations", + -4, + 1, + "template", + 1, + "span", + 3, + "Copied!", + -3, + 1, "p", 3, "\n The intention was that postponed evaluation would become the default at some point in the future. After the ", @@ -5296,6 +18292,242 @@ " function has been added. You should call this to access annotations at runtime:\n ", -1, 1, + "div", + 2, + "data-is-repl", + "true", + 2, + "data-syntax-language", + "pycon", + 2, + "aria-label", + "Code block", + 1, + "div", + 1, + "pre", + 1, + "code", + 1, + "span", + 3, + ">", + 3, + ">", + 3, + ">", + -1, + 1, + "span", + 3, + "import", + -1, + 1, + "span", + 3, + "inspect", + -1, + 1, + "span", + 3, + ">", + 3, + ">", + 3, + ">", + -1, + 1, + "span", + 3, + "def", + -1, + 1, + "span", + 3, + "mean", + -1, + 1, + "span", + 3, + "(", + -1, + 1, + "span", + 3, + "numbers", + -1, + 1, + "span", + 3, + ":", + -1, + 1, + "span", + 3, + "list", + -1, + 1, + "span", + 3, + "[", + -1, + 1, + "span", + 3, + "int", + -1, + 1, + "span", + 3, + "|", + -1, + 1, + "span", + 3, + "float", + -1, + 1, + "span", + 3, + "])", + -1, + 1, + "span", + 3, + "-", + 3, + ">", + -1, + 1, + "span", + 3, + "float", + -1, + 1, + "span", + 3, + ":", + -1, + 1, + "span", + 3, + "... ", + -1, + 1, + "span", + 3, + "return", + -1, + 1, + "span", + 3, + "sum", + -1, + 1, + "span", + 3, + "(", + -1, + 1, + "span", + 3, + "numbers", + -1, + 1, + "span", + 3, + ")", + -1, + 1, + "span", + 3, + "/", + -1, + 1, + "span", + 3, + "len", + -1, + 1, + "span", + 3, + "(", + -1, + 1, + "span", + 3, + "numbers", + -1, + 1, + "span", + 3, + ")", + -1, + 1, + "span", + 3, + "...", + -1, + 1, + "span", + 3, + ">", + 3, + ">", + 3, + ">", + -1, + 1, + "span", + 3, + "inspect", + -1, + 1, + "span", + 3, + ".", + -1, + 1, + "span", + 3, + "get_annotations", + -1, + 1, + "span", + 3, + "(", + -1, + 1, + "span", + 3, + "mean", + -1, + 1, + "span", + 3, + ")", + -1, + 1, + "span", + 3, + "{'numbers': list[int | float], 'return': ", + 3, + "<", + 3, + "class 'float'", + 3, + ">", + 3, + "}", + -4, + 1, + "template", + 1, + "span", + 3, + "Copied!", + -3, + 1, "p", 3, "\n Check out ", @@ -5349,6 +18581,107 @@ " comparison of version strings until now. While it’s never been good practice, it’s been possible to do the following:\n ", -1, 1, + "div", + 2, + "data-syntax-language", + "python", + 2, + "aria-label", + "Code block", + 1, + "div", + 1, + "pre", + 1, + "code", + 1, + "span", + 3, + "# bad_version_check.py", + -1, + 1, + "span", + 3, + "import", + -1, + 1, + "span", + 3, + "sys", + -1, + 1, + "span", + 3, + "# Don't do the following", + -1, + 1, + "span", + 3, + "if", + -1, + 1, + "span", + 3, + "sys", + -1, + 1, + "span", + 3, + ".", + -1, + 1, + "span", + 3, + "version", + -1, + 1, + "span", + 3, + "<", + -1, + 1, + "span", + 3, + "\"3.6\"", + -1, + 1, + "span", + 3, + ":", + -1, + 1, + "span", + 3, + "raise", + -1, + 1, + "span", + 3, + "SystemExit", + -1, + 1, + "span", + 3, + "(", + -1, + 1, + "span", + 3, + "\"Only Python 3.6 and above is supported\"", + -1, + 1, + "span", + 3, + ")", + -4, + 1, + "template", + 1, + "span", + 3, + "Copied!", + -3, + 1, "p", 3, "\n In Python 3.10, this code will raise ", @@ -5380,6 +18713,117 @@ "\n The correct way to compare version numbers is to use tuples of numbers:\n ", -1, 1, + "div", + 2, + "data-syntax-language", + "python", + 2, + "aria-label", + "Code block", + 1, + "div", + 1, + "pre", + 1, + "code", + 1, + "span", + 3, + "# good_version_check.py", + -1, + 1, + "span", + 3, + "import", + -1, + 1, + "span", + 3, + "sys", + -1, + 1, + "span", + 3, + "if", + -1, + 1, + "span", + 3, + "sys", + -1, + 1, + "span", + 3, + ".", + -1, + 1, + "span", + 3, + "version_info", + -1, + 1, + "span", + 3, + "<", + -1, + 1, + "span", + 3, + "(", + -1, + 1, + "span", + 3, + "3", + -1, + 1, + "span", + 3, + ",", + -1, + 1, + "span", + 3, + "6", + -1, + 1, + "span", + 3, + "):", + -1, + 1, + "span", + 3, + "raise", + -1, + 1, + "span", + 3, + "SystemExit", + -1, + 1, + "span", + 3, + "(", + -1, + 1, + "span", + 3, + "\"Only Python 3.6 and above is supported\"", + -1, + 1, + "span", + 3, + ")", + -4, + 1, + "template", + 1, + "span", + 3, + "Copied!", + -3, + 1, "p", 1, "a", @@ -5410,6 +18854,84 @@ " to make sure you’re handling versions correctly:\n ", -1, 1, + "div", + 2, + "data-is-repl", + "true", + 2, + "data-syntax-language", + "console", + 2, + "aria-label", + "Code block", + 1, + "div", + 1, + "pre", + 1, + "code", + 1, + "span", + 3, + "$ ", + -1, + 3, + "python", + 1, + "span", + -1, + 3, + "-m", + 1, + "span", + -1, + 3, + "pip", + 1, + "span", + -1, + 3, + "install", + 1, + "span", + -1, + 3, + "flake8-2020\n\n", + 1, + "span", + 3, + "$ ", + -1, + 3, + "flake8", + 1, + "span", + -1, + 3, + "bad_version_check.py", + 1, + "span", + -1, + 3, + "good_version_check.py\n", + 1, + "span", + 3, + "bad_version_check.py:3:4: YTT103 `sys.version` compared to string", + -1, + 1, + "span", + 3, + " (python3.10), use `sys.version_info`", + -4, + 1, + "template", + 1, + "span", + 3, + "Copied!", + -3, + 1, "p", 3, "\n With the ", diff --git a/packages/readabilityjs/test/test-pages/realpython/expected.html b/packages/readabilityjs/test/test-pages/realpython/expected.html index 0fe34c658..04cdd5b16 100644 --- a/packages/readabilityjs/test/test-pages/realpython/expected.html +++ b/packages/readabilityjs/test/test-pages/realpython/expected.html @@ -31,22 +31,147 @@

    Python is often lauded for being a user-friendly programming language. While this is true, there are certain parts of Python that could be friendlier. Python 3.10 comes with a host of more precise and constructive error messages. In this section, you’ll see some of the newest improvements. The full list is available in the documentation.

    Think back to writing your first Hello World program in Python:

    +
    +
    + 
    # hello.py
+
+print("Hello, World!)
+
    +
    +

    Maybe you created a file, added the famous call to print(), and saved it as hello.py. You then ran the program, eager to call yourself a proper Pythonista. However, something went wrong:

    +
    +
    + 
    $ python hello.py
+  File "/home/rp/hello.py", line 3
+    print("Hello, World!)
+                        ^
+SyntaxError: EOL while scanning string literal
+
    +
    +

    There was a SyntaxError in the code. EOL, what does that even mean? You went back to your code, and after a bit of staring and searching, you realized that there was a missing quotation mark at the end of your string.

    One of the more impactful improvements in Python 3.10 is better and more precise error messages for many common issues. If you run your buggy Hello World in Python 3.10, you’ll get a bit more help than in earlier versions of Python:

    +
    +
    + 
    $ python hello.py
+  File "/home/rp/hello.py", line 3
+    print("Hello, World!)
+          ^
+SyntaxError: unterminated string literal (detected at line 3)
+
    +
    +

    The error message is still a bit technical, but gone is the mysterious EOL. Instead, the message tells you that you need to terminate your string! There are similar improvements to many different error messages, as you’ll see below.

    A SyntaxError is an error raised when your code is parsed, before it even starts to execute. Syntax errors can be tricky to debug because the interpreter provides imprecise or sometimes even misleading error messages. The following code is missing a curly brace to terminate the dictionary:

    +
    +
    + 
     1# unterminated_dict.py
+ 2
+ 3months = {
+ 4    10: "October",
+ 5    11: "November",
+ 6    12: "December"
+ 7
+ 8print(f"{months[10]} is the tenth month")
+
    +
    +

    The missing closing curly brace that should have been on line 7 is an error. If you run this code with Python 3.9 or earlier, you’ll see the following error message:

    +
    +
    + 
      File "/home/rp/unterminated_dict.py", line 8
+    print(f"{months[10]} is the tenth month")
+    ^
+SyntaxError: invalid syntax
+
    +
    +

    The error message highlights line 8, but there are no syntactical problems in line 8! If you’ve experienced your share of syntax errors in Python, you might already know that the trick is to look at the lines before the one Python complains about. In this case, you’re looking for the missing closing brace on line 7.

    In Python 3.10, the same code shows a much more helpful and precise error message:

    +
    +
    + 
      File "/home/rp/unterminated_dict.py", line 3
+    months = {
+             ^
+SyntaxError: '{' was never closed
+
    +
    +

    This points you straight to the offending dictionary and allows you to fix the issue in no time.

    There are a few other ways to mess up dictionary syntax. A typical one is forgetting a comma after one of the items:

    +
    +
    + 
     1# missing_comma.py
+ 2
+ 3months = {
+ 4    10: "October"
+ 5    11: "November",
+ 6    12: "December",
+ 7}
+
    +
    +

    In this code, a comma is missing at the end of line 4. Python 3.10 gives you a clear suggestion on how to fix your code:

    +
    +
    + 
      File "/home/real_python/missing_comma.py", line 4
+    10: "October"
+        ^^^^^^^^^
+SyntaxError: invalid syntax. Perhaps you forgot a comma?
+
    +
    +

    You can add the missing comma and have your code back up and running in no time.

    Another common mistake is using the assignment operator (=) instead of the equality comparison operator (==) when you’re comparing values. Previously, this would just cause another invalid syntax message. In the newest version of Python, you get some more advice:

    +
    +
    + 
    >>> if month = "October":
+  File "<stdin>", line 1
+    if month = "October":
+       ^^^^^^^^^^^^^^^^^
+SyntaxError: invalid syntax. Maybe you meant '==' or ':=' instead of '='?
+
    +
    +

    The parser suggests that you maybe meant to use a comparison operator or an assignment expression operator instead.

    Take note of another nifty improvement in Python 3.10 error messages. The last two examples show how carets (^^^) highlight the whole offending expression. Previously, a single caret symbol (^) indicated just an approximate location.

    The final error message improvement that you’ll play with for now is that attribute and name errors can now offer suggestions if you misspell an attribute or a name:

    +
    +
    + 
    >>> import math
+>>> math.py
+AttributeError: module 'math' has no attribute 'py'. Did you mean: 'pi'?
+
+>>> pint
+NameError: name 'pint' is not defined. Did you mean: 'print'?
+
+>>> release = "3.10"
+>>> relaese
+NameError: name 'relaese' is not defined. Did you mean: 'release'?
+
    +
    +

    Note that the suggestions work for both built-in names and names that you define yourself, although they may not be available in all environments. If you like these kinds of suggestions, check out BetterErrorMessages, which offers similar suggestions in even more contexts.

    The improvements you’ve seen in this section are just some of the many error messages that have gotten a face-lift. The new Python will be even more user-friendly than before, and hopefully, the new error messages will save you both time and frustration going forward.

    @@ -70,23 +195,196 @@

    At its core, structural pattern matching is about defining patterns to which your data structures can be matched. In this section, you’ll study a practical example where you’ll work with data that are structured differently, even though the meaning is the same. You’ll define several patterns, and depending on which pattern matches your data, you’ll process your data appropriately.

    This section will be a bit light on explanations of the possible patterns. Instead, it will try to give you an impression of the possibilities. The next section will step back and explain the patterns in more detail.

    Time to match your first pattern! The following example uses a match ... case block to find the first name of a user by extracting it from a user data structure:

    +
    +
    + 
    >>> user = {
+...     "name": {"first": "Pablo", "last": "Galindo Salgado"},
+...     "title": "Python 3.10 release manager",
+... }
+
+>>> match user:
+...     case {"name": {"first": first_name}}:
+...         pass
+...
+
+>>> first_name
+'Pablo'
+
    +
    +

    You can see structural pattern matching at work in the highlighted lines. user is a small dictionary with user information. The case line specifies a pattern that user is matched against. In this case, you’re looking for a dictionary with a "name" key whose value is a new dictionary. This nested dictionary has a key called "first". The corresponding value is bound to the variable first_name.

    For a practical example, say that you’re processing user data where the underlying data model changes over time. Therefore, you need to be able to process different versions of the same data.

    In the next example, you’ll use data from randomuser.me. This is a great API for generating random user data that you can use during testing and development. The API is also an example of an API that has changed over time. You can still access the old versions of the API.

    You may expand the collapsed section below to see how you can use requests to obtain different versions of the user data using the API:

    You can get a random user from the API using requests as follows:

    +
    +
    + 
    # random_user.py
+
+import requests
+
+def get_user(version="1.3"):
+    """Get random users"""
+    url = f"https://randomuser.me/api/{version}/?results=1"
+    response = requests.get(url)
+    if response:
+        return response.json()["results"][0]
+
    +
    +

    get_user() gets one random user in JSON format. Note the version parameter. The structure of the returned data has changed quite a bit between earlier versions like "1.1" and the current version "1.3", but in each case, the actual user data are contained in a list inside the "results" array. The function returns the first—and only—user in this list.

    At the time of writing, the latest version of the API is 1.3 and the data has the following structure:

    +
    +
    + 
    {
+    "gender": "female",
+    "name": {
+        "title": "Miss",
+        "first": "Ilona",
+        "last": "Jokela"
+    },
+    "location": {
+        "street": {
+            "number": 4473,
+            "name": "Mannerheimintie"
+        },
+        "city": "Harjavalta",
+        "state": "Ostrobothnia",
+        "country": "Finland",
+        "postcode": 44879,
+        "coordinates": {
+            "latitude": "-6.0321",
+            "longitude": "123.2213"
+        },
+        "timezone": {
+            "offset": "+5:30",
+            "description": "Bombay, Calcutta, Madras, New Delhi"
+        }
+    },
+    "email": "ilona.jokela@example.com",
+    "login": {
+        "uuid": "632b7617-6312-4edf-9c24-d6334a6af52d",
+        "username": "brownsnake482",
+        "password": "biatch",
+        "salt": "ofk518ZW",
+        "md5": "6d589615ca44f6e583c85d45bf431c54",
+        "sha1": "cd87c931d579bdff77af96c09e0eea82d1edfc19",
+        "sha256": "6038ede83d4ce74116faa67fb3b1b2e6f6898e5749b57b5a0312bd46a539214a"
+    },
+    "dob": {
+        "age": 64
+    },
+    "registered": {
+        "date": ,
+        "age": 15
+    },
+    "phone": "07-369-318",
+    "cell": "048-284-01-59",
+    "id": {
+        "name": "HETU",
+        "value": "NaNNA204undefined"
+    },
+    "picture": {
+        "large": "https://randomuser.me/api/portraits/women/28.jpg",
+        "medium": "https://randomuser.me/api/portraits/med/women/28.jpg",
+        "thumbnail": "https://randomuser.me/api/portraits/thumb/women/28.jpg"
+    },
+    "nat": "FI"
+}
+
    +
    +

    One of the members that changed between different versions is "dob", the date of birth. Note that in version 1.3, this is a JSON object with two members, "date" and "age".

    Compare the result above with a version 1.1 random user:

    +
    +
    + 
    {
+    "gender": "female",
+    "name": {
+        "title": "miss",
+        "first": "ilona",
+        "last": "jokela"
+    },
+    "location": {
+        "street": "7336 myllypuronkatu",
+        "city": "kurikka",
+        "state": "central ostrobothnia",
+        "postcode": 53740
+    },
+    "email": "ilona.jokela@example.com",
+    "login": {
+        "username": "blackelephant837",
+        "password": "sand",
+        "salt": "yofk518Z",
+        "md5": "b26367ea967600d679ee3e0b9bda012f",
+        "sha1": "87d2910595acba5b8e8aa8b00a841bab08580e2f",
+        "sha256": "73bd0d205d0dc83ae184ae222ff2e9de5ea4039119a962c4f97fabd5bbfa7aca"
+    },
+    "registered": ,
+    "phone": "04-636-931",
+    "cell": "048-828-40-15",
+    "id": {
+        "name": "HETU",
+        "value": "366-9204"
+    },
+    "picture": {
+        "large": "https://randomuser.me/api/portraits/women/24.jpg",
+        "medium": "https://randomuser.me/api/portraits/med/women/24.jpg",
+        "thumbnail": "https://randomuser.me/api/portraits/thumb/women/24.jpg"
+    },
+    "nat": "FI"
+}
+
    +
    +

    Observe that in this older format, the value of the "dob" member is a plain string.

    In this example, you’ll work with the information about the date of birth (dob) for each user. The structure of these data has changed between different versions of the Random User API:

    +
    +
    + 
    # Version 1.1
+"dob": 
+
+# Version 1.3
+"dob": {"date": , "age": 64}
+
    +
    +

    Note that in version 1.1, the date of birth is represented as a simple string, while in version 1.3, it’s a JSON object with two members: "date" and "age". Say that you want to find the age of a user. Depending on the structure of your data, you’d either need to calculate the age based on the date of birth or look up the age if it’s already available.

    Traditionally, you would detect the structure of the data with an if test, maybe based on the type of the "dob" field. You can approach this differently in Python 3.10. Now, you can use structural pattern matching instead:

    +
    +
    + 
     1# random_user.py (continued)
+ 2
+ 3from datetime import datetime
+ 4
+ 5def get_age(user):
+ 6    """Get the age of a user"""
+ 7    match user:
+ 8        case {"dob": {"age": int(age)}}:
+ 9            return age
+10        case {"dob": dob}:
+11            now = datetime.now()
+12            dob_date = datetime.strptime(dob, "%Y-%m-%d %H:%M:%S")
+13            return now.year - dob_date.year
+
    +
    +

    The match ... case construct is new in Python 3.10 and is how you perform structural pattern matching. You start with a match statement that specifies what you want to match. In this example, that’s the user data structure.

    One or several case statements follow match. Each case describes one pattern, and the indented block beneath it says what should happen if there’s a match. In this example:

      @@ -103,8 +401,36 @@

    One important feature of pattern matching is that at most one pattern will be matched. Since the pattern on line 10 matches any dictionary with "dob", it’s important that the more specific pattern on line 8 comes first.

    Before looking closer at the details of the patterns and how they work, try calling get_age() with different data structures to see the result:

    +
    +
    + 
    >>> import random_user
+
+>>> users11 = random_user.get_user(version="1.1")
+>>> random_user.get_age(users11)
+55
+
+>>> users13 = random_user.get_user(version="1.3")
+>>> random_user.get_age(users13)
+64
+
    +
    +

    Your code can calculate the age correctly for both versions of the user data, which have different dates of birth.

    Look closer at those patterns. The first pattern, {"dob": {"age": int(age)}}, matches version 1.3 of the user data:

    +
    +
    + 
    {
+    ...
+    "dob": {"date": , "age": 64},
+    ...
+}
+
    +
    +

    The first pattern is a nested pattern. The outer curly braces say that a dictionary with the key "dob" is required. The corresponding value should be a dictionary. This nested dictionary must match the subpattern {"age": int(age)}. In other words, it needs to have an "age" key with an integer value. That value is bound to the name age.

    The second pattern, {"dob": dob}, matches the older version 1.1 of the user data:

    This second pattern is a simpler pattern than the first one. Again, the curly braces indicate that it will match a dictionary. However, any dictionary with a "dob" key is matched because there are no other restrictions specified. The value of that key is bound to the name dob.

    @@ -159,22 +485,120 @@

    You already used several of them in the example in the previous section. In particular, you used mapping patterns to unravel data stored in dictionaries. In this section, you’ll learn more about how some of these work. All the details are available in the PEPs mentioned above.

    A capture pattern is used to capture a match to a pattern and bind it to a name. Consider the following recursive function that sums a list of numbers:

    +
    +
    + 
     1def sum_list(numbers):
+ 2    match numbers:
+ 3        case []:
+ 4            return 0
+ 5        case [first, *rest]:
+ 6            return first + sum_list(rest)
+
    +
    +

    The first case on line 3 matches the empty list and returns 0 as its sum. The second case on line 5 uses a sequence pattern with two capture patterns to match lists with one or more elements. The first element in the list is captured and bound to the name first. The second capture pattern, *rest, uses unpacking syntax to match any number of elements. rest will bind to a list containing all elements of numbers except the first one.

    sum_list() calculates the sum of a list of numbers by recursively adding the first number in the list and the sum of the rest of the numbers. You can use it as follows:

    +
    +
    + 
    >>> sum_list([4, 5, 9, 4])
+22
+
    +
    +

    The sum of 4 + 5 + 9 + 4 is correctly calculated to be 22. As an exercise for yourself, you can try to trace the recursive calls to sum_list() to make sure you understand how the code sums the whole list.

    sum_list() handles summing up a list of numbers. Observe what happens if you try to sum anything that isn’t a list:

    +
    +
    + 
    >>> print(sum_list("4594"))
+None
+
+>>> print(sum_list(4594))
+None
+
    +
    +

    Passing a string or a number to sum_list() returns None. This occurs because none of the patterns match, and the execution continues after the match block. That happens to be the end of the function, so sum_list() implicitly returns None.

    Often, though, you want to be alerted about failed matches. You can add a catchall pattern as the final case that handles this by raising an error, for example. You can use the underscore (_) as a wildcard pattern that matches anything without binding it to a name. You can add some error handling to sum_list() as follows:

    +
    +
    + 
    def sum_list(numbers):
+    match numbers:
+        case []:
+            return 0
+        case [first, *rest]:
+            return first + sum_list(rest)
+        case _:
+            wrong_type = numbers.__class__.__name__
+            raise ValueError(f"Can only sum lists, not {wrong_type!r}")
+
    +
    +

    The final case will match anything that doesn’t match the first two patterns. This will raise a descriptive error, for instance, if you try to calculate sum_list(4594). This is useful when you need to alert your users that some input was not matched as expected.

    Your patterns are still not foolproof, though. Consider what happens if you try to sum a list of strings:

    +
    +
    + 
    >>> sum_list(["45", "94"])
+TypeError: can only concatenate str (not "int") to str
+
    +
    +

    The base case returns 0, so therefore the summing only works for types that you can add with numbers. Python doesn’t know how to add numbers and text strings together. You can restrict your pattern to only match integers using a class pattern:

    +
    +
    + 
    def sum_list(numbers):
+    match numbers:
+        case []:
+            return 0
+        case [int(first), *rest]:
+            return first + sum_list(rest)
+        case _:
+            raise ValueError(f"Can only sum lists of numbers")
+
    +
    +

    Adding int() around first makes sure that the pattern only matches if the value is an integer. This might be too restrictive, though. Your function should be able to sum both integers and floating-point numbers, so how can you allow this in your pattern?

    To check whether at least one out of several subpatterns match, you can use an OR pattern. OR patterns consist of two or more subpatterns, and the pattern matches if at least one of the subpatterns does. You can use this to match when the first element is either of type int or type float:

    +
    +
    + 
    def sum_list(numbers):
+    match numbers:
+        case []:
+            return 0
+        case [int(first) | float(first), *rest]:
+            return first + sum_list(rest)
+        case _:
+            raise ValueError(f"Can only sum lists of numbers")
+
    +
    +

    You use the pipe symbol (|) to separate the subpatterns in an OR pattern. Your function now allows summing a list of floating-point numbers:

    +
    +
    + 
    >>> sum_list([45.94, 46.17, 46.72])
+138.82999999999998
+
    +
    +

    There’s a lot of power and flexibility within structural pattern matching, even more than what you’ve seen so far. Some things that aren’t covered in this overview are:

    • Using guards to restrict patterns
      • @@ -188,10 +612,42 @@

      Matching Literal Patterns

      A literal pattern is a pattern that matches a literal object like an explicit string or number. In a sense, this is the most basic kind of pattern and allows you to emulate switch ... case statements seen in other languages. The following example matches a specific name:

      +
      +
      + 
      def greet(name):
+    match name:
+        case "Guido":
+            print("Hi, Guido!")
+        case _:
+            print("Howdy, stranger!")
+
      +
      +

      The first case matches the literal string "Guido". In this case, you use _ as a wildcard to print a generic greeting whenever name is not "Guido". Such literal patterns can sometimes take the place of if ... elif ... else constructs and can play the same role that switch ... case does in some other languages.

      One limitation with structural pattern matching is that you can’t directly match values stored in variables. Say that you’ve defined bdfl = "Guido". A pattern like case bdfl: will not match "Guido". Instead, this will be interpreted as a capture pattern that matches anything and binds that value to bdfl, effectively overwriting the old value.

      You can, however, use a value pattern to match stored values. A value pattern looks a bit like a capture pattern but uses a previously defined dotted name that holds the value that will be matched against.

      You can, for example, use an enumeration to create such dotted names:

      +
      +
      + 
      import enum
+
+class Pythonista(str, enum.Enum):
+    BDFL = "Guido"
+    FLUFL = "Barry"
+
+def greet(name):
+    match name:
+        case Pythonista.BDFL:
+            print("Hi, Guido!")
+        case _:
+            print("Howdy, stranger!")
+
      +
      +

      The first case now uses a value pattern to match Pythonista.BDFL, which is "Guido". Note that you can use any dotted name in a value pattern. You could, for example, have used a regular class or a module instead of the enumeration.

      To see a bigger example of how to use literal patterns, consider the game of FizzBuzz. This is a counting game where you should replace some numbers with words according to the following rules:

        @@ -201,11 +657,71 @@

      FizzBuzz is sometimes used to introduce conditionals in programming education and as a screening problem in interviews. Even though a solution is quite straightforward, Joel Grus has written a full book about different ways to program the game.

      A typical solution in Python will use if ... elif ... else as follows:

      +
      +
      + 
      def fizzbuzz(number):
+    mod_3 = number % 3
+    mod_5 = number % 5
+
+    if mod_3 == 0 and mod_5 == 0:
+        return "fizzbuzz"
+    elif mod_3 == 0:
+        return "fizz"
+    elif mod_5 == 0:
+        return "buzz"
+    else:
+        return str(number)
+
      +
      +

      The % operator calculates the modulus, which you can use to test divisibility. Namely, if a modulus b is 0 for two numbers a and b, then a is divisible by b.

      In fizzbuzz(), you calculate number % 3 and number % 5, which you then use to test for divisibility with 3 and 5. Note that you must do the test for divisibility with both 3 and 5 first. If not, numbers that are divisible by both 3 and 5 will be covered by either the "fizz" or the "buzz" cases instead.

      You can check that your implementation gives the expected result:

      +
      +
      + 
      >>> fizzbuzz(3)
+fizz
+
+>>> fizzbuzz(14)
+14
+
+>>> fizzbuzz(15)
+fizzbuzz
+
+>>> fizzbuzz(92)
+92
+
+>>> fizzbuzz(65)
+buzz
+
      +
      +

      You can confirm for yourself that 3 is divisible by 3, 65 is divisible by 5, and 15 is divisible by both 3 and 5, while 14 and 92 aren’t divisible by either 3 or 5.

      An if ... elif ... else structure where you’re comparing one or a few variables several times over is quite straightforward to rewrite using pattern matching instead. For example, you can do the following:

      +
      +
      + 
      def fizzbuzz(number):
+    mod_3 = number % 3
+    mod_5 = number % 5
+
+    match (mod_3, mod_5):
+        case (0, 0):
+            return "fizzbuzz"
+        case (0, _):
+            return "fizz"
+        case (_, 0):
+            return "buzz"
+        case _:
+            return str(number)
+
      +
      +

      You match on both mod_3 and mod_5. Each case pattern then matches either the literal number 0 or the wildcard _ on the corresponding values.

      Compare and contrast this version with the previous one. Note how the pattern (0, 0) corresponds to the test mod_3 == 0 and mod_5 == 0, while (0, _) corresponds to mod_3 == 0.

      As you saw earlier, you can use an OR pattern to match on several different patterns. For example, since mod_3 can only take the values 0, 1, and 2, you can replace case (_, 0) with case (1, 0) | (2, 0). Remember that (0, 0) has already been covered.

      @@ -232,31 +748,154 @@

      PEP 604 will probably be the most widely used of these changes going forward, but you’ll get a brief overview of each of the features in this section.

      You can use union types to declare that a variable can have one of several different types. For example, you’ve been able to type hint a function calculating the mean of a list of numbers, floats, or integers as follows:

      +
      +
      + 
      from typing import List, Union
+
+def mean(numbers: List[Union[float, int]]) -> float:
+    return sum(numbers) / len(numbers)
+
      +
      +

      The annotation List[Union[float, int]] means that numbers should be a list where each element is either a floating-point number or an integer. This works well, but the notation is a bit verbose. Also, you need to import both List and Union from typing.

      In Python 3.10, you can replace Union[float, int] with the more succinct float | int. Combine this with the ability to use list instead of typing.List in type hints, which Python 3.9 introduced. You can then simplify your code while keeping all the type information:

      +
      +
      + 
      def mean(numbers: list[float | int]) -> float:
+    return sum(numbers) / len(numbers)
+
      +
      +

      The annotation of numbers is easier to read now, and as an added bonus, you didn’t need to import anything from typing.

      A special case of union types is when a variable can have either a specific type or be None. You can annotate such optional types either as Union[None, T] or, equivalently, Optional[T] for some type T. There is no new, special syntax for optional types, but you can use the new union syntax to avoid importing typing.Optional:

      In this example, address is allowed to be either None or a string.

      You can also use the new union syntax at runtime in isinstance() or issubclass() tests:

      +
      +
      + 
      >>> isinstance("mypy", str | int)
+True
+
+>>> issubclass(str, int | float | bytes)
+False
+
      +
      +

      Traditionally, you’ve used tuples to test for several types at once—for example, (str, int) instead of str | int. This old syntax will still work.

      Type aliases allow you to quickly define new aliases that can stand in for more complicated type declarations. For example, say that you’re representing a playing card using a tuple of suit and rank strings and a deck of cards by a list of such playing card tuples. A deck of cards is then type hinted as list[tuple[str, str]].

      To simplify type annotation, you define type aliases as follows:

      +
      +
      + 
      Card = tuple[str, str]
+Deck = list[Card]
+
      +
      +

      This usually works okay. However, it’s often not possible for the type checker to know whether such a statement is a type alias or just the definition of a regular global variable. To help the type checker—or really, help the type checker help you—you can now explicitly annotate type aliases:

      +
      +
      + 
      from typing import TypeAlias
+
+Card: TypeAlias = tuple[str, str]
+Deck: TypeAlias = list[Card]
+
      +
      +

      Adding the TypeAlias annotation clarifies the intention, both to a type checker and to anyone reading your code.

      Type guards are used to narrow down union types. The following function takes in either a string or None but always returns a tuple of strings representing a playing card:

      +
      +
      + 
      def get_ace(suit: str | None) -> tuple[str, str]:
+    if suit is None:
+        suit = "♠"
+    return (suit, "A")
+
      +
      +

      The highlighted line works as a type guard, and static type checkers are able to realize that suit is necessarily a string when it’s returned.

      Currently, the type checkers can only use a few different constructs to narrow down union types in this way. With the new typing.TypeGuard, you can annotate custom functions that can be used to narrow down union types:

      +
      +
      + 
      from typing import Any, TypeAlias, TypeGuard
+
+Card: TypeAlias = tuple[str, str]
+Deck: TypeAlias = list[Card]
+
+def is_deck_of_cards(obj: Any) -> TypeGuard[Deck]:
+    # Return True if obj is a deck of cards, otherwise False
+
      +
      +

      is_deck_of_cards() should return True or False depending on whether obj represents a Deck object or not. You can then use your guard function, and the type checker will be able to narrow down the types correctly:

      +
      +
      + 
      def get_score(card_or_deck: Card | Deck) -> int:
+    if is_deck_of_cards(card_or_deck):
+        # Calculate score of a deck of cards
+    ...
+
      +
      +

      Inside of the if block, the type checker knows that card_or_deck is, in fact, of the type Deck. See PEP 647 for more details.

      The final new typing feature is Parameter Specification Variables, which is related to type variables. Consider the definition of a decorator. In general, it looks something like the following:

      +
      +
      + 
      import functools
+from typing import Any, Callable, TypeVar
+
+R = TypeVar("R")
+
+def decorator(func: Callable[..., R]) -> Callable[..., R]:
+    @functools.wraps(func)
+    def wrapper(*args: Any, **kwargs: Any) -> R:
+        ...
+    return wrapper
+
      +
      +

      The annotations mean that the function returned by the decorator is a callable with some parameters and the same return type, R, as the function passed into the decorator. The ellipsis (...) in the function header correctly allows any number of parameters, and each of those parameters can be of any type. However, there’s no validation that the returned callable has the same parameters as the function that was passed in. In practice, this means that type checkers aren’t able to check decorated functions properly.

      Unfortunately, you can’t use TypeVar for the parameters because you don’t know how many parameters the function will have. In Python 3.10, you’ll have access to ParamSpec in order to type hint these kinds of callables properly. ParamSpec works similarly to TypeVar but stands in for several parameters at once. You can rewrite your decorator as follows to take advantage of ParamSpec:

      +
      +
      + 
      import functools
+from typing import Callable, ParamSpec, TypeVar
+
+P = ParamSpec("P")
+R = TypeVar("R")
+
+def decorator(func: Callable[P, R]) -> Callable[P, R]:
+    @functools.wraps(func)
+    def wrapper(*args: P.args, **kwargs: P.kwargs) -> R:
+        ...
+    return wrapper
+
      +
      +

      Note that you also use P when you annotate wrapper(). You can also use the new typing.Concatenate to add types to ParamSpec. See the documentation and PEP 612 for details and examples.

      @@ -267,25 +906,111 @@

      As an example, consider the following table of Lego sets:

      One way to represent these data in plain Python would be with each column as a list. It could look something like this:

      +
      +
      + 
      >>> names = ["Louvre", "Diagon Alley", "Saturn V", "Millennium Falcon", "NYC"]
+>>> set_numbers = ["21024", "75978", "92176", "75192", "21028"]
+>>> num_pieces = [695, 5544, 1969, 7541, 598]
+
      +
      +

      Note that you have three independent lists, but there’s an implicit correspondence between their elements. The first name ("Louvre"), the first set number ("21024"), and the first number of pieces (695) all describe the first Lego set.

      zip() can be used to iterate over these three lists in parallel:

      +
      +
      + 
      >>> for name, num, pieces in zip(names, set_numbers, num_pieces):
+...     print(f"{name} ({num}): {pieces} pieces")
+...
+Louvre (21024): 695 pieces
+Diagon Alley (75978): 5544 pieces
+Saturn V (92176): 1969 pieces
+Millennium Falcon (75192): 7541 pieces
+NYC (21028): 598 pieces
+
      +
      +

      Note how each line collects information from all three lists and shows information about one particular set. This is a very common pattern that’s used in a lot of different Python code, including in the standard library.

      You can also add list() to collect the contents of all three lists in a single, nested list of tuples:

      +
      +
      + 
      >>> list(zip(names, set_numbers, num_pieces))
+[('Louvre', '21024', 695),
+ ('Diagon Alley', '75978', 5544),
+ ('Saturn V', '92176', 1969),
+ ('Millennium Falcon', '75192', 7541),
+ ('NYC', '21028', 598)]
+
      +
      +

      Note how the nested list closely resembles the original table.

      The dark side of using zip() is that it’s quite easy to introduce a subtle bug that can be hard to discover. Note what happens if there’s a missing item in one of your lists:

      +
      +
      + 
      >>> set_numbers = ["21024", "75978", "75192", "21028"]  # Saturn V missing
+
+>>> list(zip(names, set_numbers, num_pieces))
+[('Louvre', '21024', 695),
+ ('Diagon Alley', '75978', 5544),
+ ('Saturn V', '75192', 1969),
+ ('Millennium Falcon', '21028', 7541)]
+
      +
      +

      All the information about the New York City set disappeared! Additionally, the set numbers for Saturn V and Millennium Falcon are wrong. If your datasets are bigger, these kinds of errors can be very hard to discover. And even when you observe that something’s wrong, it’s not always easy to diagnose and fix.

      The issue is that you assumed that the three lists have the same number of elements and that the information is in the same order in each list. After set_numbers gets corrupted, this assumption is no longer true.

      PEP 618 introduces a new strict keyword parameter to zip() that you can use to confirm all sequences have the same length. In your example, it would raise an error alerting you to the corrupted list:

      +
      +
      + 
      >>> list(zip(names, set_numbers, num_pieces, strict=True))
+Traceback (most recent call last):
+  File "<stdin>", line 1, in <module>
+ValueError: zip() argument 2 is shorter than argument 1
+
      +
      +

      When the iteration reaches the New York City Lego set, the second argument set_numbers is already exhausted, while there are still elements left in the first argument names. Instead of silently giving the wrong result, your code fails with an error, and you can take action to find and fix the mistake.

      There are use cases when you want to combine sequences of unequal length. Expand the box below to see how zip() and itertools.zip_longest() handle these:

      The following idiom divides the Lego sets into pairs:

      +
      +
      + 
      >>> num_per_group = 2
+>>> list(zip(*[iter(names)] * num_per_group))
+[('Louvre', 'Diagon Alley'), ('Saturn V', 'Millennium Falcon')]
+
      +
      +

      There are five sets, a number that doesn’t divide evenly into pairs. In this case, the default behavior of zip(), where the last element is dropped, might make sense. You could use strict=True here as well, but that would raise an error when your list can’t be split into pairs. A third option, which could be the best in this case, is to use zip_longest() from the itertools standard library.

      As the name suggests, zip_longest() combines sequences until the longest sequence is exhausted. If you use zip_longest() to divide the Lego sets, it becomes more explicit that New York City doesn’t have any pairing:

      +
      +
      + 
      >>> from itertools import zip_longest
+
+>>> list(zip_longest(*[iter(names)] * num_per_group, fillvalue=""))
+[('Louvre', 'Diagon Alley'),
+ ('Saturn V', 'Millennium Falcon'),
+ ('NYC', '')]
+
      +
      +

      Note that 'NYC' shows up in the last tuple together with an empty string. You can control what’s filled in for missing values with the fillvalue parameter.

      While strict is not really adding any new functionality to zip(), it can help you avoid those hard-to-find bugs.

      @@ -307,15 +1032,82 @@

    You can use each function to describe a certain aspect of the relationship between two variables. As an example, say that you have data from a set of blog posts—the number of words in each blog post and the number of views each post has had over some time period:

    +
    +
    + 
    >>> words = [7742, 11539, 16898, 13447, 4608, 6628, 2683, 6156, 2623, 6948]
+>>> views = [8368, 5901, 3978, 3329, 2611, 2096, 1515, 1177, 814, 467]
+
    +
    +

    You now want to investigate whether there’s any (linear) relationship between the number of words and number of views. In Python 3.10, you can calculate the correlation between words and views with the new correlation() function:

    +
    +
    + 
    >>> import statistics
+
+>>> statistics.correlation(words, views)
+0.454180067865917
+
    +
    +

    The correlation between two variables is always a number between -1 and 1. If it’s close to 0, then there’s little correspondence between them, while a correlation close to -1 or 1 indicates that the behaviors of the two variables tend to follow each other. In this example, a correlation of 0.45 indicates that there’s a tendency for posts with more words to have more views, although it’s not a strong connection.

    You can also calculate the covariance between words and views. The covariance is another measure of the joint variability between two variables. You can calculate it with covariance():

    +
    +
    + 
    >>> import statistics
+
+>>> statistics.covariance(words, views)
+5292289.977777777
+
    +
    +

    In contrast to correlation, covariance is an absolute measure. It should be interpreted in the context of the variability within the variables themselves. In fact, you can normalize the covariance by the standard deviation of each variable to recover Pearson’s correlation coefficient:

    +
    +
    + 
    >>> import statistics
+
+>>> cov = statistics.covariance(words, views)
+>>> σ_words, σ_views = statistics.stdev(words), statistics.stdev(views)
+>>> cov / (σ_words * σ_views)
+0.454180067865917
+
    +
    +

    Note that this matches your earlier correlation coefficient exactly.

    A third way of looking at the linear correspondence between the two variables is through simple linear regression. You do the linear regression by calculating two numbers, slope and intercept, so that the (squared) error is minimized in the approximation number of views = slope × number of words + intercept.

    In Python 3.10, you can use linear_regression():

    +
    +
    + 
    >>> import statistics
+
+>>> statistics.linear_regression(words, views)
+LinearRegression(slope=0.2424443064354672, intercept=1103.6954940247645)
+
    +
    +

    Based on this regression, a post with 10,074 words could expect about 0.2424 × 10074 + 1104 = 3546 views. However, as you saw earlier, the correlation between the number of words and the number of views is quite weak. Therefore, you shouldn’t expect this prediction to be very accurate.

    The LinearRegression object is a named tuple. This means that you can unpack the slope and intercept directly:

    +
    +
    + 
    >>> import statistics
+
+>>> slope, intercept = statistics.linear_regression(words, views)
+>>> slope * 10074 + intercept
+3546.0794370556605
+
    +
    +

    Here, you use slope and intercept to predict the number of views on a blog post with 10,074 words.

    You still want to use some of the more advanced packages like pandas and statsmodels if you do a lot of statistical analysis. With the new additions to statistics in Python 3.10, however, you have the chance to do basic analysis more easily without bringing in third-party dependencies.

    @@ -328,10 +1120,60 @@

    When you open a text file, the default encoding used to interpret the characters is system dependent. In particular, locale.getpreferredencoding() is used. On Mac and Linux, this usually returns "UTF-8", while the result on Windows is more varied.

    You should therefore always specify an encoding when you attempt to open a text file:

    +
    +
    + 
    with open("some_file.txt", mode="r", encoding="utf-8") as file:
+    ...  # Do something with file
+
    +
    +

    If you don’t explicitly specify an encoding, the preferred locale encoding is used, and you could experience that a file that can be read on one computer fails to open on another.

    Python 3.7 introduced UTF-8 mode, which allows you to force your programs to use UTF-8 encoding independent of the locale encoding. You can enable UTF-8 mode by giving the -X utf8 command-line option to the python executable or by setting the PYTHONUTF8 environment variable.

    In Python 3.10, you can activate a warning that will tell you when a text file is opened without a specified encoding. Consider the following script, which doesn’t specify an encoding:

    +
    +
    + 
    # mirror.py
+
+import pathlib
+import sys
+
+def mirror_file(filename):
+    for line in pathlib.Path(filename).open(mode="r"):
+        print(f"{line.rstrip()[::-1]:>72}")
+
+if __name__ == "__main__":
+    for filename in sys.argv[1:]:
+        mirror_file(filename)
+
    +
    +

    The program will echo one or more text files back to the console, but with each line reversed. Run the program on itself with the encoding warning enabled:

    +
    +
    + 
    $ python -X warn_default_encoding mirror.py mirror.py
+/home/rp/mirror.py:7: EncodingWarning: 'encoding' argument not specified
+  for line in pathlib.Path(filename).open(mode="r"):
+                                                             yp.rorrim #
+
+                                                          bilhtap tropmi
+                                                              sys tropmi
+
+                                              :)emanelif(elif_rorrim fed
+                  :)"r"=edom(nepo.)emanelif(htaP.bilhtap ni enil rof
+                             )"}27>:]1-::[)(pirtsr.enil{"f(tnirp
+
+                                              :"__niam__" == __eman__ fi
+                                       :]:1[vgra.sys ni emanelif rof
+                                           )emanelif(elif_rorrim
+
    +
    +

    Note the EncodingWarning printed to the console. The command-line option -X warn_default_encoding activates it. The warning will disappear if you specify an encoding—for example, encoding="utf-8"—when you open the file.

    There are times when you want to use the user-defined local encoding. You can still do so by explicitly using encoding="locale". However, it’s recommended to use UTF-8 whenever possible. You can check out PEP 597 for more information.

    @@ -343,10 +1185,54 @@

    In Python 3.10, two new asynchronous built-in functions are added: aiter() and anext(). In practice, these functions call the .__aiter__() and .__anext__() special methods—analogous to the regular iter() and next()—so no new functionality is added. These are convenience functions that make your code more readable.

    In other words, in the newest version of Python, the following statements—where things is an asynchronous iterable—are equivalent:

    +
    +
    + 
    >>> it = things.__aiter__()
+>>> it = aiter(things)
+
    +
    +

    In either case, it ends up as an asynchronous iterator. Expand the following box to see a complete example using aiter() and anext():

    The following program counts the number of lines in several files. In practice, you use Python’s ability to iterate over files to count the number of lines. The script uses asynchronous iteration in order to handle several files concurrently.

    Note that you need to install the third-party aiofiles package with pip before running this code:

    +
    +
    + 
    # line_count.py
+
+import asyncio
+import sys
+import aiofiles
+
+async def count_lines(filename):
+    """Count the number of lines in the given file"""
+    num_lines = 0
+
+    async with aiofiles.open(filename, mode="r") as file:
+        lines = aiter(file)
+        while True:
+            try:
+                await anext(lines)
+                num_lines += 1
+            except StopAsyncIteration:
+                break
+
+    print(f"{filename}: {num_lines}")
+
+async def count_all_files(filenames):
+    """Asynchronously count lines in all files"""
+    tasks = [asyncio.create_task(count_lines(f)) for f in filenames]
+    await asyncio.gather(*tasks)
+
+if __name__ == "__main__":
+    asyncio.run(count_all_files(filenames=sys.argv[1:]))
+
    +
    +

    asyncio is used to create and run one asynchronous task per filename. count_lines() opens one file asynchronously and iterates through it using aiter() and anext() in order to count the number of lines.

    @@ -359,7 +1245,29 @@

    Context managers are great for managing resources in your programs. Until recently, though, their syntax has included an uncommon wart. You haven’t been allowed to use parentheses to break long with statements like this:

    +
    +
    + 
    with (
+    read_path.open(mode="r", encoding="utf-8") as read_file,
+    write_path.open(mode="w", encoding="utf-8") as write_file,
+):
+    ...
+
    +
    +

    In earlier versions of Python, this causes an invalid syntax error message. Instead, you need to use a backslash (\) if you want to control where you break your lines:

    +
    +
    + 
    with read_path.open(mode="r", encoding="utf-8") as read_file, \
+     write_path.open(mode="w", encoding="utf-8") as write_file:
+    ...
+
    +
    +

    While explicit line continuation with backslashes is possible in Python, PEP 8 discourages it. The Black formatting tool avoids backslashes completely.

    In Python 3.10, you’re now allowed to add parentheses around with statements to your heart’s content. Especially if you’re employing several context managers at once, like in the example above, this can help improve the readability of your code. Python’s documentation shows a few other possibilities with this new syntax.

    One small fun fact: parenthesized with statements actually work in version 3.9 of CPython. Their implementation came almost for free with the introduction of the PEG parser in Python 3.9. The reason that this is called a Python 3.10 feature is that using the PEG parser is voluntary in Python 3.9, while Python 3.9, with the old LL(1) parser, doesn’t support parenthesized with statements.

    @@ -408,14 +1316,72 @@

    The sys module contains a lot of information about your system, the current Python runtime, and the script currently being executed. You can, for example, inquire about the paths where Python looks for modules with sys.path and see all modules that have been imported in the current session with sys.modules.

    In Python 3.10, sys has two new attributes. First, you can now get a list of the names of all modules in the standard library:

    +
    +
    + 
    >>> import sys
+
+>>> len(sys.stdlib_module_names)
+302
+
+>>> sorted(sys.stdlib_module_names)[-5:]
+['zipapp', 'zipfile', 'zipimport', 'zlib', 'zoneinfo']
+
    +
    +

    Here, you can see that there are around 300 modules in the standard library, several of which start with the letter z. Note that only top-level modules and packages are listed. Subpackages like importlib.metadata don’t get a separate entry.

    You will probably not be using sys.stdlib_module_names all that often. Still, the list ties in nicely with similar introspection features like keyword.kwlist and sys.builtin_module_names.

    One possible use case for the new attribute is to identify which of the currently imported modules are third-party dependencies:

    +
    +
    + 
    >>> import pandas as pd
+>>> import sys
+
+>>> {m for m in sys.modules if "." not in m} - sys.stdlib_module_names
+{'__main__', 'numpy', '_cython_0_29_24', 'dateutil', 'pytz',
+ 'six', 'pandas', 'cython_runtime'}
+
    +
    +

    You find the imported top-level modules by looking at names in sys.modules that don’t have a dot in their name. By comparing them to the standard library module names, you find that numpy, dateutil, and pandas are some of the imported third-party modules in this example.

    The other new attribute is sys.orig_argv. This is related to sys.argv, which holds the command-line arguments given to your program when it was started. In contrast, sys.orig_argv lists the command-line arguments passed to the python executable itself. Consider the following example:

    +
    +
    + 
    # argvs.py
+
+import sys
+
+print(f"argv: {sys.argv}")
+print(f"orig_argv: {sys.orig_argv}")
+
    +
    +

    This script echoes back the orig_argv and argv lists. Run it to see how the information is captured:

    +
    +
    + 
    $ python -X utf8 -O argvs.py 3.10 --upgrade
+argv: ['argvs.py', '3.10', '--upgrade']
+orig_argv: ['python', '-X', 'utf8', '-O', 'argvs.py', '3.10', '--upgrade']
+
    +
    +

    Essentially, all arguments—including the name of the Python executable—end up in orig_argv. This is in contrast to argv, which only contains the arguments that aren’t handled by python itself.

    Again, this is not a feature that you’ll use a lot. If your program needs to concern itself with how it’s being run, you’re usually better off relying on information that’s already exposed instead of trying to parse this list. For example, you can choose to use the strict zip() mode only when your script is not running with the optimized flag, -O, like this:

    +
    +
    + 
    list(zip(names, set_numbers, num_pieces, strict=__debug__))
+
    +
    +

    The __debug__ flag is set when the interpreter starts. It’ll be False if you’re running python with -O or -OO specified, and True otherwise. Using __debug__ is usually preferable to "-O" not in sys.orig_argv or some similar construct.

    One of the motivating use cases for sys.orig_argv is that you can use it to spawn a new Python process with the same or modified command-line arguments as your current process.

    @@ -426,9 +1392,32 @@ Annotations were introduced in Python 3 to give you a way to attach metadata to variables, function parameters, and return values. They are most commonly used to add type hints to your code.

    One challenge with annotations is that they must be valid Python code. For one thing, this makes it hard to type hint recursive classes. PEP 563 introduced postponed evaluation of annotations, making it possible to annotate with names that haven’t yet been defined. Since Python 3.7, you can activate postponed evaluation of annotations with a __future__ import:

    +
    +
    + 
    from __future__ import annotations
+
    +
    +

    The intention was that postponed evaluation would become the default at some point in the future. After the 2020 Python Language Summit, it was decided to make this happen in Python 3.10.

    However, after more testing, it became clear that postponed evaluation didn’t work well for projects that use annotations at runtime. Key people in the FastAPI and the Pydantic projects voiced their concerns. At the last minute, it was decided to reschedule these changes for Python 3.11.

    To ease the transition into future behavior, a few changes have been made in Python 3.10 as well. Most importantly, a new inspect.get_annotations() function has been added. You should call this to access annotations at runtime:

    +
    +
    + 
    >>> import inspect
+
+>>> def mean(numbers: list[int | float]) -> float:
+...     return sum(numbers) / len(numbers)
+...
+
+>>> inspect.get_annotations(mean)
+{'numbers': list[int | float], 'return': <class 'float'>}
+
    +
    +

    Check out Annotations Best Practices for details.

    @@ -437,12 +1426,51 @@

    Python 3.10 is the first version of Python with a two-digit minor version number. While this is mostly an interesting fun fact and an indication that Python 3 has been around for quite some time, it does also have some practical consequences.

    When your code needs to do something specific based on the version of Python at runtime, you’ve gotten away with doing a lexicographical comparison of version strings until now. While it’s never been good practice, it’s been possible to do the following:

    +
    +
    + 
    # bad_version_check.py
+
+import sys
+
+# Don't do the following
+if sys.version < "3.6":
+    raise SystemExit("Only Python 3.6 and above is supported")
+
    +
    +

    In Python 3.10, this code will raise SystemExit and stop your program. This happens because, as strings, "3.10" is less than "3.6".

    The correct way to compare version numbers is to use tuples of numbers:

    +
    +
    + 
    # good_version_check.py
+
+import sys
+
+if sys.version_info < (3, 6):
+    raise SystemExit("Only Python 3.6 and above is supported")
+
    +
    +

    sys.version_info is a tuple object you can use for comparisons.

    If you’re doing these kinds of comparisons in your code, you should check your code with flake8-2020 to make sure you’re handling versions correctly:

    +
    +
    + 
    $ python -m pip install flake8-2020
+
+$ flake8 bad_version_check.py good_version_check.py
+bad_version_check.py:3:4: YTT103 `sys.version` compared to string
+                          (python3.10), use `sys.version_info`
+
    +
    +

    With the flake8-2020 extension activated, you’ll get a recommendation about replacing sys.version with sys.version_info.

    From e54c691c09d2445d0cab12ef822030811decbf59 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Thu, 25 Jan 2024 11:18:18 +0800 Subject: [PATCH 37/59] create a job for finding thumbnail --- packages/api/package.json | 1 + packages/api/src/jobs/find_thumbnail.ts | 174 ++++++++++++++++++++++++ packages/api/src/queue-processor.ts | 8 +- packages/api/src/services/save_page.ts | 4 +- packages/api/src/utils/createTask.ts | 51 ++----- 5 files changed, 193 insertions(+), 45 deletions(-) create mode 100644 packages/api/src/jobs/find_thumbnail.ts diff --git a/packages/api/package.json b/packages/api/package.json index 7d5420686..0f1b60eba 100644 --- a/packages/api/package.json +++ b/packages/api/package.json @@ -71,6 +71,7 @@ "graphql-shield": "^7.5.0", "highlightjs": "^9.16.2", "html-entities": "^2.3.2", + "image-size": "^1.0.2", "intercom-client": "^3.1.4", "ioredis": "^5.3.2", "jsonwebtoken": "^8.5.1", diff --git a/packages/api/src/jobs/find_thumbnail.ts b/packages/api/src/jobs/find_thumbnail.ts new file mode 100644 index 000000000..06ed28a3e --- /dev/null +++ b/packages/api/src/jobs/find_thumbnail.ts @@ -0,0 +1,174 @@ +import axios, { AxiosResponse } from 'axios' +import sizeOf from 'image-size' +import { parseHTML } from 'linkedom' +import { + findLibraryItemById, + updateLibraryItem, +} from '../services/library_item' +import { logger } from '../utils/logger' + +interface Data { + libraryItemId: string + userId: string +} + +interface ImageSize { + src: string + width: number + height: number +} + +export const THUMBNAIL_JOB = 'find-thumbnail' + +const fetchImage = async (url: string): Promise => { + console.log('fetching image', url) + try { + // get image file by url + return await axios.get(url, { + responseType: 'arraybuffer', + timeout: 10000, // 10s + maxContentLength: 20000000, // 20mb + }) + } catch (e) { + logger.error('fetch image error', e) + return null + } +} + +const getImageSize = async (src: string): Promise => { + try { + const response = await fetchImage(src) + if (!response) { + return null + } + + // eslint-disable-next-line @typescript-eslint/no-unsafe-argument + const buffer = Buffer.from(response.data, 'binary') + + // get image size + const { width, height } = sizeOf(buffer) + + if (!width || !height) { + return null + } + + return { + src, + width, + height, + } + } catch (e) { + logger.error(e) + return null + } +} + +export const fetchAllImageSizes = async (content: string) => { + const dom = parseHTML(content).document + + // fetch all images by src and get their sizes + const images = dom.querySelectorAll('img[src]') + if (!images || images.length === 0) { + console.log('no images') + return [] + } + + return Promise.all( + Array.from(images).map((image) => { + const src = image.getAttribute('src') + if (!src) { + return null + } + + return getImageSize(src) + }) + ) +} + +// credit: https://github.com/reddit-archive/reddit/blob/753b17407e9a9dca09558526805922de24133d53/r2/r2/lib/media.py#L706 +export const _findThumbnail = (imagesSizes: (ImageSize | null)[]) => { + // find the largest and squarest image as the thumbnail + let thumbnail = '' + let largestArea = 0 + for (const imageSize of Array.from(imagesSizes)) { + if (!imageSize) { + continue + } + + let area = imageSize.width * imageSize.height + + // ignore small images + if (area < 5000) { + logger.info('ignore small', { src: imageSize.src }) + continue + } + + // penalize excessively long/wide images + const ratio = + Math.max(imageSize.width, imageSize.height) / + Math.min(imageSize.width, imageSize.height) + if (ratio > 1.5) { + logger.info('penalizing long/wide', { src: imageSize.src }) + area /= ratio * 2 + } + + // penalize images with "sprite" in their name + if (imageSize.src.toLowerCase().includes('sprite')) { + logger.info('penalizing sprite', { src: imageSize.src }) + area /= 10 + } + + if (area > largestArea) { + largestArea = area + thumbnail = imageSize.src + } + } + + return thumbnail +} + +export const findThumbnail = async (data: Data) => { + const { libraryItemId, userId } = data + + const item = await findLibraryItemById(libraryItemId, userId) + if (!item) { + logger.info('page not found') + return false + } + + const thumbnail = item.thumbnail + if (thumbnail) { + logger.info('thumbnail already set') + // pre-cache thumbnail first if exists + const image = await fetchImage(thumbnail) + if (!image) { + logger.info('thumbnail image not found') + item.thumbnail = undefined + } + } + + logger.info('pre-caching all images...') + // pre-cache all images in the content and get their sizes + const imageSizes = await fetchAllImageSizes(item.readableContent) + // find thumbnail from all images if thumbnail not set + if (!item.thumbnail && imageSizes.length > 0) { + logger.info('finding thumbnail...') + const thumbnail = _findThumbnail(imageSizes) + if (!thumbnail) { + logger.info('no thumbnail found from content') + return false + } + + // update page with thumbnail + await updateLibraryItem( + libraryItemId, + { + thumbnail, + }, + userId + ) + logger.info(`thumbnail updated: ${thumbnail}`) + } + + return true +} diff --git a/packages/api/src/queue-processor.ts b/packages/api/src/queue-processor.ts index 233bf36e7..2810ac6d6 100644 --- a/packages/api/src/queue-processor.ts +++ b/packages/api/src/queue-processor.ts @@ -2,17 +2,18 @@ /* eslint-disable @typescript-eslint/restrict-template-expressions */ /* eslint-disable @typescript-eslint/require-await */ /* eslint-disable @typescript-eslint/no-misused-promises */ -import { Job, QueueEvents, Worker, Queue, JobType } from 'bullmq' +import { Job, Queue, QueueEvents, Worker, JobType } from 'bullmq' import express, { Express } from 'express' import { SnakeNamingStrategy } from 'typeorm-naming-strategies' import { appDataSource } from './data_source' import { env } from './env' +import { findThumbnail, THUMBNAIL_JOB } from './jobs/find_thumbnail' import { refreshAllFeeds } from './jobs/rss/refreshAllFeeds' import { refreshFeed } from './jobs/rss/refreshFeed' import { savePageJob } from './jobs/save_page' +import { updatePDFContentJob } from './jobs/update_pdf_content' import { redisDataSource } from './redis_data_source' import { CustomTypeOrmLogger } from './utils/logger' -import { updatePDFContentJob } from './jobs/update_pdf_content' export const QUEUE_NAME = 'omnivore-backend-queue' @@ -119,8 +120,9 @@ const main = async () => { case 'update-pdf-content': { return updatePDFContentJob(job.data) } + case THUMBNAIL_JOB: + return findThumbnail(job.data) } - return true }, { connection: workerRedisClient, diff --git a/packages/api/src/services/save_page.ts b/packages/api/src/services/save_page.ts index 7931fc48e..7cd7b1768 100644 --- a/packages/api/src/services/save_page.ts +++ b/packages/api/src/services/save_page.ts @@ -170,8 +170,8 @@ export const savePage = async ( if (!isImported && !parseResult.parsedContent?.previewImage) { try { // create a task to update thumbnail and pre-cache all images - const taskId = await enqueueThumbnailTask(user.id, slug) - logger.info('Created thumbnail task', { taskId }) + const job = await enqueueThumbnailTask(user.id, clientRequestId) + logger.info('Created thumbnail task', { job }) } catch (e) { logger.error('Failed to create thumbnail task', e) } diff --git a/packages/api/src/utils/createTask.ts b/packages/api/src/utils/createTask.ts index 09c84b62d..88eb3a2d5 100644 --- a/packages/api/src/utils/createTask.ts +++ b/packages/api/src/utils/createTask.ts @@ -22,6 +22,8 @@ import { CreateTaskError } from './errors' import { stringToHash } from './helpers' import { logger } from './logger' import View = google.cloud.tasks.v2.Task.View +import { getBackendQueue } from '../queue-processor' +import { THUMBNAIL_JOB } from '../jobs/find_thumbnail' // Instantiates a client. const client = new CloudTasksClient() @@ -579,50 +581,19 @@ export const enqueueExportToIntegration = async ( export const enqueueThumbnailTask = async ( userId: string, - slug: string -): Promise => { - const { GOOGLE_CLOUD_PROJECT } = process.env + libraryItemId: string +) => { + const queue = await getBackendQueue() + if (!queue) { + return undefined + } const payload = { userId, - slug, + libraryItemId, } - - const headers = { - Cookie: `auth=${generateVerificationToken({ id: userId })}`, - } - - // If there is no Google Cloud Project Id exposed, it means that we are in local environment - if (env.dev.isLocal || !GOOGLE_CLOUD_PROJECT) { - if (env.queue.thumbnailTaskHandlerUrl) { - // Calling the handler function directly. - setTimeout(() => { - axios - .post(env.queue.thumbnailTaskHandlerUrl, payload, { - headers, - }) - .catch((error) => { - logError(error) - }) - }, 0) - } - return '' - } - - const createdTasks = await createHttpTaskWithToken({ - payload, - taskHandlerUrl: env.queue.thumbnailTaskHandlerUrl, - requestHeaders: headers, - queue: 'omnivore-thumbnail-queue', + return queue.add(THUMBNAIL_JOB, payload, { + priority: 100, }) - - if (!createdTasks || !createdTasks[0].name) { - logger.error(`Unable to get the name of the task`, { - payload, - createdTasks, - }) - throw new CreateTaskError(`Unable to get the name of the task`) - } - return createdTasks[0].name } export interface RssSubscriptionGroup { From 4c765378e390d971e59f50aeed664b019ddabb57 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Thu, 25 Jan 2024 11:32:30 +0800 Subject: [PATCH 38/59] do not retry if find_thumbnail job fails --- packages/api/src/jobs/find_thumbnail.ts | 4 +++- packages/api/src/services/save_email.ts | 16 +++++++++------- packages/api/src/services/save_page.ts | 8 ++++---- packages/api/src/utils/createTask.ts | 7 +++++-- 4 files changed, 21 insertions(+), 14 deletions(-) diff --git a/packages/api/src/jobs/find_thumbnail.ts b/packages/api/src/jobs/find_thumbnail.ts index 06ed28a3e..2e1231962 100644 --- a/packages/api/src/jobs/find_thumbnail.ts +++ b/packages/api/src/jobs/find_thumbnail.ts @@ -5,6 +5,7 @@ import { findLibraryItemById, updateLibraryItem, } from '../services/library_item' +import { createImageProxyUrl, createThumbnailUrl } from '../utils/imageproxy' import { logger } from '../utils/logger' interface Data { @@ -139,8 +140,9 @@ export const findThumbnail = async (data: Data) => { const thumbnail = item.thumbnail if (thumbnail) { logger.info('thumbnail already set') + const proxyUrl = createThumbnailUrl(thumbnail) // pre-cache thumbnail first if exists - const image = await fetchImage(thumbnail) + const image = await fetchImage(proxyUrl) if (!image) { logger.info('thumbnail image not found') item.thumbnail = undefined diff --git a/packages/api/src/services/save_email.ts b/packages/api/src/services/save_email.ts index 0c0f4f456..c080345ad 100644 --- a/packages/api/src/services/save_email.ts +++ b/packages/api/src/services/save_email.ts @@ -1,5 +1,5 @@ import { LibraryItem, LibraryItemState } from '../entity/library_item' -import { enqueueThumbnailTask } from '../utils/createTask' +import { enqueueThumbnailJob } from '../utils/createTask' import { cleanUrl, generateSlug, @@ -132,12 +132,14 @@ export const saveEmail = async ( await updateReceivedEmail(input.receivedEmailId, 'article', input.userId) - // create a task to update thumbnail and pre-cache all images - try { - const taskId = await enqueueThumbnailTask(input.userId, slug) - logger.info('Created thumbnail task', { taskId }) - } catch (e) { - logger.error('Failed to create thumbnail task', e) + if (!newLibraryItem.thumbnail) { + // create a task to update thumbnail and pre-cache all images + try { + const job = await enqueueThumbnailJob(input.userId, newLibraryItem.id) + logger.info('Created thumbnail job', { taskId: job }) + } catch (e) { + logger.error('Failed to create thumbnail job', e) + } } return newLibraryItem diff --git a/packages/api/src/services/save_page.ts b/packages/api/src/services/save_page.ts index 7cd7b1768..f8fd4340f 100644 --- a/packages/api/src/services/save_page.ts +++ b/packages/api/src/services/save_page.ts @@ -13,7 +13,7 @@ import { SaveResult, } from '../generated/graphql' import { authTrx } from '../repository' -import { enqueueThumbnailTask } from '../utils/createTask' +import { enqueueThumbnailJob } from '../utils/createTask' import { cleanUrl, generateSlug, @@ -170,10 +170,10 @@ export const savePage = async ( if (!isImported && !parseResult.parsedContent?.previewImage) { try { // create a task to update thumbnail and pre-cache all images - const job = await enqueueThumbnailTask(user.id, clientRequestId) - logger.info('Created thumbnail task', { job }) + const job = await enqueueThumbnailJob(user.id, clientRequestId) + logger.info('Created thumbnail job', { job }) } catch (e) { - logger.error('Failed to create thumbnail task', e) + logger.error('Failed to enqueue thumbnail job', e) } } diff --git a/packages/api/src/utils/createTask.ts b/packages/api/src/utils/createTask.ts index 88eb3a2d5..ad97d358b 100644 --- a/packages/api/src/utils/createTask.ts +++ b/packages/api/src/utils/createTask.ts @@ -14,10 +14,12 @@ import { ArticleSavingRequestStatus, CreateLabelInput, } from '../generated/graphql' +import { THUMBNAIL_JOB } from '../jobs/find_thumbnail' import { queueRSSRefreshFeedJob } from '../jobs/rss/refreshAllFeeds' +import { getBackendQueue } from '../queue-processor' import { redisDataSource } from '../redis_data_source' import { signFeatureToken } from '../services/features' -import { generateVerificationToken, OmnivoreAuthorizationHeader } from './auth' +import { OmnivoreAuthorizationHeader } from './auth' import { CreateTaskError } from './errors' import { stringToHash } from './helpers' import { logger } from './logger' @@ -579,7 +581,7 @@ export const enqueueExportToIntegration = async ( return createdTasks[0].name } -export const enqueueThumbnailTask = async ( +export const enqueueThumbnailJob = async ( userId: string, libraryItemId: string ) => { @@ -593,6 +595,7 @@ export const enqueueThumbnailTask = async ( } return queue.add(THUMBNAIL_JOB, payload, { priority: 100, + attempts: 1, }) } From be0a2e8c52e7123c96153fd7058e5a8481304c94 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Thu, 25 Jan 2024 17:20:26 +0800 Subject: [PATCH 39/59] use different redis server for mq and cache --- packages/api/src/queue-processor.ts | 4 +-- packages/api/src/redis_data_source.ts | 35 +++++++++++++++++---------- packages/api/src/server.ts | 2 +- packages/api/src/util.ts | 21 +++++++++++++--- 4 files changed, 42 insertions(+), 20 deletions(-) diff --git a/packages/api/src/queue-processor.ts b/packages/api/src/queue-processor.ts index 2810ac6d6..d99d79f10 100644 --- a/packages/api/src/queue-processor.ts +++ b/packages/api/src/queue-processor.ts @@ -40,8 +40,8 @@ const main = async () => { const port = process.env.PORT || 3002 redisDataSource.setOptions({ - REDIS_URL: env.redis.url, - REDIS_CERT: env.redis.cert, + cache: env.redis.cache, + mq: env.redis.mq, }) appDataSource.setOptions({ diff --git a/packages/api/src/redis_data_source.ts b/packages/api/src/redis_data_source.ts index ef7cf55e3..26da7d9fe 100644 --- a/packages/api/src/redis_data_source.ts +++ b/packages/api/src/redis_data_source.ts @@ -1,9 +1,14 @@ import Redis, { RedisOptions } from 'ioredis' import { env } from './env' +import { logger } from './utils/logger' +type RedisClientType = 'cache' | 'mq' +type RedisDataSourceOption = { + url?: string + cert?: string +} export type RedisDataSourceOptions = { - REDIS_URL?: string - REDIS_CERT?: string + [key in RedisClientType]: RedisDataSourceOption } export class RedisDataSource { @@ -22,8 +27,9 @@ export class RedisDataSource { async initialize(): Promise { if (this.isInitialized) throw 'Error already initialized' - this.redisClient = createIORedisClient('app', this.options) - this.workerRedisClient = createIORedisClient('worker', this.options) + this.redisClient = createIORedisClient('cache', this.options) + this.workerRedisClient = + createIORedisClient('mq', this.options) || this.redisClient // if mq is not defined, use cache this.isInitialized = true return Promise.resolve(this) @@ -45,17 +51,21 @@ export class RedisDataSource { } const createIORedisClient = ( - name: string, + name: RedisClientType, options: RedisDataSourceOptions ): Redis | undefined => { - const redisURL = options.REDIS_URL + const option = options[name] + const redisURL = option.url if (!redisURL) { - throw 'Error: no redisURL supplied' + logger.info(`no redisURL supplied: ${name}`) + return undefined } + + const redisCert = option.cert const tls = - redisURL.startsWith('rediss://') && options.REDIS_CERT + redisURL.startsWith('rediss://') && redisCert ? { - ca: options.REDIS_CERT, + ca: redisCert, rejectUnauthorized: false, } : undefined @@ -92,7 +102,6 @@ const createIORedisClient = ( return new Redis(redisURL, redisOptions) } -export const redisDataSource = new RedisDataSource({ - REDIS_URL: env.redis.url, - REDIS_CERT: env.redis.cert, -}) +export const redisDataSource = new RedisDataSource( + env.redis as RedisDataSourceOptions +) diff --git a/packages/api/src/server.ts b/packages/api/src/server.ts index e52b0312b..58a7e4d2b 100755 --- a/packages/api/src/server.ts +++ b/packages/api/src/server.ts @@ -159,7 +159,7 @@ const main = async (): Promise => { await appDataSource.initialize() // redis is optional for the API server - if (env.redis.url) { + if (env.redis.cache.url) { await redisDataSource.initialize() } diff --git a/packages/api/src/util.ts b/packages/api/src/util.ts index 691b58063..3f1570c26 100755 --- a/packages/api/src/util.ts +++ b/packages/api/src/util.ts @@ -4,6 +4,11 @@ import * as dotenv from 'dotenv' import os from 'os' +interface redisConfig { + url?: string + cert?: string +} + export interface BackendEnv { pg: { host: string @@ -105,8 +110,8 @@ export interface BackendEnv { } } redis: { - url?: string - cert?: string + mq: redisConfig + cache: redisConfig } } @@ -156,6 +161,8 @@ const nullableEnvVars = [ 'SUBSCRIPTION_FEED_MAX', 'REDIS_URL', 'REDIS_CERT', + 'MQ_REDIS_URL', + 'MQ_REDIS_CERT', 'IMPORTER_METRICS_COLLECTOR_URL', 'INTERNAL_API_URL', ] // Allow some vars to be null/empty @@ -295,8 +302,14 @@ export function getEnv(): BackendEnv { }, } const redis = { - url: parse('REDIS_URL'), - cert: parse('REDIS_CERT')?.replace(/\\n/g, '\n'), // replace \n with new line + mq: { + url: parse('MQ_REDIS_URL'), + cert: parse('MQ_REDIS_CERT')?.replace(/\\n/g, '\n'), // replace \n with new line + }, + cache: { + url: parse('REDIS_URL'), + cert: parse('REDIS_CERT')?.replace(/\\n/g, '\n'), // replace \n with new line + }, } return { From 7e892358065b8db2080ab5c6b742d8ea8fbabe10 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Thu, 25 Jan 2024 17:33:17 +0800 Subject: [PATCH 40/59] update content-fetch --- packages/api/src/utils/helpers.ts | 12 +-- .../content-fetch/src/redis_data_source.ts | 73 ++++++++++--------- 2 files changed, 42 insertions(+), 43 deletions(-) diff --git a/packages/api/src/utils/helpers.ts b/packages/api/src/utils/helpers.ts index 5b7ca9b83..df4fcb014 100644 --- a/packages/api/src/utils/helpers.ts +++ b/packages/api/src/utils/helpers.ts @@ -22,11 +22,10 @@ import { PageType, Profile, Recommendation, - ResolverFn, SearchItem, } from '../generated/graphql' import { createPubSubClient } from '../pubsub' -import { Claims, WithDataSourcesContext } from '../resolvers/types' +import { redisDataSource } from '../redis_data_source' import { validateUrl } from '../services/create_page_save_request' import { updateLibraryItem } from '../services/library_item' import { Merge } from '../util' @@ -391,17 +390,10 @@ export const setRecentlySavedItemInRedis = async ( url: string ) => { // save the url in redis for 26 hours so rss-feeder won't try to re-save it - if (!redisClient) { - console.info( - 'not setting recently saved item because redis is not configured' - ) - return - } - // save the url in redis for 8 hours so rss-feeder won't try to re-save it const redisKey = `recent-saved-item:${userId}:${url}` const ttlInSeconds = 60 * 60 * 26 try { - return redisClient.set(redisKey, 1, 'EX', ttlInSeconds, 'NX') + return await redisClient.set(redisKey, 1, 'EX', ttlInSeconds, 'NX') } catch (error) { logger.error('error setting recently saved item in redis', { redisKey, diff --git a/packages/content-fetch/src/redis_data_source.ts b/packages/content-fetch/src/redis_data_source.ts index 1a95f9d6e..917f9a867 100644 --- a/packages/content-fetch/src/redis_data_source.ts +++ b/packages/content-fetch/src/redis_data_source.ts @@ -1,8 +1,12 @@ import Redis, { RedisOptions } from 'ioredis' +type RedisClientType = 'cache' | 'mq' +type RedisDataSourceOption = { + url?: string + cert?: string +} export type RedisDataSourceOptions = { - REDIS_URL?: string - REDIS_CERT?: string + [key in RedisClientType]: RedisDataSourceOption } export class RedisDataSource { @@ -14,8 +18,12 @@ export class RedisDataSource { constructor(options: RedisDataSourceOptions) { this.options = options - this.cacheClient = createRedisClient('cache', this.options) - this.queueRedisClient = createRedisClient('queue', this.options) + const cacheClient = createIORedisClient('cache', this.options) + if (!cacheClient) throw 'Error initializing cache redis client' + + this.cacheClient = cacheClient + this.queueRedisClient = + createIORedisClient('mq', this.options) || this.cacheClient // if mq is not defined, use cache } setOptions(options: RedisDataSourceOptions): void { @@ -32,46 +40,45 @@ export class RedisDataSource { } } -const createRedisClient = (name: string, options: RedisDataSourceOptions) => { - const redisURL = options.REDIS_URL - const cert = options.REDIS_CERT?.replace(/\\n/g, '\n') // replace \n with new line +const createIORedisClient = ( + name: RedisClientType, + options: RedisDataSourceOptions +): Redis | undefined => { + const option = options[name] + const redisURL = option.url if (!redisURL) { - throw 'Error: no redisURL supplied' + console.log(`no redisURL supplied: ${name}`) + return undefined } - const redisOptions: RedisOptions = { - name, - connectTimeout: 10000, // 10 seconds - tls: cert + const redisCert = option.cert + const tls = + redisURL.startsWith('rediss://') && redisCert ? { - cert, - rejectUnauthorized: false, // for self-signed certs + ca: redisCert, + rejectUnauthorized: false, } - : undefined, + : undefined + + const redisOptions: RedisOptions = { + tls, + name, + connectTimeout: 10000, maxRetriesPerRequest: null, offlineQueue: false, } - - const redis = new Redis(redisURL, redisOptions) - - redis.on('connect', () => { - console.log('Redis connected', name) - }) - - redis.on('error', (err) => { - console.error('Redis error', err, name) - }) - - redis.on('close', () => { - console.log('Redis closed', name) - }) - - return redis + return new Redis(redisURL, redisOptions) } export const redisDataSource = new RedisDataSource({ - REDIS_URL: process.env.REDIS_URL, - REDIS_CERT: process.env.REDIS_CERT, + cache: { + url: process.env.REDIS_CACHE_URL, + cert: process.env.REDIS_CACHE_CERT, + }, + mq: { + url: process.env.MQ_REDIS_URL, + cert: process.env.MQ_REDIS_CERT, + }, }) // eslint-disable-next-line @typescript-eslint/no-misused-promises From e4e07c8acf9f7088764f7d7680c635dc765a5bae Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Thu, 25 Jan 2024 17:34:54 +0800 Subject: [PATCH 41/59] cache fetched content for 24 hours --- packages/content-fetch/src/request_handler.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/content-fetch/src/request_handler.ts b/packages/content-fetch/src/request_handler.ts index 56356ec1e..ed81b3ffb 100644 --- a/packages/content-fetch/src/request_handler.ts +++ b/packages/content-fetch/src/request_handler.ts @@ -53,8 +53,8 @@ interface FetchResult { } export const cacheFetchResult = async (fetchResult: FetchResult) => { - // cache the fetch result for 4 hours - const ttl = 4 * 60 * 60 + // cache the fetch result for 24 hours + const ttl = 24 * 60 * 60 const key = `fetch-result:${fetchResult.finalUrl}` const value = JSON.stringify(fetchResult) return redisDataSource.cacheClient.set(key, value, 'EX', ttl, 'NX') From 52419c39da4579f320df36deb72067d72ba66efd Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Thu, 25 Jan 2024 17:40:25 +0800 Subject: [PATCH 42/59] resolve conflicts --- packages/api/src/utils/createTask.ts | 2 -- packages/content-fetch/src/redis_data_source.ts | 8 ++------ 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/packages/api/src/utils/createTask.ts b/packages/api/src/utils/createTask.ts index ad97d358b..94233a454 100644 --- a/packages/api/src/utils/createTask.ts +++ b/packages/api/src/utils/createTask.ts @@ -24,8 +24,6 @@ import { CreateTaskError } from './errors' import { stringToHash } from './helpers' import { logger } from './logger' import View = google.cloud.tasks.v2.Task.View -import { getBackendQueue } from '../queue-processor' -import { THUMBNAIL_JOB } from '../jobs/find_thumbnail' // Instantiates a client. const client = new CloudTasksClient() diff --git a/packages/content-fetch/src/redis_data_source.ts b/packages/content-fetch/src/redis_data_source.ts index 917f9a867..aa985348a 100644 --- a/packages/content-fetch/src/redis_data_source.ts +++ b/packages/content-fetch/src/redis_data_source.ts @@ -26,10 +26,6 @@ export class RedisDataSource { createIORedisClient('mq', this.options) || this.cacheClient // if mq is not defined, use cache } - setOptions(options: RedisDataSourceOptions): void { - this.options = options - } - async shutdown(): Promise { try { await this.queueRedisClient?.quit() @@ -72,8 +68,8 @@ const createIORedisClient = ( export const redisDataSource = new RedisDataSource({ cache: { - url: process.env.REDIS_CACHE_URL, - cert: process.env.REDIS_CACHE_CERT, + url: process.env.REDIS_URL, + cert: process.env.REDIS_CERT, }, mq: { url: process.env.MQ_REDIS_URL, From ecbf402266697e53fe6f7a3c304a862f688136d4 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Thu, 25 Jan 2024 17:48:13 +0800 Subject: [PATCH 43/59] fix tests --- packages/api/test/global-setup.ts | 2 +- packages/api/test/global-teardown.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/api/test/global-setup.ts b/packages/api/test/global-setup.ts index dce893a86..31f9a3d86 100644 --- a/packages/api/test/global-setup.ts +++ b/packages/api/test/global-setup.ts @@ -7,7 +7,7 @@ export const mochaGlobalSetup = async () => { await createTestConnection() console.log('db connection created') - if (env.redis.url) { + if (env.redis.cache.url) { await redisDataSource.initialize() console.log('redis connection created') } diff --git a/packages/api/test/global-teardown.ts b/packages/api/test/global-teardown.ts index d76129414..2e476c55e 100644 --- a/packages/api/test/global-teardown.ts +++ b/packages/api/test/global-teardown.ts @@ -10,7 +10,7 @@ export const mochaGlobalTeardown = async () => { await appDataSource.destroy() console.log('db connection closed') - if (env.redis.url) { + if (env.redis.cache.url) { await redisDataSource.shutdown() console.log('redis connection closed') } From 7556c6da5f617b7f97cbc0cc4651f9a125e3b421 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Fri, 26 Jan 2024 17:15:39 +0800 Subject: [PATCH 44/59] move trigger rule into a job --- packages/api/src/jobs/trigger_rule.ts | 138 ++++++++++++++++++++ packages/api/src/resolvers/article/index.ts | 9 +- packages/api/src/services/labels.ts | 40 ++---- packages/api/src/services/rules.ts | 17 ++- packages/api/src/services/user.ts | 26 ++++ packages/api/src/utils/sendNotification.ts | 4 +- 6 files changed, 199 insertions(+), 35 deletions(-) create mode 100644 packages/api/src/jobs/trigger_rule.ts diff --git a/packages/api/src/jobs/trigger_rule.ts b/packages/api/src/jobs/trigger_rule.ts new file mode 100644 index 000000000..3ba72fd4d --- /dev/null +++ b/packages/api/src/jobs/trigger_rule.ts @@ -0,0 +1,138 @@ +import { LibraryItem, LibraryItemState } from '../entity/library_item' +import { Rule, RuleAction, RuleActionType, RuleEventType } from '../entity/rule' +import { addLabelsToLibraryItem } from '../services/labels' +import { + SearchArgs, + searchLibraryItems, + updateLibraryItem, +} from '../services/library_item' +import { findEnabledRules } from '../services/rules' +import { sendPushNotifications } from '../services/user' +import { logger } from '../utils/logger' + +interface Data { + id: string + userId: string + ruleEventType: RuleEventType + subscription: string + image: string + content: string + readingProgressPercent: number +} + +interface RuleActionObj { + userId: string + action: RuleAction + libraryItem: LibraryItem +} + +type RuleActionFunc = (obj: RuleActionObj) => Promise + +const addLabels = async (obj: RuleActionObj) => { + const labelIds = obj.action.params + + return addLabelsToLibraryItem( + labelIds, + obj.libraryItem.id, + obj.userId, + 'system' + ) +} + +const archivePage = async (obj: RuleActionObj) => { + return updateLibraryItem( + obj.libraryItem.id, + { archivedAt: new Date(), state: LibraryItemState.Archived }, + obj.userId + ) +} + +const markPageAsRead = async (obj: RuleActionObj) => { + return updateLibraryItem( + obj.libraryItem.id, + { + readingProgressTopPercent: 100, + readingProgressBottomPercent: 100, + readAt: new Date(), + }, + obj.userId + ) +} + +const sendNotification = async (obj: RuleActionObj) => { + const item = obj.libraryItem + const message = { + title: item.author || item.siteName || 'Omnivore', + body: item.title, + } + + return sendPushNotifications(obj.userId, message, 'rule') +} + +const getRuleAction = (actionType: RuleActionType): RuleActionFunc => { + switch (actionType) { + case RuleActionType.AddLabel: + return addLabels + case RuleActionType.Archive: + return archivePage + case RuleActionType.MarkAsRead: + return markPageAsRead + case RuleActionType.SendNotification: + return sendNotification + } +} + +const triggerActions = async (userId: string, rules: Rule[], data: Data) => { + const actionPromises: Promise[] = [] + + for (const rule of rules) { + const itemId = data.id + const searchArgs: SearchArgs = { + includeContent: false, + includeDeleted: false, + includePending: false, + size: 1, + query: `(${rule.filter}) AND includes:${itemId}`, + } + + const libraryItems = await searchLibraryItems(searchArgs, userId) + if (libraryItems.count === 0) { + logger.info(`No pages found for rule ${rule.id}`) + continue + } + + const libraryItem = libraryItems.libraryItems[0] + + for (const action of rule.actions) { + const actionFunc = getRuleAction(action.type) + const actionObj: RuleActionObj = { + userId, + action, + libraryItem, + } + + actionPromises.push(actionFunc(actionObj)) + } + } + + try { + await Promise.all(actionPromises) + } catch (error) { + logger.error(error) + } +} + +export const triggerRule = async (data: Data) => { + const { userId, ruleEventType } = data + + // get rules by calling api + const rules = await findEnabledRules(userId, ruleEventType) + if (rules.length === 0) { + console.log('No rules found') + return false + } + + await triggerActions(userId, rules, data) + + return true +} diff --git a/packages/api/src/resolvers/article/index.ts b/packages/api/src/resolvers/article/index.ts index 03cce990e..9eaf31c59 100644 --- a/packages/api/src/resolvers/article/index.ts +++ b/packages/api/src/resolvers/article/index.ts @@ -92,6 +92,7 @@ import { import { traceAs } from '../../tracing' import { analytics } from '../../utils/analytics' import { isSiteBlockedForParse } from '../../utils/blocked' +import { authorized } from '../../utils/gql-utils' import { cleanUrl, errorHandler, @@ -102,7 +103,6 @@ import { titleForFilePath, userDataToUser, } from '../../utils/helpers' -import { authorized } from '../../utils/gql-utils' import { contentConverter, getDistillerResult, @@ -908,7 +908,12 @@ export const setFavoriteArticleResolver = authorized< const labels = await findOrCreateLabels([label], uid) // adds Favorites label to item - await addLabelsToLibraryItem(labels, id, uid) + await addLabelsToLibraryItem( + labels.map((l) => l.id), + id, + uid, + 'user' + ) return { success: true, diff --git a/packages/api/src/services/labels.ts b/packages/api/src/services/labels.ts index 46ec8eb0a..564a1e2da 100644 --- a/packages/api/src/services/labels.ts +++ b/packages/api/src/services/labels.ts @@ -5,7 +5,6 @@ import { Label } from '../entity/label' import { createPubSubClient, EntityType, PubsubClient } from '../pubsub' import { authTrx } from '../repository' import { CreateLabelInput, labelRepository } from '../repository/label' -import { libraryItemRepository } from '../repository/library_item' type AddLabelsToLibraryItemEvent = { pageId: string @@ -124,43 +123,28 @@ export const saveLabelsInLibraryItem = async ( } export const addLabelsToLibraryItem = async ( - labels: Label[], + labelIds: string[], libraryItemId: string, userId: string, - source: LabelSource = 'user', - pubsub = createPubSubClient() + source: LabelSource = 'user' ) => { await authTrx( async (tx) => { - const libraryItem = await tx - .withRepository(libraryItemRepository) - .findOneByOrFail({ id: libraryItemId, user: { id: userId } }) - - if (libraryItem.labels) { - labels.push(...libraryItem.labels) - } - - // save new labels - await tx.getRepository(EntityLabel).save( - labels.map((l) => ({ - labelId: l.id, - libraryItemId, - source, - })) + await tx.query( + `INSERT INTO omnivore.entity_labels (label_id, library_item_id, source) + SELECT id, $1, $2 FROM omnivore.labels + WHERE id = ANY($3) + AND NOT EXISTS ( + SELECT 1 FROM omnivore.entity_labels + WHERE label_id = labels.id + AND library_item_id = $1 + )`, + [libraryItemId, source, labelIds] ) }, undefined, userId ) - - if (source === 'user') { - // create pubsub event - await pubsub.entityCreated( - EntityType.LABEL, - { pageId: libraryItemId, labels, source }, - userId - ) - } } export const saveLabelsInHighlight = async ( diff --git a/packages/api/src/services/rules.ts b/packages/api/src/services/rules.ts index 01f86c8ce..5f27a3839 100644 --- a/packages/api/src/services/rules.ts +++ b/packages/api/src/services/rules.ts @@ -1,6 +1,6 @@ -import { ILike } from 'typeorm' -import { Rule, RuleAction } from '../entity/rule' -import { authTrx } from '../repository' +import { ArrayContainedBy, ArrayContains, ILike } from 'typeorm' +import { Rule, RuleAction, RuleEventType } from '../entity/rule' +import { authTrx, getRepository } from '../repository' export const createRule = async ( userId: string, @@ -53,3 +53,14 @@ export const deleteRules = async (userId: string) => { userId ) } + +export const findEnabledRules = async ( + userId: string, + eventType: RuleEventType +) => { + return getRepository(Rule).findBy({ + user: { id: userId }, + enabled: true, + eventTypes: ArrayContainedBy([eventType]), + }) +} diff --git a/packages/api/src/services/user.ts b/packages/api/src/services/user.ts index 1a4dc58d1..1a74500bb 100644 --- a/packages/api/src/services/user.ts +++ b/packages/api/src/services/user.ts @@ -1,9 +1,15 @@ +import { Notification } from 'firebase-admin/messaging' import { DeepPartial, FindOptionsWhere, In } from 'typeorm' import { Profile } from '../entity/profile' import { StatusType, User } from '../entity/user' import { authTrx, getRepository, queryBuilderToRawSql } from '../repository' import { userRepository } from '../repository/user' import { SetClaimsRole } from '../utils/dictionary' +import { + PushNotificationType, + sendMulticastPushNotifications, +} from '../utils/sendNotification' +import { findDeviceTokensByUserId } from './user_device_tokens' export const deleteUser = async (userId: string) => { await authTrx( @@ -120,3 +126,23 @@ export const batchDelete = async (criteria: FindOptionsWhere) => { SetClaimsRole.ADMIN ) } + +export const sendPushNotifications = async ( + userId: string, + notification: Notification, + notificationType: PushNotificationType, + data?: { [key: string]: string } +) => { + const tokens = await findDeviceTokensByUserId(userId) + if (tokens.length === 0) { + throw new Error('No device tokens found') + } + + const message = { + notification, + data, + tokens: tokens.map((token) => token.token), + } + + return sendMulticastPushNotifications(userId, message, notificationType) +} diff --git a/packages/api/src/utils/sendNotification.ts b/packages/api/src/utils/sendNotification.ts index 9c6550653..26adf675c 100644 --- a/packages/api/src/utils/sendNotification.ts +++ b/packages/api/src/utils/sendNotification.ts @@ -60,7 +60,7 @@ export const sendMulticastPushNotifications = async ( }) logger.info('sending multicast message: ', message) - const res = await getMessaging().sendMulticast(message) + const res = await getMessaging().sendEachForMulticast(message) logger.info('send notification result: ', res.responses) return res @@ -75,7 +75,7 @@ export const sendBatchPushNotifications = async ( messages: Message[] ): Promise => { try { - const res = await getMessaging().sendAll(messages) + const res = await getMessaging().sendEach(messages) logger.info(`success count: ${res.successCount}`) return res From e03895c8b72949ba22da43c402bedc040e6c0f89 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Fri, 26 Jan 2024 17:40:51 +0800 Subject: [PATCH 45/59] enqueue trigger rule job once page is created or updated --- packages/api/src/jobs/trigger_rule.ts | 28 ++++++++++++++--------- packages/api/src/pubsub.ts | 26 +++++++++++++++++++-- packages/api/src/queue-processor.ts | 3 +++ packages/api/src/services/library_item.ts | 7 +++++- packages/api/src/utils/createTask.ts | 14 ++++++++++++ 5 files changed, 64 insertions(+), 14 deletions(-) diff --git a/packages/api/src/jobs/trigger_rule.ts b/packages/api/src/jobs/trigger_rule.ts index 3ba72fd4d..72d902a90 100644 --- a/packages/api/src/jobs/trigger_rule.ts +++ b/packages/api/src/jobs/trigger_rule.ts @@ -10,14 +10,10 @@ import { findEnabledRules } from '../services/rules' import { sendPushNotifications } from '../services/user' import { logger } from '../utils/logger' -interface Data { - id: string +export interface TriggerRuleJobData { + libraryItemId: string userId: string ruleEventType: RuleEventType - subscription: string - image: string - content: string - readingProgressPercent: number } interface RuleActionObj { @@ -26,6 +22,8 @@ interface RuleActionObj { libraryItem: LibraryItem } +export const TRIGGER_RULE_JOB_NAME = 'trigger-rule' + type RuleActionFunc = (obj: RuleActionObj) => Promise const addLabels = async (obj: RuleActionObj) => { @@ -43,7 +41,9 @@ const archivePage = async (obj: RuleActionObj) => { return updateLibraryItem( obj.libraryItem.id, { archivedAt: new Date(), state: LibraryItemState.Archived }, - obj.userId + obj.userId, + undefined, + true ) } @@ -55,7 +55,9 @@ const markPageAsRead = async (obj: RuleActionObj) => { readingProgressBottomPercent: 100, readAt: new Date(), }, - obj.userId + obj.userId, + undefined, + true ) } @@ -82,11 +84,15 @@ const getRuleAction = (actionType: RuleActionType): RuleActionFunc => { } } -const triggerActions = async (userId: string, rules: Rule[], data: Data) => { +const triggerActions = async ( + userId: string, + rules: Rule[], + data: TriggerRuleJobData +) => { const actionPromises: Promise[] = [] for (const rule of rules) { - const itemId = data.id + const itemId = data.libraryItemId const searchArgs: SearchArgs = { includeContent: false, includeDeleted: false, @@ -122,7 +128,7 @@ const triggerActions = async (userId: string, rules: Rule[], data: Data) => { } } -export const triggerRule = async (data: Data) => { +export const triggerRule = async (data: TriggerRuleJobData) => { const { userId, ruleEventType } = data // get rules by calling api diff --git a/packages/api/src/pubsub.ts b/packages/api/src/pubsub.ts index ea1bcf987..02c35f3d4 100644 --- a/packages/api/src/pubsub.ts +++ b/packages/api/src/pubsub.ts @@ -1,7 +1,9 @@ import { PubSub } from '@google-cloud/pubsub' import express from 'express' +import { RuleEventType } from './entity/rule' import { env } from './env' import { ReportType } from './generated/graphql' +import { enqueueTriggerRuleJob } from './utils/createTask' import { deepDelete } from './utils/helpers' import { buildLogger } from './utils/logger' @@ -41,11 +43,21 @@ export const createPubSubClient = (): PubsubClient => { Buffer.from(JSON.stringify({ userId, email, name, username })) ) }, - entityCreated: ( + entityCreated: async ( type: EntityType, data: T, userId: string ): Promise => { + // queue trigger rule job + if (type === EntityType.PAGE) { + const libraryItemId = (data as T & { id: string }).id + await enqueueTriggerRuleJob({ + userId, + ruleEventType: RuleEventType.PageCreated, + libraryItemId, + }) + } + const cleanData = deepDelete( data as T & Record, [...fieldsToDelete] @@ -56,11 +68,21 @@ export const createPubSubClient = (): PubsubClient => { Buffer.from(JSON.stringify({ type, userId, ...cleanData })) ) }, - entityUpdated: ( + entityUpdated: async ( type: EntityType, data: T, userId: string ): Promise => { + // queue trigger rule job + if (type === EntityType.PAGE) { + const libraryItemId = (data as T & { id: string }).id + await enqueueTriggerRuleJob({ + userId, + ruleEventType: RuleEventType.PageUpdated, + libraryItemId, + }) + } + const cleanData = deepDelete( data as T & Record, [...fieldsToDelete] diff --git a/packages/api/src/queue-processor.ts b/packages/api/src/queue-processor.ts index d99d79f10..cd89e1c69 100644 --- a/packages/api/src/queue-processor.ts +++ b/packages/api/src/queue-processor.ts @@ -14,6 +14,7 @@ import { savePageJob } from './jobs/save_page' import { updatePDFContentJob } from './jobs/update_pdf_content' import { redisDataSource } from './redis_data_source' import { CustomTypeOrmLogger } from './utils/logger' +import { triggerRule, TRIGGER_RULE_JOB_NAME } from './jobs/trigger_rule' export const QUEUE_NAME = 'omnivore-backend-queue' @@ -122,6 +123,8 @@ const main = async () => { } case THUMBNAIL_JOB: return findThumbnail(job.data) + case TRIGGER_RULE_JOB_NAME: + return triggerRule(job.data) } }, { diff --git a/packages/api/src/services/library_item.ts b/packages/api/src/services/library_item.ts index 11a7ee92a..73e9fde77 100644 --- a/packages/api/src/services/library_item.ts +++ b/packages/api/src/services/library_item.ts @@ -698,7 +698,8 @@ export const updateLibraryItem = async ( id: string, libraryItem: QueryDeepPartialEntity, userId: string, - pubsub = createPubSubClient() + pubsub = createPubSubClient(), + skipPubSub = false ): Promise => { const updatedLibraryItem = await authTrx( async (tx) => { @@ -726,6 +727,10 @@ export const updateLibraryItem = async ( userId ) + if (skipPubSub) { + return updatedLibraryItem + } + await pubsub.entityUpdated>( EntityType.PAGE, { diff --git a/packages/api/src/utils/createTask.ts b/packages/api/src/utils/createTask.ts index 94233a454..60e74da5d 100644 --- a/packages/api/src/utils/createTask.ts +++ b/packages/api/src/utils/createTask.ts @@ -17,6 +17,8 @@ import { import { THUMBNAIL_JOB } from '../jobs/find_thumbnail' import { queueRSSRefreshFeedJob } from '../jobs/rss/refreshAllFeeds' import { getBackendQueue } from '../queue-processor' +import { TriggerRuleJobData, TRIGGER_RULE_JOB_NAME } from '../jobs/trigger_rule' +import { getBackendQueue } from '../queue-processor' import { redisDataSource } from '../redis_data_source' import { signFeatureToken } from '../services/features' import { OmnivoreAuthorizationHeader } from './auth' @@ -648,4 +650,16 @@ export const enqueueRssFeedFetch = async ( } } +export const enqueueTriggerRuleJob = async (data: TriggerRuleJobData) => { + const queue = await getBackendQueue() + if (!queue) { + return undefined + } + + return queue.add(TRIGGER_RULE_JOB_NAME, data, { + removeOnComplete: true, + removeOnFail: true, + }) +} + export default createHttpTaskWithToken From 30fef30384195bae4f07f28ebb0e93623051262f Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Fri, 26 Jan 2024 20:32:56 +0800 Subject: [PATCH 46/59] resolve conflicts --- packages/api/src/utils/createTask.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/api/src/utils/createTask.ts b/packages/api/src/utils/createTask.ts index 60e74da5d..fc213d33e 100644 --- a/packages/api/src/utils/createTask.ts +++ b/packages/api/src/utils/createTask.ts @@ -16,7 +16,6 @@ import { } from '../generated/graphql' import { THUMBNAIL_JOB } from '../jobs/find_thumbnail' import { queueRSSRefreshFeedJob } from '../jobs/rss/refreshAllFeeds' -import { getBackendQueue } from '../queue-processor' import { TriggerRuleJobData, TRIGGER_RULE_JOB_NAME } from '../jobs/trigger_rule' import { getBackendQueue } from '../queue-processor' import { redisDataSource } from '../redis_data_source' From 29cb15de328f214c587c434fb78a3026d013cf0a Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Fri, 26 Jan 2024 21:50:21 +0800 Subject: [PATCH 47/59] use nitter handler for twitter new domain x.com --- packages/content-handler/src/websites/nitter-handler.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/content-handler/src/websites/nitter-handler.ts b/packages/content-handler/src/websites/nitter-handler.ts index 36ebfd8c2..f8d927d6f 100644 --- a/packages/content-handler/src/websites/nitter-handler.ts +++ b/packages/content-handler/src/websites/nitter-handler.ts @@ -31,7 +31,7 @@ interface Tweet { export class NitterHandler extends ContentHandler { // matches twitter.com and nitter.net urls URL_MATCH = - /((twitter\.com)|(nitter\.net))\/(?:#!\/)?(\w+)\/status(?:es)?\/(\d+)(?:\/.*)?/ + /((x\.com)|(twitter\.com)|(nitter\.net))\/(?:#!\/)?(\w+)\/status(?:es)?\/(\d+)(?:\/.*)?/ INSTANCES = [ { value: 'https://nitter.moomoo.me', score: 0 }, { value: 'https://nitter.net', score: 1 }, // the official instance @@ -316,7 +316,7 @@ export class NitterHandler extends ContentHandler { parseTweetUrl = (url: string) => { const match = url.match(this.URL_MATCH) return { - domain: match?.[1], + domain: match?.[1]?.replace('x', 'twitter'), username: match?.[4], tweetId: match?.[5], } From fd80724d79d45264bb08faedbb6f4a82282a8a0a Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Fri, 26 Jan 2024 22:09:53 +0800 Subject: [PATCH 48/59] fallback to guid if link of rss item is not available --- packages/api/src/jobs/rss/refreshFeed.ts | 15 ++++++++++++--- packages/content-fetch/src/request_handler.ts | 2 -- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/packages/api/src/jobs/rss/refreshFeed.ts b/packages/api/src/jobs/rss/refreshFeed.ts index ad09fa909..4253f8aec 100644 --- a/packages/api/src/jobs/rss/refreshFeed.ts +++ b/packages/api/src/jobs/rss/refreshFeed.ts @@ -10,6 +10,7 @@ import { updateSubscriptions, } from '../../services/update_subscription' import createHttpTaskWithToken from '../../utils/createTask' +import { logger } from '../../utils/logger' import { RSSRefreshContext } from './refreshAllFeeds' type FolderType = 'following' | 'inbox' @@ -475,16 +476,24 @@ const processSubscription = async ( // save each item in the feed for (const item of feed.items) { try { + const guid = item.guid || item.link // use published or updated if isoDate is not available for atom feeds const isoDate = item.isoDate || item.published || item.updated || item.created - console.log('Processing feed item', item.links, item.isoDate, feedUrl) - if (!item.links || item.links.length === 0) { + logger.info('Processing feed item', { + guid, + links: item.links, + isoDate, + feedUrl, + }) + + if (!item.links || item.links.length === 0 || !guid) { throw new Error('Invalid feed item') } - const link = getLink(item.links, feedUrl) + // fallback to guid if link is not available + const link = getLink(item.links, feedUrl) || guid if (!link) { throw new Error('Invalid feed item link') } diff --git a/packages/content-fetch/src/request_handler.ts b/packages/content-fetch/src/request_handler.ts index ed81b3ffb..5fb1d0066 100644 --- a/packages/content-fetch/src/request_handler.ts +++ b/packages/content-fetch/src/request_handler.ts @@ -68,7 +68,6 @@ export const contentFetchRequestHandler: RequestHandler = async (req, res) => { // users is used when saving article for multiple users let users = body.users || [] const userId = body.userId - const folder = body.folder // userId is used when saving article for a single user if (userId) { users = [ @@ -104,7 +103,6 @@ export const contentFetchRequestHandler: RequestHandler = async (req, res) => { rssFeedUrl, savedAt, publishedAt, - folder, users, } From ac5acf7f21735e3590dc0d664377453b7039aff4 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Fri, 26 Jan 2024 22:34:55 +0800 Subject: [PATCH 49/59] fix: add rss label and feed url to the pdf in rss feed item --- packages/api/src/generated/graphql.ts | 3 + packages/api/src/generated/schema.graphql | 3 + packages/api/src/jobs/save_page.ts | 117 ++++++---------------- packages/api/src/schema.ts | 3 + packages/api/src/services/save_file.ts | 7 +- 5 files changed, 44 insertions(+), 89 deletions(-) diff --git a/packages/api/src/generated/graphql.ts b/packages/api/src/generated/graphql.ts index 33ecceb9b..9d908a941 100644 --- a/packages/api/src/generated/graphql.ts +++ b/packages/api/src/generated/graphql.ts @@ -2283,8 +2283,11 @@ export type SaveFileInput = { clientRequestId: Scalars['ID']; folder?: InputMaybe; labels?: InputMaybe>; + publishedAt?: InputMaybe; + savedAt?: InputMaybe; source: Scalars['String']; state?: InputMaybe; + subscription?: InputMaybe; uploadFileId: Scalars['ID']; url: Scalars['String']; }; diff --git a/packages/api/src/generated/schema.graphql b/packages/api/src/generated/schema.graphql index 203d7b9bd..2df4f681e 100644 --- a/packages/api/src/generated/schema.graphql +++ b/packages/api/src/generated/schema.graphql @@ -1718,8 +1718,11 @@ input SaveFileInput { clientRequestId: ID! folder: String labels: [CreateLabelInput!] + publishedAt: Date + savedAt: Date source: String! state: ArticleSavingRequestStatus + subscription: String uploadFileId: ID! url: String! } diff --git a/packages/api/src/jobs/save_page.ts b/packages/api/src/jobs/save_page.ts index 0e87eb30c..1eebf1adb 100644 --- a/packages/api/src/jobs/save_page.ts +++ b/packages/api/src/jobs/save_page.ts @@ -8,6 +8,7 @@ import { } from '../generated/graphql' import { redisDataSource } from '../redis_data_source' import { userRepository } from '../repository/user' +import { saveFile } from '../services/save_file' import { savePage } from '../services/save_page' import { logger } from '../utils/logger' @@ -46,17 +47,6 @@ interface UploadFileResponse { } } -interface CreateArticleResponse { - data: { - createArticle: { - createdArticle: { - id: string - } - errorCodes: string[] - } - } -} - interface FetchResult { finalUrl: string title?: string @@ -174,57 +164,6 @@ const uploadPdf = async ( return uploadResult.id } -const sendCreateArticleMutation = async (userId: string, input: unknown) => { - const data = JSON.stringify({ - query: `mutation CreateArticle ($input: CreateArticleInput!){ - createArticle(input:$input){ - ... on CreateArticleSuccess{ - createdArticle{ - id - } - } - ... on CreateArticleError{ - errorCodes - } - } - }`, - variables: { - input, - }, - }) - - const auth = await signToken({ uid: userId }, JWT_SECRET) - try { - const response = await axios.post( - `${REST_BACKEND_ENDPOINT}/graphql`, - data, - { - headers: { - Cookie: `auth=${auth as string};`, - 'Content-Type': 'application/json', - }, - timeout: REQUEST_TIMEOUT, - } - ) - - if ( - response.data.data.createArticle.errorCodes && - response.data.data.createArticle.errorCodes.length > 0 - ) { - console.error( - 'error while creating article', - response.data.data.createArticle.errorCodes[0] - ) - return null - } - - return response.data.data.createArticle - } catch (error) { - console.error('error creating article', error) - return null - } -} - const sendImportStatusUpdate = async ( userId: string, taskId: string, @@ -298,25 +237,37 @@ export const savePageJob = async (data: Data, attemptsMade: number) => { const { title, contentType } = fetchedResult let content = fetchedResult.content + const user = await userRepository.findById(userId) + if (!user) { + logger.error('Unable to save job, user can not be found.', { + userId, + url, + }) + // if the user is not found, we do not retry + return false + } + // for pdf content, we need to upload the pdf if (contentType === 'application/pdf') { - const encodedUrl = encodeURI(url) - const uploadFileId = await uploadPdf(url, userId, articleSavingRequestId) - const uploadedPdf = await sendCreateArticleMutation(userId, { - url: encodedUrl, - articleSavingRequestId, - uploadFileId, - state, - labels, - source, - folder, - rssFeedUrl, - savedAt, - publishedAt, - }) - if (!uploadedPdf) { - throw new Error('error while saving uploaded pdf') + + const result = await saveFile( + { + url, + uploadFileId, + state: state ? (state as ArticleSavingRequestStatus) : undefined, + labels, + source, + folder, + subscription: rssFeedUrl, + savedAt, + publishedAt, + clientRequestId: articleSavingRequestId, + }, + user + ) + if (result.__typename == 'SaveError') { + throw new Error(result.message || result.errorCodes[0]) } isSaved = true @@ -331,16 +282,6 @@ export const savePageJob = async (data: Data, attemptsMade: number) => { state = ArticleSavingRequestStatus.Failed } - const user = await userRepository.findById(userId) - if (!user) { - logger.error('Unable to save job, user can not be found.', { - userId, - url, - }) - // if the user is not found, we do not retry - return false - } - // for non-pdf content, we need to save the page const result = await savePage( { diff --git a/packages/api/src/schema.ts b/packages/api/src/schema.ts index 877b4bfb4..d18b91012 100755 --- a/packages/api/src/schema.ts +++ b/packages/api/src/schema.ts @@ -545,6 +545,9 @@ const schema = gql` state: ArticleSavingRequestStatus labels: [CreateLabelInput!] folder: String + savedAt: Date + publishedAt: Date + subscription: String } input ParseResult { diff --git a/packages/api/src/services/save_file.ts b/packages/api/src/services/save_file.ts index e6c58148b..dd727ec38 100644 --- a/packages/api/src/services/save_file.ts +++ b/packages/api/src/services/save_file.ts @@ -14,6 +14,7 @@ export const saveFile = async ( const uploadFile = await findUploadFileById(input.uploadFileId) if (!uploadFile) { return { + __typename: 'SaveError', errorCodes: [SaveErrorCode.Unauthorized], } } @@ -24,6 +25,7 @@ export const saveFile = async ( if (!uploadFileData) { return { + __typename: 'SaveError', errorCodes: [SaveErrorCode.Unknown], } } @@ -34,6 +36,8 @@ export const saveFile = async ( { state: (input.state as unknown as LibraryItemState) || undefined, folder: input.folder || undefined, + savedAt: input.savedAt || undefined, + publishedAt: input.publishedAt || undefined, }, user.id ) @@ -43,7 +47,8 @@ export const saveFile = async ( await createAndSaveLabelsInLibraryItem( input.clientRequestId, user.id, - input.labels + input.labels, + input.subscription ) return { From 5c365021f30db3daa6eebd191a5f3564425d895c Mon Sep 17 00:00:00 2001 From: Alex T <79433869+hereAlexT@users.noreply.github.com> Date: Sun, 28 Jan 2024 14:25:16 +1100 Subject: [PATCH 50/59] add shortcut cmd+enter to save note add shortcut cmd+enter to save note --- .../components/templates/article/HighlightNoteModal.tsx | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/packages/web/components/templates/article/HighlightNoteModal.tsx b/packages/web/components/templates/article/HighlightNoteModal.tsx index d2bab8fb5..c505f96b3 100644 --- a/packages/web/components/templates/article/HighlightNoteModal.tsx +++ b/packages/web/components/templates/article/HighlightNoteModal.tsx @@ -78,6 +78,13 @@ export function HighlightNoteModal( event.preventDefault() event.stopPropagation() }} + onKeyDown={(event) => { + if ((event.metaKey || event.ctrlKey) && event.key === 'Enter') { + event.preventDefault() + saveNoteChanges() + props.onOpenChange(false) + } + }} >
    { From 8aea5c9a90e2cb93b4a0468c26a05e9e9a5702c7 Mon Sep 17 00:00:00 2001 From: Alex T <79433869+hereAlexT@users.noreply.github.com> Date: Sun, 28 Jan 2024 14:29:14 +1100 Subject: [PATCH 51/59] * feat(HighlightNoteModal.tsx): add keyboard shortcut to save note changes and close modal --- .../components/templates/article/HighlightNoteModal.tsx | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/packages/web/components/templates/article/HighlightNoteModal.tsx b/packages/web/components/templates/article/HighlightNoteModal.tsx index d2bab8fb5..c505f96b3 100644 --- a/packages/web/components/templates/article/HighlightNoteModal.tsx +++ b/packages/web/components/templates/article/HighlightNoteModal.tsx @@ -78,6 +78,13 @@ export function HighlightNoteModal( event.preventDefault() event.stopPropagation() }} + onKeyDown={(event) => { + if ((event.metaKey || event.ctrlKey) && event.key === 'Enter') { + event.preventDefault() + saveNoteChanges() + props.onOpenChange(false) + } + }} > { From 411b0c305ad0727f6e30ab9e4a99c7f2a69e440c Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Mon, 29 Jan 2024 11:37:35 +0800 Subject: [PATCH 52/59] Improve the library view contrast --- packages/web/components/tokens/stitches.config.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/web/components/tokens/stitches.config.ts b/packages/web/components/tokens/stitches.config.ts index 75cd5e1dc..2e82e9c68 100644 --- a/packages/web/components/tokens/stitches.config.ts +++ b/packages/web/components/tokens/stitches.config.ts @@ -169,7 +169,7 @@ export const { styled, css, theme, getCssText, globalCss, keyframes, config } = thBackground3: '#FFFFFF', thBackground4: '#EBEBEB', thBackground5: '#F5F5F5', - thBackgroundActive: 'rgb(255, 234, 159)', + thBackgroundActive: '#FFEA9F', thBackgroundContrast: '#FFFFFF', thLeftMenuBackground: '#FCFCFC', thLibraryBackground: '#F3F3F3', @@ -281,7 +281,7 @@ const darkThemeSpec = { thBackground3: '#242424', thBackground4: '#3D3D3D', thBackground5: '#3D3D3D', - thBackgroundActive: '#2E2E2E', + thBackgroundActive: '#3D3D3D', thBackgroundContrast: '#000000', thLeftMenuBackground: '#1D1D1D', thLibraryBackground: '#333333', From a2bdddf8c9e8b572a4d9cda6a357af2969ac8c8e Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Mon, 29 Jan 2024 12:30:31 +0800 Subject: [PATCH 53/59] fix tests --- packages/api/src/services/save_file.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/api/src/services/save_file.ts b/packages/api/src/services/save_file.ts index dd727ec38..c0e2a5e75 100644 --- a/packages/api/src/services/save_file.ts +++ b/packages/api/src/services/save_file.ts @@ -36,8 +36,10 @@ export const saveFile = async ( { state: (input.state as unknown as LibraryItemState) || undefined, folder: input.folder || undefined, - savedAt: input.savedAt || undefined, - publishedAt: input.publishedAt || undefined, + savedAt: input.savedAt ? new Date(input.savedAt) : undefined, + publishedAt: input.publishedAt + ? new Date(input.publishedAt) + : undefined, }, user.id ) From 4f061546bfbfb3e4f94bb79027d32dfb985a15aa Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Mon, 29 Jan 2024 12:55:19 +0800 Subject: [PATCH 54/59] remove api call to upload file request --- packages/api/src/jobs/save_page.ts | 78 ++------- packages/api/src/resolvers/article/index.ts | 4 +- .../api/src/resolvers/upload_files/index.ts | 151 +---------------- packages/api/src/services/upload_file.ts | 155 ++++++++++++++++++ 4 files changed, 172 insertions(+), 216 deletions(-) diff --git a/packages/api/src/jobs/save_page.ts b/packages/api/src/jobs/save_page.ts index 1eebf1adb..82d5b246a 100644 --- a/packages/api/src/jobs/save_page.ts +++ b/packages/api/src/jobs/save_page.ts @@ -10,13 +10,13 @@ import { redisDataSource } from '../redis_data_source' import { userRepository } from '../repository/user' import { saveFile } from '../services/save_file' import { savePage } from '../services/save_page' +import { uploadFile } from '../services/upload_file' import { logger } from '../utils/logger' const signToken = promisify(jwt.sign) const IMPORTER_METRICS_COLLECTOR_URL = env.queue.importerMetricsUrl const JWT_SECRET = env.server.jwtSecret -const REST_BACKEND_ENDPOINT = `${env.server.internalApiUrl}/api` const MAX_ATTEMPTS = 2 const REQUEST_TIMEOUT = 30000 // 30 seconds @@ -82,86 +82,32 @@ const uploadToSignedUrl = async ( } } -const getUploadIdAndSignedUrl = async ( - userId: string, - url: string, - articleSavingRequestId: string -) => { - const auth = await signToken({ uid: userId }, JWT_SECRET) - const data = JSON.stringify({ - query: `mutation UploadFileRequest($input: UploadFileRequestInput!) { - uploadFileRequest(input:$input) { - ... on UploadFileRequestError { - errorCodes - } - ... on UploadFileRequestSuccess { - id - uploadSignedUrl - } - } - }`, - variables: { - input: { - url: encodeURI(url), - contentType: 'application/pdf', - clientRequestId: articleSavingRequestId, - }, - }, - }) - - try { - const response = await axios.post( - `${REST_BACKEND_ENDPOINT}/graphql`, - data, - { - headers: { - Cookie: `auth=${auth as string};`, - 'Content-Type': 'application/json', - }, - timeout: REQUEST_TIMEOUT, - } - ) - - if ( - response.data.data.uploadFileRequest.errorCodes && - response.data.data.uploadFileRequest.errorCodes?.length > 0 - ) { - console.error( - 'Error while getting upload id and signed url', - response.data.data.uploadFileRequest.errorCodes[0] - ) - return null - } - - return response.data.data.uploadFileRequest - } catch (e) { - console.error('error getting upload id and signed url', e) - return null - } -} - const uploadPdf = async ( url: string, userId: string, articleSavingRequestId: string ) => { - const uploadResult = await getUploadIdAndSignedUrl( - userId, - url, - articleSavingRequestId + const result = await uploadFile( + { + url, + contentType: 'application/pdf', + clientRequestId: articleSavingRequestId, + }, + userId ) - if (!uploadResult) { + if (!result.uploadSignedUrl) { throw new Error('error while getting upload id and signed url') } + const uploaded = await uploadToSignedUrl( - uploadResult.uploadSignedUrl, + result.uploadSignedUrl, 'application/pdf', url ) if (!uploaded) { throw new Error('error while uploading pdf') } - return uploadResult.id + return result.id } const sendImportStatusUpdate = async ( diff --git a/packages/api/src/resolvers/article/index.ts b/packages/api/src/resolvers/article/index.ts index 03cce990e..80990d2f7 100644 --- a/packages/api/src/resolvers/article/index.ts +++ b/packages/api/src/resolvers/article/index.ts @@ -87,11 +87,13 @@ import { import { parsedContentToLibraryItem } from '../../services/save_page' import { findUploadFileById, + itemTypeForContentType, setFileUploadComplete, } from '../../services/upload_file' import { traceAs } from '../../tracing' import { analytics } from '../../utils/analytics' import { isSiteBlockedForParse } from '../../utils/blocked' +import { authorized } from '../../utils/gql-utils' import { cleanUrl, errorHandler, @@ -102,7 +104,6 @@ import { titleForFilePath, userDataToUser, } from '../../utils/helpers' -import { authorized } from '../../utils/gql-utils' import { contentConverter, getDistillerResult, @@ -111,7 +112,6 @@ import { parsePreparedContent, } from '../../utils/parser' import { getStorageFileDetails } from '../../utils/uploads' -import { itemTypeForContentType } from '../upload_files' export enum ArticleFormat { Markdown = 'markdown', diff --git a/packages/api/src/resolvers/upload_files/index.ts b/packages/api/src/resolvers/upload_files/index.ts index b22395458..74caee344 100644 --- a/packages/api/src/resolvers/upload_files/index.ts +++ b/packages/api/src/resolvers/upload_files/index.ts @@ -1,55 +1,17 @@ -/* eslint-disable @typescript-eslint/no-unused-vars */ -import normalizeUrl from 'normalize-url' -import path from 'path' -import { LibraryItemState } from '../../entity/library_item' -import { UploadFile } from '../../entity/upload_file' import { env } from '../../env' import { MutationUploadFileRequestArgs, - PageType, UploadFileRequestError, - UploadFileRequestErrorCode, UploadFileRequestSuccess, - UploadFileStatus, } from '../../generated/graphql' -import { validateUrl } from '../../services/create_page_save_request' -import { - createLibraryItem, - findLibraryItemByUrl, - updateLibraryItem, -} from '../../services/library_item' +import { uploadFile } from '../../services/upload_file' import { analytics } from '../../utils/analytics' -import { generateSlug } from '../../utils/helpers' import { authorized } from '../../utils/gql-utils' - -import { - contentReaderForLibraryItem, - generateUploadFilePathName, - generateUploadSignedUrl, -} from '../../utils/uploads' - -const isFileUrl = (url: string): boolean => { - const parsedUrl = new URL(url) - return parsedUrl.protocol == 'file:' -} - -export const itemTypeForContentType = (contentType: string) => { - if (contentType == 'application/epub+zip') { - return PageType.Book - } - return PageType.File -} - export const uploadFileRequestResolver = authorized< UploadFileRequestSuccess, UploadFileRequestError, MutationUploadFileRequestArgs ->(async (_, { input }, ctx) => { - const { authTrx, uid, log } = ctx - let uploadFileData: { id: string | null } = { - id: null, - } - +>(async (_, { input }, { uid }) => { analytics.track({ userId: uid, event: 'file_upload_request', @@ -59,112 +21,5 @@ export const uploadFileRequestResolver = authorized< }, }) - let title: string - let fileName: string - try { - const url = normalizeUrl(new URL(input.url).href, { - stripHash: true, - stripWWW: false, - }) - title = decodeURI(path.basename(new URL(url).pathname, '.pdf')) - fileName = decodeURI(path.basename(new URL(url).pathname)).replace( - /[^a-zA-Z0-9-_.]/g, - '' - ) - - if (!fileName) { - fileName = 'content.pdf' - } - - if (!isFileUrl(url)) { - try { - validateUrl(url) - } catch (error) { - log.info('illegal file input url', error) - return { - errorCodes: [UploadFileRequestErrorCode.BadInput], - } - } - } - } catch { - return { errorCodes: [UploadFileRequestErrorCode.BadInput] } - } - - uploadFileData = await authTrx((t) => - t.getRepository(UploadFile).save({ - url: input.url, - user: { id: uid }, - fileName, - status: UploadFileStatus.Initialized, - contentType: input.contentType, - }) - ) - - if (uploadFileData.id) { - const uploadFileId = uploadFileData.id - const uploadFilePathName = generateUploadFilePathName( - uploadFileId, - fileName - ) - const uploadSignedUrl = await generateUploadSignedUrl( - uploadFilePathName, - input.contentType - ) - - // If this is a file URL, we swap in a special URL - const attachmentUrl = `https://omnivore.app/attachments/${uploadFilePathName}` - if (isFileUrl(input.url)) { - await authTrx(async (tx) => { - await tx.getRepository(UploadFile).update(uploadFileId, { - url: attachmentUrl, - status: UploadFileStatus.Initialized, - }) - }) - } - - let createdItemId: string | undefined = undefined - if (input.createPageEntry) { - // If we have a file:// URL, don't try to match it - // and create a copy of the item, just create a - // new item. - const item = await findLibraryItemByUrl(input.url, uid) - if (item) { - await updateLibraryItem( - item.id, - { - state: LibraryItemState.Processing, - }, - uid - ) - createdItemId = item.id - } else { - const itemType = itemTypeForContentType(input.contentType) - const uploadFileId = uploadFileData.id - const item = await createLibraryItem( - { - id: input.clientRequestId || undefined, - originalUrl: isFileUrl(input.url) ? attachmentUrl : input.url, - user: { id: uid }, - title, - readableContent: '', - itemType, - uploadFile: { id: uploadFileData.id }, - slug: generateSlug(uploadFilePathName), - state: LibraryItemState.Processing, - contentReader: contentReaderForLibraryItem(itemType, uploadFileId), - }, - uid - ) - createdItemId = item.id - } - } - - return { - id: uploadFileData.id, - uploadSignedUrl, - createdPageId: createdItemId, - } - } else { - return { errorCodes: [UploadFileRequestErrorCode.FailedCreate] } - } + return uploadFile(input, uid) }) diff --git a/packages/api/src/services/upload_file.ts b/packages/api/src/services/upload_file.ts index c281a3384..009157747 100644 --- a/packages/api/src/services/upload_file.ts +++ b/packages/api/src/services/upload_file.ts @@ -1,5 +1,39 @@ +import normalizeUrl from 'normalize-url' +import path from 'path' +import { LibraryItemState } from '../entity/library_item' import { UploadFile } from '../entity/upload_file' +import { + PageType, + UploadFileRequestErrorCode, + UploadFileRequestInput, + UploadFileStatus, +} from '../generated/graphql' import { authTrx, getRepository } from '../repository' +import { generateSlug } from '../utils/helpers' +import { logger } from '../utils/logger' +import { + contentReaderForLibraryItem, + generateUploadFilePathName, + generateUploadSignedUrl, +} from '../utils/uploads' +import { validateUrl } from './create_page_save_request' +import { + createLibraryItem, + findLibraryItemByUrl, + updateLibraryItem, +} from './library_item' + +const isFileUrl = (url: string): boolean => { + const parsedUrl = new URL(url) + return parsedUrl.protocol == 'file:' +} + +export const itemTypeForContentType = (contentType: string) => { + if (contentType == 'application/epub+zip') { + return PageType.Book + } + return PageType.File +} export const findUploadFileById = async (id: string) => { return getRepository(UploadFile).findOne({ @@ -22,3 +56,124 @@ export const setFileUploadComplete = async (id: string, userId?: string) => { userId ) } + +export const uploadFile = async ( + input: UploadFileRequestInput, + uid: string +) => { + let uploadFileData: { id: string | null } = { + id: null, + } + let title: string + let fileName: string + try { + const url = normalizeUrl(new URL(input.url).href, { + stripHash: true, + stripWWW: false, + }) + title = decodeURI(path.basename(new URL(url).pathname, '.pdf')) + fileName = decodeURI(path.basename(new URL(url).pathname)).replace( + /[^a-zA-Z0-9-_.]/g, + '' + ) + + if (!fileName) { + fileName = 'content.pdf' + } + + if (!isFileUrl(url)) { + try { + validateUrl(url) + } catch (error) { + logger.info('illegal file input url', error) + return { + errorCodes: [UploadFileRequestErrorCode.BadInput], + } + } + } + } catch { + return { + errorCodes: [UploadFileRequestErrorCode.BadInput], + } + } + + uploadFileData = await authTrx((t) => + t.getRepository(UploadFile).save({ + url: input.url, + user: { id: uid }, + fileName, + status: UploadFileStatus.Initialized, + contentType: input.contentType, + }) + ) + + if (uploadFileData.id) { + const uploadFileId = uploadFileData.id + const uploadFilePathName = generateUploadFilePathName( + uploadFileId, + fileName + ) + const uploadSignedUrl = await generateUploadSignedUrl( + uploadFilePathName, + input.contentType + ) + + // If this is a file URL, we swap in a special URL + const attachmentUrl = `https://omnivore.app/attachments/${uploadFilePathName}` + if (isFileUrl(input.url)) { + await authTrx(async (tx) => { + await tx.getRepository(UploadFile).update(uploadFileId, { + url: attachmentUrl, + status: UploadFileStatus.Initialized, + }) + }) + } + + let createdItemId: string | undefined = undefined + if (input.createPageEntry) { + // If we have a file:// URL, don't try to match it + // and create a copy of the item, just create a + // new item. + const item = await findLibraryItemByUrl(input.url, uid) + if (item) { + await updateLibraryItem( + item.id, + { + state: LibraryItemState.Processing, + }, + uid + ) + createdItemId = item.id + } else { + const itemType = itemTypeForContentType(input.contentType) + const uploadFileId = uploadFileData.id + const item = await createLibraryItem( + { + id: input.clientRequestId || undefined, + originalUrl: isFileUrl(input.url) ? attachmentUrl : input.url, + user: { id: uid }, + title, + readableContent: '', + itemType, + uploadFile: { id: uploadFileData.id }, + slug: generateSlug(uploadFilePathName), + state: LibraryItemState.Processing, + contentReader: contentReaderForLibraryItem(itemType, uploadFileId), + }, + uid + ) + createdItemId = item.id + } + } + + return { + id: uploadFileData.id, + uploadSignedUrl, + createdPageId: createdItemId, + } + } else { + return { + errorCodes: [UploadFileRequestErrorCode.FailedCreate], + } + } +} From 58d384f4f653221d90b71aef522be7f3e4e74b23 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Mon, 29 Jan 2024 14:11:07 +0800 Subject: [PATCH 55/59] fix pdf saving --- packages/api/src/jobs/save_page.ts | 20 ++-- packages/api/src/services/save_file.ts | 26 ++--- packages/api/src/services/upload_file.ts | 116 ++++++++------------ packages/api/test/resolvers/article.test.ts | 27 ++++- 4 files changed, 87 insertions(+), 102 deletions(-) diff --git a/packages/api/src/jobs/save_page.ts b/packages/api/src/jobs/save_page.ts index 82d5b246a..3dd702ab9 100644 --- a/packages/api/src/jobs/save_page.ts +++ b/packages/api/src/jobs/save_page.ts @@ -35,18 +35,6 @@ interface Data { taskId?: string } -interface UploadFileResponse { - data: { - uploadFileRequest: { - id: string - uploadSignedUrl: string - uploadFileId: string - createdPageId: string - errorCodes?: string[] - } - } -} - interface FetchResult { finalUrl: string title?: string @@ -63,6 +51,12 @@ const uploadToSignedUrl = async ( contentType: string, contentObjUrl: string ) => { + logger.info('uploading to signed url', { + uploadSignedUrl, + contentType, + contentObjUrl, + }) + try { const stream = await axios.get(contentObjUrl, { responseType: 'stream', @@ -92,6 +86,7 @@ const uploadPdf = async ( url, contentType: 'application/pdf', clientRequestId: articleSavingRequestId, + createPageEntry: true, }, userId ) @@ -116,6 +111,7 @@ const sendImportStatusUpdate = async ( isImported?: boolean ) => { try { + logger.info('sending import status update') const auth = await signToken({ uid: userId }, JWT_SECRET) await axios.post( diff --git a/packages/api/src/services/save_file.ts b/packages/api/src/services/save_file.ts index c0e2a5e75..a004f1558 100644 --- a/packages/api/src/services/save_file.ts +++ b/packages/api/src/services/save_file.ts @@ -30,20 +30,18 @@ export const saveFile = async ( } } - if (input.state || input.folder) { - await updateLibraryItem( - input.clientRequestId, - { - state: (input.state as unknown as LibraryItemState) || undefined, - folder: input.folder || undefined, - savedAt: input.savedAt ? new Date(input.savedAt) : undefined, - publishedAt: input.publishedAt - ? new Date(input.publishedAt) - : undefined, - }, - user.id - ) - } + await updateLibraryItem( + input.clientRequestId, + { + state: + (input.state as unknown as LibraryItemState) || + LibraryItemState.Succeeded, + folder: input.folder || undefined, + savedAt: input.savedAt ? new Date(input.savedAt) : undefined, + publishedAt: input.publishedAt ? new Date(input.publishedAt) : undefined, + }, + user.id + ) // add labels to item await createAndSaveLabelsInLibraryItem( diff --git a/packages/api/src/services/upload_file.ts b/packages/api/src/services/upload_file.ts index 009157747..1df0ab97f 100644 --- a/packages/api/src/services/upload_file.ts +++ b/packages/api/src/services/upload_file.ts @@ -17,11 +17,7 @@ import { generateUploadSignedUrl, } from '../utils/uploads' import { validateUrl } from './create_page_save_request' -import { - createLibraryItem, - findLibraryItemByUrl, - updateLibraryItem, -} from './library_item' +import { createLibraryItem } from './library_item' const isFileUrl = (url: string): boolean => { const parsedUrl = new URL(url) @@ -61,9 +57,6 @@ export const uploadFile = async ( input: UploadFileRequestInput, uid: string ) => { - let uploadFileData: { id: string | null } = { - id: null, - } let title: string let fileName: string try { @@ -97,7 +90,7 @@ export const uploadFile = async ( } } - uploadFileData = await authTrx((t) => + const uploadFileData = await authTrx((t) => t.getRepository(UploadFile).save({ url: input.url, user: { id: uid }, @@ -106,74 +99,53 @@ export const uploadFile = async ( contentType: input.contentType, }) ) + const uploadFileId = uploadFileData.id + const uploadFilePathName = generateUploadFilePathName(uploadFileId, fileName) + const uploadSignedUrl = await generateUploadSignedUrl( + uploadFilePathName, + input.contentType + ) - if (uploadFileData.id) { - const uploadFileId = uploadFileData.id - const uploadFilePathName = generateUploadFilePathName( - uploadFileId, - fileName - ) - const uploadSignedUrl = await generateUploadSignedUrl( - uploadFilePathName, - input.contentType - ) - - // If this is a file URL, we swap in a special URL - const attachmentUrl = `https://omnivore.app/attachments/${uploadFilePathName}` - if (isFileUrl(input.url)) { - await authTrx(async (tx) => { - await tx.getRepository(UploadFile).update(uploadFileId, { - url: attachmentUrl, - status: UploadFileStatus.Initialized, - }) + // If this is a file URL, we swap in a special URL + const attachmentUrl = `https://omnivore.app/attachments/${uploadFilePathName}` + if (isFileUrl(input.url)) { + await authTrx(async (tx) => { + await tx.getRepository(UploadFile).update(uploadFileId, { + url: attachmentUrl, + status: UploadFileStatus.Initialized, }) - } - - let createdItemId: string | undefined = undefined - if (input.createPageEntry) { - // If we have a file:// URL, don't try to match it - // and create a copy of the item, just create a - // new item. - const item = await findLibraryItemByUrl(input.url, uid) - if (item) { - await updateLibraryItem( - item.id, - { - state: LibraryItemState.Processing, - }, - uid - ) - createdItemId = item.id - } else { - const itemType = itemTypeForContentType(input.contentType) - const uploadFileId = uploadFileData.id - const item = await createLibraryItem( - { - id: input.clientRequestId || undefined, - originalUrl: isFileUrl(input.url) ? attachmentUrl : input.url, - user: { id: uid }, - title, - readableContent: '', - itemType, - uploadFile: { id: uploadFileData.id }, - slug: generateSlug(uploadFilePathName), - state: LibraryItemState.Processing, - contentReader: contentReaderForLibraryItem(itemType, uploadFileId), - }, - uid - ) - createdItemId = item.id - } - } + }) + } + const itemType = itemTypeForContentType(input.contentType) + if (input.createPageEntry) { + // If we have a file:// URL, don't try to match it + // and create a copy of the item, just create a + // new item. + const item = await createLibraryItem( + { + id: input.clientRequestId || undefined, + originalUrl: isFileUrl(input.url) ? attachmentUrl : input.url, + user: { id: uid }, + title, + readableContent: '', + itemType, + uploadFile: { id: uploadFileData.id }, + slug: generateSlug(uploadFilePathName), + state: LibraryItemState.Processing, + contentReader: contentReaderForLibraryItem(itemType, uploadFileId), + }, + uid + ) return { - id: uploadFileData.id, + id: uploadFileId, uploadSignedUrl, - createdPageId: createdItemId, - } - } else { - return { - errorCodes: [UploadFileRequestErrorCode.FailedCreate], + createdPageId: item.id, } } + + return { + id: uploadFileId, + uploadSignedUrl, + } } diff --git a/packages/api/test/resolvers/article.test.ts b/packages/api/test/resolvers/article.test.ts index 601b2a498..c25d1447f 100644 --- a/packages/api/test/resolvers/article.test.ts +++ b/packages/api/test/resolvers/article.test.ts @@ -218,14 +218,18 @@ const savePageQuery = ( ` } -const saveFileQuery = (url: string, uploadFileId: string) => { +const saveFileQuery = ( + clientRequestId: string, + url: string, + uploadFileId: string +) => { return ` mutation { saveFile ( input: { url: "${url}", source: "test", - clientRequestId: "${generateFakeUuid()}", + clientRequestId: "${clientRequestId}", uploadFileId: "${uploadFileId}", } ) { @@ -832,8 +836,23 @@ describe('Article API', () => { let query = '' let url = '' let uploadFileId = '' + let itemId = '' + + before(async () => { + const item = await createLibraryItem( + { + user: { id: user.id }, + originalUrl: 'https://blog.omnivore.app/setBookmarkArticle', + slug: 'test-with-omnivore', + readableContent: '

    test

    ', + title: 'test title', + readingProgressBottomPercent: 100, + readingProgressTopPercent: 80, + }, + user.id + ) + itemId = item.id - before(() => { sinon.replace( uploads, 'getStorageFileDetails', @@ -842,7 +861,7 @@ describe('Article API', () => { }) beforeEach(() => { - query = saveFileQuery(url, uploadFileId) + query = saveFileQuery(itemId, url, uploadFileId) }) after(() => { From d5d581fc54f8d8afea4b8e16c1f4d6ac018be1fb Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Mon, 29 Jan 2024 11:48:47 +0800 Subject: [PATCH 56/59] remove redundant logs --- .../reports/content_display_report_created.ts | 21 +++---- packages/api/src/jobs/find_thumbnail.ts | 10 +--- packages/api/src/jobs/rss/refreshAllFeeds.ts | 13 ++-- packages/api/src/jobs/rss/refreshFeed.ts | 60 ++++++++++--------- packages/api/src/jobs/save_page.ts | 16 ++--- packages/api/src/jobs/update_pdf_content.ts | 3 +- packages/api/src/redis_data_source.ts | 2 +- packages/api/src/routers/svc/content.ts | 1 - packages/api/src/routers/svc/following.ts | 6 +- packages/api/src/routers/svc/links.ts | 6 +- packages/api/src/routers/svc/rss_feed.ts | 4 +- packages/api/src/routers/svc/user.ts | 2 +- packages/api/src/services/reports.ts | 21 +++---- packages/api/src/services/save_email.ts | 1 - packages/api/src/services/save_page.ts | 3 +- packages/api/src/utils/createTask.ts | 3 +- packages/api/src/utils/helpers.ts | 1 - packages/readabilityjs/Readability.js | 7 +-- 18 files changed, 85 insertions(+), 95 deletions(-) diff --git a/packages/api/src/events/reports/content_display_report_created.ts b/packages/api/src/events/reports/content_display_report_created.ts index d89dcfc22..4345e0601 100644 --- a/packages/api/src/events/reports/content_display_report_created.ts +++ b/packages/api/src/events/reports/content_display_report_created.ts @@ -23,16 +23,17 @@ export class ContentDisplayReportSubscriber ${report.user.id} for URL: ${report.originalUrl} ${report.reportComment}` - logger.info(message) - - if (!env.dev.isLocal) { - // If we are in the local environment, just log a message, otherwise email the report - await sendEmail({ - to: env.sender.feedback, - subject: 'New content display report', - text: message, - from: env.sender.message, - }) + // If we are in the local environment, just log a message, otherwise email the report + if (env.dev.isLocal) { + logger.info(message) + return } + + await sendEmail({ + to: env.sender.feedback, + subject: 'New content display report', + text: message, + from: env.sender.message, + }) } } diff --git a/packages/api/src/jobs/find_thumbnail.ts b/packages/api/src/jobs/find_thumbnail.ts index 2e1231962..85427dafc 100644 --- a/packages/api/src/jobs/find_thumbnail.ts +++ b/packages/api/src/jobs/find_thumbnail.ts @@ -5,7 +5,7 @@ import { findLibraryItemById, updateLibraryItem, } from '../services/library_item' -import { createImageProxyUrl, createThumbnailUrl } from '../utils/imageproxy' +import { createThumbnailUrl } from '../utils/imageproxy' import { logger } from '../utils/logger' interface Data { @@ -22,7 +22,7 @@ interface ImageSize { export const THUMBNAIL_JOB = 'find-thumbnail' const fetchImage = async (url: string): Promise => { - console.log('fetching image', url) + logger.info('fetching image', url) try { // get image file by url return await axios.get(url, { @@ -70,7 +70,6 @@ export const fetchAllImageSizes = async (content: string) => { // fetch all images by src and get their sizes const images = dom.querySelectorAll('img[src]') if (!images || images.length === 0) { - console.log('no images') return [] } @@ -100,7 +99,6 @@ export const _findThumbnail = (imagesSizes: (ImageSize | null)[]) => { // ignore small images if (area < 5000) { - logger.info('ignore small', { src: imageSize.src }) continue } @@ -109,13 +107,11 @@ export const _findThumbnail = (imagesSizes: (ImageSize | null)[]) => { Math.max(imageSize.width, imageSize.height) / Math.min(imageSize.width, imageSize.height) if (ratio > 1.5) { - logger.info('penalizing long/wide', { src: imageSize.src }) area /= ratio * 2 } // penalize images with "sprite" in their name if (imageSize.src.toLowerCase().includes('sprite')) { - logger.info('penalizing sprite', { src: imageSize.src }) area /= 10 } @@ -139,7 +135,6 @@ export const findThumbnail = async (data: Data) => { const thumbnail = item.thumbnail if (thumbnail) { - logger.info('thumbnail already set') const proxyUrl = createThumbnailUrl(thumbnail) // pre-cache thumbnail first if exists const image = await fetchImage(proxyUrl) @@ -154,7 +149,6 @@ export const findThumbnail = async (data: Data) => { const imageSizes = await fetchAllImageSizes(item.readableContent) // find thumbnail from all images if thumbnail not set if (!item.thumbnail && imageSizes.length > 0) { - logger.info('finding thumbnail...') const thumbnail = _findThumbnail(imageSizes) if (!thumbnail) { logger.info('no thumbnail found from content') diff --git a/packages/api/src/jobs/rss/refreshAllFeeds.ts b/packages/api/src/jobs/rss/refreshAllFeeds.ts index 024ae0884..34b373aac 100644 --- a/packages/api/src/jobs/rss/refreshAllFeeds.ts +++ b/packages/api/src/jobs/rss/refreshAllFeeds.ts @@ -5,6 +5,7 @@ import { getBackendQueue } from '../../queue-processor' import { validateUrl } from '../../services/create_page_save_request' import { RssSubscriptionGroup } from '../../utils/createTask' import { stringToHash } from '../../utils/helpers' +import { logger } from '../../utils/logger' export type RSSRefreshContext = { type: 'all' | 'user-added' @@ -44,7 +45,7 @@ export const refreshAllFeeds = async (db: DataSource): Promise => { ['RSS', 'ACTIVE', 'following', 'ACTIVE'] )) as RssSubscriptionGroup[] - console.log(`rss: checking ${subscriptionGroups.length}`, { + logger.info(`rss: checking ${subscriptionGroups.length}`, { refreshContext, }) @@ -53,11 +54,11 @@ export const refreshAllFeeds = async (db: DataSource): Promise => { await updateSubscriptionGroup(group, refreshContext) } catch (err) { // we don't want to fail the whole job if one subscription group fails - console.error('error updating subscription group') + logger.error('error updating subscription group') } } const finishTime = new Date() - console.log( + logger.info( `rss: finished queuing subscription groups at ${finishTime.toISOString()}`, { refreshContext, @@ -74,18 +75,18 @@ const updateSubscriptionGroup = async ( let feedURL = group.url const userList = JSON.stringify(group.userIds.sort()) if (!feedURL) { - console.error('no url for feed group', group) + logger.error('no url for feed group', group) return } if (!userList) { - console.error('no userlist for feed group', group) + logger.error('no userlist for feed group', group) return } try { feedURL = validateUrl(feedURL).toString() } catch (err) { - console.log('not refreshing invalid feed url: ', { feedURL }) + logger.error('not refreshing invalid feed url: %s', feedURL) } const jobid = `refresh-feed_${stringToHash(feedURL)}_${stringToHash( userList diff --git a/packages/api/src/jobs/rss/refreshFeed.ts b/packages/api/src/jobs/rss/refreshFeed.ts index 4253f8aec..86917c04f 100644 --- a/packages/api/src/jobs/rss/refreshFeed.ts +++ b/packages/api/src/jobs/rss/refreshFeed.ts @@ -97,11 +97,11 @@ const isFeedBlocked = async (feedUrl: string) => { // if the feed has failed to fetch more than certain times, block it const maxFailures = parseInt(process.env.MAX_FEED_FETCH_FAILURES ?? '10') if (result && parseInt(result) > maxFailures) { - console.log('feed is blocked: ', feedUrl) + logger.info('feed is blocked: %s', feedUrl) return true } } catch (error) { - console.error('Failed to check feed block status', feedUrl, error) + logger.error('Failed to check feed block status', { feedUrl, error }) } return false @@ -117,7 +117,7 @@ const incrementFeedFailure = async (feedUrl: string) => { return result } catch (error) { - console.error('Failed to block feed', feedUrl, error) + logger.error('Failed to block feed', { feedUrl, error }) return null } } @@ -165,7 +165,7 @@ export const fetchAndChecksum = async (url: string) => { return { url, content: dataStr, checksum: hash.digest('hex') } } catch (error) { - console.log(`Failed to fetch or hash content from ${url}.`, error) + logger.info(`Failed to fetch or hash content from ${url}.`, error) return null } } @@ -209,7 +209,7 @@ const parseFeed = async (url: string, content: string) => { // otherwise the error will be caught by the outer try catch return await parser.parseString(content) } catch (error) { - console.log(error) + logger.info(error) return null } } @@ -220,7 +220,7 @@ const isItemRecentlySaved = async (userId: string, url: string) => { const result = await redisDataSource.redisClient?.get(key) return !!result } catch (err) { - console.error('error checking if item is old', err) + logger.error('error checking if item is old', err) } // If we failed to check, assume the item is good return false @@ -256,7 +256,7 @@ const createTask = async ( ) => { const isRecentlySaved = await isItemRecentlySaved(userId, item.link) if (isRecentlySaved) { - console.log('Item recently saved', item.link) + logger.info('Item recently saved %s', item.link) return true } @@ -264,7 +264,7 @@ const createTask = async ( return createItemWithPreviewContent(userId, feedUrl, item) } - console.log(`adding fetch content task ${userId} ${item.link.trim()}`) + logger.info(`adding fetch content task ${userId} ${item.link.trim()}`) return addFetchContentTask(fetchContentTasks, userId, folder, item) } @@ -292,7 +292,7 @@ const fetchContentAndCreateItem = async ( }) return !!task } catch (error) { - console.error('Error while creating task', error) + logger.error('Error while creating task', error) return false } } @@ -334,7 +334,7 @@ const createItemWithPreviewContent = async ( }) return !!task } catch (error) { - console.error('Error while creating task', error) + logger.error('Error while creating task', error) return false } } @@ -454,7 +454,11 @@ const processSubscription = async ( let lastValidItem: RssFeedItem | null = null if (fetchResult.checksum === lastFetchedChecksum) { - console.log('feed has not been updated', feedUrl, lastFetchedChecksum) + logger.info( + 'feed has not been updated %s, %s', + feedUrl, + lastFetchedChecksum + ) return } const updatedLastFetchedChecksum = fetchResult.checksum @@ -464,12 +468,12 @@ const processSubscription = async ( failedAt: Date | undefined const feedLastBuildDate = feed.lastBuildDate - console.log('Feed last build date', feedLastBuildDate) + logger.info('Feed last build date %s', feedLastBuildDate) if ( feedLastBuildDate && new Date(feedLastBuildDate) <= new Date(mostRecentItemDate) ) { - console.log('Skipping old feed', feedLastBuildDate) + logger.info('Skipping old feed %s', feedLastBuildDate) return } @@ -522,7 +526,7 @@ const processSubscription = async ( // skip old items if (isOldItem(feedItem, mostRecentItemDate)) { - console.log('Skipping old feed item', feedItem.link) + logger.info('Skipping old feed item %s', feedItem.link) continue } @@ -545,7 +549,7 @@ const processSubscription = async ( itemCount = itemCount + 1 } catch (error) { - console.error('Error while saving RSS feed item', { error, item }) + logger.error('Error while saving RSS feed item', { error, item }) failedAt = new Date() } } @@ -554,7 +558,7 @@ const processSubscription = async ( if (!lastItemFetchedAt && !failedAt) { // the feed has been fetched before, no new valid items found if (mostRecentItemDate || !lastValidItem) { - console.log('No new valid items found') + logger.info('No new valid items found') return } @@ -568,7 +572,9 @@ const processSubscription = async ( folder ) if (!created) { - console.error('Failed to create task for feed item', lastValidItem.link) + logger.error('Failed to create task for feed item', { + url: lastValidItem.link, + }) failedAt = new Date() } @@ -589,14 +595,14 @@ const processSubscription = async ( refreshedAt, failedAt, }) - console.log('Updated subscription', updatedSubscription) + logger.info('Updated subscription', updatedSubscription) } export const refreshFeed = async (request: any) => { if (isRefreshFeedRequest(request)) { return _refreshFeed(request) } - console.log('not a feed to refresh') + logger.info('not a feed to refresh') return false } @@ -613,36 +619,36 @@ export const _refreshFeed = async (request: RefreshFeedRequest) => { refreshContext, } = request - console.log('Processing feed', feedUrl, { refreshContext: refreshContext }) + logger.info('Processing feed', feedUrl, { refreshContext: refreshContext }) try { const isBlocked = await isFeedBlocked(feedUrl) if (isBlocked) { - console.log('feed is blocked: ', feedUrl) + logger.info('feed is blocked: %s', feedUrl) throw new Error('feed is blocked') } const fetchResult = await fetchAndChecksum(feedUrl) if (!fetchResult) { - console.error('Failed to fetch RSS feed', feedUrl) + logger.error('Failed to fetch RSS feed %s', feedUrl) await incrementFeedFailure(feedUrl) throw new Error('Failed to fetch RSS feed') } const feed = await parseFeed(feedUrl, fetchResult.content) if (!feed) { - console.error('Failed to parse RSS feed', feedUrl) + logger.error('Failed to parse RSS feed %s', feedUrl) await incrementFeedFailure(feedUrl) throw new Error('Failed to parse RSS feed') } let allowFetchContent = true if (isContentFetchBlocked(feedUrl)) { - console.log('fetching content blocked for feed: ', feedUrl) + logger.info('fetching content blocked for feed: %s', feedUrl) allowFetchContent = false } - console.log('Fetched feed', feed.title, new Date()) + logger.info('Fetched feed %s at %s', feed.title, new Date()) const fetchContentTasks = new Map() // url -> FetchContentTask // process each subscription sequentially @@ -664,7 +670,7 @@ export const _refreshFeed = async (request: RefreshFeedRequest) => { feed ) } catch (error) { - console.error('Error while processing subscription', { + logger.error('Error while processing subscription', { error, subscriptionId, }) @@ -682,7 +688,7 @@ export const _refreshFeed = async (request: RefreshFeedRequest) => { return true } catch (error) { - console.error('Error while saving RSS feeds', { + logger.error('Error while saving RSS feeds', { feedUrl, subscriptionIds, error, diff --git a/packages/api/src/jobs/save_page.ts b/packages/api/src/jobs/save_page.ts index 3dd702ab9..0c933260f 100644 --- a/packages/api/src/jobs/save_page.ts +++ b/packages/api/src/jobs/save_page.ts @@ -71,7 +71,7 @@ const uploadToSignedUrl = async ( timeout: REQUEST_TIMEOUT, }) } catch (error) { - console.error('error uploading to signed url', error) + logger.error('error uploading to signed url', error) return null } } @@ -129,7 +129,7 @@ const sendImportStatusUpdate = async ( } ) } catch (e) { - console.error('error while sending import status update', e) + logger.error('error while sending import status update', e) } } @@ -149,7 +149,7 @@ const getCachedFetchResult = async (url: string) => { throw new Error('fetch result is not valid') } - console.log('fetch result is cached', url) + logger.info('fetch result is cached', url) return fetchResult } @@ -172,7 +172,7 @@ export const savePageJob = async (data: Data, attemptsMade: number) => { state = data.state try { - console.log(`savePageJob: ${userId} ${url}`) + logger.info(`savePageJob: ${userId} ${url}`) // get the fetch result from cache const fetchedResult = await getCachedFetchResult(url) @@ -218,7 +218,7 @@ export const savePageJob = async (data: Data, attemptsMade: number) => { } if (!content) { - console.log('content is not fetched', url) + logger.info('content is not fetched', url) // set the state to failed if we don't have content content = 'Failed to fetch content' state = ArticleSavingRequestStatus.Failed @@ -250,16 +250,16 @@ export const savePageJob = async (data: Data, attemptsMade: number) => { isSaved = true } catch (e) { if (e instanceof Error) { - console.error('error while saving page', e.message) + logger.error('error while saving page: %s', e.message) } else { - console.error('error while saving page', 'unknown error') + logger.error('error while saving page: unknown error') } throw e } finally { const lastAttempt = attemptsMade === MAX_ATTEMPTS - 1 if (lastAttempt) { - console.log('last attempt reached', data.url) + logger.info('last attempt reached %s', data.url) } if (taskId && (isSaved || lastAttempt)) { diff --git a/packages/api/src/jobs/update_pdf_content.ts b/packages/api/src/jobs/update_pdf_content.ts index 60850fd0a..ff346a534 100644 --- a/packages/api/src/jobs/update_pdf_content.ts +++ b/packages/api/src/jobs/update_pdf_content.ts @@ -1,5 +1,4 @@ import { - UpdateContentMessage, isUpdateContentMessage, updateContentForFileItem, } from '../services/update_pdf_content' @@ -9,6 +8,6 @@ export const updatePDFContentJob = async (data: unknown): Promise => { if (isUpdateContentMessage(data)) { return await updateContentForFileItem(data) } - logger.log('update_pdf_content data is not a update message', { data }) + logger.info('update_pdf_content data is not a update message', { data }) return false } diff --git a/packages/api/src/redis_data_source.ts b/packages/api/src/redis_data_source.ts index 26da7d9fe..051bcbcc0 100644 --- a/packages/api/src/redis_data_source.ts +++ b/packages/api/src/redis_data_source.ts @@ -45,7 +45,7 @@ export class RedisDataSource { await this.workerRedisClient?.quit() await this.redisClient?.quit() } catch (err) { - console.error('error while shutting down redis', err) + logger.error('error while shutting down redis', err) } } } diff --git a/packages/api/src/routers/svc/content.ts b/packages/api/src/routers/svc/content.ts index f32d4078b..c545b4f32 100644 --- a/packages/api/src/routers/svc/content.ts +++ b/packages/api/src/routers/svc/content.ts @@ -13,7 +13,6 @@ export function contentServiceRouter() { const router = express.Router() router.post('/search', async (req, res) => { - logger.info('search req', req) const { message: msgStr, expired } = readPushSubscription(req) logger.info('read pubsub message', { msgStr, expired }) diff --git a/packages/api/src/routers/svc/following.ts b/packages/api/src/routers/svc/following.ts index 261dbf4ee..834d6c531 100644 --- a/packages/api/src/routers/svc/following.ts +++ b/packages/api/src/routers/svc/following.ts @@ -53,15 +53,13 @@ export function followingServiceRouter() { const router = express.Router() router.post('/save', async (req, res) => { - logger.info('save following item request', req.body) - if (req.query.token !== process.env.PUBSUB_VERIFICATION_TOKEN) { - console.log('query does not include valid token') + logger.info('query does not include valid token') return res.sendStatus(403) } if (!isSaveFollowingItemRequest(req.body)) { - console.error('Invalid request body', req.body) + logger.error('Invalid request body', req.body) return res.status(400).send('INVALID_REQUEST_BODY') } diff --git a/packages/api/src/routers/svc/links.ts b/packages/api/src/routers/svc/links.ts index 7d6e1d0db..05d69225d 100644 --- a/packages/api/src/routers/svc/links.ts +++ b/packages/api/src/routers/svc/links.ts @@ -29,7 +29,7 @@ const getPruneMessage = (msgStr: string): PruneMessage => { return obj } } catch (err) { - console.log('error deserializing event: ', { msgStr, err }) + logger.error('error deserializing event: ', { msgStr, err }) } // default to prune following folder items older than 30 days @@ -43,7 +43,6 @@ export function linkServiceRouter() { const router = express.Router() router.post('/create', async (req, res) => { - logger.info('create link req', req) const { message: msgStr, expired } = readPushSubscription(req) logger.info('read pubsub message', { msgStr, expired }) @@ -71,7 +70,6 @@ export function linkServiceRouter() { userId: msg.userId, url: msg.url, }) - logger.info('create link request', request) res.status(200).send(request) } catch (err) { @@ -81,8 +79,6 @@ export function linkServiceRouter() { }) router.post('/prune', async (req, res) => { - logger.info('prune expired items in folder') - const { message: msgStr, expired } = readPushSubscription(req) if (!msgStr) { diff --git a/packages/api/src/routers/svc/rss_feed.ts b/packages/api/src/routers/svc/rss_feed.ts index 7b2ef2ccb..4f7b345d8 100644 --- a/packages/api/src/routers/svc/rss_feed.ts +++ b/packages/api/src/routers/svc/rss_feed.ts @@ -23,11 +23,11 @@ export function rssFeedRouter() { if (redisDataSource.workerRedisClient) { await queueRSSRefreshAllFeedsJob() } else { - console.log('unable to fetchAll feeds, redis is not configured') + logger.info('unable to fetchAll feeds, redis is not configured') return res.status(500).send('Expired') } } catch (error) { - logger.info('error fetching rss feeds', error) + logger.error('error fetching rss feeds', error) return res.status(500).send('Internal Server Error') } diff --git a/packages/api/src/routers/svc/user.ts b/packages/api/src/routers/svc/user.ts index cf9519c69..ec6857ff0 100644 --- a/packages/api/src/routers/svc/user.ts +++ b/packages/api/src/routers/svc/user.ts @@ -23,7 +23,7 @@ const getCleanupMessage = (msgStr: string): CleanupMessage => { return obj } } catch (err) { - console.log('error deserializing event: ', { msgStr, err }) + logger.error('error deserializing event: ', { msgStr, err }) } return { diff --git a/packages/api/src/services/reports.ts b/packages/api/src/services/reports.ts index f7829dadf..e2aa573d5 100644 --- a/packages/api/src/services/reports.ts +++ b/packages/api/src/services/reports.ts @@ -33,18 +33,19 @@ export const saveContentDisplayReport = async ( ${report.user.id} for URL: ${report.originalUrl} ${report.reportComment}` - logger.info(message) - - if (!env.dev.isLocal) { - // If we are in the local environment, just log a message, otherwise email the report - await sendEmail({ - to: env.sender.feedback, - subject: 'New content display report', - text: message, - from: env.sender.message, - }) + // If we are in the local environment, just log a message, otherwise email the report + if (env.dev.isLocal) { + logger.info(message) + return !!report } + await sendEmail({ + to: env.sender.feedback, + subject: 'New content display report', + text: message, + from: env.sender.message, + }) + return !!report } diff --git a/packages/api/src/services/save_email.ts b/packages/api/src/services/save_email.ts index c080345ad..8ecd25d53 100644 --- a/packages/api/src/services/save_email.ts +++ b/packages/api/src/services/save_email.ts @@ -75,7 +75,6 @@ export const saveEmail = async ( existingLibraryItem.id, input.userId ) - logger.info('updated page from email', updatedLibraryItem) return updatedLibraryItem } diff --git a/packages/api/src/services/save_page.ts b/packages/api/src/services/save_page.ts index f8fd4340f..52c814acf 100644 --- a/packages/api/src/services/save_page.ts +++ b/packages/api/src/services/save_page.ts @@ -58,7 +58,6 @@ const shouldParseInBackend = (input: SavePageInput): boolean => { return ( ALREADY_PARSED_SOURCES.indexOf(input.source) === -1 && FORCE_PUPPETEER_URLS.some((regex) => { - console.log('REGEX: ', regex) return regex.test(input.url) }) ) @@ -241,7 +240,7 @@ export const parsedContentToLibraryItem = ({ rssFeedUrl?: string | null folder?: string | null }): DeepPartial & { originalUrl: string } => { - console.log('save_page: state', { url, state, itemId }) + logger.info('save_page: state', { url, state, itemId }) return { id: itemId || undefined, slug, diff --git a/packages/api/src/utils/createTask.ts b/packages/api/src/utils/createTask.ts index fc213d33e..4e3ed31ce 100644 --- a/packages/api/src/utils/createTask.ts +++ b/packages/api/src/utils/createTask.ts @@ -67,7 +67,7 @@ const createHttpTaskWithToken = async ({ > => { // If there is no Google Cloud Project Id exposed, it means that we are in local environment if (env.dev.isLocal || !project) { - console.error( + logger.error( 'error: attempting to create a cloud task but not running in google cloud.' ) return null @@ -172,7 +172,6 @@ export const createAppEngineTask = async ({ } logger.info('Sending task:') - logger.info(task) // Send create task request. const request = { parent: parent, task: task } const [response] = await client.createTask(request) diff --git a/packages/api/src/utils/helpers.ts b/packages/api/src/utils/helpers.ts index df4fcb014..c53b1f630 100644 --- a/packages/api/src/utils/helpers.ts +++ b/packages/api/src/utils/helpers.ts @@ -25,7 +25,6 @@ import { SearchItem, } from '../generated/graphql' import { createPubSubClient } from '../pubsub' -import { redisDataSource } from '../redis_data_source' import { validateUrl } from '../services/create_page_save_request' import { updateLibraryItem } from '../services/library_item' import { Merge } from '../util' diff --git a/packages/readabilityjs/Readability.js b/packages/readabilityjs/Readability.js index b268c0f2e..2a6aced84 100644 --- a/packages/readabilityjs/Readability.js +++ b/packages/readabilityjs/Readability.js @@ -3036,16 +3036,16 @@ Readability.prototype = { // detect language from the html content const languages = (await cld.detect(content, { isHTML: true })).languages; - console.log('Detected languages: ', languages); + this.log('Detected languages: ', languages); if (languages.length > 0) { code = languages[0].code; } - console.log('Getting language name from code: ', code); + this.log('Getting language name from code: ', code); let lang = new Intl.DisplayNames(['en'], {type: 'language'}); return lang.of(code); } catch (error) { - console.error('Failed to get language', error); + this.log('Failed to get language', error); return 'English'; } }, @@ -3082,7 +3082,6 @@ Readability.prototype = { this._removeScripts(this._doc); this._prepDocument(); - console.log(this._doc.body.innerHTML); var metadata = this._getArticleMetadata(jsonLd); this._articleTitle = metadata.title; From 809c572ca783bcfc9460abf0aa51030d63ea8bd1 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Mon, 29 Jan 2024 12:10:48 +0800 Subject: [PATCH 57/59] reduce the size of the log entry by truncating any string values to 500 characters --- packages/api/src/utils/logger.ts | 14 +++++--------- packages/inbound-email-handler/src/index.ts | 2 -- 2 files changed, 5 insertions(+), 11 deletions(-) diff --git a/packages/api/src/utils/logger.ts b/packages/api/src/utils/logger.ts index 99daf5a7b..f4019893d 100644 --- a/packages/api/src/utils/logger.ts +++ b/packages/api/src/utils/logger.ts @@ -34,11 +34,9 @@ export class CustomTypeOrmLogger } logQuery(query: string, parameters?: any[], queryRunner?: QueryRunner) { - this.logger.info( - `query: ${query} -- PARAMETERS: ${super.stringifyParams( - parameters || [] - )}` - ) + this.logger.info(query, { + parameters, + }) } log( @@ -133,10 +131,8 @@ const truncateObjectDeep = (object: any, length: number): any => { class GcpLoggingTransport extends LoggingWinston { log(info: any, callback: (err: Error | null, apiResponse?: any) => void) { - const sizeInfo = jsonStringify(info).length - if (sizeInfo > MAX_LOG_SIZE) { - info = truncateObjectDeep(info, 500) as never // the max length for string values is 500 - } + // reduce the size of the log entry by truncating any string values to 500 characters + info = truncateObjectDeep(info, 500) as never super.log(info, callback) } } diff --git a/packages/inbound-email-handler/src/index.ts b/packages/inbound-email-handler/src/index.ts index d6874d866..eefadb6a3 100644 --- a/packages/inbound-email-handler/src/index.ts +++ b/packages/inbound-email-handler/src/index.ts @@ -117,8 +117,6 @@ export const inboundEmailHandler = Sentry.GCPFunction.wrapHttpFunction( } } const headers = parseHeaders(parsed.headers) - console.log('parsed: ', parsed) - console.log('headers: ', headers) // original sender email address const from = parsed['from'] From 5faa24556ebc1fabc26cb21dba91a8da8a6a55fd Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Mon, 29 Jan 2024 19:11:26 +0800 Subject: [PATCH 58/59] fix: string interpolation in logs --- packages/api/src/jobs/rss/refreshAllFeeds.ts | 2 +- packages/api/src/jobs/rss/refreshFeed.ts | 26 +++++++++----------- packages/api/src/jobs/save_page.ts | 4 +-- 3 files changed, 14 insertions(+), 18 deletions(-) diff --git a/packages/api/src/jobs/rss/refreshAllFeeds.ts b/packages/api/src/jobs/rss/refreshAllFeeds.ts index 34b373aac..5a50e41ce 100644 --- a/packages/api/src/jobs/rss/refreshAllFeeds.ts +++ b/packages/api/src/jobs/rss/refreshAllFeeds.ts @@ -86,7 +86,7 @@ const updateSubscriptionGroup = async ( try { feedURL = validateUrl(feedURL).toString() } catch (err) { - logger.error('not refreshing invalid feed url: %s', feedURL) + logger.error(`not refreshing invalid feed url: ${feedURL}`) } const jobid = `refresh-feed_${stringToHash(feedURL)}_${stringToHash( userList diff --git a/packages/api/src/jobs/rss/refreshFeed.ts b/packages/api/src/jobs/rss/refreshFeed.ts index 86917c04f..d111f248d 100644 --- a/packages/api/src/jobs/rss/refreshFeed.ts +++ b/packages/api/src/jobs/rss/refreshFeed.ts @@ -97,7 +97,7 @@ const isFeedBlocked = async (feedUrl: string) => { // if the feed has failed to fetch more than certain times, block it const maxFailures = parseInt(process.env.MAX_FEED_FETCH_FAILURES ?? '10') if (result && parseInt(result) > maxFailures) { - logger.info('feed is blocked: %s', feedUrl) + logger.info(`feed is blocked: ${feedUrl}`) return true } } catch (error) { @@ -256,7 +256,7 @@ const createTask = async ( ) => { const isRecentlySaved = await isItemRecentlySaved(userId, item.link) if (isRecentlySaved) { - logger.info('Item recently saved %s', item.link) + logger.info(`Item recently saved ${item.link}`) return true } @@ -454,11 +454,7 @@ const processSubscription = async ( let lastValidItem: RssFeedItem | null = null if (fetchResult.checksum === lastFetchedChecksum) { - logger.info( - 'feed has not been updated %s, %s', - feedUrl, - lastFetchedChecksum - ) + logger.info('feed has not been updated', { feedUrl, lastFetchedChecksum }) return } const updatedLastFetchedChecksum = fetchResult.checksum @@ -468,12 +464,12 @@ const processSubscription = async ( failedAt: Date | undefined const feedLastBuildDate = feed.lastBuildDate - logger.info('Feed last build date %s', feedLastBuildDate) + logger.info(`Feed last build date ${feedLastBuildDate || 'N/A'}`) if ( feedLastBuildDate && new Date(feedLastBuildDate) <= new Date(mostRecentItemDate) ) { - logger.info('Skipping old feed %s', feedLastBuildDate) + logger.info(`Skipping old feed ${feedLastBuildDate}`) return } @@ -526,7 +522,7 @@ const processSubscription = async ( // skip old items if (isOldItem(feedItem, mostRecentItemDate)) { - logger.info('Skipping old feed item %s', feedItem.link) + logger.info(`Skipping old feed item ${feedItem.link}`) continue } @@ -624,31 +620,31 @@ export const _refreshFeed = async (request: RefreshFeedRequest) => { try { const isBlocked = await isFeedBlocked(feedUrl) if (isBlocked) { - logger.info('feed is blocked: %s', feedUrl) + logger.info(`feed is blocked: ${feedUrl}`) throw new Error('feed is blocked') } const fetchResult = await fetchAndChecksum(feedUrl) if (!fetchResult) { - logger.error('Failed to fetch RSS feed %s', feedUrl) + logger.error(`Failed to fetch RSS feed ${feedUrl}`) await incrementFeedFailure(feedUrl) throw new Error('Failed to fetch RSS feed') } const feed = await parseFeed(feedUrl, fetchResult.content) if (!feed) { - logger.error('Failed to parse RSS feed %s', feedUrl) + logger.error(`Failed to parse RSS feed ${feedUrl}`) await incrementFeedFailure(feedUrl) throw new Error('Failed to parse RSS feed') } let allowFetchContent = true if (isContentFetchBlocked(feedUrl)) { - logger.info('fetching content blocked for feed: %s', feedUrl) + logger.info(`fetching content blocked for feed: ${feedUrl}`) allowFetchContent = false } - logger.info('Fetched feed %s at %s', feed.title, new Date()) + logger.info('Fetched feed', { title: feed.title, at: new Date() }) const fetchContentTasks = new Map() // url -> FetchContentTask // process each subscription sequentially diff --git a/packages/api/src/jobs/save_page.ts b/packages/api/src/jobs/save_page.ts index 0c933260f..778334bb3 100644 --- a/packages/api/src/jobs/save_page.ts +++ b/packages/api/src/jobs/save_page.ts @@ -250,7 +250,7 @@ export const savePageJob = async (data: Data, attemptsMade: number) => { isSaved = true } catch (e) { if (e instanceof Error) { - logger.error('error while saving page: %s', e.message) + logger.error(`error while saving page: ${e.message}`) } else { logger.error('error while saving page: unknown error') } @@ -259,7 +259,7 @@ export const savePageJob = async (data: Data, attemptsMade: number) => { } finally { const lastAttempt = attemptsMade === MAX_ATTEMPTS - 1 if (lastAttempt) { - logger.info('last attempt reached %s', data.url) + logger.info(`last attempt reached ${data.url}`) } if (taskId && (isSaved || lastAttempt)) { From 8d8309d3bb1f0137af0942c23add83822a720bdd Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Mon, 29 Jan 2024 21:22:26 +0800 Subject: [PATCH 59/59] fix: empty client request id in save-page job data when saving pdf in rss item --- packages/api/src/jobs/save_page.ts | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/packages/api/src/jobs/save_page.ts b/packages/api/src/jobs/save_page.ts index 778334bb3..1f2d276e4 100644 --- a/packages/api/src/jobs/save_page.ts +++ b/packages/api/src/jobs/save_page.ts @@ -90,7 +90,7 @@ const uploadPdf = async ( }, userId ) - if (!result.uploadSignedUrl) { + if (!result.uploadSignedUrl || !result.createdPageId) { throw new Error('error while getting upload id and signed url') } @@ -102,7 +102,11 @@ const uploadPdf = async ( if (!uploaded) { throw new Error('error while uploading pdf') } - return result.id + + return { + uploadFileId: result.id, + itemId: result.createdPageId, + } } const sendImportStatusUpdate = async ( @@ -191,12 +195,12 @@ export const savePageJob = async (data: Data, attemptsMade: number) => { // for pdf content, we need to upload the pdf if (contentType === 'application/pdf') { - const uploadFileId = await uploadPdf(url, userId, articleSavingRequestId) + const uploadResult = await uploadPdf(url, userId, articleSavingRequestId) const result = await saveFile( { url, - uploadFileId, + uploadFileId: uploadResult.uploadFileId, state: state ? (state as ArticleSavingRequestStatus) : undefined, labels, source, @@ -204,7 +208,7 @@ export const savePageJob = async (data: Data, attemptsMade: number) => { subscription: rssFeedUrl, savedAt, publishedAt, - clientRequestId: articleSavingRequestId, + clientRequestId: uploadResult.itemId, }, user )