diff --git a/packages/api/src/routers/svc/emails.ts b/packages/api/src/routers/svc/emails.ts index 070e1e359..4bb9aa6df 100644 --- a/packages/api/src/routers/svc/emails.ts +++ b/packages/api/src/routers/svc/emails.ts @@ -4,6 +4,8 @@ import { sendEmail } from '../../utils/sendEmail' import { analytics } from '../../utils/analytics' import { getNewsletterEmail } from '../../services/newsletters' import { env } from '../../env' +import { findNewsletterUrl, isProbablyNewsletter } from '../../utils/parser' +import { saveNewsletterEmail } from '../../services/save_newsletter_email' interface ForwardEmailMessage { from: string @@ -48,6 +50,21 @@ export function emailsServiceRouter() { return } + if (isProbablyNewsletter(data.html)) { + console.log('handling as newsletter', data) + await saveNewsletterEmail({ + email: data.to, + title: data.subject, + content: data.html, + author: data.from, + url: + (await findNewsletterUrl(data.html)) || + 'https://omnivore.app/no_url', + }) + res.status(200).send('Newsletter') + return + } + // get user from newsletter email const newsletterEmail = await getNewsletterEmail(data.to) diff --git a/packages/api/src/routers/svc/newsletters.ts b/packages/api/src/routers/svc/newsletters.ts index 2a668115b..463da1747 100644 --- a/packages/api/src/routers/svc/newsletters.ts +++ b/packages/api/src/routers/svc/newsletters.ts @@ -1,43 +1,13 @@ import express from 'express' -import { - createPubSubClient, - readPushSubscription, -} from '../../datalayer/pubsub' -import { - getNewsletterEmail, - updateConfirmationCode, -} from '../../services/newsletters' -import { - SaveContext, - saveEmail, - SaveEmailInput, -} from '../../services/save_email' -import { initModels } from '../../server' -import { kx } from '../../datalayer/knex_config' -import { analytics } from '../../utils/analytics' -import { env } from '../../env' -import { sendMulticastPushNotifications } from '../../utils/sendNotification' -import { getDeviceTokensByUserId } from '../../services/user_device_tokens' -import { messaging } from 'firebase-admin' -import { ContentReader } from '../../generated/graphql' -import { UserDeviceToken } from '../../entity/user_device_tokens' -import { UserArticleData } from '../../datalayer/links/model' -import { ArticleData } from '../../datalayer/article/model' -import MulticastMessage = messaging.MulticastMessage +import { readPushSubscription } from '../../datalayer/pubsub' +import { updateConfirmationCode } from '../../services/newsletters' +import { saveNewsletterEmail } from '../../services/save_newsletter_email' interface SetConfirmationCodeMessage { emailAddress: string confirmationCode: string } -interface NewsletterMessage { - email: string - content: string - url: string - title: string - author: string -} - export function newsletterServiceRouter() { const router = express.Router() @@ -111,7 +81,7 @@ export function newsletterServiceRouter() { try { // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment - const data: NewsletterMessage = JSON.parse(message) + const data = JSON.parse(message) if ( !('email' in data) || @@ -124,79 +94,15 @@ export function newsletterServiceRouter() { return } - // get user from newsletter email - const newsletterEmail = await getNewsletterEmail(data.email) - - if (!newsletterEmail) { - console.log('newsletter email not found', data.email) - res.status(200).send('Not Found') - return - } - - analytics.track({ - userId: newsletterEmail.user.id, - event: 'newsletter_email_received', - properties: { - url: data.url, - title: data.title, - author: data.author, - env: env.server.apiEnv, - }, - }) - - const ctx: SaveContext = { - models: initModels(kx, false), - pubsub: createPubSubClient(), - } - const input: SaveEmailInput = { - url: data.url, - originalContent: data.content, - title: data.title, - author: data.author, - } - - const result = await saveEmail(ctx, newsletterEmail.user.id, input) + const result = await saveNewsletterEmail(data) if (!result) { - console.log('newsletter not created:', input) - res.status(200).send(result) + console.log('Error createing newsletter link from data', data) + res.status(500).send('Error creating newsletter link') return } - // send push notification - const deviceTokens = await getDeviceTokensByUserId( - newsletterEmail.user.id - ) - - if (!deviceTokens) { - console.log('Device tokens not set:', newsletterEmail.user.id) - res.status(200).send('Device token Not Found') - return - } - - const link = await ctx.models.userArticle.getForUser( - newsletterEmail.user.id, - result.articleId - ) - - if (!link) { - console.log( - 'Newsletter link not found:', - newsletterEmail.user.id, - result.articleId - ) - res.status(200).send(result) - return - } - - if (deviceTokens.length) { - const multicastMessage = messageForLink(link, deviceTokens) - await sendMulticastPushNotifications( - newsletterEmail.user.id, - multicastMessage, - 'newsletter' - ) - } - + // We always send 200 if it was a valid message + // because we don't want the res.status(200).send('newsletter created') } catch (e) { console.log(e) @@ -211,43 +117,3 @@ export function newsletterServiceRouter() { return router } - -const messageForLink = ( - link: ArticleData & UserArticleData, - deviceTokens: UserDeviceToken[] -): MulticastMessage => { - let title = '📫 - An article was added to your Omnivore Inbox' - - if (link.author) { - title = `📫 - ${link.author} has published a new article` - } - - const pushData = !link - ? undefined - : { - link: Buffer.from( - JSON.stringify({ - id: link.id, - url: link.url, - slug: link.slug, - title: link.title, - image: link.image, - author: link.author, - isArchived: link.isArchived, - contentReader: ContentReader.Web, - readingProgressPercent: link.articleReadingProgress, - readingProgressAnchorIndex: link.articleReadingProgressAnchorIndex, - }) - ).toString('base64'), - } - - return { - notification: { - title: title, - body: link.title, - imageUrl: link.image || undefined, - }, - data: pushData, - tokens: deviceTokens.map((token) => token.token), - } -} diff --git a/packages/api/src/services/save_email.ts b/packages/api/src/services/save_email.ts index 6c4bde97c..dbb0a8ed5 100644 --- a/packages/api/src/services/save_email.ts +++ b/packages/api/src/services/save_email.ts @@ -2,7 +2,7 @@ import { PubsubClient } from '../datalayer/pubsub' import { DataModels } from '../resolvers/types' import { generateSlug, stringToHash, validatedDate } from '../utils/helpers' import { - parseMetadata, + parseUrlMetadata, parseOriginalContent, parsePreparedContent, } from '../utils/parser' @@ -44,14 +44,15 @@ export const saveEmail = async ( const slug = generateSlug(title) const pageType = parseOriginalContent(url, input.originalContent) - const metadata = await parseMetadata(url) + const metadata = await parseUrlMetadata(url) const articleToSave = { originalHtml: input.originalContent, content: content, description: metadata?.description || parseResult.parsedContent?.excerpt, - title: title, - author: input.author, + title: metadata?.title || parseResult.parsedContent?.title || title, + author: + metadata?.author || parseResult.parsedContent?.byline || input.author, url: normalizeUrl(parseResult.canonicalUrl || url, { stripHash: true, stripWWW: false, diff --git a/packages/api/src/services/save_newsletter_email.ts b/packages/api/src/services/save_newsletter_email.ts new file mode 100644 index 000000000..e00bacb91 --- /dev/null +++ b/packages/api/src/services/save_newsletter_email.ts @@ -0,0 +1,137 @@ +import { MulticastMessage } from 'firebase-admin/messaging' +import { ArticleData } from '../datalayer/article/model' +import { kx } from '../datalayer/knex_config' +import { UserArticleData } from '../datalayer/links/model' +import { createPubSubClient } from '../datalayer/pubsub' +import { UserDeviceToken } from '../entity/user_device_tokens' +import { env } from '../env' +import { ContentReader } from '../generated/graphql' +import { initModels } from '../server' +import { analytics } from '../utils/analytics' +import { sendMulticastPushNotifications } from '../utils/sendNotification' +import { getNewsletterEmail } from './newsletters' +import { SaveContext, saveEmail, SaveEmailInput } from './save_email' +import { getDeviceTokensByUserId } from './user_device_tokens' + +interface NewsletterMessage { + email: string + content: string + url: string + title: string + author: string +} + +// Returns true if the link was created successfully. Can still fail to +// send the push but that is ok and we wont retry in that case. +export const saveNewsletterEmail = async ( + data: NewsletterMessage +): Promise => { + // get user from newsletter email + const newsletterEmail = await getNewsletterEmail(data.email) + + if (!newsletterEmail) { + console.log('newsletter email not found', data.email) + return false + } + + analytics.track({ + userId: newsletterEmail.user.id, + event: 'newsletter_email_received', + properties: { + url: data.url, + title: data.title, + author: data.author, + env: env.server.apiEnv, + }, + }) + + const ctx: SaveContext = { + models: initModels(kx, false), + pubsub: createPubSubClient(), + } + const input: SaveEmailInput = { + url: data.url, + originalContent: data.content, + title: data.title, + author: data.author, + } + + const result = await saveEmail(ctx, newsletterEmail.user.id, input) + if (!result) { + console.log('newsletter not created:', input) + return false + } + + // send push notification + const deviceTokens = await getDeviceTokensByUserId(newsletterEmail.user.id) + + if (!deviceTokens) { + console.log('Device tokens not set:', newsletterEmail.user.id) + return true + } + + const link = await ctx.models.userArticle.getForUser( + newsletterEmail.user.id, + result.articleId + ) + + if (!link) { + console.log( + 'Newsletter link not found:', + newsletterEmail.user.id, + result.articleId + ) + return true + } + + if (deviceTokens.length) { + const multicastMessage = messageForLink(link, deviceTokens) + await sendMulticastPushNotifications( + newsletterEmail.user.id, + multicastMessage, + 'newsletter' + ) + } + + return true +} + +const messageForLink = ( + link: ArticleData & UserArticleData, + deviceTokens: UserDeviceToken[] +): MulticastMessage => { + let title = '📫 - An article was added to your Omnivore Inbox' + + if (link.author) { + title = `📫 - ${link.author} has published a new article` + } + + const pushData = !link + ? undefined + : { + link: Buffer.from( + JSON.stringify({ + id: link.id, + url: link.url, + slug: link.slug, + title: link.title, + image: link.image, + author: link.author, + isArchived: link.isArchived, + contentReader: ContentReader.Web, + readingProgressPercent: link.articleReadingProgress, + readingProgressAnchorIndex: link.articleReadingProgressAnchorIndex, + }) + ).toString('base64'), + } + + return { + notification: { + title: title, + body: link.title, + imageUrl: link.image || undefined, + }, + data: pushData, + tokens: deviceTokens.map((token) => token.token), + } +} diff --git a/packages/api/src/utils/axios-handler.ts b/packages/api/src/utils/axios-handler.ts index dbe4b5619..6e44e1e90 100644 --- a/packages/api/src/utils/axios-handler.ts +++ b/packages/api/src/utils/axios-handler.ts @@ -3,6 +3,7 @@ import { DOMWindow } from 'jsdom' export class AxiosHandler { name = 'axios' + // eslint-disable-next-line @typescript-eslint/no-unused-vars shouldPrehandle = (url: URL, _dom: DOMWindow): boolean => { const host = this.name + '.com' // check if url ends with axios.com diff --git a/packages/api/src/utils/createTask.ts b/packages/api/src/utils/createTask.ts index 20b9fbf3d..75eed7395 100644 --- a/packages/api/src/utils/createTask.ts +++ b/packages/api/src/utils/createTask.ts @@ -170,6 +170,7 @@ export const getTask = async ( const request: protos.google.cloud.tasks.v2.GetTaskRequest = { responseView: View.FULL, + // eslint-disable-next-line @typescript-eslint/no-explicit-any toJSON(): { [p: string]: any } { return {} }, diff --git a/packages/api/src/utils/golang-handler.ts b/packages/api/src/utils/golang-handler.ts index cc9a6e368..3a3037a59 100644 --- a/packages/api/src/utils/golang-handler.ts +++ b/packages/api/src/utils/golang-handler.ts @@ -3,6 +3,7 @@ import { DOMWindow } from 'jsdom' export class GolangHandler { name = 'golangweekly' + // eslint-disable-next-line @typescript-eslint/no-unused-vars shouldPrehandle = (url: URL, _dom: DOMWindow): boolean => { const host = this.name + '.com' // check if url ends with golangweekly.com diff --git a/packages/api/src/utils/parser.ts b/packages/api/src/utils/parser.ts index ecac81caf..c32b539ed 100644 --- a/packages/api/src/utils/parser.ts +++ b/packages/api/src/utils/parser.ts @@ -346,16 +346,15 @@ const getJSONLdLinkMetadata = async ( } type Metadata = { + title?: string + author?: string description: string previewImage: string } -export const parseMetadata = async ( - url: string -): Promise => { +export const parsePageMetadata = (html: string): Metadata | undefined => { try { - const res = await axios.get(url) - const window = new JSDOM(res.data).window + const window = new JSDOM(html).window // get open graph metadata const description = @@ -368,9 +367,99 @@ export const parseMetadata = async ( .querySelector("head meta[property='og:image']") ?.getAttribute('content') || '' - return { description: description, previewImage: previewImage } + const title = + window.document + .querySelector("head meta[property='og:title']") + ?.getAttribute('content') || undefined + + const author = + window.document + .querySelector("head meta[name='author']") + ?.getAttribute('content') || undefined + + // TODO: we should be able to apply the JSONLD metadata + // here too + + return { title, author, description, previewImage } } catch (e) { - console.log('failed to got:', url, e) + console.log('failed to parse page:', html, e) return undefined } } + +export const parseUrlMetadata = async ( + url: string +): Promise => { + try { + const res = await axios.get(url) + return parsePageMetadata(res.data) + } catch (e) { + console.log('failed to get:', url, e) + return undefined + } +} + +// Attempt to determine if an HTML blob is a newsletter +// based on it's contents. +// TODO: when we consolidate the handlers we could include this +// as a utility method on each one. +export const isProbablyNewsletter = (html: string): boolean => { + const dom = new JSDOM(html).window + const domCopy = new JSDOM(dom.document.documentElement.outerHTML) + const article = new Readability(domCopy.window.document, { + debug: false, + keepTables: true, + }).parse() + + if (!article || !article.content) { + return false + } + + // substack newsletter emails have tables with a *post-meta class + if (dom.document.querySelector('table[class$="post-meta"]')) { + return true + } + + // If the article has a header link, and substack icons its probably a newsletter + const href = findNewsletterHeaderHref(dom.window) + const heartIcon = dom.document.querySelector( + 'table tbody td span a img[src*="HeartIcon"]' + ) + const recommendIcon = dom.document.querySelector( + 'table tbody td span a img[src*="RecommendIconRounded"]' + ) + if (href && (heartIcon || recommendIcon)) { + return true + } + + return false +} + +const findNewsletterHeaderHref = (dom: DOMWindow): string | undefined => { + const postLink = dom.document.querySelector('h1 a ') + if (postLink) { + return postLink.getAttribute('href') || undefined + } + return undefined +} + +// Given an HTML blob tries to find a URL to use for +// a canonical URL. +export const findNewsletterUrl = async ( + html: string +): Promise => { + const dom = new JSDOM(html).window + const href = findNewsletterHeaderHref(dom.window) + if (href) { + // Try to make a HEAD request so we get the redirected URL, since these + // will usually be behind tracking url redirects + return axios({ + method: 'HEAD', + url: href, + }) + .then((res) => res.request.res.responseUrl as string | undefined) + .catch((e) => href) + } + + return undefined +} diff --git a/packages/api/src/utils/usernamePolicy.ts b/packages/api/src/utils/usernamePolicy.ts index fa64c469e..20bd5fe32 100644 --- a/packages/api/src/utils/usernamePolicy.ts +++ b/packages/api/src/utils/usernamePolicy.ts @@ -94,6 +94,7 @@ const RESERVED_NAMES = new Set([ 'mine', 'mis', 'news', + 'no_url', 'oauth', 'oauth_clients', 'offers', diff --git a/packages/api/src/utils/wikipedia-handler.ts b/packages/api/src/utils/wikipedia-handler.ts index 153a6cd0e..05fc4b5d4 100644 --- a/packages/api/src/utils/wikipedia-handler.ts +++ b/packages/api/src/utils/wikipedia-handler.ts @@ -3,6 +3,7 @@ import { DOMWindow } from 'jsdom' export class WikipediaHandler { name = 'wikipedia' + // eslint-disable-next-line @typescript-eslint/no-unused-vars shouldPrehandle = (url: URL, _dom: DOMWindow): boolean => { return url.hostname.endsWith('wikipedia.org') } diff --git a/packages/api/test/services/save_newsletter_email.test.ts b/packages/api/test/services/save_newsletter_email.test.ts new file mode 100644 index 000000000..bf29de3ac --- /dev/null +++ b/packages/api/test/services/save_newsletter_email.test.ts @@ -0,0 +1,44 @@ +import 'mocha' +import { expect } from 'chai' +import 'chai/register-should' +import { + createTestUser, + deleteTestUser, +} from '../db' +import { createNewsletterEmail } from '../../src/services/newsletters' +import { saveNewsletterEmail } from '../../src/services/save_newsletter_email' +import { getRepository } from 'typeorm' +import { Link } from '../../src/entity/link' + +describe('saveNewsletterEmail', () => { + const username = 'fakeUser' + after(async () => { + await deleteTestUser(username) + }) + + it('adds the newsletter to the library', async () => { + const user = await createTestUser(username) + const email = await createNewsletterEmail(user.id) + + await saveNewsletterEmail({ + email: email.address, + content: 'fake content', + url: 'https://example.com', + title: 'fake title', + author: 'fake author', + }) + + const links = await getRepository(Link).find({ + where: { + user: user, + }, + relations: ['page'], + }) + + expect(links.length).to.equal(1) + expect(links[0].page.url).to.equal('https://example.com') + expect(links[0].page.title).to.equal('fake title') + expect(links[0].page.author).to.equal('fake author') + expect(links[0].page.content).to.contain('fake content') + }).timeout(10000) +}) diff --git a/packages/api/test/utils/data/substack-forwarded-newsletter.html b/packages/api/test/utils/data/substack-forwarded-newsletter.html new file mode 100644 index 000000000..ebfa93cee --- /dev/null +++ b/packages/api/test/utils/data/substack-forwarded-newsletter.html @@ -0,0 +1 @@ +


---------- Forwarded message ---------
From: Andrew Methven <slowchinese@substack.com>
Date: Fri, Feb 18, 2022 at 11:57 PM
Subject: Companies that eat people
To: <XXXXXXXXXX@gmail.com>


Slow Chinese 每周漫闻 ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌

Companies that eat people

Slow Chinese 每周漫闻

Andrew Methven
Feb 19

The phrase, ‘eating people’ (吃人 chī rén), is used to criticise companies in China that exploit their employees.

It’s originally from Lǔ Xùn’s (鲁迅), A Madman's Diary (狂人日记 kuángrén rìjì), published in 1918:

我翻开历史一查，这历史没有年代。歪歪斜斜的每页上都写着“仁义道德”几个字，我横竖睡不着，仔细看了半夜，才从字缝里看出来，满本上都写着两个字“吃人"！

As I look through the pages of history, I see there are no dates. On each page, written messily, are the characters, ‘benevolence and morality’. I can’t sleep. I read into the night. Finally, I find hidden between the characters across the page, the words, ‘eating people’.

The times have changed since Lu Xun made that observation more than 100 years ago, but the culture of ‘eating people’ has not, according to social media comments this week, such as this one:

吃人的事实，从来没有变过，历朝历代都是如此 - The reality of [companies] exploiting their employees is nothing new. It’s been the same throughout history.

Two of China’s biggest tech companies, Tencent and Bilibili, have recently been accused of ‘eating people’, abusing and exploiting their staff.

So that’s what we discuss this week.

  • Conversations worth consuming: interview with Zhāng Yìfēi 张义飞 a former employee of Tencent

  • Words of the week: coverage and social media commentary of a recent death of a Bilibili employee allegedly due to overwork.

The audio version of this newsletter is already live - become a member to access it in your podcast app!

Use this link to claim a one-month free trial of the membership to give the full experience a go:

One-month free trial

1. CONVERSATIONS WITH CONSUMING

腾讯带头“反内卷”：光子工作室拒绝996，保障双休_游戏

Interview with Zhang Yifei

Two weeks ago a 25-year-old programmer at Tencent, Zhāng Yìfēi 张义飞, became an Internet sensation after standing up to his bosses at the company. He announced in an internal group chat that he was quitting his job, which then went viral on social media.

If 20-hour days is what the company wants, he wrote, ‘I’ll resign tomorrow’

36Kr interviewed Zhang this week (in Chinese). He talks more about the overtime culture at Tencent, and why he dared to take on his company in such a public way - he already had another job lined up.

There are some excellent words in his description of life as a working person at Tencent.

Useful words

  • 卡 kǎ - stop, block

    什么时候离职的？有人卡你吗 - When did you leave your job? Did they try to stop you?

  • 剥削 bō xuē - exploit

    加班严重、996工作制、互联网巨头压榨剥削员工等话题再次被拿来讨论

    - Topics such as serious overtime, the 996 work system, and the exploitation of employees by Internet giants are being discussed again.

  • 底气 dǐ qì - confidence, back up

    自己已经提前拿到其他公司的offer，比较有底气 - I already had an offer from another company, so I was relatively confident about doing it.

  • 忌惮 jì dàn - fear, be afraid of

    如果一些互联网大厂因此忌惮、不录用我，我正好也不想去这种加班严重的地方 - If some big Internet companies are afraid to hire me, that’s fine by me. I also don't want to work in a company with such heavy overtime.

  • 手软 shǒu ruǎn - ‘soft hand’, forgiving

    不要特立独行，搞小团体，否则他不会手软 - Do not march to a different beat or form small cliques. He will come down hard on this kind of behaviour.

  • 打硬仗 dǎ yìng zhàng - fight a hard war

    张小龙管理下的企业微信，经常会强调用小而精的团队打硬仗 - The company Wechat, under Zhang Xiaolong’s management, would often emphasise using a small and efficient team to work on tight deadlines.

    • Note: a common phrase used in Chinese companies when a team is working intensely on a project or against a ridiculous deadline.

    • Related: 打胜仗 dǎ shèngzhàng - win a war

  • 喊口号 hǎn kǒuhào - shouting slogans

    但大家普遍的看法是，不想看到空洞地喊口号，只想看到具体行为 - The general view is they don’t want to see people shouting empty slogans. They want to see action.

Idioms

  • 初出茅庐 chūchū máolú - ‘just come out of the thatched cottage’; inexperienced, wet behind the ears

    但对于大众而言，互联网巨头和初出茅庐的应届生，相比较下毕竟力量悬殊 - There’s no comparison between the power of the big internet companies and graduate employees with no experience.

    • More: 悬殊 xuán shū - disparity

    • More: first covered in 3 April newsletter last year.

  • 昏昏沉沉 hūnhūn chénchén - feeling sleepy

    来这里入职两个月，感到昏昏沉沉，记忆力下降很多

    - I’ve been here for two months. I feel tired and my memory has declined a lot.

  • 热火朝天 rèhuǒ cháotiān - ‘hot fire face sky’; vigorously, with energy

    到点的时候，差不多一半人还没走，都在热火朝天地讨论工作

    - When it was time to finish at the end of the day, around half of the team stayed behind to talk energetically about their work.

2. WORDS OF THE WEEK

上海之旅-前往Bilibili总部！ - 哔哩哔哩

Bilibili eats people

A man who headed a content moderation department at Chinese video-streaming site Bilibili died last week after suffering a cerebral hemorrhage while working a Chinese New Year holiday shift.

The company was heavily criticised (Sohu - in Chinese) of having a toxic work culture.

One of the top comments on social media adapted the line from Lu Xun’s A Madman’s Diary. But instead of looking through the pages of history, overworked netizens find the same message hidden in their payslips:

我翻开工资单一查，这工资单没有工资，歪歪斜斜的每条都写着“迟到扣款”四个字。我横竖睡不着，仔细看了半夜，才从字缝里看出来，满本上都写着两个字“吃人"！

I glance at my payslip. I don’t see any pay on it. All I see are the four characters scrawled across the page: ‘fined for being late’. I can’t sleep. I look at it deep into the night. I finally find hidden between the characters across the page, written the words: ‘eating people’.

The words shared below are from the Sohu article and also from social media comments.

Useful words

  • 猝死 cù sǐ - sudden death, die suddenly

    他2月5日凌晨脑出血猝死 - He died suddenly in the early hours of the morning on 5 February.

  • 嗝屁 gé pì - hiccup, to die

    对大企业嗝屁几个不算啥，对于各个家庭你就是唯一呀 - A few people dying means nothing to a big company, but for a family it’s their only child!

    • Related: 翘辫子 qiào biàn zi - make braids - kick the bucket (Qing Dynasty reference relating to when men had to remove their braids).

      他因为加班严重而翘辫子了 - He died because of too much overtime.

  • 企图 qǐ tú - try to, seek to do something (negative)

    通过各种企图将这件事压下来，我决定发声 - Bilibili attempted to suppress the situation through different means, so I decided to speak up.

    • Note: similar to 试图 shì tú, but more negative connotations

  • 腐朽 fǔ xiǔ - degenerate, rotten

    道出了资本家的腐朽和恶臭 - Reeks of the stench and rot of capitalism.

  • 压垮 yā kuǎ - crush

    就是无情的压榨现有劳动力，能压垮一个是一个，多招一个人算我输 - It’s the callous exploitation of the current employees. The company tries to squeeze as much as possible from each and every one of them rather than hiring one more employee.

Idioms

  • 血汗工厂 xuèhàn gōngchǎng - ‘blood sweat factory’ - sweat shop

    B站因员工猝死一事，被推进了“血汗工厂”的舆论漩涡 - Bilibili has been dragged into a public debate about the company being a ‘sweatshop’ due to the sudden death of an employee.

    • Note: The pronunciation of 血 is normally xiě in colloquial phrases, and xuè in technical terms. But the rule is vague and not very helpful. In this phrase it's always xuè. But in 血汗钱 xiěhàn qián, ‘hard earned money’, xiě is more common. So confusing!

  • 混淆视听 hùnxiáo shìtīng - to muddle or confuse an issue

    晚9到早9确实不属于加班，因为是大夜班的正常时间，大厂就这样混淆视听？ - 9pm to 9am does not count as overtime. But that’s because the night shift is a normal working shift for these big tech companies. They are muddling up the matter.

  • 枯燥乏味 kūzào fáwèi - boring

    做审核的确工作强度很大，而且枯燥乏味 - Being a content moderator is a very intense job. It’s also extremely boring.

    • More: 枯燥无味 kūzào wúwèi - boring (same meaning)

  • 恬不知耻 tián bù zhī chǐ - shameless

    觉得正常吗？居然还能如此恬不知耻的说“没有让他加班” - Is this normal? How can they be so shameless in saying the company did not ‘ask him to work overtime’?

  • 难上加难 nán shàng jiā nán - very difficult

    只要企业做大了，普通职工想维权难上加难 - When the company gets big it’s almost impossible for employees to protect their rights.

    • Related: 雪上加霜 xuěshàng jiāshuāng - make matters worse

Colloquial phrases

  • 万变不离其宗 wàn biàn bùlí qízōng - make ten thousand changes but remain the same in essence

    好像这些大公司公关都是万变不离其宗，核心就是推卸责任！- It seems the PR of these big companies tells a nice story, but in essence they don’t change. They are merely avoiding their responsibility.

  • 不见棺材不落泪 bùjiàn guāncai bù luò lèi - won’t cry until they see the coffin

    这也说得出口啊！真是不见血不掉泪啊 - They actually say this? Do they really have to let somebody die before they accept they are in the wrong?

    • More: I wrote more about this colloquialism in SupChina’s phrase of the week.

    • Related: 不到黄河心不死 bù dào huánghé xīn bù sǐ - not to stop until one reaches the Yellow River; refuse to give up until all hope is gone

3. RECOMMENDATIONS

Become a member of the community

As a member of the community you get access to unique resources to help you master modern Mandarin, learn, use, and understand Chinese language the way people speak it today.

  • 📚 Resources: Pleco downloads, word lists, and example sentences print-outs and audio download for each issue.

  • 🔉 Audio: audio version of the newsletter delivered as a member-only podcast every Saturday morning (before the free newsletter is published)

  • 🤓 Archive: full database of all words and phrases in the archive (nearly 1,300!) searchable according to word-type, sector and topic with audio and example sentences for each entry, updated weekly.

Use this link to claim a one-month free trial of the membership to give the full experience a go.

One-month free trial

That’s it for this week.

I look forward to seeing you in your inbox same time next weekend.

Andrew

+++

ps - please do share this newsletter on your social channels and with your networks

Share Slow Chinese 每周漫闻

Like

© 2022 Andrew Methven Unsubscribe
548 Market Street PMB 72296, San Francisco, CA 94104

Publish on Substack

\ No newline at end of file diff --git a/packages/api/test/utils/data/substack-forwarded-welcome-email.html b/packages/api/test/utils/data/substack-forwarded-welcome-email.html new file mode 100644 index 000000000..f975441aa --- /dev/null +++ b/packages/api/test/utils/data/substack-forwarded-welcome-email.html @@ -0,0 +1 @@ +


---------- Forwarded message ---------
From: Andrew Methven <slowchinese@substack.com>
Date: Thu, Dec 9, 2021 at 11:27 PM
Subject: How can Slow Chinese 每周漫闻 help you?
To: <XXXXXXXXXX@gmail.com>


Thank you for subscribing to for Slow Chinese 每周漫闻 ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌

Thanks so much for subscribing to Slow Chinese 每周漫闻 and welcome aboard!

I’m excited to help you improve and practice your Chinese language skills.

Here’s a quick way I can help:

Reply to this email and tell me about your story of learning Chinese and what challenges you currently have with the language.

I’ll reply with a specific suggestion to help you.

Also, to make sure the next issue of the newsletter doesn’t land in your spam folder, add my email address to your contacts.

Thanks!

Andrew

© 2021 Andrew Methven Unsubscribe
548 Market Street PMB 72296, San Francisco, CA 94104

Publish on Substack

\ No newline at end of file diff --git a/packages/api/test/utils/data/substack-post.html b/packages/api/test/utils/data/substack-post.html new file mode 100644 index 000000000..bcaf32931 --- /dev/null +++ b/packages/api/test/utils/data/substack-post.html @@ -0,0 +1,385 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Code Block Syntax Highlighting - Omnivore + + + + + + + + + + + + + + + + + + + + + + + + + +

Omnivore

Share this post
Code Block Syntax Highlighting
blog.omnivore.app

Code Block Syntax Highlighting

Highlighted <code> in Omnivore

Omnivore
Feb 28
1		Share
  • Edit post
  • Pin on home page
  • Exclude from Top

After open sourcing Omnivore, we received some great feature suggestions from our community. One of them was syntax highlighting on posts with code.

This weekend we added support for highlighting of <code> blocks for Web and iOS users using highlight.js. Try it out by saving a blog post with code snippets or a GitHub README.

Screenshot of Omnivore’s code block syntax highlighting

If you’d like to join our community please star us on GitHub, and/or join our Discord.

1		ShareShare
TopNew
Getting Started with OmnivoreLearn the best ways to save links with Omnivore
Omnivore
Oct 13, 2021
2		Share
Share this post
Getting Started with Omnivore
blog.omnivore.app
  • Edit post
  • Pin on home page
  • Exclude from Top
New PDF ViewerToday we are happy to launch our new PDF viewer. It is available in our latest iOS release (1.3.0) and on the web. The new PDF viewer supports…
Omnivore
Nov 12, 2021
Share
Share this post
New PDF Viewer
blog.omnivore.app
  • Edit post
  • Pin on home page
  • Exclude from Top
Our new distraction free PDF readerDistraction free PDF reading, in-app feedback, and content improvements
Omnivore
Nov 18, 2021
Share
Share this post
Our new distraction free PDF reader
blog.omnivore.app
  • Edit post
  • Pin on home page
  • Exclude from Top
Our updated web appFaster article and library loading
Omnivore
Jan 5
Share
Share this post
Our updated web app
blog.omnivore.app
  • Edit post
  • Pin on home page
  • Exclude from Top
October 27th UpdatesUnarchive on iOS, new keyboard shortcuts, and reader improvements
Omnivore
Oct 27, 2021
Share
Share this post
October 27th Updates
blog.omnivore.app
  • Edit post
  • Pin on home page
  • Exclude from Top
Highlight PDFs on iOSand we are hiring!
Omnivore
Dec 2, 2021
Share
Share this post
Highlight PDFs on iOS
blog.omnivore.app
  • Edit post
  • Pin on home page
  • Exclude from Top
November 5th UpdatesFaster reader loading and simpler controls on iOS
Omnivore
Nov 5, 2021
Share
Share this post
November 5th Updates
blog.omnivore.app
  • Edit post
  • Pin on home page
  • Exclude from Top
Highlight your PDFsAdd and share highlights from your saved PDFs
Omnivore
Nov 24, 2021
Share
Share this post
Highlight your PDFs
blog.omnivore.app
  • Edit post
  • Pin on home page
  • Exclude from Top
© 2022 Omnivore
Privacy ∙ Terms ∙ Collection notice
Publish on Substack
Substack is the home for great writing
+ +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/packages/api/test/utils/data/substack-private-forwarded-newsletter.html b/packages/api/test/utils/data/substack-private-forwarded-newsletter.html new file mode 100644 index 000000000..071bfeb77 --- /dev/null +++ b/packages/api/test/utils/data/substack-private-forwarded-newsletter.html @@ -0,0 +1,2 @@ +


---------- Forwarded message ---------
From: giggs <darkgiggsxx@gmail.com>
Date: Wed, Mar 2, 2022 at 5:29 PM
Subject: Fwd: The German Retreat From Nuclear Power
To: Radek <radoslaw.jurga@gmail.com>



---------- Forwarded message ---------
De : Bismarck Analysis <bismarck@substack.com>
Date: mer. 2 mars 2022 à 15:02
Subject: The German Retreat From Nuclear Power
To: <darkgiggsxx@gmail.com>


Germany's economic priorities are torn between industrial growth and energy degrowth. As its last nuclear plants are shut down, the choice is between relying on fossil fuels or closing factories. ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌

The German Retreat From Nuclear Power

Germany's economic priorities are torn between industrial growth and energy degrowth. As its last nuclear plants are shut down, the choice is between relying on fossil fuels or closing factories.

Samo Burja
Mar 2Share
Isar Nuclear Power Plant near Landshut, Germany in 2016. The Isar station is scheduled to be shut down by the end of 2022. Photo by Dennis Hansch. Source.
+
\ No newline at end of file diff --git a/packages/api/test/utils/parser.test.ts b/packages/api/test/utils/parser.test.ts new file mode 100644 index 000000000..5d20a24ba --- /dev/null +++ b/packages/api/test/utils/parser.test.ts @@ -0,0 +1,51 @@ +import 'mocha' +import { expect } from 'chai' +import 'chai/register-should' +import { JSDOM } from 'jsdom' +import fs from 'fs' +import { findNewsletterUrl, isProbablyNewsletter, parsePageMetadata } from '../../src/utils/parser' + +const load = (path: string): string => { + return fs.readFileSync(path, 'utf8') +} + +describe('isProbablyNewsletter', () => { + it('returns true for substack newsletter', () => { + const html = load('./test/utils/data/substack-forwarded-newsletter.html') + isProbablyNewsletter(html).should.be.true + }) + it('returns true for private forwarded substack newsletter', () => { + const html = load('./test/utils/data/substack-private-forwarded-newsletter.html') + isProbablyNewsletter(html).should.be.true + }) + it('returns false for substack welcome email', () => { + const html = load('./test/utils/data/substack-forwarded-welcome-email.html') + isProbablyNewsletter(html).should.be.false + }) +}) + +describe('findNewsletterUrl', async () => { + it('gets the URL from the header if it is a newsletter', async () => { + const html = load('./test/utils/data/substack-forwarded-newsletter.html') + const url = await findNewsletterUrl(html) + // Not sure if the redirects from substack expire, this test could eventually fail + expect(url).to.startWith('https://newsletter.slowchinese.net/p/companies-that-eat-people-217') + }) + it('returns undefined if it is not a newsletter', async () => { + const html = load('./test/utils/data/substack-forwarded-welcome-email.html') + const url = await findNewsletterUrl(html) + expect(url).to.be.undefined + }) +}) + +describe('parseMetadata', async () => { + it('gets author, title, image, description', async () => { + const html = load('./test/utils/data/substack-post.html') + const metadata = await parsePageMetadata(html) + expect(metadata?.author).to.deep.equal('Omnivore') + expect(metadata?.title).to.deep.equal('Code Block Syntax Highlighting') + expect(metadata?.previewImage).to.deep.equal('https://cdn.substack.com/image/fetch/w_1200,h_600,c_fill,f_jpg,q_auto:good,fl_progressive:steep,g_auto/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2F2ab1f7e8-2ca7-4011-8ccb-43d0b3bd244f_1490x2020.png') + expect(metadata?.description).to.deep.equal('Highlighted in Omnivore') + + }) +})