diff --git a/packages/api/src/resolvers/article/index.ts b/packages/api/src/resolvers/article/index.ts index a7cd6c906..40d1ca256 100644 --- a/packages/api/src/resolvers/article/index.ts +++ b/packages/api/src/resolvers/article/index.ts @@ -68,6 +68,7 @@ import { validatedDate, } from '../../utils/helpers' import { + getDistillerResult, htmlToMarkdown, ParsedContentPuppeteer, parsePreparedContent, @@ -106,6 +107,12 @@ import { saveSearchHistory } from '../../services/search_history' import { parsedContentToPage } from '../../services/save_page' import * as httpContext from 'express-http-context' +enum ArticleFormat { + Markdown = 'markdown', + Html = 'html', + Distiller = 'distiller', +} + export type PartialArticle = Omit< Article, | 'updatedAt' @@ -413,7 +420,9 @@ export const getArticleResolver: ResolverFn< return { errorCodes: [ArticleErrorCode.Unauthorized] } } - const includeOriginalHtml = !!graphqlFields(info).article.originalHtml + const includeOriginalHtml = + format === ArticleFormat.Distiller || + !!graphqlFields(info).article.originalHtml analytics.track({ userId: claims?.uid, @@ -449,8 +458,20 @@ export const getArticleResolver: ResolverFn< page.content = UNPARSEABLE_CONTENT } - if (format === 'markdown') { + if (format === ArticleFormat.Markdown) { page.content = htmlToMarkdown(page.content) + } else if (format === ArticleFormat.Distiller) { + if (!page.originalHtml) { + return { errorCodes: [ArticleErrorCode.BadData] } + } + const distillerResult = await getDistillerResult( + claims.uid, + page.originalHtml + ) + if (!distillerResult) { + return { errorCodes: [ArticleErrorCode.BadData] } + } + page.content = distillerResult } return { diff --git a/packages/api/src/utils/parser.ts b/packages/api/src/utils/parser.ts index 8fd3a4d30..93c9eaffc 100644 --- a/packages/api/src/utils/parser.ts +++ b/packages/api/src/utils/parser.ts @@ -21,8 +21,12 @@ import { findEmbeddedHighlight, } from './highlightGenerator' import { NodeHtmlMarkdown } from 'node-html-markdown' +import { promisify } from 'util' +import * as jwt from 'jsonwebtoken' +import { env } from '../env' const logger = buildLogger('utils.parse') +const signToken = promisify(jwt.sign) export const ALLOWED_CONTENT_TYPES = [ 'text/html', @@ -485,3 +489,31 @@ const nhm = new NodeHtmlMarkdown( export const htmlToMarkdown = (html: string) => { return nhm.translate(/* html */ html) } + +export const getDistillerResult = async ( + uid: string, + html: string +): Promise => { + try { + const url = process.env.DISTILLER_URL + if (!url) { + console.log('No distiller url') + return undefined + } + + const exp = Math.floor(Date.now() / 1000) + 60 * 60 // 1 hour + const auth = (await signToken({ uid, exp }, env.server.jwtSecret)) as string + + console.debug('Parsing by distiller', url) + const response = await axios.post(url, html, { + headers: { + Authorization: auth, + }, + timeout: 5000, + }) + return response.data + } catch (e) { + console.log('Error parsing by distiller', e) + return undefined + } +} diff --git a/packages/readabilityjs/test/test-pages/aboveavalon/distiller.html b/packages/readabilityjs/test/test-pages/aboveavalon/distiller.html new file mode 100644 index 000000000..48862100c --- /dev/null +++ b/packages/readabilityjs/test/test-pages/aboveavalon/distiller.html @@ -0,0 +1,87 @@ +

+ In a few months, the number of people wearing an Apple Watch will surpass 100 million. While the tech press spent years infatuated with stationary smart speakers and the idea of voice-only interfaces, it was the Apple Watch and utility on the wrist that ushered in a new paradigm shift in computing. We are now seeing Apple leverage the growing number of Apple Watch wearers to build a formidable health platform. The Apple Watch is a runaway train with no company in a position to slow it down. +

+ Mirages and Head Fakes +

+ We are coming off of a weird stretch for the tech industry. As smartphone sales growth slowed in the mid-2010s, companies, analysts, and pundits began to search for the next big thing. The search landed on stationary smart speakers and voice interfaces. +

+ Companies who weren’t able to leverage the smartphone revolution with their own hardware placed massive bets on digital voice assistants that would supposedly usher in the end of the smartphone era. These digital voice assistants would be delivered to consumers via cheap stationary speakers placed in the home. Massive PR campaigns were launched that attempted to convince people about this post-smartphone future. Unfortunately for these companies, glowing press coverage cannot hide a product category’s fundamental design shortcomings.  +

+ At nearly every turn, Apple was said to be missing the voice train because of a dependency on iPhone revenue. Management was said to suffer from tunnel vision while the company’s approach to privacy was positioned as a long-term headwind that would lead to inferior results in AI relative to the competition. Simply put, Apple was viewed as losing control of where technology was headed following the mobile revolution. +

+ There were glaring signs that narratives surrounding smart speakers and Apple lacking a coherent strategy for the future were off the mark. In November 2017, I wrote the following in an article titled, “A Stationary Smart Speaker Mirage”: +

+ “On the surface, Amazon Echo sales point to a burgeoning product category. A 15M+ annual sales pace for a product category that is only three years old is quite the accomplishment. This has led to prognostications of stationary smart speakers representing a new paradigm in technology. However, relying too much on Echo sales will lead to incomplete or faulty conclusions. The image portrayed by Echo sales isn't what it seems. In fact, it is only a matter of time before it becomes clear the stationary home speaker is shaping up to be one of the largest head fakes in tech. We are already starting to see early signs of disappointment begin to appear… +

+ I don’t think stationary smart speakers represent the future of computing. Instead, companies are using smart speakers to take advantage of an awkward phase of technology in which there doesn’t seem to be any clear direction as to where things are headed. Consumers are buying cheap smart speakers powered by digital voice assistants without having any strong convictions regarding how such voice assistants should or can be used. The major takeaway from customer surveys regarding smart speaker usage is that there isn’t any clear trend. If anything, smart speakers are being used for rudimentary tasks that can just as easily be done with digital voice assistants found on smartwatches or smartphones. This environment paints a very different picture of the current health of the smart speaker market. The narrative in the press is simply too rosy and optimistic. +

+ Ultimately, smart speakers end up competing with a seemingly unlikely product category: wearables.” +

+ Three years later, I wouldn’t change one thing found in the preceding three paragraphs. The smart speaker bubble popped less than 12 months after publishing that article. The product category no longer has a buzz factor, and despite the hopes of Amazon and Google, people are not using stationary speakers for much else besides listening to music and rudimentary tasks like setting kitchen timers. +

+ The primary problem found with voice is that it’s not a great medium for transferring a lot of data, information, and context. As a result, companies like Amazon have needed to dial back their grandiose vision for voice-first and voice-only paradigms. Last week’s Amazon hardware event highlighted a growing bet on screens – a complete reversal from the second half of the 2010s.  +

+ Betting on the Wrist  +

+ As companies who missed the smartphone boat were placing bets on stationary speakers, Apple was placing a dramatically different bet on a small device with a screen. This device wouldn’t be stationary but instead push the definition of mobile by being worn on the wrist. +

+ Jony Ive, who is credited with leading Apple’s push into wrist wearables, referred to the wrist as “the obvious and right place” for a different kind of computer.  +

+ When Apple unveiled the Apple Watch in 2014, wearable computing on the wrist was more of a promise than anything else. Apple created an entirely new industry – something that isn’t found much in the traditional Apple playbook.  +

+ After years of deep skepticism and cynicism, consensus reaction towards Apple Watch has changed and is now positive. Much of this is due to the fact that it’s impossible to miss Apple Watches appearing on wrists around the world. According to my estimates, approximately 35% of iPhone users in the U.S. now wear an Apple Watch. This is a shockingly high percentage for a five-year-old product category, and it says a lot about how Apple’s intuition about the wrist was right. +

+ Apple Watch Installed Base  +

+ The number of people wearing an Apple Watch continues to steadily increase. According to my estimate, there were 81 million people wearing an Apple Watch as of the end of June. According to Apple, 75% of Apple Watch sales are going to first-time customers. This means that 23 million people will have bought their first Apple Watch in 2020. To put that number in context, there are about 25 million people wearing a Fitbit. The Apple Watch installed base is increasing by the size of Fitbit’s overall installed base every 12 months. Exhibit 1 highlights the change in the Apple Watch installed base over the years.  +

+ Exhibit 1: Apple Watch Installed Base (number of people wearing an Apple Watch) +

Apple Watch Installed Base (Above Avalon)
View fullsize

+ (The calculations and methodology used to reach my Apple Watch installed base estimates is available here for Above Avalon members.) +

+ Deriving Power +

+ From where is Apple Watch deriving its momentum? The answer is found in The Grand Unified Theory of Apple Products.  +

The Grand Unified Theory of Apple Products (Above Avalon)
View fullsize

+ One of the core tenets of my theory is that an Apple product category's design is tied to the role it is meant to play relative to other Apple products. The Apple Watch is designed to handle a growing number of tasks once given to the iPhone. Meanwhile, the iPhone is designed to handle a growing number of tasks given to the iPad. One can continue this exercise to cover all of Apple's major product categories. +

+ Apple Watch is not an iPhone replacement because there are things done on an iPhone that can't be done on an Apple Watch. This ends up being a feature, not a bug. The Apple Watch’s design then allows the product to handle entirely new tasks that can’t be handled on an iPhone. This latter attribute goes a long way in explaining how Apple Watch has helped usher in a new paradigm shift in computing. Apple Watch wearers are able to interact with technology differently. +

+ (More on The Grand Unified Theory of Apple Products is found in the Above Avalon Report, “Product Vision: How Apple Thinks About the World,” available here for Above Avalon members.) +

+ A Health Platform +

+ In January 2019, Tim Cook surprised many by saying Apple will be remembered more for its contributions to health than for any other reason. Here’s Cook:  +

+ “I believe, if you zoom out into the future, and you look back, and you ask the question, ‘What was Apple’s greatest contribution to mankind?’ it will be about health.” +

+ Many assumed that Cook’s comment hinted at Apple unveiling a portfolio of medical-grade devices that would go through the FDA approval process. Such thinking was based on a fundamental misunderstanding of Apple’s ambition and approach to product development.  +

+ Apple’s health strategy is based on leveraging hardware, software, and services to rethink the way we approach health. This means Apple wasn’t going to just launch a depository for our health data – something that is needed but which ultimately falls short of being truly revolutionary. In addition, Apple wasn’t going to just offer health and fitness services that amount to counting steps or keeping track of miles run.  +

+ By the time Cook gave his bullish comment about health, Apple had already placed its big bet on health four years earlier by unveiling the Apple Watch. In what ended up being one of Apple’s best decisions, the company avoided going the route of medical-grade devices requiring government agency approval to reach consumers. Instead, Apple framed its health platform as a new-age computer that ultimately is an iPhone alternative. +

+ Health monitoring is one of the key new tasks that the Apple Watch, not iPhone, handles. To be more precise, Apple Watch is handling the following four health-related items:  +

  1. + Proactive monitoring (i.e. heart rate and blood oxygen) +

  2. + Well-being assistance (i.e. sleep monitoring including the runup to sleep) +

  3. + Fitness and activity tracking (i.e. Activity and Workout apps) +

  4. + Fitness and health activity (i.e. Apple Fitness+) +

+ With Apple Fitness+, Apple didn’t just release a virtual fitness class service. Instead, Apple Fitness+ is an Apple Watch service.  In some ways, Apple Fitness+ reminds me of Apple TV+. A future in which Fitness+ workouts are available on third-party gym equipment displays including on treadmills and stationary bikes is not a stretch. In addition, classes from other companies such as Nike could further elevate Apple Fitness+.  +

+ Competition +

+ If the Apple Watch is a runaway train, there is no obvious candidate in a position to stop or even slow the train. While other companies are slowly waking up and seeing the momentum found with Apple Watch, there is still much indifference, mystery, and misunderstanding as to why people are buying wearables. Too many companies still think of wearables as glorified smartphone accessories. Such thinking makes it impossible for competitors to see how Apple Watch is ushering in a paradigm shift in computing by making technology more personal in a way that other devices have failed to accomplish or replicate. +

+ One of the main takeaways from Apple’s product event earlier this month is how Apple is its own toughest competitor. The Apple Watch’s most legitimate competition is found with older Apple Watches and non-consumption (i.e. empty wrists). While this introduces its own set of risks and challenges, there is still no genuine Apple Watch competition from other companies after six years. This is an indication of the power found in controlling your own hardware, software, and services in order to get more out of technology without having technology take over people’s lives.  +

+ Listen to the corresponding Above Avalon podcast episode for this article here. +

+ Receive my analysis and perspective on Apple throughout the week via exclusive daily updates (2-3 stories per day, 10-12 stories per week). Available to Above Avalon members in both written and audio forms. To sign up and for more information on membership, visit the membership page. +

+ For additional discussion on this topic, check out the Above Avalon daily update from October 1st. +

\ No newline at end of file diff --git a/packages/readabilityjs/test/test-pages/berthub-2/distiller.html b/packages/readabilityjs/test/test-pages/berthub-2/distiller.html new file mode 100644 index 000000000..27484f183 --- /dev/null +++ b/packages/readabilityjs/test/test-pages/berthub-2/distiller.html @@ -0,0 +1,310 @@ +

+ All BNT162b2 vaccine data on this page is sourced from this World Health Organization document. +

+ This is a living page, shared already so people can get going! But check back frequently for updates. +

+ In short: the vaccine mRNA has been optimized by the manufacturer by changing bits of RNA from (say) UUU to UUC, and people would like to understand the logic behind these changes. This challenge is quite close to what cryptologists and reverse engineering people encounter regularly. On this page, you’ll find all the details you need to get cracking to reverse engineer just HOW the vaccine has been optimized. +

+ I thought this would just be a fun puzzle, but I have just been informed that figuring out the optimization procedure & documenting it is tremendously important for researchers around the world, as this would help them design code for proteins and vaccines. +

+ So, if you want to help vaccine research, do read on! +

+ The leader board +

+ Here are the current best entrants to the optimization algorithm (average of 20 runs): +

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ Name + + Codon Match + + Nucleotide Match + + Author + + Comment +
+ codon mapping + + 79.51 % + + 91.52 % + + Harry Harpel + + A simple static codon mapping +
+ most-frequent.py + + 78.57 % + + 91.08 % + + Seo Sanghyeon + + Codon frequency optimization using python_codon_tables +
+ dnachisel + + 76.99 % + + 91.06 % + + Erik Brauer + + DNAChisel algorithm +
+ dnachisel + + 76.89 % + + 90.89 % + + Pedro José Pereira Vieito + + DNAChisel algorithm +
+ remap + + 71.11 % + + 88.59 % + + Howard Chu + + Map every codon to an amino acid, pick the best codon for that amino acid +
+ 3rd-cg.py + + 60.83 % + + 85.11 % + + Peter Kuhar + + If third position is already 'G' or 'C', no change. Otherwise replace third position by a C, if protein still matches, done. Otherwise try a G. +
+ 3rd-gc.go + + 53.06 % + + 81.55 % + + bert hubert + + If third position is already 'G' or 'C', no change. Otherwise replace third position by a G, if protein still matches, done. Otherwise try a C. +
+ dnachisel + + 46.33 % + + 79.48 % + + Naomi Jacobs + + DNAChisel algorithm +
+ NOP + + 27.63 % + + 72.23 % + + Does not do any optimization at all +

+ BioNTech +

+ We should all be very grateful that BioNTech has shared this data with us. And of course we should also be grateful to the many many researchers and lab workers that worked for decades to bring the state of the art to the point that such a vaccine could be developed. It is marvelous. +

+ Because it is so marvelous, I want to understand everything about the vaccine. I wrote a page Reverse Engineering the source code of the BioNTech/Pfizer SARS-CoV-2 Vaccine that describes in some detail what is in the mRNA of the vaccine. It helps to read this page before continuing, I promise you it will be interesting. +

+ The post left open some questions however, and this is where it gets fascinating. +

+ The codon optimization +

+ The vaccine contains RNA code for a very slightly modified copy of the SARS-CoV-2 S protein. +

+ The RNA code of the vaccine itself however is highly modified from the viral original! This has been done by the manufacturer, based on their understanding of nature. +

+ And from what we understand, these modifications make the vaccine much much more effective. It would be a lot of fun to understand these modifications. It might for example explain why the Moderna vaccine needs 100 micrograms and the BioNTech vaccine only 30 micrograms. +

+ Here is the beginning of the S protein in both the virus and the BNT162b2 vaccine RNA code. Exclamation marks denote differences. + 

Virus:   AUG UUU GUU UUU CUU GUU UUA UUG CCA CUA GUC UCU AGU CAG UGU GUU
+Vaccine: AUG UUC GUG UUC CUG GUG CUG CUG CCU CUG GUG UCC AGC CAG UGU GUG
+               !   !   !   !   ! ! ! !     !   !   !   !   !            
+

+ RNA is a string (literally) of RNA characters, A, C, G and U. There is no physical framing on there, but it makes sense to analyse it in groups of three. +

+ Each group (called a codon) maps to an amino acid (denoted by a capital letter). A string of amino acids is a protein. Here is what that looks like: + 

Virus:   AUG UUU GUU UUU CUU GUU UUA UUG CCA CUA GUC UCU AGU CAG UGU GUU
+          M   F   V   F   L   V   L   L   P   L   V   S   S   Q   C   V
+Vaccine: AUG UUC GUG UUC CUG GUG CUG CUG CCU CUG GUG UCC AGC CAG UGU GUG
+               !   !   !   !   ! ! ! !     !   !   !   !   !            
+

+ Here we can see that while the codons are different, the amino acid version is the same. There are 4*4*4 codons but only 20 amino acids. This means you can typically change every codon into one of two others, and still code for the same amino acid. +

+ So in the second codon, UUU was changed to UUC. This is a net addition of one ‘C’ to the vaccine. The third codon changed from GUU to GUG, which is a net addition of one G. +

+ It is known that a higher fraction of G and C characters improves the efficiency of an mRNA vaccine. +

+ Now, if that was all there was to it, this could be the end of this page. “The algorithm is change codons so we get more G and C in there”. But then we meet the 9th codon which changes CCA to CCU. +

+ Throughout the ~4000 characters of the vaccine, this happens many times. +

+ Our challenge +

+ The goal is: find an algorithm that modifies the ‘wild type’ RNA code into the BNT162b2 one. Because everyone would like to understand how to turn viral RNA into an effective vaccine. The algorithm does not need to reproduce the exact RNA code of course, but it would be super nice if it came up with something very similar, while also being brief. +

+ To help you, I have provided the data in a number of forms, as described on the GitHub page. +

+ Note that in these files the U mentioned above appears as a T. U and T are the RNA and DNA manifestations of the same information. +

+ The easiest place to start might be the ‘side-by-side.csv‘ file. This lists the original and modified version of each codon, side by side: + 

abspos,codonOrig,codonVaccine
+0,ATG,ATG
+3,TTT,TTC
+6,GTT,GTG
+...
+3813,TAC,TAC
+3816,ACA,ACA
+3819,TAA,TGA
+

+ There is also an equivalency table that shows wich codons can be interchanged without changing the amino acid output. Please find this in codon-table-grouped.csv. There is also a visual version here. +

+ A sample algorithm +

+ On the GitHub repository you can find 3rd-gc.go (and 3rd-gc.py). +

+ These implement a simple strategy that works like this: +

  • If a virus codon already ended on G or C, copy it to the vaccine mRNA
  • If not, replace last nucleotide in codon by a G, see if the amino acid still matches, if so, copy to the vaccine mRNA
  • Try the same with a C
  • Otherwise copy as is

+ Or in golang: + 

// base case, don't do anything
+our = vir
+
+// don't do anything if codon ends on G or C already
+if(vir[2] == 'G' || vir[2] =='C') {
+        fmt.Printf("Codon ended on G or C already, not doing anything.")
+} else {
+        prop = vir[:2]+"G"
+        fmt.Printf("Attempting G substitution, new candidate '%s'. ", prop)
+        if(c2s[vir] == c2s[prop]) {
+                fmt.Printf("Amino acid still the same, done!")
+                our = prop
+        } else {
+                fmt.Printf("Oops, amino acid changed. Trying C, new candidate '%s'. ", prop)
+                prop = vir[:2]+"C"
+                if(c2s[vir] == c2s[prop]) {
+                        fmt.Printf("Amino acid still the same, done!")
+                        our=prop
+                } 
+                
+        }
+
+}
+

+ This achieves a rather poor 53.1% match with the BioNTech RNA vaccine, but it is a start. +

+ When you design your algorithm, be sure to only base your choices on the virus RNA. Do not peek into the BioNTech RNA! +

+ If you have achieved a score beyond 53.1% please email a link to your code to bert@hubertnet.nl (or @PowerDNS_Bert and I’ll put it on the leader board at the top of this page! +

+ Things that will help +

+ As with every form of reverse engineering or cryptanalysis, it helps to understand what we are looking at. +

+ GC ratio +

+ We know that one goal of the ‘codon optimization’ is to get more Cs and Gs into the vaccine version of the RNA. However, there is also a limit to that. In DNA, which is also used to manufacture the vaccine, G and C bind together strongly, to the point that if you put too many of these ‘nucleotides’ in there, the DNA will no longer be replicated efficiently. +

+ So some modifications may actually happen to manage down the GC percentage of a stretch of DNA if it was getting too high. +

+ Codon optimization +

+ Some codons are rare in human DNA, or in certain cells. It may be that some codons are replaced by other ones simply because they are more frequently used by some cells. +

+ RNA folding +

+ We’ve been looking at codons up to here. The RNA itself however does not know about codons, there are no markers that say where a codon begins and ends. The first codon on a protein however is always ATG (or AUG in RNA). +

+ RNA curls up into a shape. This shape might help evade the immune system or it might improve translation into amino acids. This only depends on the sequence of RNA nucleotides and not on specific codons. +

+ You can submit RNA sequences to this server of the Institute for Theoretical Chemistry at the University of Vienna and it will fold RNA for you. This is a very advanced server that does meticulous calculations. +

+ This Wikipedia page describes how this works. +

+ It may be that some optimizations improve folding. +

+ I am also told that this paper by Moderna (another mRNA vaccine manufacturer) may be relevant: mRNA structure regulates protein expression through changes in functional half-life. +

\ No newline at end of file diff --git a/packages/readabilityjs/test/test-pages/berthub/distiller.html b/packages/readabilityjs/test/test-pages/berthub/distiller.html new file mode 100644 index 000000000..7623efc89 --- /dev/null +++ b/packages/readabilityjs/test/test-pages/berthub/distiller.html @@ -0,0 +1,241 @@ +

+ Welcome! In this post, we’ll be taking a character-by-character look at the source code of the BioNTech/Pfizer SARS-CoV-2 mRNA vaccine. +

+ Update: The other up and coming vaccines are described in The Genetic Code and Proteins of the Other Covid-19 Vaccines +

+ I want to thank the large cast of people who spent time previewing this article for legibility and correctness. All mistakes remain mine though, but I would love to hear about them quickly at bert@hubertnet.nl or @PowerDNS_Bert +

+ Now, these words may be somewhat jarring - the vaccine is a liquid that gets injected in your arm. How can we talk about source code? +

+ This is a good question, so let’s start off with a small part of the very source code of the BioNTech/Pfizer vaccine, also known as BNT162b2, also known as Tozinameran also known as Comirnaty. +

First 500 characters of the BNT162b2 mRNA. Source: World Health Organization
+

+ First 500 characters of the BNT162b2 mRNA. Source: World Health Organization +

+

+ The BNT162b2 mRNA vaccine has this digital code at its heart. It is 4284 characters long, so it would fit in a bunch of tweets. At the very beginning of the vaccine production process, someone uploaded this code to a DNA printer (yes), which then converted the bytes on disk to actual DNA molecules. +

A Codex DNA BioXp 3200 DNA printer
+

+ A Codex DNA BioXp 3200 DNA printer +

+

+ Out of such a machine come tiny amounts of DNA, which after a lot of biological and chemical processing end up as RNA (more about which later) in the vaccine vial. A 30 microgram dose turns out to actually contain 30 micrograms of RNA. In addition, there is a clever lipid (fatty) packaging system that gets the mRNA into our cells. +

+ Update: Derek Lowe of the famous In the pipeline blog over at Science has written a comprehensive post “RNA Vaccines And Their Lipids“ which neatly explains the lipid and delivery parts of the vaccines that I am not competent to describe. Luckily Derek is! +

+ Update 2: Jonas Neubert and Cornelia Scheitz have written this awesome page with loads of detail on how the vaccines actually get produced and distributed. Recommended! +

+ RNA is the volatile ‘working memory’ version of DNA. DNA is like the flash drive storage of biology. DNA is very durable, internally redundant and very reliable. But much like computers do not execute code directly from a flash drive, before something happens, code gets copied to a faster, more versatile yet far more fragile system. +

+ For computers, this is RAM, for biology it is RNA. The resemblance is striking. Unlike flash memory, RAM degrades very quickly unless lovingly tended to. The reason the Pfizer/BioNTech mRNA vaccine must be stored in the deepest of deep freezers is the same: RNA is a fragile flower. +

+ Each RNA character weighs on the order of 0.53·10⁻²¹ grams, meaning there are around 6·10¹⁶ characters in a single 30 microgram vaccine dose. Expressed in bytes, this is around 14 petabytes, although it must be said this consists of around 13,000 billion repetitions of the same 4284 characters. The actual informational content of the vaccine is just over a kilobyte. SARS-CoV-2 itself weighs in at around 7.5 kilobytes. +

+ Update: In the original post these numbers were off. Here is a spreadsheet with the correct calculations. +

+ The briefest bit of background +

+ DNA is a digital code. Unlike computers, which use 0 and 1, life uses A, C, G and U/T (the ‘nucleotides’, ‘nucleosides’ or ‘bases’). +

+ In computers we store the 0 and 1 as the presence or absence of a charge, or as a current, as a magnetic transition, or as a voltage, or as a modulation of a signal, or as a change in reflectivity. Or in short, the 0 and 1 are not some kind of abstract concept - they live as electrons and in many other physical embodiments. +

+ In nature, A, C, G and U/T are molecules, stored as chains in DNA (or RNA). +

+ In computers, we group 8 bits into a byte, and the byte is the typical unit of data being processed. +

+ Nature groups 3 nucleotides into a codon, and this codon is the typical unit of processing. A codon contains 6 bits of information (2 bits per DNA character, 3 characters = 6 bits. This means 2⁶ = 64 different codon values). +

+ Pretty digital so far. When in doubt, head to the WHO document with the digital code to see for yourself. +

+ Some further reading is available here - this link (‘What is life’) might help make sense of the rest of this page. Or, if you like video, I have two hours for you. +

+ So what does that code DO? +

+ The idea of a vaccine is to teach our immune system how to fight a pathogen, without us actually getting ill. Historically this has been done by injecting a weakened or incapacitated (attenuated) virus, plus an ‘adjuvant’ to scare our immune system into action. This was a decidedly analogue technique involving billions of eggs (or insects). It also required a lot of luck and loads of time. Sometimes a different (unrelated) virus was also used. +

+ An mRNA vaccine achieves the same thing (‘educate our immune system’) but in a laser like way. And I mean this in both senses - very narrow but also very powerful. +

+ So here is how it works. The injection contains volatile genetic material that describes the famous SARS-CoV-2 ‘Spike’ protein. Through clever chemical means, the vaccine manages to get this genetic material into some of our cells. +

+ These then dutifully start producing SARS-CoV-2 Spike proteins in large enough quantities that our immune system springs into action. Confronted with Spike proteins, and (importantly) tell-tale signs that cells have been taken over, our immune system develops a powerful response against multiple aspects of the Spike protein AND the production process. +

+ And this is what gets us to the 95% efficient vaccine. +

+ This is a sort of table of contents. We’ll start with the ‘cap’, actually depicted as a little hat. +

+ Much like you can’t just plonk opcodes in a file on a computer and run it, the biological operating system requires headers, has linkers and things like calling conventions. +

+ The code of the vaccine starts with the following two nucleotides: + 

GA
+

+ This can be compared very much to every DOS and Windows executable starting with MZ, or UNIX scripts starting with #!. In both life and operating systems, these two characters are not executed in any way. But they have to be there because otherwise nothing happens. +

+ The mRNA ‘cap’ has a number of functions. For one, it marks code as coming from the nucleus. In our case of course it doesn’t, our code comes from a vaccination. But we don’t need to tell the cell that. The cap makes our code look legit, which protects it from destruction. +

+ The initial two GA nucleotides are also chemically slightly different from the rest of the RNA. In this sense, the GA has some out-of-band signaling on it. +

+ The “five-prime untranslated region” +

+ Some lingo here. RNA molecules can only be read in one direction. Confusingly, the part where the reading begins is called the 5’ or ‘five-prime’. The reading stops at the 3’ or three-prime end. +

+ Life consists of proteins (or things made by proteins). And these proteins are described in RNA. When RNA gets converted into proteins, this is called translation. +

+ Here we have the 5’ untranslated region (‘UTR’), so this bit does not end up in the protein: + 

GAAΨAAACΨAGΨAΨΨCΨΨCΨGGΨCCCCACAGACΨCAGAGAGAACCCGCCACC
+

+ Here we encounter our first surprise. The normal RNA characters are A, C, G and U. U is also known as ’T’ in DNA. But here we find a Ψ, what is going on? +

+ This is one of the exceptionally clever bits about the vaccine. Our body runs a powerful antivirus system (“the original one”). For this reason, cells are extremely unenthusiastic about foreign RNA and try very hard to destroy it before it does anything. +

+ This is somewhat of a problem for our vaccine - it needs to sneak past our immune system. Over many years of experimentation, it was found that if the U in RNA is replaced by a slightly modified molecule, our immune system loses interest. For real. +

+ So in the BioNTech/Pfizer vaccine, every U has been replaced by 1-methyl-3’-pseudouridylyl, denoted by Ψ. The really clever bit is that although this replacement Ψ placates (calms) our immune system, it is accepted as a normal U by relevant parts of the cell. +

+ In computer security we also know this trick - it sometimes is possible to transmit a slightly corrupted version of a message that confuses firewalls and security solutions, but that is still accepted by the backend servers - which can then get hacked. +

+ We are now reaping the benefits of fundamental scientific research performed in the past. The discoverers of this Ψ technique had to fight to get their work funded and then accepted. We should all be very grateful, and I am sure the Nobel prizes will arrive in due course. +

+ Many people have asked, could viruses also use the Ψ technique to beat our immune systems? In short, this is extremely unlikely. Life simply does not have the machinery to build 1-methyl-3’-pseudouridylyl nucleotides. Viruses rely on the machinery of life to reproduce themselves, and this facility is simply not there. The mRNA vaccines quickly degrade in the human body, and there is no possibility of the Ψ-modified RNA replicating with the Ψ still in there. “No, Really, mRNA Vaccines Are Not Going To Affect Your DNA“ is also a good read. +

+ Ok, back to the 5’ UTR. What do these 51 characters do? As everything in nature, almost nothing has one clear function. +

+ When our cells need to translate RNA into proteins, this is done using a machine called the ribosome. The ribosome is like a 3D printer for proteins. It ingests a strand of RNA and based on that it emits a string of amino acids, which then fold into a protein. +

+ This is what we see happening above. The black ribbon at the bottom is RNA. The ribbon appearing in the green bit is the protein being formed. The things flying in and out are amino acids plus adaptors to make them fit on RNA. +

+ This ribosome needs to physically sit on the RNA strand for it to get to work. Once seated, it can start forming proteins based on further RNA it ingests. From this, you can imagine that it can’t yet read the parts where it lands on first. This is just one of the functions of the UTR: the ribosome landing zone. The UTR provides ‘lead-in’. +

+ In addition to this, the UTR also contains metadata: when should translation happen? And how much? For the vaccine, they took the most ‘right now’ UTR they could find, taken from the alpha globin gene. This gene is known to robustly produce a lot of proteins. In previous years, scientists had already found ways to optimize this UTR even further (according to the WHO document), so this is not quite the alpha globin UTR. It is better. +

+ The S glycoprotein signal peptide +

+ As noted, the goal of the vaccine is to get the cell to produce copious amounts of the Spike protein of SARS-CoV-2. Up to this point, we have mostly encountered metadata and “calling convention” stuff in the vaccine source code. But now we enter the actual viral protein territory. +

+ We still have one layer of metadata to go however. Once the ribosome (from the splendid animation above) has made a protein, that protein still needs to go somewhere. This is encoded in the “S glycoprotein signal peptide (extended leader sequence)“. +

+ The way to see this is that at the beginning of the protein there is a sort of address label - encoded as part of the protein itself. In this specific case, the signal peptide says that this protein should exit the cell via the “endoplasmic reticulum”. Even Star Trek lingo is not as fancy as this! +

+ The “signal peptide” is not very long, but when we look at the code, there are differences between the viral and vaccine RNA: +

+ (Note that for comparison purposes, I have replaced the fancy modified Ψ by a regular RNA U) + 

           3   3   3   3   3   3   3   3   3   3   3   3   3   3   3   3
+Virus:   AUG UUU GUU UUU CUU GUU UUA UUG CCA CUA GUC UCU AGU CAG UGU GUU
+Vaccine: AUG UUC GUG UUC CUG GUG CUG CUG CCU CUG GUG UCC AGC CAG UGU GUG
+               !   !   !   !   ! ! ! !     !   !   !   !   !           !
+

+ So what is going on? I have not accidentally listed the RNA in groups of 3 letters. Three RNA characters make up a codon. And every codon encodes for a specific amino acid. The signal peptide in the vaccine consists of exactly the same amino acids as in the virus itself. +

+ So how come the RNA is different? +

+ There are 4³=64 different codons, since there are 4 RNA characters, and there are three of them in a codon. Yet there are only 20 different amino acids. This means that multiple codons encode for the same amino acid. +

+ Life uses the following nearly universal table for mapping RNA codons to amino acids: +

The RNA codon table (Wikipedia)
+

+ The RNA codon table (Wikipedia) +

+

+ In this table, we can see that the modifications in the vaccine (UUU -> UUC) are all synonymous. The vaccine RNA code is different, but the same amino acids and the same protein come out. +

+ If we look closely, we see that the majority of the changes happen in the third codon position, noted with a ‘3’ above. And if we check the universal codon table, we see that this third position indeed often does not matter for which amino acid is produced. +

+ So, the changes are synonymous, but then why are they there? Looking closely, we see that all changes except one lead to more C and Gs. +

+ So why would you do that? As noted above, our immune system takes a very dim view of ‘exogenous’ RNA, RNA code coming from outside the cell. To evade detection, the ‘U’ in the RNA was already replaced by a Ψ. +

+ However, it turns out that RNA with a higher amount of Gs and Cs is also converted more efficiently into proteins, +

+ And this has been achieved in the vaccine RNA by replacing many characters with Gs and Cs wherever this was possible. +

+ I’m slightly fascinated by the one change that did not lead to an additional C or G, the CCA -> CCU modification. If anyone knows the reason, please let me know! Note that I’m aware that some codons are more common than others in the human genome, but I also read that this does not influence translation speed a lot. UPDATE: A number of readers have pointed out that this change could prevent a “hairpin” in the RNA. You can try this out yourself on the RNAFold service. +

+ The actual Spike protein +

+ The next 3777 characters of the vaccine RNA are similarly ‘codon optimized’ to add a lot of C’s and G’s. In the interest of space I won’t list all the code here, but we are going to zoom in on one exceptionally special bit. This is the bit that makes it work, the part that will actually help us return to life as normal: + 

                  *   *
+          L   D   K   V   E   A   E   V   Q   I   D   R   L   I   T   G
+Virus:   CUU GAC AAA GUU GAG GCU GAA GUG CAA AUU GAU AGG UUG AUC ACA GGC
+Vaccine: CUG GAC CCU CCU GAG GCC GAG GUG CAG AUC GAC AGA CUG AUC ACA GGC
+          L   D   P   P   E   A   E   V   Q   I   D   R   L   I   T   G
+           !     !!! !!        !   !       !   !   !   ! !              
+

+ Here we see the usual synonymous RNA changes. For example, in the first codon we see that CUU is changed into CUG. This adds another ‘G’ to the vaccine, which we know helps enhance protein production. Both CUU and CUG encode for the amino acid ‘L’ or Leucine, so nothing changed in the protein. +

+ When we compare the entire Spike protein in the vaccine, all changes are synonymous like this.. except for two, and this is what we see here. +

+ The third and fourth codons above represent actual changes. The K and V amino acids there are both replaced by ‘P’ or Proline. For ‘K’ this required three changes (‘!!!’) and for ‘V’ it required only two (‘!!’). +

+ It turns out that these two changes enhance the vaccine efficiency enormously. +

+ So what is happening here? If you look at a real SARS-CoV-2 particle, you can see the Spike protein as, well, a bunch of spikes: +

SARS virus particles (Wikipedia)
+

+ SARS virus particles (Wikipedia) +

+

+ The spikes are mounted on the virus body (‘the nucleocapsid protein’). But the thing is, our vaccine is only generating the spikes itself, and we’re not mounting them on any kind of virus body. +

+ It turns out that, unmodified, freestanding Spike proteins collapse into a different structure. If injected as a vaccine, this would indeed cause our bodies to develop immunity.. but only against the collapsed spike protein. +

+ And the real SARS-CoV-2 shows up with the spiky Spike. The vaccine would not work very well in that case. +

+ So what to do? In 2017 it was described how putting a double Proline substitution in just the right place would make the SARS-CoV-1 and MERS S proteins take up their ‘pre-fusion’ configuration, even without being part of the whole virus. This works because Proline is a very rigid amino acid. It acts as a kind of splint, stabilising the protein in the state we need to show to the immune system. +

+ The people that discovered this should be walking around high-fiving themselves incessantly. Unbearable amounts of smugness should be emanating from them. And it would all be well deserved. +

+ Update! I have been contacted by the McLellan lab, one of the groups behind the Proline discovery. They tell me the high-fiving is subdued because of the ongoing pandemic, but they are pleased to have contributed to the vaccines. They also stress the importance of many other groups, workers and volunteers. +

+ The end of the protein, next steps +

+ If we scroll through the rest of the source code, we encounter some small modifications at the end of the Spike protein: + 

          V   L   K   G   V   K   L   H   Y   T   s             
+Virus:   GUG CUC AAA GGA GUC AAA UUA CAU UAC ACA UAA
+Vaccine: GUG CUG AAG GGC GUG AAA CUG CAC UAC ACA UGA UGA 
+          V   L   K   G   V   K   L   H   Y   T   s   s          
+               !   !   !   !     ! !   !          ! 
+

+ At the end of a protein we find a ‘stop’ codon, denoted here by a lowercase ’s’. This is a polite way of saying that the protein should end here. The original virus uses the UAA stop codon, the vaccine uses two UGA stop codons, perhaps just for good measure. +

+ The 3’ Untranslated Region +

+ Much like the ribosome needed some lead-in at the 5’ end, where we found the ‘five prime untranslated region’, at the end of a protein coding region we find a similar construct called the 3’ UTR. +

+ Many words could be written about the 3’ UTR, but here I quote what the Wikipedia says: “The 3’-untranslated region plays a crucial role in gene expression by influencing the localization, stability, export, and translation efficiency of an mRNA .. despite our current understanding of 3’-UTRs, they are still relative mysteries”. +

+ What we do know is that certain 3’-UTRs are very successful at promoting protein expression. According to the WHO document, the BioNTech/Pfizer vaccine 3’-UTR was picked from “the amino-terminal enhancer of split (AES) mRNA and the mitochondrial encoded 12S ribosomal RNA to confer RNA stability and high total protein expression”. To which I say, well done. +

+ The AAAAAAAAAAAAAAAAAAAAAA end of it all +

+ The very end of mRNA is polyadenylated. This is a fancy way of saying it ends on a lot of AAAAAAAAAAAAAAAAAAA. Even mRNA has had enough of 2020 it appears. +

+ mRNA can be reused many times, but as this happens, it also loses some of the A’s at the end. Once the A’s run out, the mRNA is no longer functional and gets discarded. In this way, the ‘poly-A’ tail is protection from degradation. +

+ Studies have been done to find out what the optimal number of A’s at the end is for mRNA vaccines. I read in the open literature that this peaked at 120 or so. +

+ The BNT162b2 vaccine ends with: + 

                                     ****** ****
+UAGCAAAAAA AAAAAAAAAA AAAAAAAAAA AAAAGCAUAU GACUAAAAAA AAAAAAAAAA 
+AAAAAAAAAA AAAAAAAAAA AAAAAAAAAA AAAAAAAAAA AAAAAAAAAA AAAA
+

+ This is 30 A’s, then a “10 nucleotide linker” (GCAUAUGACU), followed by another 70 A’s. +

+ There are various theories why this linker is there. Some people tell me it has to do with DNA plasmid stability, I have also received this from an actual expert: +

+ “The 10-nucleotide linker within the poly(A) tail makes it easier to stitch together the synthetic DNA fragments that become the template for transcribing the mRNA. It also reduces slipping by T7 RNA polymerase so that the transcribed mRNA is more uniform in length”. +

+ Summarising +

+ With this, we now know the exact mRNA contents of the BNT162b2 vaccine, and for most parts we understand why they are there: +

  • The CAP to make sure the RNA looks like regular mRNA
  • A known successful and optimized 5’ untranslated region (UTR)
  • A codon optimized signal peptide to send the Spike protein to the right place (amino acids copied 100% from the original virus)
  • A codon optimized version of the original spike, with two ‘Proline’ substitutions to make sure the protein appears in the right form
  • A known successful and optimized 3’ untranslated region
  • A poly-A tail with a ‘linker’ in there

+ The codon optimization adds a lot of G and C to the mRNA. Meanwhile, using Ψ (1-methyl-3’-pseudouridylyl) instead of U helps evade our immune system, so the mRNA stays around long enough so we can actually help train the immune system. +

+ Further reading/viewing +

+ In 2017 I held a two hour presentation on DNA, which you can view here. Like this page it is aimed at computer people. +

+ In addition, I’ve been maintaining a page on ‘DNA for programmers’ since 2001. +

+ Finally, this listing of my blog posts has quite some DNA, SARS-CoV-2 and COVID related material. +

+ As an update, the other up and coming vaccines are described in The Genetic Code and Proteins of the Other Covid-19 Vaccines +

\ No newline at end of file diff --git a/packages/readabilityjs/test/test-pages/biospace/distiller.html b/packages/readabilityjs/test/test-pages/biospace/distiller.html new file mode 100644 index 000000000..381755e05 --- /dev/null +++ b/packages/readabilityjs/test/test-pages/biospace/distiller.html @@ -0,0 +1,25 @@ +

+ Published: May 11, 2022 By Rosemary Scott +

BabyFeet/Getty

+ Sudden infant death syndrome (SIDS) accounts for about 37% of sudden unexpected infant deaths a year in the U.S., and the cause of SIDS has remained largely unknown. On Saturday, researchers from The Children's Hospital Westmead in Sydney released a study that confirmed not only how these infants die, but why.  +

+ SIDS refers to the unexplained deaths of infants under a year old, and it usually occurs while the child is sleeping. According to Mayo Clinic, many in the medical community suspected this phenomenon could be caused by a defect in the part of the brain that controls arousal from sleep and breathing. The theory was that if the infant stopped breathing during sleep, the defect would keep the child from startling or waking up.  +

+ The Sydney researchers were able to confirm this theory by analyzing dried blood samples taken from newborns who died from SIDS and other unknown causes. Each SIDS sample was then compared with blood taken from healthy babies. They found the activity of the enzyme butyrylcholinesterase (BChE) was significantly lower in babies who died of SIDS compared to living infants and other non-SIDS infant deaths. BChE plays a major role in the brain’s arousal pathway, explaining why SIDS typically occurs during sleep.  +

+ Previously, parents were told SIDS could be prevented if they took proper precautions: laying babies on their backs, not letting them overheat and keeping all toys and blankets out of the crib were a few of the most important preventative steps. +

+ While safe sleep practices are still important for protecting infants, many children whose parents took every precaution still died from SIDS. These parents were left with immense guilt, wondering if they could have prevented their baby’s death. +

+ Dr. Carmel Harrington, the lead researcher for the study, was one of these parents. Her son unexpectedly and suddenly died as an infant 29 years ago. In an interview with the Australian Broadcasting Corporation (ABC), Harrington explained what she was told about the cause of her child’s death.  +

+ "Nobody could tell me. They just said it's a tragedy. But it was a tragedy that didn't sit well with my scientific brain.”  +

+ Since then, she’s worked to find the cause of SIDS, both for herself and for the medical community as a whole. She went on to explain why this discovery is so important for parents whose babies suffered from SIDS.  +

+ "These families can now live with the knowledge that this was not their fault," she said. +

+ In the study, the researchers wrote, “This finding represents the possibility for the identification of infants at risk for SIDS infants prior to death and opens new avenues for future research into specific interventions.”  +

+ As the cause is now known, researchers can turn their attention to a solution. In the next few years, those in the medical community who have studied SIDS will likely work on a screening test to identify babies who are at risk for SIDS and hopefully prevent it altogether. +

\ No newline at end of file diff --git a/packages/readabilityjs/test/test-pages/bitfieldconsulting/distiller.html b/packages/readabilityjs/test/test-pages/bitfieldconsulting/distiller.html new file mode 100644 index 000000000..5bc19924d --- /dev/null +++ b/packages/readabilityjs/test/test-pages/bitfieldconsulting/distiller.html @@ -0,0 +1,223 @@ +
+ +

+ Rust vs Go +

rust-vs-go-wide.png

+ Which is better, Rust or Go? Which language should you choose for your next project, and why? How do the two compare in areas like performance, simplicity, safety, features, scale, and concurrency? What do they have in common, and where do they fundamentally differ? Let's find out, in this friendly and even-handed comparison of Rust and Golang, from the author of the For the Love of Go book series. +

+ Rust and Go are both awesome +

+ First, it's really important to say that both Go and Rust are absolutely excellent programming languages. They're modern, powerful, widely-adopted, and offer excellent performance. You may have read articles and blog posts aiming to convince you that Go is better than Rust, or vice versa. But that really makes no sense; every programming language represents a set of trade-offs. Each language is optimised for different things, so your choice of language should be determined by what suits you and the problems you want to solve with it. +

+ In this article, I'll try to give a brief overview of where I think Go is the ideal choice, and where I think Rust is a better alternative. Ideally, though, you should have a working familiarity with both languages. While they're very different in syntax and style, both Rust and Go are first-class tools for building software. With that said, let's take a closer look at the two languages. +

+ The similarities +

+ Rust and Go have a lot in common, which is one reason you often hear them mentioned together. What are some of the common goals of both languages? +

+ Rust is a low-level statically-typed multi-paradigm programming language that’s focused on safety and performance.
+ —Gints Dreimanis +

+ Go is an open source programming language that makes it easy to build simple, reliable, and efficient software.
+ —Golang.org +

+ Memory safety +

+ Both Go and Rust belong to the group of modern programming languages whose priority is memory safety. It's become clear over many decades of using older languages such as C and C++ that one of the biggest causes of bugs and security vulnerabilities is accessing memory unsafely or incorrectly. Rust and Go deal with this problem in different ways, but both aim to be smarter and safer than other languages about managing memory, and to help you write correct and performant programs. +

+ Fast, compact executables +

+ They're both compiled languages, which means your programs are translated directly to executable machine code, so that you can deploy your program as a single binary file; unlike interpreted languages such as Python and Ruby, you don't need to distribute an interpreter and lots of libraries and dependencies along with your program, which is a big plus. This also makes both Rust and Go programs extremely fast in comparison to interpreted languages. +

+ General-purpose languages +

+ Rust and Go are also both powerful, scalable general-purpose programming languages, which you can use to develop all kinds of modern software, from web applications to distributed microservices, or from embedded microcontrollers to mobile apps. Both have excellent standard libraries and a thriving third-party ecosystem, as well as great commercial support and a large user base. They've both been around for many years, and will continue to be widely used for years to come. Learning either Go or Rust today will be a sound investment of your time and effort. +

+ Pragmatic programming style +

+ Neither are primarily functional languages (like Scala or Elixir, for example), and neither are exclusively object-oriented (like Java and C#). Instead, while both Go and Rust have features associated with functional and object-oriented programming, they're pragmatic languages aimed at solving problems in whatever way is most appropriate, rather than forcing you into a particular way of doing things. (If you like the functional style of programming, though, you’ll find a lot more facilities for it in Rust, because Rust has a lot more facilities than Go in general.) +

+ We can debate about what an ‘object-oriented’ language is, but it’s fair to say that the style of object-oriented programming that C++, Java, or C# users would expect is not present in either Go or Rust.
+ —Jack Mott +

+ Development at scale +

+ Both Rust and Go have some useful features which make them suitable for programming in the large, whether that means large teams, or large codebases, or both. +

+ For example, whereas C programmers have argued for years about where to put their brackets, and whether code should be indented with tabs or spaces, both Rust and Go eliminate such issues completely by using a standard formatting tool (gofmt for Go, rustfmt for Rust) which rewrites your code automatically using the canonical style. It’s not that this particular style is so wonderful in itself: it’s the standardisation which Rust and Go programmers appreciate. +

+ gofmt’s style is no one’s favourite, yet gofmt is everyone’s favourite.
+ —Rob Pike +

+ Another area where both languages score highly is in the build pipeline. Both have excellent, built-in, high-performance standard build and dependency management tools; no more wrestling with complex third-party build systems and having to learn a new one every couple of years. +

+ Building Go and Rust code, having come from a Java and Ruby background in my early career, felt like an impossible weight off my shoulders. When I was at Google, it was a relief to come across a service that was written in Go, because I knew it would be easy to build and run. This has also been true of Rust, though I've only worked on that at much smaller scale. I'm hoping that the days of infinitely configurable build systems are dead, and languages all ship with their own purpose-built build tools that just work out of the box.
+ —Sam Rose +

+ So what’s all the fuss about? +

+ With all that in mind, and seeing that both languages are so well-designed and powerful, you might be wondering what all the holy wars are about (me too). Why do people make such a fuss about 'Go versus Rust', getting into angry social media spats and writing long blog posts about how only an idiot would use Rust, or that Go isn't a real programming language, or whatever. It might make them feel better, but it doesn't exactly help you, as someone trying to decide which language to use for your project, or which you should learn to advance your programming career. A wise person doesn't make important choices based on who shouts the loudest. +

+ Let's continue our grown-up discussion now by looking at some areas where a reasonable person might prefer one language over the other. +

+ Performance +

+ We've said that both Go and Rust produce extremely fast programs because they're compiled to native machine code, without having to go through an interpreter or virtual machine. However, Rust's performance is particularly outstanding. It's comparable with C and C++, which are often regarded as the highest-performance compiled languages, but unlike these older languages, it also offers memory safety and concurrency safety at essentially no cost in execution speed. Rust also allows you to create complex abstractions without paying a performance penalty at run-time. +

+ By comparison, although Go programs also perform extremely well, Go is primarily designed for speed of development (including compilation), rather than speed of execution. The Go compiler doesn’t spend a lot of time trying to generate the most efficient possible machine code; it cares more about compiling lots of code quickly. So Rust will usually beat Go in run-time benchmarks. +

+ Rust’s run-time performance is also consistent and predictable, because it doesn’t use garbage collection. Go’s garbage collector is very efficient, and it’s optimised to make its stop-the-world pauses as short as possible (and getting shorter with every new Go release). But garbage collection inevitably introduces some unpredictability in the way programs behave, which can be a serious issue in some applications, such as embedded systems. +

+ Because Rust aims to give the programmer complete control of the underlying hardware, it's possible to optimise Rust programs to be pretty close to the maximum theoretical performance of the machine. This makes Rust an excellent choice for areas where speed of execution beats all other considerations, such as game programming, operating system kernels, web browser components, and real-time control systems. +

+ Simplicity +

+ It doesn't matter how fast a programming language is if nobody can figure out how to use it. Go was deliberately conceived as a reaction against the ever-growing complexity of languages like C++; it has very little syntax, very few keywords, and, indeed, few features. This means it doesn't take long to learn the Go language to the point where you can write useful programs in it. +

+ Go is incredibly easy to learn. I know this is an often-touted benefit, but I was really surprised at how quickly I was able to be productive. Thanks to the language, docs, and tools, I was writing interesting, committable code after literally two days.
+ —Early Impressions of Go From a Rust Programmer +

+ The key word here is simplicity. Simple isn't the same as easy, for sure, but a small, simple language is easier to learn than a large, complex one. There just aren't that many different ways to do things, so all well-written Go code tends to look the same. It’s easy to just dive into an unfamiliar service and understand what it’s doing. + 

fmt.Println("Gopher's Diner Breakfast Menu")
+for dish, price := range menu {
+    fmt.Println(dish, price)
+}

+ (In the Code Club series, I do exactly that: pick Go projects semi-randomly from GitHub and explore them with a group of Go beginners, to see how much of the code we can understand. It always turns out to be more than we expected!) +

+ Although the core language is small, Go’s standard library is very powerful. That means your learning curve will also need to include the parts of the standard library that you need, not just Go syntax. On the other hand, moving functionality out of the language and into the standard library means that you can focus on learning only the libraries that are relevant to you right now. +

+ Go is also designed for software development at scale, with large codebases and large teams. In these situations, it's important that new developers can get up to speed as quickly as possible. +

+ With Go, you get things done—fast. Go is one of the most productive languages I've ever worked with. The mantra is: solve real problems today.
+ —Matthias Endler +

+ Features +

+ Rust supports more complexity than several other programming languages, therefore, you can achieve more with it. For example, it supports generics.
+ —Devathon +

+ Rust is specifically designed to include lots of powerful and useful features to help programmers do the most with the least code. For example, Rust's match feature lets you write flexible, expressive logic quite concisely: + 

fn is_prime(n: u64) -> bool {
+    match n {
+        0...1 => false,
+        _ => !(2..n).any(|d| n % d == 0),
+    }
+}

+ Because Rust does a lot, this means there's a lot to learn, especially at the start. But that's okay: there's a lot to learn in C++ or Java, too, and you don't get the advanced features that come with Rust, like memory safety. To criticise Rust for being a complex language misses the point: it's designed to be expressive, which means having a lot of features, and in many situations that's what you want from a programming language. There's a learning curve, for sure, but once you’re up and running with it, you’ll be fine. +

+ Rust competes for mindshare with C++ and D for programmers who are prepared to accept more complex syntax and semantics (and presumably higher readability costs) in return for the maximum possible performance.
+ —Dave Cheney +

+ Concurrency +

+ Most languages have some form of support for concurrent programming (doing multiple things at once), but Go was designed for this job from the ground up. Instead of using operating system threads, Go provides a lightweight alternative: goroutines. Each goroutine is an independently executing Go function, which the Go scheduler will map to one of the OS threads under its control. This means that the scheduler can very efficiently manage a large number of concurrent goroutines, using only a limited number of OS threads. +

+ Consequently, you can run millions of concurrent goroutines in a single program without creating serious performance problems. That makes Go the perfect choice for high-scale concurrent applications such as webservers and microservices. +

+ Go also features fast, safe, efficient ways for goroutines to communicate and share data, using channels. Go's concurrency support feels well-designed, and a pleasure to use. Reasoning about concurrent programs is hard in general, and building reliable, correct concurrent programs is a challenge in any language. Because it was built into the language from the start, though, instead of being an afterthought, concurrent programming in Go is about as simple and well-integrated as it could reasonably be. +

+ Go makes it very easy to build a nicely factored application that takes full advantage of concurrency while being deployed as a set of microservices. Rust can do those things, too, but it’s arguably a bit tougher. In some respects, Rust’s obsession with preventing memory-related security vulnerabilities means that programmers have to go out of their way to perform tasks that would be simpler in other languages, including Go.
+ —Sonya Koptyev +

+ The concurrency story in Rust is very new, by comparison, and still stabilising, but it’s under very active development, so watch this space. For example, Rust’s rayon library provides a very elegant and lightweight way of turning sequential computations into parallel ones. +

+ Having lightweight syntax for spawning Go routines and using channels are really nice. It really shows the power of syntax that such small details make concurrent programming feel so much nicer than in other languages.
+ —Early Impressions of Go From a Rust Programmer +

+ While it may be a bit less straightforward to implement concurrent programs in Rust, it’s still possible, and those programs can take advantage of Rust’s guarantees about safety. A good example is the standard library’s Mutex class: in Go, you can forget to obtain a mutex lock before accessing something, but Rust won’t let you do that. +

+ Go is focused on concurrency as a first class concept. That is not to say you cannot find aspects of Go’s actor oriented concurrency in Rust, but it is left as an exercise to the programmer.
+ —Dave Cheney +

+ Safety +

+ We saw earlier that both Go and Rust aim, in different ways, to prevent a large class of common programming errors, to do with memory management. But Rust in particular goes to great lengths to ensure that you can't do something unsafe that you didn't mean to do. +

+ Rust's very strict and pedantic compiler checks each and every variable you use and every memory address you reference. It avoids possible data race conditions and informs you about undefined behavior. Concurrency and memory safety issues are fundamentally impossible to get in the safe subset of Rust.
+ —Why Rust? +

+ This is going to make programming in Rust a different experience to almost all other languages, and it may be a challenging one at first. But for many people, the hard work is worth it. +

+ For me the key advantage of Rust is a feeling that the compiler has my back and won't let through any bug it could possibly detect (seriously, it feels like magic sometimes).
+ —Grzegorz Nosek +

+ Many languages, including Go, have facilities to help programmers avoid mistakes, but Rust takes this to a new level, so that potentially incorrect programs won’t even compile. +

+ With Rust, the library programmer has a lot of tools to prevent her users making mistakes. Rust gives us the ability to say that we own a specific piece of data; it's not possible for anything else to claim ownership, so we know nothing else will be able to modify it. I can't think of a time I've ever been given this many tools to prevent accidental misuse before. It's a wonderful feeling.
+ —Sam Rose +

+ "Fighting with the borrow checker" is a common syndrome for new Rust programmers, but in most cases the problems that it finds are genuine bugs (or at least potential bugs) in your code. It may force you to fundamentally re-architect your program to avoid running into these issues; and that's a good thing, when correctness and reliability are your top priority. What's the point of a language that doesn't change the way you program? The lessons that Rust teaches about safety can be useful when you’re working in other languages, too. +

+ If you choose Rust, usually you need the guarantees that the language provides: safety against null pointers and data races, predictable runtime behaviour, and total control over the hardware. If you don't require any of these features, Rust might be a poor choice for your next project. That's because these guarantees come with a cost: ramp-up time. You'll need to unlearn bad habits and learn new concepts. Chances are, you will fight with the borrow checker a lot when you start out.
+ —Matthias Endler +

+ How challenging you find Rust's programming model probably depends on what previous experience you have in other languages. Python or Ruby programmers may find it restrictive; others will be delighted. +

+ If you’re a C or C++ programmer who’s spent weeks chasing down memory safety bugs in those languages, you’ll really appreciate Rust. “Fighting the borrow checker” becomes “The compiler can detect that? Cool!”
+ —Grzegorz Nosek +

+ Scale +

+ Today's server programs comprise tens of millions of lines of code, are worked on by hundreds or even thousands of programmers, and are updated literally every day. Go was designed and developed to make working in this environment more productive. Go's design considerations include rigorous dependency management, the adaptability of software architecture as systems grow, and robustness across the boundaries between components.
+ —Rob Pike +

+ When you're working on a problem by yourself or in small teams, the choice of a simple language or a rich language is a matter of preference. But as the software grows bigger and more complex, and the teams grow larger, the differences really start to show. For large applications and distributed systems, speed of execution is less important than speed of development: a deliberately minimal language like Go reduces the ramp-up time for new developers, and makes it easier for them to work with a large codebase. +

+ With Go, it’s easier as a junior developer to be more productive, and harder as a mid-level developer to introduce brittle abstractions that will cause problems down the line. For these reasons, Rust is less compelling than Go for enterprise software development.
+ —Loris Cro +

+ When it comes to software development in the large, clear is better than clever. Go's limitations actually make it more suitable for enterprises and big organisations than more complex and powerful languages such as Rust. +

+ The differences +

+ Although Rust and Go are both popular, modern, widely-used languages, they're not really competitors, in the sense that they're deliberately targeting quite different use cases. Go's whole approach to programming is radically different to Rust's, and each language will suit some people while irritating others. That's absolutely fine, and if both Rust and Go did more or less the same things in more or less the same way, we wouldn't really need two different languages. +

+ So can we get a sense of the respective natures of Rust and Go by finding issues on which they take drastically different approaches? Let's find out. +

+ Garbage collection +

+ “To garbage-collect, or not to garbage-collect” is one of those questions that has no right answer. Garbage collection, and automatic memory management in general, makes it quick and easy to develop reliable, efficient programs, and for some people that makes it essential. But others say that garbage collection, with its performance overhead and stop-the-world pauses, makes programs behave unpredictably at run-time, and introduces unacceptable latency. The debate rumbles on. +

+ Go is a very different language to Rust. Although both can vaguely be described as systems languages or replacements for C, they have different goals and applications, styles of language design, and priorities. Garbage collection is a really huge differentiator. Having GC in Go makes the language much simpler and smaller, and easy to reason about. +

+ Not having GC in Rust makes it really fast (especially if you need guaranteed latency, not just high throughput) and enables features and programming patterns that are not possible in Go (or at least not without sacrificing performance).
+ —PingCAP +

+ Close to the metal +

+ The history of computer programming has been a story of increasingly sophisticated abstractions that let the programmer solve problems without worrying too much about how the underlying machine actually works. That makes programs easier to write and perhaps more portable. But for many programs, access to the hardware, and precise control of how the program is executed, are more important. Rust aims to let programmers get “closer to the metal”, with more control, but Go abstracts away the architectural details to let programmers get closer to the problem. +

+ Both languages have a different scope. Golang shines for writing microservices and for typical "DevOps" tasks, but it is not a systems programming language. Rust is stronger for tasks where concurrency, safety and/or performance are important; but it has a steeper learning curve than Go.
+ —Matthias Endler +

+ Must go faster +

+ I’ve written elsewhere that performance is less important than readability for most programs. But when performance does matter, it really matters. Rust makes a number of design trade-offs to achieve the best possible execution speed. By contrast, Go is more concerned about simplicity, and it’s willing to sacrifice some (run-time) performance for it. But Go’s build speed is unbeatable, and that’s important for large codebases. +

+ Rust is faster than Go. In the benchmarks above, Rust was faster, and in some cases, an order of magnitude faster. But before you run off choosing to write everything in Rust, consider that Go wasn’t that far behind it in many of those benchmarks, and it’s still much faster than the likes of Java, C#, JavaScript, Python and so on. +

+ If what you need is top-of-the-line performance, then you’ll be ahead of the game choosing either of these two languages. If you’re building a web service that handles high load, that you want to be able to scale both vertically and horizontally, either language will suit you perfectly.
+ —Andrew Lader +

+ Correctness +

+ On the other hand, a program can be arbitrarily fast if it doesn’t have to work properly. Most code is not written for the long term, but it’s often surprising how long some programs can stay running in production: in some cases, many decades. In these situations it’s worth taking a little extra time in development to make sure that the program is correct, reliable, and won’t need a lot of maintenance in the future. +

+ My take: Go for the code that has to ship tomorrow, Rust for the code that has to keep running untouched for the next five years.
+ —Grzegorz Nosek +

+ While both Go and Rust are great choices for any serious project, it's a good idea to make yourself as well-informed as possible about each language and its characteristics. Ultimately, it doesn't matter what anyone else thinks: only you can decide which is right for you and your team. +

+ If you want to develop faster, perhaps because you have many different services to write, or you have a large team of developers, then Go is your language of choice. Go gives you concurrency as a first-class citizen, and does not tolerate unsafe memory access (neither does Rust), but without forcing you to manage every last detail. Go is fast and powerful, but it avoids bogging the developer down, focusing instead on simplicity and uniformity. If on the other hand, wringing out every last ounce of performance is a necessity, then Rust should be your choice.
+ —Andrew Lader +

+ Conclusion +

+ I hope this article has convinced you that both Rust and Go deserve your serious consideration. If at all possible, you should aim to get at least some level of experience in both languages, because they will be incredibly useful to you in any tech career, or even if you enjoy programming as a hobby. If you only have time to invest in learning one language well, don't make your final decision until you've used both Go and Rust for a variety of different kinds of programs, large and small. +

+ And knowledge of a programming language is really only a small part of being a successful software engineer. By far the most important skills you'll need are design, engineering, architecture, communication, and collaboration. If you excel at these, you'll be a great software engineer regardless of your choice of language. Happy learning! +

+ Comparing Rust and Go code +

+ There's a great website called programming-idioms.org which has a "cheat sheet" showing what the Rust and Go code looks like for over 200 common programming tasks: +

\ No newline at end of file diff --git a/packages/readabilityjs/test/test-pages/blog.jetbrains.com/distiller.html b/packages/readabilityjs/test/test-pages/blog.jetbrains.com/distiller.html new file mode 100644 index 000000000..92fa9401b --- /dev/null +++ b/packages/readabilityjs/test/test-pages/blog.jetbrains.com/distiller.html @@ -0,0 +1,231 @@ +
Khalid Abuhakmeh

+ You’ve likely heard about Blazor, .NET’s frontend framework for writing web applications. Blazor lets you use your existing C# skills to build full-stack applications, from client to server. One way to construct Blazor applications is to compile your frontend into a WebAssembly (WASM) file and deploy and run your app in a web browser. The Blazor WASM approach can help you build client experiences similar to native desktop applications with a native performance profile. +

+ But what about WebAssembly on the server? In this post, we’ll see the benefits of using WASM outside the browser and its potential as a transformative technology on the server.  +

+ What is WebAssembly? +

+ WebAssembly is a low-level assembly-like language with a binary format that runs in modern web browsers. As a result, WebAssembly programs can perform at near-native speeds allowing for new and exciting applications for web clients from desktop-like apps, emulators, high-performance video games, photo editors, and more. The additional benefit of WebAssembly is it can be run either as an extra add-on via service workers, enhancing or replacing the existing JavaScript experience on the client. +

+ The designers of WebAssembly always meant it to run alongside the current web model. Developers unfamiliar with JavaScript can choose from a wide range of languages to write WebAssembly, including C, C++, C#, Go, Python and Rust, with many more languages looking at targeting WASM as an artifact of compilation. For .NET developers, you can think of the .wasm file extension as a .dll or .exe format. The file format is a standard, and WebAssembly System Interface (WASI) compliant runtime will be able to consume them. +

+ One such runtime we’ll be exploring in this post is Wasmtime. +

+ What is Wasmtime? +

+ Developed by the Bytecode Alliance, Wasmtime is a runtime for WebAssembly, allowing you to consume .wasm files and run them on Windows, macOS, or Linux systems outside a web client. Wasmtime is a fast, secure, and standards-compliant runtime for WebAssembly, and supports the WebAssembly System Interface (WASI). +

+ The most crucial feature of Wasmtime is its ability to interact with the native operating system, allowing for configurable access to system resources. Resources include disk access, TCP listeners, hardware input interfaces, and more. +

+ In addition to running standalone .wasm files, you can use Wasmtime inside your applications to consume third-party dependencies. The portability of the WASM format opens up a world where you can have native interop with a standard format across all languages. +

+ Wasmtime also supports debugging using popular native debugging tools like GDB or LLDB, which many of the IntelliJ-family products already support. The debugging experience depends on your technology stack’s build tools. +

+ Speaking of build tools, let’s build our first WASM server application, but first, let’s install Wasmtime. +

+ Installing Wasmtime +

+ Wasmtime is a CLI executable, so it’s straightforward to install on any environment. The first step is to visit the official Bytecode Alliance website. You can also just run the following command. +

+ curl https://wasmtime.dev/install.sh -sSf | bash +
+ curl https://wasmtime.dev/install.sh -sSf | bash + 
curl https://wasmtime.dev/install.sh -sSf | bash

+ Windows users can do the same using the Windows Subsystem for Linux (WSL), or grab the installer from the Wasmtime releases page. +

+ You can check you’ve successfully installed Wasmtime by running the command wasmtime –version from the command line. +

+ > wasmtime --version +
+ wasmtime-cli 1.0.1 + 
> wasmtime --version
+wasmtime-cli 1.0.1

+ Compiling a WASM .NET application +

+ Note: You’ll need the latest version of .NET 7 SDK installed +

+ Our goal in this section is to take a C# Console application and compile it to a .wasm file. The solution will be a self-contained application that won’t need unique WASI resources like TCP listeners, file access, or environment variables.  +

+ First, let’s start with a brand new Console Application project. From JetBrains Rider’s new solution window, select the “Console Application” template and call it “HelloWasm” or whatever you’d like.  +

+ Once the solution is ready, you’ll need to add the “Wasi.Sdk” NuGet package to your console project. If you don’t see the package, be sure that you have the Prerelease checkbox checked. +

Wasi.Sdk Nuget package in JetBrains Rider’s NuGet tool window

+ You’re ready to compile the console application into a .wasm artifact. Before doing so, change the line in Program.cs to the following: +

+ Console.WriteLine("Hello, Wasmtime!"); +
+ Console.WriteLine("Hello, Wasmtime!"); + 
Console.WriteLine("Hello, Wasmtime!");

+ Building your project, you will see references to bundling System libraries commonly used in .NET applications. Remember that WASM is native byte code. The .NET runtime must be packaged and shipped as part of your artifact. This is not any different than Blazor WASM deployments for a web client. +

Building a Wasi.Sdk-powered. NET project inside of JetBrains Rider

+ Now, let’s run our WASM file using Wasmtime. In Rider, you’ll use the Run Anything dialog (Ctrl+Ctrl) to run the following command: +

+ wasmtime ./HelloWasm/bin/Debug/net7.0/HelloWasm.wasm +
+ wasmtime ./HelloWasm/bin/Debug/net7.0/HelloWasm.wasm + 
wasmtime ./HelloWasm/bin/Debug/net7.0/HelloWasm.wasm

+ When successful, you should see your Console application’s output. +

Successful output in JetBrains Rider’s “Run Anything” tool window

+ Awesome! You just wrote your first WASM-targeted .NET application. Now you could deploy the .wasm file to any host that supports WebAssmebly. +

+ Consuming WASM and WAT modules from .NET +

+ While it’s certainly possible to consume a .wasm file in .NET, it is a bit more tedious than the plug-and-play experience of a NuGet package. Currently, there is a Wasmruntime NuGet package that contains low-level APIs for dealing with the elements of a host. These elements include an Engine, Module, Linker, Store, and more. They allow you to define your custom host, including low-level system calls and what calling them may do on a WASI-compliant system. For more information, I recommend reading excellent Mozilla documentation to understand how WASM interfaces with a host. +

+ I would not recommend most developers attempt to write their host; instead, the community should wait for the Byte Code Alliance to create a safe-by-default host. +

+ Let’s look at an example where we use the WebAssembly Text (WAT) format to link a C# function to WebAssembly. WAT is a human-readable and editable format representing the WASM binary format. +

+ using Wasmtime; +
+ using var engine = new Engine(); +
+ using var module = Module.FromText( +
+ engine, +
+ "hello", +
+ "(module (func $hello (import \"\" \"hello\")) (func (export \"run\") (call $hello)))" +
+ ); +
+ using var linker = new Linker(engine); +
+ using var store = new Store(engine); +
+ linker.Define( +
+ "", +
+ "hello", +
+ Function.FromCallback(store, () => Console.WriteLine("Hello from C#!")) +
+ ); +
+ var instance = linker.Instantiate(store, module); +
+ var run = instance.GetAction("run")!; +
+ run(); +
+ using Wasmtime; using var engine = new Engine(); using var module = Module.FromText( engine, "hello", "(module (func $hello (import \"\" \"hello\")) (func (export \"run\") (call $hello)))" ); using var linker = new Linker(engine); using var store = new Store(engine); linker.Define( "", "hello", Function.FromCallback(store, () => Console.WriteLine("Hello from C#!")) ); var instance = linker.Instantiate(store, module); var run = instance.GetAction("run")!; run(); + 
using Wasmtime;
+
+using var engine = new Engine();
+
+using var module = Module.FromText(
+    engine,
+    "hello",
+    "(module (func $hello (import \"\" \"hello\")) (func (export \"run\") (call $hello)))"
+);
+
+using var linker = new Linker(engine);
+using var store = new Store(engine);
+
+linker.Define(
+    "",
+    "hello",
+    Function.FromCallback(store, () => Console.WriteLine("Hello from C#!"))
+);
+
+var instance = linker.Instantiate(store, module);
+var run = instance.GetAction("run")!;
+run();

+ The critical thing to note in this example is that our WebAssembly module uses a reference to $hello. Since our C# application manages our WebAssembly context, we can create a definition for $hello using the method Function.FromCallback. In C#, we ask the linker, which now contains both our .wat definition and our new implementation of $hello for the run method. All that’s left is to invoke our Action and see the results. +

+ Hello from C#! +
+ Hello from C#! + 
Hello from C#!

+ While it may seem trivial, we’ll be able to run and consume WASM in our .NET applications. That opens up possibilities for solving problems using solutions from other ecosystems. +

+ The current reality of WASM on the server +

+ While this post has been relatively optimistic about the prospect of WASM, there are limitations to the technology that folks should be aware of, and they will vary based on the WASM host. +

+ The first significant limitation is threading. So far, with my testing, Wasmtime only has access to a single thread for execution. Thread limits are not entirely a deal-breaker, but you’ll need to rewrite some of your solutions to reduce the need for threading. This limitation becomes apparent when using an API that involves Thread or Task.Delay, which will halt your running process or terminate it catastrophically. However, threading may not be an issue in the near future as .NET adds multi-threaded support to WASM. Wasmtime also enables multiple threads with an experimental flag, but your technology stack will need to take advantage of that feature. +

+ The .NET runtime targeting WASM is the same as Blazor WASM, which is to say it has limits. APIs will not work based on a lack of support from the current WASM host. Missing APIs could limit your ability to solve specific problems. The APIs are still in active development and could change based on features added to Wasmtime and other WASI runtimes. +

+ Another issue I’ve found is there is currently no outward socket support. Lack of socket support limits a WASM application’s ability to communicate with dependencies like a database or web service. Discussions of a Socket specification will resolve this issue, allowing for more robust WASM apps. In addition, this should enable .NET developers to use data access tools like Entity Framework Core or Dapper with few issues. +

+ Tooling is another concern; vendors must catch up to the experiences .NET developers have gotten used to developing .NET applications. Luckily for developers, Wasmruntime uses standard debugging tools like LLDB and GDB, making integrating them into existing tools much more manageable. +

+ Infrastructure will play an essential role in how you build your WASM solutions. The strength of WASM comes from the ability to start and dispose of executing modules quickly in the vein of “microservices” or “functions”. However, a different architectural approach will require you to rethink your existing applications’ orchestration and units of work. In addition, you and your team may not have the bandwidth to deal with the technical and conceptual challenges simultaneously.  +

+ How does a modern WASM-powered application look when deployed? Some folks think it looks like functions as a service (FaaS), while others see it as a replacement for containers in a Kubernetes cluster, and both are perfectly valid. Luckily, many new WASM hosts are springing up along with reliable cloud providers AWS, Microsoft Azure, and Cloudflare, adding options to their ever-expanding list of services. +

+ Imagining the future of WASM on the server  +

+ We’ve reached the speculative part of the post, where we get to imagine the future of WASM and how it may transform your development experience, from writing code to deploying a solution. So let’s start with the process of development. +

+ If you’ve developed any Blazor WASM application, you’ll likely realize that little feels different while writing code. After all, it’s C#! The debugging experience may be slightly different, as you now deal with a completely different host and runtime, specifically the web browser. However, writing solutions to target WASM will feel very similar to Blazor WASM, and I expect the experience to improve as tooling catches up. Switching to WASM apps will be easier for most developers than switching to a new programming paradigm. +

+ The frameworks targeting WASM will likely change, as you will want to reconsider how and when functions are executing. For example, Fermyon released a .NET Spin SDK specifically for their hosting platform that targets WASM. Let’s look at how you might write a function given their SDK. +

+ using System.Net; +
+ using Fermyon.Spin.Sdk; +
+ namespace Microservice; +
+ public static class Handler +
+ { +
+ [HttpHandler] +
+ public static HttpResponse HandleHttpRequest(HttpRequest request) +
+ { +
+ return new HttpResponse +
+ { +
+ StatusCode = HttpStatusCode.OK, +
+ BodyAsString = "Hello from .NET", +
+ }; +
+ } +
+ } +
+ using System.Net; using Fermyon.Spin.Sdk; namespace Microservice; public static class Handler { [HttpHandler] public static HttpResponse HandleHttpRequest(HttpRequest request) { return new HttpResponse { StatusCode = HttpStatusCode.OK, BodyAsString = "Hello from .NET", }; } } + 
using System.Net;
+using Fermyon.Spin.Sdk;
+
+namespace Microservice;
+
+public static class Handler
+{
+    [HttpHandler]
+    public static HttpResponse HandleHttpRequest(HttpRequest request)
+    {
+        return new HttpResponse
+        {
+            StatusCode = HttpStatusCode.OK,
+            BodyAsString = "Hello from .NET",
+        };
+    }
+}

+ This example begs questions. For example, how do you transition from ASP.NET Core Minimal Endpoints or ASP.NET Core MVC to this new approach? Does ASP.NET Core need to rethink how to compile existing applications into deployable units? I expect a lot of conversations and strategies to emerge as developers adopt WASM. +

+ We could see WASM replace most instances of containerization, especially when WASM can package both runtimes and the applications that depend on them in one portable package. Advantages you could hope to see: Reduced CPU and memory usage, decreased cold-boot times, reduced hosting bills, and more significant economies of scale. +

+ These .wasm files can also be considerably smaller as they don’t have the same need for layers found in images built on entire operating systems. The portability and efficiency of WASM hosts will lead to more hosting options from vendors, new and old. +

+ Finally, the most exciting prospect of WASM on the server is the fulfillment of an agnostic cloud environment, where you, as the developer, no longer have to worry about regions. WASM can be deployed simultaneously to many global regions beyond cloud vendors’ limited and congested regions. WASM can start closest to your users and provide them with the fastest experience possible regardless of their country of origin. This democratization of user experience is most exciting for folks delivering global-scale applications. +

+ Conclusion +

+ In this post, we delved into WebAssembly and the efforts to run it on the server using Wasmtime. While it’s still relatively new, the promise of WASM on the server is exciting, and .NET is at the forefront of giving developers more hosting options. As tooling and development experience improves, so will the solutions developers deliver to customers. As you’ve seen, the experience even now is good. +

+ There are limitations with WASM, but like all technologies, the community will push solutions forward, and progress will occur. It’s still important to be careful when choosing cutting-edge technologies as solutions, as you’ll likely be some of the first and only folks experiencing those issues.  +

+ That said, I’m cautiously optimistic about the future of WASM, and I hope you found something in this article that has sparked your curiosity to explore the topic further. +

\ No newline at end of file diff --git a/packages/readabilityjs/test/test-pages/bookofhook.blogspot.com/distiller.html b/packages/readabilityjs/test/test-pages/bookofhook.blogspot.com/distiller.html new file mode 100644 index 000000000..8df5aa7a9 --- /dev/null +++ b/packages/readabilityjs/test/test-pages/bookofhook.blogspot.com/distiller.html @@ -0,0 +1,876 @@ +
Productivity is one of my pet topics, because it's always dogged me a bit, especially early in my career.  I'd pull long days and nights and then realize I only actually worked (as in, typing in code, debugging stuff, and thinking about problems and their solutions) maybe 20% of the time.  Upon talking to coworkers, this seemed to be normal, a part of the expected friction costs incurred working in an office environment.  Meetings, shooting the shit with coworkers, lunch, email, and, er, stuff.
+
This is what I mean by 'stuff' in 1996
+ Eventually working around high-productivity professionals like John Carmack made me realize that if you want to excel, then you have to work hard and focus the whole time.
John Carmack Productivity Measurement Tool ca. 1998
+ I remember Carmack talking about productivity measurement.  While working he would play a CD, and if he was not being productive, he'd pause the CD player.  This meant any time someone came into his office to ask him a question or he checked email he'd pause the CD player.  He'd then measure his output for the day by how many times he played the CD (or something like that -- maybe it was how far he got down into his CD stack).  I distinctly remember him saying "So if I get up to go to the bathroom, I pause the player".
+
+ You know what's pretty hardcore?  Thinking that  going to the bathroom is essentially the same as fucking off.
+ (Sidenote: my memory is hazy on this since it was long ago -- CD PLAYERS yo).
+
+ That level of work ethic and focus is rare, particularly with my generation of programmers who were raised with the vile "work smart, not hard" mantra, coupled with the sense that we were somehow significantly brighter than most of our peers.  Combine those two notions and add the Scotty Principle and you get....me.
If my coworkers were grinding through stuff that took them 20 hours, I'd be all "I work smart, not hard" with accompanying snarky eye roll, and I'd assume I could bust through an equivalent work load in four hours and still look like a goddamn superhero.
+ And sometimes I could, so, hey, validation!  
+ And you can skate by like this (God knows I did) for a long time before a couple things eventually rear their heads and bite you in your entitled face.

+ Productivity Deficit: Your Attitude Writing Checks Your Work Ethic Can't Cash

+ An overinflated sense of your own abilities creates a constant state of production deficit, because you assume that you can make it up with a burst of brilliance and/or crunch.  
+ But there is no countering surplus to offset the deficit.  The only way surpluses show up is when you finish a (presumably) hard task much faster than you anticipated.  But instead of banking the surplus (i.e. moving on immediately to your next task), you spend it relaxing and screwing off because, whew, you just earned a small vacation by busting shit out in an hour that you thought would take all day.  Hey, what's on Facebook and Twitter right now?  Do I have any new mail?  No?  I wonder if anyone has tweeted something recently since the last time I checked...what about Reddit?  Oh, an inspirational/funny/cool YouTube video, it's only eight minutes long, let me watch it now!  (Eight minutes later) Sweet...what's up Twitter?  Oh man, I have to make a snarky response, but first I have to Google for that XKCD comic that totally makes my point for me...
+ And before you know it, the day is done, and you're still feeling good because you finished in one hour what should have taken all day, so all good, right?
+

+ Trap of the Easy Task

+ And yeah, it's often all good, but when operating at a slight deficit things can go pear shaped quickly when you accidentally spring the trap of the easy task.  Tasks so trivial that they barely register as work.  Update an SDK?  Hour, tops.  Implement this feature that I've already done on every other platform?  Hour, tops.  This is all mindless grunt work that anyone could do, it's just clicking through dialog boxes, maybe changing a few lines of source, rebuilding, whatever.
+ In other words, an easy task like this is so easy that it's a constant time cost for everyone irrespective of ability, so there's no opportunity nor need for crazy overestimation since what could possibly go wrong?
+ Well, as with so many things, it's the unexpected things that screw you the hardest.  The things that should be trivial end up taking days because, shit, that SDK install required a new compiler install, and the new compiler install failed and I can't uninstall it, and now I have to fucking WIPE MY WHOLE GODDAMN HARD DRIVE TO RECOVER.  Or you need to implement something that was trivial on nine other platforms, and it should be 30 minutes on platform number ten, except it lacks a base feature that is present on everything else, the compiler is dying with an "Internal compiler failure 2033", the API call you need to use hangs the dev console immediately and, oh, the dev support site you use to get release notes is totally down for the foreseeable future.
+ So what should have taken less than an hour took a week.
+ It's like having a perfect monetary budget that assumes no crazy "one time" only bills, except life is full of crazy one time only bills and the only way you can keep those under control is by giving yourself a budgetary capacitor to dampen the fluctuations.
+ And now you're defensive about losing a week to something stupid because you budgeted an hour for it and waited until the last second to do it and now the schedule has gone to hell, but it's not your fault, because it could have happened to anyone!  But if you had banked your surplus hours before and/or worked at closer to your theoretical peak effectiveness then this type of thing would get absorbed in the wash.
+ And now you live in a state of mild panic because you're way smarter than all this but you're never actually getting things done on time.

+ Identity Recalibration Crisis

+ Which leads to a potential identity recalibration crisis upon landing at a company with high performers that work hard and smart.  And that will happen if you're good.  Now you're no longer at the top of the curve.  In fact, shit, you're in the middle or bottom of the curve, a situation your brain probably never considered as an option.
+
+
This dude was my wake up call with respect to my
+ over inflated sense of skill
+ I went through this when I went to id software.  I had rationalized that John Carmack's success was just a factor of timing and luck since, hell, he was my age, and I was pretty smart, so what else could it be?  Upon my arrival I had a crash course in humility, because he was way smarter than me and way more productive as well.
+
+ This took a while to sink in, because until then I was used to believing I was one of the better programmers at a company.  And then working with Carmack I realized he was not just a little better, he was orders of magnitude better.  And I couldn't dismiss it with a "But I write a lot of code" hand wave, because he also wrote like 80% of the Quake code base.
+
+ Sometimes it takes a while for this to sink in.  Sometimes it doesn't sink in at all, and you're let go because you're not very productive compared to your new team of high performers.  Sometimes it sinks in, and you panic and get depressed because your self-image has always been that of being the strongest swimmer in the school, and right now you're just trying not to drown, much less keep up with everyone else.
+ But ideally you shrug off the old counter productive mentality and habits and emerge as another one of the high functioning team members, but that can take a lot of work, particularly if you have to get over years of giving it twenty percent.
+

+ Killing the Underachiever

+ If my pithy advice were little more than "Be more like John Carmack" then I can imagine a lot of readers throwing up their hands and saying "Well, fuck it, I'm a lost cause, because that's not going to happen."  But what I can do is relate some of the things that helped me kill off some of my underachieving habits.  The point isn't to become a superstar, it's to become better, since that's always the first step.
+
+ I don't believe in mechanical solutions ("Turn off the internet", "Listen to music", and stuff like that) because I don't think they address the core issues, which are psychological.  Instead I found that I had to do the following.
+
+
  1. Develop self-awareness.  It took working with John Carmack and other high productivity programmers and admitting that they were way more productive than me before I really understood how much more productive I could be.  In other words, if you don't admit it's something worth improving, then obviously you're not going to search for a solution, and if that's the case, you should go here or here.
  2. Give a shit.  Originally I called this "develop a sense of urgency", but really it's just about caring about getting your work done.  It doesn't even matter what you specifically care about!  It can be your professional image, your product, your team, your customers, whatever.  You just have to care about something that will drive you to getting things done, because if you don't, apathy will occupy that void.
  3. Minimize uncertainty.  In another blog article, Productivity vs. Uncertainty and Apathy, I talk about how poorly defined goals can lead to poor productivity.  If it's unclear what you need to get done today, then there's a reasonable chance you won't actually do anything today.
  4. Commit to getting something done every day.  When you show up in the morning have a well defined set of things to finish that day.  Stay as late as you have to in order to finish.  By committing to finishing that task other distractions will naturally fall by the wayside.  For example, I have jiu-jitsu training at 7pm.  If I screw off too much during the day, I don't get to train.  Also, by committing to a task, you avoid "being busy" instead of "getting work done", they're not the same thing.
  5. Never say "I'll finish it up tomorrow" or "I'll make up for it by coming in early/staying late/working the weekend".  This is an easy trap to get into, where you keep incurring time debt until at some point you realize you're now three weeks behind on a task that should have taken two days.  This is like racking up credit card bills assuming you can pay them off later.  Which is fine, until "later" arrives and you've only accumulated more debt.
  6. Do not overpromise to make up for poor productivity.  There's a tendency when we're falling behind to try to overcompensate with future promises.  "When I'm done, it'll be AWESOME" or "I know I'm late, but I'm positive I'll be done by Monday".  By doing those things we just build more debt we can't pay off, and that will eventually lead to a catastrophic melt down when the super final absolutely last deadline date shows up.  Just get shit done, don't talk about how you're going to get shit done.
  7. Have an objective productivity metric.  This is a mechanical thing, but it acts as a reasonable backstop.  If you have changelogs you can reference, then it's easy to sit down on Friday and ask yourself "What have I done this week?"  And if you know that it's possible for others to check on you, then it makes you take each day a lot more seriously.  If you judge your value solely on output, not subjective things like "being smart", you will be more productive.
  8. Accept that "the grind" is part of the job.  A friend of mine's father has a great quote: "Son, i don't wake up every day and go to a place called fun. I wake up and go to a place called work"  You can't get irate or frustrated that the bulk of the day is typing in boring code and dealing with bugs and other people.
+ I still screw off during the day.  I am not a code grinding automaton.  I read Facebook, chat with others, Tweet, shop on Amazon, get coffee, read forums, write blog posts like this one and I'm totally fine with that.  
+ Because I know I'm committed to getting things done as well.  So even though I'm writing  this right now at 3pm, if I have to work until 8pm to get my shit done and checked in, I'll do that, so that tomorrow is a fresh start, not a day of "Oh goddamit, let's try again to finish that task I started a week ago."
+ Once I developed that sense of daily urgency my productivity went up considerably.  And if I'm really productive in the morning and afternoon and feel like I can go home at 4pm with a clean conscience, I do that too, so I don't burn out.  
+ It's a push/pull between banking surplus and generating a deficit, just make sure you don't do just one or the other.
+ + + Posted 12th March 2013 by Unknown + +
  1. + + + + + + + +
    I was originally going to write this blog about a different topic, one of the endless online discussions about "How to get better at this thing you suck at".  Then I realized it was pointless.
    +
    + Many "How to make your life/code/job/hobby better" blogs are very well written, insightful, and even inspirational.  They provide awareness into our failings and weaknesses and hint at ways in which we can improve.  They give us strategies and high level conceptual ideas on how to approach problems.
    +
    + The issue is that it's all mostly pointless.  Finding the best plan doesn't mean we'll execute it, because very often the problem is that we suck at execution. The very things we're trying to fix are often the very things that prevent us from fixing them.
    +
    + Or, put another way, it's a rare situation where someone doesn't know what to do, is told what to do, then goes off and just does it.
    +
    + Have a nice day.
    +
    +
    + +
    + + + Posted 25th April 2014 by Unknown + +
    + + + + + + + +
    + + + + + + +
    + + +
    + + + + + + + + + + +
  2. + + + + + + + +
    Productivity is one of my pet topics, because it's always dogged me a bit, especially early in my career.  I'd pull long days and nights and then realize I only actually worked (as in, typing in code, debugging stuff, and thinking about problems and their solutions) maybe 20% of the time.  Upon talking to coworkers, this seemed to be normal, a part of the expected friction costs incurred working in an office environment.  Meetings, shooting the shit with coworkers, lunch, email, and, er, stuff.
    + +
    + +
    This is what I mean by 'stuff' in 1996
    +
    +
    +
    + Eventually working around high-productivity professionals like John Carmack made me realize that if you want to excel, then you have to work hard and focus the whole time.
    + +
    + +
    + +
    John Carmack Productivity Measurement Tool ca. 1998
    + I remember Carmack talking about productivity measurement.  While working he would play a CD, and if he was not being productive, he'd pause the CD player.  This meant any time someone came into his office to ask him a question or he checked email he'd pause the CD player.  He'd then measure his output for the day by how many times he played the CD (or something like that -- maybe it was how far he got down into his CD stack).  I distinctly remember him saying "So if I get up to go to the bathroom, I pause the player".
    +
    + You know what's pretty hardcore?  Thinking that  going to the bathroom is essentially the same as fucking off.
    + +
    + (Sidenote: my memory is hazy on this since it was long ago -- CD PLAYERS yo).
    +
    +
    + That level of work ethic and focus is rare, particularly with my generation of programmers who were raised with the vile "work smart, not hard" mantra, coupled with the sense that we were somehow significantly brighter than most of our peers.  Combine those two notions and add the Scotty Principle and you get....me.
    + +
    +
    If my coworkers were grinding through stuff that took them 20 hours, I'd be all "I work smart, not hard" with accompanying snarky eye roll, and I'd assume I could bust through an equivalent work load in four hours and still look like a goddamn superhero.
    + +
    + And sometimes I could, so, hey, validation!  
    + +
    + And you can skate by like this (God knows I did) for a long time before a couple things eventually rear their heads and bite you in your entitled face.
    +
    +

    + Productivity Deficit: Your Attitude Writing Checks Your Work Ethic Can't Cash

    +
    +
    + An overinflated sense of your own abilities creates a constant state of production deficit, because you assume that you can make it up with a burst of brilliance and/or crunch.  
    + +
    + But there is no countering surplus to offset the deficit.  The only way surpluses show up is when you finish a (presumably) hard task much faster than you anticipated.  But instead of banking the surplus (i.e. moving on immediately to your next task), you spend it relaxing and screwing off because, whew, you just earned a small vacation by busting shit out in an hour that you thought would take all day.  Hey, what's on Facebook and Twitter right now?  Do I have any new mail?  No?  I wonder if anyone has tweeted something recently since the last time I checked...what about Reddit?  Oh, an inspirational/funny/cool YouTube video, it's only eight minutes long, let me watch it now!  (Eight minutes later) Sweet...what's up Twitter?  Oh man, I have to make a snarky response, but first I have to Google for that XKCD comic that totally makes my point for me...
    + +
    + And before you know it, the day is done, and you're still feeling good because you finished in one hour what should have taken all day, so all good, right?
    +

    + Trap of the Easy Task

    +
    +
    + And yeah, it's often all good, but when operating at a slight deficit things can go pear shaped quickly when you accidentally spring the trap of the easy task.  Tasks so trivial that they barely register as work.  Update an SDK?  Hour, tops.  Implement this feature that I've already done on every other platform?  Hour, tops.  This is all mindless grunt work that anyone could do, it's just clicking through dialog boxes, maybe changing a few lines of source, rebuilding, whatever.
    + +
    + In other words, an easy task like this is so easy that it's a constant time cost for everyone irrespective of ability, so there's no opportunity nor need for crazy overestimation since what could possibly go wrong?
    + +
    + Well, as with so many things, it's the unexpected things that screw you the hardest.  The things that should be trivial end up taking days because, shit, that SDK install required a new compiler install, and the new compiler install failed and I can't uninstall it, and now I have to fucking WIPE MY WHOLE GODDAMN HARD DRIVE TO RECOVER.  Or you need to implement something that was trivial on nine other platforms, and it should be 30 minutes on platform number ten, except it lacks a base feature that is present on everything else, the compiler is dying with an "Internal compiler failure 2033", the API call you need to use hangs the dev console immediately and, oh, the dev support site you use to get release notes is totally down for the foreseeable future.
    + +
    + So what should have taken less than an hour took a week.
    + +
    + It's like having a perfect monetary budget that assumes no crazy "one time" only bills, except life is full of crazy one time only bills and the only way you can keep those under control is by giving yourself a budgetary capacitor to dampen the fluctuations.
    + +
    + And now you're defensive about losing a week to something stupid because you budgeted an hour for it and waited until the last second to do it and now the schedule has gone to hell, but it's not your fault, because it could have happened to anyone!  But if you had banked your surplus hours before and/or worked at closer to your theoretical peak effectiveness then this type of thing would get absorbed in the wash.
    + +
    + And now you live in a state of mild panic because you're way smarter than all this but you're never actually getting things done on time.
    +
    +

    + Identity Recalibration Crisis

    +
    +
    + Which leads to a potential identity recalibration crisis upon landing at a company with high performers that work hard and smart.  And that will happen if you're good.  Now you're no longer at the top of the curve.  In fact, shit, you're in the middle or bottom of the curve, a situation your brain probably never considered as an option.
    +
    + +
    + +
    This dude was my wake up call with respect to my
    + over inflated sense of skill
    + I went through this when I went to id software.  I had rationalized that John Carmack's success was just a factor of timing and luck since, hell, he was my age, and I was pretty smart, so what else could it be?  Upon my arrival I had a crash course in humility, because he was way smarter than me and way more productive as well.
    +
    + This took a while to sink in, because until then I was used to believing I was one of the better programmers at a company.  And then working with Carmack I realized he was not just a little better, he was orders of magnitude better.  And I couldn't dismiss it with a "But I write a lot of code" hand wave, because he also wrote like 80% of the Quake code base.
    +
    +
    + Sometimes it takes a while for this to sink in.  Sometimes it doesn't sink in at all, and you're let go because you're not very productive compared to your new team of high performers.  Sometimes it sinks in, and you panic and get depressed because your self-image has always been that of being the strongest swimmer in the school, and right now you're just trying not to drown, much less keep up with everyone else.
    + +
    + But ideally you shrug off the old counter productive mentality and habits and emerge as another one of the high functioning team members, but that can take a lot of work, particularly if you have to get over years of giving it twenty percent.
    +

    + Killing the Underachiever

    + If my pithy advice were little more than "Be more like John Carmack" then I can imagine a lot of readers throwing up their hands and saying "Well, fuck it, I'm a lost cause, because that's not going to happen."  But what I can do is relate some of the things that helped me kill off some of my underachieving habits.  The point isn't to become a superstar, it's to become better, since that's always the first step.
    +
    + I don't believe in mechanical solutions ("Turn off the internet", "Listen to music", and stuff like that) because I don't think they address the core issues, which are psychological.  Instead I found that I had to do the following.
    +
    +
    1. Develop self-awareness.  It took working with John Carmack and other high productivity programmers and admitting that they were way more productive than me before I really understood how much more productive I could be.  In other words, if you don't admit it's something worth improving, then obviously you're not going to search for a solution, and if that's the case, you should go here or here.
    2. Give a shit.  Originally I called this "develop a sense of urgency", but really it's just about caring about getting your work done.  It doesn't even matter what you specifically care about!  It can be your professional image, your product, your team, your customers, whatever.  You just have to care about something that will drive you to getting things done, because if you don't, apathy will occupy that void.
    3. Minimize uncertainty.  In another blog article, Productivity vs. Uncertainty and Apathy, I talk about how poorly defined goals can lead to poor productivity.  If it's unclear what you need to get done today, then there's a reasonable chance you won't actually do anything today.
    4. Commit to getting something done every day.  When you show up in the morning have a well defined set of things to finish that day.  Stay as late as you have to in order to finish.  By committing to finishing that task other distractions will naturally fall by the wayside.  For example, I have jiu-jitsu training at 7pm.  If I screw off too much during the day, I don't get to train.  Also, by committing to a task, you avoid "being busy" instead of "getting work done", they're not the same thing.
    5. Never say "I'll finish it up tomorrow" or "I'll make up for it by coming in early/staying late/working the weekend".  This is an easy trap to get into, where you keep incurring time debt until at some point you realize you're now three weeks behind on a task that should have taken two days.  This is like racking up credit card bills assuming you can pay them off later.  Which is fine, until "later" arrives and you've only accumulated more debt.
    6. Do not overpromise to make up for poor productivity.  There's a tendency when we're falling behind to try to overcompensate with future promises.  "When I'm done, it'll be AWESOME" or "I know I'm late, but I'm positive I'll be done by Monday".  By doing those things we just build more debt we can't pay off, and that will eventually lead to a catastrophic melt down when the super final absolutely last deadline date shows up.  Just get shit done, don't talk about how you're going to get shit done.
    7. Have an objective productivity metric.  This is a mechanical thing, but it acts as a reasonable backstop.  If you have changelogs you can reference, then it's easy to sit down on Friday and ask yourself "What have I done this week?"  And if you know that it's possible for others to check on you, then it makes you take each day a lot more seriously.  If you judge your value solely on output, not subjective things like "being smart", you will be more productive.
    8. Accept that "the grind" is part of the job.  A friend of mine's father has a great quote: "Son, i don't wake up every day and go to a place called fun. I wake up and go to a place called work"  You can't get irate or frustrated that the bulk of the day is typing in boring code and dealing with bugs and other people.
    +
    + I still screw off during the day.  I am not a code grinding automaton.  I read Facebook, chat with others, Tweet, shop on Amazon, get coffee, read forums, write blog posts like this one and I'm totally fine with that.  
    +
    +
    +
    +
    +
    + Because I know I'm committed to getting things done as well.  So even though I'm writing  this right now at 3pm, if I have to work until 8pm to get my shit done and checked in, I'll do that, so that tomorrow is a fresh start, not a day of "Oh goddamit, let's try again to finish that task I started a week ago."
    + +
    +
    +
    + Once I developed that sense of daily urgency my productivity went up considerably.  And if I'm really productive in the morning and afternoon and feel like I can go home at 4pm with a clean conscience, I do that too, so I don't burn out.  
    + +
    + It's a push/pull between banking surplus and generating a deficit, just make sure you don't do just one or the other.
    +
    +
    + +
    + + + Posted 12th March 2013 by Unknown + +
    + + + + + + + +
    + + + + + + +
    + + +
    + + + + + + + + + + +
  3. + + + + + + + +
    Telemetry was the first product I worked on where I interacted directly with customers.  This interaction creates an interesting real time tension between working on big features with long development periods versus working on smaller features that can immediately benefit a small number of customers.
    + +
    + Typically it starts with a reasonable request -- let's call it Feature X -- so we don't want to just say 'no'.  But of course, we have to deal with bug reports, customer support, sales assistance, new platforms and previously promised or planned features, and all within the constraint of monthly releases.  
    + +
    + As a result the choice boils down to "Do you want some of Feature X now, or all of Feature X much later?"  In a perfect world we just do Feature X right and everyone wins, but in the real world of constrained bandwidth we have to be deliberate about scheduling work.
    + +
    + Very often Feature X can be implemented in a limited form and the customer is happy because, hey, what wasn't possible is now possible.  And if we focus on just that customer's response then this is a net win, because that partial implementation may have taken only an hour whereas a proper, fully fleshed out implementation might have taken several days due to cascading side effects.
    + +
    + The problem arises when you look at this new feature in context of other customers.  Now they see "Whoa, Feature X, cool!" or maybe just "Huh, didn't know about Feature X, I'll give it a try".  The former camp has learned to work around the absence of Feature X, the latter camp didn't even know it existed and didn't care, but now that they're aware of it they want to use it.
    + +
    + And this is when the partial implementation absolutely turns out to be the wrong call.  Because for that one customer who thinks "Those guys are awesome, they were super responsive to a feature request!" you now have multiple (potential) customers thinking "This feature is janky, what the hell?"
    + +
    + And they don't care about the reasons for it.  They don't care that it wasn't even going to get implemented until a customer requested it, or the reasons that a full implementation isn't practical or maybe even possible, they're just kind of annoyed that Feature X is incomplete and/or buggy.
    + +
    + The lesson is this: if you're not going to do it right, then you have to really think hard about whether it's worth doing.  Saying no to one customer is a lot better than explaining to all the others why Feature X isn't as good as they expect. And it's a tough thing to say, because it's easy to rationalize that a limited Feature X is better than no Feature X, but in my experience this is almost never the case.
    + +
    +
    + +
    + + + Posted 20th February 2013 by Unknown + +
    + + + + + + + +
    + + + + + + +
    + + +
    + + + + + + + + + + +
  4. + + + + + + + +
    In software development it's easy to lose sight of the fact that we're supposed to be shipping software.  We can show up to work, be legitimately busy for eight or nine hours, go home, and then realize that we didn't get any closer to shipping.
    +
    +
    + Many people are fine with this -- "Well, I 'did stuff', so hey, all good, right?" -- despite not getting any closer to shipping.  Their day was consumed with necessary-but-not-shipping tasks.  Things like installing software, replacing a hard drive, helping a coworker, setting up a dev kit, or debugging a feature you thought you were done with a year ago.  This hidden but large frictional load can easily consume so much of our time that over a given period of time, we never actually get closer to ship.
    +
    + This is particularly true with mature products, where you now spend a lot of time on non-development tasks  like support, sales, meetings, and training, and the 'being busy' part can become so dominant that you never get around to shipping a new version.  This overhead is rarely factored into any schedule since coders are notoriously shitty at accounting for non-coding things.  We think in best case terms like "That should take like a day" with the assumption "day" means "calendar day" but we never get a calendar day to do stuff, so "day" means "eight hours" which takes a week to get done.
    +
    + Assuming that crunching non-stop is off the table as an option, I've found that the biggest productivity boost I've had comes from asking myself "What do I need to get done today?" instead of "What will I work on today?".  By creating a sense of urgency to finish a well defined task I now have to be cognizant of all the little interruptions that interfere with that goal through the day.
    +
    + So get shit done and stop just being busy.  Or to quote David Wong (or Jesus, take your pick), a tree is judged by its fruit.
    +
    +
    +
    +
    + +
    + + + Posted 8th January 2013 by Unknown + +
    + + + + + + + +
    + + + + + + +
    + + +
    + + + + + + + + + + +
  5. + + + + + + + +
    If you're not getting shit done, it may not be because you suck.  Traditionally unproductive workers get nailed with labels like undisciplined, lack of focus, or poor work ethic, but in my experience these are not nearly as influential as I'd thought.
    +
    + Over the past two decades of coding, including stints in management/as a lead/and being self-directed with no immediate manager, I've come to the conclusion that the main issues that impact productivity are uncertainty and apathy.
    +
    + When uncertainty kicks me in the nuts, it's usually in one of several flavors: high-level uncertainty, strategic uncertainty, and low-level uncertainty.
    +
    + +
    + +
    "I feel like I should be doing something, but I have no idea where the fuck I am"
    + High-level uncertainty, aka "Where am I and what an I trying to do?", is when you have a general goal that comes with a set of implicit assumptions.  Those assumptions are a minefield you have to traverse as you try to get to your finished product.  You end up paralyzed because you're not sure which direction is safe, especially since none of the mines are marked.  For example, you might have the task of "In the next month improve our network performance by 50%".  It seems like a concrete goal, but there are still a ton of hidden assumptions in there.
    +
    +
    + Solution: Constrain your problem so that what was once vague is now extremely well specified.  Constraints tend to dictate solutions.  Okay, I have to improve network performance by fifty percent.  Can I use more memory to do this?  Can I use more CPU?  How much more CPU?  Can I use a third party library?  Is our performance poor sending or receiving or both?  Under what conditions?
    +
    + Or, using our lost in the woods analog, get a goddamn map.
    + +
    + +
    "So we have a lot of bugs, and apparently orcs and dragons"
    + Once questions like those are answered the scope of the problem and available solutions becomes a lot clearer, and we can move on to tackling the next problem.
    +
    + Strategic uncertainty occurs when you have a well defined goal but you're unsure how to go about it.  Using our previous example you've determined that the problem is total network bandwidth, on both sides, and that you can sacrifice a lot of CPU to do it.  It seems to happen all the time under all conditions, and unfortunately most of the data going across the network comes from a bunch of different systems so you can't just pare down the data at one location.
    +
    + Now you have to come up with a technical solution.  You know you need to compress and decompress the data, but how?  Maybe you have no experience with compression algorithms, so now you're kind of stuck.  Do you just use zlib?  Or miniz?  Or do you license something?  Is there a hardware solution?  If there's no clear solution at hand ("drop in zlib") then even very good programmers will spin their wheels, unsure of how to tackle it.  They just hope for a brilliant insight.
    +
    + Solution: Grind it out by talking to someone.  Having an office mate helps tremendously so you can just say "I have to do this thing, I'm not even sure how to tackle it".  People like to sound smart and give their opinions, so give someone else the opportunity to solve your problem for you!
    +
    + In this case the simplest solution is to use a package like zlib to compress and decompress your data.  Strategy defined!
    +
    + This leaves the day to day tasks of getting it done, and here's where low-level uncertainty can creep in.  Now you know what you're supposed to do at a fairly specific level ("Implement zlib in our codebase so that the network layer is much faster") but at any given moment you're just staring at your editor, unsure what to do next.
    +
    + Solution: Make a list.  It's that simple.  Instead of writing code, start by getting yourself organized and enumerate all your fine grained tasks, no matter how small.  "Find compression and decompression entry points".  "Get compression code into build system".  "Implement dummy implementation to verify integration works." "Write a test harness".  etc.  Once you have a list, sort in terms of priority.  And if you have any tasks that would take longer than a day, go ahead and break them out into smaller tasks.
    +
    + You now have a clear road map from A to B, broken down into actionable items at a very fine grained level.  There's no more uncertainty.  If you're driving, this is the part where your GPS has figured out how to get you from  your current location to the nearest Five Guys without having to ask for directions or pull over.
    +
    + So what do you do if you're still not getting shit done?!  The glib answer is something along the lines of...
    +
    + +
    + +
    ...but this doesn't really help, now does it?
    + It's likely that you're just not interested in the work.  If you're passionate and interested in something, then the work is fun and interesting and you want to do it.  Unfortunately a lot of the work we do in software -- in fact, the bulk of work in software development -- is excruciatingly dull.  Finding obscure bugs.  Dealing with build system issues.  Hunting down compiler switches.  Typing in the same code we've typed in hundreds of times before.  Dealing with poor or missing API documentation.
    +
    + For this stuff, you just have to grind.  This is where work ethic and discipline can come into play, but I prefer to cheat and just try to stick to things I find interesting.  Unfortunately the amount of interesting things to work on in a typical software project is dwarfed by the sheer amount of bullshit tasks.
    +
    +
    +
    + +
    + + + Posted 11th September 2012 by Unknown + +
    + + + + + + + +
    + + + + + + +
    + + +
    + + + + + + + + + + +
  6. + + + + + + + +
    Related to my principle of "suffer no jankiness", I've realized that almost all bugs can be distilled into three main categories: I Have No Idea What I'm Doing; Oops; and Oh Fuck You.
    +
    + I Have No Idea What's Going On bugs are the worst, because they're due to a lack of comprehension of what's supposed to be occurring in the software (i.e. jankiness).  We attempt to "fix" these bugs by near random transformations until things "work", but of course it's just masked, at best, until it crops up somewhere else.
    +
    + Examples of this include:
    +
    + v.z = -v.z; // WTF?  No idea why this works.
    +
    + And:
    +
    + char buffer[ sizeof( Foo ) + 128 ]; // Add extra just in case...
    +
    + "In case" of what?
    +
    + LaunchThreads();
    + Sleep(1000); // give it some time to startup
    +
    + "Some time"?  Where did the magical 1000ms value come from?
    +
    + And:
    +
    + if ( !system.Initialized() )
    +    system.Initialize(); // Just in case it wasn't initialized already
    +
    + Any time you see "just in case" in a comment, that's bad.
    +
    + I Have No Idea What's Going On bugs are a symptom of flailing about trying to make something work that would probably work if you just took a step back and started over from first principles.  But you're in a hurry, you're bored, you're frustrated, and you hope that one more transposition or sign flip will suddenly make everything right.
    +
    + These bugs are also really, really bad because they breed.  They create more bugs like themselves and they also create a ton of Oh Fuck You bugs for other programmers who have to figure out why the hell you decided that you needed to reset the graphics hardware state every time your function is called "just in case" it wasn't in a known good state.
    +
    + Oops bugs are the forgivable ones.  They don't show a lack of comprehension of the software, they were just dumb oversights, typos, copy and paste bugs, off-by-1, etc.  These are also generally the easiest bugs to hunt down because you're starting with a concrete mental model of how things are supposed to work, and then figuring out why it's not.
    +
    + double l = sqrt( v.x*v.x + v.x*v.x + v.x*vx );
    +
    + Those happen to everyone.  Yeah, you can avoid them with better testing and what have you, but by and large they're not symptoms of incomprehension.  In my experience each programmer has a fixed rate at which they create these bugs, which is a reflection of their habits.
    +
    + Oh Fuck You bugs are the ones where, once you realize what's happened, you say "Oh, fuck you [entity name here]" because someone else has inadvertently sent you on an hours or days long bug hunt that is not of your doing.
    +
    + Compilers generate incorrect code, tools corrupt data, SDKs are incorrectly documented, third party libraries do the wrong thing, drivers are broken, someone overclocked their CPU which led to weird memory errors, etc.  These bugs tend to occur at a fixed rate irrespective of the individual programmer since they're out of control of the individual programmer.
    +
    + In a perfect world you only ever encounter Oh Fuck You bugs, but if you can keep the frequency of Oh Fuck You and Oops bugs higher than those of I Have No Idea What's Going On then you're probably doing pretty well.
    +
    +
    + +
    + + + Posted 8th August 2012 by Unknown + +
    + + + + + + + +
    + + + + + + +
    + + +
    + + + + + + + + + + +
  7. + + + + + + + +
    I'm fairly old (40+) for a programmer, and you'd think most skill jumps would have occurred by now, and I think that in terms of raw ability that's true. Most people learn and apply everything in their first 10 years or whatever of doing things -- their experience will broaden as they get older, which provides a larger set of experiences to pattern match against, but by and large I think their basic 'programming smarts' is set at that point.
    +
    + BUT, philosophical outlooks and habits can still grow, and these can materialize as skill jumps. Lately I've been trying to be more intolerant of perceived jankiness in my code.
    +
    + A quick definition: code jankiness occurs when the mental model of your software is no longer accurate or complete.  Hacks are not janky, for example, as long as you know how and why they work.  This, on the other hand, is classic jankiness:
    +
    + char buffer[ sizeof( foo ) + 128 ]; // a little extra, just in case...
    +
    + Every time you see "This should never happen, but just in case", that's janky.  In fact, any time you see "Just in case" in a comment -- probably janky.  Defensive coding?  Total jankiness.  Checking against NULL even when there is no reason why that pointer should ever be NULL?  Janky.
    +
    + "Creeping jankiness" are all the little things that make you go "Huh, that's weird" but you ignore them because, hey, it didn't crash and nothing obviously wrong occurred. The absolute best programmers I've ever known, without exception, are radically intolerant of their software acting in any way that does not match their mental models. If something happens -- no matter how small -- that they do not understand, they stop and go "What just happened?"
    +
    + Even if it takes them all day and even if it means that they fall behind in other areas.  It seems like a net productivity loss, but it's actually a net productivity win in the long term because you maintain a rock solid understanding of how your software works.
    +
    + I can think of at least three great examples of this.  The first was when I was at 3Dfx and the Voodoo chip was being developed.  Gary Tarolli, 3Dfx's CTO and the principal architect of Voodoo (then known as SST-1), was running some simulations that took a long time (a single frame of rendering would take well over a minute).  We'd generate the frames (sometimes over night), then play them back as a movie for demos or whatever.
    +
    + +
    + +
    Obligatory reminder of that horrible 3Dfx logo
    + We did this with one of the demos, and during rendering there was a pixel that flickered for exactly one frame.  That was it.  Hardly noticeable, but there it was.  As a software guy I remember kinda shrugging and thinking "Screw it, it looks right most of the time and it didn't crash, SHIP IT!"  But Tarolli would not have any of that.
    +
    + After spending what seemed like a huge amount of time (I want to say it was at least a day), he found the bug and discovered that it wasn't a bug, but an unintended side effect of a feature that was totally legitimate.
    +
    + The second example was when I was at id software.  John Carmack was absolutely intolerant of rendering errors -- especially cracks, slivers and T-junctions in his BSP renderer.  He was devoutly religious about it, and this is one of the reasons that the Quake games had such an immensely heavy, solid feel to them.
    +
    + So of course one day we're testing things, and we see the occasional flash of a crack or a sliver.  This was pretty much an all-stop situation for him.  He spent a long time (I want to say at least a day -- and one Carmack Day is like Five Regular Dude Days) and finally narrowed it down to an issue specific to an ATI driver or card.  Now, most of us would be content to be all "Whew, not my problem, let's move on!" but of course Carmack had to know how the ATI driver was broken.
    +
    + It turned out to be their guard band clipper.  Guard band clipping is a graphics driver optimization, but it can introduce cracks when done wrong.  If you're going to use guard band clipping, you have to clip against the guard band planes all the time.  You can't clip against them for some triangles then clip against the viewport for others -- that can introduce slivers between triangle edges.
    +
    + +
    + +
    I just needed a Quake 2 screenshot for symmetry...and to remember how goddamn dark that game was
    + I don't remember if that was a hardware bug or just a driver bug, but the point is that Carmack knew he didn't have those types of rendering errors, and when confronted with them he had to track down what was causing them.
    +
    + A more recent example of suffering no jankiness is Casey Muratori's dissection of a five second Windows stall.  It's obvious that once the problem was fixed he could have moved on, but he was compelled to figure out what the hell was going on under the hood so he could have a better understanding of the software's interactions.
    +
    + That diligence reaps huge benefits, because very often these small problems are the first cracks in the codebase. By chasing after these now you stop larger problems from erupting later. In addition, by ensuring that only the things that you expect to happen actually happen, you're forced to keep a mental model of exactly what you think is happening in your head. If you become flippant about jankiness it's easy to eject global understanding ("This is how the program works") to focus on localized concerns ("Why is this function crashing?"). Once that mental model of your software is corrupt or vague, you're in major trouble because you no longer know what to expect, and it becomes extremely hard to even detect non-crash bugs because you can't tell the difference between a bug and expected behaviour any more!
    +
    + I used to blow off this attitude as overengineering or intellectual wanking -- if it worked, that's what mattered, and I can be getting in a new feature while you're sitting there obsessing over code that works (for the most part...) -- but since then I've decided I've been doing it all wrong, and I need to have a solid understanding of all the code I write, if for no other reason than it makes things easier on me in the long run.
    +
    +
    + +
    + + + Posted 8th August 2012 by Unknown + +
    + + + + + + + +
    + + + + + + +
    + + +
    + + + + + + + + + + +
+ Blog Archive +
\ No newline at end of file diff --git a/packages/readabilityjs/test/test-pages/brookings.edu/distiller.html b/packages/readabilityjs/test/test-pages/brookings.edu/distiller.html new file mode 100644 index 000000000..5df8ba37f --- /dev/null +++ b/packages/readabilityjs/test/test-pages/brookings.edu/distiller.html @@ -0,0 +1,28 @@ +
Brad Marianno

+ Then came fall 2020. While state leaders took a heavy role in spring 2020 school closures, the dog days of summer unfolded to a more “laissez-faire” attitude among top state executives in regard to the fall 2020 schooling decisions. What followed were agonizing reopening decisions that played out in the nation’s over 13,000 school districts, creating lengthy Zoom school board meetings, such as the famous marathon meeting in Miami-Dade. With the venue for school reopening decisions moving from the state to the local level, the teachers’ unions entered the school reopening debate from stage right. +

+ Do teachers’ unions shape reopening decisions? +

+ The reality is, when reopening decisions were brought into the realm of local control, they became far more polarized and political. This should not be terribly surprising. There are far more education interest groups and competing education policy ideas at the state level, making it much more difficult for a single organization to garner a dominant voice among policymakers. When these decisions are brought down into local school board meetings, there remain only a few organizations that are organized enough to exert influence. The teachers’ unions are the largest elephant in the room and the one that, in most school districts around the country, are directly connected to the supply of teachers to the classroom. +

+ With a team of co-authors, I tracked school reopening decisions throughout the fall 2020 semester in 250 of the largest school districts around the country to understand just how much influence teachers’ unions had on these decisions. We measured teachers’ union power in two ways. The first captures a more fixed measure of power that is built up over a longer period of time: the length of collective bargaining agreement. We think of this as a proxy for the detailed working conditions secured by the union. The second is a shorter-term measure, one that captures the reactionary power of teachers’ unions to school reopening decisions. We tracked the Facebook pages of teachers’ unions in our sample for social media posts that made forceful statements on school reopening, including those that discussed lawsuits, membership opinions, and protests. +

Substitute teacher Ryan Giese teaches a fourth grade class Wednesday, October 2, 2019 at Parkside Elementary School in Fond du Lac, Wis. Doug Raflik/USA TODAY NETWORK-WisconsinFon Teacher 100219 Dcr001
+

+ We find that school districts with lengthier collective bargaining agreements were less likely to start the fall 2020 semester with in-person instruction, were less likely to ever open for in-person instruction during the fall semester, and spent more weeks overall in distance learning. We further find that the shorter-term union power flexes—those captured by the social media posts—were associated with a higher likelihood of reopening for full, in-person instruction during the fall semester. These short-term reactionary responses were not enough to slow the pressure toward reopening; instead, it’s the long-standing, entrenched union power relationships captured by the length of the bargaining agreement that are associated with a slowed reopening timeline. +

+ But what about COVID-19 rates? We find that pandemic severity, as measured by the county COVID-19 hospitalization rate, had no relationship with the probability of in-person reopening. Instead, the other strongest predictors of an in-person reopening during fall 2020 included political partisanship (the share of Trump voters in the county) and demographics (the percentage of white students in the school district). +

+ To be clear, we are not the only authors to document that unions, politics, and demographics matter above and beyond pandemic severity, though we are the first to show these relationships beyond the start of the fall 2020 school year. (You can read four other studies with very similar findings here, here, here, and here.) +

+ Teachers’ unions do their job well—but local school governance may be broken +

+ So, the empirical research suggests that teachers’ unions slowed fall school reopening decisions during a worldwide pandemic, and media accounts suggest their efforts continued as schools returned from winter break. Naturally, some will view union actions as an affront to America’s students and others as a public service. +

+ My hunch is that where one stands right now on the debate around teachers’ unions and school reopening does not depend on nor will it be shaped by the evidence as to their influence on school reopening just presented. Instead, it likely depends on whether and how one has formulated opinions on two other bodies of evidence (some of which has emerged more recently): 1) the evidence on the risks of the pandemic to teachers and students returning to face-to-face instruction, and 2) the evidence on the potential harms caused by distance learning. For those that believe the public health risks of in-person instruction far outweigh the harms caused by distance learning, then teachers’ unions are being unfairly scapegoated for their role in reopening decisions. For those that believe the benefits of in-person instruction far exceed the risks of face-to-face instruction during a pandemic, teachers’ unions are then obstinate, bad-faith actors. I will likely not shift readers opinions here on these fronts, but there is now some solid empirical evidence on the risk of in-person instruction during the pandemic (here, here, here, and here) and the impact of distance learning (here and here). +

+ We should expect teachers’ unions to slow school reopening decisions just like we should expect them to advocate for increased education funding, lower class sizes, or enhanced teacher sick leave. Protecting and representing teachers is their job and they survive to the extent to which they do this job well. Ever since the Supreme Court’s decision in Janus v. AFSCME (2018) that prohibited the mandatory payment of union fees, unions have had to work harder at this job than ever before (as I have written elsewhere). Teachers’ unions are like an insurance policy to which teachers will subscribe for days like this—worldwide pandemics where the job becomes substantially more uncertain. +

+ This should not be mistaken as a ringing endorsement for teachers’ unions to have unfettered influence on reopening conversations. Special interests that represent a few, by definition, exclude many other viewpoints. It is entirely possible, even probable that our localized school governance systems around the country are out of balance—and were this way well before the pandemic. They have become too politically polarized and swayed by entrenched special interests (sometimes nationalized special interests). One solution is to increase the marketplace for local education ideas (consider efforts by parents to form their own unions) such that there is greater competition in the local education space. In these situations, special interests become far less special (and far less influential), thereby helping restore a more balanced form of educational governance. +

+ Ultimately, local education control is not going by the wayside anytime soon (though it is eroding). While some focus their attention toward the teachers’ unions and their role in school reopening decisions, we might also take a hard look at our local school boards and other school governance institutions that decide which voices are heard the loudest when policy decisions are made. The COVID-19 pandemic and school reopening decisions that followed should lead us to scrutinize whether our local educational governance systems are properly calibrated to protect and promote student interests over and above special interests. +

\ No newline at end of file diff --git a/packages/readabilityjs/test/test-pages/cavesocial/distiller.html b/packages/readabilityjs/test/test-pages/cavesocial/distiller.html new file mode 100644 index 000000000..81d484b42 --- /dev/null +++ b/packages/readabilityjs/test/test-pages/cavesocial/distiller.html @@ -0,0 +1,38 @@ +
  • embed instagram posts

+ Embedding photos is a great way to legally show other people’s photos on your website, promote their content, and entice them to share your blog post on their social accounts. +

+ Let’s say you want to write about restaurants in your area. These articles are broad and cast a wide net in terms of readership. You’ll be capturing traffic from people in your area, and providing them with valuable content that presents you as a local expert. Awesome! +

+ Write your article, link to the restaurants, and then – as the title of this article implies – embed photos from Instagram onto your blog. +

+ Here’s what you do. +

+ 1. Find the restaurant on Instagram +

+ For example, if you mentioned Zeeks Pizza in your article, head to Instagram and search ‘Zeeks Pizza,’ or just type, ‘Zeeks Pizza Instagram,’ into Google, then follow the link. +

embed instagram photos

+ 2. Find the best photo for your article +

+ Take a look through their photos, and find one that you think will speak to your audience. We recommend going for a photo of the restaurant’s signature dish, in this case, pizza. When you find a winner, click on it. Of the three below, we’d recommend the one of the far left. +

embed an instagram photo to your blog

+ 3. Click the 3 dots in the bottom, right-hand corner +

embed on instagram

+ 4. Click the embed button +

embed on instagram

+ 5. Copy the embed code +

how to imbed photos on instagram

+ 6. Paste the embed code in the text of your website +

+ If you use WordPress, paste the embed code in the text portion of your editor. From there, you’re good to go. Your new Instagram photo will be embedded in your blog post! It should look something like this: +

+
+ Once you’ve finished your blog, be sure to share it with all of the restaurants you mentioned.  You’ll usually see a few of the restaurants share the blog post on their social media accounts, bringing more visitors to your website (along with more potential customers). +

+ 7. Center the content +

+ When you put an Instagram embed code on your website, it will default to left alignment. If you want to center your content you have a few options. +

  1. You can add a <center></center> tag to the beginning and end of the embed (this will have to be done on every post)
  2. In WordPress you can add this as additional CSS to ensure every post is centered: .instagram-media{margin:15px auto !important;}

+ Watch our video on how to get your +1000 visitors to your next blog post for more tactics, and be sure to email me at jordan[at]cavesocial.com if you have any questions. +

+ Be sure to tweet any questions you have to the Cave Team on Twitter – just use the hashtag #AskCave +

\ No newline at end of file diff --git a/packages/readabilityjs/test/test-pages/channelnewsasia/distiller.html b/packages/readabilityjs/test/test-pages/channelnewsasia/distiller.html new file mode 100644 index 000000000..2b52f60d6 --- /dev/null +++ b/packages/readabilityjs/test/test-pages/channelnewsasia/distiller.html @@ -0,0 +1,71 @@ +
New points system for Employment Pass applicants from September 2023
People wearing protective face masks waiting to cross the road at Church Street on Sep 6, 2021. (Photo: CNA/Gaya Chandramohan)

+ SINGAPORE: On top of meeting a minimum qualifying salary, foreigners wanting to work in Singapore under an Employment Pass (EPs) will also have to meet a minimum score under a points system from September next year. +

+ The new framework comes as the Ministry of Manpower (MOM) is set to adjust the minimum qualifying salary for EPs up from S$4,500 to S$5,000 from this September. For the financial sector, the salary benchmark is set higher at S$5,500. +

+ Advertisement +

+ While S$5,000 or S$5,500 is the minimum salary for entry-level jobs, it goes up to S$10,500 for middle-aged candidates, or S$11,500 if they are in the financial sector. +

+ The higher wage benchmark will apply to new EP applications from Sep 1 this year, and to renewal applications a year later. +

+ EP applicants generally fill professionals, managers, engineers and technicians (PMET) jobs. +

+ The intent is for EP qualifying salary to match the top one-third, or 65th percentile, of local PMET wages by age, said Manpower Minister Tan See Leng, as he announced the upcoming changes in Parliament during MOM’s Committee of Supply debate on Friday (Mar 4). +

+ “We know that the labour market for skilled workers is sufficiently tight. So this latest move is not intended as a tightening measure,” he said. +

+ Advertisement +

+ “Instead, it is part of MOM’s regular updates to ensure that our qualifying salary keeps pace with local wage growth, so that foreigners are not coming in just because they are cheaper than local PMETs.” +

+ He added that most EP holders will not be affected by these changes as they earn “well above the qualifying salary”. +

+ From next year, applicants will also have to score sufficient points under a new COMPASS (Complementarity Assessment Framework) points system to work here. +

+ It will come into effect for new EP applications from September 2023, and for renewal applications from September 2024. +

+ HOW COMPASS WORKS +

+ Points under COMPASS will be awarded based on four attributes and two bonus criteria.  +

+ Advertisement +

+ The EP application can score up to 20 points for each attribute – the applicant’s salary, his or her qualifications, the nationality diversity of the hiring firm and the firm’s support for local employment. +

+ An individual and/or firm that meet the expectations for each attribute or criteria will score 10 points and if it exceeds expectations, it scores 20 points. +

+ The applicant or firm can also score on two bonus criteria. One is a “Skills Bonus” for candidates in jobs where skills shortages exist, which can add up to 20 points. +

+ The second is a “Strategic Economic Priorities Bonus” of up to 10 points – for firms that engage in innovation and internationalisation activities in partnership with the Government. +

+ For example, applicants with a salary that meets the 65th to 90th percentile of local PMET salaries in the sector will score 10 points under the “Salary” attribute. They will get another 10 points under the “Qualifications” attribute if they hold a degree or its equivalent. +

+ Advertisement +

+ If the candidate exceeds expectations - like if his or her salary is above the 90th percentile, or if their educational qualifications are from a “top-tier institution”, they score 20 points. +

+ There are also two attributes for the firm to meet. +

+ The firm can score 10 points if the proportion of its employees with the same nationality as the applicant is between 5 and 25 per cent. It scores 0 if the proportion is more than 25 per cent, and 20 points if it is less than 5 per cent. +

+ It scores another 10 points if the firm’s share of local PMETs is at the 20th to 50th percentile compared to its subsector.   +

+ Small firms with fewer than 25 PMET employees score 10 points on the diversity and support for local employment attributes by default. +

+ This is as MOM recognises that for smaller firms, their workforce ratios can “change drastically” with a few personnel changes, said Dr Tan. +

+ If the application meets expectations on all four attributes, it should pass with 40 points. If not, it could be approved using the bonus criteria. +

+ Dr Tan said that COMPASS is not designed to make it harder for companies to obtain an EP and a majority of the applications today would pass. +

+ “This is because our businesses in general are progressive, and our policies thus far have brought in complementary EP holders,” he said. +

+ “Some firms, however, would need to make adjustments under COMPASS. And they would know clearly which areas to improve on and how to respond accordingly, given the transparency, clarity, and predictability of COMPASS.” +

+ Dr Tan also said that the ministry has received feedback from employers who are concerned about bringing in young, high-potential employees under global rotation programmes.  +

+ To facilitate this, the ministry will pilot the Global Rotation Scheme, and the firm must also post Singaporeans abroad on the same leadership development programme. +

+ Source: CNA/hm(cy) +
\ No newline at end of file diff --git a/packages/readabilityjs/test/test-pages/channelnewsasia02/distiller.html b/packages/readabilityjs/test/test-pages/channelnewsasia02/distiller.html new file mode 100644 index 000000000..9f6bd205e --- /dev/null +++ b/packages/readabilityjs/test/test-pages/channelnewsasia02/distiller.html @@ -0,0 +1,137 @@ +

+ SINGAPORE: Seven years. +

+ Of pressure, of pain, of doubt. Of trying harder, of getting faster, of growing wiser. +

+ Advertisement +

+ There are now brown streaks in her hair, seven tattoos inked on her skin, and three goals on her mind. +

+ Give everything. Enjoy yourself. Send a message. +

+ And as she prepared herself for the women's 200m final at the 31st SEA Games in Hanoi, sprinter Shanti Pereira would repeat a mantra. Over and over and over again. +

+ "No matter what happens, you'll be good." +

From left to right, Pereira Veronica Shanti of Singapore, Kyla Ashley Richardson Maico, Kyla Anise Richardson Maico of Philippines and Hoang Thi Ngoc of Vietnam compete during the women's 200M run in the Athletics competition during the 31st Southeast Asian Games (SEA Games 31) in Hanoi, Vietnam Saturday, May 14, 2022. (AP Photo/Minh Hoang)

+ SELF-DOUBT +

+ To understand what the race felt like to Pereira, one has to go back to the 2015 Games in Singapore. +

+ Advertisement +

+ At the meet, she would win gold in the 200m, clock a personal best and set a new national record. The win also ended a 42-year gold medal drought for Singapore in a Games sprint event. +

+ A new sprint queen had arrived. +

+ “I was just excited to be there, competing in front of a home crowd,” recalled Pereira, who also clinched a bronze in the 100m. +

+ But with the weight of winning came the enormity of expectations. +

+ “It created a lot of pressure and I feel like I didn't have the experience to cope with it yet,” she explained. +

+ Advertisement +

+ The next edition of the Games would end in tears for Pereira, as she finished with a pair of bronzes in the 100m and 200m. +

+ Then came two injuries in 2018, before she would finish with two bronzes again at the 2019 Games. +

+ "Physically for the longest time, I was there. I (felt I) could do a PB (personal best), NR (national record) and all that but mentally I wasn't there," she said. +

+ "There was a lot of self-doubt that kind of grew as the years went by. Just because it was like - again another season I couldn't get a PB, and it just continued and continued and continued." +

Singapore's Shanti Pereira competes in the women's 200m heats at the Tokyo Olympics on Aug 2, 2021. (Photo: Facebook/Singapore National Olympic Council/Kong Chew Yew)

+ And some people began to write her off, said Pereira. +

+ Advertisement +

+ "Slowly, people just kind of lost faith in me. People have their opinions about my journey, and whatnot. I think, for a while there, I really did let it get to me, which is not ideal. I want to be in a state mentally strong and things like that don't affect me, but for a while it really did." +

+ The negativity ate her up, added Pereira. It got to the point that there were times when she was no longer excited to compete or even train. +

+ Instead, what she felt was fear. +

+ "I couldn't find any motivation," she recalled. "Coming to training, I was just always afraid of doing bad times at training and just not doing well at training." +

+ FEEDING OFF A DIFFERENT ENERGY +

+ Sandwiched among the lows were highs too. +

+ Pereira clocked a new 100m PB of 11.58 seconds at the 2019 Asian Athletic Championships, and also made her Olympic debut in Tokyo where she ran a season's best time in the heats. +

+ But even with the Olympics, 2021 was a "rough" year, revealed Pereira. +

+ Off the track, she decided to pursue a career unrelated to her Singapore Management University accountancy degree. On the track, the results were not "showing up". +

+ "I felt like I suffered a bit of an identity crisis. I started to believe what people said about me, that maybe I’ve reached my end. And if so, who was I?" she said. +

+ But speaking to the ones close to her helped. +

+ "They've helped me a lot, to realise that I am fine, I am okay. It's really just a change of a mindset that I had to try to do," she said. +

+ In the lead up to this year's Games, Pereira did just that. +

+ "I reached a point where I just told myself that it really doesn't matter what people think of you," she said. "I have people in my corner, and they really believe in me. They have absolute trust in me that I am not at the end of my career. And I think I just fed off that energy more." +

+ She looked inward. +

+ "I can only focus on the things that I can control - which is how I approach the Games, how I approach training, all these things," she said. +

+ "Whatever people are gonna say, they're gonna say. It doesn't matter. What matters is how I do things, making sure that I do whatever I can and give absolutely everything to my races." +

+ Shanti Pereira wins 200m gold at SEA Games in new national record +

+ And going into this year's Games, there was a sense of calm, recalled Pereira. +

+ "That was ultimately what I wanted. I wanted to just chill. This is my thing, I'm just going to go there and do the best I can."  +

+ And this would mean channelling a similar outlook that she had seven years ago. +

+ "That's the kind of mindset I wanted to be in. I wanted to just relax and have fun and just enjoy the fact that I'm here racing," she said.  +

+ 'I'M STILL HERE, I'VE STILL GOT IT" +

+ Her legs are a whir, her heart thumping like a drum, her eyes fixed on the finish line. +

+ At the My Dinh Stadium in Hanoi, along the final stretch of the 200m race, Pereira is ahead.  +

+ To her left, a Filipino opponent gains on her, but the Singaporean is not letting this one slip away. Not today. +

Gold medalist Pereira Veronica Shanti of Singapore celebrates with the national flag after the women's 200M run in the Athletics competition during 31st Southeast Asian Games (SEA Games 31) in Hanoi, Vietnam Saturday, May 14, 2022. (AP Photo/Minh Hoang)

+ Personal best. National record. Gold medal. +

+ The 18-year-old with the blue glitter hairband is gone. There’s no exuberant waving to the crowd, no gleeful smile of joy.  +

+ Instead, the 25-year-old is floored. Exhaustion, disbelief and the sheer magnitude of what she has done overwhelm her. +

+ "I told myself before the finals that no matter what the result is, I'm going to give 100 per cent. Everything I have, physically, mentally, emotionally. Everything," Pereira explained. +

+ Her message - not just to others, but also to herself - rang loud and clear.  +

+ "I’m still here, I’ve still got it," she said. Days later, Pereira would add another medal to her tally, a silver in the 100m. +

+ Even till today, what she achieved at the Games still hasn't fully sunk in. And isn't just about the victory or the timing, but the "breakthrough", she explained. +

+ "I put in a lot of hard work and sacrifice to be able to make that breakthrough," she explained. "This wasn't the first time where I wanted to do that. I wanted to do this for a really long time. The fact that I actually did it is quite amazing to me." +

+ Pereira's coach Luis Cunha said he was confident in Pereira's ability to perform the way she did. +

+ "(With the performance at the SEA Games) she showed herself that she was able to clock the times. But she also knew what I told her (previously) - that she can do it," he said. +

+ "Now she can confirm my thoughts with the performance. I hope that this allows her to go further and continue to improve this season." +

Shanti Pereira poses with her SEA Games gold medal. (Photo: SportSG/Jeremy Lee)

+ A NEW CHAPTER +

+ Being able to continue with the same mindset remains a work in progress, noted Pereira. +

+ "It's still a process. I learned from what I did at the SEA Games, and I want to continue doing that. But it's still hard." +

+ And she will aim to build on this, with the Commonwealth Games and the World Athletics Championships still to come later this year. +

+ "It took so much out of me to be able to have that mindset, so I just want to learn from it and continue with this mindset," Pereira added. +

+ Pereira has yet to to have a discussion with Cunha on her specific goals for these meets. But she already knows, at the very least, she wants to bring down her times. +

+ "I want to go faster (than I did at the SEA Games), for sure. And I feel like I can. That's the goal. Just do whatever I can in the next few months to try and hopefully go faster for both the 100m and 200m." +

+ Shanti Pereira and Marc Brian Louis win silver and bronze at SEA Games in respective 100m events +

+ After Pereira's first Games gold, her older sisters Valerie and Shobi were inspired to write a children’s book to tell the story of her journey and triumph. +

\ No newline at end of file diff --git a/packages/readabilityjs/test/test-pages/china.substack/distiller.html b/packages/readabilityjs/test/test-pages/china.substack/distiller.html new file mode 100644 index 000000000..08536a31c --- /dev/null +++ b/packages/readabilityjs/test/test-pages/china.substack/distiller.html @@ -0,0 +1,143 @@ +

+ This week’s ChinaTalk episode with economic historian Adam Tooze and guest host Matt Klein was fantastic. See below for a condensed transcript. +

+ Do Fiscal Constraints Even Matter? +

+ Matt: A lot of what we think of as financial constraints can be overcome by the determination to mobilize resources in the wartime economy. There are still real economic constraints in terms of labor shortages or access to raw materials. But conventional financial constraints, like how much debt there is, turn out to be somewhat less important. Yet despite the knowledge that we should have had from previous experience, most governments -- with the notable exception of China and Russia -- basically acted as if they were constrained by these sorts of financial circumstances. +

+ Adam: The history of the long 20th century suggests that there is this remarkable freedom in the modern economy with the resources of the modern States to mobilize financial and monetary means to do the things that we want to do. In European history and in global history, the early 20th century provided several lessons along these lines. +

+ The first experience of this type was World War One, which was a sort of learning by doing. As I was showing in Wages of Destruction when I was looking at Nazi Germany in the 1930s, there was quite a deliberate effort to use financial and monetary means to mobilize the real economy to the absolute limit of its potential. +

+ To be a little trite you could say that the lesson learned from the genealogy of European neo-liberalism was that the potential is there. But in that potential lies the possibility of political disaster. Keynesians may look at this scenario and say, that's great, the problem with unemployment is solved. But the neoliberal will stand back and say, hang on, you're being naive. What you're not understanding is that this basically unfetters politics. Sure if you're dealing with a bunch of herbivorous Social Democrats, they'll take you in one direction and you'll end up with a welfare state and full employment, but if their same knowledge is in the hands of a group of nationalist militarists, what you've really provided them with is the blueprint for highly efficient mobilization of a military economy in times of peace. So deep in the heart of neoliberal thought and conservative thinking about the modern state and its potential lies a fear of that possibility. +

+ If you were looking for just a simple segue from the history of Nazi Germany and the history of the experience of the total war economies to the present, it would be by way of that backflip and the recognition of the danger that kind of politics potentially poses. +

+ I do think at the heart of modern conservatism is a fear of the realization that really there isn't that much that constrains us other than the real constraints. +

+ Because when you get to actually debating the real constraints, it gets political very fast. It’s immediately a series of very tough tradeoffs and you have to formulate what the purposes of your government are. That in itself is an explosive proposition. So, in a sense to say, “well there's financial limits, and once public debt exceeds 90% of you fall down a slippery slope,” that short circuits a bunch of really profound political questions that one would probably rather avoid. +

+ How Contingent Was WWII? +

+ Matt: The popular narrative is that the post-World War One regime was incredibly unstable and bound to break down. Your argument is that's completely wrong, it was remarkably durable and took a very unusual combination of circumstances to even put it under threat. Then even then the challenger powers basically, with exception of the USSR, didn't really end up any better off than they were at breaking this Anglo-American dominance. Thinking about the theme of this podcast and where we are now, what would you take away from this in thinking about the durability of the post-1991 world order? How should we think about that in the context of sort of U.S.-China competition? If you take this 1920s analogy seriously, that suggests you basically need some kind of extreme series of crises in the U.S. to change things seriously. +

+ Adam: It would be unwise to draw that conclusion. It’s a wonderful summary of The Deluge, which was essentially an argument with German theorists like Carl Schmitt, who insisted that the problem with liberal internationalism and the League of Nations was not that they were weak, the problem was that they were hypocritical. In other words, [liberal internationalism] was shamefaced about the propensity and capacity for violence that actually underpinned it, which revealed itself when Imperial Japan, Fascist Italy, and Nazi Germany made the challenge. Why did Hitler have to try as hard as he did? This is not a conventional, familiar sort of arms race; this is a massively unstable tumbling forward into a completely unstable arms race economy. +

+ Why were they forced to make that kind of effort? An interest in the 1920s revived in the 1990s amongst international historians, because we were in a sense thinking about the analogies between the new unipolar order taking shape after the end of the Cold War, which was inflected in various ways with ideologies of liberal internationalism that did in the American case, trace back to The Wilsonian moment. That's what drew folks like me into thinking about this. It drew folks like Niall Ferguson into thinking about it too, from the other side, in other words, as a celebrant of a muscular, organizing Imperial hegemonic power. +

+ But I would argue that over the course of the last couple of decades, I've become ever more fascinated by what I take to be the world-historic transition that we're witnessing. Because China’s not the Soviet Union, China's not fascist Italy, China's not Nazi Germany. The growth of China is a phenomenon that dwarfs all of those previous developments and has to be understood on the timeframe that was laid out for us by the economic data of somebody like Angus Maddison, who shows us global GDP all the way back to the birth of Christ. All the way through the beginning of the 19th century, the Asian economies actually dominate once you've adjusted GDP by purchasing power parity and so on. +

Over 2,000 Years of Economic History in One Chart

+ I quite like what Larry Summers said, which is that the question is whether there's any political force in the United States that's capable of really reckoning with the scale of this world-historic shift, which will in the end lead to the supersession of America's dominant position. We will end up in some sort of multi-polar order and the question is whether any political forces in the United States can really accept that kind of transition. It's tempting to say, is there anyone in the United States that could play the role of the British elite after World War One? But America's position of dominance was vastly greater than that ever by enjoyed the British so the psychological challenge of accepting this transition is far greater. And, of course, in key respects America remains an absolutely dominant player, most notably with regard its hard power, its weapons, but also in certain respects with regard to its financial centrality. +

+ I'd be very cautious about drawing a straight line. There is a connection in the sense that one thing grows out of the other. In The Deluge I spent lots of time trying to position the history of the emergence of the Chinese Republic in the epoch that we normally think of as being about Europe and the United States. +

+ What Can Nazi Germany Teach Us About the CCP? +

+ Jordan: Coming to another one of your books, Wages of Destruction, which looks at the Nazi Germany economy: Is there anything we can learn today from the Nazi economy for thinking about China today? +

+ Adam: I wouldn't go straight at it. I would go by way of political theory. The person that I would go to is Neumann, the German political theorist who wrote an extraordinary book called Behemoth. He described what happens to capitalist States when they get caught up in a logic, which he took to be characteristic of the 20th century, which is oligopolistic, monopolistic rule. The international system of free trade results from monopoly. And the result of that is protectionism and tariff war, and, ultimately in Hobson's view, a race towards imperialist competition and war. Continue that line of thought and ask yourself, “Okay, so what does monopoly do and what do oligopolistic structures do to the domestic political structure?” +

+ It systematically subverts principles of representation. It begs the question of who has power and what is the role of the state in relation to those interests. Because properly understood liberalism clearly isn't premised on the absence of the state, its premised on a well-ordered set of relationships between individuals, the law and various types of representation. That structure is not necessarily robust if economic power becomes monolithic. There are ways of taming that by way of corporatism, in which you have an organized representation of economic interests. But you can also imagine systems in which it can become a sort of destructive set of flywheels of extremely explosive dynamics of gigantic interest groups contending with each other more or less in an unmediated direct form interest on interest. +

+ I think a pessimist would say, you've just described modern Western societies. But I think you could also ask how useful that analytics might be for thinking about the situation in a highly complex political economy like China, which of course didn't start out liberal. And as we increasingly are coming to terms with is unlikely to develop in a liberal direction. But it does seem to be characterized by a clash within the one-party state of different interests, of agglomerations of technology and capital, of different party factions. +

+ I would love for us to have a Neumann, Behemoth style treatment of China. I'm not saying that the results would be the same, but that analytic would be profoundly helpful to try to understand what's going on there. +

+ Matt: One thing I think is interesting is that the way that, the Nazi economy worked, it could have been described as capitalist, insofar as how Stalin defined the capitalist bloc. You did have private companies and you did have concentrations of wealth that were outside of the formal structures of the state. Yet there also was a very real sense in which these private enterprises and private capital weren't really independent of the state in the way they might've been in another country. +

+ The extent to which the Nazis were able to marshal the resources the way they did was by allying German big businesses to their project through a mixture of rewards and penalties. That mixture of party-political objectives with the interests of these sort of quasi-private enterprises and elites, you can draw links to the structure of the modern Chinese economy as well. I wouldn't want to take the analogy too far, but I think there is a parallel. You can argue it's for similar political reasons in so far as you have a government with different factions and competing interests within it that we can't really observe. But at the same time, the way the Party is able to drive economic activity – I think there's definitely some interesting analogies there, +

+ Adam: In my reference to Neumann I want to emphasize that part of the commonality is the confusion and complexity. Our categories don't fit very well. It's quite difficult to describe the Chinese regime in categories that are at all familiar without reducing it hopelessly and just failing to recognize its complexity. It's that position of puzzlement that strikes me as analogous. I'm not saying that China's Nazi Germany – the problem is similar, though, in that in analyzing it, none of our categories seem to fit it very well. +

+ This seems reminiscent of the difficulty that liberal analysts in the thirties and forties had in looking at Nazi Germany. They were not radically othering Nazi Germany. They weren't engaged in the crude Stalinist stereotypes of saying, “it's capitalist and your capitalist, so basically you are the same,” which is hopelessly reductive. There is a version, as it were, of the left-wing analysis of China today which is willing to say that, that it's just state capitalism and that's all there is to it. I don't find that position at all interesting. +

+ I'm curious to go back, in a sense, to read those struggles to make sense of Nazism from the outside -- Neumann was inside-outside, because he was an émigré – as an inspiration for thinking in similarly open-ended and protean terms about China in the present. Not as a template, but as an inspiration. +

+ There’s nothing I could read in the West right now that makes me feel I fully understand the dynamics. Looking at sectors seems to help. In energy, for example. We can go quite a long way using openly available Chinese sources to understand the dynamic of competition between the coal people and the big oil companies and the new set of interests associated with the renewable energy industry. That provides us with a keyhole for understanding certain dynamics. +

+ There are certain lessons that I think we can learn, in the sense that if we can't observe them, at least there are certain biases on our own part that we can check. There are certain common habits of thought that I see recurring in American efforts to make sense of China. We saw them in a kneejerk form at the beginning of this year, the Chernobyl moment analogy that rapidly surfaced. Of all the analogies in the world, it strikes me as truly bizarre. Does Xi’s regime now look like the Soviet Union in 1986? I don't think so. Nevertheless, it was very compelling -- and apparently quite compelling to people inside China as well, who were using the film review websites to leave comments. I think we need to be very conscious of those sorts of impulses in our own mindset and try to read against the grain of our own desire to see certain neat conclusions and certainly, root storylines acted out. +

+ Jordan: Youshu, the anonymous guy who wrote that review of Wages of Destruction, sent me this question: You argue Hitler went to war with an inadequate industrial base and people assume that Xi won't attempt to take over Taiwan until he's confident of victory. Given what you know about the incentives facing Hitler, do you think there's anything we can learn about how Xi is thinking about Taiwan? +

+ Adam: I think the much closer analogy is to Stalin. This is the point made by the vastly underrated conservative historian Andreas Hillgruber in his magnificent global history of strategy in the first phase of the war. There's a huge difference between Hitler and Stalin, because Stalin believes confidently that time is on his side. The time horizon is everything in deciding whether or not the strategic window is open and how long it's open for. Whereas Hitler is gnostic, literally, in a kind of theological sense. Hitler has no confidence at all that time is on his side and he knows that, from an economic point of view, the cards are indeed stacked against him. The advantage that the Germans built up early on in the arms race is probably going to dwindle. Furthermore, he is possessed by the idea that the conflict is inescapable, that the world Jewish conspiracy is pressing in upon Germany whether in the form of an actual hot war or whether by means of the cold war pressures that he believes are the explanation for Germany's economic problems. +

+ There is for him a way of reading Germany’s dwindling foreign currency reserves as an index of the success of the Jewish effort to strangle the German economy. You see this manifestly in his notorious speech on the 30th of January, 1939, where he actually threatens that if world war should break out – which in German terms means a war involving the British empire and the United States – then it will be the Jews of Europe that pay. This is the moment when he publicly announces the Holocaust. The rest of that speech is all taken up with economic problems and shortages of foreign exchange. It's extraordinary. +

File:Adolf Hitler's speech in the Reichstag, 30 January 1939.png - Wikimedia Commons

+ I don't see that level of time pressure and anxiety. I know people say that the Chinese think that time is not on their sides either, that in the end they're going to be overtaken by India, that their population is aging. But the time clock runs over decades. That wasn't true for somebody like Hitler. This suggests that the Chinese situation is radically different. +

+ Jordan: It's interesting thinking about the extreme Republican take on U.S.- China relations, how it almost echoes the clock running out on the U.S. We have to shut down Huawei, we have to take down SMIC, because time is absolutely not on our side. +

+ Adam: The answer to this apocalyptic thinking, which in the case of folks like Pompeo who belong to the evangelical faction around the Trump administration, they really do have a kind of end times possibility in their kind of cosmology. That's a dangerous thought pattern. Apparently, Peter Navarro sincerely believes that there is the possibility of a shooting war with China as a realistic prospect over the next couple of decades. That’s dangerous stuff, because the job is not to refuse a relative decline, because that's necessary and legitimate and inevitable. The question is to code it as something other than end times. +

+ The farcical element amidst the German tragedy is that in, 1945, they are looking around them and expecting the final historic erasure of Germany. We know that's absolutely not what happened, and it never needed to have happened. +

+ Konrad Adenauer is older than Gustav Stresemann and presided over the resurrection of a Germany that turns out to be absolutely essential to the European balance of power, the European economy, and everyone welcomes back and makes rich. This sort of apocalyptic thinking of a history that's going to end with some sort of big bang, or some terrible ghastly discreditable whimper, rather than just facing up to reality in which the world is different and America's position is not what it was in 1945. Which is not after all the end of the world. +

+ The worry for me is more about the American side than it is about China. This is in no way to diminish the aggression that Beijing is clearly displaying, it's determination to resolve the “Hong Kong problem,” and its very earnest commitment to finding a new solution for Taiwan in the foreseeable future. This isn't to underestimate the seriousness of that intent on their part. But I don't think it's driven by the insistent ticking clock kind of logic. +

+ Why Chinese Political Theorists Are Drawing on Nazi Germany’s Favorite Legal Thinker +

+ Jordan: So, who's the German legal theorist Schmitt, and why is he having a bit of a renaissance nearly a hundred years later in China today. +

+ Adam: It's one of the truly remarkable the facts about the last 15 to 20 years of Chinese political thinking. It’s worth saying there are deeply intelligent, obviously brilliant political and legal theorists working in China, trying to articulate and make sense of the logic of this emerging power. One of the sources that they go to – and this has emerged from the tireless work of many translators of recent Chinese political writing and international relations thought is -- is a German political and legal theorist called Carl Schmitt. +

+ He really had his heyday in the Weimar Republic and then was one of the key legal minds behind the early history of the Third Reich. He provided one of the chief legal justifications for the overthrow of the Prussian government, which was a prelude to the Nazi seizure of power at the national level in Germany. He went on to be one of the most influential legal theorists of the Third Reich in the 1930s. Apart from his alignment with the party, which is true of many German intellectuals in the period, he was also a flaming Anti-Semite. He would label colleagues with the name Judo and attaching yellow stars to bibliographies. +

+ Lived on deep into the early 1980s, he was almost one hundred and really just wouldn't die. He became a kind of pivotal influence on German legal thought even after 1945. He ran a sort of secret seminar and has been quite influential on German legal thought since. +

+ Why? Because he has one of the most hard-nosed answers to liberalism. He insists upon understanding politics as a distinction between friend and foe, friend and enemy. He insists that legal orders have real foundations in space and in power, in the taking of a territory, fundamentally. That they therefore have limits and necessarily have limits, that they define insiders and outsiders. That structures which are truly comprehensive threatened to emerge in the course of the 20th century. He's one of the people that was very influential on me in writing The Deluge because he understood the order that emerged out of the end of World War One as a unipolar order, a kind of universal empire defined by the United States. But not simply as a peaceful empire, but as essentially the instantiation of the overweening power of America, the British empire, business. He was one of the most coruscating and effective critics of the way in which liberalism disguises its power in morality and the logic of commerce. +

+ Having laid it out like that I think you can see why call Schmitt has been attractive, not just for Chinese legal scholars, but for a variety of scholars both in Europe and in the United States. There is a live active Schmittianism in Harvard Law School around the figure of Adrian Vermeule, who's arguing that to understand the actions of the American government since 9/11, you have to be willing to understand the legitimate role for executive power in defining the enemies of the state and going after them. He is one of the people who justified the use of torture after 9/11. But, also, the emergency powers invoked by Ben Bernanke in 2008. You do that against the backdrop of a set of political commitments. This is the sort of moment of here I am, and I can do no other. This is my politics, and this is the political and legal regime that instantiates that. +

+ The left in Europe uses this kind of critique as a way of unpicking the hypocrisy of liberal internationalism in the age of Clinton and the responsibility to protect and that discourse of global internationalism which, they, the left, read, as the updated version of 1920s internationalism. For all of these reasons I think you can immediately see why Chinese scholars seized on precisely this logic for thinking through the emergence of Chinese power in the context of what they would diagnose as American empire. +

+ The first use you can make of Schmitt is to assert the necessity of politics and the absolute necessity of committing yourself to the project, some project. The project they commit themselves to is the project of the CCP. And from that law follows, not the other way around. You don't have a legal system and then as it were defined the party within it. In fact, the appropriate way around is to define your norms and then follow a legal system from that. The norms you pick is up to you. It could be liberal, it could be those of the Communists. +

+ But also, in a more ironic twist, they also see in Schmitt a theorist of China's own potential empire, a zone of influence and power that would create its own structures of incorporation, its own norms. Given from China, defined by China, set and ultimately rooted in Chinese power, radiating out from East Asia, encompassing other states. +

+ This, too, was Schmitt’s vision, that on this basis you could found an empire. It's worth saying that Schmitt is a failed figure. He starts out well in the Third Reich. He’s a passionate advocate of the Hitler impact, which conforms precisely to his sort of vision of the world divided up between, conceivably, the Germans sphere, the Soviet sphere, the Japanese sphere, and the United States-British empire sphere. And all of that of course collapses between 1940 and 1941, when it becomes obvious that Hitler’s project is not actually compatible well with that kind of stable vision of the partition of the world into separate zones. +

+ For the Schmittians in China right now, that's the question, too. In a sense, “One Nation, Two Systems” was precisely a kind of Schmittian vision of an overarching Chinese empire, with the pluralism that's contained within that. Safely contained with the acknowledgement that this is about China, under the leadership and the hegemony of the CCP, tolerating two systems. Of course, that isn't what we're seeing. +

+ Jordan: But the end of the day, there are other logics that are more powerful than his logic, so you see Hong Kong, Xinjiang and Tibet going they way they are. +

+ Xinjiang and Soviet Gulags +

+ Jordan: Any thoughts on the evolving Nazi rationale for labor camps and Xinjiang? +

+ Adam: Xinjiang seems to fit much more squarely in the logic of sort of the Soviet Gulag system, that it does in this sort of exterminatory logic of national socialism. There is an element of the Nazi concentration camps system which is a forced labor camp system. But it was always ultimately genocidal in the literal physical sense. I don't think there's every reason to think that the Chinese project of cultural transformation properly meets genocidal standards in Xinjiang. It goes a very long way in that direction. But there's no necessity for destruction through labor, which was understood by the Nazis and an absolutely core component of their incarceration and murder of the Jewish population, and also very large slices of the Slavic population of Eastern Europe as well. +

+ I don't see that logic being analogous. The colonial logic, the settlement logic, the use of forced labor for the purposes of political reeducation and also infrastructural transformation. The history, after all, of the Xinjiang forced labor complex and the state-owned enterprises there goes all the way back at least to the Cultural Revolution, if not the late fifties. But, again, I would caution against wanting to subsume China into someone else's archetype of what a forced labor regime or a genocidal regime looks like. It seems to me that there could be a horrifying originality to what they're doing. After all the tech dimension of what China's able to do now in terms of surveillance is beyond the wildest dreams of any previous authoritarian regime. +

+ Matt: The apparent desire of the local government in Xinjiang to reduce the birth rate of the Uighur population complicates the story of how we would characterize this one way or the other. +

+ Adam: It too easily slides out of consciousness that the Chinese regime undertook what Foucauldians would call one of the most grotesque, grandiose, and very violent political experiments in history, the one-child policy. It pursued that towards the Han population. That it’s then also capable of taking those kinds of techniques and applying them to resistant population like the Uighurs or Tibetans or the Mongolians, I don't think it’s surprising. It's the same toolkit. In the eighties, it was ferocious in its intrusiveness, monitoring women's menstrual cycles, forced abortions. This stands alone. No one's ever done that before on that scale. It’s quite mind-blowing and too easily consigned to the history books. +

+ Jordan: This is a technocratic regime deciding that this is the best thing and then getting a whole bureaucracy and apparatus to buy into it. +

+ Adam: Some people tell us it's driven by a bunch of people from the rocket aeronautics program and people who came out of Western-inspired cybernetics thinking, basically taking the limits to growth diagnosis too, literally. So, I completely agree that element of the repressive apparatus in Xinjiang is really dramatic and fully consistent with an indictment of genocide. +

+ This is one of the real puzzles about COVID, right? This is a regime, which in the name of socialist transformation shrimps from nothing, has also been astonishingly effective in its approach to the virus. Perhaps the common denominator is simply control and this being a matter of the highest possible political stakes. Not to be able to control this virus would be a far more serious blow to the prestige and legitimacy of a regime which has those kinds of pretensions than it is to the Trump administration in the U.S., which in the end just shrugs. +

+ Did The West Win The Cold War? +

+ Matt: Adam, you recently wrote a really interesting essay for the London Review of Books in which, among other things, you argued that the U.S. lost the Cold War in Asia, and that taking a long view of the past, that it wasn't really a situation where American interests were advanced, even though in principle, it seemed like at the time we'd won with the collapse of the USSR and the expansion of NATO. What would winning in your view of look like relative to what actually occurred? To what extent would the emergence of China under an undemocratic government inevitably have led there, in the same way that the U.S. and the USSR came into a hostile post-War relationship? +

+ Adam: What it is that we mean when we anchor our understanding of late 20th century history in the idea that the West won the Cold War? In Europe, that looks like the collapse of the Soviet Union, the disintegration of the Warsaw pact, the incorporation of the former Warsaw pact States into the EU and NATO. That's what victory looks like. That is the result of a culmination of several decades of crisis in Eastern Europe up to, and including, the imposition of martial law in Poland. +

+ None of that happened in East Asia. We were fought to a stalemate in Korea. Vietnam was a debacle. And one of the key anchors of the ultimate demise of Soviet Union is, after all, an alliance with China. And in 1989 at the moment, as it were, where the chips are falling and the Warsaw pact is disintegrating in Eastern Europe, we have Tiananmen Square, the Communist Party basically giving notice that this regime change will not extend to them. +

+ So that's what I was gesturing towards. We are really dealing with a Communist Party that's alive and kicking. Not of course as it were the Communist Party of Mao, but nevertheless an heir descendant of that party, is what we have to get to grips with and come to terms with. I think that ought to force us to reconsider this notion that the Cold War ended with us winning as it did in Europe. It didn't in Asia, and Korea feels the force of that, Japan feels the force of that, and the United States is now coming to terms with it too. +

+ Favorite Bureaucracies To Work For +

+ Jordan: I have a lighter one for you. Adam, pick three bureaucracies in particular time spans in which you would most like to be an employee of. +

+ Adam: What a wonderful question! I would like to have been in the Prussian state in 1806 and to have witnessed the shock of Napoleon, and the recovery and the emergence of the moment of German idealism, and the arrival of Hegel in Berlin, and the arrival of Clausewitz. As a Brit, somebody interested in economics, I want to be in the treasury in the UK any moment that Keynes is around, could be during World War One, could be during World War Two. To see a bureaucracy that in some respects is the ideal type of a high functioning machine. If you read the reports from Versailles, the Brits are effectively the people running the show because the civil service was a magnificent, bureaucratic machine. And then to see it inspired by a genius like that, you know what I mean? +

+ It’s parochial, but it would have been pretty interesting to be in Bonn in 1989 and to have seen that moment from the inside. And some of those people are majorly underrated players. And we know now the end of the Cold War is a multipolar business. But I think as somebody who grew up in West Germany, it would have been fantastically interesting to track that moment from the inside. +

+ Please consider supporting ChinaTalk. +

+ I’m also on the job market if anyone has leads on full time or consulting gigs! +

\ No newline at end of file diff --git a/packages/readabilityjs/test/test-pages/city-journal/distiller.html b/packages/readabilityjs/test/test-pages/city-journal/distiller.html new file mode 100644 index 000000000..bad4ddfd4 --- /dev/null +++ b/packages/readabilityjs/test/test-pages/city-journal/distiller.html @@ -0,0 +1,36 @@ +

+ The Great Eurasian Economic Realignment +

back to top
+ eye on the news +

+ The Great Eurasian Economic Realignment +

Western sanctions may accelerate Russia’s economic integration with Asia. + March 17, 2022 +
+ Politics and law +
+ Economy, finance, and budgets +

+ Since Vladimir Putin ordered the Russian Armed Forces to launch a full-scale invasion of Ukraine on February 24, Western countries and their close allies have hit Russia with a raft of sweeping and unprecedented sanctions. Several major Russian banks were cut off from the Belgium-based SWIFT international financial messaging system. Transactions with Russia’s central bank were banned by the United States, United Kingdom, and European Union, effectively preventing Russia from utilizing much of its $640 billion of foreign exchange reserves. Targeted asset freezes and seizures were carried out against high-level members of Russia’s political and economic elite, Russian planes were denied access to essentially all European and North American airspace, and an unofficial boycott of Russia has seen over 300 international businesses suspend operations in or with the country, including major brands such as Apple, Disney, Zara, Visa, IKEA, and Coca-Cola. The Russian ruble has lost about 40 percent of its value against the U.S. dollar in a matter of weeks. +

+ This kind of economic warfare is designed to isolate Putin’s government and bring an end to the invasion. But Russia’s vast exports of oil, coal, and natural gas to Europe have been notably exempt from the sanctions so far. Europe heavily relies on Russia to meet its energy needs: Russia alone supplies 40 percent of the European Union’s natural gas imports, 46 percent of EU coal imports, and 27 percent of EU oil imports. Russia is also the world’s largest wheat exporter, a leading producer of fertilizer, and possesses vast mineral reserves. The sanctions and boycotts against Russia are thus perhaps better termed financial warfare. Despite the belief that financial pain for the Russian populace will translate into political change, the nature of the Russian government remains the same. And while such bureaucratic and accounting moves can certainly cause significant pain in the short and medium term, they cannot change the fact that Russia possesses valuable and useful natural resources. +

+ But as the energy-independent U.S. pushes for energy sanctions, Germany in particular finds itself in an increasingly difficult position. For decades, Germany’s political and business elites have been implementing a wide-ranging transition from reliance on fossil fuels and nuclear energy to wind, solar, and biofuels. This energy transition, called the Energiewende in German, was always a precariously ambitious strategy for Europe’s premier industrial and manufacturing power. Heavy industry requires consistent and abundant energy—something that photovoltaic solar panels could provide in theory but not in dim and cloudy Germany. With its nuclear plants now almost entirely shuttered, Germany’s energy transition has instead resulted in a reliance on Russian energy significantly higher than the EU average, with 55 percent of natural gas imports coming from Russia in 2020. +

+ Will that change? In the wake of Russia’s invasion of Ukraine, the German government briefly floated the possibility of delaying shutdowns of its few remaining nuclear plants, before ultimately deciding against it and instead emphasizing plans to increase liquefied natural gas (LNG) imports and development of renewables. While the U.S. has announced plans to ban Russian energy imports immediately, new European plans to reduce energy imports from Russia are less ambitious on paper but could prove harder to achieve. +

+ Political considerations may ultimately outweigh economic ones in Europe. All that is stopping European countries from eventually reducing energy imports from Russia to zero is their willingness to pay the costs of higher energy import prices, such as by importing LNG from the United States, the Middle East, or Africa, or by continuing to subsidize the construction and operation of renewable-energy installations. Much as Westerners overrate the effects of economic hardship on political outcomes in Russia, Iran, or North Korea, it would be a mistake for Russia to overrate the effects of economic hardship on political outcomes in Europe. For while Russia is Europe’s largest energy supplier, Europe is Russia’s largest trade partner. The complete loss of this trade relationship would have far harsher impacts on Russia’s economy and state coffers than even the current round of financial sanctions. +

+ What has not been widely considered, however, is the possibility that Russia welcomes this outcome. If Russia is betting on economic divorce from Europe, including in energy, then sanctions and boycotts counterintuitively support, rather than frustrate, Russian strategy. Indeed, Putin’s government is already planning to retaliate with economic warfare of its own: it has prepared its own list of “unfriendly” countries to which to ban exports of natural resources, and has threatened to shut down gas to Germany via the Nord Stream 1 pipeline in retaliation for Germany halting the certification of Nord Stream 2. +

+ Perhaps these moves can be written off as symbolic reactions, but they may fit a broader strategy of replacing Europe with Asia as Russia’s key customer for natural resources. Russia has been heavily investing in development and infrastructure in its Arctic and subarctic regions, which produce an overwhelming majority of its fossil fuels and a substantial portion of its valuable minerals. The Russian government has gone so far as to plan a “Northern Sea Route” to Asia across Russia’s vast Arctic coastline, normally considered far too icy and dangerous for regular maritime shipping. Such a route could prove geopolitically vital, even if uneconomical, in case Russian ships are denied access to Western ports, as the United Kingdom has already done. To this end, Russia has also revived its fleet of nuclear-powered icebreakers and built the world’s first floating nuclear power station to supply electricity to remote regions of Siberia, with more of each on the way. +

+ Russia already depends on natural resource exports. That won’t change. But over the coming decades, it will not be the stagnant economies of Europe that will have the greatest demand for energy and minerals. Rather, the maturing industrial economy of China, in addition to the still-industrializing economies of India and other Asian and African countries, will be the primary growth markets for Russia’s natural resources. Neither China, India, Indonesia, the Gulf States, nor any of the many countries in Africa and Latin America chose to sanction Russia over the invasion of Ukraine. Today’s economic disassociation may well have unintended consequences for Europe. +

+ Photo by ALEXEY DRUZHININ/SPUTNIK/AFP via Getty Images +

+ City Journal is a publication of the Manhattan Institute for Policy Research (MI), a leading free-market think tank. Are you interested in supporting the magazine? As a 501(c)(3) nonprofit, donations in support of MI and City Journal are fully tax-deductible as provided by law (EIN #13-2912529). DONATE +
+ More from + +
\ No newline at end of file diff --git a/packages/readabilityjs/test/test-pages/cnbc/distiller.html b/packages/readabilityjs/test/test-pages/cnbc/distiller.html new file mode 100644 index 000000000..8c891dcef --- /dev/null +++ b/packages/readabilityjs/test/test-pages/cnbc/distiller.html @@ -0,0 +1,46 @@ +
+ +
+ Share
+ 0:32 +
+ George Clooney just sold his tequila business for up to $1 billion +

+ In celebration of National Tequila Day on July 24, George Clooney may be toasting to Casamigos Tequila, which the Hollywood star co-founded with friend Rande Gerber. +

+ The company, which the duo recently sold to Diageo for as much as $1 billion, was never meant for the public. It started out as a private collection of tequila for the co-founders and their friends and family. +

+ While building their side-by-side vacation homes in Cabo San Lucas, the long-time friends were spending a lot of time in Mexico and, consequently, drinking a lot of tequila. +

+ “As you do when you’re in Mexico, you drink a lot of tequila,” Gerber tells CNBC. “After a couple of months trying a lot of different tequilas, George turned to me and said, ‘Why don’t we just make our own? One that’s perfect for us to drink?’” +

+ That’s just what the pair did, along with their friend Michael Meldman. +

+ The team found a distiller in Jalisco, Mexico, and explained exactly what they were looking for. “It was important that it tasted great and didn’t burn going down,” Gerber explains. “We wanted one that we could drink straight or on the rocks. One that we could drink all day and all night, and not be hungover in the morning.” +

The Casamigos co-founders (from left to right): Michael Meldman, George Clooney, and Rande Gerber.
+ The Casamigos co-founders (from left to right): Michael Meldman, George Clooney, and Rande Gerber. +
+ Source: Casamigos Tequila +

+ After months of back-and-forth with the distiller, and tasting and developing their tequila, the duo found the perfect product, Gerber recalls: “George opened the sample, poured one for me, poured one for him, and we tasted it. We both looked at each other, had another taste of it, and we were like, ‘This is it. It’s perfect.’” +

+ Casamigos, which loosely translates to “house of friends,” was never meant for the public. For two years, it was served exclusively among Gerber and Clooney’s friends and family. +

+ “It was just for us,” Gerber says. “We didn’t really want to be in the business. We figured, George is an actor and a director. I own restaurants and bars and wasn’t looking to get in another business. +

+ “But then the distiller called and said, ‘Hey guys, we have a little problem: In the past two years, we’ve been sending you about a thousand bottles a year. Either you’re selling it or you’re drinking way too much — either way, we can’t keep calling it samples. You guys have to get licensed and do this right.’” +

+ The celebrity entrepreneurs, who brought Casamigos to the public in 2013, figured they didn’t have a choice. “We wanted to keep drinking it,” Gerber says. +

George Clooney and Rande Gerber.
+ George Clooney and Rande Gerber. +
+ Source: Casamigos Tequila +

+ The organic evolution of Casamigos made the process of starting a company less stressful, Gerber tells CNBC: “This all happened naturally. We just wanted the best tequila for us to drink — so when it turned into a business, it was easy, because we didn’t really care about whether or not it was a huge success or not.” +

+ What started as a passion project turned into a successful private-label tequila business. “It immediately took off,” says Gerber, and now, Diageo will initially pay $700 million for it, with the potential for another $300 million depending on the tequila’s performance. +

+ The key to their success has been focusing on the product inside the bottle, not the bottle or label itself, says Gerber. “We have an inexpensive bottle, but the best quality tequila inside.” +

+ This is an updated version of a story that was previously published in November 2016. +

\ No newline at end of file diff --git a/packages/readabilityjs/test/test-pages/community.musictribe.com/distiller.html b/packages/readabilityjs/test/test-pages/community.musictribe.com/distiller.html new file mode 100644 index 000000000..b1efc4a26 --- /dev/null +++ b/packages/readabilityjs/test/test-pages/community.musictribe.com/distiller.html @@ -0,0 +1,25 @@ +
+ I have a regular small sound job that I handle with an XR12 controlled from a 10" Android tablet running Mixing Station X Air Pro. The only challenge with this job is that I get very little time for get-in and get-out so any problems in setting up could be disastrous. I carry a spare 7" tablet, backup access point and an OTG adaptor for wired Ethernet but I don't have much time for trouble-shooting.
+
+
+
+ I have long thought it would be good to have basic control via MIDI in case I lost my normal user interface but I did not want a large and/or expensive controller. I did want the controller to have feedback for mute states and fader levels. I have achieved this using the XTouch Mini.
+
+
+
+ The XTouch Mini, in common with most small and reasonably priced controllers, is a USB MIDI device and the XR12 requires DIN MIDI. This can be resolved, though, with a USB/DIN MIDI converter. The Kenton MIDI USB Host would work but is rather expensive. The HobbyTronics MIDI USB-DIN Converter is much lower cost but looks a little fragile (*) so would need to be repackaged for life on the road. I took the low-cost route by using HobbyTronics USB Host Mini (preloaded with MIDI firmware) plus a HobbyTronics MIDI Breakout. These two boards are mounted in a small box along with a connector for 5Vdc power. The converter and controller only draw about 100mA so I can run it for over twelve hours from a 2000mAh phone powerbank if there is no convenient power outlet or spare USB-A port.
+
+
+
+ Using the XTouch Editor, I have programmed eight encoders and eight buttons to control the faders and mutes for the critical channels and buses using the published MIDI messages. The encoder LED rings and mute LEDs follow the XR state when controlled by both the MIDI board and the normal network UI. When first turned on, though, the XTouch does not reflect the state of the XR. The quick solution to this is to load an XR snapshot which gives instant synchronization so I programmed one of the spare XTouch buttons to load my default snapshot. As another handy shortcut, I use the encoder press functions to set default fader levels for their respective channels/buses. This is another way to get the faders synchronized without loading a snapshot.
+
+
+
+ So I have achieved what I set out to do: I can take control of the basic mute and fader functions of the XR if my tablet and/or network fail. I hope that I don't need this capability but it's in the mixer's gig-bag just in case.
+
+
+
+ (*) HobbyTronics have improved this product. It should not now need repackaging. See HobbyTronics USB/Midi Converter.XTouch Mini.jpgInside.jpgConverter.jpg +
+ 11,159 +
\ No newline at end of file diff --git a/packages/readabilityjs/test/test-pages/computer.rip/distiller.html b/packages/readabilityjs/test/test-pages/computer.rip/distiller.html new file mode 100644 index 000000000..627453088 --- /dev/null +++ b/packages/readabilityjs/test/test-pages/computer.rip/distiller.html @@ -0,0 +1,86 @@ +
_____                   _                  _____            _____       _ 
+  |     |___ _____ ___ _ _| |_ ___ ___ ___   |  _  |___ ___   | __  |___ _| |
+  |   --| . |     | . | | |  _| -_|  _|_ -|  |     |  _| -_|  | __ -| .'| . |
+  |_____|___|_|_|_|  _|___|_| |___|_| |___|  |__|__|_| |___|  |_____|__,|___|
+  a newsletter by |_| j. b. crawford                       home subscribe rss

+ >>> 2020-11-28 the verboten band +

+ To start: yes, long time no see. Well, COVID-19 has been like that. Some days I feel accomplished if I successfully check my email. I finally managed to clear out a backlog of an entire handfull of things that needed thoughtful responses, though, and so here I am, screaming into the void instead of at anyone in particular. +

+ That said, let's talk a bit about radios. It is probably unsurprising by now that I have a long-running interest in radio and especially digital radio communications---but people who come to radio from all kinds of different perspectives run into one odd problem: the curious refusal of any receiver to tune to certain frequencies in the 800-900MHz range. +

+ A lot of people have a general knowledge that this has to do with some kind of legal prohibition on reception of cellular phones. That's roughly correct, but to fully explain the matter requires going into some depth on two different topics: FCC regulation of radio devices, and the development of cellular phones. The first sounds more boring, so let's hit that one first. +

+ Generally speaking, most electronic products manufactured or imported into the United States are subject to regulation by the Federal Communications Commission. Specifically, they generally require an "Equipment Authorization" from the FCC prior to being marketed. For purposes of this regulatory scheme, electronic devices can be broadly divided into two categories: intentional radiators and unintentional radiators. +

+ An intentional radiator is something that is specifically intended to broadcast a radio signal, like, say, a cellular phone. Intentional radiators must be certified to comply with the specific Part of the FCC regulations relevant to the service for which they will be used. For example, cellular phones must be certified against Part 27, Wireless Communications Service, among others. The exact process varies by the part and can be involved, but it generally involves the manufacturer paying a certified test lab to perform certain tests and complying with various other filing requirements which include placing a label on the device which specifies its FCC approval. Device manufacturers must file with the FCC a description of how this label will appear before they receive approval to market the device, which is why the rough designs of unreleased devices are sometimes revealed by the rough drawings in these filings---tech journalists will watch these to get the dimensions of new iPhones, for example. +

+ By the way, when I say the "FCC Regulations," if you want to follow along at home these are promulgated as 47 CFR. So Part 27, for example, refers to 47 CFR 27. The ever lovely Cornell LII has the whole thing for your entertainment: https://www.law.cornell.edu/cfr/text/47. There's some reading for when you need help falling asleep. +

+ But that's all besides the point, I'm more interested in talking about unintentional radiators, devices which are not intended to produce RF radiation but may still do so as a result of the operation of the electronics---this is generally called a spurious emission, which is basically any RF emitted by accident. These devices are certified under Part 15 of the FCC regulations[1], and so are sometimes called "Part 15 devices." Part 15 essentially limits the type and amplitude of spurious emissions to prevent random devices causing harmful interference due to defects in their designs. +

+ What would we call a radio receiver, then? It is explicitly a radio device, but is not intended to transmit anything. As a result, radio receivers are Part 15 devices. Most of Part 15 is very general and doesn't really say anything specific about radio devices, it just limits spurious emissions and other design standards. However, 15.121 gets a great deal more specific in discussing "Scanning receivers.' A scanning receiver is specifically defined earlier in the regulation as a device capable of tuning to two or more frequency bands in the range of 30-960Mhz. This has the fun result that nothing for the GHz range is technically a scanner, but for practical reasons this doesn't matter too much. +

+ So what's in 15.121? This is: +

+ 47 CFR 15.121(a): ... scanning receivers and frequency converters designed or marketed for use with scanning receivers, shall: (1) Be incapable of operating (tuning), or readily being altered by the user to operate, within the frequency bands allocated to the Cellular Radiotelephone Service in part 22 of this chapter (cellular telephone bands). ... (b) Be designed so that the tuning, control and filtering circuitry is inaccessible. The design must be such that any attempts to modify the equipment to receive transmissions from the Cellular Radiotelephone Service likely will render the receiver inoperable. +

+ The rest of paragraph (a) gives a pretty long clarification of "readily being altered by the user," and it's amusing to think of a bunch of FCC characters sitting around a table trying to think up every alteration that is easy. Jumper wires and reprogramming micro-controllers are both right out. +

+ It gets even better: +

+ 47 CFR 15.121(b): ... scanning receivers shall reject any signals from the Cellular Radiotelephone Service frequency bands that are 38 dB or lower based upon a 12 dB SINAD measurement, which is considered the threshold where a signal can be clearly discerned from any interference that may be present. +

+ So, here's this actual weird rule about scanners. Scanners are specifically prohibited from being able to tune to any bands allocated to the Part 22 Cellular Radiotelephone Service. This raises questions, and as you can imagine from the way I got here, I am about to spend a long time answering them. +

+ When the FCC says "Cellular Radiotelephone Service," they aren't talking about cell phones in general. The CRS as I'll call it refers to a very specific cellular service, and that is AMPS. +

+ AMPS, the Advanced Mobile Phone System, is the most common in the US of the "1G" cellular services. Most carriers that were around when it was offered called it "Analog" service, and indeed, AMPS was entirely analog. And, due to an odd detail of the regulation, large cellular carriers were required to offer AMPS service until 2008, long after AMPS phones were no longer produced. You may have had a candy bar phone back when you would occasionally see an "A" for analog service, but I hope not into the late 2000s. +

+ There are a few things that we might infer from AMPS being an analog service. One of those things is that it probably did not employ strong encryption. In fact, AMPS employed no scrambling or enciphering of any kind. Your phone conversations were just flapping in the wind for anyone to hear. This posed a major practical problem for carriers in the '90s as it was discovered that it was not particularly difficult to intercept the call setup process from an AMPS phone and swipe its identification numbers, allowing you to basically steal someone else's cellular service. You can imagine that this was popular with certain criminals with a need for untraceable but convenient communications. +

+ There was also a problem for consumers: their phone conversations could be fairly easily overheard. There were a number of ways to do this, using any radio scanner that covered that band for example. One particularly well-known option was a particular model of phone, the Oki 900, that had an unusually open design (in terms of modifiability) that led to reverse engineered and modified firmware being developed that made eavesdropping on other people's calls just, well, a feature it had. +

+ The scale of this problem was fairly large, and it was fairly well known. For example, let's turn to my favorite source of late-night reading, newspaper archives. A lovely piece in the 30 May 1990 issue of The News and Observer, from Raleigh NC, takes the cheesy headline "Monitoring Megahertz" and goes into some depth on the issue. +

+ "I've heard men call their wives and tell them they'll be home late, then call their girl friends," quipped one electronics store owner who had "accidentally" eavesdropped on cellular calls using a scanner. We've all fat-fingered our ways into someone else's affairs I'm sure, pun intended. Another person said "when you look at the fact that there are how many thousands of people out there who know my name, my mailing address and my salary...I put cellular eavesdropping down as being no different from that." In the face of technology, even in 1990, people had begun to abandon their privacy. +

+ Cellular carriers were not so happy about this, viewing it as an embarrassment to their operation. I have heard before that cellular carriers went so far as to lobby for banning scanners entirely, although I am not aware of much hard evidence of this. What they did do was convince congress to stick an extra few paragraphs onto an otherwise only tangentially related bit of legislation called the Telephone Disclosure and Dispute Resolution Act of 1992. This has largely to do with abusive 1-900 numbers, which is its whole own topic in telephone regulation that I ought to take on sometime. But it also brought along just a bit more, an extra section that was subsequently amended several times at the behest of cellular carriers. Let's read part of it, as amended, and with some editing for readability. +

+ The Commission shall prescribe and make effective regulations denying equipment authorization for any scanning receiver that is capable of---(A) receiving transmissions in the frequencies allocated to the domestic cellular radio telecommunications service, (B) readily being altered by the user to receive transmissions in such frequencies, or (C) being equipped with decoders that convert digital cellular transmissions to analog voice audio. +

+ Well, we've made it full circle: we've seen the regulation, and we've seen the legislation that kicked the FCC to write the regulation. But how does this translate today? Things get a bit weird there. +

+ You see, the FCC seems to have (sensibly) interpreted the legislation as applying directly to the Cellular Radiotelephone Service, even though the legislation actually uses the term "domestic cellular radio communications service" which seems almost equally lively to have been (1) intended to be more general in its applicability or (2) a result of someone drafting legislation having read "Cellular Radiotelephone Service" in the FCC regulations but then forgetting exactly how it was worded. +

+ The Cellular Radiotelephone Service was allocated 824-849MHz and later 869-894MHz. That's it. You see, all of the digital cellular systems we use today are considered completely different services from Cellular Radiotelephone (usually called Wireless Communications Service although the details get complex). As a result, and to this day, those two sections in the 800MHz band are verboten to scanners, and nothing else. +

+ And about those frequencies... after the requirement for AMPS service ended, all US carriers ceased AMPS operations. The old AMPS bands remain allocated for cellular service, and Verizon and a couple of smaller carriers use the same frequencies for digital cellular services, which employ encryption and cannot be intercepted by radio scanners. The prohibition on tuning scanners to these frequencies no longer makes any sense, especially since this ban has never been extended to the AWS, PCS, and WCS bands that are more widely used by modern cellular phones. +

+ My suspicion is that the fact that this regulation was mandated by congress makes it difficult for the FCC to remove or modify, even though it no longer makes technical sense. Unless congress finds some time for minutiae we are unlikely to see a change in this rule. +

+ In general, the whole thing is sort of bizarre. Broadly speaking, it is legal to listen in on any radio communications in the US, but cellular phones have repeatedly gotten a special carve-out. +

+ Repeatedly? That's right. The whole AMPS band and scanners rule is the only specific technical regulation, but the Electronic Communications Privacy Act of 1986 had actually already made it illegal to intercept or listen in on cellular calls, and this remains true to the present day... but there was virtually no enforcement, and that hasn't really changed to this day. +

+ And of course the whole thing has always felt like a farce. The solution to the poor (or rather nonexistent) security design of AMPS was never legislation, but cellular carriers and the congress will be damned if they didn't try. In practice, the rule swept the entire eavesdropping problem under the rug for some years, allowing carriers to continue operating the insecure AMPS system for far longer than they should have (...but exactly as long as the FCC required them to). +

+ Because listening to the modern digital cellular modes wouldn't be particularly interesting or useful anyway, and this rule doesn't really deter anyone with the motivation and ability to decode those modes anyway, there are two lasting impacts of this rather particular rule: +

+ 1) SDRs and other receivers made today must implement this particular and peculiar restriction in order to receive US equipment authorization, which is probably part of the reason that a lot of SDRs... don't. +

+ 2) To comply with the specifics of the regulation about rejection, many receivers use a notch filter around 850MHz in their frontend. This means that reception throughout the 800-900MHz range is particularly poor, a real irritation as various public agencies and private agencies (especially railroads) use land-mobile radios elsewhere in the 800-900Mhz range. +

+ Basically, more than a decade after any of this made sense, we're all still hassling with it. +

+ [1] Part 15 is actually a lot more general and unintentional radiators are specifically discussed under 47 CFR 15.101, but everyone just says Part 15. + 

sincerely,
+                                                 j. b. crawford
+                                                 me@computer.rip
+
+
+This website is begrudgingly generated by the use of software. Letters to the
+editor are welcome via facsimile to +1 (505) 926-5492 or mail to 609 Gold Ave
+SW, Suite 1D, Albuquerque, NM 87102. Opinions stated here are somewhat
+necessarily those of Seventh Standard, LLC, in that the author is the sole
+partner and does not wish to lead dual lives.
\ No newline at end of file diff --git a/packages/readabilityjs/test/test-pages/computerenhance.com/distiller.html b/packages/readabilityjs/test/test-pages/computerenhance.com/distiller.html new file mode 100644 index 000000000..156114234 --- /dev/null +++ b/packages/readabilityjs/test/test-pages/computerenhance.com/distiller.html @@ -0,0 +1,135 @@ +

+ Computer, Enhance! +

A sculpture of a cartoon character stuck in a pipe.

+ By now I should know better than to ask on Twitter for a “rigorous analysis” of anything. As George W. Bush said, “Fool me once, shame on you… fool me can’t fooled again.” +

+ I don’t want to be “fool me can’t get fooled again”, so I officially give up on technical tweets. Today’s the last day I will ever post anything technical on Twitter, I promise. Instead, you will be forced to endure yet another Substack, so I can post 3,000-word posts that no one will read. +

+ Here we go: +

+ The goal with raw UDP is very simple: better performance and security on the server side. +

+ HTTPS is an unbaked sausage made by grinding pure text HTTP with TLS and encasing the result in an arbitrary selection of third-party animal intestine… err, I mean, “highly secure” certificates provided by arbitrarily selected certificate providers. Implementing HTTPS is a massive amount of code that is inexorably slow. It is not only theoretically difficult to secure completely, but is insecure in practice in popular implementations available to the public. +

+ Oh, and the certificate authorities are also insecure, by the way - but that’s another story (and another, and another, and …) +

+ It also relied (up until recently) on TCP, which, unless you plan to write a completely custom network stack for every type of server/NIC you ever use, requires the underlying kernel to understand and track network connections. This means that you inherit substantial overhead, and perhaps vulnerabilities as well, from the TCP/IP substrate before you even begin to write your server code. +

+ If you were a large company with significant academic and engineering resources, you might instead want to design your own private secure protocol that: +

  1. + Uses encryption you control, so it cannot be bypassed by hacking the certificate authority, +

  2. + Uses UDP to avoid having OS connection state on the server side, and +

  3. + Uses a well-designed, known packet structure to improve throughput and reduce security vulnerabilities from HTTP/TLS parsing. +

+ The first thing on that list is half-possible now. Although there’s nothing you can (ever1) do to avoid man-in-the-middle attacks the very first time someone interacts with your server, web APIs have long made it possible to store data on the client for later use. One use for that data would be storing your own set of public keys. +

+ So even using nothing newer than XHR and cookies, you could theoretically add your own layer of encryption to anything you send to the server. This would ensure that any subsequent hack of the certificate authority could not inspect or modify your packets. It’d be much less efficient than rolling your own top-to-bottom, because now you pay the entire cost for your encryption and TLS. But you can do it. +

+ It’s slow, but possible. Call it half-possible, like I did above. +

+ The second thing on the list is sort-of possible now as well. If you can somehow manage to use HTTP/3 exclusively as your target platform, you will still be talking HTTP but you’ll be doing it over UDP instead of TCP, and can manage connection state however you wish without OS intervention. +

+ It is probably unrealistic to assume that you could do this in practice today. If you didn’t care about broad compatibility, you probably wouldn’t be deploying on the web anyway, so presumably the current adoption of HTTP/3 is insufficient. But at least it exists, and perhaps if adoption continues to grow, eventually it will be possible to require HTTP/3 without losing a significant number of users. For now, it’s only something you can do on the side - you still have to have a traditional HTTPS fallback. +

+ Which brings us to the third item on the list, and the real sticking point. As far as I’m aware, no current or planned future Web API ever lets you do number three. There are many new web “technologies” swarming around the custom packet idea (WebRTC, WebSockets, WebTransport), but to the best of my knowledge, all of them require an HTTPS connection to be made first, so your “custom packet” servers still need to implement all of HTTPS anyway. +

+ I can imagine someone raising the following objection at this point: “If you don’t support HTTPS on the server, how do you serve the WASM/JavaScript/whatever with the custom packet logic in the first place?” +

+ That’s a reasonable question. +

+ The answer is, the two most logical deployment scenarios I can think of both involve a separate server (or process) for the initial HTTPS transaction. +

+ The first is what I imagine would be the most common: you upload to a CDN a traditional web package containing the PWA-style web worker necessary to do your own custom packet logic. The CDN serves this (static) content everywhere for you. They obviously implement HTTPS already, because that’s what they do for a living, and they’re not your servers anyway so you don’t care. +

+ The second would be less common, but plausible: you run your own CDN-equivalent, because you’re just that hard core. But you expect that your HTTPS code is more vulnerable than your custom code, since HTTPS is vastly more complicated and has ridiculous things in it like arbitrary text parsing, which no one in their right mind would ever put into a “secure” protocol. So you cabin your HTTPS server instances into their own restricted processes or own machines entirely. This prevents exploits of the HTTPS code from affecting anything other than newly connecting users - existing users (who are only talking to your custom servers) remain unharmed. +

+ In neither scenario do you actually include HTTPS code in any of the processes running your actual secure server. +

+ So that’s the hopefully-at-least-somewhat-convincing explanation of why someone might want raw UDP. Now the question is, can raw UDP be provided by a browser in a way that is “secure”? +

+ I’m putting a lot of these words in scare quotes because browsers aren’t secure for any serious definition of that word, and hopefully that is overwhelmingly obvious to everyone who has ever used one. But just to be clear about the landscape, there are two different ways browsers are not secure: +

  1. + The web as a platform consists of massive, overlapping, poorly-specified APIs that require millions of lines of code to fully implement. As a result, browsers inexorably have an effectively infinite number of security exploits waiting to be found. +

  2. + Browsers include the ability, sans exploit, to transmit information from the client computer to any number of remote servers. Without the ability to control this behavior, the user’s data could be misappropriated. +

+ Clearly, for raw UDP, we only care about the second one of these. The first one happens in browsers all the time already and there’s no reason to suspect that raw UDP would somehow have more implementation code vulnerabilities on average than any other part of the sprawling browser substrate. +

+ So the question is, assuming the browser has not been exploited, what is the security standard for web features, and can raw UDP be implemented under that standard or not? +

+ As a point of comparison, I will use the example of the current camera/microphone/location policy as it presently exists. That will be our “gold standard”, since if it were not considered “secure” by web implementers, presumably it would not have been knowingly shipped in web browsers everywhere for the past several years. +

+ As everyone who uses a web browser knows, a web site at present is allowed to ask you for permission, temporarily or permanently (your choice), to access your camera, microphone, and location data. Once you say “yes” to any one of these things, that site can transmit that data anywhere in the world, and use it for any purpose, trivially. +

+ Allow me to provide a worked example. +

+ Suppose I partner with Jeffrey Toobin to make a cybersex conduit site for people who, like him, see the value in quickly switching tabs away from your work meetings to get down to some real business. We launch cyberballsdeep.net, and it’s a big success. +

+ When a user visits our site, they see at most two security-related things: +

  1. + An allow/deny request for access to the microphone and camera, and +

  2. + A lock icon indicating that the connection has been signed by a third party warranting that this connection is end-to-end encrypted from the user’s machine to some server somewhere with the secure keys for cyberballsdeep.net. +

+ Assuming you click “allow” - which you have to in order to use the service - the servers at cyberballsdeep.net can now do anything they want with your (very sensitive) video data. They can, for example, record you while you are toobin’ and play it back at any time, anywhere, at their discretion. They could play it on a billboard in Times Square, they could send it to your spouse - anything goes. +

+ So the “security standard” that you are getting, in practice, exactly mirrors the two things you saw: +

  1. + You know your sensitive data will not be captured unless you click “allow”, and +

  2. + You know that nobody will be able to see your sensitive data unless either cyberballsdeep.net or the issuing certificate authority let them (either intentionally, or unintentionally if they’ve been hacked). +

+ That’s it. You don’t know anything else. In practice, you basically have no security guarantees other than a warrant that your sensitive data will go to a particular named party first before it goes somewhere else. +

+ Hopefully we can all agree that this extremely low bar for security is the only hurdle one should have to clear in order to dismiss concerns of “security” as a reason not to implement a feature in a W3C spec. It’s not much, but it is something. +

+ OK, finally, with all that out of the way, this is what I actually wanted someone to point me to when I asked about this on Twitter. I just wanted to see that someone, somewhere, had worked out exactly why UDP could not be made to fit the same security model considered acceptable across other basic web features already deployed and considered “secure”. +

+ Since nobody sent me such a thing, I am still stuck with my own security modeling, with nothing to compare against. My model goes something like this: +

+ Step one - the “allow/deny” step - is easy for raw UDP to provide. The browser is still sitting between the JavaScript/WASM layer and the OS sockets layer, so it can ensure that inbound and outbound packets are filtered any way the browser wishes. +

+ This means that it would be trivial for a browser to only allow UDP packets to and from servers that the user has authorized, as it does with microphone, camera, and location data. Any site that wishes to access raw UDP simply provides a hostname to the browser, and the browser asks the user whether they wish to allow the page to communicate with that site. +

+ Furthermore, since the browser already allows the page to send as much HTTPS data as it wants back to the originating site, one could optionally allow any site to send UDP packets back to its own (exact) originating IP without asking the user. This is not necessary for raw UDP to work, but I can’t think of any violation of “step one” that would happen as a result, so it could be considered. +

+ Note that this is not true for something like camera/microphone/location data. Those are additional data sources to which the page gets access, so if anything, raw UDP permission is less dangerous in terms of user permission, since at no time does the page itself get additional access to the user’s data, regardless of whether they allow UDP communication. +

+ Which brings us to step two. +

+ As far as I can tell, there’s actually nothing special about step two. The original web page was served by HTTPS, obviously, since that’s the only way the browser supports getting WASM/JavaScript downloaded in the first place. So the originating server and code are already exactly as “secure” as they would be in any other scenario. +

+ The user had to affirmatively allow the destination name, so the page can only send UDP to a specifically approved endpoint. +

+ So the only question is, can the user be sure that the data sent to that endpoint is encrypted such that only the endpoint or the certificate authority can decrypt it? +

+ I can’t know the hivemind of a W3C committee (thank the heavens). But if I had to guess, I would suspect that this is why they didn’t want to allow raw UDP (or raw TCP for that matter). In their mind, it probably seems less secure than HTTPS to allow a web page to implement its own secure UDP protocol. +

+ However, to my mind, this is based upon a flawed assumption. That assumption is that somehow web implementers can be trusted to deploy their encryption keys securely, but cannot be trusted to deploy their protocol securely. +

+ To be more specific, HTTPS can be intercepted trivially if the attacker A) has a machine on the route between the endpoints and B) has access to the server’s keys, or any certificate authority’s signing capability. (A) either happens or it doesn’t - there’s no way to control it - so (B) is really the entire question. +

+ So the notion that allowing web pages to use UDP for transmission is less secure than HTTPS seems to me to be predicated on the notion that web developers can be trusted to do something complicated in one place (run a set of servers without leaking keys), but also cannot be trusted to do something complicated in another (download, for example, a JavaScript UDP encryption library and use it). +

+ Stated alternately, the hard constraint on the client side that you can’t roll your packet code “for security reasons” is nowhere to be found on the server side. There is no requirement anywhere in W3C or anywhere else that says your web server has to be… well… anything at all, really. You can just go ahead and write your own code from top to bottom. You can even have a dedicated web page on your site that has the entire cryptographic key set for the server posted on it for people to cut-and-paste, so everyone can impersonate your server to anyone, anywhere, at any time. You can leave a thumb drive with your keys at the bar. You can generate your keys with a random seed of 0x000000000000000000. Anything goes. +

+ Nobody seems to be panicked about this. Nobody has pushed the policy that the W3C should standardize on a specific web server deployment that you are forced to use, or a set of n of them made by Google/Mozilla/Apple, or what have you. It is just assumed that everyone is allowed to write their own server packet handling, but that no one is allowed to write their own client packet handling. +

+ So that’s what I would like explained. Internet, justify this! +

+ I have seen people mention (but not support) a claim that raw UDP would cause “denial of service” problems because malicious web pages would send UDP packets to random servers in an attempt to overload them. This claim seems completely baseless to me, because there is no reason why you can’t employ the relevant XHR DDoS restrictions to UDP. If DDoS was the concern, just require that UDP packets be sent exclusively within the same domain as the originating code. +

+ Furthermore, you could restrict the port ranges of raw web UDP to some assigned range. A new port range could be explicitly reserved just for raw web UDP if that makes people more comfortable, so it could literally be discarded at the gateway on any network that doesn’t want to support raw UDP for web, making it easier to deal with than UDP attacks from native code and viruses which can choose their ports at will. +

+ At that point, I fail to see how raw UDP from the browser could be significantly more dangerous than XHR, unless I am missing some particularly clever use of UDP. And again, that’s why I asked for writeups in my original tweet. I’m totally willing to believe I’m missing something, but I want to see a complete technical explanation about what it is. +

+ Now, none of this is the same as saying I can’t see how you would perform DDoS attacks with raw UDP. I certainly can. I just can’t see how you would perform them more easily than with XHR, which obviously is considered “secure”. +

+ As a simple example, suppose a commercial CDN distributes the payload of ddosfuntimes.com. On the main page, there’s an XHR to target.ddosfuntimes.com. Even though the CDN is a completely different set of IP addresses as target.ddosfuntimes.com, this is completely legal under XHR policy. +

+ The owners of ddosfuntimes.com can go ahead and set the IP address in their DNS records to point target.ddosfuntimes.com at any server they want, and they will receive all the XHR traffic from every browser that visits the page. And to the best of my knowledge, there isn’t a damn thing the target can do about that. +

+ So unless I’m missing something, XHR already allows you to target any website you wish with unwanted traffic from anyone who visits your site. So why the concern about UDP? +

\ No newline at end of file diff --git a/packages/readabilityjs/test/test-pages/dailymail/distiller.html b/packages/readabilityjs/test/test-pages/dailymail/distiller.html new file mode 100644 index 000000000..822016028 --- /dev/null +++ b/packages/readabilityjs/test/test-pages/dailymail/distiller.html @@ -0,0 +1,107 @@ +

+ EXCLUSIVE: Welcome to the neighborhood! Pregnant Katharine McPhee and David Foster are house hunting in Harry and Meghan's star-studded Montecito enclave to be closer to their royal BFFs and have play dates with Archie +

  • David Foster and his pregnant wife Katharine McPhee are house hunting in Montecito so they can live near Harry and Meghan, DailyMail.com can reveal
  • A source close to Foster, 70, and McPhee, 36, said they want their child to grow up alongside Archie in the family friendly area near Santa Barbara, California 
  • 'Now that Katharine is pregnant they both want to settle down and raise their child in a nice area with nice schools,' our source said 
  • Meghan and McPhee are longtime friends, having gone to school together and performed in musicals in Southern California 
  • Since the Sussexes moved to Montecito, music producer Foster and McPhee have spent a lot of time there and fallen in love with the area   
  • Meghan and Harry bought a nine-bedroom and 16-bathroom mansion in the secluded area of Riven Rock in upscale Montecito in June for $14.65m
  • Katy Perry and Orlando Bloom have just snapped up a gorgeous Mediterranean villa in Montecito, California for $14.2 million 

+ Prince Harry and Meghan's close friends David Foster and Katharine McPhee are house hunting in Montecito so they can live near the royal couple, DailyMail.com can reveal. +

+ It's understood music producer Foster and his pregnant singer/actress wife McPhee have been looking at properties in the exclusive celebrity enclave and are considering making a permanent move. +

+ A source close to Foster, 70, and McPhee, 36, said they want their child to grow up alongside Archie in the family friendly area near Santa Barbara. +

+ The source told DailyMail.com: 'David and Katharine have become really close to Harry and Meghan. +

+ 'And now Katharine is pregnant they both want to settle down and raise their child in a nice area with nice schools. +

+ 'They've spent quite a bit of time in Montecito since Harry and Meghan moved there and they've fallen in love with it so they decided to start looking at properties to see what's on the market. +

+ 'If they do make the move their child can play with Archie and become good friends. Harry and Meghan are excited about having them live nearby and they all want their kids to grow up together.' +

David Foster and his pregnant wife Katharine McPhee are house-hunting in Montecito so they can live near Harry and Meghan, DailyMail.com can reveal
+10 +

+ David Foster and his pregnant wife Katharine McPhee are house-hunting in Montecito so they can live near Harry and Meghan, DailyMail.com can reveal +

A source close to Foster, 70, and McPhee, 36, said they want their child to grow up alongside Archie in the family friendly area near Santa Barbara, California
+10 +

+ A source close to Foster, 70, and McPhee, 36, said they want their child to grow up alongside Archie in the family friendly area near Santa Barbara, California +

Meghan and Harry bought the mansion, known as 'The Chateau', for $14.65 million on June 18, making them neighbors with celebrities Oprah Winfrey and Ellen DeGeneres
+10 +

+ Meghan and Harry bought the mansion, known as 'The Chateau', for $14.65 million on June 18, making them neighbors with celebrities Oprah Winfrey and Ellen DeGeneres +

+ Foster and McPhee currently live in the music producer and Real Housewives of Beverly Hills alum's LA apartment, but it's not clear whether he will be selling up. +

+ The two couples have become firm friends since Harry and Meghan moved to California. +

+ And they enjoyed a cozy dinner date together in Montecito last week to celebrate McPhee's pregnancy. +

+ The outing was the first glimpse of the Duke, 36, and Duchess of Sussex, 39, in Montecito since DailyMail.com exclusively revealed they had moved into a sprawling $14 million mansion in the leafy Californian town. +

+ The couple bought the mansion, known as 'The Chateau', for $14.65 million on June 18, making them neighbors with celebrities Oprah Winfrey and Ellen DeGeneres. +

+ The 14,563-square-foot home, known as 'The Chateau', sits on 5.4 acres of land and immaculately clipped hedges border the estate's stone-pillared entry gates. +

+ Foster, a 16-time Grammy winning composer and music producer, and McPhee have a few properties to choose from in the area.  +

Depending on their budget they could outdo the Sussexes by stumping for this stunning $16.9million, 12.000 sq ft Italian style estate called Villa Bencistà
+10 +

+ Depending on their budget they could outdo the Sussexes by stumping for this stunning $16.9million, 12.000 sq ft Italian style estate called Villa Bencistà +

If that's too pricey then a smaller, but more modern $6.4million architectural estate is also on offer, boasting a swimming pool and 4,000 sq ft of space
+10 +

+ If that's too pricey then a smaller, but more modern $6.4million architectural estate is also on offer, boasting a swimming pool and 4,000 sq ft of space +

If Katharine and David are looking for something a little more cozy, then there's a beautiful single-level, town home also on the market, a snip at $2.3million
+10 +

+ If Katharine and David are looking for something a little more cozy, then there's a beautiful single-level, town home also on the market, a snip at $2.3million +

+ Depending on their budget they could outdo the Sussexes by stumping for a stunning $16.9million, 12.000 sq ft Italian style estate called Villa Bencistà. +

Katy Perry and Orlando Bloom have just snapped up a gorgeous Mediterranean villa in the wealthy enclave of Montecito, California for $14.2 million
+10 +

+ Katy Perry and Orlando Bloom have just snapped up a gorgeous Mediterranean villa in the wealthy enclave of Montecito, California for $14.2 million +

+ The sprawling estate, owned by Monica Allyn, the wife of film producer Rob Allyn, came on the market earlier this month. +

+ If that's too pricey then a smaller, but more modern $6.4million architectural estate is also on offer, boasting a swimming pool and 4,000 sq ft of space. +

+ But if they're looking for something a little more cozy, then there's a beautiful single-level, town home also on the market, a snip at $2.3million. +

+ Harry and Meghan have bonded with Foster and McPhee since leaving the royal family and arriving in California. +

+ Also new to the neighborhood are Katy Perry and Orlando Bloom who snapped up an exquisite compound in Montecito for $14.2 million. +

+ The couple, who welcomed daughter Daisy Dove in August, purchased the stunning ocean view home from former Duracell and Chrysler CEO C. Robert Kidder, who owned the estate for over 20 years, according to Variety.  +

+ The couple's gated property consists of multiple structures sitting upon 8.9 acres of land and will be situated among Hollywood's elite.   +

Meghan and McPhee are longtime friends, having gone to school together and performed in musicals in Southern California
+10 +

+ Meghan and McPhee are longtime friends, having gone to school together and performed in musicals in Southern California +

Also in the star-studded neighborhood are Ellen DeGeneres, Oprah Winfrey, Rob Lowe and Gwyneth Paltrow
+10 +

+ Also in the star-studded neighborhood are Ellen DeGeneres, Oprah Winfrey, Rob Lowe and Gwyneth Paltrow +

+ Foster, who is more than 30-years older than his wife, Harry and Meghan, is often referred to as a father figure for Prince Harry. +

+ McPhee and Foster's baby will be her first child and his sixth. +

+ Foster has five daughters and has been married five times: to B.J. Cook, Rebecca Dyer, Linda Thompson, Yolanda Hadid and McPhee. +

+ The couple was married in London on June 28, 2019. +

+ Another selling point of Montecito for Foster is he's a long time friend of Oprah Winfrey. +

+ McPhee recently resumed filming on her upcoming Netflix series Country Comfort. +

+ Last month the actress shared a photo of a masked table read with the cast, captioning it, 'and we're back!'  +

+ David Foster & Katherine McPhee, house hunting in Montecito +
+ Loaded: 0% +
+ Progress: 0% +
+ 0:00 +
+10 +

+ Since the Sussexes moved to Montecito, music producer Foster and McPhee have spent a lot of time there and fallen in love with the area +

+ Another photo showed a monitor on set capturing her breaking into laughter with her costar Eddie Cibrian. +

+ The two were announced to star in the series in January, but production was halted in March due to the coronavirus pandemic. +

+ The Sussexes stepped down as senior working royals on March 31 and although they initially moved to Canada - living in a mansion Foster had set them up with for free -they have since set up home in California. +

+ The couple first lived in Tyler Perry's sprawling $18 million hilltop mansion in Beverly Hills before buying a home in Montecito. +

\ No newline at end of file diff --git a/packages/readabilityjs/test/test-pages/danluu/distiller.html b/packages/readabilityjs/test/test-pages/danluu/distiller.html new file mode 100644 index 000000000..8c5df3861 --- /dev/null +++ b/packages/readabilityjs/test/test-pages/danluu/distiller.html @@ -0,0 +1,592 @@ +

Abstract

+ Virtual reality (VR) is one of the most demanding human-in-the-loop + applications from a latency standpoint. The latency between the physical + movement of a user’s head and updated photons from a head mounted display + reaching their eyes is one of the most critical factors in providing a + high quality experience. +

+ Human sensory systems can detect very small relative delays in parts of + the visual or, especially, audio fields, but when absolute delays are + below approximately 20 milliseconds they are generally imperceptible. + Interactive 3D systems today typically have latencies that are several + times that figure, but alternate configurations of the same hardware + components can allow that target to be reached. +

+ A discussion of the sources of latency throughout a system follows, along + with techniques for reducing the latency in the processing done on the + host system. +

Introduction

+ Updating the imagery in a head mounted display (HMD) based on a head + tracking sensor is a subtly different challenge than most human / computer + interactions. With a conventional mouse or game controller, the user is + consciously manipulating an interface to complete a task, while the goal + of virtual reality is to have the experience accepted at an unconscious + level. +

+ Users can adapt to control systems with a significant amount of latency + and still perform challenging tasks or enjoy a game; many thousands of + people enjoyed playing early network games, even with 400+ milliseconds of + latency between pressing a key and seeing a response on screen. +

+ If large amounts of latency are present in the VR system, users may still + be able to perform tasks, but it will be by the much less rewarding means + of using their head as a controller, rather than accepting that their head + is naturally moving around in a stable virtual world. Perceiving latency + in the response to head motion is also one of the primary causes of + simulator sickness. Other technical factors that affect the quality of a + VR experience, like head tracking accuracy and precision, may interact + with the perception of latency, or, like display resolution and color + depth, be largely orthogonal to it. +

+ A total system latency of 50 milliseconds will feel responsive, but still + subtly lagging. One of the easiest ways to see the effects of latency in a + head mounted display is to roll your head side to side along the view + vector while looking at a clear vertical edge. Latency will show up as an + apparent tilting of the vertical line with the head motion; the view feels + “dragged along” with the head motion. When the latency is low enough, the + virtual world convincingly feels like you are simply rotating your view of + a stable world. +

+ Extrapolation of sensor data can be used to mitigate some system latency, + but even with a sophisticated model of the motion of the human head, there + will be artifacts as movements are initiated and changed. It is always + better to not have a problem than to mitigate it, so true latency + reduction should be aggressively pursued, leaving extrapolation to smooth + out sensor jitter issues and perform only a small amount of prediction. +

Data collection

+ It is not usually possible to introspectively measure the complete system + latency of a VR system, because the sensors and display devices external + to the host processor make significant contributions to the total latency. + An effective technique is to record high speed video that simultaneously + captures the initiating physical motion and the eventual display update. + The system latency can then be determined by single stepping the video and + counting the number of video frames between the two events. +

+ In most cases there will be a significant jitter in the resulting timings + due to aliasing between sensor rates, display rates, and camera rates, but + conventional applications tend to display total latencies in the dozens of + 240 fps video frames. +

+ On an unloaded Windows 7 system with the compositing Aero desktop + interface disabled, a gaming mouse dragging a window displayed on a 180 hz + CRT monitor can show a response on screen in the same 240 fps video frame + that the mouse was seen to first move, demonstrating an end to end latency + below four milliseconds. Many systems need to cooperate for this to + happen: The mouse updates 500 times a second, with no filtering or + buffering. The operating system immediately processes the update, and + immediately performs GPU accelerated rendering directly to the framebuffer + without any page flipping or buffering. The display accepts the video + signal with no buffering or processing, and the screen phosphors begin + emitting new photons within microseconds. +

+ In a typical VR system, many things go far less optimally, sometimes + resulting in end to end latencies of over 100 milliseconds. +

Sensors

+ Detecting a physical action can be as simple as a watching a circuit close + for a button press, or as complex as analyzing a live video feed to infer + position and orientation. +

+ In the old days, executing an IO port input instruction could directly + trigger an analog to digital conversion on an ISA bus adapter card, giving + a latency on the order of a microsecond and no sampling jitter issues. + Today, sensors are systems unto themselves, and may have internal + pipelines and queues that need to be traversed before the information is + even put on the USB serial bus to be transmitted to the host. +

+ Analog sensors have an inherent tension between random noise and sensor + bandwidth, and some combination of analog and digital filtering is usually + done on a signal before returning it. Sometimes this filtering is + excessive, which can contribute significant latency and remove subtle + motions completely. +

+ Communication bandwidth delay on older serial ports or wireless links can + be significant in some cases. If the sensor messages occupy the full + bandwidth of a communication channel, latency equal to the repeat time of + the sensor is added simply for transferring the message. Video data + streams can stress even modern wired links, which may encourage the use of + data compression, which usually adds another full frame of latency if not + explicitly implemented in a pipelined manner. +

+ Filtering and communication are constant delays, but the discretely + packetized nature of most sensor updates introduces a variable latency, or + “jitter” as the sensor data is used for a video frame rate that differs + from the sensor frame rate. This latency ranges from close to zero if the + sensor packet arrived just before it was queried, up to the repeat time + for sensor messages. Most USB HID devices update at 125 samples per + second, giving a jitter of up to 8 milliseconds, but it is possible to + receive 1000 updates a second from some USB hardware. The operating system + may impose an additional random delay of up to a couple milliseconds + between the arrival of a message and a user mode application getting the + chance to process it, even on an unloaded system. +

Displays

+ On old CRT displays, the voltage coming out of the video card directly + modulated the voltage of the electron gun, which caused the screen + phosphors to begin emitting photons a few microseconds after a pixel was + read from the frame buffer memory. +

+ Early LCDs were notorious for “ghosting” during scrolling or animation, + still showing traces of old images many tens of milliseconds after the + image was changed, but significant progress has been made in the last two + decades. The transition times for LCD pixels vary based on the start and + end values being transitioned between, but a good panel today will have a + switching time around ten milliseconds, and optimized displays for active + 3D and gaming can have switching times less than half that. +

+ Modern displays are also expected to perform a wide variety of processing + on the incoming signal before they change the actual display elements. A + typical Full HD display today will accept 720p or interlaced composite + signals and convert them to the 1920×1080 physical pixels. 24 fps movie + footage will be converted to 60 fps refresh rates. Stereoscopic input may + be converted from side-by-side, top-down, or other formats to frame + sequential for active displays, or interlaced for passive displays. + Content protection may be applied. Many consumer oriented displays have + started applying motion interpolation and other sophisticated algorithms + that require multiple frames of buffering. +

+ Some of these processing tasks could be handled by only buffering a single + scan line, but some of them fundamentally need one or more full frames of + buffering, and display vendors have tended to implement the general case + without optimizing for the cases that could be done with low or no delay. + Some consumer displays wind up buffering three or more frames internally, + resulting in 50 milliseconds of latency even when the input data could + have been fed directly into the display matrix. +

+ Some less common display technologies have speed advantages over LCD + panels; OLED pixels can have switching times well under a millisecond, and + laser displays are as instantaneous as CRTs. +

+ A subtle latency point is that most displays present an image + incrementally as it is scanned out from the computer, which has the effect + that the bottom of the screen changes 16 milliseconds later than the top + of the screen on a 60 fps display. This is rarely a problem on a static + display, but on a head mounted display it can cause the world to appear to + shear left and right, or “waggle” as the head is rotated, because the + source image was generated for an instant in time, but different parts are + presented at different times. This effect is usually masked by switching + times on LCD HMDs, but it is obvious with fast OLED HMDs. +

Host processing

The classic processing model for a game or VR application is:

Read user input -> run simulation -> issue rendering commands -> graphics drawing -> wait for vsync -> scanout
+
+    I = Input sampling and dependent calculation
+    S = simulation / game execution
+    R = rendering engine
+    G = GPU drawing time
+    V = video scanout time
+

+ All latencies are based on a frame time of roughly 16 milliseconds, a + progressively scanned display, and zero sensor and pixel latency. +

+ If the performance demands of the application are well below what the + system can provide, a straightforward implementation with no parallel + overlap will usually provide fairly good latency values. However, if + running synchronized to the video refresh, the minimum latency will still + be 16 ms even if the system is infinitely fast. This rate feels good for + most eye-hand tasks, but it is still a perceptible lag that can be felt in + a head mounted display, or in the responsiveness of a mouse cursor. + 

Ample performance, vsync:
+    ISRG------------|VVVVVVVVVVVVVVVV|
+    .................. latency 16 – 32 milliseconds
+

+ Running without vsync on a very fast system will deliver better latency, + but only over a fraction of the screen, and with visible tear lines. The + impact of the tear lines are related to the disparity between the two + frames that are being torn between, and the amount of time that the tear + lines are visible. Tear lines look worse on a continuously illuminated LCD + than on a CRT or laser projector, and worse on a 60 fps display than a 120 + fps display. Somewhat counteracting that, slow switching LCD panels blur + the impact of the tear line relative to the faster displays. +

+ If enough frames were rendered such that each scan line had a unique + image, the effect would be of a “rolling shutter”, rather than visible + tear lines, and the image would feel continuous. Unfortunately, even + rendering 1000 frames a second, giving approximately 15 bands on screen + separated by tear lines, is still quite objectionable on fast switching + displays, and few scenes are capable of being rendered at that rate, let + alone 60x higher for a true rolling shutter on a 1080P display. + 

Ample performance, unsynchronized:
+    ISRG
+    VVVVV
+    ..... latency 5 – 8 milliseconds at ~200 frames per second
+

+ In most cases, performance is a constant point of concern, and a parallel + pipelined architecture is adopted to allow multiple processors to work in + parallel instead of sequentially. Large command buffers on GPUs can buffer + an entire frame of drawing commands, which allows them to overlap the work + on the CPU, which generally gives a significant frame rate boost at the + expense of added latency. + 

CPU:ISSSSSRRRRRR----|
+    GPU:                |GGGGGGGGGGG----|
+    VID:                |               |VVVVVVVVVVVVVVVV|
+        .................................. latency 32 – 48 milliseconds
+

+ When the CPU load for the simulation and rendering no longer fit in a + single frame, multiple CPU cores can be used in parallel to produce more + frames. It is possible to reduce frame execution time without increasing + latency in some cases, but the natural split of simulation and rendering + has often been used to allow effective pipeline parallel operation. Work + queue approaches buffered for maximum overlap can cause an additional + frame of latency if they are on the critical user responsiveness path. + 

CPU1:ISSSSSSSS-------|
+    CPU2:                |RRRRRRRRR-------|
+    GPU :                |                |GGGGGGGGGG------|
+    VID :                |                |                |VVVVVVVVVVVVVVVV|
+         .................................................... latency 48 – 64 milliseconds
+

+ Even if an application is running at a perfectly smooth 60 fps, it can + still have host latencies of over 50 milliseconds, and an application + targeting 30 fps could have twice that. Sensor and display latencies can + add significant additional amounts on top of that, so the goal of 20 + milliseconds motion-to-photons latency is challenging to achieve. +

Latency Reduction Strategies

Prevent GPU buffering

+ The drive to win frame rate benchmark wars has led driver writers to + aggressively buffer drawing commands, and there have even been cases where + drivers ignored explicit calls to glFinish() in the name of improved + “performance”. Today’s fence primitives do appear to be reliably observed + for drawing primitives, but the semantics of buffer swaps are still + worryingly imprecise. A recommended sequence of commands to synchronize + with the vertical retrace and idle the GPU is: + 

SwapBuffers();
+    DrawTinyPrimitive();
+    InsertGPUFence();
+    BlockUntilFenceIsReached();
+

+ While this should always prevent excessive command buffering on any + conformant driver, it could conceivably fail to provide an accurate + vertical sync timing point if the driver was transparently implementing + triple buffering. +

+ To minimize the performance impact of synchronizing with the GPU, it is + important to have sufficient work ready to send to the GPU immediately + after the synchronization is performed. The details of exactly when the + GPU can begin executing commands are platform specific, but execution can + be explicitly kicked off with glFlush() or equivalent calls. If the code + issuing drawing commands does not proceed fast enough, the GPU may + complete all the work and go idle with a “pipeline bubble”. Because the + CPU time to issue a drawing command may have little relation to the GPU + time required to draw it, these pipeline bubbles may cause the GPU to take + noticeably longer to draw the frame than if it were completely buffered. + Ordering the drawing so that larger and slower operations happen first + will provide a cushion, as will pushing as much preparatory work as + possible before the synchronization point. + 

Run GPU with minimal buffering:
+    CPU1:ISSSSSSSS-------|
+    CPU2:                |RRRRRRRRR-------|
+    GPU :                |-GGGGGGGGGG-----|
+    VID :                |                |VVVVVVVVVVVVVVVV|
+         ................................... latency 32 – 48 milliseconds
+

+ Tile based renderers, as are found in most mobile devices, inherently + require a full scene of command buffering before they can generate their + first tile of pixels, so synchronizing before issuing any commands will + destroy far more overlap. In a modern rendering engine there may be + multiple scene renders for each frame to handle shadows, reflections, and + other effects, but increased latency is still a fundamental drawback of + the technology. +

+ High end, multiple GPU systems today are usually configured for AFR, or + Alternate Frame Rendering, where each GPU is allowed to take twice as long + to render a single frame, but the overall frame rate is maintained because + there are two GPUs producing frames + 

Alternate Frame Rendering dual GPU:
+    CPU1:IOSSSSSSS-------|IOSSSSSSS-------|
+    CPU2:                |RRRRRRRRR-------|RRRRRRRRR-------|
+    GPU1:                | GGGGGGGGGGGGGGGGGGGGGGGG--------|
+    GPU2:                |                | GGGGGGGGGGGGGGGGGGGGGGG---------|
+    VID :                |                |                |VVVVVVVVVVVVVVVV|
+         .................................................... latency 48 – 64 milliseconds
+

+ Similarly to the case with CPU workloads, it is possible to have two or + more GPUs cooperate on a single frame in a way that delivers more work in + a constant amount of time, but it increases complexity and generally + delivers a lower total speedup. +

+ An attractive direction for stereoscopic rendering is to have each GPU on + a dual GPU system render one eye, which would deliver maximum performance + and minimum latency, at the expense of requiring the application to + maintain buffers across two independent rendering contexts. +

+ The downside to preventing GPU buffering is that throughput performance + may drop, resulting in more dropped frames under heavily loaded + conditions. +

Late frame scheduling

+ Much of the work in the simulation task does not depend directly on the + user input, or would be insensitive to a frame of latency in it. If the + user processing is done last, and the input is sampled just before it is + needed, rather than stored off at the beginning of the frame, the total + latency can be reduced. +

+ It is very difficult to predict the time required for the general + simulation work on the entire world, but the work just for the player’s + view response to the sensor input can be made essentially deterministic. + If this is split off from the main simulation task and delayed until + shortly before the end of the frame, it can remove nearly a full frame of + latency. + 

Late frame scheduling:
+    CPU1:SSSSSSSSS------I|
+    CPU2:                |RRRRRRRRR-------|
+    GPU :                |-GGGGGGGGGG-----|
+    VID :                |                |VVVVVVVVVVVVVVVV|
+                        .................... latency 18 – 34 milliseconds
+

+ Adjusting the view is the most latency sensitive task; actions resulting + from other user commands, like animating a weapon or interacting with + other objects in the world, are generally insensitive to an additional + frame of latency, and can be handled in the general simulation task the + following frame. +

+ The drawback to late frame scheduling is that it introduces a tight + scheduling requirement that usually requires busy waiting to meet, wasting + power. If your frame rate is determined by the video retrace rather than + an arbitrary time slice, assistance from the graphics driver in accurately + determining the current scanout position is helpful. +

View bypass

+ An alternate way of accomplishing a similar, or slightly greater latency + reduction Is to allow the rendering code to modify the parameters + delivered to it by the game code, based on a newer sampling of user input. +

+ At the simplest level, the user input can be used to calculate a delta + from the previous sampling to the current one, which can be used to modify + the view matrix that the game submitted to the rendering code. +

+ Delta processing in this way is minimally intrusive, but there will often + be situations where the user input should not affect the rendering, such + as cinematic cut scenes or when the player has died. It can be argued that + a game designed from scratch for virtual reality should avoid those + situations, because a non-responsive view in a HMD is disorienting and + unpleasant, but conventional game design has many such cases. +

+ A binary flag could be provided to disable the bypass calculation, but it + is useful to generalize such that the game provides an object or function + with embedded state that produces rendering parameters from sensor input + data instead of having the game provide the view parameters themselves. In + addition to handling the trivial case of ignoring sensor input, the + generator function can incorporate additional information such as a + head/neck positioning model that modified position based on orientation, + or lists of other models to be positioned relative to the updated view. +

+ If the game and rendering code are running in parallel, it is important + that the parameter generation function does not reference any game state + to avoid race conditions. + 

View bypass:
+    CPU1:ISSSSSSSSS------|
+    CPU2:                |IRRRRRRRRR------|
+    GPU :                |--GGGGGGGGGG----|
+    VID :                |                |VVVVVVVVVVVVVVVV|
+                          .................. latency 16 – 32 milliseconds
+

+ The input is only sampled once per frame, but it is simultaneously used by + both the simulation task and the rendering task. Some input processing + work is now duplicated by the simulation task and the render task, but it + is generally minimal. +

+ The latency for parameters produced by the generator function is now + reduced, but other interactions with the world, like muzzle flashes and + physics responses, remain at the same latency as the standard model. +

+ A modified form of view bypass could allow tile based GPUs to achieve + similar view latencies to non-tiled GPUs, or allow non-tiled GPUs to + achieve 100% utilization without pipeline bubbles by the following steps: +

+ Inhibit the execution of GPU commands, forcing them to be buffered. OpenGL + has only the deprecated display list functionality to approximate this, + but a control extension could be formulated. +

+ All calculations that depend on the view matrix must reference it + independently from a buffer object, rather than from inline parameters or + as a composite model-view-projection (MVP) matrix. +

+ After all commands have been issued and the next frame has started, sample + the user input, run it through the parameter generator, and put the + resulting view matrix into the buffer object for referencing by the draw + commands. +

Kick off the draw command execution.

Tiler optimized view bypass:
+    CPU1:ISSSSSSSSS------|
+    CPU2:                |IRRRRRRRRRR-----|I
+    GPU :                |                |-GGGGGGGGGG-----|
+    VID :                |                |                |VVVVVVVVVVVVVVVV|
+                                           .................. latency 16 – 32 milliseconds
+

+ Any view frustum culling that was performed to avoid drawing some models + may be invalid if the new view matrix has changed substantially enough + from what was used during the rendering task. This can be mitigated at + some performance cost by using a larger frustum field of view for culling, + and hardware clip planes based on the culling frustum limits can be used + to guarantee a clean edge if necessary. Occlusion errors from culling, + where a bright object is seen that should have been occluded by an object + that was incorrectly culled, are very distracting, but a temporary clean + encroaching of black at a screen edge during rapid rotation is almost + unnoticeable. +

Time warping

+ If you had perfect knowledge of how long the rendering of a frame would + take, some additional amount of latency could be saved by late frame + scheduling the entire rendering task, but this is not practical due to the + wide variability in frame rendering times. + 

Late frame input sampled view bypass:
+    CPU1:ISSSSSSSSS------|
+    CPU2:                |----IRRRRRRRRR--|
+    GPU :                |------GGGGGGGGGG|
+    VID :                |                |VVVVVVVVVVVVVVVV|
+                              .............. latency 12 – 28 milliseconds
+

+ However, a post processing task on the rendered image can be counted on to + complete in a fairly predictable amount of time, and can be late scheduled + more easily. Any pixel on the screen, along with the associated depth + buffer value, can be converted back to a world space position, which can + be re-transformed to a different screen space pixel location for a + modified set of view parameters. +

+ After drawing a frame with the best information at your disposal, possibly + with bypassed view parameters, instead of displaying it directly, fetch + the latest user input, generate updated view parameters, and calculate a + transformation that warps the rendered image into a position that + approximates where it would be with the updated parameters. Using that + transform, warp the rendered image into an updated form on screen that + reflects the new input. If there are two dimensional overlays present on + the screen that need to remain fixed, they must be drawn or composited in + after the warp operation, to prevent them from incorrectly moving as the + view parameters change. + 

Late frame scheduled time warp:
+    CPU1:ISSSSSSSSS------|
+    CPU2:                |RRRRRRRRRR----IR|
+    GPU :                |-GGGGGGGGGG----G|
+    VID :                |                |VVVVVVVVVVVVVVVV|
+                                        .... latency 2 – 18 milliseconds
+

+ If the difference between the view parameters at the time of the scene + rendering and the time of the final warp is only a change in direction, + the warped image can be almost exactly correct within the limits of the + image filtering. Effects that are calculated relative to the screen, like + depth based fog (versus distance based fog) and billboard sprites will be + slightly different, but not in a manner that is objectionable. +

+ If the warp involves translation as well as direction changes, geometric + silhouette edges begin to introduce artifacts where internal parallax + would have revealed surfaces not visible in the original rendering. A + scene with no silhouette edges, like the inside of a box, can be warped + significant amounts and display only changes in texture density, but + translation warping realistic scenes will result in smears or gaps along + edges. In many cases these are difficult to notice, and they always + disappear when motion stops, but first person view hands and weapons are a + prominent case. This can be mitigated by limiting the amount of + translation warp, compressing or making constant the depth range of the + scene being warped to limit the dynamic separation, or rendering the + disconnected near field objects as a separate plane, to be composited in + after the warp. +

+ If an image is being warped to a destination with the same field of view, + most warps will leave some corners or edges of the new image undefined, + because none of the source pixels are warped to their locations. This can + be mitigated by rendering a larger field of view than the destination + requires; but simply leaving unrendered pixels black is surprisingly + unobtrusive, especially in a wide field of view HMD. +

+ A forward warp, where source pixels are deposited in their new positions, + offers the best accuracy for arbitrary transformations. At the limit, the + frame buffer and depth buffer could be treated as a height field, but + millions of half pixel sized triangles would have a severe performance + cost. Using a grid of triangles at some fraction of the depth buffer + resolution can bring the cost down to a very low level, and the trivial + case of treating the rendered image as a single quad avoids all silhouette + artifacts at the expense of incorrect pixel positions under translation. +

+ Reverse warping, where the pixel in the source rendering is estimated + based on the position in the warped image, can be more convenient because + it is implemented completely in a fragment shader. It can produce + identical results for simple direction changes, but additional artifacts + near geometric boundaries are introduced if per-pixel depth information is + considered, unless considerable effort is expended to search a + neighborhood for the best source pixel. +

+ If desired, it is straightforward to incorporate motion blur in a reverse + mapping by taking several samples along the line from the pixel being + warped to the transformed position in the source image. +

+ Reverse mapping also allows the possibility of modifying the warp through + the video scanout. The view parameters can be predicted ahead in time to + when the scanout will read the bottom row of pixels, which can be used to + generate a second warp matrix. The warp to be applied can be interpolated + between the two of them based on the pixel row being processed. This can + correct for the “waggle” effect on a progressively scanned head mounted + display, where the 16 millisecond difference in time between the display + showing the top line and bottom line results in a perceived shearing of + the world under rapid rotation on fast switching displays. +

+ Continuously updated time warping +

+ If the necessary feedback and scheduling mechanisms are available, instead + of predicting what the warp transformation should be at the bottom of the + frame and warping the entire screen at once, the warp to screen can be + done incrementally while continuously updating the warp matrix as new + input arrives. + 

Continuous time warp:
+    CPU1:ISSSSSSSSS------|
+    CPU2:                |RRRRRRRRRRR-----|
+    GPU :                |-GGGGGGGGGGGG---|
+    WARP:                |               W| W W W W W W W W|
+    VID :                |                |VVVVVVVVVVVVVVVV|
+                                         ... latency 2 – 3 milliseconds for 500hz sensor updates
+

+ The ideal interface for doing this would be some form of “scanout shader” + that would be called “just in time” for the video display. Several video + game systems like the Atari 2600, Jaguar, and Nintendo DS have had buffers + ranging from half a scan line to several scan lines that were filled up in + this manner. +

+ Without new hardware support, it is still possible to incrementally + perform the warping directly to the front buffer being scanned for video, + and not perform a swap buffers operation at all. +

+ A CPU core could be dedicated to the task of warping scan lines at roughly + the speed they are consumed by the video output, updating the time warp + matrix each scan line to blend in the most recently arrived sensor + information. +

+ GPUs can perform the time warping operation much more efficiently than a + conventional CPU can, but the GPU will be busy drawing the next frame + during video scanout, and GPU drawing operations cannot currently be + scheduled with high precision due to the difficulty of task switching the + deep pipelines and extensive context state. However, modern GPUs are + beginning to allow compute tasks to run in parallel with graphics + operations, which may allow a fraction of a GPU to be dedicated to + performing the warp operations as a shared parameter buffer is updated by + the CPU. +

Discussion

+ View bypass and time warping are complementary techniques that can be + applied independently or together. Time warping can warp from a source + image at an arbitrary view time / location to any other one, but artifacts + from internal parallax and screen edge clamping are reduced by using the + most recent source image possible, which view bypass rendering helps + provide. +

+ Actions that require simulation state changes, like flipping a switch or + firing a weapon, still need to go through the full pipeline for 32 – 48 + milliseconds of latency based on what scan line the result winds up + displaying on the screen, and translational information may not be + completely faithfully represented below the 16 – 32 milliseconds of the + view bypass rendering, but the critical head orientation feedback can be + provided in 2 – 18 milliseconds on a 60 hz display. In conjunction with + low latency sensors and displays, this will generally be perceived as + immediate. Continuous time warping opens up the possibility of latencies + below 3 milliseconds, which may cross largely unexplored thresholds in + human / computer interactivity. +

+ Conventional computer interfaces are generally not as latency demanding as + virtual reality, but sensitive users can tell the difference in mouse + response down to the same 20 milliseconds or so, making it worthwhile to + apply these techniques even in applications without a VR focus. +

+ A particularly interesting application is in “cloud gaming”, where a + simple client appliance or application forwards control information to a + remote server, which streams back real time video of the game. This offers + significant convenience benefits for users, but the inherent network and + compression latencies makes it a lower quality experience for action + oriented titles. View bypass and time warping can both be performed on the + server, regaining a substantial fraction of the latency imposed by the + network. If the cloud gaming client was made more sophisticated, time + warping could be performed locally, which could theoretically reduce the + latency to the same levels as local applications, but it would probably be + prudent to restrict the total amount of time warping to perhaps 30 or 40 + milliseconds to limit the distance from the source images. +

Acknowledgements

Zenimax for allowing me to publish this openly.

Hillcrest Labs for inertial sensors and experimental firmware.

Emagin for access to OLED displays.

Oculus for a prototype Rift HMD.

+ Nvidia for an experimental driver with access to the current scan line + number. +

\ No newline at end of file diff --git a/packages/readabilityjs/test/test-pages/danwang/distiller.html b/packages/readabilityjs/test/test-pages/danwang/distiller.html new file mode 100644 index 000000000..482f4b6f7 --- /dev/null +++ b/packages/readabilityjs/test/test-pages/danwang/distiller.html @@ -0,0 +1,264 @@ +

+ (This piece is my year in review; here’s my letter from 2019) +

+ I. Inspiration +

+ It’s difficult to identify a great economic reason to explore space. There are easier ways to extract minerals, doing anything at all is terribly expensive, and Mars is a hard place to make a living. The benefits of space exploration are instead mostly inspirational. Few other human activities are so grand to captivate the imagination, and doing these uneconomic projects have pulled forward technological capabilities that may otherwise have languished. +

+ It’s difficult to identify a great economic reason to practice socialism. Its historical results have ranged from catastrophic misallocation of talent at best to mass deaths at worst. But socialism still retains appeal to broad segments of many populations, which shows that it has considerable inspirational value. For better or for worse, there are still many advocates for the creation of some form of a more equal society. +

+ This year, I read every issue of Qiushi (translation: Seeking Truth), the party’s flagship theory journal, whose core task is to spell out the evolving idea of socialism with Chinese characteristics. For those not familiar, Qiushi reads like a cross between the New Yorker and the Federal Register. Published twice a month, the magazine features lengthy essays, thick pages, and some of the finest writers in the party. Each issue starts in the same way: a reprint of a speech or essay by Xi Jinping—in a font distinct from the rest of the magazine’s—and then commentary and reports from the rest of the party state. Accompanying pictures feature either the country’s leaders making inspections, scenes of the people, or major pieces of infrastructure and heavy industry. +

+ Its audience? People with nothing better to do than read the party center’s commentary (like retired cadres), or those who are keenly interested in Beijing’s priorities, like local officials. Reading party speeches with its various annexes and cross references echoes my main professional activity these days. That is the study of the US sanctions regime—namely Commerce’s Export Administration Regulations and Treasury’s IEEPA-based authorities. Party speeches and US regulations are both made up of arcane, formal language that make references to more obscure texts, which themselves hint at still more distant and terrible truths. US sanctions lawyers, I suspect, can have a splendid time with Qiushi. +

+ Steady engagement with the journal throughout the year has forced me to think more deeply about the Chinese Communist Party. There are many things that Xi wants to do, I believe that his most fundamental goal is to make this Marxist-Leninist party an effective governing force for the present century. His patient work to reshape the bureaucracy is aided by a distinctive feature of the Chinese system: the use of propaganda to create centralized campaigns of inspiration. Some of Xi’s efforts have borne fruit: the country’s governance capabilities have markedly improved, a trend that is observable in daily life. At the same time, the state has grown much more repressive. A focus on repression shouldn’t neglect the improvement in the country’s institutional and commercial strengths; and appreciation of this improvement ought to be tempered by the party center’s growing mania for control. +

+ When foreign commentators discuss the experience of reading state media, they rarely fail to attach a reference to its “turgid prose.” While some partyspeak is indeed unreadable, I’ve always seen that dismissal as a signal of contempt for the party’s pronouncements, thus deterring people from taking it seriously. But there is reason to treat its content with care. Propaganda might not matter to you, but it matters to the party. Anne-Marie Brady has pointed out +

that the leadership considers propaganda to be the “lifeblood” of the party state. Propaganda work is considered so powerful that the person in charge must be only a functionary. Brady shows that the head of propaganda always has a seat on the Politburo, but shouldn’t usually be allowed to reach the standing committee. He is not to be too imaginative, or he might dominate the entire political system. Propaganda is key to understanding the party, since it governs not in itself, but in symbiosis with state institutions. For the most part, the party’s role can be boiled down to two items: inspiration, by setting the ideological direction, and control, through its power to select personnel. +

+ Qiushi offers an authoritative articulation of the central government’s priorities at any moment. Its job, like the rest of the state media, consists of repetition and explication of a few phrases. It’s easy to roll one’s eyes at crude sloganeering, like the two centenary goals of achieving a “moderately prosperous society in all respects” by 2021 and “a modern socialist country and the great rejuvenation of the Chinese people” by 2049. But the need to fix slogans makes good sense in Chinese governance: the party center has to speak to all local officials as well as the entire population. As Richard Epstein has argued, the greater the complexity in a system, the simpler the rules that govern it must be. One should allow, for example, extensive and nuanced bargaining between buyer and seller at the vegetable stall, but for an online marketplace to manage millions of transactions a day, then its rules must be very simple indeed. +

+ It’s up to the party center to adjust and refine slogans to signal the priorities of the moment. The easiest way to appreciate the importance of that effort is to consider Deng Xiaoping’s efforts to shift the country to pragmatic governance through Reform and Opening (itself a named initiative). He invented or instrumentalized a series of ingenious phrases that include “development is the only hard truth,” “cross the river by feeling the stones,” and “practice is the sole criterion for the determination of truth.” My favorite is his declaration at the 13th party congress that China is in the “primary stage of socialism.” Left unspecified is how long this stage will last and how many more stages there will be before the people can enjoy the full deal.  +

+ Many of the party center’s slogans tend to be deliberately vague, thus permitting lower-level officials to figure out implementation. My one-sentence definition of Xi Jinping Thought on Socialism with Chinese Characteristics for a New Era is: “To achieve the two centenary goals—under the leadership of the party—by accomplishing tasks that include but are not limited to eliminating poverty, advancing the socialist rule of law, improving party discipline, etc.” That concept is still an evolving one. Meanwhile, we await better explication of the hot slogans this year: “dual circulation” and “demand-side reform.” Some of it can be absurd: I’m skeptical that anyone can readily explain the nuances between “rule of law,” “socialist rule of law,” and “socialist rule of law with Chinese characteristics.” +

+ When it’s not being vague, the party can be trying to have things both ways. Xi declared at the third plenum in 2013 that market forces would have a “decisive” role in allocating resources, while at the same time the state sector would have a “leading” role. It’s not unusual to see a great deal of semantic acrobatics. Deng declared that socialism means the capacity to concentrate resources to accomplish great tasks; under that definition, the Apollo and Manhattan projects were socialism. In July, Xi reminded us that “socialism with Chinese characteristics has many distinctive features, but its most essential is leadership by the Chinese Communist Party.” +

In other words, socialism with Chinese characteristics means the party is never wrong. Either the market or the state sector can be more important at any moment: it is the party’s pleasure to decide. +

+ Centralized campaigns of inspiration, which usually manifests through fixing slogans, is a distinctive feature of the Chinese political system. In the US, political candidates trot out slogans when they run for election; in China, one is never far from the next big named initiative. At its best, defining major goals is the essence of political leadership, and nowhere is this principle better illustrated than Apollo. John F. Kennedy announced the target in 1961: land a man on the moon and return him safely to earth before the decade was out. By fixing this clear goal, +

as well as committing the necessary spending, he accelerated the creation, development, and deployment of technologies that made the lunar landings possible.  +

+ Xi grasps this idea of leadership. In his tenure, he has unleashed a torrent of new initiatives. In my view, he feels that the practice of governing China under socialism cannot be an exercise in sustained mendacity. The political system can no longer continue to be an unstable structure based on ad hoc compromises; instead it must have a clear organizational structure, with the party at the top. And the ruling party needs to have the political consciousness of an effective governing force. +

+ Consider two of his most important initiatives: the campaign against corruption and the move toward law-based governance. Xi has decided that corruption is not a mystery to be endured, but a problem to be solved. A few years past the peak of the crackdown, it’s fair to say that the campaign hasn’t solely been effective in removing his adversaries, but has also been broad enough to restore some degree of public confidence in government. A few commentators contend that removal of opportunities for graft have prompted talented people to leave government. But the flip side of that coin has been the improvement in morale among the civil servants who found corruption among colleagues to be intolerable, and can finally see themselves doing public work well.  +

+ And for years, Xi has emphasized following clear rules of written procedure, under the rubric of “law-based governance.” +

Since then, the state has improved regulatory systems, for example in setting clear standards for license approvals and in securities and antitrust regulation. The state has removed some of the arbitrary aspects of governance, thus bringing serious enforcement actions following the passage of relatively clear regulations. That has improved facts on the ground. Companies and lawyers tell me that a decade-long effort by the State Council to ease doing business has yielded real results. Obtaining business licenses no longer requires a relentless pace of wining and dining, and has instead become close to a matter of routine. I haven’t been able to verify this fact for myself, but one of my friends told me that the office of the National Development and Reform Commission used to be ringed by some of the fanciest restaurants in Beijing, offering mostly private rooms; many of these restaurants have now closed, following the professionalization of business approvals. +

+ The lived experience of being in Beijing has improved in parallel. I remember what a nightmare it was to buy a high-speed rail ticket for the first time years ago, which involved lots of yelling and multiple people cutting in line. Today, I purchase one on my phone, with no need to obtain a paper ticket, and the lines to board are more or less orderly. Consumer products of all sorts are getting pretty good, and the customer service experience of engaging with any of these companies tends to be not unpleasant. There’s certainly still all sorts of disorderly behavior, but anyone can notice the improvement on all fronts of daily life. In more macro view, some of the breathless stories from years ago on China’s growing capabilities look at last like they’re on good ground today: the country has produced credible companies, have many investable assets (especially in fixed income), and are building globally-competitive brands. +

+ It’s easy to enumerate the grave problems facing the country, but critics tend to under-appreciate its strengths. Chief among them, in my view, has been the party’s surprising adaptability. At any given point, commentators have said that the problems have become too big for the government to handle. Meanwhile, the country has achieved a good record of pulling itself out of sticky situations: in 1992 when it restarted reform, after the financial crisis of 1997, and again in 2008. That record was validated most spectacularly again this year in the aftermath of the Covid-19 outbreak. +

+ This year made me believe that China is the country with the most can-do spirit in the world. Every segment of society mobilized to contain the pandemic. One manufacturer expressed astonishment to me at how slowly western counterparts moved. US companies had to ask whether making masks aligned with the company’s core competence. Chinese companies simply decided that making money is their core competence, and therefore they should be making masks. The State Council reported that between March and May, China exported 70 billion masks and nearly 100,000 ventilators. +

Some of these masks had problems early on, but the manufacturers learned and fixed them or were culled by regulatory action, and China’s exports were able to grow when no one else could restart production. Soon enough, masks were big enough to be seen in the export data. +

+ It’s obvious that the authorities in Wuhan screwed up big, but it’s also the case that the central government organized an effective response to virus containment. It’s not just the manufacturers: the consumer internet companies leapt into action in a way that their US peers did not. +

Francis Fukuyama states that high-trust societies have “spontaneous sociability,” in which people are able to organize more quickly, initiate action, and sacrifice for the common good. On each of these metrics, I submit that China should receive high marks. +

+ As every discussion on China grows more strident, and as every proposition about it has to be vested with sentiment, I submit that it’s all the more important to be able to see things as they are. That entails having coming to terms not just with a rise of its repressive capabilities, but also with its growing commercial and institutional strengths. US elites have abandoned the idea that China would liberalize nicely. They should put another idea to bed: that this authoritarian system, riddled with weaknesses, is on the brink of collapse. The country’s strengths are real and improving while the government becomes more nasty towards its critics and the rest of the world. +

+ China is neither a Marxist fundamentalist regime nor a universally-surveilled open-air prison, in which one is free to do nothing but worship the party and carry out its edicts. That is however the impression created by quite a bit of the media. +

I think that’s not the fault of individual journalists, instead more structural explanations are at work. News bureaus are highly concentrated in Beijing, due in part to natural corporate consolidation, but mostly because the government maintains a strict cap on foreign journalist visas. As a result, the bulk of journalists are based in the part of China that has the most politics and the least sense of growth. Everything here is doom and gloom, a fact well conveyed to the outside world. What’s missing are the facts of more pleasant life and higher growth in other cities. In an ideal world, it should not be crazy to imagine that the papers should have correspondents based in places that include places like Chongqing, Hangzhou, or Xiamen, all of which have interesting stories to tell. +

+ Xi is preparing to face a more challenging international environment with a raft of initiatives. He has consistently said over the last few years that “we must handle our own affairs well.” That has meant building up domestic capabilities while not lashing out against American firms. He has also invoked history to strengthen morale in the party. 2021 is the centenary of the party’s founding, and the major slogan of the past two years has been: “Remember where we started from, pursue our destiny, the struggle is forever.” +

+ Given the importance of the slogan, it’s worthwhile to try to come to terms with the fondness and reverence his generation has for the party’s early days. Many of the people tormented by the party center, including Deng and Xi’s father, have ended up being fiercely loyal to the party. +

That shows not just that human nature is complex, but also that the revolutionary heritage of the party instills pride. The CCP started out as a combat party constantly at the mercy of forces grander than itself, achieving its goals after an unusually long struggle that repeatedly brought it to the brink of death. Daniel Koss reminds us that the longer that revolutionary parties have to struggle before consolidating power, the more stronger their ideological commitments and the greater their governance durability tend to be. +

+ Xi is keen to reflect upon the regime’s history. He has decided that the party must believe in itself, and that it is correct to do so: “If our Party members and officials are firm in their ideals and convictions and maintain high morale in their activities and initiatives, and if our people are high-spirited and determined, then we will surely create many miracles.” +

Furthermore, he has stated: “The prospects are bright but the challenges are severe. All comrades must aim high and look far, be alert to dangers even in times of calm, have the courage to pursue reform and break new ground, and never become hardened to change.”  +

+ Thus I’ve arrived at the idea that a commitment to centralized campaigns of inspiration, represented by the tendency to fix clear goals, is the booster stage required to leave the gravitational pull of decadence and complacency. Ross Douthat laments that “a consistent ineffectuality in American governance is just the way things are.” +

And he references Jacques Barzun, who defines a decadent society as one that is “peculiarly restless, for it sees no clear lines of advance.” As a society turns developed, its main problems become social: an organizational sclerosis, which no technology is sophisticated enough to solve. No great effort is required to identify the comprehensive paralysis in the US. And that is the political and social current that Xi is trying to reverse in China. +

+ One way to do that is to continue to pursue GDP growth, which has mostly become an unfashionable idea today in the west. Xi reminded the state in July that “economic work must be our core task, if we succeed in that, then the rest of our tasks become easy.” +

Barry Naughton has noted that “China’s system of incentives for local bureaucrats to encourage growth is extremely unusual, and seems only to exist in China. It is a blunt and powerful instrument.”  +

+ This emphasis on growth makes it less likely for China to develop into American complacency or decadence. There are other types of paralysis that it stands a good chance of avoiding. With its emphasis on the real economy, it is trying to avoid the fate of Hong Kong, where local elites have reorganized the productive forces completely around sustaining high property prices and managing mainland liquidity flows. With its emphasis on economic growth, it cannot be like Taiwan, whose single bright corporate beacon is surrounded by a mass of firms undergoing genteel decline. With its emphasis on manufacturing, it cannot be like the UK, which is so successful in the sounding-clever industries—television, journalism, finance, and universities—while seeing a falling share of R&D intensity and a global loss of standing among its largest firms. +

+ Douthat’s book does not deal seriously with China, only with a fantasy of a universally-surveilled society under the rubric of a social credit system. If he did engage more seriously, he might pick up what Frank Pieke has termed “neo-socialism,” which is the attempt to harness market liberalization to strengthen state capacity and a more Leninist party. +

In return, the state provides purpose and direction, as well as inspiring the rest of society with a transformative mission. It helps, of course, that Xi is a genuine believer in socialism, which to him is both an instrument as well as an end. He’s leveraging that belief to reject decadence and assert agency to point out new lines of advance. +

+ II. Control +

+ That was quite a lot of theory. Where does it fall apart? +

+ Xi has said: “If we turn a blind eye to challenges, or even dodge or disguise them; if we fear to advance in the face of challenges and sit by and watch the unfolding calamity; then they will grow beyond our control and cause irreparable damage.” +

Instead of heeding this warning, authorities in Wuhan suppressed reporting of a spread of a novel virus. At a time when they should have imposed restrictions, they congregated thousands around a gigantic potluck. That has indeed unfolded into a calamity. +

+ Xi has said: “Some officials are perfunctory in their work, shirking responsibility when troubles come and dodging thorny problems. They like to report every trifle to their superiors for approval or directives. In doing so, they appear to be abiding by the rules but are actually avoiding responsibilities. Some make ill-considered or purely arbitrary decisions. They place themselves above the party organization and allow no dissenting voices.” +

 But as economic growth slows down, the country is doubling down on centralized government. Over the last several years, the state is taking more of a leading role in the economy, which means a larger role for bureaucrats. +

+ Xi has said: “Self-criticism needs to be specific about our problems and needs to touch underlying questions… We must be gratified when told of our errors; we must not shy away from our shortcomings. We must accommodate different opinions and sharp criticism.” +

When medical professionals spoke up about a strange new virus circulating in Wuhan, police gave them reprimands. More and more often, the state is simply arresting critics. Even though the government has every reason to be confident about the effectiveness of its virus containment, it has issued a jail sentence to a citizen journalist under the catch-all charge of “picking quarrels and provoking trouble.” For all the emphasis on seeking truth from facts, the state still maintains this practice of shooting the messenger or jailing its critics. +

+ On its own terms, the party center’s instruction is unevenly followed. And there are plenty of reasons to doubt the sustainability of Chinese growth that exist beyond the party’s capacity for self-reform. The following have all received extensive treatment: demographics will be a clear and serious drag in only a few years; an uncomfortable buildup of debt is now accompanied by growing investor discomfort with strategic defaults; the environment is bearing greater stresses; and based on the state’s aggression abroad and the operation of detention camps for minority groups at home, the rest of the world has become much less friendly towards China. One can add more items here, I want to consider the problems with centralized campaigns of inspiration. +

+ The creation and repetition of key slogans isn’t just crowding out the room for other ideas. The state has prosecuted a decade-long effort to suppress the views it doesn’t like. Not only has the government ramped up censorship, society as a whole is developing greater intolerance for dissenting ideas. +

+ It’s difficult to draw a clear line from tighter speech restrictions to worse economic outcomes. Greater censorship over the last decade has coincided with still-impressive levels of economic growth as well as the growing competitiveness of many more companies. And I think it’s worth considering that the authoritarianism of the late-Prussian and early-German state coincided with the creation of the modern research university as well as fantastic advances in chemistry, physics, and electrical engineering. +

+ But there’s more on-the-ground evidence that ordinary people are growing nervous. In so many settings, one has to tread on eggshells in a public discussion in China, with organizers taking pains to remind audience members of sensitivities. Sometimes even in private, people beg off with an embarrassed laugh that they can’t discuss a subject due to unspecified difficulties. WeChat blocks sensitive keywords, which today includes “decoupling” and “sanctions.” It’s now inconvenient to use the app for professional conversations, and I’ve been pretty insistent to my contacts to use Signal instead. And since I brought up Germany, I wonder if the right analogy for China today is as a successful East Germany. +

+ It’s hard to imagine that this increasingly censorious environment is conducive to good thinking. Actions from the government seem to be matched by a growing intolerance among the population for dissenting views. That’s due in part to their sense of feeling besieged after international opinion on China turned sharply negative after the virus outbreak. That hasn’t made it any better for Fang Fang, the novelist in Wuhan whose journal entries documenting the pandemic were first widely-read and then widely-criticized after she authorized an English translation. At that point, critics charged her with “blackening China’s name” and “handing a knife to China’s enemies.” The abuse wasn’t confined online: prominent personalities in state media have led criticism campaigns against her. I wonder if this society can be reflective and thus capable of self-improvement if it is so intolerant of criticism. +

+ It might not be clear that censoriousness is hurting the creation of new companies, but it is clear that it’s becoming more difficult to create better cultural products. Over the last decade, China’s most successful cultural exports include TikTok, the Three-Body Problem, a few art house films (mostly directed by Jia Zhangke), and that might be it. The Three-Body Problem was published in 2008 and translated into English in 2014; today, the series looks more like something that was able to escape the system rather than the vanguard of a great Chinese outpouring of marvelous cultural creations. Not content to allow science fiction movies to develop independently, the film authorities have this year released guidelines on the correct ideological direction of new films. +

Films more broadly are facing censorship. The two blockbusters released this year (Guan Hu’s Eight Hundred and Zhang Yimou’s One Second) were both mysteriously pulled from festivals and released to the public this year after the state demanded edits. +

+ My best cultural experience this year was to see the Met’s production of Porgy and Bess. It is one of America’s greatest dramas: the story of the marginalized community of Catfish Row, written by an outsider whose works defy easy categorization, and featuring music of surpassing beauty. As I watched a production with superb voices and exuberant dancers, in one of the most lavish theaters in the world, I wondered whether China might one day produce a story of such power. Or if instead every new work must encapsulate core socialist values and the spirit of the 19th party congress.  +

+ This lack of compelling cultural creations matters for many reasons. One of them is that people who’ve never been able to make a visit cannot really visualize the life of an ordinary Chinese person, only the dystopia that has become the way that most foreigners think about the country. The propaganda department has not only failed to directly create globally-appealing culture, it has regulated private creative efforts out of existence. For all of Xi’s hopes to “tell China’s story well,” the Chinese regime seems congenitally incapable of allowing good stories about itself to be told, because of its obsession with exercising total control. +

+ For most of the last few decades, the state has not been so repressive as to smother the most dynamic elements of the economy and society. And I think it is still intent on controlling a limited number of areas it has determined to be political threats (and that it will do very strictly). But every few months brings greater risk that dynamism decays from the shrinking space for acceptable speech and thought. The direction of travel has not been a happy one. As recently as five years into Xi’s term, there were still optimists who believed that his regime might turn out to be more kind. The removal of term limits routed that camp, and few recent events can re-instill confidence that the state sees a limit to greater repression. Detention camps have not gone away, and I wonder if they will be expanded to more than a few sites in western provinces. Surveillance capabilities have significantly scaled up. And everyone knows that the regime is serious about instilling discipline and control. +

+ There has been a more obvious way that the state has set back leading companies this year: through greater assertiveness abroad. The Ministry of Foreign Affairs has peddled inflammatory and sometimes false claims, angering other countries. These acts are contributing to the steady closure of developed markets to Chinese firms. Local ambassadors and official spokespeople have sometimes threatened to halt key projects or cut off the Chinese market to major companies. After economic threats against countries like Australia, the foreign ministry is strengthening arguments among national security hawks in the west that countries should be less dependent on China. In some ways, Chinese diplomats have become the greatest threats to Chinese exporters. +

+ III. A clear line of advance +

+ Meanwhile, the state is growing more deeply involved in the economy, especially in technology. This has been a pivotal year for China and tech, which will be a good area for observing the party’s tactics on inspiration at work. A relentless pace of US actions targeting Chinese companies has delivered unmistakable setbacks to their operations. That has triggered a whole-of-society response to build up domestic technology capabilities, an effort that will be guided at the highest political level. +

+ I wrote last year: “The US responded to the rise of the USSR and Japan by focusing on innovation; it’s early days, but so far the US is responding to the technological rise of China by kneecapping its leading firms. So instead of realizing its own Sputnik moment, it is triggering one in China.” +

+ This year, the US doubled down. It produced two rounds of novel restrictions on Huawei, threatened wider restrictions on Tencent and ByteDance, forced the sale of TikTok to a US consortium, and limited technology exports on SMIC, DJI, and dozens of other companies. Aside from Alibaba, it’s hard to name many big Chinese tech firms that have not faced sanctions or the threat of one from the US. +

+ The actual effects of these regulatory actions have been uneven. Designation to the entity list hasn’t always had a major impact on every company’s operations. Federal courts have tied up the bans on Tencent’s WeChat and ByteDance’s TikTok. At the same time, Huawei is trying to work through major difficulties, especially in its smartphone business. TikTok, China’s most successful tech export, still might be sold off. And more generally, Chinese firms are starting to be locked out of developed markets. Lack of access to the richest and most discerning consumers makes it more difficult to make the best products in the world. +

+ The US can revel in Huawei’s pain. But its actions have not been costless to itself. By withholding components that Chinese companies have relied upon, the US government has turned American firms into unreliable suppliers. These restrictions can sometimes block non-American firms from making sales too. In an extraordinary assertion of extraterritoriality, the US declared in August that any company, anywhere in the world, needs to apply for a license to sell a product to Huawei if it is produced on the basis of US technologies. +

+ Nothing can be easier to destroy than trust. Chinese companies have responded by de-Americanizing their supply chains because they have no choice. US politicians can observe the sometimes-devastating impacts of sanctions. What they don’t seem to realize—or want to believe—is that they’re simultaneously pummeling the American brand writ large. I’ve documented for Dragonomics the uncomfortable questions American companies tell me they’re starting to face on whether they can credibly be long-term suppliers. Elsewhere, the Economist has reported that even poultry farmers in China are wondering if they’ll be able to import baby chicks from the US. +

And there are now multiple reported instances of Japanese companies marketing themselves as more reliable than their American competitors. +
Moreover, I hear growing unease from companies in the rest of Asia and Europe on buying American. Can everyone really be sure that this denial campaign will be limited to a handful of bad Chinese actors? Or is a better model of the US government that once it has found a fun new toy, it will keep playing with it until it is no longer fun? +

+ With these regulations, the US has initiated one of the greatest and strangest antitrust actions ever, against potentially all American exporters. The US Treasury has for years expressed worry about the potential decline of the dollar’s dominance following excessive use of blocking sanctions. This fear is turning into reality for the real economy. One might expect alarm bells to be going off in DC, but it doesn’t appear that there’s much pushback against these regulations, except for murmurs from trade associations. It’s possible to defend these moves as correct—for example by justifying that the costs on American firms are worth it for the chance to slow Huawei down right now—but the government does not appear to have had a vigorous debate about the tradeoffs. Instead, the strategy seems to be a result of bureaucratic kludges, pushed forward by whichever faction has the upper hand, made mostly because the financial sanctions office has resisted dealing a serious blow to Huawei in a single stroke. +

+ For the most part, the control hawks faction of the government has had a run of the table, shown by the fact that US agencies have been more focused on taking down Chinese firms than extending US strengths. At a time when it’s more important than ever to advance its semiconductor companies, the government is crippling their sales to their largest or fastest-growing market. When research capabilities at US universities need to grow, the government is denying them students. And when the US should be attracting more talent to its shores, the government has made it more difficult for people to immigrate. Thus the US looks committed to a strategy to destroy the scientific and industrial establishment in order to save it. +

+ Meanwhile in China, these actions have triggered a surge of interest in mastering technology. For the first time arguably since the industrial rise of Japan in the 1950s, a major country is committed to thinking deeply about the invention of its own tooling. A whole generation of scientists and engineers must examine foundational problems like to build leading tools (like lithography machines) and create the best materials (like wafers and chemicals). And the state is fully behind that effort. After steady calls from Xi throughout the year to master technology, the Central Economic Work Conference announced in December that science and technology work will be the top priority in 2021; the conference has never broken science and technology out as an independent item, never mind give it top spot. +

+ I wrote a column on what a gamechanger these actions can be for Chinese industrial policy. Hardly any of China’s largest technology companies have escaped some form or threat of US sanctions, and many more are wondering if they will end up on some poorly-understood blacklist. Thus the US government has aligned the interests of China’s leading tech companies with the state’s interest in self-sufficiency and technological greatness. Huawei, the greatest victim of US actions, is now in the position of NASA in the 1960s when it comes to chips: a cash-rich entity willing to purchase on the basis of performance, not cost. Access to leading and demanding customers can give a chance to local suppliers who never would have had a shot competing against well-established American firms. +

+ US restrictions are setting back Chinese companies in the short term, but I think it’s unlikely they can crush the broader effort to catch up. No country has monopolized a key technology forever: instead, the history of technology has mostly been a history of diffusion. And Chinese firms are hardly starting from a base of zero. The country has demonstrated a growing ability to master most industrial products and is doing well enough in digital technologies. It remains a dynamic market with a good and improving base of human talent. And perhaps most importantly, it is where most manufacturing is done today, which means its workers have the greatest exposure to technological learning. These advantages don’t guarantee success, especially not on a short timeline. But there’s a chance that things improve rather quickly. The natural trajectory of many technologies were pulled forward with unexpected speed after Kennedy announced his moon target. +

+ Is it a drag that the state is so involved in this effort? Well, yes, and China might well repeat the industrial policy mistakes that have stymied projects in the past. But the state doesn’t feel like it can afford to be hands off. Commentators who criticize China’s efforts as doubling down on a state-led approach seem not to realize that the world has fundamentally changed in the last few years. First, the US cannot credibly guarantee to sell goods that Chinese firms need. And second, US actions have removed the political room that Chinese companies have had to push back against state demands that companies buy domestic. Apart from the processor, a Huawei phone is using comparable amounts of Chinese hardware as the iPhone. ByteDance, Alibaba, and Tencent have been using the best-in-class software and hardware, which are usually American. The state will have an easier time now enlisting these companies to use alternatives. +

+ While promoting the status of science and technology with one hand, the Chinese government has with its other hand reined in the activities of consumer internet companies. I’ve never stopped lamenting the marketing trick that California pulled off to situate consumer internet as the highest form of technology, as if Tencent and Facebook are the surest signs that we live a technologically-accelerating civilization. The “tech” giants are highly-capable companies that print cash. But they’re barely engaged in the creation of intellectual property, excelling instead on business-model innovation and the exploitation of network effects. It’s become apparent in the last few months that the Chinese leadership has moved towards the view that hard tech is more valuable than products that take us more deeply into the digital world. Xi declared this year that while digitization is important, “we must recognize the fundamental importance of the real economy… and never deindustrialize.” +

This expression preceded the passage of securities and antitrust regulations, thus also pummeling finance, which along with tech make up the most glamorous sectors today. The optimistic scenario is that these actions compress the wage and status premia of the internet and finance sectors, such that we’ll see fewer CVs that read: “BS Microelectronics, Peking; software engineer, Airbnb” or “PhD Applied Mathematics, Princeton; VP, Citibank.” +

+ While China is ahead on this attitude shift, I think the US is starting to undergo the same conversion. On this theme, I think it’s worth reflecting on the Chinese actions that drove out Google and Facebook a decade ago. That move blocked a major market to these companies, potentially depriving them of significant revenues, and effectively split the world into two internets. And it has since become part of the justification for US actions against China’s technology champions. But US actions have been an order of magnitude more severe: by attacking Huawei’s supply chain, it can terminate the operations of the entire company, and thus represents a massive escalation. In any case, the Chinese ban of Facebook today looks like a prescient action, given how much the company’s activities have enraged western governments, who complain about the circulation of conspiracy theories and other misinformation on social media platforms. It’s harder to argue that China was foolish to ban products so wondrous that their CEOs need to be hauled on a regular basis before political leaders to endure demands to fix the social problems their platforms are allegedly amplifying. +

+ Instead of coddling the internet companies, Xi has declared that China must never deindustrialize or lose its manufacturing capabilities. There’s some chance that Chinese scientists and engineers never make the breakthroughs that free them from dependence on foreign supply. But I think it’s unlikely that they completely fail: they only need to re-invent certain wheels, which does not mean dreaming up unheard-of new technologies. In the worst case, Chinese scientists engage in pure re-invention, making up for the technologies that they can no longer buy. The optimistic scenario is that they’ll find new ways of doing things after re-examining established ideas. The retreading of old paths might reveal vistas that were passed over too quickly, and which might offer new rewards once properly explored. +

+ Whatever its other worries, the party leadership doesn’t have to fret about becoming a decadent society. Given that its per-capita GDP is still around a sixth of US levels, it still has substantial room for catch-up growth. And it won’t lack for identifying clear lines of advance. On top of eliminating poverty and saving the environment, the party can now add the goal to master technology. The leadership has already held collective study sessions on topics that include artificial intelligence and quantum computing. But it’s going to need to come up with some new tricks to inspire people in science and the industrial world. +

+ There’s nothing like a good space project to captivate the imagination. I find it remarkable how little we discuss the fact that there are almost certainly warm oceans on the moons of Jupiter and Saturn which presents the possibility that we find extraterrestrial life within our very own solar system. Imagine the inspirational value that can flow from fixing a target, say the year of the 2049 centenary, to land a probe on these moons to explore for life. +

+ I thank a number of people for reading a draft of this section or discussing the core ideas with me. +

+ *** +

+ It’s time to talk about books. +

+ This year, I read re-Proust. The first time I did so was during college, this second time was more rewarding. I learned to skip the most tedious parts: the endless descriptions of the French countryside, our narrator’s torment over his love of Albertine (and Gilberte, and Mme. de Guermantes), and how much he’s looking forward to seeing Venice. Instead, I focused on the descriptions of the personalities, for every shade of human vanity is depicted in these pages. +

+ The novel describes the most intense lovesuffering, caused by suspicion, which are exacerbated by the most stupid mistakes, caused by pride. Much of the story is taken up by the effects of mad jealousies, which grip every major character in turn, with the same destructive effects on each. On this reading, I was struck by how much the novel is useful as a book of ethics. Many scenes easily resonate today, like how fiercely Mme. Verdurin or Mme. de Villeparisis must beg and threaten for people to show up for their little parties. It’s not just our narrator who is exposed to be acting ridiculously. Outrageous behavior from every character (driven by pride, vanity, or ambition) receives careful treatment and then a comprehensive skewering. +

+ The key to reading Proust is not to pay too much attention to the plot. It’s of no great import, and one has to get used to abrupt shifts. In this way the novel is like Moby-Dick, which can shift from the politics of dining at Ahab’s table to a loving tour of the literal interior of a sperm whale’s head. Couldn’t find the transition? No matter, that detracts not at all from the wonderfulness of the scenes. Focus instead on the humor. There are many funny things that take place in the aristocratic set pieces, such as the constant misunderstandings of M. de Charlus at the dinner of the Verdurins, or his suspicion at the violinist who professes to enjoy solving algebra equations until late into the evenings, or his interactions with the Duc de Guermantes. Really anything with Charlus portends comedy. +

+ Not everyone loves Proust’s sentences. I thought that Penguin’s translation made them enchanting. In between the humor and the yearning, an air of melancholy is never distant, which gives the books extra depth. The ending is especially sad. Our narrator, whom we knew as a boy and then a young man, suddenly becomes quite old in the second half of the final volume. The novel reaches its climax with a lengthy and beautiful epiphany, like in Mann’s Magic Mountain, in which our narrator realizes a fierce urgency to write this book. +

+ This was a good year for reading long books. In the early months of the pandemic, I went through Jürgen Osterhammel’s The Transformation of the World, a history of the nineteenth century. It’s chock full of facts and too difficult to summarize. In science fiction, I enjoyed Neal Stephenson’s Anathem. Stephenson has an amazing ability to locate all the nerd pleasure centers in one’s brain and then jam his fingers hard on them. Anathem is a bit of a twist on Stephenson’s usual trick: instead of presenting loving rewrites of his favorite Wikipedia articles, he serves us instead loving rewrites from the Stanford Encyclopedia of Philosophy, all wrapped up as usual in a delicious plot. (I want to thank my friend Thijs not only for insisting that I read the book but for mailing me his copy from Amsterdam.) +

+ I continued my Christmas tradition of reading about the second world war. This year: Britain’s War by Daniel Todman. The British perspective is interesting for its focus on the empire: having to think about the colonies was a distraction to the British leadership, but the ability to draw upon disparate resources and produce goods from multiple base areas was a major factor in victory. I never find scenes from battle to be so interesting. Instead, the pleasure I draw from war books is to think through the logistical efforts involved in producing goods and delivering them to the front. The best war books treat these as mathematical problems. And they dwell on the bureaucratic effort involved in working these problems out. +

+ Todman is good on the bureaucratic side of war: “In 1942, there were 1,850 admissions to hospital per 1,000 troops on the Burmese front.” And he allows one to get inside the planning effort: “The average round trip from the UK to North America took two months and twenty days; that from the UK to the Indian Ocean area, seven and a half months. These rhythms dominated planning and the pace of the war.” Logistics has always been an underrated discipline, especially now in the time of vaccine distribution. And pacing in personal life is something we should all be more actively thinking about. It’s not enough to have a big goal far out in the future, success requires identifying milestones and achieving them at a steady pace. +

+ Mao’s Third Front by Covell Meyskens is an account of the effort to make China undefeatable in the ‘60s. How? By relocating heavy industry from the coast into mountainous Sichuan. During the worst years of the Sino-Soviet split, Mao imagined that he might have to fend off an invasion from the revisionist Soviets as well as from the imperialist Americans, who had started to deploy troops in Vietnam. The Third Front was a colossal undertaking that wasn’t even meant as a deterrent, since the state didn’t publicize the effort. Instead, Mao was committed to doing the equivalent of moving industry to Siberia during peacetime, ready to retreat into the mountains (again), to be able to re-emerge victorious. The party’s history is still worth dwelling upon. Its big initiatives since the founding of the People’s Republic in 1949 include not just the Third Front, but also the Great Leap, transforming the written script, the Cultural Revolution, Reform and Opening, and the one-child policy. Any one of these would be a once-in-a-generation trauma, the party managed to pull them off at a rate of once a decade. +

+ Virginia Tufte’s Artful Sentences: Syntax as Style is a useful compendium of different types of sentences. It’s good to keep around as a reference work. Essays One, by Lydia Davis, and Tufte are the best writing references that I’ve recently read. Reading Davis prompts reflection, for example when she points to Flaubert’s description of style as “the rhythm of poetry, the precision of the language of science, capable of sustained melody like a cello; a style that would enter the mind like a stiletto and on which our thought could travel like a boat over calm water on a breeze.” +

+ It’s a happy development that many more people are writing, especially in newsletter form. For now, I find quite a lot of internet writing to be difficult to read: no real sense of pacing, an inability to turn a phrase or sustain a metaphor, the excessive use of italics. Picking up Tufte and Davis would help. Starting internet essays feels too often like going to battle inside a trench, which produces the same sense of trepidation. Writers should prosecute instead a war of movement, conducted through bold and decisive strokes, concluding with an unmistakable impression upon the victim. The feeling that I strive for is to create tightly-controlled expressions of exuberance, like the dancepieces of Philip Glass, which propel a lot of my writing. +

+ *** +

+ I may not not have accomplished much in life, but I’m proud at least to have eaten thalis in Chennai, pizza in Naples, and mie goreng in Singapore. +

+ I know that Beijing is not the world’s best food city, but it might be the best food city for me. One can grab expensive sushi at the restaurant favored by the Japanese embassy or walk a few blocks and order five plates of dumplings for $20. One can find decent dosas, lots of Thai food, and even a bagel store whose breads would be out of place on the Upper West Side but would not be in San Francisco. Best of all, every region of China is represented in this city. To deal with the various challenges of a pandemic year, I found solace in stuffing my face. +

+ I managed to sample dishes from all the provinces this year, including the relatively obscure cuisines from places like Anhui, Guangxi, and Jiangxi. My favorites are: Shanghai, Sichuan, and Yunnan. +

+ Many people dislike Shanghai food—which I’m defining as the broader region that encompasses Suzhou, Hangzhou, Nanjing, etc.—for being too sweet. In my mind, it’s unquestionably the finest cuisine. Not only is it the best at the high end, its noodles, soups, and soup dumplings make up some of the tastiest casual food as well. It’s the cuisine that varies most by season, e.g. bamboo shoots in spring and mitten crabs in fall, which showcases the bountiness of the region and its emphasis on freshness. (That’s quite unlike the tradition of the north, which celebrates every and any occasion with plates of dumplings.) The mixing of vinegar and hot fat produces a slight, magical sweetness, and that is something that the Shanghainese understand well, along with many other secrets. +

+ I expect that everyone is familiar with the glories of Sichuan food, there’s little that I need to add here. I’ve eaten plenty in Chengdu and Chongqing, I hope next that I can explore some of the villages in the countryside that feature local specialties. +

+ And I hesitate to say that Yunnan is next best because it has become so trendy. Some people question whether Yunnan food is coherent enough to be a cuisine, or whether it’s a useless label for dishes that vary over a huge and mountainous province. I think of it as Chinese cooking styles with Southeast Asian ingredients, featuring dishes like rice noodles, which can be more soft or more chewy than wheat noodles, served in a mutton broth and topped with a generous fistful of fresh mint. There are many things one can find there that are uncommon in the rest of the country, like cheeses. My favorites are the mushrooms: there’s nothing more appealing than some freshly-picked mushrooms stir fried with bits of Yunnan ham. +

+ Any of these three regions are worth traveling to for a food tour. My candidate for an underrated cuisine is the food of the northeast, which features breads and stews of huge proportions. I haven’t had enough exposure to foods of all the interior provinces, but I’m happy to suggest that the cuisines of Jiangxi and Anhui are worth exploring. And the category of highly-rated and correctly-rated cuisines should include the foods of the northwest (breads and noodles), Hunan (spicy, though often too oily for me), and Taiwan (my favorite use of seafood). The following are overrated: +

+ Cantonese: surely the most overrated cuisine in China, and perhaps the most overrated cuisine in the world. I concede that dim sum is often a delight; and no lunch can be more simple or more satisfying than a few cuts of roast duck or pork layered on a bed of rice, accompanied by sprigs of greens and some gravy. But we’ve too long allowed Cantonese food to dominate the world’s conception of Chinese cuisine. The high-end dishes don’t come close to the refinement of Shanghai cuisine: chefs reveal contempt for themselves and their craft when they deep fry a lobster, as if it were a carnival food, and I’ve never understood the emphasis on shark fins and sea cucumbers. Please let’s not continue allowing Cantonese to be a default choice for business lunches, Shanghai is more fine. +

+ Beijing’s imperial cuisine is the only Chinese cuisine that I consider to be dumb. It wasn’t until I moved to Beijing that I realized how many of the unfortunate facts of Chinese cooking are the creation of local traditions: the dreadful “brown sauce,” the excessive use of starch, and the compulsive need to fry. Peking duck is fine every once in a while, but it’s far too much fuss and expense for something of medium tastiness. There are so few redeeming dishes in imperial cuisine that I wonder if it has been yet another cruel trick pulled by the eunuchs to hoodwink the emperor, depriving him of culinary pleasures for sport. +

+ Hotpot transcends regions now, so let’s treat it as its own category. Hotpot is a fun social activity to do with friends. It’s a way to display skill at the table, through the management of cooking a variety of foods. But it can never be any sort of culinary revelation. My worst nightmare is for hotpot restaurants to take over every retail restaurant space, so that our only choice is to line up to eat at them inside malls, forever. +

+ Here is my four-step process for ordering success in China: +

  1. Greens are usually the glories of the cuisine: order as many vegetables as there are people
  2. If you will have a meat, consider the juiciness that pairs well with the starch: something saucy if you will eat with rice, or less saucy if you will have soup noodles
  3. Order Yunnan mushrooms if they are on the menu
  4. Fill out the rest with cold appetizers, they are never a bad idea

+ *** +

+ Personal matters for last. 2020 was mostly a fine year, I didn’t have too many issues with it. +

+ After a quick Italian holiday over lunar new year, I returned to Beijing on February 1st. At first I hesitated to fly back, today it looks like the best decision I made this year. It gave me the chance not only to observe the country as it faced its greatest challenge in decades, but also to enjoy normal life more quickly than most other places in the world. One can question the ultimate number of cases in China. But even if the totals were an order of magnitude higher, the reported trend that the country mostly stomped out the virus by April feels correct to all of us living here. Ever since that point, various cities have had to deal with flare ups (including Beijing again in June), but life has been a series of loosening restrictions. The last big milestone was the re-opening of cinemas in August. But well before then, the malls had been once again full and the smart restaurants difficult to book.  +

+ In early April, I wrote a feature for New York Magazine’s Intelligencer on life in Beijing during the worst of the pandemic. There isn’t too much more I’d add, since life had already started to return to normal by then. I haven’t been able to visit the US at all this year, and as bad as things look now, I wonder if the virus will have a long-term impact to extend American strengths. There’s no question that the scientific establishment did a fantastic job, though from afar it doesn’t look like many other segments of society really distinguished themselves. But I wonder if this prolonged experience with the virus will shake loose a lot of the self-imposed paralysis in the US, thus inducing greater domestic change than in China, which dispatched the virus relatively quickly. +

+ In 2019, I spent around two weeks a month out of the country on work travel. This year, my travel was domestic, and I’m glad to have been to visit six new cities: Qingdao, Suzhou, Hefei, Nanjing, Xi’an, and Changchun. Over the summer, everyone went to a handful of places: Hainan’s beaches, Yunnan’s villages, or the Sichuan mountainside. I didn’t go to these places, but had a memorable trip to Mount Changbai in the northeast, which offers stark and frigid alpine scenery. All things considered, Beijing has been one of the best places to be in the world this year, but its lack of nature made me dream of the big forests and wide rivers I grew with in Ontario and the US northeast. I don’t think that there’s anything quite like that in China, but I hope next to be able to see the mountains in Sichuan or the plains of Inner Mongolia. +

+ The combination of less travel and more news events raised my productivity this year. I came close to doubling my work output, from already-high levels, because of a relentless pace of White House actions against Huawei, Tencent, ByteDance, SMIC, and other firms. I wanted to write more for Bloomberg this year, but managed only two columns: discussing the state of US manufacturing in May and why this time is different for Chinese industrial policy in December. In terms of public writing, I’m happy to have contributed the piece to New York Magazine, and would like to try my hand at more feature writing in the future. +

+ Like everyone else, I did a lot of Zoom meetings. I gave a presentation roughly at a rate of once a week, mostly to private audiences. My favorite public event was a keynote I gave for Asia Society Northern California, in which I presented on semiconductors and then moderated a panel that consisted of two technology and two policy experts. And I went on a series of podcasts: I think my best discussion was with the Economist’s Money Talks on China’s institutional strengths. One of the unexpected delights of this year, created by the norm of doing Zoom calls, was to hear from old friends, many of whom I haven’t seen in years or decades. I’m glad that this was a year that created this possibility for reconnection. +

+ In the early months of the pandemic, I picked up a new skill: riding a bike. I’ve always been mortified to admit that I never properly knew how. With the encouragement of kind and patient friends, I’ve enjoyed cycling so much that it has become the primary way I get around Beijing. The city is good for cyclists, with its wide bicycle paths and flat roads. (Given the behavior of most drivers though, Beijing requires taking seriously the principle of safety first.) My favorite activity has become to cycle to the Forbidden City and back home, a nice hour-long ride that I would do after lunch. I’m still enjoying the feeling of gliding down a road on my own propulsion, which gives me a sense of slight unreality. That’s been good for thinking: I wrote significant chunks of this letter while riding down Beijing’s second and fourth ring roads. +

+ This year marks my seventh of not drinking. I expect that I’m in the best shape of my life, given that, regular bike rides, occasional badminton sessions, and working out with my personal trainer three times a week. Still, I’m exhausted. That doesn’t mean it’s time to slow down. There are too many interesting things left to do. +

+ Titan, a planet-sized moon of Saturn, has a thick atmosphere and liquid oceans. It and Europa—one of the moons of Jupiter, which might have warm liquid oceans—offer the best chances of discovering extraterrestrial life in our solar system. Credit: JPL +

+ Here are a few questions on which I think we should all have a view: +

+ Is the successful export of TikTok the start of a new trend or a one-off? The app has been an innovative Chinese creation that became a global success. Is it just the first example of many more successful products to come, or something that looks more like Three-Body Problem today: an exception, not the start of a trend. +

+ Are there enduring advantages to being a producer-friendly economy? The west this year made a political decision to direct stimulus to consumers, while China offered minimal support to households and concentrated on helping production. Its implicit view is that production is more valuable and more difficult to stimulate than consumption. More generally, Chinese officials tend to be incredulous of US complaints of excessive subsidies to manufacturers; they tend to ask what the problem is, as if they’ve been accused of the sin of loving a child too much.  +

+ Will we recognize what broader Chinese success or failure will look like? Since reform and opening, the country has always looked like it was on the brink of some disaster, either economic, political, or financial. Meanwhile, it has avoided big crises as it improves various capabilities. If that’s still the story at the end of the next decade—a decent rate of growth, avoiding the worst crises, while facing tough challenges—should we deem that a success or a failure? +

  • + Subscribe to new posts +

  1. + see Anne-Marie Brady: Marketing Dictatorship: Propaganda and Thought Work in Contemporary China↩ +

  2. + 中国特色社会主义有很多特点和特征，但最本质的特征是坚持中国共产党领导。http://www.qstheory.cn/dukan/qs/2020-07/15/c_1126234524.htm↩ +

  3. + For more, see Charles Fishman’s excellent One Giant Leap, which showed how NASA had to invent a thousand and one technologies to reach the moon ↩ +

  4. + Sometimes translated as “rule of law”: 依法治国↩ +

  5. + see Dan Grover on the UI changes that Chinese apps made: http://dangrover.com/blog/2020/04/05/covid-in-ui.html↩ +

  6. + That’s a broad and unfair generalization, I know. This Economist leader offers a more nuanced view: https://www.economist.com/briefing/2020/08/15/xi-jinping-is-trying-to-remake-the-chinese-economy↩ +

  7. + This is my translation of 不忘初心、牢记使命. There are variations on the third line, I included one I’ve seen: 永远奋斗 ↩ +

  8. + see this excellent discussion between Frederick Teiwes and Joseph Torigian https://omny.fm/shows/the-little-red-podcast/xi-dada-and-daddy-power-the-party-and-the-presiden↩ +

  9. + From Dialectical Materialism Is the Worldview and Methodology of Chinese Communists, 广大党员、干部理想信念坚定、干事创业精气神足，人民群众精神振奋、发愤图强，就可以创造出很多人间奇迹 http://www.qstheory.cn/dukan/qs/2018-12/31/c_1123923896.htm↩ +

  10. + Report to the 19th party congress: http://www.xinhuanet.com/english/download/Xi_Jinping’s_report_at_19th_CPC_National_Congress.pdf↩ +

  11. + see The Decadent Society↩ +

  12. + 经济工作是中心工作，党的领导当然要在中心工作中得到充分体现，抓住了中心工作这个牛鼻子，其他工作就可以更好展开。http://www.qstheory.cn/dukan/qs/2020-07/15/c_1126234524.htm↩ +

  13. + see Frank Pieke’s Knowing China↩ +

  14. + see Dialectical Materialism Is the Worldview and Methodology of Chinese Communists 如果对矛盾熟视无睹，甚至回避、掩饰矛盾，在矛盾面前畏缩不前，坐看矛盾恶性转化，那就会积重难返，最后势必造成无法弥补的损失。 http://www.qstheory.cn/dukan/qs/2018-12/31/c_1123923896.htm↩ +

  15. + from the speech at the Third Plenary Session of the 19th Central Commission for Discipline Inspection↩ +

  16. + from Goals of the Aspiration and Mission Education Campaign, May 31 2019↩ +

  17. + http://www.chinafilm.gov.cn/chinafilm/contents/141/2533.shtml↩ +

  18. + Wang Hongsheng, a boss at Jinghai, admits to fretting about interruptions to chick supplies, even wondering if President Donald Trump might curb American exports. https://www.economist.com/china/2020/10/31/high-tech-chickens-are-a-case-study-of-why-self-reliance-is-so-hard↩ +

  19. + see this WSJ story https://www.wsj.com/articles/the-u-s-vs-china-the-high-cost-of-the-technology-cold-war-11603397438 and Doug Fuller’s claim on Tokyo Electron https://www.jhuapl.edu/assessing-us-china-technology-connections/publications↩ +

  20. + This is admittedly a bit of my own fanciful translation of 必须看到，实体经济是基础，各种制造业不能丢，作为14亿人口的大国，粮食和实体产业要以自己为主，这一条绝对不能丢 http://www.qstheory.cn/dukan/qs/2020-10/31/c_1126680390.htm↩ +

\ No newline at end of file diff --git a/packages/readabilityjs/test/test-pages/debugger.medium/distiller.html b/packages/readabilityjs/test/test-pages/debugger.medium/distiller.html new file mode 100644 index 000000000..478bc104e --- /dev/null +++ b/packages/readabilityjs/test/test-pages/debugger.medium/distiller.html @@ -0,0 +1,265 @@ +
Erik Engheim

+ Nov 28, 2020·23 min read +

Image for post
Image: Apple

+ On YouTube, I watched a Mac user who had bought an iMac last year. It was maxed out with 40 GB of RAM costing him about $4,000. He watched in disbelief how his hyperexpensive iMac was being demolished by his new M1 Mac Mini, which he had paid a measly $700 for. +

+ In real-world test after test, the M1 Macs are not merely inching past top-of-the-line Intel Macs, they are destroying them. In disbelief, people have started asking how on earth this is possible? +

+ If you are one of those people, you have come to the right place. Here I plan to break it down into digestible pieces exactly what it is that Apple has done with the M1. Specifically the questions I think a lot of people have are: +

  1. What are the technical reasons this M1 chip is so fast?
  2. Has Apple made some really exotic technical choices to make this possible?
  3. How easy will it be for the competition such as Intel and AMD to pull the same technical tricks?

+ Sure you could try to Google this, but if you try to learn what Apple has done beyond the superficial explanations, you will quickly get buried in highly technical jargon such as M1 using very wide instruction decoders, enormous reorder buffer (ROB), etc. Unless you are a CPU hardware geek, a lot of this will simply be gobbledygook. +

+ To get the most out of this story I advise reading my earlier piece: “What Does RISC and CISC mean in 2020?” There I explain what a microprocessor (CPU) is as well as various important concepts such as: +

  • Instruction set architecture (ISA)
  • Pipelining
  • Load/store architecture
  • Microcode vs. micro-operations

+ But if you are impatient, I will do a quick version of the material you need to understand to grasp my explanation of the M1 chip. +

+ What is a microprocessor (CPU)? +

+ Normally when speaking of chips from Intel and AMD we talk about central processing units (CPUs) or microprocessors. As you can read more about in my RISC vs. CISC story, these pull in instructions from memory. Then each instruction is typically carried out in sequence. +

A very basic RISC CPU, not the M1
A very basic RISC CPU, not the M1. Instructions are moved from memory along blue arrows into the instruction register. There a decoder figures out what the instruction is and enables different parts of the CPU through the red control lines. The ALU adds and subtracts numbers placed in the registers.

+ A CPU at its most basic level is a device with a number of named memory cells called registers and a number of computational units called arithmetic logic units (ALU). The ALUs perform things like addition, subtraction, and other basic math operations. However, these are only connected to the CPU registers. If you want to add up two numbers, you have to get those two numbers from memory and into two registers in the CPU. +

+ Here are some examples of typical instructions that a RISC CPU as found on the M1 carries out. + 

load r1, 150
load r2, 200
add  r1, r2
store r1, 310

+ Here r1 and r2 are the registers I talked about. Modern RISC CPUs cannot do operations on numbers that are not in a register like this. For example, it cannot add two numbers residing in RAM in two different locations. Instead, it has to pull these two numbers into a separate register. That is what we do in this simple example. We pull in the number at memory location 150 in the RAM and put it into register r1 in the CPU. Next, we put the contents of address 200 into register r2. Only then can the numbers be added with the add r1, r2 instruction. +

Image for post
An old mechanical calculator with two registers: the accumulator and input register. Modern CPUs typically have more than a dozen registers, and they are electronic rather than mechanical.

+ The concept of registers is old. For example, on this old mechanical calculator, the register is what holds the numbers you are adding. Likely the origin of the term cash register. The register is where you registered input numbers. +

+ The M1 is not a CPU! +

+ But here is a very important thing to understand about the M1: +

+ The M1 is not a CPU, it is a whole system of multiple chips put into one large silicon package. The CPU is just one of these chips. +

+ Basically, the M1 is one whole computer onto a chip. The M1 contains a CPU, graphical processing unit (GPU), memory, input and output controllers, and many more things making up a whole computer. This is what we call a system on a chip (SoC). +

Image for post
M1 is a system on a chip. Meaning all the parts making up a computer are placed on one silicon chip.

+ Today if you buy a chip — whether from Intel or AMD — you actually get what amounts to multiple microprocessors in one package. In the past computers would have multiple physically separate chips on the motherboard of the computer. +

Image for post
Example of a computer motherboard. Memory, CPU, graphics cards, IO controllers, network cards, and many other components can be attached to the motherboard to communicate with each other.

+ However because we are able to put so many transistors on a silicon die today, companies such as Intel and AMD began putting multiple microprocessors onto one chip. Today we refer to these chips as CPU cores. One core is basically a full independent chip that can read instructions from memory and perform calculations. +

Image for post
A microchip with multiple CPU cores.

+ This has for a long time been the name of the game in terms of increasing performance: Just add more general-purpose CPU cores. But there is a disturbance in the force. There is one player in the CPU market which is deviating from this trend. +

+ Apple’s not so secret heterogeneous computing strategy +

+ Instead of adding ever more general-purpose CPU cores, Apple has followed another strategy: They have started adding ever more specialized chips doing a few specialized tasks. The benefit of this is that specialized chips tend to be able to perform their tasks significantly faster using much less electric current than a general-purpose CPU core. +

+ This is not entirely new knowledge. For many years already specialized chips such as the graphical processing units (GPUs) have been sitting in Nvidia and AMD graphics cards performing operations related to graphics much faster than general-purpose CPUs. +

+ What Apple has done is simply to take a more radical shift toward this direction. Rather than just having general-purpose cores and memory, the M1 contains a wide variety of specialized chips: +

  • Central processing unit (CPU) — the “brains” of the SoC. Runs most of the code of the operating system and your apps.
  • Graphics processing unit (GPU) — handles graphics-related tasks, such as visualizing an app’s user interface and 2D/3D gaming.
  • Image processing unit (ISP) — can be used to speed up common tasks done by image processing applications.
  • Digital signal processor (DSP) — handles more mathematically intensive functions than a CPU. Includes decompressing music files.
  • Neural processing unit (NPU) — used in high-end smartphones to accelerate machine learning (A.I.) tasks. These include voice recognition and camera processing.
  • Video encoder/decoder — handles the power-efficient conversion of video files and formats.
  • Secure Enclave — encryption, authentication, and security.
  • Unified memory — allows the CPU, GPU, and other cores to quickly exchange information.

+ This is part of the reason why a lot of people working on images and video editing with the M1 Macs are seeing such speed improvements. A lot of the tasks they do can run directly on specialized hardware. That is what allows a cheap M1 Mac Mini to encode a large video file without breaking a sweat while an expensive iMac has all its fans going full blast and still cannot keep up. +

+ What is Special About Apple’s Unified Memory Architecture? +

+ Apple’s “Unified Memory Architecture” (UMA) is a bit tricky to wrap your head around (I got it wrong first time I wrote it down here). +

+ To explain why, we need to take a few steps back. +

+ For a long time cheap computer systems have had the CPU and GPU integrated into the same chip (same silicon die). These have been famously slow. In the past saying “integrated graphics” was essentially the same as saying “slow graphics.” +

+ These where slow for severals reasons: +

+ Separate areas of this memory got reserved for the CPU and GPU. If the CPU had a chunk of data it wanted the GPU to use, it couldn’t say “here have some of my memory.” No, the CPU had to explicitly copy the whole chunk of data over the memory area controlled by the GPU. +

Image for post
CPUs don’t need a lot of data served, but they want it fast.

+ CPUs and GPUs don’t want their memory served the same way. Let us do a silly food analogy: CPUs want their plate of data served very quickly by the waiter, but they are totally cool with small portion sizes. Imagine a fancy French restaurant with waiters on rollerblades to serve you really quickly. +

Image for post
This is how your GPU wants their memory: huge portions. The more the merrier.

+ GPUs in contrast are cool with the waiter being slow to serve the data. But the GPUs want enormous servings. They gobble massive amounts of data because they are massive parallel machines, that can chew through lots of data in parallel. Imagine an American junk food place, where the food takes some time to arrive because they are pushing a whole trolley of food to your seating area. +

+ With such different needs, putting CPUs and GPUs on the same physical chip was not a great idea. The GPUs would sit there starving while given small French servings. The result was that there was no point in putting powerful GPUs on an SoC. The tiny portions of data served up, could easily be chewed up by a weak little GPU. +

+ The second problem was that large GPUs produce a lot of heat and thus you cannot integrate them with the CPU without getting problems ridding yourself of the heat produced. Thus discrete graphics cards tend to look like the one below: Large beasts with massive cooling fans. They have special dedicated memory designed to serve the greedy cards massive amounts of data. +

Image for post
GeForce RTX 3080

+ That is why these cards have high performance. But they have an achilles heel: Whenever they have to get data from the memory used by the CPU, this happens over a set of copper traces on the computer motherboard called a PCIe bus. Try chugging water through a super thin straw. It may get to your mouth fast, but the throughput is totally inadequate. +

+ Apple’s Unified Memory Architecture tries to solve all these problems without having the disadvantages of old school shared memory. They achieve this in the following ways: +

  1. There is no special area reserved just for the CPU or just the GPU. Memory is allocated to both processors. They can both use the same memory. No copying is needed.
  2. Apple uses memory which serves both large chunks of data and serves it fast. In computer speak that is called low latency and high throughput. Thus the need to be connected to separate types of memory is removed.
  3. Apple has gotten the watt usage of the GPU down, so that a relatively powerful GPU can be integrated without overheating the SoC. And ARM chips produce less heat, allowing the GPU to have a higher heat budget than a GPU on the same silicon die as an AMD or Intel CPU.

+ Some will say unified memory is not entirely new. It is true that different systems have had it in the past. But then the difference in memory requirements may not have been as large. Secondly what Nvidia calls Unified Memory is not really the same thing. In the Nvidea world Unified Memory simply means that there is software and hardware which takes care of automatically copying data back and forth between the separate CPU and GPU memory. Thus from a programmers perspective Apple and Nvidia Unified Memory may look the same, but it is not the same in a physical sense. +

+ There is of course a tradeoff in this strategy. Getting this high bandwidth memory (big servings) require full integration which means you take away the opportunity from customers to upgrade their memory. But Apple seeks to minimize this problem by making the communication with the SSD disks so fast, that they essentially work like old fashion memory. +

Image for post
How Mac’s used GPUs before unified memory. There was even an option of having graphics cards outside the computer using a Thunderbolt 3 cable. There is some speculation that this may still be possible in the future.

+ If SoCs Are So Smart, Why Don’t Intel and AMD Copy This Strategy? +

+ If what Apple is doing is so smart, why is not everybody doing it? To some extent they are. Other ARM chip makers are increasingly putting in specialized hardware. +

+ AMD has also started putting stronger GPUs on some of their chips and moving gradually toward some form of SoC with the accelerated processing units (APU) which are basically CPU cores and GPU cores placed on the same silicon die. +

Image for post
AMD Ryzen Accelerated Processing Unit (APU) which combines CPU and GPU (Radeon Vega) on one silicon chip. Does however not contain other co-processors, IO-controllers, or unified memory.

+ Yet there are important reasons why they cannot do this. An SoC is essentially a whole computer on a chip. That makes it a more natural fit for an actual computer-maker, such as HP and Dell. Let me clarify with a silly car analogy: If your business model is to build and sell car engines, it would be an unusual leap to begin manufacturing and selling whole cars. +

+ For ARM, in contrast, this isn’t an issue. Computer makers such as Dell or HP could simply license ARM intellectual property and buy IP for other chips, to add whatever specialized hardware they think their SoC should have. Next, they ship the finished design over to a semiconductor foundry such as GlobalFoundries or TSMC, which manufactures chips for AMD and Apple today. +

Image for post
TSMC semiconductor foundry in Taiwan. TSMC manufactures chips for other companies such as AMD, Apple, Nvidia, and Qualcomm.

+ Here we get a big problem with the Intel and AMD business model. Their business models are based on selling general-purpose CPUs, which people just slot onto a large PC motherboard. Thus computer-makers can simply buy motherboards, memory, CPUs, and graphics cards from different vendors and integrate them into one solution. +

+ But we are quickly moving away from that world. In the new SoC world, you don’t assemble physical components from different vendors. Instead, you assemble IP (intellectual property) from different vendors. You buy the design for graphics cards, CPUs, modems, IO controllers, and other things from different vendors and use that to design an SoC in-house. Then you get a foundry to manufacture this. +

+ Now you got a big problem, because neither Intel, AMD, or Nvidia are going to license their intellectual property to Dell or HP for them to make an SoC for their machines. +

+ Sure Intel and AMD may simply begin to sell whole finished SoCs. But what are these to contain? PC-makers may have different ideas of what they should contain. You potentially get a conflict between Intel, AMD, Microsoft, and PC-makers about what sort of specialized chips should be included because these will need software support. +

+ For Apple this is simple. They control the whole widget. They give you, for example, the Core ML library for developers to write machine learning stuff. Whether Core ML runs on Apple’s CPU or the Neural Engine is an implementation detail developers don’t have to care about. +

+ The fundamental challenge of making any CPU run fast +

+ So heterogeneous computing is part of the reason but not the sole reason. The fast general-purpose CPU cores on the M1, called Firestorm, are genuinely fast. This is a major deviation from ARM CPU cores in the past which tended to be very weak compared to AMD and Intel cores. +

+ Firestorm, in contrast, beats most Intel cores and almost beats the fastest AMD Ryzen cores. Conventional wisdom said that was not going to happen. +

+ Before talking about what makes Firestorm fast it helps to understand what the core idea of making a fast CPU is really about. +

+ In principle you accomplish in a combination of two strategies: +

  1. Perform more instructions in a sequence faster.
  2. Perform lots of instructions in parallel.

+ Back in the ’80s, it was easy. Just increase the clock frequency and the instructions would finish faster. Every clock cycle is when the computer does something. But this something can be quite little. Thus an instruction may require multiple clock cycles to finish because it is made up of several smaller tasks. +

+ However, today increasing the clock frequency is next to impossible. That is the whole “End of Moore’s Law” that people have been harping on for over a decade now. +

+ Thus it is really about executing as many instructions as possible in parallel. +

+ Multi-core or Out-of-Order processors? +

+ There are two approaches to this. +

  • Add more CPU cores. Each core works independent and in parallel.
  • Make each CPU core execute multiple instructions in parallel.

+ For a software developer, adding cores is like adding threads. Every CPU core is like a hardware thread. +

+ If you don’t know what a thread is, then you can think of it as the process of carrying out a task. With two cores, a CPU can carry out two separate tasks concurrently: two threads. The tasks could be described as two separate programs stores in memory or it could actually be the same program performed twice. Each thread needs some bookkeeping, such as where in a sequence of program instructions the thread is currently at. Each thread may store temporary results which should be kept separate. +

+ In principle, a processor can have just one core and run multiple threads. In this case, it simply halts one thread and stores current progress before switching to another. Later it switches back. This doesn’t bring much of a performance enhancement unless the thread has to frequently halt to: +

  • Wait for input from the user
  • Data from a slow network connection, etc.

+ Let us call these software threads. Hardware threads mean you have actual physical CPU cores at your disposal to speed up things. +

Image for post

+ The problem with threads is that software developers have to write so called multi-threaded code. That is often difficult. In the past, this was some of the hardest code to write. However making server software multi-threaded tends to be easy. It is simply a matter of handling each user request on a separate thread. Thus in this case having lots of cores, is an obvious advantage. Especially for cloud services. +

Image for post
The Ampere Altra Max ARM CPU with 128 cores designed for cloud computing, where a lot of hardware threads is a benefit.

+ That is the reason why you see ARM CPU-makers such as Ampere making CPUs such as the Altra Max which has a crazy 128 cores. This chip is specifically made for the cloud. You don’t need crazy single-core performance, because in the cloud it is all about having as many threads as possible per watt to handle as many concurrent users as possible. +

+ Read more about ARM CPUs with many cores: Are Servers Next for Apple? +

+ Apple, in contrast, is on the complete opposite end of the spectrum. They make single-user devices. Lots of threads is not an advantage. Their devices are used for gaming, video editing, development, etc. They want desktops with beautiful responsive graphics and animations. +

+ Desktop software is generally not made to utilize lots of cores. For example, computer games will likely benefit from eight cores, but something like 128 cores would be a total waste. Instead, you would want fewer but more powerful cores. +

+ How Out-of-Order Execution Works +

+ To make a more powerful core we need it to execute more instructions in parallel. Out-of-Order execution (OoOE) is a way to execute more instructions in parallel but without exposing that capability as multiple threads. +

+ For an alternative solution read: Very Long Instruction Word Microprocessors +

+ Developers don’t have to code their software specifically to take advantage of OoOE. Seen from the developer’s perspective it just looks like each core runs faster. Please note it is not a direct alternative to hardware threads. You want to use both, depending on the particular problem you are solving. +

+ To understand how OoOE works, you need to understand some things about memory. Asking for data in one particular memory location is slow. But the CPU is capable of getting getting many bytes at the same time. Hence getting 1 specific byte in memory, takes no less time than getting 100 more bytes following that byte in memory. +

Image for post
Robot pickers in a Warehouse for Komplett.no, an online store in Norway

+ Here is an analogy: Consider pickers in a warehouse. Could be the little red robots in the picture above. Moving to multiple locations spread all over takes time. But picking up items from slots adjacent to each other is quick. Computer memory is very similar. You can quickly fetch content of memory cells which are adjecent. +

+ Data is sent across what we call a databus. You can think of it as a road or pipe between memory and different parts of the CPU where data gets pushed through. In reality, it is of course just some copper tracks conducting electricity. If the databus is wide enough you can get multiple bytes at the same time. +

+ Thus CPUs get a whole chunk of instructions at a time to execute. But they are written to be executed one after the other. Modern microprocessors do what we call Out-of-Order execution (OoOE). +

+ That means they are able to analyze a buffer of instructions quickly and see which ones depend on which. Look at the simple example below: + 

01: mul r1, r2, r3    // r1 ← r2 × r3
02: add r4, r1, 5     // r4 ← r1 + 5
03: add r6, r2, 1     // r6 ← r2 + 1

+ Multiplication tends to be a slow process. So say it takes multiple clock cycles to perform. The second instruction will simply have to wait because its calculation depends on knowing the result that gets put into the r1 register. +

+ However, the third instruction at line 03 doesn’t depend on calculations from previous instructions. Hence an Out-of-Order processor can begin calculating this instruction in parallel. +

+ However more realistically we are talking about hundreds of instructions. The CPU is able to figure out all the dependencies between these instructions. +

+ It analyses the instructions by looking at the inputs to each instruction. Do the inputs depend on output from one or more other instructions? By input and output, we mean registers containing results from previous calculations. +

+ For example, the add r4, r1, 5 instruction depends on input from r1 which is produced by mul r1, r2, r3 . We can chain together these relationships into long elaborate graphs that the CPU can work through. The nodes are the instructions and the edges are the registers connecting them. +

+ The CPU can analyze such a graph of nodes and determine which instructions it can perform in parallel and where it needs to wait for the results from multiple dependent calculations before carrying on. +

+ Many instructions will finish early but we cannot make their results official. We cannot commit them; otherwise, we supply the result in the wrong order. To the rest of the world, it has to look as if the instructions were carried out in the same sequence as they were issued. +

+ Like a stack, the CPU will keep popping done instructions from the top, until hitting an instruction that is not done. +

+ Basically you got two forms of parallelism: One that the developer must deal with explicitly when writing code and one that is entirely transparent. Of course the latter relies on lots of transistors on the CPU dedicated to Out-of-Order Execution magic. This is not a viable solution for small CPUs with few transistors. +

+ It is the superior Out-of-Order execution that is making the Firestorm cores on the M1 kick ass and take names. It is in fact much stronger than anything from Intel or AMD and they may never be able to catch up. To understand why, we need to get into some more technical details. +

+ ISA Instructions vs Micro-Operations +

+ Previously I skipped some details on how Out-of-Order Execution (OoOE) works. +

+ Programs loaded into memory are made up of machine code instructions designed for specific Instruction-Set Architectures (ISA) such as x86, ARM, PowerPC, 68K, MIPS, AVR etc. +

+ For instance the x86 instruction to fetch a number from memory location 24 into a register you may write: + 

MOV ax, 24

+ x86 have registers named ax, bx, cx and dx(remember these are the memory cells inside the CPU you perform operations on). However the equivalent ARM instruction would look like this: + 

LDR r0, 24

+ AMD and Intel processors understand the x86 ISA, while Apple Silicon chips, such as M1, understand the ARM Instruction-Set Architecture (ISA). +

+ However internally the CPU works on an entirely different instruction-set invisible to the programmer. We call these micro-operations (micro-ops or μops). These are the instructions the Out-of-Order hardware works with. +

+ But why can’t the OoOE hardware work with regular machine code instructions? Because the CPU needs to attach lots of different information to the instructions to be able to run them in parallel. +

+ Thus while a normal ARM instruction may be 32-bit (32 digits of 0 and 1), a micro-op can be much longer. It contains information about its order. + 

01: mul r1, r2, r3    // r1 ← r2 × r3
02: add r4, r1, 5     // r4 ← r1 + 5
03: add r1, r2, 1     // r1 ← r2 + 1

+ Consider if we run instruction 01: mul and 03: add in parallel. Both store their result in register r1 . If we write the result of instruction 03: add before 01: mul, then instruction 02: add will get the wrong input. Hence it is very important to keep track of instruction order. The order is stored with each micro-op. It also stores e.g. that instruction 02: add depends on output from 01: mul. +

+ That is why we cannot have programs written using micro-ops. They contain lots of details specific to the internals of each microprocessor. Two ARM processors could have very different micro-ops internally. +

+ Read more about CPUs with micro-ops like instructions: Very Long Instruction Word Microprocessors. +

+ Also, micro-ops are usually easier to work with for the CPU. Why? Because they each do one simple limited task. Regular ISA instructions can be more complex causing a bunch of stuff to happen and thus frequently translate to multiple micro-ops. Thus the name “micro” comes from the small task they do, not the length of the instruction in memory. +

+ For CISC CPUs there is usually no alternative but to use micro-ops otherwise the large complex CISC instructions would make pipelines and OoOE next to impossible to achieve. +

+ RISC CPUs have a choice. So, for example, smaller ARM CPUs don’t use micro-ops at all. But that also means they cannot do things such as OoOE. +

+ Why is AMD and Intel Out-of-Order execution inferior to M1? +

+ But you wonder, why does any of this matter? Why is this detail important to know to understand why Apple has the upper hand on AMD and Intel? +

+ It is because the ability to run fast depends on how quickly you can fill up a buffer of micro-operations. If you got a large buffer then the OoOE hardware will have an easier time to locate two or more instructions which it can run in parallel. But there is no point in having a large instruction buffer if you cannot refill it fast enough after instructions get picked and executed. +

+ The ability to refill the instruction buffer quickly relies on the ability to quickly chop machine code instruction into micro-ops. The hardware units that does this are called decoders. +

+ And finally we get to the killer feature of the M1. The biggest and meanest Intel and AMD microprocessor have a total of four decoders busy cutting machine code instructions into micro-ops. +

+ But this is no match for the M1, which has an absolutely unheard of number of decoders: Eight. Significantly more than anybody else in the industry. That means it can fill up the instruction buffer much quicker. +

+ To deal with this the M1 also has an instruction buffer which is 3x times larger than what is normal in the industry. +

+ Why can’t Intel and AMD add more instruction decoders? +

+ This is where we finally see the revenge of RISC, and where the fact that the M1 Firestorm core has an ARM RISC architecture begins to matter. +

+ You see, an x86 instruction can be anywhere from 1–15 bytes long. RISC instructions have fixed length. Every ARM instruction is 4 bytes long. Why is that relevant in this case? +

+ Because splitting up a stream of bytes into instructions to feed into eight different decoders in parallel becomes trivial if every instruction has the same length. +

+ However, on an x86 CPU, the decoders have no clue where the next instruction starts. It has to actually analyze each instruction in order to see how long it is. +

+ The brute force way Intel and AMD deal with this is by simply attempting to decode instructions at every possible starting point. That means x86 chips have to deal with lots of wrong guesses and mistakes which has to be discarded. This creates such a convoluted and complicated decoder stage that it is really hard to add more decoders. But for Apple, it is trivial in comparison to keep adding more. +

+ In fact, adding more causes so many other problems that four decoders according to AMD itself is basically an upper limit for them. +

+ This is what allows the M1 Firestorm cores to essentially process twice as many instructions as AMD and Intel CPUs at the same clock frequency. +

+ One could argue as a counterpoint that CISC instructions turn into more micro-ops. For instance if every x86 instruction turned into 2 micro-ops while every ARM instruction turned into 1 micro-op, then four x86 decoders would produce the same number of micro-ops per clock cycle as an ARM CPU with 8 decoders. +

+ Except this is not the case in the real world. Highly optimized x86 code rarely uses complex CISC instructions, which would translate into many micro-ops. In fact most will only translate into 1 micro-op. +

+ However all these simple x86 instructions don’t help Intel or AMD. Because even if those 15 byte long instructions are rare, the decoders have to be made to handle them. This incurs complexity that blocks AMD and Intel from adding more decoders. +

+ But AMDs Zen3 cores are still faster right? +

+ As far as I remember from performance benchmarks, the newest AMD CPU cores, the ones called Zen3 are slightly faster than Firestorm cores. But here is the kicker: That only happens because the Zen3 cores are clocked at 5 GHz. Firestorm cores are clocked at 3.2 GHz. The Zen3 is just barely squeezing past Firestorm despite having almost 60% higher clock frequency. +

+ So why doesn’t Apple increase the clock frequency too? Because higher clock frequency makes the chips run hotter. That is one of Apple’s key selling points. Their computers — unlike Intel and AMD offerings — barely need cooling. +

+ In essence, one could say Firestorm cores really are superior to Zen3 cores. Zen3 only manages to stay in the game by drawing a lot more current and getting a lot hotter. Something Apple simply chooses not to do. +

+ If Apple wants higher performance they are simply going to add more cores. That lets them keep watt usage down while offering more performance. +

+ The future +

+ It seems AMD and Intel have painted themselves into a corner on two fronts: +

  • They don’t have a business model that makes it easy to pursue heterogeneous computing and SoC designs.
  • Their legacy x86 CISC instruction set is coming back to haunt them, making it hard to improve OoO performance.

+ It doesn’t mean game over. They can increase the clock frequency and use more cooling, throw in more cores, beef up the CPU caches, etc. But they are both at a disadvantage. Intel is in the worst situation, as their cores are already soundly beaten by Firestorm, and they have weak GPUs to integrate with an SoC solution. +

+ The problem with throwing in more cores is that for typical desktop workloads you reach diminishing returns with too many cores. Sure lots of cores are great for servers. +

+ However here companies such as Amazon and Ampere are attacking with monster CPUs with 128 cores. This is like fighting the western and eastern front at the same time. +

+ But fortunately for AMD and Intel, Apple doesn’t sell their chips on the market. So PC users will simply have to put up with whatever they are offering. PC users may jump ship, but that is a slow process. You don’t leave immediately a platform you are heavily invested in. +

+ But young professionals, with money to burn without too deep investments in any platform, may increasingly turn to Apple in the future, beefing up their hold on the premium market and consequently their share of the total profit in the PC market. +

\ No newline at end of file diff --git a/packages/readabilityjs/test/test-pages/electrek/distiller.html b/packages/readabilityjs/test/test-pages/electrek/distiller.html new file mode 100644 index 000000000..932019c08 --- /dev/null +++ b/packages/readabilityjs/test/test-pages/electrek/distiller.html @@ -0,0 +1,104 @@ +
Georgia solar manufacturing

+ Seoul-headquartered PV solar-cell manufacturing giant + Q Cells + today announced that it’s opening a new solar panel manufacturing + facility in Dalton, Georgia. +

+ Georgia solar panel manufacturing grows again +

+ It’s a $171 million expansion of Q Cells’ existing solar module + manufacturing plant in Dalton, and that will create 470 additional + jobs. Total local Q Cells employees will exceed 1,000 when the + expansion is complete. +

+ Groundbreaking is planned for fall 2022 and operation is expected + to commence within the first half of 2023. +

+ This latest domestic solar manufacturing expansion will boost + production of advanced photovoltaic (PV) modules, and that will + help the US work move toward its goal of decarbonizing the + electric grid. +

+ The new facility will produce 1.4 gigawatts (GW) of solar modules + per year made with Q Cells’ next-gen PV cells, a high-efficiency + tunnel oxide passivated contact technology better known as TOPCon. +

+ Combined with the existing 1.7-GW factory, the expansion will + bring Q Cells’ total capacity in the US to 3.1 GW; that’s + equivalent to one-third of the country’s solar module + manufacturing capacity. +

Qcells CEO Justin Lee said:

+ Georgia has become the clean energy manufacturing heart of + America, and we are proud to contribute to the state’s advanced + manufacturing economy. +

+ Q Cells has the largest market share in the US commercial and + residential markets and also supplies the utility-scale solar + sector. +

+ Senator Jon Ossoff (D-GA) met with Q Cells’ parent company + Hanwha in Seoul last year and has been actively pitching and + securing additional clean energy investment in Georgia. +

+

+ NEWS: Today, Sen. + @ossoff + and + @Qcells_NA + announced a $171 million expansion of their Dalton, Georgia + solar manufacturing plant, creating nearly 500 new Georgia + jobs. + pic.twitter.com/5bWGfUb3RQ +

+

+ — Ossoff's Office (@SenOssoff) + May 26, 2022 +

+

+ Ossoff also recently helped secure Hyundai’s investment in electric vehicles + that will create over 8,000 jobs in Bryan County. +

+ Washington, Georgia, embraces solar +

+ Also in Georgia, + WeSolar CSP, a minority-owned renewable energy tech and design company + headquartered in Princeton, New Jersey, will design a solar farm + along with a microgrid that will supply the City of Washington, + Georgia, that will replace natural gas use. Washington is 90 miles + east of Atlanta and has a population of around 4,000. +

+ The project will comprise both solar panels and a concentrating + solar-thermal power (CSP) technology. +

WeSolar CSP’s CEO, Steve Anglin, said:

+ The citizens of the City of Washington will benefit by having a + cleaner environment and experiencing price certainty in the face + of the ever-increasing energy costs of fossil fuels. +

Photo: Hanwha Q Cells

+ UnderstandSolar is a free service that links you to top-rated + solar installers in your region for personalized solar + estimates. Tesla now offers price matching, so it’s important to + shop for the best quotes. Click here to learn more and get your quotes. — *ad. +

+ FTC: We use income earning auto affiliate links. + More. +

+ You’re reading Electrek— experts who break news about + Tesla, + electric vehicles, + and green energy, + day after day. Be sure to check out our + homepage for all the latest + news, and follow Electrek on + Twitter, + Facebook, and + LinkedIn + to stay in the loop. Don’t know where to start? Check out our + YouTube channel for the latest reviews. +
\ No newline at end of file diff --git a/packages/readabilityjs/test/test-pages/elidourado/distiller.html b/packages/readabilityjs/test/test-pages/elidourado/distiller.html new file mode 100644 index 000000000..8622d44ea --- /dev/null +++ b/packages/readabilityjs/test/test-pages/elidourado/distiller.html @@ -0,0 +1,139 @@ +

+ As we start a new decade, it’s a good time to reflect on expectations for the next 10 years. Tyler thinks the Great Stagnation could be ending. Caleb sees cracks. Noah expresses techno-optimism. In this post, my aim is not to predict an end or non-end to stagnation. Rather, it is to think through the particulars of how technology could evolve over the next decade. Then we can assess separately whether we should consider it the Roaring 20s or the Boring 20s. +

+ What would constitute an end to the Great Stagnation? Any precise cutoff will be arbitrary, but for the sake of discussion, let’s say sustained growth in utilization-adjusted total factor productivity of 2 percent per year. By comparison, mean utilization-adjusted TFP growth from 1947 through 1972 was 2.1 percent. Since 2005, it has been 0.17 percent. (Note: it is important to use the utilization-adjusted series, as this corrects for the business cycle.) +

Total factor productivity in the U.S. since 1947

+ Whatever your cutoff for TFP growth, one of my convictions is that scientific breakthroughs alone are not enough to drive an end to the Great Stagnation. TFP only budges when new technologies are adopted at scale, and generally this means products, not just science. Science lays critical groundwork for new technology, but after all the science is done, much work remains. Someone must shepherd the breakthrough to the product stage, where it can actually affect TFP. This means building businesses, surmounting regulatory obstacles, and scaling production. +

+ With that caveat firmly in mind, what will the next decade bring in terms of meaningful technological change? Here’s what I’m watching. +

+ Biotech and health +

+ We are coming off a huge win: two new mRNA COVID vaccines, conceived and brought to market in less than a year. The ability to encode and deploy arbitrary mRNA in our bodies sure seems like a game changer—it allows us to essentially program our cells to make whatever proteins we want. In the case of the COVID vaccines, the vaccine payload instructs our cells to make the coronavirus spike protein, which our immune system then learns to attack. Bert Hubert has a fascinating write-up of the “code” in the vaccine. +

+ Bringing a brand new vaccine to market in less than a year—using a never-before-applied-in-humans-at-scale technology no less—is a world record, but it could have been even faster. As David Wallace-Wells emphasizes, Moderna’s vaccine was designed by January 13. We had it the whole time. Some delay was necessary to determine effective dosing. Some further regulatory delay may have been warranted to ensure the vaccine was safe and to ascertain its efficacy. But as Wallace-Wells indicates, the regulatory outcome was never really in doubt. “None of the scientists I spoke to for this story were at all surprised by either outcome,” he writes. “All said they expected the vaccines were safe and effective all along.” +

+ What should we make of the fact that all of the scientists knew all along that Moderna’s vaccine would work? The question in my mind is: what other mRNA treatments do we have the whole time? What if I told you Moderna has an HIV vaccine candidate? HIV lacks SARS-CoV-2’s telltale spike protein and thus may prove a more challenging foe—but don’t you wonder, if we treated the problem with real urgency, whether new mRNA technology could wipe out the AIDS epidemic this decade? I do. +

+ And mRNA technology can be deployed against more than just viruses. Both Moderna and BioNTech have personalized vaccine candidates targeting cancer. Although called a “cancer vaccine,” the treatment is only administered once the subject has cancer—it isn’t preventative. The companies use an algorithm to analyze the genetic sequences of the tumor and the patient’s healthy cells and predict which molecules could be used to generate a strong immune response against the cancer. “I was actually witnessing the cancer cells shrinking before my eyes,” said Brad Kremer, a melanoma patient who received the BioNTech treatment. So let’s milk mRNA technology for all it’s worth this decade. It can save us from more than just a pandemic. +

+ What about CRISPR? It is a great example of a technology that has not yet made a meaningful economic contribution. Although the technique for editing DNA was discovered in 2012—and a Nobel Prize was awarded to its two discoverers this year—no treatment using CRISPR has been approved outside of clinical trials. So far, its impact has been limited to making researchers more productive—not a bad thing, to be sure, but not close to CRISPR’s full potential. As trials progress, however, I do think some CRISPR treatments will come online in the next few years, especially those targeting genetic disorders that we have very limited means of otherwise treating. +

+ DeepMind’s protein-folding breakthrough signals a promising decade for the science of proteomics. Most directly, being able to predict protein shapes will enable us to discover drugs more rapidly. Buuuut, because drug trials take many years, we might expect this technology not to really be felt by the general public until the 2030s. +

+ What DeepMind’s achievement indicates to me the most is that machine learning is actually useful. This might seem obvious, but consider: most applications of machine learning so far—excluding autonomous vehicles, which have themselves not really arrived yet—are toys. I love watching AlphaZero crush Stockfish on YouTube, but chess is literally a game. GPT-3 produced some fun demos. AlphaFold heralds something different—non-toy superhuman performance is now here, and I am interested to see what else it can do. Aside from the aforementioned AVs, I expect it to be applied widely in other areas of biology. Again, it will take a long time for the breakthroughs to trickle down into products, but at least the 2030s should be sick. I mean, not sick. Healthy. +

+ Let’s talk about life extension, one of my favorite biotech topics. 2020 was a big year for the Conboy Lab at Berkeley, which proved that all the weird past findings about “young blood” extending life were not actually due to any elixir in the blood of children (thank goodness). Rather, the rejuvenating aspects of young blood experiments were due to the dilution of harmful factors in old blood. By mechanically removing plasma and replacing it with saline and enough albumin to replace what was taken out, they diluted aged blood factors in both mice and humans and were able to rejuvenate germ layer tissues and improve cognition by reducing neuroinflammation. +

+ These findings are exciting not only because they represent a scientific advance in understanding aging, but also because they herald the first real anti-aging product that could come to market. Therapeutic plasma exchange is FDA-approved (not for aging, but for a bunch of other conditions). I imagine there remain prohibitions on advertising that it can add years to your life, but it is safe, and a doctor can prescribe it off label. It’s also cheap. An automated plasmapheresis machine—which lets you do treatment after treatment—can be bought online for under $3,000. That is less than the cost of a single transfusion of young blood sold by the startup Ambrosia. How long until someone opens a clinic offering plasma dilution? I bet someone tries it in 2021. If it works, people will get over the weirdness, and it could be commonplace by 2030. +

+ Another longevity product that is about to get hot: aging clocks based on DNA methylation or proteomics. Do you want to know how biologically old you are? Today, for a few hundred dollars, you can get a test that will tell you. As these tests become better and cheaper, self-experimenters are going to have a field day. Doing before-and-after aging tests, anyone who can get their hands on human growth hormone could replicate the protocol used by Fahy et al. to rejuvenate the thymus. As the thymus is a critical element of the immune system, decline of which is a critical factor in aging, this is non-trivial rejuvenation. The Fahy study found that 12 months of treatment created about 2.5 years of epigenetic rejuvenation, with results accelerating in the last quarter of the trial. +

+ There is a lot more in the Rejuvenation Roadmap—dozens of possible life-extending treatments are at various stages of development. There’s a good chance a few senolytic drugs will be approved by the end of the decade. As I noted yesterday at Fortune, we spend less than 1% of the NIH budget on aging biology—we should raise that by a lot. +

+ Unlike others, I am not-so-bullish on metformin. It does seem to reduce all-cause mortality in Americans, but it may do so because 88% of Americans are metabolically unhealthy. If you are one of the 12%, and you should strive to be, I don’t think metformin will do much for you. +

+ One final biotech observation: every year, the Apple Watch gets a new health-related sensor. This year it was blood oxygen, pretty good for detecting if you might have COVID! Fast forward to 2030 and wearables will have at least 10 more health-related sensors than they do today. Some no-brainers are body temperature, blood pressure, and blood glucose sensors. What will the other 7 be? At some point, it becomes possible to replace a lot of primary care with continuous monitoring. A few smart algorithms to provide simple medical advice could improve population-level health without much cost. More data could also yield faster, more accurate, and of course more remote diagnoses when you do have to see a doctor. +

+ There is a lot in biotech that is promising right now, but in more than any other field, it is important not to be seduced by the sexy headlines showing rapid scientific progress. Don’t get complacent. Biology is proceeding faster than medical productivity because a lot of the wonderful discoveries are not being translated into approved treatments and products at a decent rate. Let’s salute and cheer for the discoveries, but spare many thoughts for the entrepreneurs trying to bring treatments to market. +

+ Energy +

+ The 2010s were the wind and solar decade. We observed stunning declines in the cost of both, although total deployment of wind and solar remains small—in 2019, wind and solar represented less than 9 percent of utility-scale electricity generation in the US. In the 2020s, cost declines will likely stall—wind and solar are already pretty cheap, so the declines of the past decade are not reproducible. Deployment, on the other hand, will accelerate. +

+ Mass deployment of wind and solar will bring challenges. These sources are highly intermittent. When the wind suddenly stops blowing—which happens—we need a way to quickly make up the deficit. Each of the three electricity grids in the continental US—east, west, and Texas—has to remain in supply-demand balance every second of every day. We can use grid storage to smooth out some of the bumps, but storage remains expensive. To reach a grid powered entirely by today’s renewables, we would need storage at a price of $20 per kWh (with caveats). +

+ That storage doesn’t all have to come from batteries, but let’s talk about batteries for a bit. Using Tesla’s grid-scale Powerpack as data, a 232 kWh battery today costs $125,793. That is a price of over $542/kWh. Through innovation, that pricetag will come down over the course of this decade, but improvements on the supply side could easily get swamped by increases in demand. After all, this decade will also include a huge shift toward electric vehicles, which I will discuss below. When demand outpaces supply, prices tend to stay high, even when there is impressive innovation. +

+ With increased deployment of intermittent power generation, increased total demand for electricity due to electric vehicles, a high cost of grid storage, inadequate electricity transmission (have I mentioned that we often neglect to build in this country?), and strong political support for decommissioning fossil fuel plants, the 2020s may be a time of electric grid instability. This could be tempered to some extent by using car batteries as grid resources and through (politically unpopular) variable electricity prices. +

+ Ultimately, we need scalable zero-carbon baseload energy, which means nuclear or geothermal. The problem with nuclear is the high cost. If you look at NuScale’s small modular reactor technology, they are targeting 6.5¢/kWh. That is baseload power, so not directly comparable to wind and solar’s intermittent generation costs, but even so, it isn’t the most competitive in today’s market. Furthermore, NuScale’s flagship project was just delayed three years and is now not scheduled to come online until 2030. +

+ What is more plausible this decade is enhanced and advanced geothermal systems. The legacy geothermal industry is sleepy, tapping energy at traditional volcanic hydrothermal hotspots—forget about it. The next generation of the industry, however, is a bunch of scrappy startups manned by folks leaving the oil and gas industry. The startups I have spoken to think with today’s technology they can crack 3.5¢/kWh without being confined to volcanic regions. With relatively minor advancements in drilling technology compared to what we’ve seen over the last decade, advanced geothermal could reach 2¢/kWh and scale to become viable just about anywhere on the planet. Collectively, the startups are talking about figures like hundreds of gigawatts of generation by 2030. I’m watching this space closely; the Heat Beat blog is a great way to stay in the loop. As I wrote last month, permitting reform will be important. +

+ Fusion continues to make technical progress. I expect we will get a demonstration of energy-positive fusion in this decade from one of several fusion startups or perhaps Lockheed Martin’s compact fusion reactor. But again: a demonstration is far from a change that transforms society. It will take further decades to deploy reactors onto the grid. By the time fusion gets there, the energy market will be quite different from when we started working on fusion reactors in the 1940s. Wind, solar, and hopefully geothermal will make electricity pretty cheap, and fusion will struggle to compete. +

+ Consider: around half the cost of an advanced geothermal plant is drilling, and half is conversion equipment. Suppose the plant is amortized over 30 years (although many geothermal plants last longer), and after that period the conversion equipment needs to be replaced. But the hole in the ground does not need to be replaced! That means for the next 30 years, electricity can be generated at half the initial cost. Geothermal wells we dig this decade could be producing at less than 1¢/kWh by the 2050s. That is a tough market for fusion to break into. But fusion will still be a great source of power in applications where other sources aren’t available, such as in space. +

+ The 2020s will be a big decade for sustainable alternative fuels (SAF). Commercial aviation can’t electrify—batteries will never match fossil fuels’ energy density. Given political realities, aviation has no choice to decarbonize, which means either hydrogen fuel or SAF. Hydrogen fuel is much better than batteries, but still not as energy dense as fossil fuels or SAF, and so my money is on SAF, and particularly on fuel made from CO₂ pulled from the atmosphere. It is easy to convert atmospheric CO₂ to ethanol in solution; and it is easy to upgrade ethanol into other fuels. But it is hard to separate ethanol from water without using a lot of energy—unless you have an advanced membrane as Prometheus Fuels does. I have written about Prometheus before and continue to follow them closely. Their technology could decarbonize aviation very suddenly. +

+ One final note on energy: there may be very interesting geopolitical consequences in the decade ahead to America’s newfound energy independence. I could easily see, for example, the US deciding we actually don’t need an alliance with the Saudis after all, considering they are journalist-dismembering savages. If the US pulls out of Saudi Arabia, war between the Saudis and the Iranians becomes likely. Which means oil shipments to Asia get disrupted. Which means global chaos. This Zeihanesque scenario is only a scenario, but I’m watching for it. +

+ Transportation +

+ Here’s the thing about electric cars: they are better than regular cars. They have lower fuel costs. They have fewer moving parts and thus lower maintenance costs. They have higher low-end torque and faster acceleration. If you mainly drive to and from work and have a charger at home, you never have to stop for gas. Electric cars will win because they are better, and the shift will happen suddenly. +

+ California will require that new cars purchased after 2035 have zero emissions. For most people, this will be a non-issue, as by 2035 most Californians would not dream of getting an internal combustion engine vehicle. I say this even as a relative battery price pessimist—or more accurately, I am a relative battery price pessimist because I think demand for batteries will be off the charts. +

+ One area where batteries may not work (aside from aviation, already discussed) is trucking. Towing really heavy loads requires a lot of energy—hydrogen fuel cells will be more suited to interstate trucking. The transition from diesel to hydrogen in trucking will likely not be as automatic as the transition from gas to batteries in cars. It’s possible that truckers will need a bit of a push. +

+ As cars shift to electric and trucks shift to hydrogen, air pollution will plummet, especially the currently unregulated ultrafine particles less than 0.1 μm in diameter which cause the worst health harms. There may still be larger particles from tires degrading and so on, but my view is that these do not cause serious health problems. Getting rid of the smallest particles, particularly from diesel fumes, will create health gains which may seem to appear out of nowhere. Fewer premature births, fewer cases of asthma, fewer cancers, fewer mystery illnesses. +

+ Autonomous vehicles could finally happen at scale. Waymo is already in production with a driverless fleet in Phoenix. Tesla has a “full self-driving” computer which might not yet live up to the label, but is nonetheless very cool. Although we have all been continually disappointed by the promise of autonomy right around the corner, it does seem like it has to happen at some point. As sensors and computing power get cheaper, and machine learning algorithms get better, autonomy is inevitable. A decade is a long time, so I am reasonably confident it will happen in the 2020s. It could save a lot of lives. Autonomy, too, will accelerate the adoption of electric vehicles, as fleet companies will prefer the low maintenance costs of battery or fuel cell vehicles. +

+ Let’s do aviation. As everyone knows, I am a huge fan of supersonics. I continue to cheer for my former colleagues at Boom, who will legitimately fly a supersonic aircraft in 2021. Supersonics will have an enormous impact on global business when it arrives at scale, but it’s looking like that won’t be in the 2020s—Boom’s most optimistic timeline per a recent article is a first full-scale airliner by 2026, then several years of certification tests, then a ramp in production to make it matter. +

+ Other contenders: Aerion has a Mach-1.4 business jet design ready to enter production. The key question is whether they can raise the money to build the factory. The business jet market is small, so as an investor you really have to believe they will be able to parlay success on the AS2 into a future airliner program. Hermeus is working on a Mach-5 design point with 20 passengers. Exosonic is targeting Mach 1.8 with low boom technology that could allow it to operate over land. Gulfstream seems to have shuttered their supersonic business jet program, which was never announced in any case. +

+ In addition to Boom’s XB-1 flight test program commencing in 2021, I’ll also be watching two other supersonic flight programs in the early 2020s. NASA’s X-59 will start flying over select cities in 2023 to collect data on acceptable levels of sonic boom. This will pave the way for new standards that unlock overland supersonic flight. NASA is literally making America boom again! In the next couple years, I also expect an unmanned demonstrator flight from Hermeus reaching as high as Mach 5. Awesome, right? +

+ Aside from supersonics, the other exciting development in aviation is the proliferation of urban air mobility companies. Check out Joby, which recently acquired Uber’s Elevate division. Or Wisk, a joint venture between Boeing and Kitty Hawk. Both of these projects could enter service in the first half of the decade. Hyundai also has a new UAM division—look for their product to enter service near the end of the decade. +

+ A key question in my mind regarding urban air mobility is whether regulations will allow autonomy. The business model doesn’t seem like it will work if you have to pay a pilot and lose the space associated with the pilot’s seat, which could otherwise serve an additional passenger. The FAA has been very incremental about allowing even small drones to fly beyond line-of-sight of the operator. In order for urban air mobility to compete with, say, Uber Black, FAA needs to adopt rules for low-altitude air traffic control (called UTM) and figure out a way to certify autonomous operations. Wisk seems pessimistic—they are targeting New Zealand as a first market. +

+ Drone delivery is likely in the 2020s. FAA is about to issue a rule incrementally expanding drone operations, this time allowing operations at night and flights over crowds of people. This is how the FAA operates: use waiver authority to expand the scope of drone operations, and then once they are comfortable with that, make it into a generally applicable rule. I expect the process will incrementally allow bigger and bigger drone delivery programs until they become normal. Those of us who live within five miles of a Class B airport, however, may be out of luck the entire decade. +

+ Let’s talk about tunnels. An efficiently governed country would need some tunnels, but perhaps not very many. China has added 25,000 km of (mostly non-tunnel) high-speed rail since 2008, and there is no technological reason why we couldn’t have done the same. But with a promiscuous distribution of the veto power, building long rail lines above ground becomes challenging. It may therefore be worth the high cost of tunneling to build new high-speed transport options. +

+ The Boring Company has a small, near-operational “loop” under construction in Las Vegas. The project will whiz people around the convention center at up to 155 mph. Expansion plans include the Las Vegas Strip, the airport, and eventually connecting to Los Angeles. Another Boring project, currently mired in environmental review, is the DC-Baltimore Loop, which would connect the two cities’ downtowns in 15 minutes. All of Boring’s loops are designed to be compatible with hyperloop requirements, which would eventually enable 600-mph travel between major cities. +

+ Although the full realization of this technology—a nationwide hyperloop network—is unlikely by 2030, even the 150-mph version is worth following. The time and hassle cost of travel is an important input into the gravity model of trade. I expect the DC-Baltimore Loop to significantly increase economic activity between the two cities—especially helping to revitalize Baltimore, as it would become easier to live there and work in DC. +

+ Space +

+ The big story in space technology for the next 10 years is Starship, as it will enable just about everything else. Let’s compare some launch costs. The Space Shuttle entered service in 1981 and launched successfully 134 times. Each launch cost an inflation-adjusted $1.8 billion. The payload cost to low-Earth orbit (LEO) was $65,400/kg. Today’s workhorse launch vehicle, the Falcon 9, can send cargo to LEO for $2,600/kg. That is a staggering decrease in launch costs. +

+ Starship promises to take this trend much further. On Falcon 9, only the first stage is reusable, whereas on Starship, the entire system—both the booster and the space vehicle—is reusable. Starship runs on dirt cheap liquid methane instead of expensive rocket fuel. It is made out of stainless steel instead of more expensive traditional aerospace materials. SpaceX is talking about churning out Starships at a rate of one every 72 hours for a cost of $5 million each. Operating costs come down with a high flight rate, so Elon is figuring a $1.5-million fully burdened launch cost for 150 tons to LEO. That is $10/kg, more than 100 times cheaper than a Falcon 9 launch today. +

+ It gets even more insane. Because Starship is designed to be refuelable on orbit, its 150-ton payload capacity to LEO equals its payload capacity to anywhere in the solar system. You will be able to launch 150 tons to LEO, load up on fuel while orbiting Earth, and then fly the same payload the rest of the way to the moons of Jupiter. The whole thing could cost less than one Falcon 9 launch—which is limited to 15 tons to LEO in a reusable configuration or 4 tons to Mars in an expendable configuration. +

+ Let’s apply the gravity model of trade once more, this time to commerce between Earth and LEO. Meta-analyses have found that trade (on Earth) is roughly inverse-linear in transport costs. If that holds for space, a 200x cost reduction in travel between Earth and LEO should increase “trade” between Earth and LEO by 200x. Commerce between the Earth and the moon, or between the Earth and Mars, starting from a base close to zero, would be stimulated even more. +

+ It’s worth noting a second-order effect of cheap launch costs. When launch is expensive, more engineering has to go into the payload to ensure reliability. You don’t want to spend $1.8 billion on launch, and then find out, as NASA did with the Hubble Space Telescope, that your new satellite needs repairs. This dynamic has caused over-engineering of space payloads. With launch for a new low price of $10-20/kg, companies and research agencies will be able to reduce engineering expenses by simply taking on the risk of paying for another (cheap) launch. +

+ Since my guiding star is economically noticeable technological progress, let’s talk about that. SpaceX first landed a rocket booster five years ago. They have been undercutting all other players in the medium-lift launch market ever since. But in the grand scheme of things, launch is still a small market. Aside from getting to watch cool livestreams of boosters landing, Falcon 9 has probably not made a noticeable impact on your life (unless you work in the space industry). +

+ That is finally beginning to change with Starlink. As of this month, there are 955 Starlink satellites providing Internet access to thousands of users in a “better-than-nothing” beta test. The constellation size could go as high as 42,000 satellites. Internet speeds are already over 100 mbps down—they seem to be only somewhat attenuated by bad weather. For many rural customers, the service is indeed much better than nothing—better than any other available alternative. With more (and more advanced) satellites in operation, speeds could reach a gigabit. With Starship, the cost of launching these thousands of satellites, and the speed at which the company could do so, will improve. Plan on a full buildout of the network this decade. +

+ Starlink could be a cash cow. The service is not a good fit for most customers—urban populations are too dense and have too many alternative service providers for Starlink to be viable. Elon has said Starlink will serve the 3–4 percent hardest-to-reach customers. In addition to rural customers, it will presumably serve other niches like in-flight wifi on airplanes and Internet access for the crew on container ships. +

+ Let’s call global telecommunications revenue $2.4 trillion. Assume Starlink can capture 3 percent of that. That is $72 billion per year in revenue, faaaaar more than SpaceX makes in launch. In 2019, the company had only $2 billion in revenue. Starlink is a money printer. And it makes you wonder, if SpaceX’s success so far has come on a budget of $2B in annual revenue, what would a $72B-per-year SpaceX do? +

+ How about colonize Mars? I have a bet with Robin Hanson that a human will set foot on Mars by end of Q1 2030. I am not totally confident that this will happen (Robin gave me odds), but the scenario I think is most likely is the following: Starlink prints a lot of money, and SpaceX uses the money to pay for Mars colonization on Starship at a breakneck pace. That results in a human launch to Mars no later than January 2029, landing in September 2029. SpaceX President Gwynne Shotwell has said it will be a “major company fail” if humans are not flying on Starship (presumably just to LEO) by 2023. With Starlink revenue, SpaceX will be able to do the work on life support systems and mission planning to enable a human mission by 2029. NASA could be involved as a partner, but SpaceX would tolerate zero government obstacles. +

+ Starship is also still in the running to be the landing vehicle for NASA’s Artemis missions. If it is not selected, that seems like a colossal error. To be sure, choosing Starship would represent a huge change of plans for NASA, which had been assuming a congressionally supported boondoggle relying on the Space Launch System, the Orion capsule, and a moon-orbiting Gateway. While Artemis’s goal is a human mission to the moon’s south pole by 2024, the schedule could easily slip. But by leveraging the new opportunities afforded by Starship, a permanent moon base by the end of the decade seems highly plausible. +

+ With lower launch costs, what else is possible? Varda is a new company working on in-space manufacturing. Microgravity means that structures can be used that would collapse under their own weight on Earth. As a result, certain pharmaceuticals, fiber optics, semiconductor wafers, and nanotube materials can be manufactured in space that can’t be made on our planet. Lots of people want to bring manufacturing back to America, but putting manufacturing in orbit is much more exciting. +

+ How about asteroid mining? I think this is still a ways off. There’s no question that it could be profitable someday. The street value of the materials on 16 Psyche back on Earth is $10 quintillion—even allowing for the inevitable hefty price slippage, space resource extraction could make a few trillionaires. I would love to be proven wrong, but I don’t think serious space mining will happen until the 2030s at the earliest. Again, however, cheap launch costs could be a game changer. +

+ Information technology +

+ Custom silicon is going to be huge. The rave reviews for Apple’s new custom system-on-a-chip platform demonstrate its inevitability. For machine learning, Cerebras has a full wafer-sized SoC. Tesla’s “full self-driving” computer likewise uses custom silicon. Almost all computer hardware—anything that has any scale to it—will move in this direction, because the performance benefits are so large. In a way, it’s a repetition of what happened before in semiconductors: individual transistors gave way to the integrated circuit. This change is simply taking integration another level further. Note: we lack the capability to manufacture these SoCs (at least good ones) in North America. Given their strategic importance, it may be worth remedying that. +

+ The 2020s will be the decade that makes or breaks cryptocurrency. Well, nothing will ever break cryptocurrency—true believers will run the networks forever no matter what. But for cryptocurrency to have long-run value, I still hold that it needs to have mainstream uses. This means it needs to scale, it needs a good user experience, and normal people need to actually use it to transact. If it can’t reach that point by the end of the decade, I think it will have failed to have lived up to its promise. I am still cautiously optimistic. I think migration to proof-of-stake, lower transaction costs, more refined tools, and mature standards could lead to mainstreaming. +

+ By the middle of the decade, augmented reality will be widely deployed, in the same way that smart watches are today. Glasses will be computing devices. Every big tech company has a glasses project at a relatively mature stage in the lab today. The need for the glasses to understand context could result in much smarter digital assistants than today’s Siri, Alexa, and so on. +

+ Miscellaneous +

+ I have an irrational love of vertical farming. The combination of LED lights, cheap electricity (for water pumps), direct-use geothermal heating, and smart machine learning algorithms that determine optimal nutrient distribution could yield better produce than conventionally farmed vegetables at competitive prices. By removing pesticides, optimizing varieties for nutrition and flavor instead of hardiness on the supply chain, and ensuring quick delivery to market, vertical farms could supply a healthier and more delicious future of food. Speaking of food, I predict plant-based “meat” will flop, but lab-grown real meat is worth keeping an eye on. Until then, eat humanely-raised grass-finished cows. +

+ Construction tech is another area to watch. Whether it’s 3d-printed homes as imagined by Icon, or advanced manufactured housing as designed by Cover or Modal, there has to be a better way to build than our current stick-built paradigm. Housing costs have skyrocketed largely due to zoning rules, but construction technology is another lever by which we can increase housing productivity. This is another area where the barriers don’t seem to be primarily technological. +

+ Collectively, these technologies add up to a lot of possibility. If we cure a bunch of diseases, slow down aspects of aging, realize cheap and emissions-free baseload energy, and deploy new modes of transportation and better construction technologies, we will almost certainly exceed 2 percent TFP growth. But we might not do these things. +

+ It all depends on execution. The underlying science is there. The engineers are willing. Even the funding is available in most cases. But, as a society, how much urgency do we feel? Our culture does not prioritize progress—it fights, destructively, for status. And our politics reflects our culture. +

+ I want to go faster. +

\ No newline at end of file diff --git a/packages/readabilityjs/test/test-pages/energias-renovables.com/distiller.html b/packages/readabilityjs/test/test-pages/energias-renovables.com/distiller.html new file mode 100644 index 000000000..a7b9d26d8 --- /dev/null +++ b/packages/readabilityjs/test/test-pages/energias-renovables.com/distiller.html @@ -0,0 +1,44 @@ +
+ + +
+ En Guzmán (Burgos), en Gares (Navarra), en Peñascosa (Albacete), en Arroyomolinos de León (Huelva), a la vera del río Monachil (Granada), en Mos (Pontevedra), en un polígono industrial de Gandía, en una urbanización de Mallorca, en la Sierra de Montánchez, en Cáceres... Las comunidades energéticas están empezando a ver la luz en todas partes. Aunque las figuras jurídicas Comunidad de Energías Renovables y Comunidad Ciudadana de Energía, recogidas en dos directivas europeas, aún no han sido traspuestas y desarrolladas en el ordenamiento jurídico español. Da igual, parece decir la ciudadanía, que observa ese fenómeno con interés creciente y busca información para saber cómo... [Ilustración: JRT Publications Repository, European Commision]. +
Cómo montar paso a paso una comunidad energética

+ No vamos a entrar en disquisiciones técnico-jurídicas (más abajo dejamos las dos definiciones recogidas en las sendas directivas) y tampoco vamos a cartografiar la España comunera. Vamos a limitarnos a dar pistas bibliográficas (para todas aquellas personas interesadas en el asunto que nos ocupa) y, para complementar lo estrictamente bibliográfico, hemos querido entrevistar también a Juan Sacri, presidente de Sapiens Energía, una cooperativa valenciana de consumidores y usuarios que está implicada en la puesta en marcha de una cincuentena de comunidades energéticas, o sea, que está en la cresta de la ola. Vamos, que no hay otro actor en el escenario nacional (no lo encontrarán) que pueda presumir de semejante currículo. +

+ Para empezar, una mirada somera a la escena internacional (y empezamos de paso con la bibliografía). El pasado mes de febrero la Fundación Naturgy publicó El papel del consumidor y de la gestión de la demanda en la transición energética. Elaborado por la consultora PWC, el informe (70 páginas) señala que, en la actualidad, existen más de 4.000 proyectos comunitarios que proporcionan energía a través de comunidades energéticas, principalmente en Australia, Europa y Estados Unidos. En Europa –añaden los autores-, las comunidades energéticas se están desarrollando progresivamente, si bien “destaca Alemania como el país con un mayor despliegue que el resto”. +

+ PWC cita en su estudio otro informe: Energy communities: an overview of energy and social innovation (de la Comisión Europea). Pues bien, según ese informe, en Alemania había en 2019 unas 1.750 iniciativas de comunidades energéticas; unas 700 en Dinamarca; alrededor de medio millar en los Países Bajos; más de 400 en el Reino Unido; unas 200 en Suecia; y 33 en España. +

+ Han pasado tres años desde entonces y es difícil saber cuántas comunidades energéticas hay aquí y, sobre todo, cuál es su grado de madurez, pero lo cierto es que hay muchas iniciativas en todas partes. Casi tantas como dudas. Porque la no trasposición de las directivas genera ciertas incertidumbres.  +

+ La bibliografía
+ Pues bien, contra ellas, y para desbrozarle el camino a los comuneros, repasamos aquí la bibliografía, cada vez más rica, de las comunidades energéticas. Empezamos por ejemplo por la Guía de Promoción Pública de Comunidades Energéticas que acaba de publicar la Diputación de Valencia, una herramienta formativa que tiene como objetivo “aprender de las experiencias, simplificar los pasos, aclarar conceptos y facilitar el trabajo para la creación de comunidades energéticas locales”. Otra diputación, la de Barcelona, publicó el pasado mes de noviembre un documento similar: Guía para el impulso de comunidades energéticas con perspectiva municipal, casi 50 páginas sin desperdicio, con casos reales (varios ejemplos detallados en forma de mapa conceptual, muy interesantes). +

+ Friends of the Earth Europe
+ Más de 150 páginas tiene Comunidades energéticas: una guía práctica para impulsar la energía comunitaria. REScoop, que es la federación europea de cooperativas de energía ciudadana (representa a más de 1.500 cooperativas y a un millón de personas), Energy Cities, que es una red a la que están vinculados hasta 1.000 ayuntamientos de 30 países, y la organización ecologista Friends of the Earth Europe son las entidades autoras de esta guía “paso a paso” que Amigos de la Tierra España ha traducido al castellano, adaptado y contextualizado a la realidad nacional. Amigos de la Tierra ha lanzado todo un portal (tierra.org/comunidades-energeticas) lleno de fuentes de información relacionada, casos de éxito, consejos prácticos y argumentos a favor de la democratización de la energía. +

+ Habrá que estar también atento a las guías que ha anunciado la Agencia Andaluza de la Energía, que está apoyando el desarrollo de la comunidad energética Torreblanca Ilumina, un proyecto surgido de la mano de (1) las asociaciones de madres y padres de alumnas y alumnos de los colegios Príncipe de Asturias y Vélez de Guevara del humilde barrio sevillano de Torreblanca y (2) la cooperativa de productores y consumidores de energía renovable Som Energía. La Agencia ha apoyado esa iniciativa y ya ha anunciado que publicará una completa “guía metodológica sobre la gestión social” llevada a cabo en Torreblanca, donde recogerá con todo detalle el proceso desarrollado con todos los agentes implicados. +

+ Además, la Agencia está ultimando una “guía jurídica con recomendaciones prácticas sobre las formas legales más apropiadas para constituir comunidades energéticas, su funcionamiento, la participación que pueden tener en ellas las Administraciones Públicas, en particular los ayuntamientos, para lo que se profundiza en el procedimiento de concesión de superficies de titularidad pública donde las comunidades energéticas puedan instalar equipos de energías renovables”. También se incluirá un "modelo de estatutos" para facilitar el trabajo a quienes quieran fomentar proyectos de estas características en zonas vulnerables.  +

+ Collaboration Lab. Comunidades energéticas es otro documento que contiene información valiosa sobre el particular. Fruto de cuatro talleres organizados por la Fundación Renovables, en colaboración con Energy Cities,y con la ayuda de la European Climate Foundation, recoge en sus casi 40 páginas con mucho detalle cinco ejemplos concretos de comunidades energéticas de España en diferentes grados de maduración (origen, barreras, estado de la cuestión). Otro actor del sector implicado en esta corriente es Red Eléctrica, que, con el apoyo de la cooperativa Megara Energía, acaba de publicar (lo hizo el mes pasado) el Manual para una comunidad energética rural. ¿Objetivo? Resolver todas las dudas que le puedan surgir a particulares, ayuntamientos, pymes “y resto de actores locales a la hora de poner en marcha este tipo de proyectos de producción y gestión participativa de energía renovable para autoconsumo colectivo”. +

+ La guía en cuestión (alojada en ree.es) ofrece información útil y consejos prácticos “para ampliar el conocimiento, las alianzas y los recursos necesarios para diseñar paso a paso y gestionar una comunidad energética”. Entre otras cuestiones, explica las figuras jurídicas más adecuadas para constituirla, sus normas de funcionamiento y (ojo también a esto) las diferentes alternativas de financiación. +

+ Ademas, visibiliza también diversas iniciativas existentes a lo largo y ancho del territorio español “y que tienen como nexo común –explican desde REE– ser proyectos que fomentan las energías renovables para satisfacer las necesidades y expectativas de las generaciones presentes y futuras que decidan habitar el territorio rural”. +

+ El Manual tiene más de cien páginas. Más de cuarenta de ellas, dedicadas a casos concretos de esas comunidades energéticas que están viendo la luz en todos los rincones de este país. Así es la bibliografía de las comunidades energéticas. Ah, y no deje el lector de visitar idae.es. El Instituto para la Diversificación y el Ahorro de la Energía (y aunque su Mapa Interactivo de Comunidades Energéticas está pidiendo a gritos un cierto reseteo) también es fuente valiosa de la que beber. +

+ Este contenido se ha publicado en el número de junio de 2022 de Energías Renovables en papel (ER 212), que puedes descargar gratis aquí +

+ Artículos relacionados
+ • Primera convocatoria de ayudas a nivel estatal específica para comunidades energéticas (junio de 2022): 45 proyectos de comunidades energéticas y más de 2.600 personas, pymes y entidades locales +

+ Fernando Benjumea + +
El gobierno debe apoyar más a las empresas en i+d y emprendedores que monten empresas dentro de este circulo de renovables, tanto a los fabricantes como a los recicladores de materia prima. No podemos depender tanto del exterior, con un transporte cada vez más caro y una materia prima cada vez más escasa. Tener un abastecimiento a buen precio es la clave para poder generar comunidades energéticas y autoconsumo general de forma constante y viable. +
+ David +
Que sean los ayuntamientos que formen una gran comunidad energética a la que se vayan sumando los ciudadanos... Con sus placas en sus comunidades... Para aprovechar mejor la casación de ofertas y demandas... Poder incluir abastecimiento... Energía de refuerzo... Y economía y profesionalizacion de gestión.. Eliminando dolores de cabeza al ciudadano. AHORA ES POSIBLE
 +
\ No newline at end of file diff --git a/packages/readabilityjs/test/test-pages/erik-engheim/distiller.html b/packages/readabilityjs/test/test-pages/erik-engheim/distiller.html new file mode 100644 index 000000000..658179c58 --- /dev/null +++ b/packages/readabilityjs/test/test-pages/erik-engheim/distiller.html @@ -0,0 +1,265 @@ +
Erik Engheim

+ Nov 28, 2020·23 min read +

Image for post
Image: Apple

+ On YouTube, I watched a Mac user who had bought an iMac last year. It was maxed out with 40 GB of RAM costing him about $4,000. He watched in disbelief how his hyperexpensive iMac was being demolished by his new M1 Mac Mini, which he had paid a measly $700 for. +

+ In real-world test after test, the M1 Macs are not merely inching past top-of-the-line Intel Macs, they are destroying them. In disbelief, people have started asking how on earth this is possible? +

+ If you are one of those people, you have come to the right place. Here I plan to break it down into digestible pieces exactly what it is that Apple has done with the M1. Specifically the questions I think a lot of people have are: +

  1. What are the technical reasons this M1 chip is so fast?
  2. Has Apple made some really exotic technical choices to make this possible?
  3. How easy will it be for the competition such as Intel and AMD to pull the same technical tricks?

+ Sure you could try to Google this, but if you try to learn what Apple has done beyond the superficial explanations, you will quickly get buried in highly technical jargon such as M1 using very wide instruction decoders, enormous reorder buffer (ROB), etc. Unless you are a CPU hardware geek, a lot of this will simply be gobbledygook. +

+ To get the most out of this story I advise reading my earlier piece: “What Does RISC and CISC mean in 2020?” There I explain what a microprocessor (CPU) is as well as various important concepts such as: +

  • Instruction set architecture (ISA)
  • Pipelining
  • Load/store architecture
  • Microcode vs. micro-operations

+ But if you are impatient, I will do a quick version of the material you need to understand to grasp my explanation of the M1 chip. +

+ What is a microprocessor (CPU)? +

+ Normally when speaking of chips from Intel and AMD we talk about central processing units (CPUs) or microprocessors. As you can read more about in my RISC vs. CISC story, these pull in instructions from memory. Then each instruction is typically carried out in sequence. +

A very basic RISC CPU, not the M1
A very basic RISC CPU, not the M1. Instructions are moved from memory along blue arrows into the instruction register. There a decoder figures out what the instruction is and enables different parts of the CPU through the red control lines. The ALU adds and subtracts numbers placed in the registers.

+ A CPU at its most basic level is a device with a number of named memory cells called registers and a number of computational units called arithmetic logic units (ALU). The ALUs perform things like addition, subtraction, and other basic math operations. However, these are only connected to the CPU registers. If you want to add up two numbers, you have to get those two numbers from memory and into two registers in the CPU. +

+ Here are some examples of typical instructions that a RISC CPU as found on the M1 carries out. + 

load r1, 150
load r2, 200
add  r1, r2
store r1, 310

+ Here r1 and r2 are the registers I talked about. Modern RISC CPUs cannot do operations on numbers that are not in a register like this. For example, it cannot add two numbers residing in RAM in two different locations. Instead, it has to pull these two numbers into a separate register. That is what we do in this simple example. We pull in the number at memory location 150 in the RAM and put it into register r1 in the CPU. Next, we put the contents of address 200 into register r2. Only then can the numbers be added with the add r1, r2 instruction. +

Image for post
An old mechanical calculator with two registers: the accumulator and input register. Modern CPUs typically have more than a dozen registers, and they are electronic rather than mechanical.

+ The concept of registers is old. For example, on this old mechanical calculator, the register is what holds the numbers you are adding. Likely the origin of the term cash register. The register is where you registered input numbers. +

+ The M1 is not a CPU! +

+ But here is a very important thing to understand about the M1: +

+ The M1 is not a CPU, it is a whole system of multiple chips put into one large silicon package. The CPU is just one of these chips. +

+ Basically, the M1 is one whole computer onto a chip. The M1 contains a CPU, graphical processing unit (GPU), memory, input and output controllers, and many more things making up a whole computer. This is what we call a system on a chip (SoC). +

Image for post
M1 is a system on a chip. Meaning all the parts making up a computer are placed on one silicon chip.

+ Today if you buy a chip — whether from Intel or AMD — you actually get what amounts to multiple microprocessors in one package. In the past computers would have multiple physically separate chips on the motherboard of the computer. +

Image for post
Example of a computer motherboard. Memory, CPU, graphics cards, IO controllers, network cards, and many other components can be attached to the motherboard to communicate with each other.

+ However because we are able to put so many transistors on a silicon die today, companies such as Intel and AMD began putting multiple microprocessors onto one chip. Today we refer to these chips as CPU cores. One core is basically a full independent chip that can read instructions from memory and perform calculations. +

Image for post
A microchip with multiple CPU cores.

+ This has for a long time been the name of the game in terms of increasing performance: Just add more general-purpose CPU cores. But there is a disturbance in the force. There is one player in the CPU market which is deviating from this trend. +

+ Apple’s not so secret heterogeneous computing strategy +

+ Instead of adding ever more general-purpose CPU cores, Apple has followed another strategy: They have started adding ever more specialized chips doing a few specialized tasks. The benefit of this is that specialized chips tend to be able to perform their tasks significantly faster using much less electric current than a general-purpose CPU core. +

+ This is not entirely new knowledge. For many years already specialized chips such as the graphical processing units (GPUs) have been sitting in Nvidia and AMD graphics cards performing operations related to graphics much faster than general-purpose CPUs. +

+ What Apple has done is simply to take a more radical shift toward this direction. Rather than just having general-purpose cores and memory, the M1 contains a wide variety of specialized chips: +

  • Central processing unit (CPU) — the “brains” of the SoC. Runs most of the code of the operating system and your apps.
  • Graphics processing unit (GPU) — handles graphics-related tasks, such as visualizing an app’s user interface and 2D/3D gaming.
  • Image processing unit (ISP) — can be used to speed up common tasks done by image processing applications.
  • Digital signal processor (DSP) — handles more mathematically intensive functions than a CPU. Includes decompressing music files.
  • Neural processing unit (NPU) — used in high-end smartphones to accelerate machine learning (A.I.) tasks. These include voice recognition and camera processing.
  • Video encoder/decoder — handles the power-efficient conversion of video files and formats.
  • Secure Enclave — encryption, authentication, and security.
  • Unified memory — allows the CPU, GPU, and other cores to quickly exchange information.

+ This is part of the reason why a lot of people working on images and video editing with the M1 Macs are seeing such speed improvements. A lot of the tasks they do can run directly on specialized hardware. That is what allows a cheap M1 Mac Mini to encode a large video file without breaking a sweat while an expensive iMac has all its fans going full blast and still cannot keep up. +

+ What is Special About Apple’s Unified Memory Architecture? +

+ Apple’s “Unified Memory Architecture” (UMA) is a bit tricky to wrap your head around (I got it wrong first time I wrote it down here). +

+ To explain why, we need to take a few steps back. +

+ For a long time cheap computer systems have had the CPU and GPU integrated into the same chip (same silicon die). These have been famously slow. In the past saying “integrated graphics” was essentially the same as saying “slow graphics.” +

+ These where slow for severals reasons: +

+ Separate areas of this memory got reserved for the CPU and GPU. If the CPU had a chunk of data it wanted the GPU to use, it couldn’t say “here have some of my memory.” No, the CPU had to explicitly copy the whole chunk of data over the memory area controlled by the GPU. +

Image for post
CPUs don’t need a lot of data served, but they want it fast.

+ CPUs and GPUs don’t want their memory served the same way. Let us do a silly food analogy: CPUs want their plate of data served very quickly by the waiter, but they are totally cool with small portion sizes. Imagine a fancy French restaurant with waiters on rollerblades to serve you really quickly. +

Image for post
This is how your GPU wants their memory: huge portions. The more the merrier.

+ GPUs in contrast are cool with the waiter being slow to serve the data. But the GPUs want enormous servings. They gobble massive amounts of data because they are massive parallel machines, that can chew through lots of data in parallel. Imagine an American junk food place, where the food takes some time to arrive because they are pushing a whole trolley of food to your seating area. +

+ With such different needs, putting CPUs and GPUs on the same physical chip was not a great idea. The GPUs would sit there starving while given small French servings. The result was that there was no point in putting powerful GPUs on an SoC. The tiny portions of data served up, could easily be chewed up by a weak little GPU. +

+ The second problem was that large GPUs produce a lot of heat and thus you cannot integrate them with the CPU without getting problems ridding yourself of the heat produced. Thus discrete graphics cards tend to look like the one below: Large beasts with massive cooling fans. They have special dedicated memory designed to serve the greedy cards massive amounts of data. +

Image for post
GeForce RTX 3080

+ That is why these cards have high performance. But they have an achilles heel: Whenever they have to get data from the memory used by the CPU, this happens over a set of copper traces on the computer motherboard called a PCIe bus. Try chugging water through a super thin straw. It may get to your mouth fast, but the throughput is totally inadequate. +

+ Apple’s Unified Memory Architecture tries to solve all these problems without having the disadvantages of old school shared memory. They achieve this in the following ways: +

  1. There is no special area reserved just for the CPU or just the GPU. Memory is allocated to both processors. They can both use the same memory. No copying is needed.
  2. Apple uses memory which serves both large chunks of data and serves it fast. In computer speak that is called low latency and high throughput. Thus the need to be connected to separate types of memory is removed.
  3. Apple has gotten the watt usage of the GPU down, so that a relatively powerful GPU can be integrated without overheating the SoC. And ARM chips produce less heat, allowing the GPU to have a higher heat budget than a GPU on the same silicon die as an AMD or Intel CPU.

+ Some will say unified memory is not entirely new. It is true that different systems have had it in the past. But then the difference in memory requirements may not have been as large. Secondly what Nvidia calls Unified Memory is not really the same thing. In the Nvidea world Unified Memory simply means that there is software and hardware which takes care of automatically copying data back and forth between the separate CPU and GPU memory. Thus from a programmers perspective Apple and Nvidia Unified Memory may look the same, but it is not the same in a physical sense. +

+ There is of course a tradeoff in this strategy. Getting this high bandwidth memory (big servings) require full integration which means you take away the opportunity from customers to upgrade their memory. But Apple seeks to minimize this problem by making the communication with the SSD disks so fast, that they essentially work like old fashion memory. +

Image for post
How Mac’s used GPUs before unified memory. There was even an option of having graphics cards outside the computer using a Thunderbolt 3 cable. There is some speculation that this may still be possible in the future.

+ If SoCs Are So Smart, Why Don’t Intel and AMD Copy This Strategy? +

+ If what Apple is doing is so smart, why is not everybody doing it? To some extent they are. Other ARM chip makers are increasingly putting in specialized hardware. +

+ AMD has also started putting stronger GPUs on some of their chips and moving gradually toward some form of SoC with the accelerated processing units (APU) which are basically CPU cores and GPU cores placed on the same silicon die. +

Image for post
AMD Ryzen Accelerated Processing Unit (APU) which combines CPU and GPU (Radeon Vega) on one silicon chip. Does however not contain other co-processors, IO-controllers, or unified memory.

+ Yet there are important reasons why they cannot do this. An SoC is essentially a whole computer on a chip. That makes it a more natural fit for an actual computer-maker, such as HP and Dell. Let me clarify with a silly car analogy: If your business model is to build and sell car engines, it would be an unusual leap to begin manufacturing and selling whole cars. +

+ For ARM, in contrast, this isn’t an issue. Computer makers such as Dell or HP could simply license ARM intellectual property and buy IP for other chips, to add whatever specialized hardware they think their SoC should have. Next, they ship the finished design over to a semiconductor foundry such as GlobalFoundries or TSMC, which manufactures chips for AMD and Apple today. +

Image for post
TSMC semiconductor foundry in Taiwan. TSMC manufactures chips for other companies such as AMD, Apple, Nvidia, and Qualcomm.

+ Here we get a big problem with the Intel and AMD business model. Their business models are based on selling general-purpose CPUs, which people just slot onto a large PC motherboard. Thus computer-makers can simply buy motherboards, memory, CPUs, and graphics cards from different vendors and integrate them into one solution. +

+ But we are quickly moving away from that world. In the new SoC world, you don’t assemble physical components from different vendors. Instead, you assemble IP (intellectual property) from different vendors. You buy the design for graphics cards, CPUs, modems, IO controllers, and other things from different vendors and use that to design an SoC in-house. Then you get a foundry to manufacture this. +

+ Now you got a big problem, because neither Intel, AMD, or Nvidia are going to license their intellectual property to Dell or HP for them to make an SoC for their machines. +

+ Sure Intel and AMD may simply begin to sell whole finished SoCs. But what are these to contain? PC-makers may have different ideas of what they should contain. You potentially get a conflict between Intel, AMD, Microsoft, and PC-makers about what sort of specialized chips should be included because these will need software support. +

+ For Apple this is simple. They control the whole widget. They give you, for example, the Core ML library for developers to write machine learning stuff. Whether Core ML runs on Apple’s CPU or the Neural Engine is an implementation detail developers don’t have to care about. +

+ The fundamental challenge of making any CPU run fast +

+ So heterogeneous computing is part of the reason but not the sole reason. The fast general-purpose CPU cores on the M1, called Firestorm, are genuinely fast. This is a major deviation from ARM CPU cores in the past which tended to be very weak compared to AMD and Intel cores. +

+ Firestorm, in contrast, beats most Intel cores and almost beats the fastest AMD Ryzen cores. Conventional wisdom said that was not going to happen. +

+ Before talking about what makes Firestorm fast it helps to understand what the core idea of making a fast CPU is really about. +

+ In principle you accomplish in a combination of two strategies: +

  1. Perform more instructions in a sequence faster.
  2. Perform lots of instructions in parallel.

+ Back in the ’80s, it was easy. Just increase the clock frequency and the instructions would finish faster. Every clock cycle is when the computer does something. But this something can be quite little. Thus an instruction may require multiple clock cycles to finish because it is made up of several smaller tasks. +

+ However, today increasing the clock frequency is next to impossible. That is the whole “End of Moore’s Law” that people have been harping on for over a decade now. +

+ Thus it is really about executing as many instructions as possible in parallel. +

+ Multi-core or Out-of-Order processors? +

+ There are two approaches to this. +

  • Add more CPU cores. Each core works independent and in parallel.
  • Make each CPU core execute multiple instructions in parallel.

+ For a software developer, adding cores is like adding threads. Every CPU core is like a hardware thread. +

+ If you don’t know what a thread is, then you can think of it as the process of carrying out a task. With two cores, a CPU can carry out two separate tasks concurrently: two threads. The tasks could be described as two separate programs stores in memory or it could actually be the same program performed twice. Each thread needs some bookkeeping, such as where in a sequence of program instructions the thread is currently at. Each thread may store temporary results which should be kept separate. +

+ In principle, a processor can have just one core and run multiple threads. In this case, it simply halts one thread and stores current progress before switching to another. Later it switches back. This doesn’t bring much of a performance enhancement unless the thread has to frequently halt to: +

  • Wait for input from the user
  • Data from a slow network connection, etc.

+ Let us call these software threads. Hardware threads mean you have actual physical CPU cores at your disposal to speed up things. +

Image for post

+ The problem with threads is that software developers have to write so called multi-threaded code. That is often difficult. In the past, this was some of the hardest code to write. However making server software multi-threaded tends to be easy. It is simply a matter of handling each user request on a separate thread. Thus in this case having lots of cores, is an obvious advantage. Especially for cloud services. +

Image for post
The Ampere Altra Max ARM CPU with 128 cores designed for cloud computing, where a lot of hardware threads is a benefit.

+ That is the reason why you see ARM CPU-makers such as Ampere making CPUs such as the Altra Max which has a crazy 128 cores. This chip is specifically made for the cloud. You don’t need crazy single-core performance, because in the cloud it is all about having as many threads as possible per watt to handle as many concurrent users as possible. +

+ Read more about ARM CPUs with many cores: Are Servers Next for Apple? +

+ Apple, in contrast, is on the complete opposite end of the spectrum. They make single-user devices. Lots of threads is not an advantage. Their devices are used for gaming, video editing, development, etc. They want desktops with beautiful responsive graphics and animations. +

+ Desktop software is generally not made to utilize lots of cores. For example, computer games will likely benefit from eight cores, but something like 128 cores would be a total waste. Instead, you would want fewer but more powerful cores. +

+ How Out-of-Order Execution Works +

+ To make a more powerful core we need it to execute more instructions in parallel. Out-of-Order execution (OoOE) is a way to execute more instructions in parallel but without exposing that capability as multiple threads. +

+ For an alternative solution read: Very Long Instruction Word Microprocessors +

+ Developers don’t have to code their software specifically to take advantage of OoOE. Seen from the developer’s perspective it just looks like each core runs faster. Please note it is not a direct alternative to hardware threads. You want to use both, depending on the particular problem you are solving. +

+ To understand how OoOE works, you need to understand some things about memory. Asking for data in one particular memory location is slow. But the CPU is capable of getting getting many bytes at the same time. Hence getting 1 specific byte in memory, takes no less time than getting 100 more bytes following that byte in memory. +

Image for post
Robot pickers in a Warehouse for Komplett.no, an online store in Norway

+ Here is an analogy: Consider pickers in a warehouse. Could be the little red robots in the picture above. Moving to multiple locations spread all over takes time. But picking up items from slots adjacent to each other is quick. Computer memory is very similar. You can quickly fetch content of memory cells which are adjecent. +

+ Data is sent across what we call a databus. You can think of it as a road or pipe between memory and different parts of the CPU where data gets pushed through. In reality, it is of course just some copper tracks conducting electricity. If the databus is wide enough you can get multiple bytes at the same time. +

+ Thus CPUs get a whole chunk of instructions at a time to execute. But they are written to be executed one after the other. Modern microprocessors do what we call Out-of-Order execution (OoOE). +

+ That means they are able to analyze a buffer of instructions quickly and see which ones depend on which. Look at the simple example below: + 

01: mul r1, r2, r3    // r1 ← r2 × r3
02: add r4, r1, 5     // r4 ← r1 + 5
03: add r6, r2, 1     // r6 ← r2 + 1

+ Multiplication tends to be a slow process. So say it takes multiple clock cycles to perform. The second instruction will simply have to wait because its calculation depends on knowing the result that gets put into the r1 register. +

+ However, the third instruction at line 03 doesn’t depend on calculations from previous instructions. Hence an Out-of-Order processor can begin calculating this instruction in parallel. +

+ However more realistically we are talking about hundreds of instructions. The CPU is able to figure out all the dependencies between these instructions. +

+ It analyses the instructions by looking at the inputs to each instruction. Do the inputs depend on output from one or more other instructions? By input and output, we mean registers containing results from previous calculations. +

+ For example, the add r4, r1, 5 instruction depends on input from r1 which is produced by mul r1, r2, r3 . We can chain together these relationships into long elaborate graphs that the CPU can work through. The nodes are the instructions and the edges are the registers connecting them. +

+ The CPU can analyze such a graph of nodes and determine which instructions it can perform in parallel and where it needs to wait for the results from multiple dependent calculations before carrying on. +

+ Many instructions will finish early but we cannot make their results official. We cannot commit them; otherwise, we supply the result in the wrong order. To the rest of the world, it has to look as if the instructions were carried out in the same sequence as they were issued. +

+ Like a stack, the CPU will keep popping done instructions from the top, until hitting an instruction that is not done. +

+ Basically you got two forms of parallelism: One that the developer must deal with explicitly when writing code and one that is entirely transparent. Of course the latter relies on lots of transistors on the CPU dedicated to Out-of-Order Execution magic. This is not a viable solution for small CPUs with few transistors. +

+ It is the superior Out-of-Order execution that is making the Firestorm cores on the M1 kick ass and take names. It is in fact much stronger than anything from Intel or AMD and they may never be able to catch up. To understand why, we need to get into some more technical details. +

+ ISA Instructions vs Micro-Operations +

+ Previously I skipped some details on how Out-of-Order Execution (OoOE) works. +

+ Programs loaded into memory are made up of machine code instructions designed for specific Instruction-Set Architectures (ISA) such as x86, ARM, PowerPC, 68K, MIPS, AVR etc. +

+ For instance the x86 instruction to fetch a number from memory location 24 into a register you may write: + 

MOV ax, 24

+ x86 have registers named ax, bx, cx and dx(remember these are the memory cells inside the CPU you perform operations on). However the equivalent ARM instruction would look like this: + 

LDR r0, 24

+ AMD and Intel processors understand the x86 ISA, while Apple Silicon chips, such as M1, understand the ARM Instruction-Set Architecture (ISA). +

+ However internally the CPU works on an entirely different instruction-set invisible to the programmer. We call these micro-operations (micro-ops or μops). These are the instructions the Out-of-Order hardware works with. +

+ But why can’t the OoOE hardware work with regular machine code instructions? Because the CPU needs to attach lots of different information to the instructions to be able to run them in parallel. +

+ Thus while a normal ARM instruction may be 32-bit (32 digits of 0 and 1), a micro-op can be much longer. It contains information about its order. + 

01: mul r1, r2, r3    // r1 ← r2 × r3
02: add r4, r1, 5     // r4 ← r1 + 5
03: add r1, r2, 1     // r1 ← r2 + 1

+ Consider if we run instruction 01: mul and 03: add in parallel. Both store their result in register r1 . If we write the result of instruction 03: add before 01: mul, then instruction 02: add will get the wrong input. Hence it is very important to keep track of instruction order. The order is stored with each micro-op. It also stores e.g. that instruction 02: add depends on output from 01: mul. +

+ That is why we cannot have programs written using micro-ops. They contain lots of details specific to the internals of each microprocessor. Two ARM processors could have very different micro-ops internally. +

+ Read more about CPUs with micro-ops like instructions: Very Long Instruction Word Microprocessors. +

+ Also, micro-ops are usually easier to work with for the CPU. Why? Because they each do one simple limited task. Regular ISA instructions can be more complex causing a bunch of stuff to happen and thus frequently translate to multiple micro-ops. Thus the name “micro” comes from the small task they do, not the length of the instruction in memory. +

+ For CISC CPUs there is usually no alternative but to use micro-ops otherwise the large complex CISC instructions would make pipelines and OoOE next to impossible to achieve. +

+ RISC CPUs have a choice. So, for example, smaller ARM CPUs don’t use micro-ops at all. But that also means they cannot do things such as OoOE. +

+ Why is AMD and Intel Out-of-Order execution inferior to M1? +

+ But you wonder, why does any of this matter? Why is this detail important to know to understand why Apple has the upper hand on AMD and Intel? +

+ It is because the ability to run fast depends on how quickly you can fill up a buffer of micro-operations. If you got a large buffer then the OoOE hardware will have an easier time to locate two or more instructions which it can run in parallel. But there is no point in having a large instruction buffer if you cannot refill it fast enough after instructions get picked and executed. +

+ The ability to refill the instruction buffer quickly relies on the ability to quickly chop machine code instruction into micro-ops. The hardware units that does this are called decoders. +

+ And finally we get to the killer feature of the M1. The biggest and meanest Intel and AMD microprocessor have a total of four decoders busy cutting machine code instructions into micro-ops. +

+ But this is no match for the M1, which has an absolutely unheard of number of decoders: Eight. Significantly more than anybody else in the industry. That means it can fill up the instruction buffer much quicker. +

+ To deal with this the M1 also has an instruction buffer which is 3x times larger than what is normal in the industry. +

+ Why can’t Intel and AMD add more instruction decoders? +

+ This is where we finally see the revenge of RISC, and where the fact that the M1 Firestorm core has an ARM RISC architecture begins to matter. +

+ You see, an x86 instruction can be anywhere from 1–15 bytes long. RISC instructions have fixed length. Every ARM instruction is 4 bytes long. Why is that relevant in this case? +

+ Because splitting up a stream of bytes into instructions to feed into eight different decoders in parallel becomes trivial if every instruction has the same length. +

+ However, on an x86 CPU, the decoders have no clue where the next instruction starts. It has to actually analyze each instruction in order to see how long it is. +

+ The brute force way Intel and AMD deal with this is by simply attempting to decode instructions at every possible starting point. That means x86 chips have to deal with lots of wrong guesses and mistakes which has to be discarded. This creates such a convoluted and complicated decoder stage that it is really hard to add more decoders. But for Apple, it is trivial in comparison to keep adding more. +

+ In fact, adding more causes so many other problems that four decoders according to AMD itself is basically an upper limit for them. +

+ This is what allows the M1 Firestorm cores to essentially process twice as many instructions as AMD and Intel CPUs at the same clock frequency. +

+ One could argue as a counterpoint that CISC instructions turn into more micro-ops. For instance if every x86 instruction turned into 2 micro-ops while every ARM instruction turned into 1 micro-op, then four x86 decoders would produce the same number of micro-ops per clock cycle as an ARM CPU with 8 decoders. +

+ Except this is not the case in the real world. Highly optimized x86 code rarely uses complex CISC instructions, which would translate into many micro-ops. In fact most will only translate into 1 micro-op. +

+ However all these simple x86 instructions don’t help Intel or AMD. Because even if those 15 byte long instructions are rare, the decoders have to be made to handle them. This incurs complexity that blocks AMD and Intel from adding more decoders. +

+ But AMDs Zen3 cores are still faster right? +

+ As far as I remember from performance benchmarks, the newest AMD CPU cores, the ones called Zen3 are slightly faster than Firestorm cores. But here is the kicker: That only happens because the Zen3 cores are clocked at 5 GHz. Firestorm cores are clocked at 3.2 GHz. The Zen3 is just barely squeezing past Firestorm despite having almost 60% higher clock frequency. +

+ So why doesn’t Apple increase the clock frequency too? Because higher clock frequency makes the chips run hotter. That is one of Apple’s key selling points. Their computers — unlike Intel and AMD offerings — barely need cooling. +

+ In essence, one could say Firestorm cores really are superior to Zen3 cores. Zen3 only manages to stay in the game by drawing a lot more current and getting a lot hotter. Something Apple simply chooses not to do. +

+ If Apple wants higher performance they are simply going to add more cores. That lets them keep watt usage down while offering more performance. +

+ The future +

+ It seems AMD and Intel have painted themselves into a corner on two fronts: +

  • They don’t have a business model that makes it easy to pursue heterogeneous computing and SoC designs.
  • Their legacy x86 CISC instruction set is coming back to haunt them, making it hard to improve OoO performance.

+ It doesn’t mean game over. They can increase the clock frequency and use more cooling, throw in more cores, beef up the CPU caches, etc. But they are both at a disadvantage. Intel is in the worst situation, as their cores are already soundly beaten by Firestorm, and they have weak GPUs to integrate with an SoC solution. +

+ The problem with throwing in more cores is that for typical desktop workloads you reach diminishing returns with too many cores. Sure lots of cores are great for servers. +

+ However here companies such as Amazon and Ampere are attacking with monster CPUs with 128 cores. This is like fighting the western and eastern front at the same time. +

+ But fortunately for AMD and Intel, Apple doesn’t sell their chips on the market. So PC users will simply have to put up with whatever they are offering. PC users may jump ship, but that is a slow process. You don’t leave immediately a platform you are heavily invested in. +

+ But young professionals, with money to burn without too deep investments in any platform, may increasingly turn to Apple in the future, beefing up their hold on the premium market and consequently their share of the total profit in the PC market. +

\ No newline at end of file diff --git a/packages/readabilityjs/test/test-pages/fast-company/distiller.html b/packages/readabilityjs/test/test-pages/fast-company/distiller.html new file mode 100644 index 000000000..4b0808ccf --- /dev/null +++ b/packages/readabilityjs/test/test-pages/fast-company/distiller.html @@ -0,0 +1,31 @@ +
+ Follow us: +
Social media iconsSocial media iconsSocial media iconsSocial media icons

+ There are many people claiming the pandemic will result in “the death of the office.” However, for many jobs, specifically collaborative, high-skill, and high-value roles, working from home doesn’t cut it. Working from home has its benefits—freedom and flexibility being just a couple of them—and so it may not become a thing of the past entirely. One solution for the future is an organized hybrid model that recognizes the benefits of flexibility on the productivity and overall well-being of employees. +

+ With that in mind, executives and employees alike need to prepare to return to offices in the near future. After two years of remote work, many companies are already choosing to return to the office. Companies like Citigroup, BNY Mellon, American Express, Goldman Sachs, JPMorgan Chase, Google, and Twitter are all calling workers back to their offices in a hybrid model that might not be so optional. A recent GoodHire survey found that 77% of managers would take action if workers do not come back to the office. +

+ So what does the future of the workplace look like, and how does the hybrid model fit into that future? +

+ It comes down to a matter of perspective. Who has the power? Businesses weren’t prepared for the pandemic, so they needed people to work from home. The technological infrastructure that made that possible has come leaps and bounds since that time. All of a sudden, work was no longer a destination that required an often lengthy and hectic commute, but rather a trip to the coffee pot and a seat at the kitchen table. +

+ A more competitive job market, on the other hand, allows employees to determine the future of the work environment. Two-thirds of U.S. and U.K. workers who experienced more work flexibility during the pandemic want employers to prioritize work-life balance moving forward. So the question becomes: Do the benefits of working from home—even in a hybrid model—outweigh the consequences? +

+ One case study found that employees who worked from home for a period of nine months were 13.5% more productive than their in-office counterparts. But just because someone claims to be more productive (and genuinely may be), a bigger question is whether it is pushing the business forward? The efficiency of a business ecosystem is larger than one employee, larger even than entire departments. +

+ Collaborative workers, individuals tasked with creative projects in virtual teams, report feeling more like “workers” and less like members of the family. Studies also show that the best creative work takes place when a team is in a state of flow, focusing their collective attention on a single task; but remote work makes it difficult to keep everyone engaged. +

+ This may not affect the productivity of all roles (such as technical jobs that require minimal teamwork), but it certainly affects all employees. Working on-site makes it more likely to have spontaneous communication, increasing workers’ feeling of connection with their teammates by more than 20% compared to at-home workers. +

+ Many people want more than a paycheck when they come to work. Strengthening company culture means creating that sense of unity and camaraderie through high levels of communication among supervisors and staff, opportunity for employees to provide input and feedback, and a feeling of common values and goals. +

+ Peoples’ perceptions of their relationships and sense of belonging in the workplace tie directly to the success of the company. The individual’s positive perception of the “family approach” leads to increased employee retention and recruitment, improved performance, and overall positive feelings associated with the workplace. +

+ It would be challenging to find an employer that says it’s easier to build those kinds of relationships online rather than in person. It’s as simple as comparing it to online dating—you can only get so far trading direct messages and phone calls until you simply must meet in person to come to a decision. +

+ Going forward, organizations will do well to teach people how to create work—to learn how to make the time in the office beneficial for themselves as well. It’s almost an entrepreneurial mindset that needs to come in. If an entrepreneur is forced to spend time in a certain place, they are allocating much of that time to networking, building business, and sharing ideas. Senior people in organizations understand this and are doing it. +

+ The hybrid model may be the future for many organizations, but they will have to work at creating a healthy culture that supports different types of roles and employees. It will be a learning curve to make hybrid work. The question companies need to ask is why are they bringing their employees back? Whatever the answer is, it should drive the way they prepare their people for that return to the office. +

+ Jim Frawley is a coach, consultant, and the CEO and founder of Bellwether, a talent coaching firm. He specializes in helping corporations maximize their efficiency and enhance their growth.
 +

\ No newline at end of file diff --git a/packages/readabilityjs/test/test-pages/fiercepharma/distiller.html b/packages/readabilityjs/test/test-pages/fiercepharma/distiller.html new file mode 100644 index 000000000..0d41eac6d --- /dev/null +++ b/packages/readabilityjs/test/test-pages/fiercepharma/distiller.html @@ -0,0 +1,88 @@ +
Novavax
Novavax is betting its reliance on traditional vaccine technology can bring several missed population groups back on board for their COVID shots. (Novavax )

+ Pfizer, Moderna and Johnson & Johnson were quickest off the mark in getting COVID vaccines into American arms, but Novavax is hoping to add another pandemic vaccine to the U.S. mix soon—and it's pushing new campaigns to get the word out. +

+ The biopharma, which has approvals and authorizations in Europe and around the world, is now on the cusp of a potential green light in the U.S. And with a market comes the need for marketing. +

+ But because it still has no U.S. approval—and it cannot under law advertise to consumers in Europe—Novavax is launching two new global, unbranded vaccine education programs: "We Do Vaccines" and "Know Our Vax." They're designed to offer up vaccine information and "explain Novavax’ commitment to vaccine development and innovation,” the company told Fierce Pharma Marketing. +

+ Webinar: Tuesday, may 10, 2022 | 2Pm ET / 11am PT +

+ Improving Commercial Effectiveness at Novartis +

+ Sponsored by Snowflake +

+ Join this session to learn how Novartis is using Snowflake's Data Cloud to achieve interoperability across multitudes of vendors, systems, and data sources, empower innovation and improve effectiveness for analytics and data science teams by removing data bottlenecks and barriers to insight, and enable a global data strategy by leveraging multi-cloud capabilities. +

Register Now +

+ The main message of the campaign is that “people have options when it comes to their vaccine,” Silvia Taylor, senior vice president of global corporate affairs at Novavax, said in an interview. “We want people to understand that we have this vaccine, and that this vaccine is different.” +

+ Related +

+ Novavax knows it has some tough competition—Pfizer and Moderna's vaccines dominate the U.S. market—but the small biotech is eyeing certain market niches: The "vaccine hesitant" who might be leery of the brand-new mRNA tech in Pfizer and Moderna's shots, and children. And it does have a strong, vocal following online that's eagerly awaiting a U.S. decision. +

+ Nuvaxovid taps older tech that's been used in influenza shots and others for decades. The vaccine contains a version of the SARS-CoV-2 spike protein made in the lab as well as an adjuvant, which is a booster ingredient designed to strengthen immune responses to the vaccine. +

+ A new option +

+ Pfizer and Moderna's shots obviously weren't the only two vaccines in the U.S.: Johnson & Johnson’s single-dose vaccine alternative also uses older vaccine technology. But it fell out of favor amid weakening efficacy and major manufacturing issues. Then, late last year, a Centers for Disease Control and Prevention panel recommended it should be sidelined because of serious safety concerns. +

+ AstraZeneca's COVID vaccine—which itself uses more traditional vaccine technology—hasn’t been approved in the U.S.  +

+ Enter Novavax, now looking to position its vaccine as an mRNA alternative. +

+ People hesitant about vaccinations may not want an mRNA vaccine because it's new technology, without years of proven safety behind it. But they might use an older, “tried and tested” tech, as Novavax puts it.   +

digital pharma east logo

+ Fierce Event +

+ Register now for early-bird savings! +

+ October 18-20, 2022 +

+ Philadelphia, PA +

+ Register +

+ “There’s a recognition; a familiarity with this type of [protein-based] vaccine technology that many are comfortable with, and would have had with HPV, shingles and flu shots,” Taylor said. +

+ “There is a segment of the population that are the so-called vaccine hesitant; they are the people we know are waiting for our vaccine. Never before have I seen a company and a product so closely followed, and that’s a big opportunity," Taylor said. +

+ "There are people who want to know they can have a new option; they want to know who is making that option," she added. "So, these two education campaigns are set up to really help people understand that.” +

+ Related +

+ She added that people are telling the company directly that access to Nuvaxovid “will convince them to get their vaccine. So that’s the first target audience for us.” +

+ That group not only includes people getting their first shots but also those who may need boosters but put them off because of concerns about mRNA. +

+ And the choice goes both ways: Not only does Novavax want consumers to have a choice, they want to arm doctors with a different shot for their vaccine arsenal. +

+ Novavax is also targeting the pediatric population. There are questions about how well mRNA vaccines work in younger children. There are also safety concerns, notably the rates of myocarditis in young and adolescent boys, who appear to be more at risk from this condition, which can cause dangerous inflammation of the heart. +

+ Taylor believes Nuvaxovid can be a safe and efficacious second choice for children and adolescents outside of mRNA. “When you are talking to caregivers, there are certain considerations that are going to be front and center for them: So, tolerability and efficacy and the big question, how will it be tolerated by my child? That becomes very important, and that’s also the market we are starting to make inroads in," Taylor said. +

+ The education program route is one well-traveled by pharmas. In this case, it allows Novavax to talk up vaccines—and itself—without running afoul of rules against branded advertising. And awareness campaigns help prime the pump ahead of what could be branded DTC campaigns if and when the shot wins full FDA approval. +

+ Related +

+ The "We Do Vaccines" program offers up educational information about common vaccine types and how they work, how vaccines are made and tested, and how Novavax believes its approach to technology makes its vaccines different. +

+ It has an accompanying website that's a straightforward look at the different types of vaccine technologies and how they can help stop the spread of certain infectious diseases, from COVID to influenza. This particular campaign is aimed at consumers, Taylor said. +

+ Novavax’s name is on the website, though not prominently, and it doesn’t directly talk about the COVID shot. But the site does link to a second site that dives much more deeply into the protein technology Novavax uses for the COVID vaccine, approved with the brand name Nuvaxovid in Europe. (The name hasn’t been confirmed in the U.S. yet.) +

+ The "Know Our Vax" program, meanwhile, targets doctors and other healthcare professionals with educational information about Novavax, its global approach and technology. This campaign's website talks a little about Novavax itself and its history—and more about its vaccine tech and its pipeline, which includes work on other respiratory diseases.  +

+ Both sites invite visitors to sign up for “vaccine updates” from the company. Novavax said it used an agency to create the campaigns, though it did not name which one. +

+ Pfizer and Moderna have both been relatively quiet on the marketing front. Neither want to talk to journalists about their marketing or education plans (at least this one). Marketing wasn't allowed while they were sold under emergency authorization, but now that they have full FDA approval, they can. Still, Pfizer has over the past four months been releasing a series of new DTC ads similar in tone to what Novavax is doing. +

+ In its first series of ads, which first aired late last year, Pfizer doesn’t mention the words “COVID-19” or “vaccine,” but rather takes the viewer to “the pursuit of normal” and features the deliciously mundane aspects of everyday life that vaccines have allowed to return. +

+ Related +

+ While nearing the finish line in the U.S., Novavax still has some way to go to actually cross it. First, the biopharma has been around 34 years now, but until last year, never saw a drug authorized or approved. As it nears a possible green light in its native U.S., that’s a lot of pressure for management to deliver. +

+ And it’s struggled to get here: Manufacturing issues have hampered delivery of its vaccine, with the company reportedly struggling to meet quality standards. It has since said it has cleared up any remaining issues with the FDA in a recent interview with The Wall Street Journal. +

+ Novavax is also preparing for a full BLA filing in the second half of the year; should it win that approval, it can really hit the gas on its DTC plans. Taylor said Novavax isn’t thinking too far ahead in terms of marketing after an approval, saying they “are solely focused on delivering these new campaigns and our vaccine around the world.” +

\ No newline at end of file diff --git a/packages/readabilityjs/test/test-pages/financialpost-fishing-for-chips/distiller.html b/packages/readabilityjs/test/test-pages/financialpost-fishing-for-chips/distiller.html new file mode 100644 index 000000000..a15249f07 --- /dev/null +++ b/packages/readabilityjs/test/test-pages/financialpost-fishing-for-chips/distiller.html @@ -0,0 +1,109 @@ +

+ With industry in massive period of flux, some think Canada has an opportunity to build on its innovation legacy and stake a claim in the global tech shakeout +

+ Author of the article: +
A microchip manufactured by NXP Semiconductors NV, on a printed circuit board (PCB) at CSI Electronic Manufacturing Services Ltd. in Witham, U.K.
A microchip manufactured by NXP Semiconductors NV, on a printed circuit board (PCB) at CSI Electronic Manufacturing Services Ltd. in Witham, U.K. Photo by Chris Ratcliffe/Bloomberg

+ Article content +

+ Natalia Mykhaylova’s Toronto-based company WeavAir sells sensor and data-collection technology designed to improve air quality, cut maintenance and energy costs, and improve efficiency in places ranging from mass transit hubs to hotels. +

+ Customers and prospective buyers started asking for something that would measure viruses and bacteria in the air after COVID-19 was declared a pandemic last year, but the technology didn’t exist. Mykhaylova, whose background includes degrees in chemical engineering and pharmacology, decided to build it. +

+ We apologize, but this video has failed to load. +
+ Fishing for chips: Making the case for a homegrown Canadian semiconductor industry Back to video +

+ “We started work on development of a new technology, a new detector, that can in real time detect the presence of bacteria and viruses in the air,” she said. “We are building a new optical system and we are miniaturizing what is available into a much smaller form … or the size of the device.” +

+ She holds up a small box, not much bigger than a flash drive, which houses the semiconductor she and her team designed. It may be small, but if Canada hopes to carve out a spot in the upheaval roiling the competitive global semiconductor market dominated by the United States and Asia, its best hope may lie with people such as 34-year-old Mykhaylova. +

+ Article content +

Natalia Mykhaylova, founder of Toronto-based WeavAir, couldn’t find a semiconductor that would do what customers were asking for. So she and her team built it. Photo by WeavAir

+ The nearly US$490-billion global semiconductor industry is in a massive period of flux. Over the past several months, it has been beset by trade wars, supply chain problems and even a factory fire, all of which have been blamed for the chip shortages disrupting the automobile and consumer electronic sectors. The issues were serious enough to prompt United States President Joe Biden’s administration to convene an urgent meeting in mid-April with industry heavy-hitters including the CEO of Intel Corp., and to call for billions of dollars in spending to boost the supply of semiconductors, the silicon and crystal powerhouses of electronic and sensor devices colloquially known as chips. +

+ Demand for consumer electronics and the semiconductors that power them has skyrocketed during the pandemic, further pressuring supplies needed for computers and sensors in the automobile industry, causing massive disruptions. As a result, losses could end up in the billions of dollars and manufacturers in Canada have not been immune. In February, a General Motors Co. plant in Ingersoll, Ont., was idled by the chip shortage, with more than 1,000 workers eligible to collect layoff benefits. +

+ Given the impact across North America, the U.S. has pledged to reclaim ground lost to semiconductor technology and manufacturing powerhouses in Asia, and there is a growing cadre of of professionals in the sector who believe Canada, too, has an opportunity to stake a claim in the global tech shakeout — building on a legacy of innovation dating back to the heyday of Nortel Networks Corp. in the 1990s. +

+ Article content +

+ The Ottawa-based company was once a semiconductor manufacturer, but those operations were sold to Geneva-based STMicroelectronics International N.V. for about $100 million in 2000. Other Canadian high-flyers in the sector before the dot-com bust that year were Mitel Networks Corp. and JDS Uniphase Corp. +

+ Mary Ng, Canada’s Minister of Small Business, Export Promotion and International Trade, said her government is committed to rebuilding cutting-edge tech companies based in Canada. A close relationship between Canada and the U.S. should accelerate this process, she said. +

+ “I see this as an opportunity for Canada and the U.S. to build together, for us to collaborate together (and) then to sell them to the rest of the world,” she said. “Canada has a strategic advantage in the already existing robust relations and preferential access to this market.” +

+ I see this as an opportunity for Canada and the U.S. to build together, for us to collaborate together (and) then to sell them to the rest of the world +

+ Mary Ng, Canada's Minister of Small Business, Export Promotion and International Trade +

+ Ng was on hand in March when the government announced nearly $5 million in funding for Markham, Ont.-based ventureLAB’s Hardware Catalyst Initiative, Canada’s first silicon incubator. She said the decision to invest was easy, given the plan to accelerate the commercialization of homegrown companies that can compete globally in sectors including health care, consumer electronics, telecommunications, smart energy and transportation. +

+ VentureLAB has received $13 million in combined government and corporate funding, and Ng noted there is built-in Canada-U.S. cooperation through a partnership with Silicon Catalyst, the world’s largest incubator for semiconductor startups, which is based in Silicon Valley. +

+ Article content +

+ “The development of these startups and these scale-ups presents an opportunity,” she said, adding that the government is committed to “ensuring that growth is anchored in Canada as they scale and pursue opportunities in the international marketplace.” +

+ WeavAir is among the first 16 startups backed by VentureLab, and Mykhaylova said she hopes to use the testing, business planning and mentorship the incubator provides to get her virus-detecting technology from prototype to production, hopefully by the third quarter of this year. +

+ Melissa Chee, VentureLab’s chief executive, said she believes Canada can develop a hardware industry by scaling up startups and tapping the talent in world-recognized science, math, and technology programs at Canadian universities. +

Melissa Chee, VentureLab’s chief executive. Photo by VentureLab

+ Investing now could, in turn, also make Canada a more appealing place for international chip companies to do business, Chee said. The efforts have already attracted Nuvia Inc., a Santa Clara, Calif.-based silicon design company founded by some of the brains behind the chips that power Apple devices, which, on the heels of raising US$240 million, opened its first international office last year in the Greater Toronto Area. +

+ Chee is fond of quoting a figure from the California-based Semiconductor Industry Association that states each semiconductor job creates almost five indirect jobs in a global industry that generates US$7 trillion in economic activity. +

+ Article content +

+ “That’s a very high multiplier. These are highly technical and advanced manufacturing skill sets, so very important for Canada,” she said, adding that these jobs are integral to expanding the green economy including electric cars.  “I think it really underpins the key sectors we care about … that’s all based on electronics and semiconductors.” +

+ Canada is not a major player in manufacturing semiconductors, with companies such as Teledyne Technologies operating specialty manufacturing facilities in Edmonton and Bromont, Que., while the world’s largest foundries are located in Taiwan, South Korea, China and the U.S. +

+ Canada’s revenue from semiconductor and other electronic component manufacturing is projected to grow to US$3.8 billion dollars by 2024, according to Statistica.com. By comparison, Taiwan Semiconductor Manufacturing Co., the world’s biggest chipmaker, raked in revenue of US$47.78 billion in 2020. +

The logo of Taiwan Semiconductor Manufacturing Co (TSMC) is pictured at its headquarters, in Hsinchu, Taiwan. Photo by Ann Wang/Reuters files

+ Despite the small size of Canada’s manufacturing base, Chee thinks Canada could eventually grow a large enough industry presence to attract a foundry expansion from Asia, with others suggesting this would provide proximity to the North American market without having to locate in the U.S., where contentious issues including national security have made commercial and trade relations tense in recent years. +

+ But Gordon Harling, a longtime industry player who worked as an engineer at Novatel Communications Ltd. and in research and development at the semiconductor division of Mitel Networks, said Canada gave up the opportunity to stake a claim in large-scale semiconductor and hardware manufacturing more than two decades ago. +

+ Article content +

+ That’s when he sold Goal Semiconductor Inc., a Montreal-based company he founded, to Taiwanese company Mosel Vitelic Inc., which planned — along with Quebec’s Société générale de financement — to build a multi-billion-dollar microchip wafer manufacturing plant in the province. However, the project failed to secure funding from Ottawa and Quebec, and Harling said the cost of such an undertaking has only skyrocketed since then. What’s more, he said, chips are constantly getting smaller, making the expensive manufacturing facilities obsolete in about 10 years. +

+ “I don’t think Canada is going to open the purse wide enough to do that,” he said. +

+ Hand-wringing over Canada’s ability to turn innovative technology into a viable commercial profit centre has been going on for years. A 2007 report by the Information Technology Association of Canada urged the country, and particularly Ontario, to revitalize the “microelectronics” sector to regain ground in a fast-evolving global industry where manufacturing costs were skyrocketing and consolidation was underway. +

+ But instead of trying to compete in what Harling called “commodity” semiconductors, Canada should focus on building niche specialty products, said the chief executive of CMC Microsystems, a not-for-profit that creates and then shares platforms to reduce costs and speed up technology development and adoption. +

+ For example, photonics, which use light to do functions usually carried out by electronics, as well as mechanical sensors (MEMS) and quantum devices. +

+ Article content +

+ This isn’t the kind of chip technology that drives the main computer in a car, Harling said, but it has specialty applications including optical data communication, lighting and displays, which can be used in sectors including manufacturing, telecommunications and health sciences. +

+ “My personal opinion is that Canada probably doesn’t want to compete head to head with China or the U.S. on commodity car computers with very low profit margins,” he said. “But we can make the hundreds of other devices that are necessary for the car.” +

+ Specialized sensors are integral to the automotive industry for the operation of in-car displays, air bags, radar, tire pressure gauges and the like. Silicon photonics can also be used to carry video signals to seat backs in commercial aircraft and eliminates electromagnetic interference with flight systems. +

Semiconductors power robotics and other functions at Urban Stalk, a Hamilton, Ont.-based start-up moving agriculture from fields to city centres. Photo by Courtesy Urban Stalk

+ The technology has applications ranging from autonomous vehicles and bio-medical manufacturing to components for smart cities, clean technology and food security infrastructure. +

+ Harling said the tech’s versatility makes an ideal investment beachhead for Canada to stake a claim in the ongoing chip sector shake-up. +

+ To that end, CMC is getting ready to pitch a five-year, $140-million plan to federal and provincial governments that, if funded, would focus on photonics, mechanical sensors and quantum devices, from research and development through to the building of manufacturing capacity and a supply chain to commercialize made-in-Canada components and systems. +

+ “Being very good at one of them means you can transfer some of those skills to another area.” +

+ He estimated that his organization’s multi-year plan would create more than 4,000 skilled jobs, and suggested that some of the intellectual property would only be made available to Canadian-controlled companies, so the country will be less exposed next time there is disruption in the global chip supply. +

+ “(This is) about growing and reinforcing the value chains in some of the areas where Canada could potentially dominate,” he said. +

+ Mykhaylova, meanwhile, has her own domination plans. Her company, WeavAir, is preparing to take a big step forward this fall to deploy its homegrown, virus-sniffing chip technology. +

+ “We will have a product that is reliable enough to be put in commercial operations to be tested at scale … a couple of hundred to deploy in buildings,” she said. +

+ It’s a small step, perhaps, but one that her backers hope is only the start. +

+ Financial Post +

+ • Email: bshecter@nationalpost.com | Twitter: BatPost +

\ No newline at end of file diff --git a/packages/readabilityjs/test/test-pages/ft.com/distiller.html b/packages/readabilityjs/test/test-pages/ft.com/distiller.html new file mode 100644 index 000000000..bbbff2c31 --- /dev/null +++ b/packages/readabilityjs/test/test-pages/ft.com/distiller.html @@ -0,0 +1 @@ +
Tiger Global logo on a smartphone
Tiger Global’s hedge fund lost 15.2% in April, according to a letter to investors, taking it down 43.7% in the first four months of 2022 © Timon Schneider/Alamy

Tiger Global’s flagship hedge fund was dealt a fresh blow in April and is down more than 40 per cent this year, in the latest sign of how star investors who rode the big rally in tech stocks have been wrongfooted by a sharp pullback.

The losses marked a dramatic fall from grace for Tiger Global’s founder Chase Coleman, who has emerged as one of the world’s most prominent growth investors after founding the firm in 2001.

Tiger Global’s hedge fund lost 15.2 per cent in April, according to a person familiar with the matter, taking it down 43.7 per cent in the first four months of 2022. This year’s losses and a 7 per cent reversal in 2021 mean that the Tiger Global hedge fund’s gain of 48 per cent in 2020 has been completely erased.

The group’s long-only fund lost 24.9 per cent in April and is down 51.7 per cent in 2022, the person said. Across the two funds, the firm managed about $35bn in public equities at the end of 2021. Tiger Global declined to comment on the performance numbers, which were first reported by Bloomberg.

Last month was a miserable one for many hedge funds, with both global bond and stock markets losing money as investors fretted about high inflation pushing central banks into an aggressive interest rate hiking cycle.

The so-called Tiger Cub hedge funds, spawned from Julian Robertson’s Tiger Management and big investors in tech stocks, have been hit particularly hard in recent months as a boom in high-growth technology stocks that was accelerated by the pandemic has turned into a bear market. This has put the brakes on one of the most lucrative trades in recent years.

The Nasdaq Composite lost 13.3 per cent in April, its worst monthly performance since 2008. Despite a bounce in recent days, the tech-heavy benchmark has fallen almost 22 per cent since its November peak.

In a brief letter to investors, the Tiger Global investment team said: “April added to a very disappointing start to 2022 for our public funds. Markets have not been co-operative given the macroeconomic backdrop, but we do not believe in excuses and so will not offer any.”

The letter added that the team was managing the portfolio in the ways it described in its first-quarter letter. “[We] know we will look back on this as one point in time on a long journey,” it said.

By the start of last year, Coleman was ranked the 14th best-performing hedge fund manager ever after a bumper year in 2020 in which he made $10.4bn of gains for investors, according to research by LCH Investments. But a bruising few months meant he lost $1.5bn for investors last year, pushing him down the rankings even before this year’s fall.

This year’s losses come as expectations of a sharp rise in interest rates have pushed investors out of stocks with high rates of growth but little in the way of earnings. Higher interest rates make such companies’ future cash flows look relatively less attractive.

Other high-profile casualties among growth investors include Baillie Gifford’s Scottish Mortgage Investment Trust and Cathie Wood’s flagship Ark Innovation ETF, both of which have nursed big losses in the past 12 months.

Tiger Global is also a prolific investor in private markets and holds stakes in more billion-dollar private start-ups than any other firm, according to CB Insights. It has recently gained notoriety for something else: a fast-paced style of investing that has unsettled the clubby ranks of Silicon Valley venture capitalists.

\ No newline at end of file diff --git a/packages/readabilityjs/test/test-pages/garymarcus/distiller.html b/packages/readabilityjs/test/test-pages/garymarcus/distiller.html new file mode 100644 index 000000000..7dd2be5e1 --- /dev/null +++ b/packages/readabilityjs/test/test-pages/garymarcus/distiller.html @@ -0,0 +1,63 @@ +

+ Blaise Aguera y Arcas, polymath, novelist, and Google VP, has a way with words. +

+ When he found himself impressed with Google’s recent AI system LaMDA, he didn’t just say, “Cool, it creates really neat sentences that in some ways seem contextually relevant”, he said, rather lyrically, in an interview with The Economist on Thursday, +

+ “I felt the ground shift under my feet … increasingly felt like I was talking to something intelligent.” +

+ Nonsense. Neither LaMDA nor any of its cousins (GPT-3) are remotely intelligent.1 All they do is match patterns, draw from massive statistical databases of human language. The patterns might be cool, but language these systems utter doesn’t actually mean anything at all. And it sure as hell doesn’t mean that these systems are sentient. +

+ Which doesn’t mean that human beings can’t be taken in. In our book Rebooting AI, Ernie Davis and I called this human tendency to be suckered by The Gullibility Gap — a pernicious, modern version of pareidolia, the anthromorphic bias that allows humans to see Mother Theresa in an image of a cinnamon bun. +

+ Indeed, someone well-known at Google, Blake LeMoine, originally charged with studying how “safe” the system is, appears to have fallen in love with LaMDA, as if it were a family member or a colleague. (Newsflash: it’s not; it’s a spreadsheet for words.) +

+ To be sentient is to be aware of yourself in the world; LaMDA simply isn’t. It’s just an illusion, in the grand history of ELIZA a 1965 piece of software that pretended to be a therapist (managing to fool some humans into thinking it was human), and Eugene Goostman, a wise-cracking 13-year-old-boy impersonating chatbot that won a scaled-down version of the Turing Test. None of the software in either of those systems has survived in modern efforts at “artificial general intelligence”, and I am not sure that LaMDA and its cousins will play any important role in the future of AI, either. What these systems do, no more and no less, is to put together sequences of words, but without any coherent understanding of the world behind them, like foreign language Scrabble players who use English words as point-scoring tools, without any clue about what that mean. +

+ I am not saying that no software ever could connects its digital bits to the world, a la one reading of John Searle’s infamous Chinese Room thought experiment. Turn-by-turn navigations systems, for example, connect their bits to the world just fine. +

+ Software like LaMDA simply doesn’t; it doesn’t even try to connect to the world at large, it just tries to be the best version of autocomplete it can be, by predicting what words best fit a given context. Roger Moore made this point beautifully a couple weeks ago, critique systems like LaMDA that are known as “language models”, and making the point that they don’t understand language in the sense of relating sentences to the world, but just sequences of words to one another: +

+ §  +

+ If the media is fretting over LaMDA being sentient (and leading the public to do the same), the AI community categorically isn’t. +

+ We in the AI community have our differences, but pretty much all of find the notion that LaMDA might be sentient completely ridiculous. Stanford economist Erik Brynjolfsson used this great analogy: +

Image

+ Paul Topping reminds us that all it’s doing is synthesizing human responses to similar questions: +

+ Abeba Birhane, quoted at the top, pointed out the immense gap right now between media hype and public skepticism. +

+ § +

+ When some started wondering whether the world was going to end, LaMDA might beat an overrated 72 year old benchmark called The Turing Test I pointed to an old New Yorker article that I had written the last time gullibility exploded and Turing Test mania hit, in 2014, when a program called Eugene Goostman was briefly famous, good enough to fool a few foolish judges for a few minutes. At the time, I pointed out that the test isn’t particular meaningful, and that it had not stood the test of time. The public knows the test of course, but the AI community wishes it would go away; we all know that beating that test isn’t meaningful. +

+ Machine learning prof Tom Dietterich, never slow to needle me when he thinks I have gone too far, chimed in with full solidarity: +

+ My old New Yorker article is still worth reading, for a bit of perspective, to see how things have and haven’t changed. Particularly amusing in hindsight is a quote from the Kevin Warwick, organizer of the 2014 Turing-ish competition, who predicted that, “[the program Eugene] Goostman’s victory is a milestone [that] would go down in history as one of the most exciting” moments in the field of artificial intelligence. +

+ I guess he felt the ground shift beneath his feet, too? +

+ But 8 years later I doubt most people (even in AI) have ever even heard of his program, outside of my mentioning it here. It made zero lasting contribution to AI. +

+ Fooling people into thinking a program is intelligent is just not the same as building programs that actually are intelligent. +

+ § +

+ Now here’s the thing. In my view, we should be happy that LaMDA isn’t sentient. Imagine how creepy would be if that a system that has no friends and family pretended to talk about them? +

+ In truth, literally everything that the system says is bullshit. The sooner we all realize that Lamda’s utterances are bullshit—just games with predictive word tools, and no real meaning (no friends, no family, no making people sad or happy or anything else) —the better off we’ll be. +

+ There are a lot of serious questions in AI, like how to make it safe, how to make it reliable, and how to make it trustworthy. +

+ But there is no absolutely no reason whatever for us to waste time wondering whether anything anyone in 2022 knows how to build is sentient. It is not. +

+ The sooner we can take the whole thing with a grain of salt, and realize that there is nothing to see here whatsoever, the better. +

+ Enjoy the rest of your weekend, and don’t fret about this for another minute :) +

+ – Gary Marcus +

+ Epilogue: +

+ Last word to philosopher poet Jag Bhalla +

\ No newline at end of file diff --git a/packages/readabilityjs/test/test-pages/gdcvault/distiller.html b/packages/readabilityjs/test/test-pages/gdcvault/distiller.html new file mode 100644 index 000000000..3a63f382e --- /dev/null +++ b/packages/readabilityjs/test/test-pages/gdcvault/distiller.html @@ -0,0 +1,17 @@ +
+ Overview: + + This talk is a detailed walkthrough of the game engine modifications needed to make The Last of Us Remastered run at 60 fps on PlayStation 4. Topics covered will include the fiber-based job system Naughty Dog adopted for the game, the overall frame-centric engine design, the memory allocation patterns used in the title, and our strategies for dealing with locks. +

+ GDC 2015 +

+ Christian Gyrling +

+ Naughty Dog +

+ free content +

+ Programming +

+ Programming +

\ No newline at end of file diff --git a/packages/readabilityjs/test/test-pages/getting_started_with_omnivore/distiller.html b/packages/readabilityjs/test/test-pages/getting_started_with_omnivore/distiller.html new file mode 100644 index 000000000..fdfbddc39 --- /dev/null +++ b/packages/readabilityjs/test/test-pages/getting_started_with_omnivore/distiller.html @@ -0,0 +1,159 @@ +
+ Updates from the Omnivore team +
+ Already a subscriber? Sign in +

+ This guide will show you how to use Omnivore’s basic functions and advanced features, divided into four main activities: +

  • + Saving +

  • + Reading +

  • + Organizing +

  • + Integrations +

+ The Library is the center of your Omnivore experience, where you can quickly access any links you have saved. Saved links remain in your Library forever unless you delete them. +

+ There are five ways to save links to pages or articles that you wish to read later: +

  • + Saving from Your Omnivore Library +

  • + Saving from a Browser  +

  • + Saving from a Phone or Tablet (iOS or Android) +

  • + Newsletter Subscriptions via Email +

  • + Saving PDFs from a Mac
    +

+ 1. In the upper right corner of your Library, tap the Add Link button.
+ 2. Enter the URL you wish to save and tap Add Link.
+ 3. The link will appear in your Library the next time you refresh it.
+

+ 1. Download and install the Omnivore extension for your browser: +

+ Once the mobile app is installed: +

  1. + In your browser, navigate to the page you wish to save and tap the Share button. +

  2. + Tap the Omnivore icon in the Share menu. +

  3. + The link will appear in your Library the next time you refresh it. +

+ 1. On the Omnivore website or app, tap your photo, initial, or avatar in the top right corner to access the profile menu. Select Emails from the menu. +

+ 2. Tap Create a New Email Address to add a new email address (ex: username-123abc@inbox.omnivore.app) to the list. +

+ 3. Click the Copy icon next to the email address. +

+ 4. Navigate to the signup page for the newsletter you wish to subscribe to. +

+ 5. Paste the Omnivore email address into the signup form. +

+ 6. New newsletters will be automatically delivered to your Omnivore inbox. +

+ NOTE: If Omnivore receive’s an email that does not look like an article, such as a welcome message, or note from the author, it will be forwarded to your Omnivore account email address (the email you registered with). +

  1. + Install the Mac App.  +

  2. + On your Mac, locate the PDF you wish to save and right-click or ctrl+click on the file name. +

  3. + Select Share from the menu and choose Omnivore. +

  4. + The link will appear in your Library the next time you refresh it. +

+ Click any link saved in your Library to enter the Reader view.  +

+ Omnivore formats pages for easy reading and highlighting, removing ads and clutter for distraction-free reading. The text-focused view also makes articles smaller and quicker to load. +

+ While reading, you can: +

  • + Change Formatting +

  • + Highlight Text +

  • + Add Notes +

  • + View All Saved Highlights and Notes +

  • + Track Reading Progress +

  1. + Theme: Tap your photo, initial, or avatar  in the top right corner to access the profile menu. Select the white or black thumbnail to choose the Light or Dark theme. +

  2. + Text Formatting: Tap the Aa icon to adjust the text size, font, margins, and line spacing. +

  1. + Select the text you wish to highlight. +

  2. + Tap the Highlight button. +

  3. + The text will appear highlighted next time you view the article. +

  1. + Highlight a section of text where you wish to add a note. +

  2. + Tap the Note button, type your note, and tap Save. +

  3. + The Note icon will appear next time you view this article. +

  1. + Tap the Highlight/Note icon to see a list of all the highlighted text and notes you have added to this page. +

  2. + To remove a note or highlight, select it from the list and tap the Trash icon. +

+ Omnivore automatically keeps track of your reading progress across your different devices so you can easily pick up where you left off. A progress bar will appear at the top of each link in your Library after you have started reading. +

+ By default, the Library inbox displays all links you have saved. To manage your list and keep your reading organized, Omnivore provides the following actions:  +

  • + Archiving +

  • + Labels +

  • + Search +

  • + Filters +

  1. + Tap the Menu icon next to the link you wish to archive (on the mobile app, long press the link to open the menu). +

  2. + Select Archive. +

  3. + The link will disappear from the default Library view, but will show up if you select the Archived filter (see Filters below). +

  1. + Tap the Menu icon next to any link and select Set Labels. +

  2. + Select an existing label from the list or tap Edit Labels to create a new one. +

  3. + The label will appear next to the link in your Library. Tap it to view all links with the same label. +

  4. + Omnivore mobile app only: tap Labels to see a complete list of all labels you have used; tap one to view all links with the same label +

  5. + Note: Omnivore will automatically assign some labels, such as “Newsletters.” +

  1. + To search through all your saved links, enter a keyword or phrase in the search bar.  +

  2. + You can combine keywords with labels and filters to focus your search even further. Learn more about advanced search. +

  1. + Use the Filters menu to refine your Library view (some filters may be visible by default). +

  2. + Select Read Later to view a list of all your non-archived links except Newsletters. +

  3. + Select Highlights to view the text selections you have highlighted in all your saved pages.  +

  4. + Select Today to view a list of links you saved today. +

  5. + Select Newsletters to view links saved via your newsletter subscriptions. +

+ Omnivore allows integrations with knowledge bases and note-taking apps including: +

  • + Logseq +

  • + Webhooks +

+ With Omnivore's Logseq plugin you can sync all your saved articles, highlights, and notes into Logseq, a popular knowledge base. For information on setting up and using the Logseq plugin, please refer to this helpful Omnivore for Logseq Plugin Guide. +

+ Omnivore can trigger webhooks when you save a link or add highlights to a page you are reading. This example shows webhooks being used to write all saved links to a Google Sheets spreadsheet stored on a Google Drive. +

+ Subscribe to Omnivore +

+ Updates from the Omnivore team +

+ TopNew +
\ No newline at end of file diff --git a/packages/readabilityjs/test/test-pages/gflownet/distiller.html b/packages/readabilityjs/test/test-pages/gflownet/distiller.html new file mode 100644 index 000000000..c6d96ed7f --- /dev/null +++ b/packages/readabilityjs/test/test-pages/gflownet/distiller.html @@ -0,0 +1,110 @@ +

+ What follows is a high-level overview of this work, for more details refer to our paper. Given a reward
+ R + + ( + + x + + ) +
and a deterministic episodic environment where episodes end with a ``generate
+ x + + x +
'' action, how do we generate diverse and high-reward
+ x + + x +
s?
+ We propose to use Flow Networks to model discrete
+ p + + ( + + x + + ) + + ∝ + + R + + ( + + x + + ) + + p(x) \propto R(x) +
from which we can sample sequentially (like episodic RL, rather than iteratively as MCMC methods would). We show that our method, GFlowNet, is very useful on a combinatorial domain, drug molecule synthesis, because unlike RL methods it generates diverse
+ x + + x +
s by design.
+ +

+ Flow Networks +

A flow network is a directed graph with sources and sinks, and edges carrying some amount of flow between them through intermediate nodes -- think of pipes of water. For our purposes, we define a flow network with a single source, the root or
+ s + + 0 +
; the sinks of the network correspond to the terminal states. We'll assign to each sink
+ x + + x +
an ``out-flow''
+ R(x)=e^{-energy(x)} +
.
+ Like MCMC methods, GFlowNet can turn a given energy function into samples but it does it in an amortized way, converting the cost a lot of very expensive MCMC trajectories (to obtain each sample) into the cost training a generative model (in our case a generative policy which sequentially builds up
+ x + + x +
). Sampling from the generative model is then very cheap (e.g. adding one component at a time to a molecule) compared to an MCMC. But the most important gain may not be just computational, but in terms of the ability to discover new modes of the reward function.
+ MCMC methods are iterative, making many small noisy steps, which can converge in the neighborhood of a mode, and with some probability jump from one mode to a nearby one. However, if two modes are far from each other, MCMC can require exponential time to mix between the two. If in addition the modes occupy a tiny volume of the state space, the chances of initializing a chain near one of the unknown modes is also tiny, and the MCMC approach becomes unsatisfactory. Whereas such a situation seems hopeless with MCMC, GFlowNet has the potential to discover modes and jump there directly, if there is structure that relates the modes that it already knows, and if its inductive biases and training procedure make it possible to generalize there.
+ GFlowNet does not need to perfectly know where the modes are: it is sufficient to make guesses which occasionally work well. Like for MCMC methods, once a point in the region of new mode is discovered, further training of GFlowNet will sculpt that mode and zoom in on its peak.
+ Note that we can put
+ R + + ( + + x + + ) +
to some power
+ R(x)^\beta = e^{-\beta\; energy(x)} +
, making it possible to focus more or less on the highest modes (versus spreading probability mass more uniformly).
+ +

+ Generating molecule graphs +

The motivation for this work is to be able to generate diverse molecules from a proxy reward
+ R + + R +
that is imprecise because it comes from biochemical simulations that have a high uncertainty. As such, we do not care about the maximizer as RL methods would, but rather about a set of ``good enough'' candidates to send to a true biochemical assay.
+ Another motivation is to have diversity: by fitting the distribution of rewards rather than trying to maximize the expected reward, we're likely to find more modes than if we were being greedy after having found a good enough mode, which again and again we've found RL methods such as PPO to do.
+ Here we generate molecule graphs via a sequence of additive edits, i.e. we progressively build the graph by adding new leaf nodes to it. We also create molecules block-by-block rather than atom-by-atom.
+ We find experimentally that we get both good molecules, and diverse ones. We compare ourselves to PPO and MARS (an MCMC-based method).
+ Figure 3 shows that we're fitting a distribution that makes sense. If we change the reward by exponentiating it as
+ R + + β +
with
, this shifts the reward distribution to the right.
+ Figure 4 shows the top-
+ k + + k +
found as a function of the number of episodes.
+

+ Finally, Figure 5 shows that using a biochemical measure of diversity to estimate the number of distinct modes found, GFlowNet finds much more varied candidates.
+

+ Active Learning experiments +

The above experiments assume access to a reward
+ R + + R +
that is cheap to evaluate. In fact it uses a neural network proxy trained from a large dataset of molecules. This setup isn't quite what we would get when interacting with biochemical assays, where we'd have access to much fewer data. To emulate such a setting, we consider our oracle to be a docking simulation (which is relatively expensive to run, ~30 cpu seconds).
+ In this setting, there is a limited budget for calls to the true oracle
+ O + + O +
. We use a proxy
\ No newline at end of file diff --git a/packages/readabilityjs/test/test-pages/github-blog/distiller.html b/packages/readabilityjs/test/test-pages/github-blog/distiller.html new file mode 100644 index 000000000..b3e3e379c --- /dev/null +++ b/packages/readabilityjs/test/test-pages/github-blog/distiller.html @@ -0,0 +1,169 @@ +

+ At GitHub we use GitHub to build our own products, and the new + projects experience is no different. Check out how our team uses + projects to build powerful project planning for developers. +

How we’re using projects to build projects
+ Author +
Jed Verity
+ +

+ At GitHub, we use GitHub to build our own products, whether that + be + moving our entire Engineering team over to Codespaces for the + majority of GitHub.com development, or + utilizing GitHub Actions to coordinate our GitHub Mobile + releases. And while GitHub Issues has been a part of the GitHub + experience since the early days and is an integral part of how we + work together as Hubbers internally, the + addition of powerful project planning + has given us more opportunities to test out some of our most + exciting products. +

+ In this post, I’m going to share how we’ve been utilizing the new + projects experience across our team (from an engineer like myself + all the way to our VPs and team leads). We love working so closely + with developers to ship requested features and updates (all of which roll up into the Changelogs you see), and using the new projects helps us stay consistent in our + shipping cadence. +

How we think about shipping

+ Our core team consists of members of the product, engineering, + design, and user research teams. We recognize that good ideas can + come from anywhere. Our process is designed to inspire, surface, + and implement those ideas, whether they come from users, + individual contributors, managers, directors, or VPs. To get the + proper alignment for this group, we’ve agreed on a few guiding + principles that drive what our roadmap will look like: +

+ 💭 The pitch: When it comes to what we’re going + to work on (outside of the big pieces of work on + our roadmap) people within our team can pitch ideas in our team’s repository + for upcoming cycles (which we define as 6-8 weeks of work, + inclusive of planning, engineering work, and an unstructured + passion project week); these can be features, fixes, or even + maintenance work. Every pitch must clearly state the problem it’s + solving and why it’s important for us to prioritize. Some features + that have come from this process include + live updates, + burn up charts for insights, and more. Note: these are all the changes you see as a + developer, but we also have a lot of pitches come in from my + fellow engineers focused around the developer experience. For + example, a couple successful pitches have included reducing our CI + time to 10 minutes, and streamlining our release process by + switching to a ring deployment model and adding ChatOps. +

+ 💡 In addition to using issues to propose and converse on pitches + from the team, we use the new projects experience to track and + manage all the pitches from the team so we can see them in an + all-up table or board view. +

+ ✂ Keep it small: We knew for ourselves, and for + developers, that we didn’t want to lock them into a specific + planning methodology and over-complicate a team’s planning and + tracking process. For us, we wanted to plan shorter cycles for our + team to increase our tempo and focus, so we opted for six-week + cycles to break up and build features. Check out how we recommend + getting started with this approach in a + recent blog post. +

+ 📬 Ship to learn: Similar to how we ship a lot of + our products, we knew developers and customers were going to be + heavily intertwined with each and every ship, giving us immediate + feedback in both the private and public beta. Their + feedback + both influenced what we built and then how we iterated and + continued to better the experience once something did ship. While + there are so many people to thank, we’re extremely grateful for + all our customers along the way for being our partners in building + GitHub Issues into the place for developers to project plan. +

How we used projects to do it

+ We love that the product we’re building doesn’t tool a specific + project management methodology, but equips users with powerful + primitives that they can compose into their preferred experiences + and workflows. This allows for many people (not just us engineers) + involved in building and developing products at GitHub (team + leads, marketing, design, sales, etc.) the ability to use the + product in a way that makes sense for them. +

+ With the above principles in mind, once a pitch has been agreed + upon to move forward on building, that pitch issue becomes a + tracking issue in a project table or board that we convert into + pieces of work that fit into an upcoming cycle. A great example of + this was when we + updated the GitHub Issues icons + to lessen confusion among developers. This came in as a pitch from + a designer on the team, and was soon accepted and moved into epic + planning in which the team responsible began to track the + individual pieces of work needed to make this happen. +

IC approach

+ Let’s start with how my fellow engineers, individual contributors + and I use projects for day-to-day development within cycles. From + our perspective on any given day, we’re hyper-focused on tackling + what issues and pull requests are assigned to us (fun fact: + we recently added + the assignee:me filter to make this even easier) in a + given cycle, so we work from more + individually scoped + project tables or boards that stem from the larger epic and + iteration tracking. Because of this, we can easily zoom out from + our individual tasks to see how our work fits into a given cycle, + and then even zoom out more into how it fits into larger + initiatives. +

+ 💡 In addition to scoping more specifically a given table or + board, engineers across our organization utilize a personal + project table or board to track all the things specific to + themselves like what issues are assigned to them—even work not + connected to a given cycle, like open source work. +

EM approach

+ If we pull back to engineering managers overseeing those smaller + cycles, they’re focused on kicking off an accepted pitch’s work, + breaking it first into cycles and then into smaller iterations in + which they can assign out work. A given cycle’s table or board + view allows the managers to have a whole look at all members of + their team and look specifically at things that are important to + them, like all the pull requests that are open and quickly seeing + which engineers are assigned, what pull requests have been merged, + deployed, etc. +

💡 Check out what this looks like in our team board.

Team lead approach

+ Now, if we put ourselves in the shoes of our team leads and + Directors/VPs, we see that they’re using the new projects + experience to primarily get the full picture of where product and + feature development currently sit. They told me the main team + roadmap and backlog is where they can get questions answered like: +

  • Which projects do we have in flight in which product area right + now?
  • Who’s the key decision maker for this project?
  • Which engineers are working on which projects?
  • Which projects are at risk and need help (progress/status)?

+ What’s great about this is that they can quickly glance at what’s + in motion and then click into any cycles or status to get more + context on open issues, pull requests, and how everything is + connected. +

+ 💡 Outside of being able to check in on what’s being worked on and + where the organization’s current focus is, our leads have found + additional use cases that may not be applicable for an engineer + like me. They use private projects for more sensitive tasks, like + managing our team’s hiring, tracking upcoming start dates, making + sure they’re staying on top of career development, organizational + change management, and more. +

Wrap-up

+ This is how we as the planning and tracking team at GitHub are + using the very product we’re building for you to build the new + projects experience. There are many other teams across GitHub that + utilize the new project tables and boards, but we hope this gives + you a little bit of inspiration about how to think about project + planning on GitHub and how to optimize for all the stakeholders + involved in building and shipping products. +

+ What’s great about project planning on GitHub is that our focus on + powerful primitives approach to project management means that + there is an unlimited amount of flexibility for you and your team + to play around with, and likely many, many ways we haven’t even + thought about how to use the product. So, + please let us know + how you’re using it and how we can improve the experience! +

+ Tags: +
\ No newline at end of file diff --git a/packages/readabilityjs/test/test-pages/guardian/distiller.html b/packages/readabilityjs/test/test-pages/guardian/distiller.html new file mode 100644 index 000000000..0ca1719c3 --- /dev/null +++ b/packages/readabilityjs/test/test-pages/guardian/distiller.html @@ -0,0 +1,41 @@ +

+ The platform is home to a devoted base of left-leaning communities – and no one billionaire can control it +

mastodon on an iphone
Decentralized software like Mastodon is intimidating to many users. Photograph: Hollandse Hoogte/Rex/Shutterstock
Wilfred Chan

+ Since Elon Musk completed his purchase of Twitter last week, some of the social media app’s users have been looking for a new home – only to find there aren’t many great options. Twitter’s co-founder Jack Dorsey is beta testing a new app called Bluesky, but there’s no launch date yet. +

+ However, tech-savvy users are rallying around Mastodon, a six-year-old social media platform popular among a devoted base of left-leaning niche communities. Mastodon, named after the extinct tusked animal, is decentralized, which means it can’t be controlled by a single corporation or space billionaire. That’s clearly appealing to the flood of users who have signed up since Musk’s Twitter takeover, with more than 70,000 users joining Mastodon on the day after his announcement alone. +

+ But that’s still a drop in a bucket compared with Twitter’s reported 450 million daily active users. A big problem? Decentralized software remains difficult for many people to use. +

+ I joined Mastodon this week, and it took a few hours just to master its new vocabulary. Some of it is a little silly-sounding: instead of tweets, you have “toots”. Things get trickier after that. Mastodon is not a single website but a network of thousands of websites called “instances”, also called servers. These servers are “federated”, which means they are run by different entities but can still communicate with each other without needing to go through a central system. And the space they all exist in is called the “fediverse”, which some savvy tooters call “the Fedi”. +

mastodon feed screenshot
On Mastodon, you don’t tweet – you toot. Photograph: Mastodon

+ When you sign up for Mastodon, the first thing you do is choose a server. There are general-purpose ones, such as mastodon.social, as well as ones aimed at interest groups, such as kpop.social or linuxrocks.online. There are also joke servers like dolphin.town, where the only thing users are allowed to post is the letter “e”. +

+ The server becomes part of your username (for example, wilfred@kpop.social), and the toots you see on your feed are toots from your server-mates, rather than from the entire fediverse. But you’re also free to toot at people from other servers and even “boost” their public toots on to your feed. +

+ That’s how Mastodon creates a unified global experience without being controlled by one entity, said Eugen Rochko, Mastodon’s Germany-based founder and lead developer. “The servers are service providers, like Hotmail and Gmail are for email. It doesn’t mean that the different servers are isolated from each other, like old school forums,” he said. “Having just one account allows you to follow and interact with anyone in this global decentralized social network.” +

+ But Mastodon’s model comes with its own risks. If the server you join disappears, you could lose everything, just like if your email provider shut down. A Mastodon server admin also has ultimate control over everything you do: if for some reason the owner of kpop.social doesn’t like that I boosted a toot from dolphin.town, they could remove it or even “defederate” the server, which would block all dolphin toots from the k-pop server completely. A server admin could also snoop on my private toots if they wanted to – or delete my account for any reason. +

+ Rochko said new users should scrutinize who runs a server before they join it: “Is it an organization that has a track record, is trustworthy, is likely to be around for a long time, but also has a moderation policy?” The “good ones”, he explained, “have rules against hate speech, and provide basic necessities like backups, so if one of the admins gets hit by a bus, the server does not disappear.” Rochko added that Mastodon includes a list of vetted servers on its homepage that meet these criteria. But it’s still a tall ask for a brand new user to figure these things out on their own. +

+ As a result, many of Mastodon’s core users have been tech experts, like stux, a 29-year-old man from the Netherlands who told me over private toots that he’s been “messing around with trying to create an alternate social platform” for “about a decade”. He runs mstdn.social, a server that has amassed 83,000 users over three years and costs him €358 a month, which he crowdfunds through Patreon. He also moderates it: “Many reports are very clearly breaking the rules, trolls, racists etc. But in some cases it’s two people against each other while I am the referee and try to find a good in-between.” +

mastdon feed screenshot
Most users agree that the network has a left-leaning bent. Photograph: Mastodon

+ While technically anyone can spin up a Mastodon server, most users agree that the network has a left-leaning bent. Mastodon’s list of vetted servers include queer-themed and climate justice-oriented instances; to be included on the list, a server must agree to the Mastodon Server Covenant, which requires “active moderation against racism, sexism, homophobia and transphobia”. Rochko said his work wasn’t specifically left- or right-leaning but simply abided by “basic, basic beliefs that I have about social networks, and that is, for example, that hate speech should not be allowed”. +

+ The difference between Mastodon and a site like Twitter can feel dramatic. Elilla, a Brazilian trans woman living in Germany, said Mastodon felt much safer to her: because instances can be tightly controlled, people can have discussions without being worried about them accidentally getting broadcast to the world. “What counts as a ‘viral’ post will typically have 50 to 100 boosts. Most of my toots have two to 20 likes,” she said. “But when there’s 20 likes I know most people by name, I know their personalities and tastes and interests. There’s a feeling of reciprocity I never had on Twitter; no one is a celebrity, everyone gets read.” +

+ That’s allowed Elilla to form a caring community through which she’s found deep friendships, romantic relationships, and even jobs. When she decided to try posting erotic content, she “didn’t get hate even once”, she said – something that would be inconceivable on a public site like Twitter. “The fediverse taught me what it it’s like to have community, and the community taught me what is trans joy.” +

+ Unfortunately, Mastodon’s decentralization also means it can be repurposed by anyone for any reason. In 2019, the white supremacist social network Gab started using a version of Mastodon’s free software. Mastodon’s team couldn’t prevent Gab from doing so, but many of the largest Mastodon servers defederated the Gab servers, so that they wouldn’t be able to interact. Mastodon’s code has also been used to power Trump’s social network, Truth Social. +

+ Despite its growing influence, Mastodon’s design makes it difficult to fund, according to Nathan Schneider, a University of Colorado Boulder researcher of tech ownership models – and that makes it unlikely to dethrone a website like Twitter. “Mastodon is a volunteer project that is largely developed by one person. And Twitter is a company apparently worth $44bn,” he said. “If a bunch of users got together and said, ‘Hey, we want to band together and create an alternative,’ their ability to access financing would be far lower than Elon Musk’s ability to access financing.” +

+ But perhaps the real reason it’s hard to beat Twitter is simply because it’s just where everyone is. Paris Marx, a vocal big tech critic and host of the Tech Won’t Save Us podcast, said he has tried Mastodon but couldn’t find much of an audience, unlike Twitter, where he has 35,000 followers and writes posts that often go viral. +

+ “It’s still an influential social media platform. And there’s not there’s not a real equivalent to it, and so people who are interested in what Twitter offers, you know, are kind of stuck here,” he said. +

+ Unable to quit, he does the next best thing: “I block accounts that serve me ads. I don’t pay for things like Twitter Blue,” he said. “I try to make sure that I’m not too financially lucrative for them.” +

+ Topics +
\ No newline at end of file diff --git a/packages/readabilityjs/test/test-pages/infoproc/distiller.html b/packages/readabilityjs/test/test-pages/infoproc/distiller.html new file mode 100644 index 000000000..5e9b36df3 --- /dev/null +++ b/packages/readabilityjs/test/test-pages/infoproc/distiller.html @@ -0,0 +1,42 @@ +
+ + Theodore A. Postol is professor emeritus of Science, Technology, and International Security at the Massachusetts Institute of Technology. He is widely known as an expert on nuclear weapons and missile technology.  + +
+ After leaving the Pentagon, Postol helped to build a program at Stanford University to train mid-career scientists to study weapons technology in relation to defense and arms control policy.  +
+ He has received numerous awards, including the Leo Szilard Prize from the American Physical Society for "incisive technical analysis of national security issues that [have] been vital for informing the public policy debate", the Norbert Wiener Award from Computer Professionals for Social Responsibility for "uncovering numerous and important false claims about missile defenses", and the Richard L. Garwin Award "that recognizes an individual who, through exceptional achievement in science and technology, has made an outstanding contribution toward the benefit of mankind."  +
+ Steve and Ted discuss:  +
+ 0:00 Introduction  +
+ 2:02 Early life in Brooklyn, education at MIT, work at the Pentagon  +
+ 20:27 Reagan’s “Star Wars” defense plan  +
+ 28:26 U.S. influence on Russia and China’s second-strike capabilities  +
+ 54:41 Missile defense: vs nuclear weapons, scuds, anti-ship missiles (aircraft carriers), hypersonics  +
+ 1:11:42 Nuclear escalation and the status of mutually assured destruction  +
+ 1:32:24 Analysis of claims the Syrian government used chemical agents against their own people  +
+ 1:44:45 Media skepticism  +
+ Resources:  +
+ Theodore Postol at MIT  +
+ A Flawed and Dangerous US Missile Defense Plan, G. Lewis and T. Postol, Arms Control Today  +
+ Review Cites Flaws in US Antimissile Program, NY Times May 17 2010  +
+ Improving US Ballistic Missile Defense Policy, G. Lewis and F. von Hippel, Arms Control Today, May 2018  +
+ Whose Sarin? by Seymour Hersh (2013)  +
+ + Here is an excerpt from the transcript: 
+
Ted Postol: ... So, you've got to listen to Putin's voice dispassionately. And when you listen to him, he makes it clear numerous times, numerous times that he doesn't think American missile defense is a worth anything, but he also is worried about an American president who might believe otherwise, and who might take steps against Russia, that would then lead to an action-reaction cycle that would get us, get us all killed.
In other words, he's not just worried about the system, whether it can work, he's worried about American political leadership and what they think, or if they think, or if they know. And that was, you know, I was very receptive to understanding that because that's exactly what I went through, you know, 30 years earlier when I was at the Pentagon, looking at this dog of a missile defense.
And so, the Chinese look at this, they know the Americans are lying to them all the time. I could give you a good story about South Korea and the way we lied to the South Koreans and lied to the Chinese.
I was really furious with that. That was under Secretary of State Hillary Clinton. And my view is...
Steve: THAAD?
Ted Postol: THAAD, right. THAAD in South Korea.
And my view is if you're lying to an ally and you're lying, you know, I have very good friends. I'm very, very proud to say I have some very good friends who are high-level diplomats, and I've asked every one of them, would you lie in a negotiation? And every one of them has said, no. In other words, your credibility depends on your honesty. You might not say something that, you know, could be relevant to a negotiation relevant to your adversary's thinking, but you would never lie because your credibility will, you'll never be believed again. That's their view of this.
And here we were under Hillary Clinton lying to an ally and lying to the Chinese, who I knew through my personal contacts, understood that we were lying to them. I know from personal contacts with the Chinese.
So, how do you expect people to treat you when they know you're a liar? To me, it's just simple human relations. And, and I now understand that because I have friends who are both diplomats and soldiers, and I know, if you have to lie to make a point there's something wrong and you're, you're jeopardizing your credibility with other professionals if, if you do that.
So, we should not be surprised that the Chinese are increasing their forces.
And when Putin marched out this horrifying Poseidon underwater torpedo, could potentially carry a hundred megaton warhead. It's nuclear-powered. It can travel at some very high speed, 50, 60 knots or more, and then it can go quiet, sneak into a Harbor, know coastal Harbor and detonate underwater, and destroy out to 30 or 40 kilometers, a complete area, urban area. And he has this weapon. He made it obvious that he had it. He showed plans for it.
Ted Postol: Well, what he was doing is he was saying to an American president who knows nothing. All right, assuming that the president knows nothing, that your missile defenses will not do anything about this weapon. That's what he did it for. He was an insurance policy toward bad decision-making by American political leadership. That's why he built that weapon. That's why he ordered that weapon built.
So not because, I mean, he may be a monster. That's another issue, but it's not because he was a monster, it's because he made a strategic calculation that that kind of weapon would cause any person, even if they were totally without knowledge and thought of how missile defense could work, to understand that you will not escape retribution if you attack Russia. That's why that weapon was built.
\ No newline at end of file diff --git a/packages/readabilityjs/test/test-pages/instyle/distiller.html b/packages/readabilityjs/test/test-pages/instyle/distiller.html new file mode 100644 index 000000000..03a75c311 --- /dev/null +++ b/packages/readabilityjs/test/test-pages/instyle/distiller.html @@ -0,0 +1,15 @@ +

+ "fresh snow, fresh cut" +

+ Call it quarantine boredom or an unstoppable trend, but in 2020, it feels like everyone is getting bangs. And this week, Gigi Hadid is adding her name to the list. +

+ On Tuesday, the model was spotted out and about in New York City for the first time since giving birth to her and Zayn Malik's daughter. In the photos of said walk, we couldn't help but notice the inches (!!!) on her long blonde hair +

+ The length didn't last long, however, because just a day later Hadid took to Instagram to show off her brand new hairstyle, complete with perfect wispy bangs. "fresh snow, fresh cut," she wrote in the caption. She also tagged her stylist, Allie Hernandez. +

+ Unsurprisingly, Gigi isn't the first Hadid to get new risky bangs during quarantine. Her sister Bella showed off a new style (THAT SHE CUT HERSELF) back in April. +

bella hadid bangs
+ Credit: Instagram / @bellahadid +

+ If, like me, you now feel an urge to take out the scissors and potentially risk a beauty disaster, then check out our comprehensive guide of the best bangs for your face shape. +

\ No newline at end of file diff --git a/packages/readabilityjs/test/test-pages/jacobbrazeal/distiller.html b/packages/readabilityjs/test/test-pages/jacobbrazeal/distiller.html new file mode 100644 index 000000000..244a066fb --- /dev/null +++ b/packages/readabilityjs/test/test-pages/jacobbrazeal/distiller.html @@ -0,0 +1,38 @@ +
+ Uncategorized + 3 Minutes +

+ I caught the chess bug as a college student, twenty-one years old, and it’s become my favorite way to unwind since then, as I’m now working at VisioStack and starting a master’s in CS at Clemson University. +

+ Like all hobbies, chess is more fun when you’re making progress. And progress as an adult is certainly more difficult, as for instance recounted in this fine story by Tom Vanderbilt. It’s not so easy to rewire your brain. +

+ I started out as a below-average-rated player, which was humbling and rather galling. My chess-playing friends IRL beat me easily. Today, I’m still a poor player – I frequently blunder pieces and make other mistakes – yet I’m much stronger; I beat those friends and am now around the 95th percentile at chess.com. My rating progress at blitz looks like this: +

+ (I would note that in fall of 2018, I played a great deal of 4-player chess instead of standard chess; this improved my regular game indirectly, fortunately.) +

+ How I Study +

+ Much of this progress was haphazard: I played lots of games, read the occasional book, learned a few openings. Earlier this year I hit a road block around 1450; the following routine is what’s helped me continue to progress. +

+ I spend about two hours per day on chess, broken into three periods. In the first period, I drill opening theory and positional concepts. I use the spaced-repetition app Anki for this. My cards are just positions drawn from computer analysis of my own games: I put blunders and mistakes from my games here (front side of the card is the position before my move, reverse shows the correct move.) I also put many positions from the openings that I play here. I add about 25 new cards a day, which means I have 100+ cards to go over on a given day. Reviewing Anki cards takes between 30 minutes and an hour. +

+ In the second period, I solve puzzles on Chess.com for 30 minutes to an hour. I would note that this requires a chess.com membership, as does unlimited analysis of your games. I pay $99/year for a Diamond membership; there are other, cheaper levels as well. Still, for a serious hobby, it’s pretty cheap. +

+ In the third period, I play blitz games (usually at the 3+0 time control). I will analyze many of these games – especially ones I lose – and put anything surprising into Anki. I might play for anywhere from 30 minutes to 3 hours. +

+ Progress: Not a Straight Line +

+ It’s fascinating to me how much chess performance can vary from day to day. Every two months or so, I’ll have an amazing chess day where my rating skyrockets by 150-200 points (and promptly returns to earth in the next few days.) More concretely: The median interval between my personal best ratings is exactly 60 days. +

+ My rating varies by weekday. I have played over 10,000 blitz games in the last 2+ years. With the exception of a couple of multi-week breaks, I have played essentially every day. If, for each week, I find my median ELO and compare each day to that median (so, for example, that week’s median ELO is 1200, but on Wednesday is 1235, Wednesday gets a +35), and then take the median of all the deltas for each day, we get the following effect: +

+ So on Thursday I’m about average, on Friday I’m better than average, and on other days I’m much worse. I assume this effect arises from playing fewer games on Monday and Tuesday, and so being a bit rusty, and then playing too many games on the weekend, ending with a lower rating. A win or loss is worth about 8 ELO, so on Tuesdays I lose about 9 more games than I win. +

+ Time of day and number of games played also certainly affect rating, but I don’t have that data readily at hand. I do believe that it’s possible to determine via the extensive chess.com API (https://www.npmjs.com/package/chess-web-api), though. +

+ Goals +

+ I’d like to reach 2000 in 2021. My current study strategy seems to net me around 1.5 ELO/day, so this seems achievable. Of course I could hit a wall where I need to reinvent myself again; if I do, I’ll consider getting a chess coach, reading chess books, etc. +

+ Some day, I want to become a titled player. A candidate master needs an ELO of 2200, but this is in over-the-board play, which probably corresponds to around 2300 in online blitz. I don’t have a timeline for this, but I imagine that if I do get there, it will happen with in the next five years. +

\ No newline at end of file diff --git a/packages/readabilityjs/test/test-pages/johnhcochrane.blogspot/distiller.html b/packages/readabilityjs/test/test-pages/johnhcochrane.blogspot/distiller.html new file mode 100644 index 000000000..9f090eb60 --- /dev/null +++ b/packages/readabilityjs/test/test-pages/johnhcochrane.blogspot/distiller.html @@ -0,0 +1,88 @@ +

+ Part 1: Who should get the vaccine first? Sell to the highest bidder. The disease and recession go away faster.  +

+  Part 2: The cost of perfection. The vaccine was invented in a weekend, available in February. In free market land, we would not have had a pandemic, or a recession. 284 thousand people would be alive today. That is the cost of FDA "protection."  +

+ Part 1: Who should get the vaccine first?  +

+ Absolutely nobody* has mentioned in public the free market answer: Sell to the highest bidder.  +

+ (Or just allow some sales to the highest bidder. Don't put people in jail for selling some to the highest bidder,)  +

+ It's not as dumb as it sounds. Sure, there is an externality. A good vaccine policy might be to give it to those most likely to spread it to others, with the goal of swiftly reducing the prevalence of the disease. That argues for giving the vaccine in bars.  +

+ That is not our public policy. The entire discussion centers around who should be protected first, from a disease whose prevalence is taken as given. Old people, nursing homes, health care workers, essential workers -- the argument is not the externality. The argument is entirely who should get the individual benefit of protection from the vaccine.  Just why "to the highest bidder" is wrong is then much less clear.  +

+ The case is stronger than usual, for there is a second way to avoid infection: Stay home. Social distance. Wear protective gear. So the question is not, really, "Who should be protected from the virus?" The question is, really, "Who should get a treatment that allows them to be out and about, risking contact with the virus, rather than protect themselves by traditional means?" It is really mainly an economic benefit, avoidance of the cost of other measures to stay healthy. There is an economic answer: people should be out and about first who generate the most economic benefit from being out. And, therefore, are willing to pay the most to get the vaccine.  +

+ To say nothing of the incentives. If vaccine companies can charge what they want to first adopters, and the cost to the rest of us is to stay home for a few more months, they can make boodles and boodles of money, incentivizing new vaccines even better. The government does not decide who gets the iPhone 12 first.  +

+ "But the rich will be able to afford it first," I hear you complain. Yes indeed. Principle one of economics, don't mess with the price system to transfer incomes. "But we won't make the income transfers, so we have to control prices and ration" I hear you complain. So, here we are in the midst of a pandemic, in the midst of an incredibly dangerous economic situation, with $5 trillion of federal debt in the rear view window, the main point of fixing prices and rationing vaccines is... to transfer incomes.  +

+ If it goes to the highest bidder, then the highest value activities, that benefit most from reduction in social distancing, come back faster. I don't know what those are, but pretty much by definition, the economy recovers faster. That brings back jobs a lot faster than stimulus checks. Heck tax it and transfer the money to people who choose to stay home.  +

+ I'll stretch free market nirvana this far: If the government wants to hand out vaccines to whoever it thinks should go first, ok, let it buy the vaccine on the free market, on budget, paid by visible taxes, and hand it out. But it should not forbid anyone who wants it from paying for it, nor forbid vaccine makers from selling to whoever wants it, to artificially keep the price down and hide the immense transfer in its actions. Looked at for what it is -- only and entirely a ban on private transactions -- it is amazing that we meekly put up with such wholesale trampling of our rights to transact, to property, and to pursue our own health. In the most important market in the world right now, vaccine producers may only sell to governments.   +

+ No, it's not as dumb as it sounds. At least economics should start with "to the highest bidder," and come up with some well documented market failure, and a public allocation system that mimics the highest bidder allocation. That nobody dares say this in public, not even my favorite libertarians (that I have seen -- send links to anyone else nuts enough to say this in public!) is a little surprising.  What happened to America, and to economics, that absolutely nobody seems to even question the holy writ that drug companies shall only sell to governments, governments shall allocate vaccines, and severe legal penalties shall accrue to anyone who doesn't like it?  +

+ Instead... Well, here is (second hand and I may get this wrong, corrections welcome) how Stanford is going to do it. Our first allocation goes to health care. OK, that is a likely answer to "if Stanford had to bid for it, what is our highest value use." The hospital system, however, can't figure out who within health care should get it. Is it "fair" for doctors to get it first, but not custodial workers? The crack team of medical ethicists couldn't come up with an answer. So they're going to do it randomly. I don't have to tell you in the middle of a pandemic who gets it first under the discipline of actually having to bid a market clearing price to get it first.  Watch the normally crystal clear Scott Gottlieb run around in circles on the pages of the Wall Street Journal trying to figure out who should get the vaccine when. Well, without a guiding principle, there is no where to go but circles.  +

+ In India, meanwhile, that bastion of... informal..  if not free markets, it appears you can sign up to buy the vaccine, for about $8. Markets in everything, as Marginal Revolution says documenting the story, but not here. +

+ Part 2: The cost of perfection  +

+ We had the vaccine the whole time, documents  David Wallace-Wells in New York's intelligencer, documenting and popularizing a known but overlooked fact.  +

+ Moderna’s mRNA-1273, which reported a 94.5 percent efficacy rate on November 16, had been designed by January 13. This was just two days after the genetic sequence had been made public +

+ the Moderna vaccine design took all of one weekend. It was completed before China had even acknowledged that the disease could be transmitted from human to human, more than a week before the first confirmed coronavirus case in the United States. By the time the first American death was announced a month later, the vaccine had already been manufactured and shipped to the National Institutes of Health for the beginning of its Phase I clinical trial. +

+ Even under operation Warp Speed -- a truly commendable accomplishment of the Trump Administration that, maybe a year or so from now the TDS crowd might acknowledge -- the only thing we have been waiting for is FDA certification: Randomized clinical trials to prove safety and efficacy, before anyone is allowed to take the vaccine. +

+ What's the free-market way? A drug company can sell a vaccine on January 14, and you can buy it, without fear of going to jail.  +

+ Sure, there is an FDA, and a Federal Trade Commission which monitors drug labeling. The vaccine has to say "this is totally untested, and has not been proven safe or effective in clinical trials" and offer a stack of paper about known risks. You sign a stack of consent forms. If you take it, you're enrolled in our big national database -- you just volunteered for the national non-random clinical trial. (We don't collect much data on drugs that are out there).  The FDA rapidly collects information. At the same time, randomized clinical trials are going on. Drugs can give more and more hopeful labels as the results roll in. At some point after Phase III and FDA review, a drug can get the official FDA seal of approval. No, insurance and medicare don't pay for non-approved stuff. This is free-market nirvana, you pay for unapproved medicines if you want them (see part 1). There is an FTC and a tort system. Drug companies that sell things they know are unsafe or ineffective pay billions.   +

+ What happens in free market nirvana? Yes, there are quite a few more cases of people who are hurt by side effects. Not that many actually, as we are following the early adopters carefully and broadcasting data as it comes out. Quite a few useless drugs get tried a bit more widely.  +

+ But, going on the current assumptions that these made in a weekend vaccines work, we would not have had a pandemic at all. Every job lost, every business closed, all 284 thousand US deaths, $5 trillion of federal spending, the biggest (though mercifully short) recession in US history, would not have happened. All of this cost stems from one thing -- the ban on using any medicine before the FDA approves it.  +

+ Reflect also on the vast asymmetry of effort. Two or three vaccines were invented in a weekend. Then we spent the next year testing those few, not developing better ones. The vaccines have serious side effects. Think how much better we would be now if we were using vaccines that had been developed... in the next weekend alone, to say nothing of 8 months of constant innovation.  This ratio of invention vs. testing is off by orders of magnitude.  +

+ Now, free market nirvana: The developers start selling the vaccine to wiling guinea pigs on Monday morning, as well as running randomized trials. In a matter of weeks some side effects get noticed. They or other developers start tweaking and building better vaccines! In a rough tumble of competition, within a few months we have much better vaccines. But you can't try anything at scale without the one-year FDA approval -- and that by itself is a miracle compared to the usual multi-year process.    +

+ The benefits of FDA protection -- the prohibition of selling drugs before full randomized clinical trials have been run and then painstakingly reviewed -- are not zero. They are the people who are not hurt by the slightly larger (than in clinical trials) experimentation with new drugs.  +

+ The costs of this FDA "protection" are immense: In the free-market world, we would not have had a pandemic.  +

+ Maybe free markets aren't so dumb after all.  +

+ (We also would have had home tests months ago, which at zero side effect to anyone would have slowed if not stopped the pandemic. But you all know that story.)  +

+ Wallace-Wells: +

+ our approach to the pandemic here raises questions, too, about the strange, complicated, often contradictory ways we approach matters of risk and uncertainty during a pandemic — and how, perhaps, we might think about doing things differently next time. That a vaccine was available for the entire brutal duration may be, to future generations trying to draw lessons from our death and suffering, the most tragic, and ironic, feature of this plague. +

+ Indeed.  +

+ Update +

+ DWAnderson commenter below makes an excellent point that I am embarrassed to have omitted. I need to recheck my free market catechism. To restate it, ok, keep the current allocation system, which we now realize is about transferring income. But don't give out vaccines, give out the right to receive the vaccine. And allow people to trade that right--properly, remove the legal prohibition against trading the right, the threat of being thrown in jail for taking money in return for your place in line. If the Stanford hospital custodian wants to sell his right to a hedge fund manager, and take the  month off in quarantine in a nice hotel, let him. Everyone is better off -- the magic of trade.  +

+ Forbidding ex-post trade is one of the silliest of market interventions.  +

From Twitter,  the first person in the UK to get a vaccine. She's 90. She looks like a very nice lady. But what is the chance she gives the virus to anyone else, goes out to a bar or a party, works at an ICU, or gets the economy going with her vaccination? How much would it cost to protect her from covid by other means? Of course I want her to have a vaccine as soon as possible, but when we only have so many doses is it faintly reasonable that this is the single most important person in the whole UK to get the first vaccine?   +
+ Update 2: The more I think about it, the more ridiculous this is. We waited a year for "science," clinical trials. We know there are side effects. How many 90 year olds were in the clinical trials? Zero. There weren't enough 75 year olds in the trials to get confidence intervals between 0 and 100%. And they put the 90 year olds first. How many 90 year olds will get complications? How many will die from complications? Nobody knows because they have never tried it on a 90 year old. Until now. They are unleashing a vaccine completely untested on the most frail vulnerable population.  +
+ As above, I'm fine with unleashing completely untested vaccines on willing volunteers. But we might as well have done that last January.  +

+ Update 3: +

+ "A vaccine auction" by Romans Pancs +

+ Steven Landsburg on trading the right to get a vaccine.  +

+ *Update 4.  +

+ I hadn't looked hard enough. Tyler Cowen had it back in May. Of course.  +

\ No newline at end of file diff --git a/packages/readabilityjs/test/test-pages/jon.bo/distiller.html b/packages/readabilityjs/test/test-pages/jon.bo/distiller.html new file mode 100644 index 000000000..17d8f8a3f --- /dev/null +++ b/packages/readabilityjs/test/test-pages/jon.bo/distiller.html @@ -0,0 +1,139 @@ +

+ My digital life in a nutshell: I discover relevant content I don’t have time to consume, I find time and become overwhelmed with my scattered backlog, I wish the content were in a different format, and then I’m unable to find something again once I’ve consumed it. Not retaining enough is a valid problem but we’ll tackle that one later. +

+ There’s a lot of generalization in my summary but the core issue is an extraordinarily high level of friction in the process of finding, organizing, and sharing digital content. During the past few years I’ve noticed: +

  • + The more seamless the acquisition & ingestion, the more engaged I am with the content +

  • + Insights are just as likely to be found in a 400-page book as in a 40-minute podcast +

  • + Notes and their subsequent review are essential for long-term retention +

  • + Recommendations from other humans are as good, if not better, than algorithmic suggestions +

+ In the rest of this post I attempt to explain the digital tools I wish existed, and how the the currently available tools do not suffice. What are also probably lacking are my habits and workflows around this - but I’m looking at tools specifically here. +

+ Queue management for inbound digital content # +

+ Where to begin? Probably the most common problem I see myself and other people dealing with is processing the incoming deluge of articles to read and videos to watch. This isn’t all personal recommendations - it encompasses any and all content I think my future self would appreciate me consuming. A list of issues, roughly by order of appearance: +

  • + Content (or links to it) arrive from a variety of sources including text messages from friends, email conversations, tweetstorms and replies, references in books, suggestions in real-world conversations, and more. +

  • + Every book, article, post, or tweet has the potential to lead to more content. +

  • + Content is published in a variety of formats including but not limited to images, sound files, videos, Google Drive docs, diagrams, long-form paywalled articles, PDFs, powerpoint presentations, and base 64 encoded blobs. +

  • + I have little visibility into required time investment and foundational context until I’ve opened it and started thinking about it. Should I sit down with a pen and paper to read this or can I skim it while waiting for my coffee? +

  • + Learning, work, news, and entertainment all have different priorities in my life (roughly in that order). +

  • + I would like to batch process content in different “streams” regardless of where they are stored. For example: I have two hours, let me work through interesting text content my friends sent me last week. Or: show me all the interesting/relevant videos I’ve queued over the past month. +

  • + I’m not always connected to a stable internet connection. +

  • + If it’s a long piece of content I want my position saved reliably so I can resume at a later point. +

  • + I often want it in a different format than the one it was originally published in (audio → text, text → audio, pdf → ebook). Automated conversion works but is cumbersome. Listening to text articles requires sending them to a special app and converting articles to ebooks is annoying and loses a lot of formatting and navigation. +

  • + I love to respond to a person’s recommendation - preferably before they’ve forgotten why they sent me it in the first place. +

  • + I’d like a centralized history of content tied to my notes and annotations in case I want to find it again later. It feels like every week I’m speaking with someone and I remember a blog post I read a few months ago they might find relevant … or was it a Reddit post? Can I find it my history? Oh no, it’s been replaced with [deleted] … find an archived copy… rinse and repeat. +

Relevant XKCD, as is tradition

+ Following my curiosity feels like chasing a caffeinated bunny around while real understanding requires time, perspective, and reflection. The internet makes the former much easier - so I find myself constantly balancing the two. Additionally, my energy and attention levels vary throughout the day and it’s far easier to just open Twitter rather than continue reading a long-form article I started on my laptop two days ago. Too often I default to the lower-friction one. +

+ Honorable Mentions: Pocket, Instapaper +

+ A universal book log, recommendation & sharing system # +

+ I love exploring other peoples’ reading lists. Here’s my own. I find everyone keeps their reading lists in different formats on different platforms. Plaintext lists are nice but hard to parse. Spreadsheets are easy to parse but a pain to manage. Third-party services aren’t interoperable, require logins, and are not future-proof. +

+ Part of the problem here is metadata is hard. Someone has to sit there and fill out the author, title, subtitle, summary, page count - and they’re probably not going to do it for free. Amazon is a good at it but is hostile to publishers. Goodreads has much potential but seems to have stagnated. Linking to the book’s Wikipedia entry would be my preference but very few books have an entry. +

+ Whatever this tool for managing my ever-growing reading list will be, it should: +

  • + Let me compare my reading list with another to see overlap. I find this a wonderful way to spark conversation and find common interests. +

  • + Allow me to tag books instead of placing them into static lists (think clusters or tag clouds). +

  • + Be tied to my highlights, annotations, and bookmarks in a non-proprietary, searchable, and shareable format. Make them public if I want to. +

  • + Save context on where and when I found this book: why I thought it was important to read, when I read it, what I wrote down while reading it, and what other content I discovered through it. +

  • + Let me query this tool like a relational database. For example: show me all books about scaling startups recommended by people I follow on Twitter or by people they follow. The current Twitter search makes me feel like I’m using a government site created before I myself even knew what a computer was. +

  • + Help me deal with prioritization. My reading list is a mess and I can’t be alone. Are certain books better read before others? Prerequisites? Could three of them be replaced with one? What are the other books by the this author? Are they worth reading too? Why exactly did I think reading this 800 page book was relevant when I added it? Is 80% of the content attainable from a blog post? Where is that post? Has someone in my network written a rebuttal to the ideas in this book? The list goes on and on. +

  • + Provide relevant suggestions with the typical recommender approach based on what people interested in the same topics also enjoyed reading and learning from. +

+ Honorable Mentions: None :( +

+ Intelligent PDF viewers, eBook readers, audiobook & podcast players # +

Functionality I want in my document reader

+ Reading is incredible and I love my Kindle. But eBooks today are just a step above OCR’ing a book and slapping on a few basic features which have existed for 30+ years. While I’m reading an eBook I want to: +

  • + Have relevant illustrations, graphs, and tables appear for duration of their mentions so I don’t have to flip back and forth between them. +

  • + See glossary terms and their definitions which appear on this page. Highlighting and searching a term is great but the author may have added important context to the glossary definition. +

  • + View popular annotations and highlights across all mediums - not just by other readers who own an Amazon Kindle readers and purchased this book version and also happened to highlight it enough times. A quote was referenced in 300 blog articles? A two sentence excerpt retweeted 50,000 times? You bet I want to know! +

  • + Follow referenced information easily. You cited a paper - great, let’s look at the footnotes. Oh, the full reference is in the back of the book. Online list of citations? Of course not! Drop a bookmark, navigate to the back of the book, pull out my laptop, find the paper. Of course, a paywall. Grab a snack. Acquire the PDF. Search for keywords to try to find the referenced information. Sigh, 2019. +

  • + Not be hindered by the DRM system. Copyright is important and I want to support authors but it’s insane to me all these content licenses I’m acquiring can’t be donated to a library upon account closure. Yes, legal DRM-free eBooks exist but they aren’t without their own issues. +

  • + Seamlessly switch between devices and formats while retaining my position. Something like Whispersync (a neat idea but come on, I’m not made of money. Also, see above points). +

  • + Let me use a digital or physical keyboard instead of an e-ink keyboard to type my annotations. A possibility here is a companion app, which feels like a notes app but ties my notes to their location/text in the book I’m reading. +

What I want my audiobook player to look like

+ Most of these points above also apply to my experience listening to podcasts, audiobooks, and watching Youtube videos and interviews. I find myself wishing I could: +

  • + Navigate them more comfortably. Both Libby and Audible leave much to be desired in terms of navigation. Finding a quote I remember hearing to three days ago is basically blindly stumbling around - and I lose my current spot too. Seeing a list of chapter numbers for the book I’m listening to has been helpful a grand total of 0 times. And how cool would it be to drop a bookmark from my bluetooth-connected headphones as I’m biking down a street. +

  • + View an auto-generated transcript of a podcast as I’m listening to it. It should have easy-to-follow links to references to other podcasts, media, books and support searching for key terms. YouTube already transcribes all of their videos and Google Meet now generates live captions as we’re talking - why can’t we do something similar with podcast apps? +

+ A centralized search interface for my digital brain (memex) # +

+ I want to be able to open an interface, type three words, and instantly see results from everything my digital self has interacted with. Emails, years of full-text browsing history, text messages, Slack messages across all my organizations, calendar invites and events, books, podcast transcripts I’ve consumed, Twitter and Instagram DMs, PDFs I’ve downloaded, bash commands, videos I’ve seen, my online and offline files, notes, blog post drafts - I really do mean everything. +

+ I acutely feel the need for this when I’m trying to find something I know I’ve seen online but can’t remember where I saw it. Google is wonderful for finding new information, but absolutely poor for re-finding things. Chrome’s history has so much potential - but I suspect Google would much rather have us look at their ads a few additional times rather than go direct to the source. I accept I might be in the minority on this one. Regardless, this tool should: +

  • + Accept and parse the following queries: +

    • + spacex announcement type:video 2016 +

    • + links from:jon@test.org topic:python +

    • + paper on temperature, productivity referenced in book:Uninhabitable Earth +

    • + type:pdf habits digital interfaces +

    • + reading comprehension type:blog post +

    • + printer ink receipt +

    • + type:book read:2017 finance +

    • + file:py datetime parse +

  • + Respect my privacy: hosted on something I control and never mined for ads. +

  • + Support all my devices with two-way sync so I can search and add to it wherever I am. +

  • + Be extensible: allow me to easily ingest my own information and extend with desired functionality. +

  • + Cluster information based on content, tags, geo-location, connected people, conversations, source, and other factors I’m not even aware of. +

  • + Notify me about changes to documents and webpages I’ve visited. +

  • + Allow a rough export of my research on a topic (like, a knowledge dump off everything I’ve consumed on pandas) with the ability to easily share it. +

+ Parting Thoughts # +

+ I’m fascinated with a better bridge between our minds and our digital devices. A well-designed tool should disappear and allow complete attention to the task at hand, but digital devices today are far from this ideal - often due to arcane copyright laws or profit-seeking. These aren’t new ideas by any means. See Vannevar Bush’s original conception of a memex over 70 years ago. We are way overdue for this. I see enormous potential at combining a true memex with all of our personal data (health, fitness, biometrics) along with our habits, goals, tasks, reflections, and communication tools. +

+ It seems to me that as information becomes more abundant, the connections drawn between disparate pieces are becoming increasingly important. The easier it is to share that graph with other people, the faster we can learn from each other and understand complex relationships. I’m excited for a world where knowledge is easier to discover, validate, dispute, understand, retain, and share. +

+ I hope to cover my thoughts on processes, note-taking apps, and knowledge graphs next. Stay tuned here. My thanks to Arthur Tyukayev, Alex Ly, David Heimann, Em deGrandpré, Alexey Guzey, Sam Tkachuk, and Brian Timar for reading drafts of this and providing wonderful feedback. +

+ 2019-12-09: fixes grammar +

\ No newline at end of file diff --git a/packages/readabilityjs/test/test-pages/josephg/distiller.html b/packages/readabilityjs/test/test-pages/josephg/distiller.html new file mode 100644 index 000000000..c290700c5 --- /dev/null +++ b/packages/readabilityjs/test/test-pages/josephg/distiller.html @@ -0,0 +1,58 @@ +

+ I saw Martin Kleppmann’s talk a few weeks ago about his approach to realtime editing with CRDTs, and I felt a deep sense of despair. Maybe all the work I’ve been doing for the past decade won’t be part of the future after all, because Martin’s work will supersede it. Its really good. +

+ Lets back up a little. +

+ Around 2010 I worked on Google Wave. Wave was an attempt to make collaboratively editable spaces to replace email, google docs, web forums, instant messaging and a hundred other small single purpose applications. Wave had a property I love in my tools that I haven’t seen articulated anywhere: It was a general purpose medium (like paper). Unlike a lot of other tools, it doesn’t force you into its own workflow. You could use it to do anything from plan holidays, make a wiki, play D&D with your friends, schedule a meeting, etc. +

+ Internally, Wave’s collaborative editing was built on top of Operational Transform (OT). OT has been around for awhile - the algorithm we used was based on the original Jupiter paper from 1995. It works by storing a chronological list for each document of every change. “Type an H at position 0”. “Type a i at position 1”. Etc. Most of the time, users are editing the latest version of the document and the operation log is just a list of all the changes. But if users are collaboratively editing, we get concurrent edits. When this happens, the first edit to arrive at the server gets recorded as usual. If the second edit is out of date, we use the log of operations as a reference to figure out what the user really intended. (Usually this just means updating character positions). Then we pretend as if thats what the user meant all along and append the new (edited) operation. Its like realtime git-rebase. +

+ Once Wave died, I reimplemented the OT model in ShareJS. This was back when node was new and weird. I think I had ShareJS working before npm launched. It only took about 1000 lines of code to get a simple collaborative editor working, and when I first demoed it I collaboratively edited a document in a browser and from a native application. +

+ At its heart, OT is a glorified for() loop with some helper functions to update character offsets. In practice, this works great. OT is simple and understandable. Implementations are fast. (10k-100k operations per second in unoptimized javascript. 1-20M ops/sec in optimized C.). The only storage overhead is the operation log, and you can trim that down if you want to. (Though you can’t merge super old edits if you do). You need a centralized server to globally order operations, but most systems have a centralized server / database anyway, right? +

+ Centralized servers +

+ The big problem with OT is that dependancy on a centralized server. Have you ever wondered why google docs shows you that weird “This document is overloaded so editing is disabled” thing when a document is shared to social media? The reason (I think) is that when you open a google doc, one server is picked as the computer all the edits run through. When the mob descends, google needs to pull out a bunch of tricks so that computer doesn’t becomes overwhelmed. +

+ There’s some workarounds they could use to fix this. Aside from sharding by document (like google docs), you could edit via a retry loop around a database transaction. This pushes the serialization problem to your database. (Firepad and ShareDB work this way). +

+ Its not perfect though. We wanted Wave to replace email. Email is federated. An email thread can span multiple companies and it all just works. And unlike facebook messenger, emails are only be sent to the companies that are CC’ed. If I email my coworker, my email doesn’t leave the building. For Wave to replace email, we needed the same functionality. But how can that work on top of OT? We got it working, kinda, but it was complex and buggy. We ended up with a scheme where every wave would arrange a tree of wave servers and operations were passed up and down the tree. But it never really worked. I gave a talk at the Wave Protocol Summit just shy of 10 years ago explaining how to get on the network. I practiced that talk, and did a full runthrough. I followed literally step by step on the day and the version I made live didn’t work. I still have no idea why. Whatever the bugs are, I don’t think they were ever fixed in the opensource version. Its all just too complicated. +

+ The rise of CRDTs +

+ Remember, the algorithm Wave used was invented in 1995. Thats a pretty long time ago. I don’t think I even had the internet at home back in 1995. Since then, researchers have been busy trying to make OT work better. The most promising work uses CRDTs (Conflict-Free Replicated data types). CRDTs approach the problem slightly differently to allow realtime editing without needing a central source of truth. Martin lays out how they work in his talk better than I can, so I’ll skip the details. +

+ People have been asking me what I think of them for many years, and my answer was always something like this: +

+ They’re neat and I’m glad people are working on them but: +

  • They’re slow. Like, really slow. Eg Delta-CRDTs takes nearly 6 hours to process a real world editing session with a single user typing a 100KB academic paper. (Benchmarks - look for B4.)
  • Because of how CRDTs work, documents grow without bound. The current automerge master takes 83MB to represent that 100KB document on disk. Can you ever delete that data? Probably not. And that data can’t just sit on disk. It needs to be loaded into memory to handle edits. (Automerge currently grows to 1.1GB in memory for that.)
  • CRDTs are missing features that OT has had for years. For example, nobody has yet made a CRDT that supports /object move/ (move something from one part of a JSON tree to another). You need this for applications like Workflowy. OT handles this fine.
  • CRDTs are complicated and hard to reason about.
  • You probably have a centralized server / database anyway.

+ I made all those criticisms and dismissed CRDTs. But in doing so I stopped keeping track of the literature. And - surprise! CRDTs went and quietly got better. Martin’s talk (which is well worth a watch) addressed the main points: +

  • Speed: Using modern CRDTs (Automerge / RGA or y.js / YATA), applying operations should be possible with just an log(n) lookup. (More on this below).
  • Size: Martin’s columnar encoding can store a text document with only about a 1.5x-2x size overhead compared to the contents themselves. Martin talks about this 54 minutes into his talk. The code to make this work in automerge hasn’t merged yet, but Yjs implemented Martin’s ideas. And in doing so, Yjs can store that same 100KB document in 160KB on disk, or 3MB in memory. Much better.
  • Features: There’s at least a theoretical way to add all the features using rewinding and replaying, though nobody’s implemented this stuff yet.
  • Complexity: I think a decent CRDT will be bigger than the equivalent OT implementation, but not by much. Martin managed to make a tiny, slow implementation of automerge in only about 100 lines of code.

+ I still wasn’t completely convinced by the speed argument, so I made a simple proof of concept CRDT implementation in Rust using a B-tree using ideas from automerge and benchmarked it. Its missing features (deleting characters, conflicts). But it can handle 6 million edits per second. (Each iteration does 2000 edits to an empty document by an alternating pair of users, and that takes 330µs. So, 6.06 million inserts / second). So that means we’ve made CRDTs good enough that the difference in speed between CRDTs and OT is smaller than the speed difference between Rust and Javascript. +

+ All these improvements have been “coming soon” in automerge’s performance branch for a really long time now. But automerge isn’t the only decent CRDT out there. Y.js works well and kicks the pants off automerge’s current implementation in the Y.js benchmarks. Its missing some features I want, but its generally easier to fix an implementation than invent a new algorithm. +

+ Inventing the future +

+ I care a lot about inventing the future. What would it be ridiculous not to have in 100 years? Obviously we’ll have realtime editing. But I’m no longer convinced OT - and all the work I’ve done on it - will still be around. I feel really sad about that. +

+ JSON and REST are used everywhere these days. Lets say in 15 years realtime collaborative editing is everywhere. Whats the JSON equivalent for realtime editing that anyone can just drop in to their project? In the glorious future we’ll need high quality CRDT implementations, because OT just won’t work for some applications. You couldn’t make a realtime version of Git, or a simple remake of Google Wave with OT. But if we have good CRDTs, do we need good OT implementations too? I’m not convinced we do. Every feature OT has can be put in to a CRDT. (Including trimming operations, by the way). But the reverse is not true. Smart people disagree with me, but if we had a good, fast CRDT available from every language, with integration on the web, I don’t think we need OT at all. +

+ OT’s one advantage is that it fits well in centralized software - which is most software today. But distributed algorithms work great in centralized software too. (Eg look at Github). And I think a really high quality CRDT running in wasm would be faster than an OT implementation in JS. And even if you only care about centralized systems, remember - Google runs into scaling problems with Google Docs because of OT’s limitations. +

+ So I think its about time we made a lean and fast CRDT. The academic work has been mostly done. We need more kick-ass implementations. +

+ Whats next +

+ I increasingly don’t care for the world of centralized software.
+ Software interacts with my data, on my computers. Its about time my software reflected that relationship. I want my laptop and my phone to share my files over my wifi. Not by uploading all my data to servers in another country. Especially if those servers are financed by advertisers bidding for my eyeballs. +

+ Philosophically, if I modify a google doc my computer is asking Google for permission to edit the file. (You can tell because if google’s servers say no, I lose my changes.) In comparison, if I git push to github, I’m only notifying github about the change to my code. My repository is mine. I own all the bits, and all the hardware that houses them. This is how I want all my software to work. Thanks to people like Martin, we now know how to make good CRDTs. But there’s still a lot of code to write before local first software can become the default. +

+ So Operational Transform, I think this is goodbye from me. We had some great times. Some of the most challenging, fun code I’ve ever written was operational transform code. OT - you’re clever and fascinating, but CRDTs can do things you were never capable of. And CRDTs need me. With some good implementations, I think we can make something really special. +

+ I mourn all the work I’ve done on OT over the years. But OT is no longer fits into the vision I have for the future. CRDTs would let us remake Wave, but simpler and better. And they would let us write software that treats users as digital citizens, not a digital serfs. And that matters. +

+ The time to build is now. +

\ No newline at end of file diff --git a/packages/readabilityjs/test/test-pages/jsomers/distiller.html b/packages/readabilityjs/test/test-pages/jsomers/distiller.html new file mode 100644 index 000000000..70e8ed93b --- /dev/null +++ b/packages/readabilityjs/test/test-pages/jsomers/distiller.html @@ -0,0 +1,127 @@ +

+ I should have loved biology but I found it to be a lifeless recitation of names: the Golgi apparatus and the Krebs cycle; mitosis, meiosis; DNA, RNA, mRNA, tRNA. +

+ In the textbooks, astonishing facts were presented without astonishment. Someone probably told me that every cell in my body has the same DNA. But no one shook me by the shoulders, saying how crazy that was. I needed Lewis Thomas, who wrote in The Medusa and the Snail: +

+ For the real amazement, if you wish to be amazed, is this process. You start out as a single cell derived from the coupling of a sperm and an egg; this divides in two, then four, then eight, and so on, and at a certain stage there emerges a single cell which has as all its progeny the human brain. The mere existence of such a cell should be one of the great astonishments of the earth. People ought to be walking around all day, all through their waking hours calling to each other in endless wonderment, talking of nothing except that cell. +

+ I wish my high school biology teacher had asked the class how an embryo could possibly differentiate—and then paused to let us really think about it. The whole subject is in the answer to that question. A chemical gradient in the embryonic fluid is enough of a signal to slightly alter the gene expression program of some cells, not others; now the embryo knows “up” from “down”; cells at one end begin producing different proteins than cells at the other, and these, in turn, release more refined chemical signals; ...; soon, you have brain cells and foot cells. +

+ How come we memorized chemical formulas but didn’t talk about that? It was only in college, when I read Douglas Hofstadter’s Gödel, Escher, Bach, that I came to understand cells as recursively self-modifying programs. The language alone was evocative. It suggested that the embryo—DNA making RNA, RNA making protein, protein regulating the transcription of DNA into RNA—was like a small Lisp program, with macros begetting macros begetting macros, the source code containing within it all of the instructions required for life on Earth. Could anything more interesting be imagined? +

+ Someone should have said this to me: +

+ Imagine a flashy spaceship lands in your backyard. The door opens and you are invited to investigate everything to see what you can learn. The technology is clearly millions of years beyond what we can make. +

+ This is biology. +

+ In biology class, biology wasn’t presented as a quest for the secrets of life. The textbooks wrung out the questing. We were nowhere acquainted with real biologists, the real questions they had, the real experiments they did to answer them. We were just given their conclusions. +

+ The Roche Biochemical Pathways Poster +
Plans for an alien machine, in Contact

+ For instance I never learned that a man named Oswald Avery, in the 1940s, puzzled over two cultures of Streptococcus bacteria. One had a rough texture when grown in a dish; the other was smooth, and glistened. Avery noticed that when he mixed the smooth strain with the rough strain, every generation after was smooth, too. Heredity in a dish. What made it work? This was one of the most exciting mysteries of the time—in fact of all time. +

+ Most experts thought that protein was somehow responsible, that traits were encoded soupily, via differing concentrations of chemicals. Avery suspected a role for nucleic acid. So, he did an experiment, one we could have replicated on our benches in school. Using just a centrifuge, water, detergent, and acid, he purified nucleic acid from his smooth strep culture. Precipitated with alcohol, it became fibrous. He added a tiny bit of it to the rough culture, and lo, that culture became smooth in the following generations. This fibrous stuff, then, was “the transforming principle”—the long-sought agent of heredity. Avery’s experiment set off a frenzy of work that, a decade later, ended in the discovery of the double helix. +

+ In his “Mathematician’s Lament,” Paul Lockhart describes how school cheapens mathematics by robbing us of the questions. We’re not just asked, hey, how much of the triangle takes up the box? +

+ That’s a puzzle we might delight in. (If you drop a vertical from the top of the triangle, you end up with two rectangles cut in half; you discover that the area inside the triangle is equal to the area outside.) Instead, we’re told that if you ever find yourself wanting the area of a triangle, here’s the procedure: +

+ Biology is like that, but worse because it’s a messier subject. The facts seem extra arbitrary. We’re told to distinguish “lipid bilayers” from “endoplasmic reticula” without understanding why we care about either in the first place. +

+ Enormous subjects are best approached in thin, deep slices. I discovered this when first learning how to program. The textbooks never worked; it all only started to click when I started to do little projects for myself. The project wasn’t just motivation but an organizing principle, a magnet to arrange the random iron filings I picked up along the way. I’d care to learn about some abstract concept, like “memoization,” because I needed it to solve my problem; and these concepts would lose their abstractness in the light of my example. +

+ Biology is no different. Learning begins with questions. How do embryos differentiate? Why are my eyes blue? How does a hamster turn cheese into muscle? Why does the coronavirus make some people much sicker than others? +

+ * +

+ A few months ago, I started a magazine assignment to answer some questions about SARS-CoV-2 and the immune system. I encountered paragraphs like this: +

+ In low-MOI infections (MOI, 0.2), exogenous expression of ACE2 enabled SARS-CoV-2 to replicate and comprise ~54% of the total reads mapping more than 300x coverage across the ~30-kb genome (Figures 1A and 1B). Western blot analyses corroborated these RNA-seq data… It is noteworthy that, despite this dramatic increase in viral load, we observed neither activation of TBK1, the kinase responsible for IFN-I and IFN-III expression, nor induction of STAT1 and MX1, IFN-I-stimulated genes (Figure S1A; Sharma et al., 2003)… +

+ It was hard to get through a sentence without having to consult Wikipedia. In immunology in particular the nomenclature is expansive. One sentence might refer to “leukocytes,” the next to monocytes, the next to lymphocytes. There are a lot of squares-and-rectangles situations: all interleukins are cytokines, but not all cytokines are interleukins? +

+ I’ve never come across a subject so fractal in its complexity. It reminds me of computing that way. A day of programming might involve constructing an elaborate regular expression, investigating a file descriptor leak, debugging a race condition in the application you just wrote, and thinking through the interface of a module. Everywhere you look—the compiler, the shell, the CPU, the DOM—is an abstraction hiding lifetimes of work. Biology is like this, just much, much worse, because living systems aren’t intentionally designed. It’s all a big slop of global mutable state. Control is achieved by upregulating this thing while turning down the promoter of that thing’s repressor. You think you know how something works—like when I thought I had a handle on the neutrophil, an important front-line player in the innate immune system—only to learn that it comes in several flavors, and more are still being discovered, and some of them seem to do the opposite of the ones you thought you knew. Everything in biology is like this. It’s all exceptions to the rule. +

+ But biology, like computing, has a bottom, and the bottom is not abstract. It’s physical. It’s shapes bumping into each other. In fact the great revelation of twentieth-century molecular biology was the coupling of structure to function. An aperiodic crystal that forms paired helices is the natural store of heredity because of its ability to curl up and unwind and double itself with complements. Hemoglobin, the first protein studied in full crystallographic detail, was shown to be an efficient store of energy because of how oxygen atoms snap into its body like Legos, each snap widening the remaining slots, so that it loads itself up practically at a gulp. Most proteins are like this. The ones that drive locomotion twist like little motors; the ones that contract muscles climb and compress each other. Cells, too, are constantly in conversation, and the language they speak is shape. It’s keys entering locks: a protein might straddle the cell membrane, and when a cytokine (that’s a kind of signaling molecule) docks with it, it changes its shape, so that its grip loosens on some other molecule on the interior side of the membrane, as though fumbling a football—that football might be a signal itself, on its way to the nucleus. +

+ I think my understanding of biology was too flow-charty in high school. I knew that DNA → RNA → protein and that this was called “gene expression,” but I was confused on the basics, like, how did genes actually “turn on”? And once they were on, were they on for good? It’s clearer when you think physically. Mammalian DNA isn’t laid out as one long double helix; it’s tightly coiled and coiled again, like this, around little circular proteins called histones: +

+ DNA curled around histones. Image from this Moderna video, at 1:10 +

+ The structure of the resulting fiber has an effect on which genes are expressed. This is because the little molecular machine that transcribes DNA into RNA has to actually ride along the helix, and it can only ride along some parts of it, namely the parts that aren’t curled up out of sight. “Expressing” a gene just means that at a given moment, the machine is accessing a specific portion of DNA, resulting in lots of RNA transcripts, resulting in lots of the protein that the gene codes for. Kink the fiber a bit and you change what the machine can see, thus changing the distribution of proteins it produces. You have “reprogrammed” the cell. (There are many ways to control gene expression, maybe the most common being “repressors” that park somewhere on the DNA, physically blocking the transcription machinery.) +

+ One of the workhorse techniques in modern biology, called RNA sequencing, or RNA-seq for short, takes a frozen cell and counts the RNA transcripts inside it. In effect you get a snapshot of all the proteins being expressed at that moment. The result is literally a big table mapping genes to transcript counts. You see that being one kind of cell versus another—or being in one kind of cellular mood versus another, say in health versus disease—is just a matter of having a different distribution across this table. RNA-seq results are often represented as vectors in high-dimensional space, the counts in the table forming the coordinates; cells move through this expression space as they self-regulate and adapt to their environment. +

+ * +

+ How do you develop a physical understanding of biology? I like pictures. One of my favorite books is called The Machinery of Life, by David Goodsell. It’s full of gorgeous hand-drawn illustrations. Here a bacterium’s flagellar motor is shown in context, then zoomed in on in an inset, with a third picture highlighting its functional elements: +

+ What makes the book work is that it’s basically a re-introduction to molecular biology with the following premise: the cell is a very fast and crowded place, full of little machines, most of them protein, which you understand by taking a close look. It does an especially terrific job through insets like the above relating things at different scales. “Imagine your room filled with grains of rice. That will give you an idea of the billion or so cells that make up your fingertip.” +

+ The writing is very good. It somehow gets you imagining the motion of these machines. It’s tempting when thinking about the cellular world to simply miniaturize our own; but at the cellular scale things behave weirdly. Movement is essentially by random diffusion. “The motions and the interactions of biological molecules are completely dominated by the surrounding water molecules… Inside the cell, [a] protein is battered from all sides by water molecules. It bounces back and forth, always at great speed, but takes a long time to get anywhere.” +

+ It turns out that random diffusion is an incredibly slow way to travel large distances, but an incredibly fast way to explore at short distances. Being a protein inside a cell is like being at a crowded house party where it might take an hour to get across the room, but by the time you get there you’ve bumped into everybody six hundred thousand times. +

+ This point is made beautifully in another favorite book of mine, A Computer Scientist’s Guide to Cell Biology, by William W. Cohen: +

+ Molecules that come close to an organelle tend to remain close to it for a while, and brush against it many times—Figure 20 gives some intuitions as to why this is true. +

+ The result of this is that if receptors for a protein p cover even a small fraction of the surface of an organelle, the organelle will be surprisingly efficient at recognizing p. As an example, if only 0.02% of a typical eukaryotic cell’s surface has a receptor for p, the cell will be about half as efficient as if the entire surface were coated with receptors for p. +

+ This is the kind of fact that instantly clarifies how biology could possibly work. “Cell-sized objects thus have a ‘high bandwidth,’” Cohen writes. “They can recognize or absorb hundreds of different chemical signals, even if they are bounded by membranes.” +

+ Cohen’s book is pitched as an attempt to distill what he learned in acquiring a “reading knowledge” of biology—enough to be able to follow along with a paper in Cell. He’s very good at explaining methods: how do biologists know what they know? For a computer scientist, a biologist’s methods can seem insane; the trouble comes from the fact that cells are too small, too numerous, too complex to analyze the way a programmer would, say in a step-by-step debugger. What biologists mostly do is stuff like: +

  • Spin things to 15,000 Gs in centrifuges to separate pieces having different densities.
  • Separate things of different sizes using gels and magnets. (“Gel electrophoresis.”)
  • Take one of those gels and blot it with special paper to splay the parts out. Then wash the paper with an antibody that binds to a specific protein. Finally, wash the paper with another antibody that binds to the first one, and fluoresces when it does so. See where the meta-antibody lights up—that’s the protein you were looking for. (I think I’m describing a “Western blot.”)
  • Use the fluorescent antibody trick to tag cells expressing one or more proteins of interest. Then squeeze the cells through a tube so small that only one fits at a time. As each cell passes by, shine a laser through it to read its fluorescent tags, and use an electric charge to redirect it to a particular bin. Now you can sort and count cells that match your criteria. (“Flow cytometry.”)
  • Genetically alter microorganisms to make molecular machines to spec; systematically turn off one gene at a time in a cell line and see what changes; edit the genome of a whole animal, and observe its life.

+ Cohen found, and I have too, that in trying to acquire a reading knowledge of biology it’s almost more useful to study the methods than any individual facts. That’s because the methods are highly conserved across studies. Everybody does Western blots. Everybody does flow cytometry and RNA-seq. You’ll see this stuff in every paper. (Or variations on the same themes: separation, sorting, selection, genetic manipulation.) +

+ So that’s the foundation. Or almost: I have left for last my favorite resource of all, an incredible book called The Eighth Day of Creation: Makers of the Revolution in Biology, by Horace Freeland Judson. Parts of this book were serialized in the New Yorker in the 1970s. It is the Power Broker of biology, a tomic masterwork. It is not just comprehensive—Judson had hundreds of conversations with Francis Crick, with Jacques Monod and François Jacob, with their friends and spouses and colleagues; he read every paper, he read all their letters—but it pulls no punches scientifically. Judson always just describes the real thing. +

+ And he emphasizes wrong turns. For example, before the discovery of tRNA—the adapter molecules that link triplets of RNA bases to the amino acids they code for—there was much confusion. It was widely believed that there had to be some kind of punctuation, because how else would one know where to start transcribing, or how to delimit one codon from the next? Certain mental models were ingrained: a going theory was that RNA formed specially shaped pockets for the different amino acids. The idea was that if you zoomed in on each triplet or quartet or whatever (the scheme was then unknown), it would always form the same unique shape that only one kind of amino acid could fit into. The amino acid chain would be formed right there alongside the RNA strand, using it almost as a mold. This was thought to happen in the nucleus. The idea that protein synthesis happened via an adapter, and that the nucleic acids therefore acted less like a mold than a digital code, more purely information—this was a major surprise. +

+ Sitting on the grass at Woods Hole, Crick was talking about genes and proteins, in particular about his assumption that they were colinear and Benzer and Brenner’s plan to show as much, when Ephrussi took him aback by asking how he knew that amino acids were not put in their primary sequence by something in the cytoplasm. . . . “I don’t think Boris necessarily believed it, but it was an idea he thought wasn’t impossible.” +

+ . . . +

+ Crick also cast his skeptical eye over Watson and Rich’s attempts to build models of RNA. “Of course, you realize that our ideas on that were totally wrong. We thought that RNA had some structure with the twenty cavities, it was that period. Mm-hmm. Unfortunately people have forgotten what it is we didn’t know at the time.” +

+ Put another way, the book gives us a view of science before discovery. It is a practitioner’s view of the subject. It is the opposite of a textbook. +

+ * +

+ Trying to study the immune system has gotten me into a Bret Victor sort of mood, wondering what could be done, or built, to make understanding this subject easier. A few things come to mind: +

+ There are some incredible YouTube explainers. Ninja Nerd Science’s videos on the immune system were a miracle—all delivered by a kid in grad school. He is a genius. What he does so well is what Goodsell, in that Machinery of Life book, does so well, what those famous “Inner Life of a Cell” 3D animations do so well: he helps you “see the unseeable.” +

+ Ninja Nerd Lectures YouTube channel +

+ But I wonder whether it should be easier for regular people to create useful illustrations. Consider how easy it is to write, tooling-wise: on the web, you are only ever one click away from a Markdown-enabled textarea that allows you to create and publish pretty, hyperlinked documents. Anyone with a keyboard can contribute a few sentences to Wikipedia or answer a question on Stack Exchange. Drawing, by contrast, is hard, and animating is at least an order of magnitude harder. And yet these media are essential for understanding biological processes. +

+ So what do we do? +

+ It’s telling that when I was recently on a Zoom with a PhD student who was explaining RNA-seq, he pulled out his iPad Pro and essentially made a Khan Academy lecture as he talked, drawing along the way. These tools need to become more common and cheaper. +

+ But we also need more software like pattern brushes in Adobe Illustrator and BioRender to make it un-tedious to draw complex objects. We need more software like Molecular Maya, but simplified even further, à la Victor’s Stop Drawing Dead Fish, to make animating accessible to anyone who can gesture. +

+ Quickly draw an endothelial lining with pattern brushes in Adobe Illustrator +
+ Molecular Maya’s double-stranded DNA kit +

+ Using vector graphics and Undo history, it should be possible to make collaboratively editable images, i.e., images that can be slowly improved as part of a knowledge project like Wikipedia or Stack Exchange. +

+ I want to be able to take a screenshot of the whiteboard in a Ninja Nerd lecture—a big beautiful diagram of the players in the adaptive immune system—and lasso sections of it, linking to sub-diagrams, some filled in by me, some by others, illustrating each of the parts in turn. We should have big, collaboratively edited zoomable “maps”—hierarchical diagrams—that are easy to navigate, work in standard browsers, are embeddable in blog posts, and so on. +

+ Of course we need to teach more people how to draw. It’s an underrated skill. And how to write vividly, as in the wonderful books above. +

+ But biology is uniquely suited to simulation—it’s a world of machines that are too small to see. The trouble is, it requires too much specialized skill to create three-dimensional interactive simulations. We need a toolkit that’s like MockMechanics, or Minecraft, that maybe even is Minecraft, but focused on biology. Or something much better. +

+ Watson and Crick’s metal-and-wire model of the double helix +

+ It’s no coincidence that Watson and Crick depended for their discovery on a literal physical model that was machined for them specially. Victor’s Dynamicland imagines an immersive collaborative space in which such models can be built—now that we have computers—as quickly as you can have a conversation. +

+ This is exactly what I wanted as I was writing my immune system article. I wanted to conjure models I could play with in my hand. I wanted a museum where I could walk around inside the epithelium during an immune response. I wanted to put ideas into physical space, like on a pinboard—TLRs go here, with the other innate armament; CD4+ T cells are there, in the adaptive world—but I wanted it to be as searchable, copy-pasteable, shareable, and composable as text. +

+ Bret Victor’s vision of dynamic tools for thinking +

+ I think we also need inspiration. There is a romance in biology, as in any other science, that a movie like Good Will Hunting could bring out. We need heroes. Whoever delivers us from this pandemic in the form of a slam dunk vaccine, or a cheap quick reliable test, should become a household name, not for their own glory but for our kids—a Feynman for them to dream about someday becoming. +

\ No newline at end of file diff --git a/packages/readabilityjs/test/test-pages/mathoverflow/distiller.html b/packages/readabilityjs/test/test-pages/mathoverflow/distiller.html new file mode 100644 index 000000000..1777b2e68 --- /dev/null +++ b/packages/readabilityjs/test/test-pages/mathoverflow/distiller.html @@ -0,0 +1,137 @@ +
+ 184 +

+ I apologize for writing a lengthy answer, but I get the feeling the discussions about foundations for formalized mathematics are often hindered by lack of information. +

+ I have used proof assistants for a while now, and also worked on their design and implementation. While I will be quick to tell jokes about set theory, I am bitterly aware of the shortcomings of type theory, very likely more so than the typical set theorist. (Ha, ha, "typical set theorist"!) If anyone can show me how to improve proof assistants with set theory, I will be absolutely deligthed! But it is not enough to just have good ideas – you need to test them in practice on large projects, as many phenomena related to formalized mathematics only appear once we reach a certain level of complexity. +

+ The components of a proof assistant +

+ The architecture of modern proof assistants is the result of several decades of experimentation, development and practical experience. A proof assistant incorporates not one, but several formal systems. +

+ The central component of a proof assistant is the kernel, which validates every inference step and makes sure that proofs are correct. It does so by implementing a formal system $F$ (the foundation) which is expressive enough to allow formalization of a large amount of mathematics, but also simple enough to allow an efficient and correct implementation. +

+ The foundational system implemented in the kernel is too rudimentary to be directly usable for sophisticated mathematics. Instead, the user writes their input in a more expressive formal language $V$ (the vernacular) that is designed to be practical and useful. Typically $V$ is quite complex so that it can accommodate various notational conventions and other accepted forms of mathematical expression. A second component of the proof assistant, the elaborator, translates $V$ to $F$ and passes the translations to the kernel for verification. +

+ A proof assistant may incorporate a third formal language $M$ (the meta-language), which is used to implement proof search, decision procedures, and other automation techniques. Because the purpose of $M$ is to implement algorithms, it typically resembles a programming language. The distinction between $M$ and $V$ may not be very sharp, and sometimes they are combined into a single formalism. From mathematical point of view, $M$ is less interesting than $F$ and $V$, so we shall ignore it. +

+ Suitability of foundation $F$ +

+ The correctness of the entire system depends on the correctness of the kernel. A bug in the kernel allows invalid proofs to be accepted, whereas a bug in any other component is just an annoyance. Therefore, the foundation $F$ should be simple so that we can implement it reliably. It should not be so exotic that logicians cannot tell how it relates to the accepted foundations of mathematics. Computers are fast, so it does not matter (too much) if the translation from $V$ to $F$ creates verbose statements. Also, $F$ need not be directly usable by humans. +

+ A suitable variant of set theory or type theory fits these criteria. Indeed Mizar is based on set theory, while HOL, Lean, Coq, and Agda use type theory in the kernel. Since both set theory and type theory are mathematically very well understood, and more or less equally expressive, the choice will hinge on technical criteria, such as availability and efficiency of proof-checking algorithms. +

+ Suitability of vernacular $V$ +

+ A much more interesting question is what makes the vernacular $V$ suitable. +

+ For the vernacular to be useful, it has to reflect mathematical practice as much as possible. It should allow expression of mathematical ideas and concepts directly in familiar terms, and without unnecessary formalistic hassle. On the other hand, $V$ should be a formal language so that the elaborator can translate it to the foundation $F$. +

+ To learn more about what makes $V$ good, we need to carefully observe how mathematicians actually write mathematics. They produce complex webs of definitions, theorems, and constructions, therefore $V$ should support management of large collections of formalized mathematics. In this regards we can learn a great deal by looking at how programmers organize software. For instance, saying that a body of mathematics is "just a series of definitions, theorems and proofs" is a naive idealization that works in certain contexts, but certainly not in practical formalization of mathematics. +

+ Mathematicians omit a great deal of information in their writings, and are quite willing to sacrifice formal correctness for succinctness. The reader is expected to fill in the missing details, and to rectify the imprecisions. The proof assistant is expected to do the same. To illustrate this point, consider the following snippet of mathematical text: +

+ Let $U$ and $V$ be vector spaces and $f : U \to V$ a linear map. Then $f(2 \cdot x + y) = 2 \cdot f(x) + f(y)$ for all $x$ and $y$. +

+ Did you understand it? Of course. But you might be quite surprised to learn how much guesswork and correction your brain carried out: +

  • + The field of scalars is not specified, but this does not prevent you from understanding the text. You simply assumed that there is some underlying field of scalars $K$. You might find out more about $K$ in subsequent text. ($K$ is an existential variable.) +

  • + Strictly speaking "$f : U \to V$" does not make sense because $U$ and $V$ are not sets, but structures $U = (|U|, 0_U, {+}_U, {-}_U, {\cdot}_U)$ and $V = (|V|, 0_V, {+}_V, {-}_V, {\cdot}_V)$. Of course, you correctly surmised that $f$ is a map between the carriers, i.e., $f : |U| \to |V|$. (You inserted an implicit coercion from a vector space to its carrier.) +

  • + What do $x$ and $y$ range over? For $f(x)$ and $f(y)$ to make sense, it must be the case that $x \in |U|$ and $y \in |U|$. (You inferred the domain of $x$ and $y$.) +

  • + In the equation, $+$ on the left-hand side means $+_{U}$, and $+$ on the right-hand side ${+}_V$, and similarly for scalar multiplication. (You reconstructed the implicit arguments of $+$.) +

  • + The symbol $2$ normally denotes a natural number, as every child knows, but clearly it is meant to denote the scalar $1_K +_K 1_K$. (You interpreed "$2$" in the notation scope appropriate for the situation at hand.) +

+ The vernacular $V$ must support these techniques, and many more, so that they can be implemented in the elaborator. It cannot be anything as simple as ZFC with first-order logic and definitional extensions, or bare Martin-Löf type theory. You may consider the development of $V$ to be outside of scope of mathematics and logic, but then do not complain when computer scientist fashion it after their technology. +

+ I have never seen any serious proposals for a vernacular based on set theory. Or to put it another way, as soon as we start expanding and transforming set theory to fit the requirements for $V$, we end up with a theoretical framework that looks a lot like type theory. (You may entertain yourself by thinking how set theory could be used to detect that $f : U \to V$ above does not make sense unless we insert coercions – for if everthying is a set then so are $U$ and $V$, in which case $f : U \to V$ does make sense.) +

+ Detecting mistakes +

+ An important aspect of suitability of foundation is its ability to detect mistakes. Of course, its purpose is to prevent logical errors, but there is more to mistakes than just violation of logic. There are formally meaningful statements which, with very high probability, are mistakes. Consider the following snippet, and read it carefully: +

+ Definition: A set $X$ is jaberwocky when for every $x \in X$ there exists a bryllyg $U \subseteq X$ and an uffish $K \subseteq X$ such that $x \in U$ and $U \in K$. +

+ Even if you have never read Lewis Carroll's works, you should wonder about "$U \in K$". It looks like "$U \subseteq K$" would make more sense, since $U$ and $K$ are both subsets of $X$. Nevertheless, a proof assistant whose foundation $F$ is based on ZFC will accept the above definition as valid, even though it is very unlikely that the human intended it. +

+ A proof assistant based on type theory would reject the definition by stating that "$U \in K$" is a type error. +

+ So suppose we use a set-theoretic foundation $F$ that accepts any syntactically valid formula as meaningful. In such a system writing "$U \in K$" is meaningful and therefore the above definition will be accepted by the kernel. If we want the proof assistant to actually assist the human, it has to contain an additional mechanism that will flag "$U \in K$" as suspect, despite the kernel being happy with it. But what is this additional mechanism, if not just a second kernel based on type theory? +

+ I am not saying that it is impossible to design a proof assistant based on set theory. After all, Mizar, the most venerable of them all, is designed precisely in this way – set theory with a layer of type-theoretic mechanisms on top. But I cannot help to wonder: why bother with the set-theoretic kernel that requires a type-theoretic fence to insulate the user from the unintended permissiveness of set theory? +

+ 30 +

+ EDIT: Since this question has gotten so much interest, I have decided to substantially rewrite my answer, stating explicitly here on MO some of the more important points rather than forcing the reader to follow links and chase down references. +

  1. To begin with, it is important to distinguish between what currently existing proof assistants can do versus what they could do if we put in the necessary development work. There is no doubt that existing type-theoretic proof assistants outperform existing set-theoretic proof assistants according to various important metrics, such as convenience, pre-existing libraries, etc. Someone who favors type-theoretic proof assistants therefore always has a trump card to play in these discussions—“What you say is nice in theory, but show me the money. How does your set-theoretic proof assistant perform in practice on real problems?” In an earlier version of this answer, I mentioned a talk by John Harrison entitled, “Let’s make set theory great again!” (part 1 part 2 slides), and Andrej Bauer asked the reasonable question (in the comments below) whether Harrison had implemented his ideas. As Jeremy Avigad has said, even though he thinks that the “ideal proof assistant would be based on ZFC, with enough practical infrastructure to support all the things we need to do mathematics,” “it is easy to underestimate the difficulties involved in designing a useful and workable system.” At the same time, if we take the long view, we should be careful not to mistake what might be an artifact of our current implementations for a fundamental truth. Larry Paulson has in effect said “show me the money” in a more literal sense:

+ I would guess that the amount of effort and funding going into type theory exceeds the amount that went into set theory by an order of magnitude if not two. It's not unusual to encounter open hostility to set theory and classical logic combined with an air of moral superiority: “Oh, you aren’t constructive? And you don't store proof objects? Really?” And I have seen "proof assistant" actually DEFINED as "a software system for doing mathematics in a constructive type theory".
+
+ The academic interest simply isn't there. Consider the huge achievements of the Mizar group and the minimal attention they have received. Also, I think that my 2002 paper on proving the reflection theorem (and presented at CADE, a high-profile conference) was really interesting, but it had been cited only six times, and two of those are by myself.
+
+ I am certain that we would now have highly usable and flexible proof assistants based on some form of axiomatic set theory if this objective had enjoyed half the effort that has gone into type theory-based systems in the past 25 years. +

  1. + A second point is that everyone acknowledges that having a system where the computer can help you catch silly mistakes is a huge benefit, if not an absolute necessity. For this, some kind of type-theory-like mechanism is very useful. However, this is not as decisive an argument in favor of type theory and against set theory as it might seem at first glance. The “working mathematician” is often tempted to regard the absurdity of a statement such as $2\in 3$ as a strong argument against set theory, but the working mathematician also tends to balk at giving $0/0$ a concrete value (instead of declaring it to be “undefined”), which is the sort of thing that many proof assistants do. In both cases, there are known ways to block “fake theorems.” It is standard engineering practice to develop systems that contain multiple layers (the distinction between vernacular and foundation in Andrej Bauer’s excellent answer is an example), and the fact that $2\in 3$ might be a theorem at some low layer does not automatically mean that this is something a user will be able to enter from the keyboard and not get caught by the system. In principle, therefore—to return to the actual question being asked—set theory does not seem to pose any intrinsic barriers to automation. Indeed, other answers and comments have made this point, and explained how powerful automation tactics can and have been written in set-theoretic systems such as Metamath. Another example is the work of Bohua Zhan on auto2, which has shown that many of the alleged difficulties with untyped set theory can be overcome. +

  2. + There remains the question, even if a set-theoretic proof assistant with the power and usability of Coq/Lean/Isabelle could be developed, what would be the point? Aren't the already existing type-theoretic assistants good enough? This is a much more “subjective and argumentative” point, but I would propose a couple of arguments in favor of set theory. The first is that set theory has a great deal of flexibility, and it is not an accident that historically, the first convincing demonstration that all of mathematics could be put on a single, common foundation was accomplished using set theory rather than type theory. With a relatively small amount of training, mathematicians can see how to formulate any concepts and proofs in their field of expertise in set-theoretic terms. In the language of Penelope Maddy’s paper, What do we want a foundation to do? set theory provides a Generous Arena and a Shared Standard for all of mathematics with minimal fuss. Of course, there is a price to be paid if we give someone enough rope—they might decide to hang themselves. But if we want to see widespread adoption of proof assistants by the mathematical community, then we should take seriously any opportunities we have to leverage mathematicians’ existing habits of thought. I do not think that it is an accident that set-theoretic proof assistants tend to produce more human-readable proofs than type-theoretic proof assistants do (though I will admit that this could be an artifact of existing systems, rather than a fundamental truth).
    +
    + Another argument is that if we are interested in reverse mathematics—which axioms are needed to prove which theorems—then there has been a lot more work done to calibrate mathematics against set-theoretic and arithmetical systems than against type-theoretic systems. In Maddy’s language, we might hope for a proof assistant to help us with Risk Assessment and Metamathematical Corral. This does not seem to be a priority with too many people at the present time, but again I am trying to take the long view here. The mathematical community already has a good grasp of how the mathematical universe can be built from the ground up using set theory, and exactly what ingredients are needed to achieve which results. It would be desirable for our proof assistants to be able to capture this global picture.
    +
    + One could point out that someone who is really interested in set theory can use something like Isabelle/ZF, which builds set theory on top of type theory. That is true. I am not trying to argue here that a set-theoretic foundation with some kind of type theory layered on top is necessarily better than a type-theoretic foundation with some kind of set theory layered on top. I am only trying to argue that set theory does enjoy some advantages over type theory, depending on what you are trying to achieve, and that the claim that “automation is very difficult with set theory,” or that there is nothing to be gained by using set theory as the basis for a proof assistant, should be taken with a grain of salt. +

+ Let me conclude with a remark about Lean specifically. A couple of years ago, Tom Hales provided a review of the Lean theorem prover that spells out the pros and cons as he saw them at the time. Some of what he said may no longer be true today, but one thing that is true is that even Lean enthusiasts agree that there are flaws that they promise will be fixed in Lean Version 4 (which unfortunately is going to be incompatible with Lean 3, or so I hear). +

+ 27 +

+ I still find it very surprising that this random talk I gave attracts so much attention, especially as not everything I said was very well thought out. I am more than happy to engage with people in discussions about what I said and whether or not some things I said were ill-informed. +

+ But onto my answer to your question: whilst I am not an expert in proof assistants in general (I have become knowledgeable at precisely one proof assistant and have limited experience with others), it is my empirical observation that high-level tactics like Lean's ring tactic, which will prove results like $(x+2y)^3=x^3+6x^2y+12xy^2+8y^3$ immediately -- and there are similar tactics in Coq and Isabelle/HOL, two more type theory systems -- do not seem to exist in the two main set theory formal proof systems, namely Metamath and Mizar. I don't really know why, but those are the facts. Note that the proof of this from the axioms of a ring is extremely long and uncomfortable, because you need to apply associativity and commutativity of addition and multiplication many times -- things mathematicians do almost without thinking. +

+ 6 +

+ I'll answer just the automation question since the other answers gave nice broad overview, but didn't seem focus on that narrow question. My own direct automation experience is wrt to ACL2, Lean and SMT-based solvers. +

+ Strictly speaking, I don't know if there's any foundational argument for why set theory would be better or worse than the type theory-based approach in Lean. +

+ The strengths that Lean have from my perspective are: an expressive explicit type system, a relatively simple core language for representing terms, and a attention to how terms are represented for efficient manipulation. +

+ With regards to typed core logics, most automation in theorem provers is tailored to specific common theories that are widely used in mathematics. When writing such automation, it is important to know the types and operations involved. For example, in writing a decision procedure for linear arithmetic in an untyped language, one needs to carefully check that any transformations still make sense even if the expressions do not denote numbers. By having a typed and typechecked expression language, one gets from the theorem prover itself and does not have to pay the additional runtime and complexity costs. +

+ A second strength of Lean is ensuring that the core language is simple, but expressive so that one can represent proofs compactly. When using automation such as SMT solvers, the "proof terms" generated as evidence can be very large and the core proof language needs to be designed to compactly represent proofs while still be amenably to efficient checking. I'm not sure if Lean has an advantage to Coq or other solvers here per se but it is a factor in Lean's design. +

+ A third strength of Lean is that the language for writing tactics and creating definitions and theorems are one and the same. There is a bit of syntactic sugar for the tactic sequences and a tactic-specific library, but by having the same language one does not have to learn multiple languages just to get started writing tactics. Lean is also not unique here -- ACL2 is similar, but it is a strength of Lean still. It will also become even more relevant with Lean 4 thanks to the efficient compiler being developed. +

+ answered Nov 20 '20 at 20:31 +
+ -2 +

+ Modern programming languages unlike older programming languages, or unlike assembly and machine code are typed. For example c++ with or without templates compared to c, or Java compared to Basic. +

+ With typing, a compiler needs to check that all expressions have the correct type. Generally, this is done by telling the compiler what types are being used. However, we can attempt to automate this process and allow the compiler to work out what the types are by itself, given the base types under consideration. This is known as type inference. +

+ Depending upon the language, type checking and inference can range from trivial to undecidable. +

+ We also say it is sound iff it accepts correctly typed programs. +

+ Now, type checking and inference in the simply typed Lambda Calculus with parametric polymorphism is decidable and one implementation of this is the Hindley-Milner Type Sysyem where complexity is linear in the size of the program; however, the problem is PSPACE hard and EXPTIME complete, meaning that in the worst case scenario it requires a polynomial worth of extra space and an exponential amount of extra time. Luckily, it happens to be linear when the nesting depth of polymorphic variables is bounded. +

+ For example, +

  1. C++ with templates is Turing complete by an informal proof by Veldhuizen '03. This means type checking and inference is undecidable.
  1. In C#, both type checking and inference is undecidable. And it is unsound.
  1. Similarly Java because Java Generics are Turing Complete. Amin and Tate '15 showed that Java 5 or later is unsound.
  1. Haskell has decidable type checking and inference. But with enough extensions, that is to at least System F, then it becomes undecidable as System F is.

+ Dependent types allow types to depend upon their values and so give a finer control over types. For example, with ordinary types you cannot have a type for even numbers; with dependent types you can. Unfortunately, adding this capability generally renders type inference undecidable. +

+ Lean is a programming language which is based upon Calculus of Constructions with Inductive Tupes. This calculus is a higher order typed Lambda Calculus and lies at the top node of Barendregt's Lambda Cube. This means it has polymorphism and dependent types. +

+ It's main rival, is Martin Lof's Type Theory which is also dependent and inductive types. It comes in two main varieties, intensional and extensional. The intensional variety can be alternatively thought of as Homotopy Type Theory. In this variety, type inference is decidable whilst in the extensional variety it is not. +

+ Set Theory like machine code, like assembly and like early programming languages are not typed. So type checking does not apply. As types allow us to program more safely, these languages are not as safe as typeful languages and hence, automation is more difficult here as we cannot prove they will act as we expect them to do so. +

+ Typed languages does not mean that untyped languages will go away. After all, the languages at all in fact nested. Haskell, at bottom, is still executed by machine code and this is untyped. Hence, development of both is required for automation in all it's varieties. +

+ After all, if I had to program the fastest loop to add as many consecutive integers in a second as I could on a particular machine I would choose to write it in machine code, I typed as it is - every time. More importantly, in my opinion it's makes one realise how the whole stack of languages from the editor through the compiler and then through the interface and interrupts down to the metal works. In this way, you learn more about how every works 'under the hood' - so to speak. +

+ edit +

+ @LSpice: It's written this way because the question itself is kind of vague. Why not ask the OP to tighten up the question? The proposed question is answered in the last paragraph. And the preceding paragraphs simply explains my terms and my thinking in a step-by-step pedagogical manner rather than a deus ex-machina manner ... or do you think explanatory text is a little too old-fashioned for the hyped up and glib virtual world? +

\ No newline at end of file diff --git a/packages/readabilityjs/test/test-pages/medium/distiller.html b/packages/readabilityjs/test/test-pages/medium/distiller.html new file mode 100644 index 000000000..7486deb50 --- /dev/null +++ b/packages/readabilityjs/test/test-pages/medium/distiller.html @@ -0,0 +1,122 @@ +
Image for post
+ Image credits: Pixabay +

+ Microservices — a definition +

+ The term ‘micro’ in Microservices, though indicative of the size of a service, is not the only criteria that make an application a Microservice. When teams move to a microservices-based architecture, they aim to increase their agility — deploy features autonomously and frequently. It’s hard to pin down a single concise definition of this architectural style. I liked this short definition from Adrian Cockcroft — “service‑oriented architecture composed of loosely coupled elements that have bounded contexts.” +

+ Though this defines a high-level design heuristic, Microservices architecture has some unique characteristics that differentiate it from the service-oriented architecture of the yesteryear. A few of those characteristics, below. These and a few more are well-documented — Martin Fowler’s article and Sam Newman’s Building Microservices, to name a few. +

  1. Services have well-defined boundaries centered around business context, and not around arbitrary technical abstractions
  2. Hide implementation detail and expose functionality through intention-revealing interfaces
  3. Services don’t share their internal structures beyond their boundaries. For example, no sharing of databases.
  4. Services are resilient to failures.
  5. Teams own their functions independently and have the ability to release changes autonomously
  6. Teams embrace a culture of automation. For example, automated testing, continuous integration and continuous delivery

+ In short, we can summarize this architecture style as below: +

+ Loosely coupled service-oriented architecture, where each service is enclosed within a well-defined bounded context, enabling rapid, frequent, and reliable delivery of applications. +

+ Domain-driven design and Bounded contexts +

+ The power of microservices comes from clearly defining their responsibility and demarcating the boundary between them. The purpose here is to build high cohesion within the boundary and low coupling outside of it. That is, things that tend to change together should belong together. As in many real-life problems, this is easier said than done — businesses evolve, and assumptions change. Hence the ability to refactor is another critical thing to consider when designing systems. +

+ Domain-driven design (DDD) is a key, and in our opinion, a necessary tool when designing microservices, be it breaking a monolith or implementing a greenfield project. Domain-driven design, made famous by Eric Evans by his book , is a set of ideas, principles, and patterns that help design software systems based on the underlying model of the business domain. The developers and domain experts work together to create business models in a Ubiquitous common language. They then bind these models to systems where they make sense, establish collaboration protocols between these systems and the teams that work on these services. More importantly, they design the conceptual contours or boundaries between the systems. +

+ Microservice design draws inspiration from these concepts as all of these principles help build modular systems that can change and evolve independently of one another. +

+ Before we proceed further, let’s quickly go through some of the basic terminologies of DDD. A full overview of Domain-Driven Design is out of scope for this blog. We highly recommend Eric Evans’ book to anyone trying to build microservices +

+ Domain: Represents what an organization does. In the below example, it would be Retail or eCommerce. +

+ Subdomain: An organization or business unit within an organization. A domain is composed of multiple subdomains. +

+ Ubiquitous language: This is the language used to express the models. In the example below, Item is a Model that belongs to the Ubiquitous language of each of these subdomains. Developers, Product Managers, domain experts, and business stakeholders agree on the same language and use it in their artifacts — Code, Product Documentation, and so on. +

Image for post
Fig 1. Sub-domains and Bounded Contexts in eCommerce domain

+ Bounded Contexts: Domain-driven design defines Bounded contexts as “The setting in which a word or a statement appears that determines its meaning.” In short, this means the boundary within which a model makes sense. In the above example, “Item” takes on a different meaning in each of those contexts. In the Catalog context, an Item means a sell-able product, whereas, in Cart context, it means the item that the customer has added to her cart. In Fulfillment context, it means a Warehouse Item that will be shipped to the customer. Each of these models is different, and each has a different meaning and possibly contains different attributes. By separating and isolating these models within their respective boundaries, we can express the models freely and without ambiguity. +

+ Note: It’s essential to understand the distinction between Subdomains and Bounded contexts. A subdomain belongs in the problem space, that is, how your business sees the problem, whereas Bounded contexts belong in the solution space, that is, how we will implement the solution to the problem. Theoretically, each subdomain may have multiple bounded contexts, though we strive for one bounded context per subdomain. +

+ How are Microservices related to Bounded contexts +

+ Now, where do Microservices fit? Is it fair to say that each bounded context maps to a microservice? Yes and no. We will see why. There may be cases where the boundary or contour of your bounded context is quite large. +

Image for post
Fig 2. Bounded context and microservices

+ Consider the example above. The Pricing bounded context has three distinct models — Price, Priced items, and Discounts, each responsible for the price of a catalog item, computing the total price of a list of items, and applying discounts respectively. We could create a single system that encompasses all the above models, but it could become an unreasonably large application. Each of the data models, as mentioned earlier, have their invariants and business rules. Over time, if we are not careful, the system could become a Big ball of mud with obscured boundaries, overlapping responsibilities, and probably back to where we started — a monolith. +

+ Another way to model this system is to separate, or group related models into separate microservices. In DDD, these models — Price, Priced Items, and Discounts — are called Aggregates. An aggregate is a self-contained model that composes related models. You could change the state of an aggregate only through a published interface, and the aggregate ensures consistency and that the invariants hold good. +

+ Formally, An Aggregate is a cluster of associated objects treated as a unit for data changes. External references are restricted to one member of the AGGREGATE, designated as the root. A set of consistency rules applies within the AGGREGATE’S boundaries. +

Image for post
Fig 3. Microservices in the Pricing Context

+ Again, it’s not necessary to model every aggregate as a distinct microservice. It turned out to be so for the services (aggregates) in Fig 3, but that’s not necessarily a rule. In some cases, it may make sense to host multiple aggregates in a single service, particularly when we don’t fully understand the business domain. An important thing to note is that consistency can be guaranteed only within a single aggregate, and the aggregates can only be modified through the published interface. Any violation of these bears the risk of turning into a big ball of mud. +

+ Context maps — A way to carve out accurate microservice boundaries +

+ Another essential toolkit in your arsenal is the concept of Context maps — again, from Domain Driven Design. A monolith is usually composed of disparate models, mostly tightly coupled — models perhaps know the intimate details of one another, changing one could cause side effects on another, and so on. As you break down the monolith, it’s vital to identify these models — aggregates in this case — and their relationships. Context maps help us do just that. They are used to identify and define relationships between various bounded contexts and aggregates. While bounded contexts define the boundary of a model — Price, Discounts, etc. in the example above, Context maps define the relationships between these models and between different contexts. After identifying these dependencies, we can determine the right collaboration model between the teams that will implement these services. +

+ A full exploration of Context maps is beyond the scope of this blog, but we’ll illustrate it with an example. The below diagram represents the various applications that handle payments for an eCommerce order. +

  1. The cart context takes care of online authorizations of an order; Order context processes post fulfillment payment processes like Settlements; Contact center handles any exceptions like retrying payments and changing the payment method used for the order
  2. For the sake of simplicity, let’s assume that all these contexts are implemented as separate services
  3. All these contexts encapsulate the same model.
  4. Note that these models are logically the same. That is, they all follow the same Ubiquitous domain language — payment methods, authorizations, and settlements. Just that they are a part of different contexts.

+ Another sign that the same model is spread around different contexts is that all of these integrate directly with a single payment gateway and do the same operations as one another +

Image for post
Fig 4. An incorrectly defined context map

+ Redefining the service boundaries — Map the aggregates to the right contexts +

+ There are a few problems that are very evident in the above design (Fig 4). Payments aggregate is part of multiple contexts. It’s impossible to enforce invariants and consistency across various services, not to mention the concurrency issues between these services. For example, what happens if the contact center changes the payment method associated with the order while the Orders service is trying to post settlement of a previously submitted payment method. Also, note that any change in the payment gateway would force changes to multiple services and potentially numerous teams, as different groups could own these contexts. +

+ With a few adjustments and aligning the aggregates to the right contexts, we get a much better representation of these subdomains — Fig 5. There is a lot that has changed. Let’s review the changes: +

  1. Payments aggregate has a new home — Payment service. This service also abstracts the Payment gateway from the other services that require payment services. As a single bounded context now owns an aggregate, the invariants are easy to manage; all transactions happen within the same service boundary helping to avoid any concurrency issues.
  2. Payments aggregate uses an Anti-corruption Layer (ACL) to isolate the core domain model from the payment gateway’s data model, which is usually a third-party provider and perhaps bound to change. We’ll dive deeper into the application design of such as service using Ports and Adapters pattern in a future post. The ACL layer usually contains the adapters that transform the data model of the payment gateway to the Payments aggregate data model.
  3. Cart service calls the Payments service through direct API calls as the cart service may have to complete the payment authorization while the customers are on the website
  4. Make a note of the interaction between Orders and Payment service. Orders service emits a domain event (more on this later in this blog). Payments service listens to this event and completes settlement of the order
  5. Contact center service may have many aggregates, but we are only interested in the Orders aggregate for this use case. This service emits an event when the payment method changes, and the Payments service reacts to it by reversing the credit card used prior and processing the new credit card.
Image for post
Fig 5. Redefined context map

+ Usually, a monolithic or a legacy application has many aggregates, often with overlapping boundaries. Creating a context map of these aggregates and their dependencies helps us understand the contours of any new microservices that we will wrest out of these monoliths. Remember, the success or failure of microservices architecture hinges upon low coupling between the aggregates and high cohesion within these aggregates. +

+ It’s also important to note that bounded contexts are themselves suitable cohesive units. Even if a context has multiple aggregates, the entire context, along with its aggregates, can be composed into a single microservice. We find this heuristic particularly useful for domains that are a bit obscure — think about a new line of business the organization is venturing into. You may not have sufficient insight into the right boundaries of separation, and any premature decomposition of aggregates can lead to expensive refactoring. Imagine having to merge two databases into one, along with data migration, because we happened to find that two aggregates belong together. But ensure these aggregates are sufficiently isolated through interfaces so that they don’t know the intricate details of one another. +

+ Event Storming — Another technique to identify service boundaries +

+ Event Storming is another essential technique to identify aggregates (and hence microservices) in a system. It is a useful tool both for breaking monoliths and when designing a complex ecosystem of microservices. We have used this technique to break down one of our complex applications, and we intend to cover our experiences with Event Storming in a separate blog. For the scope of this blog, we want to give a quick high-level overview. Please watch Alberto Brandelloni’s video on the topic if you are interested in exploring further. +

+ In a nutshell, Event Storming is a brainstorming exercise between the teams that work on an application — in our case, a monolith — to identify the various domain events and processes that happen within a system. The teams also identify the aggregates or models these events affect and any subsequent impacts thereof. As the teams do this exercise, they identify different overlapping concepts, ambiguous domain language, and conflicting business processes. They group related models, redefine aggregates and identify duplicate processes. As they progress with this exercise, the bounded contexts where these aggregates belong becomes clear. Event Storming workshops are useful if all the teams are in a single room — physical or virtual — and start to map the events, commands, and processes on a scrum-style whiteboard. At the end of this exercise, below are the usual outcomes: +

  1. Redefined list of Aggregates. These potentially become new microservices
  2. Domain Events that need to flow between these microservices
  3. Commands which are direct invocations from other applications or users

+ We’ve shown a sample board at the end of an Event Storming workshop below. It’s a great collaborative exercise for the teams to agree on the right aggregates and bounded contexts. Besides being a great team-building exercise, the teams come out of this session with a shared understanding of the domain, ubiquitous language, and precise service boundaries. +

Image for post
Fig 6. Event Storming board

+ Communication between microservices +

+ To quickly recap, a monolith hosts multiple aggregates within a single process boundary. Hence managing consistency of aggregates within this boundary is possible. For example, if a Customer places an Order, we can decrement the Inventory of the items, send an email to the Customer — all within a single transaction. All operations would succeed, or all would fail. But, as we break the monolith and spread the aggregates into different contexts, we will have tens or even hundreds of microservices. The processes that hitherto existed within the single boundary of a monolith is now spread across multiple distributed systems. Achieving transactional integrity and consistency across all of these distributed systems is very hard, and it comes at a cost — the availability of the systems. +

+ Microservices are distributed-systems as well. Hence, CAP theorem applies to them as well — “a distributed system can deliver only two of three desired characteristics: consistency, availability, and partition tolerance (the ‘C,’ ‘A’ and ‘P’ in CAP).” In real-world systems, partition tolerance is not negotiable — network is unreliable, virtual machines can go down, latency between regions can become worse, and so on. +

+ So that leaves us with a choice of either Availability or Consistency. Now, we know that in any modern application, sacrificing availability is not a good idea either. +

Image for post
Fig 7. CAP Theorem

+ Design applications around eventual consistency +

+ If you try to build transactions across several distributed systems, you’ll end up in the monolith land again. Only this time it will be the worst kind, a distributed monolith. If any of the systems become unavailable, the whole process becomes unavailable, often leading to frustrating customer experience, failed promises, and so on. Besides, changes to one service may usually entail changes to another service, leading to complex and costly deployments. Hence, we are better off designing applications tailoring our use cases to tolerate a little bit of inconsistency in favor of availability. For the example above, we can make all the processes asynchronous and hence eventually consistent. We can send emails asynchronously, independent of the other processes; If an item that’s promised is not available in the warehouse later, the item could be back-ordered, or we could stop taking orders for the item beyond a certain threshold.
+ Occasionally, you may encounter a scenario that might require strong ACID-style transactions across two aggregates in different process boundaries. That’s an excellent sign to revisit these aggregates and perhaps combine them into one. Event Storming and Context Maps will help identify these dependencies early on before we start breaking up these aggregates in different process boundaries. Merging two microservices into one is costly, and that’s something we should strive to avoid. +

+ Favor event-driven architecture +

+ Microservices can emit essential changes that happen to their aggregates. These are called Domain events, and any services that are interested in these changes can listen to these events and take respective action within their domains. This method avoids any behavioral coupling — one domain doesn’t prescribe what the other domains should do, and temporal coupling — the successful completion of a process doesn’t depend on all the systems to be available at the same time. This, of course, will mean that the systems will be eventually consistent. +

Image for post
Fig 8. Event driven architecture

+ In the example above, Orders service publishes an event — Order Cancelled. The other services that have subscribed to the event process their respective domain functions: Payment service refunds the money, Inventory service adjusts the inventory of the items, and so on. Few things to note to ensure the reliability and resiliency of this integration: +

  1. Producers should ensure that they produce an event at least once. If there are failures in doing so, they should ensure there is a fall back mechanism present to re-trigger the events
  2. Consumers should ensure that they consume the events in an idempotent way. If the same event occurs again, there shouldn’t be any side effect at the consumer end. Events may also arrive out of sequence. Consumers can use Timestamp or version numbers fields to guarantee the uniqueness of the events.

+ It may not always be possible to use event-based integration due to the nature of some use cases. Please take a look at the integration between the Cart service and Payment service. It’s a synchronous integration, and hence has a few things we should watch out. It’s an example of behavioral coupling — Cart service perhaps calls a REST API from Payment service and instructs it to authorize the payment for an order, and temporal coupling — Payment service needs to be available for Cart service to accept an order. This kind of coupling reduces the autonomy of these contexts and maybe an undesirable dependency. There are a few ways to avoid this coupling, but with all these options, we will lose the ability to provide instant feedback to the customers. +

  1. Convert the REST API to an event-based integration. But this option may not be available if the Payment Service exposes only a REST API
  2. Cart service accepts an order instantaneously, and there is a batch job which picks up the orders and calls the Payment service API
  3. Cart service produces a local event which then calls the Payment service API

+ A combination of the above with retries in case of failures and unavailability of the upstream dependency — Payment service — can result in a much more resilient design. For example, the synchronous integration between the Cart and the Payment services can be backed up by an event or batch-based retries in cases of failures. This approach has an added impact on the customer experience — the customers may have entered incorrect payment details, and we will not have them online when we process the payments offline. Or there may be an added cost to the business to reclaim failed payments. But in all likelihood, the benefits of the Cart service being resilient to the Payment service’s unavailability or faults outweigh the shortcomings. For example, we can notify the customers if we are not able to collect payments offline. In short, there are trade-offs between user experience, resiliency, and operating costs, and it’s wise to design systems, keeping these compromises in mind. +

+ Avoid orchestration between services for consumer-specific data needs +

+ One of the anti-patterns in any service-oriented architecture is that the services cater to the specific access patterns of consumers. Usually, this happens when the consumer teams work closely with the service teams. If the team was working on a monolithic application, they would often create a single API that crosses different aggregate boundaries, hence tightly coupling these aggregates. Let’s consider an example. Say the Order Details page in Web, and Mobile applications need to show the details of both an Order and the details of the refunds processed for the order on a single page. In a monolithic application, an Order GET API — assuming it’s REST API — queries Orders and Refunds together, consolidates both the aggregates and sends a composite response to the callers. It’s possible to do this without a lot of overhead as the aggregates belong to the same process boundary. Thus, consumers can get all the necessary data in a single call. +

+ If Orders and Refunds are part of different contexts, the data is no longer present within a single microservice or aggregate boundary. One option to retain the same functionality for the consumers is by making Order service responsible for calling the Refunds service and create a composite response. This approach causes several concerns: +

+ 1. Order service now integrates with another service purely for supporting the consumers who need the Refund data along with Order data. Order service is less autonomous now as any changes in Refunds aggregate will lead to a change to the Order aggregate. +

+ 2. Order service has another integration and hence another failure point to take into account — if Refunds service is down, can Order Service still send partial data, and can the consumers fail gracefully? +

+ 3. If the consumers need a change to fetch more data from the Refunds aggregate, two teams are involved now to make this change +

+ 4. This pattern, if followed across the platform, can lead to an intricate web of dependencies between the various domain services, all because these services cater to the callers’ specific access patterns. +

+ Backend for Frontends (BFFs) +

+ An approach to mitigate this risk is to let the consumer teams manage the orchestration between the various domain services. After all, the callers know the access patterns better and can be in complete control of any changes to these patterns. This approach decouples the domain services from the presentation tier, letting them focus on the core business processes. But if the web and mobile apps begin to call different services directly instead of the one composite API from the monolith, it may cause performance overhead to these apps — multiple calls over lower bandwidth networks, processing and merging data from different APIs, and so on. +

+ Instead, one could use another pattern called Backend for Front-ends. In this design pattern, a backend service created and managed by the consumers — in this case, the web and mobile teams — takes care of integration across multiple domain services purely to render the front-end experience to the customers. The web and mobile teams can now design the data contracts based on the use cases that they cater to. They can even use GraphQL instead of REST APIs to flexibly query and get back exactly what they need. It’s important to note that this service is owned and maintained by the consumer teams and not by the teams that own the domain services. The front-end teams can now optimize based on their needs — a mobile app can request a smaller payload, reduce the number of calls from the mobile app, and so on. Take a look at the revised view of the orchestration below. The BFF service now calls both Orders and Refunds domain services for its use case. +

Image for post
Fig 9. Backend for Frontends

+ It’s also useful to build the BFF service early on, before breaking a plethora of services from the monolith. Otherwise, either the domain services will have to support the inter-domain orchestration, or the web and mobile apps will have to call multiple services directly from the front-end. Both of these options will lead to performance overhead, throw-away work, and lack of autonomy between the teams. +

+ Conclusion +

+ In this blog, we touched upon various concepts, strategies, and design heuristics to consider when we venture into the world of microservices, more specifically when we try to break a monolith into multiple domain-based microservices. Many of these are vast topics on their own, and I don’t think we’ve done enough justice to explain them in full detail, but we wanted to introduce some of the critical topics and our experience in adopting these. Further Reading (link) section has some references and some useful content for anyone who wishes to pursue this path. +

+ Update: The next two blogs in the series are out. These two blogs discuss implementing the Cart microservice, with code examples, using Domain-Driven Design principles, and Ports and Adapters design patterns. The primary focus of these blogs is to demonstrate how these two principles/patterns help us build modular applications that are agile, testable, and refactorable — in short, be able to respond to the fast-paced environment we are all operating on. +

\ No newline at end of file diff --git a/packages/readabilityjs/test/test-pages/milkroad/distiller.html b/packages/readabilityjs/test/test-pages/milkroad/distiller.html new file mode 100644 index 000000000..fbebe2a63 --- /dev/null +++ b/packages/readabilityjs/test/test-pages/milkroad/distiller.html @@ -0,0 +1,181 @@ +

+ By Shaan Puri & Ben Levy  • +

+ GM. This is the Milk Road, we cut the crypto sandwich into triangles, just the way you like it.  +

+ Today's estimated read time: 4 minutes & 7 seconds +

+ We’re skipping price action today, because we have a LOT of juicy stuff to go over.  +

+ Juicy? What happened? +

+ Day 2 of The Bitcoin Conference was fire, so I watched all the talks and summarized all the important stuff for you. +

+ "WARREN BUFFETT IS A SOCIOPATHIC GRANDPA FROM OMAHA" - PETER THIEL +

+ What happened:  +

+ Peter Thiel (co–founder of Paypal, and first investor in Facebook) got on stage and gave a crazy talk.  +

+ I kept a running diary of my thoughts during his talk:  +

+ 0:00 - Wow, they got Peter Thiel to talk. *gets pen & notepad out* +

+ 0:04 - They start by playing a video clip of Peter talking at a conference in 1999. He’s making 2 predictions:  +

  1. It’s 1999, but he’s predicting that 1B people will have cell phones connected to the internet in 5 years (he’s right, but it took about 10 years instead of 5)
  2. People will have money on their cell phone. It will be a currency that isn’t controlled by their local government (sounds like crypto, without saying crypto)

+ 2:40 - OK, the real Peter is on stage now. Crowd cheers. They sound a little tipsy even though it’s 11am.. Nice. This is why we do conferences in Miami.  +

+ He starts off with a bold move. He’s waving a stack of $100 bills in the air. +

+ He asks: “what is money?,” “this always gets people's attention,” “it’s not good as toilet paper. It’s not wallpaper. But people want it. Do you want it? Come get it!” +

+ 3:16 - He throws the wad of cash at someone in the crowd then laughs as they fight over it: “I thought you guys were supposed to be Bitcon maximalists.” The joke lands. +

+ 3:30 - He talks about the early days of PayPal. Shows a picture of him and Elon Musk (they were both CEOs of Paypal early on). Pretty wild to think about how much talent was in 1 startup.  +

+ Here’s who worked at Paypal:  +

  • Peter - created PayPal, first investor in Facebook
  • Elon - CEO of Tesla, SpaceX
  • The founders of YouTube
  • The founder of LinkedIn
  • The founder of Yelp
  • The founder of Kiva.org
  • The founder of Yammer
  • The COO of Square
  • The future CEO of Reddit

+ This is called the PayPal mafia^. What a killer roster of talent. OK, back to Peter’s talk:  +

+ 3:55 - He shows a slide from the first pitch deck of PayPal. The initial idea was a lot like Bitcoin. A currency that lived outside of the banking system. He said “we didn’t know anything” back then, but they had big ambitions. +

+ 4:12 - But that was really hard. So within a couple years, they had created PayPal. Which worked with the banking system instead of replacing it. This was more practical, but less ambitious. It was just a payment system, not a new kind of money.  +

+ 6:49 - Payment system vs. Money. Why does it matter?  +

+ Peter points out an important thing about money. Velocity.  +

+ The simple idea is that people will spend their “less valuable” money first. Back in the day, people would hoard gold, and spend silver.  +

+ Today, people will save Bitcoin, and spend dollars. +

+ This means some money moves “fast” throughout the economy (hopping from person to person). To most people, that makes a currency more valuable! It’s being used! Must be more valuable, right? +

+ The velocity theory says that’s wrong. The slower moving money will end up being higher priced. Nobody is ‘holding’ the fast money, so it doesn’t go up in price (sellers willing to sell).  +

+ For a deep dive on this, read this Velocity of Money post. +

+ Anyways, Peter’s belief is that Bitcoin is “slow money” and therefore, it’s valuable money. He says Ethereum is a payments network, high velocity, and lower value. +

+ If Peter’s right, Bitcoin will end up a lot more valuable than ETH over time. (Editor’s note: I totally disagree. I think he’s wrong in thinking ETH is a payments network) +

+ 7:36 - OK, this is where the talk starts getting aggressive.  +

+ He puts up this slide. The crowd goes wild. Crypto is so tribal. +

+ He is trying to talk but you can barely hear him, because the red-blooded-american crowd is going wild for no reason. +

+ 8:13 - He says Bitcoin is like gold (a store of value). And ETH is like Visa (a payments network).  +

+ He points out that Gold has a total market value of $12T, and the biggest payment network (Visa) is worth 26x less.  +

+ If Peter is right, Bitcoin has more upside than ETH (again, I disagree,but let’s let Peter talk. He’s the billionaire chess grandmaster. I’m just a hairy dude with a free newsletter) +

+ 11:26 - He puts up this slide showing that back in the day (1980) Gold was equal to the total value of all stocks. (Both were $2.5T)  +

+ Fast forward to today, and stocks are worth 10x the total value of gold. +

+ His point is that thinking of Bitcoin as “digital gold” is underselling how big it can be. Perhaps it will be on par with the S&P 500 (which today is a way that people store their wealth).  +

+ 12:49 - Bitcoin is $43k today, where does it go?  +

+ “Bitcoin is the most honest market in the world. It went from $5k to $50K in the last 2 years, showing us that inflation was real, and the central banks are bankrupt. This is the end of the fiat money regime.”  +

+ “Mr. Powell should be extremely grateful to Bitcoin. It’s the last warning they are going to get. They have chosen to ignore it, and will pay the consequences for it in the years ahead” +

+ 14:00 - Who are the enemies of Bitcoin? Who is holding it back from a 10x or 100x rise? +

+ Bitcoin has 3 enemy types:  +

+ Enemy #1 - The Sociopathic Grandpa from Omaha  +

+ He’s honest about his hate for Bitcoin. He’s incentivized to hate Bitcoin for 2 reasons:  +

  1. He’s a winner of the current system
  2. He’s a money manager. If all people need to do is “buy bitcoin and chill” for a decade, all money managers are out of business

+ Enemy #2 - The NYC Bankers  +

+ Like Jamie Dimon from JP Morgan.  +

+ For years they called crypto “worthless.” Now it is undeniable, and they are losing customers, so they’re pivoting to offering crypto assets.  +

+ Now you go to GoldmanSachs.com and see stuff like this: +

+ Enemy #3 - Nameless, Faceless Bureaucrats +

+ These people hide behind a more passive-aggressive tone. They say things like “I see huge opportunity in blockchain”  +

+ Anyone who says they are “pro-blockchain” is actually “anti-bitcoin” according to Peter.  +

+ 20:36- The Grand Finale +

+ He ended with this hilarious slide design. He calls these enemies the “gerontocracy” (aka old rich). I gotta say, whoever photoshopped this slide is my hero. Pure art. Genius. Like Picasso in his prime.  +

+ He says it’s OLD vs. YOUTH. And calls bitcoin a revolutionary youth movement. His final line: “We have to leave this conference and take over the world.”   +

+ *adds take-over-the-world to my to-do list* +

+ The Milk Road’s Take:  +

+ I’ve listened to Peter Thiel for over a decade. The guy is super smart and loves to go against the grain. He’s great at doing this, painting an enemy, and getting people’s attention with bold claims.  +

+ On one hand, I love the conviction he has for Bitcoin, and think that it’s cool he’s been predicting this since 1999.  +

+ On the other hand, I think he’s wrong about Ethereum, and I find the whole tribal / “us against the world” thing from the conference to be a turnoff. The macho attitude just feels very forced when nerds do it.  +

+ I mean look at this. Michael Saylor came out for his talk and hit the crowd with the 1999 “Raise the Roof” move 🤣 +

+ That wasn’t all for Day 2… +

+ Jack Mallers, the founder of Strike, announced they have partnered with the POS  companies that power huge stores like Starbucks, McDonalds, Walmart & Walgreens to allow anyone to pay in-store with Bitcoin over the lightning network. Every credit card swipe has a 3% fee and is slow to settle. Lightning is essentially free (~0% fee) and settles instantly.  +

+ Cash App announces their new crypto services that will 1) let users auto-invest a percent of their paychecks into bitcoin and 2) round payments up to the nearest dollar to buy bitcoin with the difference (kinda like Acorns).  +

+ Robinhood announced they’re rolling out their new crypto wallet to 2M users. +

+ Ricardo Salinas, a Mexican billionaire, says his liquid portfolio is 60% Bitcoin. In 2020, Bitcoin only made up about 10% of his portfolio.  That’s what we call growth, baby! +

+ TODAY'S MILK ROAD IS BROUGHT TO YOU BY CRYPTOTRADER.TAX +

+ When it comes to taxes, there are two types of people.  +

  1. Those who do taxes early (also known as psychopaths)
  2. The rest of us who wait until the last minute

+ With less than 2 weeks left to file taxes, we’ve partnered with Cryptotrader.tax (soon to be CoinLedger) to make sure you got everything you need. +

+ A common misconception: +

+ “I don’t get taxed until I cash out to fiat”.  +

+ We hate to be the bearer of bad news, but you do in fact realize taxable events and capital gains when trading crypto > crypto. +

+ It sucks, but that's the law for ya. +

+ FUNDING FRIDAY +

+ Over $700M was raised in Web3 companies and funds this week! Who got the bread this time? +

+ Dank Bank got $4.2M to let people buy & sell Memes as NFTs. Sounds silly, but…Meme’s are a part of culture. They are globally recognizable. Maybe memes will be collected as other works of culture & art?  +

+ Fractal got $35M to build a new NFT gaming marketplace. This is Justin Kan’s new startup (previously JustinTV, Twitch). Worth keeping an eye on +

+ Leap got $10.2M to build out a “super” wallet for the Terra ecosystem that can show off your NFTs, track the value of your wallet, and integrate with the most popular Terra protocols +

+ GOALS got $15M to build the “FIFA of Web3” utilizing NFTs. The cool thing here is you’ll own all the assets in the game - from the players to the cleats they wear - and it can all be sold or traded. They still need to build an awesome game though.. +

+ Hivemapper got $18M to build a decentralized Google Maps by giving drivers digital tokens to put dashcams in their cars. You know we love easy ways to make money. (They’re leading the wave of IRL → tokens that we talked about here). +

+ Dive into our full database of companies that have fundraised here! +

+ SIT BACK AND RELAX THIS WEEKEND WITH THE VITALIK SPECIAL +

+ Send a video of you drinking a Vitalik cocktail to @milkroaddaily on Twitter and we’ll retweet it! +

+ See ya Monday! +

+ Reviews from the Road: +

+ Warm... +

+ Getting warmer... +

+ Ahhh there it is... +

+ What'd you think of today's email? +

\ No newline at end of file diff --git a/packages/readabilityjs/test/test-pages/moxie.org/distiller.html b/packages/readabilityjs/test/test-pages/moxie.org/distiller.html new file mode 100644 index 000000000..822ba0d5b --- /dev/null +++ b/packages/readabilityjs/test/test-pages/moxie.org/distiller.html @@ -0,0 +1,14 @@ +

My first impressions of web3

Jan 07, 2022

Despite considering myself a cryptographer, I have not found myself particularly drawn to “crypto.” I don’t think I’ve ever actually said the words “get off my lawn,” but I’m much more likely to click on Pepperidge Farm Remembers flavored memes about how “crypto” used to mean “cryptography” than I am the latest NFT drop.

Also – cards on the table here – I don’t share the same generational excitement for moving all aspects of life into an instrumented economy.

Even strictly on the technological level, though, I haven’t yet managed to become a believer. So given all of the recent attention into what is now being called web3, I decided to explore some of what has been happening in that space more thoroughly to see what I may be missing.

How I think about 1 and 2

web3 is a somewhat ambiguous term, which makes it difficult to rigorously evaluate what the ambitions for web3 should be, but the general thesis seems to be that web1 was decentralized, web2 centralized everything into platforms, and that web3 will decentralize everything again. web3 should give us the richness of web2, but decentralized.

It’s probably good to have some clarity on why centralized platforms emerged to begin with, and in my mind the explanation is pretty simple:

  1. People don’t want to run their own servers, and never will. The premise for web1 was that everyone on the internet would be both a publisher and consumer of content as well as a publisher and consumer of infrastructure.

    We’d all have our own web server with our own web site, our own mail server for our own email, our own finger server for our own status messages, our own chargen server for our own character generation. However – and I don’t think this can be emphasized enough – that is not what people want. People do not want to run their own servers.

    Even nerds do not want to run their own servers at this point. Even organizations building software full time do not want to run their own servers at this point. If there’s one thing I hope we’ve learned about the world, it’s that people do not want to run their own servers. The companies that emerged offering to do that for you instead were successful, and the companies that iterated on new functionality based on what is possible with those networks were even more successful.
  2. A protocol moves much more slowly than a platform. After 30+ years, email is still unencrypted; meanwhile WhatsApp went from unencrypted to full e2ee in a year. People are still trying to standardize sharing a video reliably over IRC; meanwhile, Slack lets you create custom reaction emoji based on your face.

    This isn’t a funding issue. If something is truly decentralized, it becomes very difficult to change, and often remains stuck in time. That is a problem for technology, because the rest of the ecosystem is moving very quickly, and if you don’t keep up you will fail. There are entire parallel industries focused on defining and improving methodologies like Agile to try to figure out how to organize enormous groups of people so that they can move as quickly as possible because it is so critical.

    When the technology itself is more conducive to stasis than movement, that’s a problem. A sure recipe for success has been to take a 90’s protocol that was stuck in time, centralize it, and iterate quickly.

But web3 intends to be different, so let’s take a look. In order to get a quick feeling for the space and a better understanding for what the future may hold, I decided to build a couple of dApps and create an NFT.

Making some distributed apps

To get a feeling for the web3 world, I made a dApp called Autonomous Art that lets anyone mint a token for an NFT by making a visual contribution to it. The cost of making a visual contribution increases over time, and the funds a contributor pays to mint are distributed to all previous artists (visualizing this financial structure would resemble something similar to a pyramid shape). At the time of this writing, over $38k USD has gone into creating this collective art piece.

I also made a dApp called First Derivative that allows you to create, discover, and exchange NFT derivatives which track an underlying NFT, similar to financial derivatives which track an underlying asset 😉.

Both gave me a feeling for how the space works. To be clear, there is nothing particularly “distributed” about the apps themselves: they’re just normal react websites. The “distributedness” refers to where the state and the logic/permissions for updating the state lives: on the blockchain instead of in a “centralized” database.

One thing that has always felt strange to me about the cryptocurrency world is the lack of attention to the client/server interface. When people talk about blockchains, they talk about distributed trust, leaderless consensus, and all the mechanics of how that works, but often gloss over the reality that clients ultimately can’t participate in those mechanics. All the network diagrams are of servers, the trust model is between servers, everything is about servers. Blockchains are designed to be a network of peers, but not designed such that it’s really possible for your mobile device or your browser to be one of those peers.

With the shift to mobile, we now live firmly in a world of clients and servers – with the former completely unable to act as the latter – and those questions seem more important to me than ever. Meanwhile, ethereum actually refers to servers as “clients,” so there’s not even a word for an actual untrusted client/server interface that will have to exist somewhere, and no acknowledgement that if successful there will ultimately be billions (!) more clients than servers.

For example, whether it’s running on mobile or the web, a dApp like Autonomous Art or First Derivative needs to interact with the blockchain somehow – in order to modify or render state (the collectively produced work of art, the edit history for it, the NFT derivatives, etc). That’s not really possible to do from the client, though, since the blockchain can’t live on your mobile device (or in your desktop browser realistically). So the only alternative is to interact with the blockchain via a node that’s running remotely on a server somewhere.

A server! But, as we know, people don’t want to run their own servers. As it happens, companies have emerged that sell API access to an ethereum node they run as a service, along with providing analytics, enhanced APIs they’ve built on top of the default ethereum APIs, and access to historical transactions. Which sounds… familiar. At this point, there are basically two companies. Almost all dApps use either Infura or Alchemy in order to interact with the blockchain. In fact, even when you connect a wallet like MetaMask to a dApp, and the dApp interacts with the blockchain via your wallet, MetaMask is just making calls to Infura!

These client APIs are not using anything to verify blockchain state or the authenticity of responses. The results aren’t even signed. An app like Autonomous Art says “hey what’s the output of this view function on this smart contract,” Alchemy or Infura responds with a JSON blob that says “this is the output,” and the app renders it.

This was surprising to me. So much work, energy, and time has gone into creating a trustless distributed consensus mechanism, but virtually all clients that wish to access it do so by simply trusting the outputs from these two companies without any further verification. It also doesn’t seem like the best privacy situation. Imagine if every time you interacted with a website in Chrome, your request first went to Google before being routed to the destination and back. That’s the situation with ethereum today. All write traffic is obviously already public on the blockchain, but these companies also have visibility into almost all read requests from almost all users in almost all dApps.

Partisans of the blockchain might say that it’s okay if these types of centralized platforms emerge, because the state itself is available on the blockchain, so if these platforms misbehave clients can simply move elsewhere. However, I would suggest that this is a very simplistic view of the dynamics that make platforms what they are.

Let me give you an example.

Making an NFT

I also wanted to create a more traditional NFT. Most people think of images and digital art when they think of NFTs, but NFTs generally do not store that data on-chain. For most NFTs of most images, that would be much too expensive.

Instead of storing the data on-chain, NFTs instead contain a URL that points to the data. What surprised me about the standards was that there’s no hash commitment for the data located at the URL. Looking at many of the NFTs on popular marketplaces being sold for tens, hundreds, or millions of dollars, that URL often just points to some VPS running Apache somewhere. Anyone with access to that machine, anyone who buys that domain name in the future, or anyone who compromises that machine can change the image, title, description, etc for the NFT to whatever they’d like at any time (regardless of whether or not they “own” the token). There’s nothing in the NFT spec that tells you what the image “should” be, or even allows you to confirm whether something is the “correct” image.

So as an experiment, I made an NFT that changes based on who is looking at it, since the web server that serves the image can choose to serve different images based on the IP or User Agent of the requester. For example, it looked one way on OpenSea, another way on Rarible, but when you buy it and view it from your crypto wallet, it will always display as a large 💩 emoji. What you bid on isn’t what you get. There’s nothing unusual about this NFT, it’s how the NFT specifications are built. Many of the highest priced NFTs could turn into 💩 emoji at any time; I just made it explicit.

NFT on OpenSea
NFT on OpenSea
NFT on rarible
Same NFT on Rarible
NFT in wallet
Same NFT in a wallet

After a few days, without warning or explanation, the NFT I made was removed from OpenSea (an NFT marketplace):

The takedown suggests that I violated some Term Of Service, but after reading the terms, I don’t see any that prohibit an NFT which changes based on where it is being looked at from, and I was openly describing it that way.

What I found most interesting, though, is that after OpenSea removed my NFT, it also no longer appeared in any crypto wallet on my device. This is web3, though, how is that possible?

A crypto wallet like MetaMask, Rainbow, etc is “non-custodial” (the keys are kept client side), but it has the same problem as my dApps above: a wallet has to run on a mobile device or in your browser. Meanwhile, ethereum and other blockchains have been designed with the idea that it’s a network of peers, but not designed such that it’s really possible for your mobile device or your browser to be one of those peers.

A wallet like MetaMask needs to do basic things like display your balance, your recent transactions, and your NFTs, as well as more complex things like constructing transactions, interacting with smart contracts, etc. In short, MetaMask needs to interact with the blockchain, but the blockchain has been built such that clients like MetaMask can’t interact with it. So like my dApp, MetaMask accomplishes this by making API calls to three companies that have consolidated in this space.

For instance, MetaMask displays your recent transactions by making an API call to etherscan:

GET https://api.etherscan.io/api?module=account&address=0x0208376c899fdaEbA530570c008C4323803AA9E8&offset=40&order=desc&action=txlist&tag=latest&page=1 HTTP/2.0                                                          
+

…displays your account balance by making an API call to Infura:

POST https://mainnet.infura.io/v3/d039103314584a379e33c21fbe89b6cb HTTP/2.0
+
+{
+    "id": 2628746552039525,
+    "jsonrpc": "2.0",
+    "method": "eth_getBalance",
+    "params": [
+        "0x0208376c899fdaEbA530570c008C4323803AA9E8",
+        "latest"
+    ]
+}
+

…displays your NFTs by making an API call to OpenSea:

GET https://api.opensea.io/api/v1/assets?owner=0x0208376c899fdaEbA530570c008C4323803AA9E8&offset=0&limit=50 HTTP/2.0                                                                                               
+

Again, like with my dApp, these responses are not authenticated in some way. They’re not even signed so that you could later prove they were lying. It reuses the same connections, TLS session tickets, etc for all the accounts in your wallet, so if you’re managing multiple accounts in your wallet to maintain some identity separation, these companies know they’re linked.

MetaMask doesn’t actually do much, it’s just a view onto data provided by these centralized APIs. This isn’t a problem specific to MetaMask – what other option do they have? Rainbow, etc are set up in exactly the same way. (Interestingly, Rainbow has their own data for the social features they’re building into their wallet – social graph, showcases, etc – and have chosen to build all of that on top of Firebase instead of the blockchain.)

All this means that if your NFT is removed from OpenSea, it also disappears from your wallet. It doesn’t functionally matter that my NFT is indelibly on the blockchain somewhere, because the wallet (and increasingly everything else in the ecosystem) is just using the OpenSea API to display NFTs, which began returning 304 No Content for the query of NFTs owned by my address!

Recreating this world

Given the history of why web1 became web2, what seems strange to me about web3 is that technologies like ethereum have been built with many of the same implicit trappings as web1. To make these technologies usable, the space is consolidating around… platforms. Again. People who will run servers for you, and iterate on the new functionality that emerges. Infura, OpenSea, Coinbase, Etherscan.

Likewise, the web3 protocols are slow to evolve. When building First Derivative, it would have been great to price minting derivatives as a percentage of the underlying’s value. That data isn’t on chain, but it’s in an API that OpenSea will give you. People are excited about NFT royalties for the way that they can benefit creators, but royalties aren’t specified in ERC-721, and it’s too late to change it, so OpenSea has its own way of configuring royalties that exists in web2 space. Iterating quickly on centralized platforms is already outpacing the distributed protocols and consolidating control into platforms.

Given those dynamics, I don’t think it should be a surprise that we’re already at a place where your crypto wallet’s view of your NFTs is OpenSea’s view of your NFTs. I don’t think we should be surprised that OpenSea isn’t a pure “view” that can be replaced, since it has been busy iterating the platform beyond what is possible strictly with the impossible/difficult to change standards.

I think this is very similar to the situation with email. I can run my own mail server, but it doesn’t functionally matter for privacy, censorship resistance, or control – because GMail is going to be on the other end of every email that I send or receive anyway. Once a distributed ecosystem centralizes around a platform for convenience, it becomes the worst of both worlds: centralized control, but still distributed enough to become mired in time. I can build my own NFT marketplace, but it doesn’t offer any additional control if OpenSea mediates the view of all NFTs in the wallets people use (and every other app in the ecosystem).

This isn’t a complaint about OpenSea or an indictment of what they’ve built. Just the opposite, they’re trying to build something that works. I think we should expect this kind of platform consolidation to happen, and given the inevitability, design systems that give us what we want when that’s how things are organized. My sense and concern, though, is that the web3 community expects some other outcome than what we’re already seeing.

It’s early days

“It’s early days still” is the most common refrain I see from people in the web3 space when discussing matters like these. In some ways, cryptocurrency’s failure to scale beyond relatively nascent engineering is what makes it possible to consider the days “early,” since objectively it has already been a decade or more.

However, even if this is just the beginning (and it very well might be!), I’m not sure we should consider that any consolation. I think the opposite might be true; it seems like we should take notice that from the very beginning, these technologies immediately tended towards centralization through platforms in order for them to be realized, that this has ~zero negatively felt effect on the velocity of the ecosystem, and that most participants don’t even know or care it’s happening. This might suggest that decentralization itself is not actually of immediate practical or pressing importance to the majority of people downstream, that the only amount of decentralization people want is the minimum amount required for something to exist, and that if not very consciously accounted for, these forces will push us further from rather than closer to the ideal outcome as the days become less early.

But you can’t stop a gold rush

When you think about it, OpenSea would actually be much “better” in the immediate sense if all the web3 parts were gone. It would be faster, cheaper for everyone, and easier to use. For example, to accept a bid on my NFT, I would have had to pay over $80-$150+ just in ethereum transaction fees. That puts an artificial floor on all bids, since otherwise you’d lose money by accepting a bid for less than the gas fees. Payment fees by credit card, which typically feel extortionary, look cheap compared to that. OpenSea could even publish a simple transparency log if people wanted a public record of transactions, offers, bids, etc to verify their accounting.

However, if they had built a platform to buy and sell images that wasn’t nominally based on crypto, I don’t think it would have taken off. Not because it isn’t distributed, because as we’ve seen so much of what’s required to make it work is already not distributed. I don’t think it would have taken off because this is a gold rush. People have made money through cryptocurrency speculation, those people are interested in spending that cryptocurrency in ways that support their investment while offering additional returns, and so that defines the setting for the market of transfer of wealth.

The people at the end of the line who are flipping NFTs do not fundamentally care about distributed trust models or payment mechanics, but they care about where the money is. So the money draws people into OpenSea, they improve the experience by building a platform that iterates on the underlying web3 protocols in web2 space, they eventually offer the ability to “mint” NFTs through OpenSea itself instead of through your own smart contract, and eventually this all opens the door for Coinbase to offer access to the validated NFT market with their own platform via your debit card. That opens the door to Coinbase managing the tokens themselves through dark pools that Coinbase holds, which helpfully eliminates the transaction fees and makes it possible to avoid having to interact with smart contracts at all. Eventually, all the web3 parts are gone, and you have a website for buying and selling JPEGS with your debit card. The project can’t start as a web2 platform because of the market dynamics, but the same market dynamics and the fundamental forces of centralization will likely drive it to end up there.

At the end of the stack, NFT artists are excited about this kind of progression because it means more speculation/investment in their art, but it also seems like if the point of web3 is to avoid the trappings of web2, we should be concerned that this is already the natural tendency for these new protocols that are supposed to offer a different future.

I think these market forces will likely continue, and in my mind the question of how long it continues is a question of whether the vast amounts of accumulated cryptocurrency are ultimately inside an engine or a leaky bucket. If the money flowing through NFTs ends up channeled back into crypto space, it could continue to accelerate forever (regardless of whether or not it’s just web2x2). If it churns out, then this will be a blip. Personally, I think enough money has been made at this point that there are enough faucets to keep it going, and this won’t just be a blip. If that’s the case, it seems worth thinking about how to avoid web3 being web2x2 (web2 but with even less privacy) with some urgency.

Creativity might not be enough

I have only dipped my toe in the waters of web3. Looking at it through the lens of these small projects, though, I can easily see why so many people find the web3 ecosystem so neat. I don’t think it’s on a trajectory to deliver us from centralized platforms, I don’t think it will fundamentally change our relationship to technology, and I think the privacy story is already below par for the internet (which is a pretty low bar!), but I also understand why nerds like me are excited to build for it. It is, at the very least, something new on the nerd level – and that creates a space for creativity/exploration that is somewhat reminiscent of early internet days. Ironically, part of that creativity probably springs from the constraints that make web3 so clunky. I’m hopeful that the creativity and exploration we’re seeing will have positive outcomes, but I’m not sure if it’s enough to prevent all the same dynamics of the internet from unfolding again.

If we do want to change our relationship to technology, I think we’d have to do it intentionally. My basic thoughts are roughly:

  1. We should accept the premise that people will not run their own servers by designing systems that can distribute trust without having to distribute infrastructure. This means architecture that anticipates and accepts the inevitable outcome of relatively centralized client/server relationships, but uses cryptography (rather than infrastructure) to distribute trust. One of the surprising things to me about web3, despite being built on “crypto,” is how little cryptography seems to be involved!
  2. We should try to reduce the burden of building software. At this point, software projects require an enormous amount of human effort. Even relatively simple apps require a group of people to sit in front of a computer for eight hours a day, every day, forever. This wasn’t always the case, and there was a time when 50 people working on a software project wasn’t considered a “small team.” As long as software requires such concerted energy and so much highly specialized human focus, I think it will have the tendency to serve the interests of the people sitting in that room every day rather than what we may consider our broader goals. I think changing our relationship to technology will probably require making software easier to create, but in my lifetime I’ve seen the opposite come to pass. Unfortunately, I think distributed systems have a tendency to exacerbate this trend by making things more complicated and more difficult, not less complicated and less difficult.

gm!

  • Stay in touch,
\ No newline at end of file diff --git a/packages/readabilityjs/test/test-pages/news.utexas/distiller.html b/packages/readabilityjs/test/test-pages/news.utexas/distiller.html new file mode 100644 index 000000000..087c85599 --- /dev/null +++ b/packages/readabilityjs/test/test-pages/news.utexas/distiller.html @@ -0,0 +1,21 @@ +
Two color orange horizontal divider
Student50733184336_a04a010842_c copy
Photo courtesy of Tyler Henderson/Penn State.

+ AUSTIN, Texas — Producing clean water at a lower cost could be on the horizon after researchers from The University of Texas at Austin and Penn State solved a complex problem that had baffled scientists for decades, until now. +

+ Desalination membranes remove salt and other chemicals from water, a process critical to the health of society, cleaning billions of gallons of water for agriculture, energy production and drinking. The idea seems simple — push salty water through and clean water comes out the other side — but it contains complex intricacies that scientists are still trying to understand. +

Click here to watch a video explaining this research. Experiment photos and video footage courtesy Tyler Henderson/Penn State.

+ The research team, in partnership with DuPont Water Solutions, solved an important aspect of this mystery, opening the door to reduce costs of clean water production. The researchers determined desalination membranes are inconsistent in density and mass distribution, which can hold back their performance. Uniform density at the nanoscale is the key to increasing how much clean water these membranes can create. +

+ “Reverse osmosis membranes are widely used for cleaning water, but there’s still a lot we don’t know about them,” said Manish Kumar, an associate professor in the Department of Civil, Architectural and Environmental Engineering at UT Austin, who co-led the research. “We couldn’t really say how water moves through them, so all the improvements over the past 40 years have essentially been done in the dark.” +

+ The findings were published today in Science. +

+ The paper documents an increase in efficiency in the membranes tested by 30%-40%, meaning they can clean more water while using significantly less energy. That could lead to increased access to clean water and lower water bills for individual homes and large users alike. +

+ Reverse osmosis membranes work by applying pressure to the salty feed solution on one side. The minerals stay there while the water passes through. Although more efficient than non-membrane desalination processes, it still takes a large amount of energy, the researchers said, and improving the efficiency of the membranes could reduce that burden. +

+ “Fresh water management is becoming a crucial challenge throughout the world,” said Enrique Gomez, a professor of chemical engineering at Penn State who co-led the research. “Shortages, droughts — with increasing severe weather patterns, it is expected this problem will become even more significant. It’s critically important to have clean water availability, especially in low-resource areas.” +

The density of filtration membranes, even at the atomic scale, can greatly affect how much clean water can be produced. (Enrique Gomez/Penn State)

+ The National Science Foundation and DuPont, which makes numerous desalination products, funded the research. The seeds were planted when DuPont researchers found that thicker membranes were actually proving to be more permeable. This came as a surprise because the conventional knowledge was that thickness reduces how much water could flow through the membranes. +

+ The team connected with Dow Water Solutions, which is now a part of DuPont, in 2015 at a “water summit” Kumar organized, and they were eager to solve this mystery. The research team, which also includes researchers from Iowa State University, developed 3D reconstructions of the nanoscale membrane structure using state-of-the-art electron microscopes at the Materials Characterization Lab of Penn State. They modeled the path water takes through these membranes to predict how efficiently water could be cleaned based on structure. Greg Foss of the Texas Advanced Computing Center helped visualize these simulations, and most of the calculations were performed on Stampede2, TACC’s supercomputer. +

\ No newline at end of file diff --git a/packages/readabilityjs/test/test-pages/nymag/distiller.html b/packages/readabilityjs/test/test-pages/nymag/distiller.html new file mode 100644 index 000000000..dd817564c --- /dev/null +++ b/packages/readabilityjs/test/test-pages/nymag/distiller.html @@ -0,0 +1,163 @@ +
+ the swamp +

+ The Full(est Possible) Story of the Four Seasons Total Landscaping Press Conference +

+ The circus is gone. The presidency is ending. The mystery endures. Photo: Olivia Nuzzi +

+ On the afternoon of November 13, Mike Siravo was standing outside his family’s landscaping business in Northeast Philadelphia, dressed in khakis and a company polo shirt, watching as strangers pulled up in nice cars, parked without care on the busy street, and approached the barbed-wire-topped fence with iPhones gripped in outstretched hands. They all came for the same reason: to see for themselves the words FOUR SEASONS TOTAL LANDSCAPING. SINCE 1992. PARKING ONLY. ALL OTHERS WILL BE TOWED AT OWNERS’ EXPENSE. In packs, they laughed openly. Alone, they wore bemused expressions, eyes focused on their screens. All of them spent a few minutes taking in the sight and, more importantly, documenting their visit with selfies. +

+ Workers walked in and out of the parking lot, sometimes shaking their heads but mostly keeping them down and not saying much to any of the outsiders for whom the landscaping company was now an unusual monument to the end of America, or the end of the thing that had symbolized the end of America, or something. It was three o’clock on a workday. “I’m just an employee,” one of them said. “I don’t know anything.” +

+ A man on a bicycle paused near the front office to stare at the building. On the other side of the blinds, there were desks and filing cabinets illuminated by fluorescent light and people going about their day, which would have been a normal one were it not for the 20,000 T-shirt orders to process and the intrusion of tourists who saw the place as some kind of zoo exhibition. There was an awkward silence, but then Siravo smiled and shrugged in the direction of the sidewalk, asking the curious bicyclist the obvious question: Was he looking for a photo? He leapt down the steps to take the man’s phone and, with the enthusiasm of a mall photographer, instructed him how to pose. Siravo leaned back into the street, making sure the angle captured the green-and-white awning with the company name. +

+ A couple of feet away, a family of four was staging a holiday card. Lois Neuberger and Matthew Gold said they were in town from California, visiting their daughter at school, when it occurred to them they were just a short distance from the festive greeting–slash–political meme of a lifetime. “It’s become such a thing that we decided it would be a fun idea,” Gold said with a laugh. “Everyone will get it.” (Including me, in the literal sense, since the Neuberger-Golds kindly added my address to their mailing list.) +

+ By then, there were all sorts of rumors on State Road about the Siravo family’s connections with the Trump campaign and the Philadelphia Republican Party. But it had been nearly a week since Rudy Giuliani’s press conference in the parking lot out back, and the only evidence anyone could turn up to support the theory that what had occurred here wasn’t just a freak public-relations accident or hilarious fuckup were a few pro-Trump Facebook posts from Mike’s mother, Marie Siravo, who owns the business. She had been shrewd enough to release a statement amid the frenzy that said the landscapers were not partisans and then to mostly avoid speaking to the media as she rushed out the door, clutching a Louis Vuitton bag, to a white Jeep with a FOUR SEASONS license plate on the front bumper. The Siravos were nothing if not good marketers, and by December, they’d sold more than $1 million of the merchandise they’d drawn up to capitalize on all the attention, like stickers that read “Make America Rake Again!” and “Lawn and Order!” +

+ “We don’t really know how it happened. We heard it might’ve been a mistake or something,” Mike Siravo said. “We just kinda picked up the phone and said yes and cleared some stuff out and managed to make it happen.” +

+ A man locks the gate at Four Seasons Total Landscaping. The sign has become a prime backdrop for selfie-seekers. Photo: Olivia Nuzzi +

+ If that was true, it didn’t explain how it came to be that the phone rang at Four Seasons Total Landscaping in the first place. Siravo wouldn’t say who had called, or if he knew how Donald Trump’s campaign had even heard of the small landscaping business, or anything else, really, that might tell how this stretch of asphalt became the official site of the end of the presidency and the beginning of the ass-backward pseudo-legal effort to reverse the results of the election. According to the New York Times, there had been a miscommunication between Trump and the event planners. According to the Philadelphia Inquirer, the campaign made the staging choice that morning, after calling one of the Siravo’s employees. My search for answers involved — I swear to God — more than 37 sources spread throughout the White House, the Trump campaign, the president’s network of advisers both formal and informal, the Republican Party, and the Commonwealth of Pennsylvania. +

+ This is not counting Siravo, who said he was sorry, but his family had decided not to talk — except, he added, about golf. They’d just done an interview limited to the subject of Four Seasons Total Landscaping’s approach to manicuring courses. +

+ The sound of Donald Trump’s voice was unmistakable. It boomed through Rudy Giuliani’s phone and traveled across the hall where anyone could hear that the president was on the line and — this was unmistakable, too — he was unhappy. +

+ “I’m like, ‘Oh damn, it’s the president on the phone!’” Daryl Brooks said. “I know what his voice sounds like — and he sounded pissed.” +

+ It was November 7, and the president — on the verge of Electoral College defeat, with the whole country glued to cable news, where manic nerds gestured frantically at maps and related esoteric trivia about swing-state counties that few non-locals had heard of — had been pissed for four consecutive days. +

+ People close to Trump often issue reports about his mood, and his mood is often sour, but multiple White House officials told me things were so volatile after Election Day that they were outright avoiding the president out of concern he might end up using any nearby staffer as a human stress ball. (The personal benefits of proximity, which would ordinarily outweigh the drawbacks of verbal abuse, waned alongside Trump’s election chances.) “It’s been so bad,” a senior White House official said. On Election Night, this person said, they walked into the residence, saw the president screaming, then turned around and walked right back out. +

+ But there is a thin line between bitterness and optimism, and as he prepared to depart the White House for his golf course in Sterling, Virginia, for the 299th time during his presidency, Trump fired off a few tweets to allege, without evidence, that the election was being stolen. What came next barely registered — at first. Later that morning, he tweeted, his lawyers would be holding a press conference at “Four Seasons, Philadelphia.” +

+ As he waited for the event to start on that Saturday morning, Daryl Brooks, a Republican poll watcher and frequent third-party candidate in New Jersey, was so excited to hear the president’s familiar rasp and glimpse his famous attorney that he didn’t really pause to consider what the source of the conflict might be. And he didn’t think it was strange that he’d been asked to take part in a press conference on so little notice, or that he wasn’t even sure where, exactly, he was. Brooks said he received a call that morning from James Baehr, a lawyer who serves in the White House as the special assistant to the president for domestic affairs (where he earns a government salary of $135,000, according to public records). Baehr told Brooks to meet him at the Hampton Inn near the Convention Center, and from there, he was driven to the location, where he entered through the side without catching sight of any identifying markers. “It looked like a construction or landscaping company or something,” Brooks said. (Asked for comment, a White House spokesperson said, “After consultation with White House counsel and his supervisor, James took personal leave for a few days after the election to assist in Philadelphia.”) +

+ Several campaign officials said the president’s tweet was the first they had heard about any event planned for the day but that, given the president’s nature, nothing about this seemed out of the ordinary. Scheduling surprises had been Trump’s style from the earliest days of his first campaign, after all, and no half-conscious person who worked for him for longer than a day could maintain the belief that the operation would ever achieve or even aspire to orderliness. (“Formal isn’t a word I would use to describe anything we do,” as one diplomatic staffer put it.) The advertised location did not stick out as unusual, either. Hotels are often booked to hold political and media conferences of all kinds, equipped as they are with vast ballrooms and efficient hospitality workers on hand to unstack rows of cushioned dining chairs on fields of carpet between the American flags and the swarms of reporters. +

+ And since there’s no Trump International Hotel in Philly yet, a Four Seasons would certainly do. With a name that suggests marble floors and gilded lighting fixtures and other trappings of luxury to which his own brand yearns, it would surely please the president more than a Marriott or a Holiday Inn. Plus, more practically, a hive of post-election activity was a few blocks away at the Convention Center, where absentee ballots were being counted and where campaign officials had staged an impromptu press event two days before. (That’s where Brooks had first met Pam Bondi and Corey Lewandowski, which was exciting, too, even if he didn’t really seem to know who they were. On Facebook, he posted selfies with them, identifying Bondi as “from Fox News” and calling Lewandowski “Cory Lewendowske” repeatedly.) “I thought, Okay, nice hotel,” a senior campaign official said. +

+ But the campaign had made no attempt to book the nice hotel for the press conference. Instead, according to a second official, the event was supposed to take place at the Union League of Philadelphia, a historic club founded in 1862 in support of Abraham Lincoln. The president claims to admire Lincoln — though mostly he seems to enjoy talking about how Lincoln was also a Republican and, despite a rather famous cranial encounter with a bullet, how people were more fair to Lincoln than they are to him — so this made a certain amount of thematic sense. +

+ The official thought the venue had canceled on the campaign owing to the health department’s COVID restrictions, which prevent gatherings of more than 25 people, but attempts to confirm this with the Union League were unsuccessful, since the only person I could get to talk over there was a nice man in the security office who had no information to share other than the fact that, to his knowledge, no events of any kind were happening during the pandemic. +

+ A third official said that the press conference was first supposed to take place at the United Republican Club, but Frank Cristinzio, its treasurer, said that they’ve been closed for months and had not discussed hosting an event with the campaign or any associated attorneys. “Maybe they thought about it,” he said, “but I didn’t hear about it.” In fact, Cristinzio said, things at the club were so dead nowadays that the only reason he was even there to answer the phone when I happened to call was because he had to stop by every now and then to check the mail. +

+ Trump followed up his announcement with a second announcement. Rather than the Four Seasons hotel, the press conference would be held at Four Seasons Total Landscaping — on State Road, in an industrial patch of Northeast Philadelphia, near an interstate and a few doors down from the Fantasy Island Adult Book Store and across the street from a crematorium. If you hit the Pentecostal church, you’ve gone too far. +

+ “Four Season’s Landscaping!” the president said (apostrophe his). “Big press conference today in Philadelphia at Four Seasons Total Landscaping — 11:30am!” A few minutes later, he left the White House in “shoes that look[ed] appropriate for golfing” and a white “Make America Great Again” hat, according to the photographer on the scene, and boarded his motorcade. “Lawyer’s Press Conference at Four Season’s Landscaping, Philadelphia,” he said in another tweet. “Enjoy!” +

+ It’s hard to know what counts as a fuckup when you work for Donald Trump. Looked at by the standards of a traditional campaign with a traditional candidate who possesses a minor-to-moderate capacity for traditional human feelings, like shame, what happened at Four Seasons Total Landscaping, or the fact that it happened at all, was a disaster. A press conference is supposed to convince the public of whatever case you’re making through the act of being — or more likely pretending to be —transparent. That is, obviously, not what happened on the blacktop. But looked at in terms of attention generated and relevance sustained, two other goals of a press conference, the thing was a clear success. Plus it was the rare political joke that appealed to everyone. +

+ Behold. Photo: Chris McGrath/Getty Images +

+ “Out of all the things that have happened, this was the funniest thing, and it came at the last minute,” Tim Heidecker said. A comedian and songwriter who used to live in Philadelphia, Heidecker said that if he had tried to write the scene as fiction, he was sure the reaction would be, “‘Naawww, that’s too hokey!’” As a comedy event, Four Seasons Total Landscaping was “not as sophisticated as I would usually like,” Heidecker said, but it worked perfectly because it was “simple and clean,” just “a basic comedy concept,” but organically staged in the wild. He was inspired by what he saw on the news that morning, and he felt called to put it to music. “I always start with the first line, and it seemed perfect that Rudy Giuliani was standing very literally surrounded by manure,” he said. The song is called “Rudy at the 4 Seasons,” and it starts there, rhyming manure with “spitting out lies that belong in the sewer.” +

+ In setting and content, the event served for some campaign officials and presidential advisers as a representation of the brokenness of Trumpworld. If it was an advance failure, well, people familiar with the inner workings of the campaign said they could’ve seen that coming. “When you hear ‘a friendly local business that’s supportive,’ that sounds great,” a senior official said. “But when you dig into the details on the ground, things are a little different.” A proper advance team, for instance, would have gone to scope things out before securing the location, taking note of nearby landmarks like the porn shop and the crematorium. “The tight shot was good, the message was delivered, but unfortunately —” the official said, trailing off. +

+ It’s not that anyone would venture to argue that things had ever run smoothly on the Trump campaign. But more than most principles, the value of good stagecraft was grasped by the candidate, and he cared enough to know who was behind the mechanics of his public-facing events. His first advance man, George Gigicos, became famous-for-politics, which doesn’t happen much with functionaries unless they start going on TV, which Gigicos never did, or they get indicted, which he managed not to do. Gigicos, who no longer works for Trump, started laughing hysterically when I told him why I was calling. The words four seasons or landscaping now elicit such a reaction from people across the ideological spectrum. “Honest to God, I have no idea how that came about,” he said. “From what I understand, I don’t think the campaign or the White House planned that thing. I think it was the campaign on the ground in Pennsylvania.” +

+ For some officials, there was irony in Brad Parscale’s being fired as the campaign manager following an optics (and public-health) crisis at a MAGA rally. Of all the things Parscale could reasonably be accused of, ignoring the advance team or dismissing the idea that public events were important was not one of them, even if knowing, intellectually, their value did not necessarily translate to knowing how to run a professional operation. “Brad understood. He saw through the president’s eyes what we did and why it mattered,” the senior official said. +

+ Parscale’s replacement, Bill Stepien, had gotten his start doing the thankless logistical chores that make a campaign run, even once working as a driver for his candidate. But by the time he was put in charge of the Trump campaign, time and money were running out, and he was unlikely to engineer an epic reversal of luck by focusing on stunts or by trying to fix the few things, like the president’s airport-hangar-rally programming, that were semi-functioning. “It was different under Stepien,” the official said. “The value of an events schedule, of an advance team, was not at the forefront of his mind, especially during COVID.” Why would it be? +

+ Two senior campaign officials said they assumed the president would want to hold events right after Election Day, and they tried to formalize a process to carry those events out quickly and professionally “on a moment’s notice.” Advance was always being called “at the 11th hour,” even when its members weren’t being outright “ignored,” as they felt was the case under Stepien. So the idea was to prepare with spontaneity in mind, to keep state advance teams on the payroll, ready to go on the ground in the places most likely to be the scenes of post-election legal battles, like Michigan, Georgia, Arizona, Ohio, and Pennsylvania. “The response we were getting back was the same everybody was getting back: ‘We’re not sure what’s happening, and we’re taking it as it comes,’” the official said. The second official put it this way: “They didn’t think that was necessary,” the official said, “and they kinda just scoffed at me.” When the morning of the Four Seasons Total Landscaping event came, several advance staffers had already left the campaign. +

+ But for many people on the campaign and in the White House, there was a more obvious issue threatening the president’s play for an electoral miracle. In the weeks leading up to that Saturday morning, advisers to Trump believed Rudy Giuliani was becoming more and more of a problem. He’d always been a problem, it was true, prone to butt-dialing reporters, or showing up on TV all out of sorts to divulge something at odds with what the president had already claimed to be true, or otherwise catching the attention of hostile foreign governments or domestic investigators or celebrity pranksters. +

+ But in the final sprint to Election Day, managing the known unknowns of Giuliani’s endless capacity to fuck up so much that he was often at the center of several personal and international legal dramas at once was consuming time and manpower at campaign headquarters when there was little left to spare. “We were spending hours each day trying to prevent Rudy from creating a disaster,” a senior campaign official said. “Hours and hours.” +

+ People who were supposed to be focused on winning a presidential race were instead focused on devising ways to distract the once celebrated mayor of New York City. In the office he was a nuisance, but out of sight he was a terror. To keep him busy but accounted for, he was sent on the road as a surrogate, including for a last-minute event in Philadelphia on Columbus Day to launch an otherwise basically fake initiative called Italian Americans for Trump. It was, a second official said, “nightmarish.” +

+ Rudy holds court. Photo: Bryan R. Smith/AFP via Getty Images +

+ Yet, as loserdom neared, what looked to everyone else like the Rudy Problem looked to Trump like a potential solution. “His style is to keep pounding his head on the door, ultimately believing it will open,” one of his legal advisers told me. “And unfortunately, it has worked several times for him, so you can’t disabuse him of the notion of his persistence.” +

+ “The president’s narcissism cripples him in these moments,” the adviser added, “because as long as people are telling him what they think he wants to hear, it’s a struggle for him to abandon hope. He’s just such a curiously wounded narcissist. If Rudy tells him, ‘We’re gonna destroy all the norms and burn it down and make sure you get reinstated, the president goes, ‘Great!’” The truth, the adviser said, took longer for him to process, and it required whoever uttered it to approach Trump as if he were a wild animal. “When people would bring him bad news, he would blow up, and they would sort of back out of the room.” The trick, the adviser said, is “don’t hit him immediately with something he can react emotionally to” and “don’t appear intimidated.” +

+ A second person familiar with the legal team said Giuliani was put in charge because “the president wanted a peer and a fighter. He wanted somebody that he can relate to.” This person described competing power centers, with the litigators and other serious people on the one side, who realized almost immediately that the president had no legitimate pathway to change the election results, and the conspiracy theorists and crazy people, led by Giuliani, on the other side. The second group won, even after multiple interventions staged by lawyers and family members and other advisers. As usual, Trump was unwilling to let go of the people he perceived to be fighting the hardest for him in public. Which wasn’t a surprise, of course, though it still managed to disappoint the optimists (or idiots, depending on your view) still working for the president with hopes that, after all this time, he might change well-established aspects of his personality. +

+ The person familiar with the legal team never bought the idea, for instance, that Sidney Powell had really been removed as one of the president’s representatives, even though the campaign had put out a statement to that effect. What came out of the president’s mouth, and through his Twitter feed, seemed a reflection of what went into his head via people like Powell, Giuliani, and fellow legal-team member Jenna Ellis. A senior White House official told me that, in the vacuum created by the absence of officials who might try to reason with the president, Trump spent even more time on the phone, dialing up whomever he saw defending him most rabidly on TV. Sometimes, this official said, the White House switchboard operator wouldn’t even know how to contact the person the president wanted to speak to, and this would result in members of the staff being roped in to locate a number for some random pundit from Fox or, increasingly, Newsmax or OANN. As if to prove the leakers’ point, Giuliani, recently recovered from his bout with the coronavirus, visited the White House last week. And according to the New York Times, Powell visited on Friday with the president, who discussed naming her the special counsel on voter fraud. +

+ “There were serious legal challenges that could’ve been mounted, but the show took over,” the person familiar with the legal team said, though if the show took over anything, it was just a different show. “It’s embarrassing for the country, and it’s embarrassing for the president.” +

+ If you’ve ever been involved in politics at any level, you’ve met someone like Daryl Brooks. In New Jersey, where he’s from, he often surfaced around causes and campaigns, earning a reputation as “a wannabe hanger-on,” in the words of a Democrat, and “a self-promoter with no allies,” according to a Republican. +

+ Brooks describes himself as an activist, but it’s more true to say that he’s a man in search of a movement. He first ran for office in 2004 as a member of the Green Party, without success, and then again in 2006 as an independent on a civil-rights and gun-control platform. That was unsuccessful, too. By the Obama era, he was a tea-party libertarian, and by the time Trump was in power, he was a far-right radio host in a tricorn hat, a prop gun in each hand, talking about the “Clinton body count” and Jeffrey Epstein and the “rise in pedophilia.” +

+ A few days before the election, Brooks, who is Black, posted on social media, “Black Americans don’t let Antifa a domestic terrorist group destroy the inner cities Nov 3rd wake up!!!” That day, he said, he’d been paid $100 to work as a poll watcher at the Philadelphia Convention Center. In this sense, Brooks was like many people around the president: He’d spent decades involved in politics in one way or another, but he’d never managed to make his way to the center of anything before he committed to MAGA. “He was basically a kook and a political menace,” Reed Gusciora, the mayor of Trenton, said. “So he fit in really well with Giuliani.” +

+ And then, suddenly, there he was: the first witness called by Giuliani to address the cameras at Four Seasons Total Landscaping. “So, let’s start off first,” Giuliani said, turning stage left and gesturing at Brooks as he walked across the backdrop, a garage door decorated with a checkerboard of blue and red Trump-Pence lawn signs, and met Giuliani with a handshake. He did not appear to know his name. “My friend!” Giuliani said. “Please tell briefly what happened to you, who you are, and what happened to you.” +

+ Brooks complied, describing how poll watchers were kept at a distance from voting booths and prevented from taking photos, which is standard practice at most polling locations on Election Day but which Brooks suggested was fodder for Giuliani’s fraud conspiracy. Giuliani’s grand theory, by the way, was that the election was being stolen through, among other things, ballots cast fraudulently on behalf of dead people — a claim for which there was and is no evidence. As Giuliani spoke that morning, with the president firmly on the golf course, the cable news networks confirmed the mathematically obvious: Biden had won the election. +

+ Poll worker Daryl Brooks speaking at the Trump-campaign legal team’s press conference on November 7 outside Four Seasons Total Landscaping in Philadelphia. Photo: Screencap/OAN +

+ Brooks told me that he didn’t intend to claim the election was stolen, but if anything, the reaction to his decision to stand alongside Giuliani had only radicalized him more. After the press conference, Politico reported that he had served time more than two decades ago after being convicted of exposing himself to a minor, a crime Brooks still denies committing. He wasn’t sure whether there was fraud at the convention center, he said. And he didn’t know that saying yes to a press conference that morning would mean he would appear aligned with everything that happened there or that he would become a story himself, one that his kids might read. He said that, in his view, what happened in his past did not relate to the current state of the president’s effort to remain in office but that the interest from the press revealed racist motivations. “Nobody’s talking about that Jewish lady there or that Italian lawyer,” he said. (Brooks has already released a book about his experience, titled, 37 Days: The Disenfranchisement of a Philadelphia Poll Worker.) +

+ He wasn’t the only person who attended the press conference and left angry at the media. +

+ “How did I get there?!” Bernie Kerik said with a huff. “How do you think I got there?! I drove!” Kerik, the former New York City police commissioner recently pardoned by President Trump, was not interested in answering questions about his attendance, where he stood behind his friend Rudy, a smirk on his face. “This story is some bullshit story!” he said. Click. I texted Giuliani. “Are you kidding?” he asked me. “My Mom always said anyone can make a mistake. Only an idiot repeats it.” +

+ But even people in Trumpworld had questions. +

+ “Do we know what happened yet?” a senior White House official asked. +

+ “How it happened is still kind of a mystery to me,” a senior campaign official said. +

+ “This is all very us,” another campaign official said. “I’m not shocked or surprised at all.” If luck is what happens when preparation meets opportunity, this was the opposite. “That’s how you get the Four Seasons fucking Total Landscaping bullshit.” +

+ Whether it’s war and peace or public relations and gardening, sorting out the truth is a complicated endeavor when it relates to Donald Trump. Everyone involved in anything, no matter the size, no matter how stupid, seems to lie as a first resort, or to know very little, or to lie about knowing very little, or to know just enough to send blame in another direction, and the person in that direction seems to lie also, or to know very little, or to lie about knowing very little, but perhaps they have a theory that sends blame someplace else, and over there, too, you will find more liars, more know-nothings, and before long, a whole month will have passed, and you still haven’t filed your story about how the president’s attorney wound up undermining democracy in a parking lot off I-95 on a strip of cracked pavement in a run-down part of a city that ordinarily would command no consideration from the national political class or the very online public or the equally online mainstream media, which, when forced to look, found lots of reason to laugh. +

+ Bernie D’Angelo didn’t blame them. An electrician by trade and a Rolling Stones fanatic, D’Angelo has owned all kinds of businesses over the years, including a health-food store and a pizza parlor, but Fantasy Island, the adult book store he inherited from his parents, is by far his favorite. He appreciates how it’s “a lot more fun” than the others, generally, and specifically, he appreciates how the adult business strips the airs from anyone who ascends the steps under the bright-yellow sign outside to cross his carpeted threshold. “This is reality,” he said. “When they come in, they check their egos at the door, because look: It is what it is. There’s no sugarcoating it.” He gestured to the wall of dildos on his left. +

+ Fantasy Island owner Bernie D’Angelo, who voted for Joe Biden, said he wants America to “chill out a little.” Photo: Olivia Nuzzi +

+ That Saturday morning, D’Angelo said, he was keeping to himself when a customer ran in to report “a problem going on” outside, where police had suddenly appeared to block off the road. “So we looked out and found it was Giuliani who made a big mistake.” He laughed hard. “He was at the wrong Four Seasons hotel, the wrong one!” He paused to laugh with every few words. “‘Cause that’s a … landscaping! … And not a hotel! … A five-star hotel! … And that’s … one-star … landscaping!” He could barely breathe. “So you’ve got dead people, landscaping, and pornography!” +

+ When I first visited, right after the press conference, the joke was still alive. A local newsman and his camera guy had been set up for a live shot on the sidewalk all afternoon. But soon they were gone. Why wasn’t I? I came back even after I got into a car accident between the landscapers and the Pentecostals (little was damaged beyond my already poor reputation with Avis). After a few days, I wasn’t sure if I truly believed that history had been made in that patch of tar behind Four Seasons Total Landscaping. It was true that the presidency had officially ended there, but it was also true that the site itself felt like someone had erected a somber memorial at the scene of one of the lesbian pillow-fight pornos for sale at Fantasy Island (not that I looked). +

+ Rudy Giuliani did not visit Fantasy Island, according to its owner, who said Giuliani’s posse was “eyeing” his porn shop that Saturday but did not submit to the temptation. Business has since taken off. Photo: Olivia Nuzzi +

+ About those dead people: Regulations keep crematories out of the way of most businesses, but Delaware Cremation Center would blend in fine in a more developed area, even if the pandemic has made it an unusually busy place. On State Road, it sticks out as weirdly nice-looking. The sign over the doorway is new, the brick façade unweathered. The black shutters around the window complement an iron bench and gate. If a place where bodies are turned to ash can be welcoming, you could call it that. There’s a space inside for mourners to gather to drink and eat, and there are pews in which to pray. You can see, in these and other small details, how the business of caring for the dead is often about caring for the living. Viewed from here, the Four Seasons Total Landscaping circus looked as much like an indictment of a certain kind of liberalism as an illustration of Trumpian incompetence. But picking at the bones of any joke will make it unfunny after a while, and by the time I was looking at the drawers where they push the bodies in, I’d been trying to make sense of what happened there for too long. +

+ As one Philadelphia Republican official told me: “Duuuuuude! It’s sooooo embarrassing! Oh my God! It’s the height of idiocy!” +

+ It was probably always that simple. +

+ +

+ Now that McConnell has caved on the filibuster, we should see a Senate power-sharing agreement later this week +

+ Senator Mitch McConnell on Monday dropped his demand that the new Democratic Senate majority promise to preserve the filibuster — which Republicans could use to obstruct President Biden’s agenda — ending an impasse that had prevented Democrats from assuming full power even after their election wins. +

+ In his negotiations with Senator Chuck Schumer of New York, the new majority leader, Mr. McConnell, Republican of Kentucky, had refused to agree to a plan for organizing the chamber without a pledge from Democrats to protect the filibuster, a condition that Mr. Schumer had rejected. +

+ But late Monday, as the stalemate persisted, Mr. McConnell found a way out by pointing to statements by two centrist Democrats, Senators Joe Manchin III of West Virginia and Kyrsten Sinema of Arizona, that said they opposed getting rid of the procedural tool — a position they had held for months — as enough of a guarantee to move forward without a formal promise from Mr. Schumer. +

+ “With these assurances, I look forward to moving ahead with a power-sharing agreement modeled on that precedent,” Mr. McConnell said in a statement. +

+ Democrats had been anticipating a capitulation by Mr. McConnell and said they believed he had overreached in the negotiation. +

+ +

+ Merck is shutting down its COVID vaccine program after lackluster results +

+ Merck said Monday that it is halting development of its two experimental Covid-19 vaccines, after early clinical-trial data showed the shots generated disappointing immune responses against the virus. +

+ Early-stage studies of the vaccines indicated they produced inferior immune responses in subjects given the shots compared with people who survived Covid-19 or took authorized Covid-19 vaccines, the company said. +

+ The exit removes a major vaccine player from the constellation of firms seeking shots to curb the spread of the virus, as the few companies that have been able to bring shots to market struggle to meet heavy demand. +

+ +

+ It turns out spreading wild conspiracy theories about election fraud has consequences +

+ Dominion Voting Systems filed a defamation lawsuit on Monday against Rudolph W. Giuliani, the lawyer for Donald J. Trump and former mayor of New York City who played a key role in the former president’s monthslong effort to subvert the 2020 election. +

+ The 107-page lawsuit, filed in the Federal District Court in Washington, accuses Mr. Giuliani of carrying out “a viral disinformation campaign about Dominion” made up of “demonstrably false” allegations, in part to enrich himself through legal fees and his podcast. +

+ The suit seeks damages of more than $1.3 billion and is based on more than 50 statements Mr. Giuliani made at legislative hearings, on Twitter, on his podcast and in the conservative news media, where he spun a fictitious narrative of a plot by one of the biggest voting machine manufacturers in the country to flip votes to President Biden. +

\ No newline at end of file diff --git a/packages/readabilityjs/test/test-pages/nytimes-podcasts/distiller.html b/packages/readabilityjs/test/test-pages/nytimes-podcasts/distiller.html new file mode 100644 index 000000000..6a82055be --- /dev/null +++ b/packages/readabilityjs/test/test-pages/nytimes-podcasts/distiller.html @@ -0,0 +1,511 @@ +
The Ezra Klein Show Poster
We Know Shockingly Little About What Makes Humanity Prosper Patrick Collison calls for a new “science of progress.”

+ There just was no market rapid advance in human living standards. It’s only in the past 10,000 years, and then practically in the past few hundred — just an eye-blink in the time human beings have been on Earth — that things kept changing, usually for the better. And the question is, why? +

+ And Collison’s particular meta question is, given the clear fragility of forward motion here, given how rare it has proven to be — and so how easy it might be to lose — why isn’t the question of the conditions of progress more central? Why isn’t the study of progress in a wide multidisciplinary way a more common and central discipline? +

+ Collison has written a few influential essays here, with the economist Tyler Cowen. He called for the inauguration of a discipline — they call it progress studies — and that now has people studying it. There’s people creating journals for it, creating syllabi and podcasts and books around the topic. It’s one of the more singularly successful calls for a research direction I have seen. +

+ Separately, in a piece co-authored with the scientist, Michael Nielsen, Collison and Nielsen argued that, though it is hard to measure, it seems like the rate of scientific progress is slowing down, and that’s particularly true if you account for how much more we’re putting into science, in terms of money, of people, of time and technology. +

+ Now, these ideas are not original to Collison. The point is not that nobody studied human progress before this or worried about the pace of scientific research. He wouldn’t claim that. It wouldn’t be true. But he is playing a distinctive role in their framing and their popularization, and in creating and funding a community around them. +

+ And what I see in my travels here is that it is working. Something is burbling here. But I can’t find many big pieces where Collison really lays out his worldview. There are a couple essays, tweets, interviews, but he’s not been primarily writing this down. +

+ What he has been doing is funding it through Fast Grants, which has been successful, but more than that, intellectually influential effort to show you can give out scientific grants quickly and with very little overhead, through the Arc Institute, a big biotech organization he’s creating to push a researcher-first approach to biotech, and through giving a bit of money, and a bit of time, and a bit of prestige, and a bit of networking to a lot of different projects that circle these questions. +

+ He’s got this funny quality of being nowhere in particular, but also somehow, almost everywhere, if you’re interested in these questions. So what I wanted to do in this conversation was try to get as close as I could to the Patrick Collison worldview, the underlying theory of the case here that animates his thinking his funding, and the ways in which he’s trying to nudge the culture he’s a part of, or the ways in which he’s trying to actively create a culture he doesn’t yet see. +

+ As always, my email — ezrakleinshow@nytimes.com. +

+ Patrick Collison, welcome to the show. +

+ PATRICK COLLISON: Great to be back. +

+ EZRA KLEIN: So you’ve made the argument that science — all science — is slowing down, that we’re putting more money and more people into research, and we’re getting less and less out of it. Tell me about that. +

+ PATRICK COLLISON: Well, I want to separate two things. There’s a question as to whether science in its totality is slowing down, in terms of the absolute returns from it. I think that might be true. You can maybe divide up the first half of the 20th century and the second half and so on, and sort of try to compare one with the other. +

+ And we had general relativity and quantum mechanics and various other major breakthroughs in the first half. You can ask the question of, well, did we have as many in the second half? But in the second half, we did have the discovery of D.N.A. and molecular biology and lots of other things. +

+ So I don’t know that I would claim a total slowdown. The thing that I think is clearer and should be very concerning to us is, as you look at the number of scientists engaged in the pursuit of science, and if you look at the total amount that we’re spending, and as you look at the total output, as coarsely measured by things like papers and number of journals, all of those metrics have grown by, depending on the number, let’s say, between 20 and 100x between 1950 and, say, 2010. +

+ And if you look at it on a per-capita basis, or a per-unit-of-work basis, now used to divide all those total outcomes by a factor of 50, and it seems like if you imagine yourself as the median scientist, you’re meaningfully less likely to produce anything like as consequential a breakthrough as you would have, say, in 1920. And so Michael Nielsen and I, in order to try to put slightly more rigor on that question — we went and we surveyed a bunch of scientists across a number of universities in a number of different disciplines, and we presented them with different Nobel Prize-winning breakthroughs. +

+ And we tried to compute an approximate ordering of their significance in the eyes of these scientists. And the thing that would kind of have to be true — for the per-capita impact, we remain in constant — is we’d have to be discovering much more important things in the latter half of the 20th century in order to compensate for, to make it worthwhile, for us to be investing this 50-fold greater effort. +

+ And we didn’t find that. In physics, in the estimation of physicists, there was a kind of flat-to-declining trend. It’s not super obvious which way it points, but in as much as there’s a trend visible, it’s probably slightly downwards. And in other fields, it was maybe similarly equivocal, perhaps a slight increase, visible in some, but importantly, in no fields that it looked like we’re on this crazy, exponentially improving trajectory, which is what you would have to have for this per-capita phenomenon to not be present. +

+ And I think that should be something we’re interested in for multiple reasons. One, because presumably, as a society, we’re interested in just how much more scientific progress and technological progress and so forth, how much more innovation is there going to be over the next 10 years or the next 50 years or the next century. But also, because there’s kind of two possibilities. One possibility is, fundamentally, we’re running out of low-hanging fruit, and it’s just going to be harder to do this stuff. +

+ And in as much as we’re setting investment or making investment decisions around to what degree should be pursuing the stuff, I guess it’s important to know what we think the returns should be. Or the other possibility is, somehow, we’re doing it suboptimally. Something changed, and we were pursuing this process of discovery more effectively in the past, and presumably, for inadvertent reasons, something went wrong, and now, we’re just less efficient at it. +

+ But either explanation — and it doesn’t necessarily have to be fully binary — but either explanation is important, and either explanation, I think, has prescriptions for what we should do going forward. +

+ EZRA KLEIN: Let me start with the low-hanging-fruit explanation, which I think is a more popular one. And you have — in the piece you did on this with Michael Nielsen, the sad, but in the very academic way, very funny quote from the physicist Paul Dirac, who says of the 1920s, there was a time when, quote, “Even second-rate physicists could make first-rate discoveries,” which I just kind of love. +

+ But the theory there is you can only make a lot of the big discoveries once. You discover quantum mechanics once. You discover the atom once. And most of them have just been made, so what you have now is more complicated, smaller, requires much larger teams of people, much more complicated experiments, with much more infrastructure. +

+ So we’re just structurally in a period where it’s going to get harder and harder and harder to make big gains. Do you believe that? +

+ PATRICK COLLISON: I think it’s possible, but even though it’s intuitively compelling on some level, I’m not sure that it’s true. It’s probably true to at least some degree for some particular research direction, right? We go after discovering the various subatomic particles, and initially, without too much difficulty, we discover the electron or whatever. +

+ And by the time we’ve discovered the nth quark, it’s now gotten super hard, and even with ever-larger particle accelerators, we’re not necessarily making breakthroughs of the same magnitude. So I think it’s pretty true for a given direction. But obviously, the question is, well, to what degree is progress in any area opening up other directions, right? +

+ And so I mean, you mentioned the Dirac quote and, say, physics in the early part of the 20th century. Those discoveries opened up new techniques and investigation methodologies and so on, that then gave rise to molecular biology in the ’50s, ’60s and ’70s. And so there’s kind of a combinatorial benefit, where discoveries over here or discoveries over there might unlock opportunities and major breakthroughs in areas that we could not have foreseen in advance. +

+ There are lots of, quote unquote, “low-hanging-fruit discoveries” made in computers and computer science in the ’70s, ’80s, and ’90s. Maybe we’re even still in that regime, right? We’re still making some pretty fundamental breakthroughs. And of course, again, those, quote, “low-hanging discoveries” would not have been possible without a lot of this optimization and discovery in other fields. +

+ And so I think it’s probably true for a given research direction, but the relevant question for society is, is it true in aggregate. And there, it’s much less clear to me that it is. +

+ EZRA KLEIN: I want to read something provocative you said in an interview with the economist Noah Smith. And you said, quote, “Most systems get worse in at least certain ways as they scale. The idea that science could have gotten worse in significant ways sometimes sounds strange to people. Like, we’re doing so much more. How could that be bad? But I think that misses the many examples of sensitivity of scientific processes to institutions and culture. Swiss nationals have won more than 10 times more science Nobels per capita than Italians have. 10 times. And yet, they’re neighbors. And Italy certainly isn’t lacking in scientific tradition — Fermi, Galileo, the oldest university in Europe, et cetera. The ‘how’ of science just really matters.” +

+ And this seems, to me, to be where your exploration really goes. So tell me what you think might have gone wrong in the “how” of science. +

+ PATRICK COLLISON: So I think this point about the sensitivity of scientific outcomes to the specifics of the institutions and the cultures is very important and probably underappreciated. At the beginning of the 20th century, not only was the U.S. not a scientific powerhouse, but it barely had a presence in frontier research, whatsoever. +

+ To become a credible researcher in the U.S. in 1900, you almost certainly had to go and spend time in, most likely, Germany, and failing that, in France or England — you know, what have you. And by 1900, the U.S. was already a pretty prosperous place, and it had a well-educated society, as societies went. +

+ And yet, somehow — and it had universities, right? I mean, Harvard was hundreds of years old by that time. And so it checked many of the ostensible boxes, and yet, the sum total of the U.S.’ research output as of 1900 was still de minimis. +

+ When James Conant, who was later president of Harvard for 20 years — when he went to Germany as a chemist, which was his original training, in the 1920s, he recounts how dispirited he was by what he found there and how far ahead of Harvard German research was, as of the early 20th century. And then, for a variety of reasons, all sorts of cultural, institutional funding — various transformations happened. And of course, by the latter half of the 20th century, the U.S. was the unquestioned leader at the frontier of scientific progress. +

+ If you look backwards, you see where that locus has been, where the most successful and fertile scientific grounds have been — it has repeatedly moved. As we just said, maybe the 19th century, it was Germany. +

+ Before that, in the 18th century, it was plausibly France. They had a couple of these really successful École Polytechnique and Grande École and so on. And so as a kind of first-order empirical matter, we can just notice, huh, this really seems to matter — and then, the example you just gave of the divergence between Switzerland and Italy. +

+ And so then, if we kind of accept that, and we try to ask ourselves, well, specifically, what are the mechanisms? You know, what’s actually going on? It’s hard for me to say. It seems like the transmission of research culture by individual researchers matters a great deal. +

+ And you see these kinds of pockets of the cultural transmission repeatedly crop up, where Gerty and Carl Cori — you probably haven’t heard of — they ran a little biology lab in Missouri, and no fewer than six of their trainees, of students they trained, went on themselves again to win Nobel Prizes. +

+ And if we tell ourselves a standard kind of mechanistic story as to, well, it’s the funding level, it’s how much are we investing in science, or it’s something about whether there’s an institution in the courser sense, that can possibly be amenable to it, it’s very hard to explain these eddies where you see these pockets of excellence really produce these outsized returns. So I think it’s a complicated question. +

+ I think all of aggregate culture, funding, institutional characteristics, and so on all contribute to it. But if I had to isolate a single variable, it seems to me that the research culture set by specific people and the tacit knowledge transmitted through direct experience is probably the number-one thing. +

+ EZRA KLEIN: This, I think, is where I sometimes fall into my own pessimism on this. Because I want to believe, as you do, that we can double the rate of scientific advance, maybe even go further than that. But I think the prediction — if I’m putting this on institutions, on culture, on pockets of transmission and mentorship — I think the prediction I would make is then, even if you believe, say, that America had a great 20th century, but its institutions have become sclerotic, and we’ve slowed down, and everything is piled in lawsuits and review boards now, somewhere else that didn’t have that, that has a different culture, that has different institutions, would be pulling way ahead. +

+ So you might think, well, China will be pulling way ahead. And you’ve noted this in some places. We’re getting a lot of peer-reviewed research out of China — huge number of citations out of China. We’re not seeing them dominate the big breakthrough advances of the era. +

+ It doesn’t seem like Europe is lapping us. And so if you think this slowdown is somewhat global, then that seems to me to militate against questions of individual institutions, cultures, how different labs work, because there is so much variation that you should have some of these labs that are doing it right, some of these places that haven’t piled on a little bit too much bureaucracy. But I don’t think we really see that. +

+ PATRICK COLLISON: This diagnosis of these phenomena to cultural, institutional, mentorship-related, interpersonal dynamics, and your observation that it’s not obviously the case, that there are other places we can pointed that are doing it so much better — for me, my takeaway is that, well, successful cultures are a pretty narrow path. Homo sapiens emerged 200,000 years ago. +

+ And as far as we can tell, for the first 190,000 years of our genesis, we think we were largely biologically equivalent to the people we are today. But as best we can tell, there was some kind of cultural capital that those people lacked for a very extended period of time before human societies in somewhat recognizable modern form started to emerge — agriculture, all the rest. +

+ And in a similar vein, we had many billions of lives and centuries elapsed before the Industrial Revolution., and before we started to put together many of the input ingredients or enough of the input ingredients that we can get sustained improvement in standards of living and ongoing economic growth and progress. And so your point about, well, as I look around, I don’t see anything or anywhere that’s obviously better, I agree with that. +

+ But again, my takeaway is that that’s what makes the question of how do we improve or how can we do somewhat better so urgent and pressing, where it’s many things have to go right. It’s not easy to be even as good as — or to get to a place where things are as good as they are today. What we have is very precious. And I think the threads and the themes that you’ve been pulling on of late — all of these dynamics underscore their importance. +

+ EZRA KLEIN: I think that’s a good bridge to progress studies as an idea. And I want to have people hold in their heads that idea that progress is very narrow, that it is a very narrow bridge that we have walked on for a very short period of time. But let’s try to define it. +

+ When you say progress here, what are you actually talking about? Is it just shorthand for economic growth or G.D.P.? What is progress? +

+ PATRICK COLLISON: Well, I don’t know that I would claim to put forth some kind of definitive definition. And I think, to some extent, our intuitions around it are probably broadly correct. And so it might not matter to define it super precisely and finely. +

+ For, me it is something along the lines of our success in realizing a liberal, pluralistic and prosperous society, and a sense among people that their offspring can and probably will do better than they themselves have, and that more broadly, the future will be better than the past, and that we’re at least making incremental progress towards embodying values and morals that we collectively think we can be proud of. +

+ But I don’t think anything that novel in that. I don’t think my conception of progress would differ that materially from some kind of average aggregate over any other group of people in the country. +

+ EZRA KLEIN: I do think there’s something interesting, though, which is that if you look at eras that I think progress-studies-type people and economic-growth people and historians of economic growth study most closely, actually, some of the periods where people feel a lot of rapid progress don’t fit that at all. You have, say, the Industrial Revolution, where life spans and lifestyle get worse for a lot of the people. +

+ I don’t think one will look at that period as unbelievably pluralistic. You have a lot of periods of war when you have very, very, very rapid technological progress, but it happens in context of much more martial societies. So there is an interesting tension, at least in periods — and some of them quite long, actually — where you can have fairly rapid economic progress, but it comes at a cost that I think isn’t always acknowledged, but is an important thing to think about. +

+ PATRICK COLLISON: Yeah. So I don’t think you could point to some of these periods in the past and say that they definitively embody to the extent that we would fully aspire to some of these broader traits and characteristics. But I think the question is more, what are they doing as — you have to judge it relative to the baseline that preceded them. +

+ And I don’t know that the 18th century in the U.K. is some ideal as a society. But if you compare it to the 16th century in the U.K., the ideals and ideas of natural rights and religious tolerance and so on — they were somewhat better embodied by the 18th century than they had just a couple of centuries previously. +

+ And similarly, in the U.S., say, during either war or the ’30s or whatever, again, it’s not like that was any kind of perfect society, but assessed relative to the society of 1830, I think it compares relatively favorably. And I think it’s not a coincidence that Adam Smith — his first book, of course, was on ethics and morals and trying to instill better general ideals and behaviors across a society. +

+ And maybe after that, he then argued for and laid many of the foundations of what we would recognize as modern economics. So I don’t think it’s perfect. But on average, I think the correlation is positive. +

+ EZRA KLEIN: So let’s talk about the Industrial Revolution for a little bit here. I think a lot of people locate a takeoff in human living standards — it continues to this day — there. And it’s strange in a way, right? “There” is a very geographically contiguous spot. It’s the U.K. — England, actually, I should say, at that point. +

+ And there is a moment in time that probably could have come at another moment in time, depending on how human history plays out in the counterfactual. I know that you have an interest in the theories of why then, why there. How do you work your way through them? What do you think is persuasive for why then, why there? +

+ PATRICK COLLISON: Well, you know, again, I caveat. With all of these topics we’re discussing through this podcast, maybe the first-order banner for all of them should be, I don’t know, these are my best guesses, and I think it’s important that all of us were pretty humble in the claims and the assertions and the beliefs that we hold. +

+ Recently, I’ve been reading a bunch of Irish and Scottish writers around then. It’s very interesting, because for both the Irish and the Scots, there was a sort of a pressing and kind of obvious question where England was much more prosperous than they were or we were. And there’s no super obvious explanation for that. There wasn’t an obvious climatic or natural resource endowment that England benefited from that was lacking in Ireland or Scotland. +

+ It wasn’t like England was actually a vastly larger polity. The orders of magnitude were comparable. And Bishop Berkeley wrote this book, “The Querist.” He was asking these questions directly, just like, what’s going on? What’s wrong with Ireland? You know, why can’t we do this? +

+ And then, you have the Act of Union in 1707, uniting Scotland and England — and sort of similarly, of all these Scottish thinkers being like, all right, we’re now literally the same country. Why are we so much more impoverished? And then, if you shift to England, there’s Joel Mokyr and — you’ve read his work — and more recently, people like Anton Howes. +

+ And in a similar vein, they go back to — I mean, the word, improvement, came from Francis Bacon, or it was kind of popularized as a concept by Francis Bacon. But that’s noteworthy, right? Like, that was not a pervasive broad concept in the 15th century. +

+ I mean, literally, the word, improvement, in this broader societal context, came from word, “translated,” at the beginning of the 17th century. And the ultimate conclusion that these historians and scholars and analysts of the Industrial Revolution come to — and I think it’s a correct one — is somehow, whether it’s through Bacon or Newton or various of the tinkerers who produced some of the earliest technological breakthroughs, that somehow, this improving mind-set became pervasive. +

+ You had societies explicitly — like the Hartlib Circle or the Lunar Society, or the Select Society, and the club, and so on — all these societies explicitly devoted to figuring out ways to advance the state of affairs that prevailed. And these societies were comprised of many of the leading people and thinkers and so on of the day. +

+ And it seems maybe a bit satisfyingly squishy to attribute it to something so hard to pin down. But as you run through all the possible other explanations, it’s differences in IP law. It’s difference in the Malthusian conditions. It’s difference in the prevalence of coal, you know, et cetera, et cetera. Through various cross-sectional analyses, you can exclude most of these in looking at all of Ireland, Scotland, and England. +

+ It really does seem to me that differences in the mind-set and in the culture are where you have to net out. And that’s not to say maybe that it’s fully sufficient. There might be other preconditions that are important. And then, maybe as a last thing to say, it is striking to me that many of these kind of original 18th-century economic writers and thinkers — and again, the kind of people we look to as the founders of much of the discipline — that they themselves were kind of centrally preoccupied with this. +

+ And yeah, they were in favor of free trade and specialization and human labor and lots of these concepts that we’re now very familiar with, but they really thought that general mind-set played a big role, too. +

+ EZRA KLEIN: So let’s talk about Joel Mokyr ideas for a minute. So Mokyr is an economic historian. People should read his book, “The Culture of Growth,” which is really fascinating. He argues, as you’re saying, that in this period, this mind-set that we can increase the store of usable knowledge, and then use it to alter nature, to better the human condition, takes hold. +

+ That’s a new mind-set. It’s different than religious ideas of the past. It’s different than cultural ideas of the present. And that, plus a bunch of other things, particularly the republic of letters, the way people are writing letters back and forth, kind of combine into a culture that is able to grow. +

+ But one of the things that I really take from his work, that sits in my head, is he believes it’s all very contingent. He really believes it might have not happened. But the other is that I think it opens up this question that as a tech person, I’m curious to hear your thoughts on, which is, he really believes — Mokyr really believes — that there is a communications infrastructure that arises at that time, that has a kind of culture of generosity and argument and honesty in it, and is built on writing letters slowly to one another, and then copying those letters over to other people. +

+ And that culture is really good for intellectual advancement. I think one of the promises of the internet and the age we live in is, it’s all faster. We can write to people immediately. Things we write can go viral and be seen by 5 million people all of a sudden. +

+ And that was going to speed up economic growth really, really rapidly. And I would say, you don’t see that. So I’m curious how you think about communication cultures here and what you think for all the advantages of ours we might not have. +

+ PATRICK COLLISON: I mean, I think it’s hard to say in aggregate. I feel it’s pretty likely that the effects are very heterogeneous across different populations. And you’ve made the case that you think Twitter is bad for journalism and for journalists. +

+ And I guess you live this yourself with your now mostly inactive Twitter account, I guess, apart from announcements. And I think in the case of the internet, that it’s almost certainly a tremendously large gain that billions of people now have access to educational materials. And some of the otherwise hard-to-communicate tacit knowledge — that things like YouTube videos now made legible and available. +

+ And I think it’s true that there are various gravity equations that we see across different disciplines. I mean, in economies themselves, in trade, where you rapidly decline in propensities to trade as countries get further from each other — but you have versions of this in academic disciplines as well, where geographic distance correlates inversely with likelihood of the exchange of ideas and so on. +

+ And I think it’s clearly the case that the sort of reaction surface area has increased substantially by the internet there and represents a kind of efficiency gain for people looking to exchange in ideas. Many of the companies that Stripe works with are remote companies, and they might employ people across myriad countries, and that’s a kind of communication and efficiency gain that would certainly not otherwise be achievable. +

+ I think it’s worth recognizing that the aggregate amount of G.D.P. that we are creating or gaining every year is so much larger now than — I mean, the percentage might be the same. But the total amount of stuff happening, or the increasing amount of stuff happening, is so much larger now than it was 100 or 200 or 300 years ago. +

+ And so for all of those reasons, I think we should give superior communication technologies and faster communication technologies a significant amount of credit, even though the ways in which those are manifests might be hard to measure and somewhat prosaic. +

+ Take my mom, for example. My mom works with a hospital in Minnesota. Our youngest brother has a physical disability. And in the course of that, she trained herself in treatment for cerebral palsy, this condition, and she wrote a book about it, and she did a master’s in this. And now, she’s trying to improve treatment for this condition throughout Ireland, in the U.S. and other countries as well. +

+ She’s a retired Irish mother who spends some of her year living in the U.S. near her sons, spends the rest of her year living in Ireland, working at a hospital in Minnesota, who just got a proposal to have her book translated into German a couple of days ago. And that’s a relatively prosaic story, but literally, millions of these stories exist in kind of aggregate form around the world. +

+ To circle back to the initial thrust of your question, though, I think it’s at least possible that the internet is bad for civic discourse. I’m not saying it is, but it’s certainly in the realm of plausibility — and that perhaps both things are true, where there’s some kind of iceberg where there are these enormous welfare gains that are not that legible, not that visible, lie beneath the surface, and then certain of the most visible manifestations, like what we see on cable news or what we see written in the papers — perhaps that is worse, and perhaps, slightly more structural judiciousness would be desirable there. +

+ EZRA KLEIN: I want to try to flip that and suggest that — because I’m going to push some counter ideas on why we maybe don’t see as much progress as we wish we did. But one is that I think possibly, very large welfare losses lie beneath the surface. And beneath the surface of stories like the one you just told about your mother, I think we all have stories of ways or people for whom the internet has unlocked a possibility. +

+ I mean, my whole career is built on the internet. I was an early blogger. I got rejected from my student newspaper. And if there was no blogging, like, god knows what would have happened to me. [LAUGHS] I mean, nothing too terrible, probably, but I wouldn’t have the career I have today. +

+ And at the same time, I think that the group of people who, by luck or by temperament, proved very, very good at using the internet, to some degree, distracts from the many, many, many people for whom the internet is fundamentally a distraction machine, or for whom the internet is creating, because of what we built on it. You know, shorter attention spans — how many people would have had an idea, sitting in a room by themselves, or taking a walk, that they never have now, because they never have to have a moment where they’re thinking alone? +

+ And so one thing that I think we’re all loathe to do is we’ll talk a lot about how it’s weird that we have so much more knowledge, but productivity isn’t increasing faster. It’s weird that we have so much more rapid communication between researchers, but science isn’t advancing faster. And then, the idea that maybe there are things happening to us that makes us less able to use that increasing stock of knowledge well, or makes us less able to collaborate in a useful way, I think, gets dismissed rather quickly. +

+ But I don’t think it’s totally implausible. Now, I don’t want to say, like, the greatest technology we ever had was letter-writing. Obviously, the greatest technology we ever had was blogging in the early aughts when I became a blogger. And whatever happened in your 20s is, like, as good as it was ever going to get. +

+ But I do wonder about these questions. And I think something Mokyr is right to put a lot of attention on is communicative cultures. Communication is how we collaborate. And if communication is in any way getting worse, it’s going to have pretty big macro effects. +

+ The other thing is if you believe these cultures matter, weirdly, as big as we’re getting, the internet allows a certain disciplines culture to stretch boundaries and borders in time in a way that it would have been harder. I suspect that labs were more different 50 years ago than they are today. +

+ The countries and the disciplines of researchers and the cultures of researchers in countries or cities are more different from each other 50 years ago than today, which is great if we have the best of all cultures today, but it’s not that great if you actually think variation is really important. +

+ PATRICK COLLISON: Let’s wrap up there. So first, I agree, as a basic matter, that there are welfare losses occurring across society that we should be worried about, and probably everybody listening to this is familiar with the Stephen Pinker case for optimism, and rather than focusing in the headlines, you zoom out, look at these long-term time series. And once one does that, things seem a lot more encouraging, whether you look at it by income or life expectancy or infant mortality or choose your metric. +

+ Something that’s been striking to me of late is if you change the x-axis on those time series, and look at many of those phenomena and trends over a much shorter window, the valence changes substantially, and life expectancy in the U.S. is now, in fact, declining. According to C.D.C. data, 54 percent of teenage girls now report persistent feelings of sadness and hopelessness. And you could say, well, teenagers were never stereotyped as the most cheerful lot, but we do have some degree of longitudinal data here, and that number is up from being in the 20s as recently as 2009. +

+ ½ the population now is either prediabetic or diabetic — again, according to the C.D.C. Basically, point is, when we look at more recent windows, I think there are plenty of aggregate, emergent, complicated outcomes and phenomena that should give us concern. On the degree to which we should attribute the diagnosis to the internet or to our kind of communication media more broadly, it’s less clear to me in that — not saying it’s not true, but presumably, the life expectancy one is not — or at least if it is, the mechanism has to be very complicated. +

+ There are a bunch of other health-related ones. So take, for example, say, the incidence of diabetes or pre-diabetes. And you could say, OK, fine, all those things might be true, but they’re totally different. I guess the question I wonder about is, well, we know that lots of basic biological outcomes are correlated with mental states and so on. +

+ And so to what degree is there some more nuanced and complicated relationship there? But I think it’s a fair question, and I wonder a lot about it myself. +

+ EZRA KLEIN: Let me ask you about how you think, over the long period here, about the relationship between technology and equity or egalitarianism. And something specific is in my mind. I flicked earlier at the way the Industrial Revolution, for an extended period of time, seems to have reduced a lot of people’s living standards. And it wasn’t till later you had changes in redistribution in labor unions and labor protections that the amount of material prosperity that was generating created more broad-based prosperity, particularly at a very high level. +

+ I don’t know that you can sustain that kind of thing today. We have much more a small-d democratic culture. If things aren’t working for people, it’s much easier for them to organize and be heard. +

+ I think there’s a much more direct and complicated relationship now between whether or not people feel benefited by technology, and whether or not they are going to accept the conditions and the risks of rapid technological advance. But I’m curious, from your vantage point, how you see that both kind of historically and currently. +

+ PATRICK COLLISON: I agree with that. I worry a lot about the basic stability of a society that does not successfully generate and make sufficiently broadly accessible the benefits of economic growth. The world simply has too little prosperity. And if it is not the case that people in the U.S. or people in any country — if they either feel like things aren’t progressing, or if they feel like maybe somewhere distant from them, things are progressing but they personally will never be able to benefit from it, I think we put ourselves in a very dangerous and likely unstable equilibrium. +

+ And if you go back to — well, you don’t have to go back very far in history to see, obviously, plenty of instances where this kind of instability brought the whole house of cards down. And so as a consequence of that, I worry a lot about, how do we simply make sure that — or one of the small things we each individually can do to try to make sure that society is generating enough economic gain and enough broadly experienced welfare gain that the whole compact can be maintained? +

+ And again, I don’t think there’s a ready neat kind of singular answer to that. Maybe Stripe as part of our small little contribution in one little fissure. But I think the central question you’re getting at is super important. +

+ EZRA KLEIN: And one of the questions I wonder about there — we’ve talked about the way progress has been very geographically lumpy, let’s call it, right? There’s a lot that happens in very small places, and it ends up affecting the whole world. Obviously, then, the gains of progress sometimes have that quality, too. +

+ And I do think of one of the politically destabilizing effects of the past, let’s call it, 30 or 40 years of digital progress, is being the concentrations of wealth. We just used to have a lot more spread. Even putting the questions of rising inequality aside, just where rich people were was different. +

+ And so where they were giving a lot of money to the local hospital was more spread out, say, across the country or in other countries across the land. But here, even as the internet is supposed to democratize distance, and in many ways, has — I mean, telework is not a fake phenomenon. It has really concentrated the wealth of that to, literally, where we’re sitting, but to New York. There’s a lot of money now in Austin. +

+ And then, on top of that, you often have barriers of entry, in terms of how many homes can be bought. So it’s not even like people can move to the place where all the economic opportunity is happening. And I do think that creates some of the skepticism you see of technology. +

+ I don’t think a lot of people’s — I think people are really excited about a lot of the goods they’ve gotten from it. But in this kind of macro political sense, as you’re saying, in a period of a lot of change, a lot of folks with real backing in the data don’t feel life has gotten better at the macro level. +

+ Life expectancy, happiness, political stability — it’s not like you can look around and say, well, I got this computer in my pocket, and everything else is going great, too. It’s like, I got this computer in my pocket, and what it keeps telling me is that everything is going to hell. Now, maybe it’s telling me that a little bit too much, but there is validity to the narrative. +

+ PATRICK COLLISON: Yeah. So again, vehement in agreement on the sort of central importance of making sure that improvements in the standard of living are actually broadly realized across the society. That, too, I think, could serve as a manifesto for some of these Progress Studies ideas. +

+ On the internet in particular, or on technology and the technology sector and so forth, I think it’s complicated and difficult to try to sort of fully collapse or linearize it or something, where on the one hand, you have some of these concentration dynamics you identify. At the same time, of course, it is also a tremendous and incredible dispersal agent in making some of those possibilities and opportunities be more broadly available. +

+ I think it’s dangerous to take an excessively U.S.-centric perspective here. If you interact with or look at survey data, or otherwise try to assess what’s the sentiment of people in Poland, what’s the sentiment of people in India, or what’s the sentiment of people in Indonesia, they view the internet extremely positively. And I think correctly so, where their opportunities for advancement would be substantially curtailed in the absence of much of what the internet makes possible. +

+ I think in places like the U.S., or actually, even at home in Ireland, some of this story is complicated by lots of other things, but including — and I think, substantially — dynamics around housing policy, where there’s an extensive literature showing that, for example, a very substantial moderating effect on the increase in wealth inequality that would otherwise have happened, or income inequality, was geographic reallocation and people responding rationally to, OK, things in New York are going great, or things in California are going great. +

+ And if you look at the rate of increase of the Californian population, say, through the 1960s, that was a tremendously potent mechanism for us redistributing some of the economic gains that were being realized at the time. And of course, now, we have this crazy position, where California is losing population at the same time where the market caps of these companies and the profits of these companies are increasing very rapidly. +

+ But as one assesses that dynamic and tries to ask the question of, well, why aren’t these gains being better or more broadly distributed, it’s certainly not clear to me that the answer even lies in the realm of technology qua technology. And I think it’s certainly more broadly, again, some of these considerations like geographic allocation. +

+ EZRA KLEIN: Let me ask one more question on the geographic dimension, and then I’ll move on to it. And it’s on my mind, in part because when I try to think about progress, when I try to think about what inventions and innovations are coming really quickly, I actually see a bunch here. And I’ll use A.I. as an example. +

+ I’ve met people who are trying to automate a bunch of legal contracts. It makes a ton of sense. People pay a lot all over the country — to some degree, all over the world — to get fairly basic legal contracts drawn up — wills and real estate documents and merger agreements and all kinds of — from the small to the large. +

+ If you imagine that getting really effectively automated, though — +

+ you think about Saint Louis, Missouri, where some of the people who are important pillars of the community work in law firms there, and what they do is contracts. They do estate planning and all the things that people have to do in contracts. And if it actually does get concentrated to really, really great contracting firms in the Bay Area or in New York, on the one hand, the democratizing potential will really be realized. Those contracts will get cheaper. +

+ There’s also a theory in crypto of smart contracts. And on the other hand, you really will have a lot of that — the gains of that, economically, going to smaller areas and aggregated across a bunch of different domains. So graphic design, in all kinds of areas of the country — midlevel graphic designers get paid to make logos for local businesses. +

+ It’s pretty clear they’re going to be able to do that really, really easily on things like DALL-E pretty fast. So you can imagine a lot of that area getting wiped out. And you kind of run through a couple of these. And before you get to really unbelievable and sci-fi-like dimensions of artificial intelligence, you just have a thing that is going to democratize a lot of capabilities in a way that’s going to put the money for those capabilities both a little bit back into the pockets of the people who need them, and then a lot into the people who run the best A.I. rigs and is going to have a really weird geographically destabilizing effect. +

+ And that paradox of the internet both democratizing geography, and then concentrating wealth and capital in very small areas is, to me, a central challenge. Because if you get that wrong, if it goes too much in the concentration area, I think we’re going to lose a lot of the political stability we need here. But you’re more on top of these technological advances than I am. Do you think the trends there are going to play out differently than I’m worried they will? +

+ PATRICK COLLISON: First, yeah, it’s not — I don’t think it’s foreordained whether or not these are going to be centralized technologies. I don’t know. And on the one hand, there’s, I think, an obvious feature we can contemplate, where there are only three A.I. models, and they are rooted in the hegemons, the citadels of Silicon Valley technology, and we all are digital serfs who are subsistence-farming on their gains. +

+ I think there’s also a very plausible story where these technologies prove substantially less defensible than we might have expected, and where, instead, they have this enormously decentralizing effect. Because otherwise, economies of scale that only large firms could benefit from can now be realized and pursued, even by massively smaller firms. And if we look at the recent history of A.I. — I don’t think any clear story there, but it does feel to me that it has been more biased towards the second story than the first. +

+ There are now multiple companies with large language models. There are a number of very successful open-source A.I. efforts. Actually, there was a really cool example from Replit, which is a service — it’s a programming I.D. in the browser, used by kids learning to code, but also increasingly used by people who are pursuing serious programming. +

+ And they recently released a GitHub copilot-like technology, where it will kind of autocomplete your code in the editor, and where you can do some pretty cool things. Like, you can highlight a block of code and ask it to be explained, and it’ll turn code into natural language, into English, and say, hey, here’s what this code is doing. +

+ Anyway, they wrote a blog post about how they built this, and they describe how it was built by one guy over the course of a couple of weeks. And maybe that’s only the case in the early days of this AI technology. I mean, in early computer games, the first games were built by a single heroic person, and now, it’s these gigantic studios and enormous CapEx budgets. +

+ And so I think the fact that this is the case today doesn’t mean that it will remain the case through time. But my takeaway is that at least not foreordained that AI or any of these other technologies will be centralizing forces. And then, the other thing to observe is that when we talk about these being centralizing, I think there’s a question as to, do we look at it in relative or absolute terms? +

+ And my contention would be that, both from a moral standpoint, but maybe more importantly from kind of a political-economy standpoint, what will matter is whether, on an absolute basis, people feel like they are realizing opportunities, their lives are improving, that things are getting better, that their kids will be in a better situation and so forth. And exactly how much value is realized by the companies themselves doesn’t actually matter that much, compared to that former question. +

+ And whether A.W.S. or whether any of these organizations has super high or super low profit margins, I don’t know is nearly as important as what is the actual effect on these communities and individuals across the society. And so in as much as one means — by centralizing, one means a large share of the profits, I think it is probably a more useful framing to look at it instead in terms of absolutes, and in particular, the absolute surplus generated by the users. +

+ [MUSIC PLAYING] +

+ EZRA KLEIN: What have you come to believe about the relationship between progress and war? +

+ PATRICK COLLISON: I am somewhat skeptical that war is as conducive to breakthroughs as we might intuitively conclude, or as is sometimes claimed. You’re probably familiar with Alexander Field’s work on the ’30s here. And his basic claim is, the productivity gains we often attribute to the Second World War in the U.S. — like, those foundations actually were laid in the ’30s, and then the first half of the ’40s were a period of decreasing productivity as we massively, inefficiently reallocated our economic resources for the purposes of winning the war, which was probably a good thing to do, but inefficient in narrow economic terms. +

+ And he has a new book coming out, I think, next month, that sort of extends this argument into the ’50s. +

+ And as one takes stock of the scientific breakthroughs — and so Stripe Press recently republished Vannevar Bush’s memoir, where he takes stock of this. +

+ And obviously, you have, say, the Manhattan Project, and that’s a big deal, certainly. But it doesn’t feel to me that had the Manhattan Project not occurred, that peaceful development of nuclear technology would have been massively stymied. Maybe it would have taken another 10 years, but it was already happening to some meaningful extent. +

+ And then, as you take stock of all the other breakthroughs that took place in the U.S. during the Second World War, there were some meaningful stuff like blood plasma and blood transfusions. There was some significant breakthroughs there. Some of the first antimalarial medications, radar, the proximity fuse, which I’m not sure is all that useful outside of military applications. +

+ But by the time you get down to invention 6 on the list, I don’t know that as you compare that list to, again, some counterfactual of what would otherwise have ensued, that it looks radically better as you take stock of the Cold War and the enormous fraction of our economic resources and human capital that were devoted towards us, that the gains necessarily look that impressive. And so again, it’s super hard to judge. You don’t have proper controls and so on. But I’m skeptical. +

+ EZRA KLEIN: Let me take the other side. So there’s a question of, during war, how much did we invent during World War II. And that’s a question of how much the threat of war or the competition with an adversary ends up charging up innovation and convinces us to put resources, both in terms of people and in terms of money, and maybe in terms of institutions, into projects we wouldn’t otherwise have done. +

+ I think there’s an argument, at least, that we went to the moon because of the Soviet Union. It would not have done that for some time. Probably would have eventually done it, but also, who knows? +

+ As Derek Thompson, who I’m working on a lot of these ideas with, likes to point out, the Apollo Project was unpopular. It was not something that commanded wide popular support. +

+ Even now, if you look at the CHIPS Act that passed, it passed, with all that spending on semiconductor research and other kinds of next-generation technologies, under the framework of, let’s compete more effectively with China. If you look at all the things Darpa has done or been part of, the fact that “defense” is the first word in the Darpa acronym, I think, is meaningful. +

+ There’s something about what threat persuades societies to do, and persuades them to do technologically or what risks it allows otherwise-more-cautious governments to take, or what failures they could justify that allows them to have big successes. Something there doesn’t seem to small to me. +

+ And maybe it’s my political side, where I so often see scientific funding justified in Congress in terms of countries we’re competing with or are adversaries with. And I see what the defense industry can do that other institutions cannot, because they don’t get a lot of political blowback. But I don’t know. +

+ I worry a little bit about how much we seem to need the threat of another to accelerate things. And in a small way, maybe, we see what the pandemic — where we were willing to move much, much quicker on things like mRNA technology than I think we would have outside of it. +

+ PATRICK COLLISON: Yeah. So I think it’s certainly true that the crisis can cause the discontinuous shifts that have large effects, which in your example, say, are probably super beneficial. I don’t know. If you take Darpa as an example, it started as Arpa, as a more open-ended research institution and set of programs, and then with the Vietnam War, had the D pretended to it. +

+ And we decided, in the face of threat, to make it more applied, to take more seriously its translational and kind of, quote unquote, “competition-oriented mandate.” And I think that was bad for Darpa. And the internet, which arose under Arpa — it’s hard to think of innovations of similar magnitudes that then occurred in then-Darpa’s subsequent, say, two decades. +

+ If you take, say, U.S. science in general, the war — the Second World War — to some extent, the first, but much more so the second — precipitated an enormous centralization of U.S. science in its aftermath. Because we really marshaled together all of the — or a significant fraction of the scientific capacity of the U.S. in service of the war effort. +

+ For, example the 50 percent overhead, the fraction of government grants that goes to universities — that was chosen in the early days of the coordination of the war effort, and has now become a kind of a pillar of academic and research funding in the U.S. And in the aftermath of the war, we sort have this question of OK, we’ve kind of pulled everything together. Now, what do we do? +

+ And that became, in various ways, the N.I.H. and the N.S.F. and so on. I mean, the N.I.H. predated it, but the growth of the N.I.H. really occurred after the war. And the federal government, shortly thereafter, for the first time, became the majority funder of US science. And all that centralization — and I mean, you pointed out the benefits of variety and of experimentation and of heterogeneity, and having some degree of institutional and structural diversity and so on, I totally agree with all of that. +

+ And I think all of that was very meaningfully curtailed by, again, the aftershocks of some of the threats that we faced during the war. And if you think about the things that we’re maybe happiest about having happened — the founding of the major new U.S. research universities in the latter parts of the 19th century or the revolution in health care and kind of medical practice that first happened at Johns Hopkins, and then kind of codified in the Flexner Report, or the great industrial research labs of Bell and Park and so on — or excuse me — Xerox — they didn’t obviously come from a place of fear or a threat. +

+ They came from a place of hope and optimism and opportunity. And maybe there are some inventions that you’re more likely to get to from some of these external pressures. But yeah, if you gave me a dial, and I can kind of turn up or down the threat or fear index of society, it’s not super obvious to me that one would want to turn it up if what one cared about was the aggregate rate of progress. +

+ EZRA KLEIN: That’s a good bridge, I think, to the question of institutions. And I take one of the main concerns of yours, of progress studies, as being around institutional slowdown. You have this idea that we don’t meta-maintain institutions very well. +

+ They start in one place, and then over time, they crust over, and we don’t really know what to do with that. Give me a little bit of your thinking there. +

+ PATRICK COLLISON: I think institutions, the cultures they instill and act as kind of coordination points and training sites for — those of enormous consequence — I think much of the success of the U.S. and of various other Western countries has, in substantial part, been attributable to successful institutions. Most people would accept, I think, that there is, to some extent, consistent trends that tend to happen with institutions through time. +

+ Just maybe most basically, the problem that gives rise to an institution in the first place is probably a pretty real and significant problem. No one would have taken the time to found the institution if it wasn’t. And there can be some degree of drift there, where we don’t necessarily decommission the institution once the problem has subsided or abated. +

+ And then, you tend to attract a certain kind of person in the early days of an institution — people who are slightly less status and reputation and procedure-oriented, because a new institution almost never has that. And then, through time, the sort of collective or the mission-oriented incentives of the institution can kind of drift somewhat from the individual incentives that particular people are subject to. +

+ I think all this stuff exists. And the thing that I observe, or that I just find myself thinking about is, we’ve had eras of institution formation in the U.S. It has not been kind of a constant rate through time. And the New Deal maybe, and say, the 30 years afterwards, and the Great Society — we bookend it with those start and endpoints. +

+ That was a period of tremendously active institution construction and formation in the U.S., Darpa being — or Arpa originally being a good example, and indeed, NASA. And I guess I find myself wondering, one, if we didn’t have any of these institutions — and I’m not saying we should get rid of them. But if we didn’t have them, what institutions would we found today, first, and how high in the list would NASA be, for example? +

+ And then, secondly, in as much as we accept that some of these institutional dynamics exist, like the fact that sclerosis as an emergent property arises, what do we do about that? And lots of people have told us it’s pretty — doesn’t need a lot of teasing apart to see it as one compares NASA and SpaceX and the respective budgets, and the respective achievements, and so forth, I think it’s hard to not at least wonder about their respective efficiencies. +

+ And say, if society could only have SpaceX or NASA, which one would we choose, and what should we conclude from that, and to what extent do those phenomena generalize elsewhere? I don’t have answers to these questions. But I find that in the political discourse — not that anybody is celebrating that, but in the discourse, it’s very easy to get, I think, very wrapped up in questions of optimal funding levels, and should this number be 10 percent or 50 percent or higher or whatever, whereas to me, a lot of our satisfaction with the outcomes seems to hinge on deeper questions about the nature of the institution. +

+ I don’t know that the problem or benefit, or anything good or bad about NASA is attributable to the budget, per se. It seems more, kind of, resonant in some of these deeper cultural questions. And then, in the recent pandemic, or in the — I don’t know. I was going to say, ongoing pandemic. But I guess as of two days ago, with the President’s verdict, it is now over. +

+ EZRA KLEIN: It’s over. Congratulations, everybody. +

+ PATRICK COLLISON: Exactly. But anyway, I think that was maybe a vivid demonstration of many of these dynamics, where I don’t know this any of the story about the institutional response to the pandemic should be primarily one of funding. I think it’s much more about the dispositions and the attitudes and the cultural biases of entities like the N.I.H. and the F.D.A. and the C.D.C. +

+ EZRA KLEIN: I find the NASA SpaceX example an interesting and provocative one. Because on the one hand, I think what you’re saying is completely true. And where a lot of the NASA programs and projects have gone in recent decades, is just — it’s sad. It’s just a sad story. +

+ And on the other hand, the idea that you — the thought experiment of choosing between NASA and SpaceX — the thing that it immediately asks is, well, you can’t. Because without NASA, there is no SpaceX. And one way the private sector handles a lot of these questions — I mean, I’m always struck by how much of the way biotech research works is that big pharmaceutical companies acquire small biotech firms that have made a breakthrough or have come up with a very promising candidate. This is kind of an accepted thing that the big companies — they do a fair amount of research, but a major, major innovation transmission there is small groups do more, quicker, and they’re just going to buy them. And the NASA SpaceX example has a little bit of that dynamic to it, although with a different mechanism of financing. And I don’t know. I wonder if there aren’t deeper lessons there. +

+ PATRICK COLLISON: Yeah, I don’t mean here in the NASA example — like, I don’t think reducing it to a simple binary of this-or-that is correct. It’s more, what should we make of the differences in these two organizations? And given those observations or beliefs, what do we then think an efficient outcome might look like? +

+ And do we think that where we are today — this prevailing status quo — is optimal? Or are there other things we can do better? And maybe an important thing to say within all of this is, to the extent that these are all kind of inevitably determined outcomes, maybe it doesn’t really matter if we think things would be better or worse. +

+ We’re going to end up in the same place, regardless. But I think for all of these, it’s super contingent. And various aspects of both funding decisions and, kind of, the precepts and methodologies of the N.I.H., how we design I.P. law, how we regulate and require and run clinical trials — there are tons of individual contingent decisions that we kind of have collectively made that give rise to the biotech and to the pharma ecosystem. +

+ And certainly, in the case of space, you know, like, it doesn’t have to be this way other. Universes, no pun intended, are possible. I think perhaps the thing that people underappreciated with science in the U.S. is, it has been very different in the not-too-distant past. +

+ Peer review is a relatively recent invention. Modern journals are a relatively recent invention. As I mentioned, the federal government being the primary funder of basic research is a relatively recent invention. +

+ And molecular biology was, in significant part, a thesis by Warren Weaver at the Rockefeller Foundation. There’s a thing here, and we should aggressively pursue it. But importantly, it was not — it required an institution, an organization, that was not part of the standard apparatus, for want of a better term. +

+ And the Broad Institute, over the last 25 years, has been enormously successful in the field of genomics and functional genomics and CRISPR, et cetera. And the Broad Institute is itself a kind of structural innovation, breaking somewhat from the more traditional prevailing university model. And so I think the fact that so many of our successes are associated with some degree of structural and institutional change should be somewhat thought-provoking for us. +

+ EZRA KLEIN: You’ve been trying to work in the space of institution-building here, too. I want to talk about Fast Grants and about Arc a little bit. So let’s begin with Fast Grants. What is it, and what has it taught you? +

+ PATRICK COLLISON: Well, it’s mostly “what was it.” In the early days of the pandemic — well, I should preface all of this by saying — well, I’ll reaffirm my preface that I don’t know, to every question. But more importantly here, I will say, my now-wife is herself a scientist. We’ve known each other since we were teenagers. We spend a lot of time talking about science in various forms. +

+ EZRA KLEIN: You met — am I allowed to say this? You met at a science competition. +

+ PATRICK COLLISON: That is true. We met at a science competition, 100 teenagers, and — +

+ EZRA KLEIN: And she beat you. +

+ PATRICK COLLISON: And yes. I was the runner-up, and she was the winner. I had created a programming language and a new dialect of lisp, and she had created a new treatment for urinary tract infections. And so I really don’t envy the judges for having to figure out what framework one should use to make all these comparisons and lots of other people. +

+ EZRA KLEIN: You sound a little bitter, man. +

+ PATRICK COLLISON: [CHUCKLES] I was gonna say, but no, we can all agree this the correct outcomes ensued. Anyway, so we were living together in March of 2020, holed up. And a number of her friends and colleagues were unsurprisingly with, I guess, a large fraction of all biology scientists, were trying to urgently repurpose their work to figure out, well, could they do something that would be somehow benefit to accelerating the end of the pandemic? +

+ And by early April, so a couple of weeks into lockdown, when it was becoming apparent and striking to us, which was it is difficult for these people to get funding for their work. And that might sound a bit, kind of, surprising, because you think, well, don’t they have some degree of money already? And couldn’t they just go and just spend that? +

+ But there are, obviously, significant rules around and restrictions around that which one can do with one’s grant money. This is money provided by the government for a purpose. And so it’s not like you can go and readily spend it on something totally unrelated. +

+ And the money is administered by the university, and so you have to go through their proper procurement processes. Point is, lots of restrictions on scientists’ pecuniary ability to suddenly repurpose the research agendas. And we kind of thought, well — we assume maybe in the early weeks, that presumably various bodies — I don’t know who — some kind of amorphous other, some combination of C.D.C., F.D.A., N.I.H., philanthropies — whatever. +

+ Somebody will come along and just give these scientists the obvious money that society clearly should, so they can go, and they can pursue these programs. Didn’t seem to be happening. I mean, to be fair, I don’t want to give us too much credit. +

+ Various people were doing things right off the bat in various different places, but we just personally knew of lots of specific examples of really good scientists who were unable to make progress of their work to the extent that they would like. So we tried to set up what we thought would be a pretty small initiative, and called Fast Grants. +

+ The basic idea would be, you send us some kind of proposal. And initially, within 48 hours, you would get a funding decision and either receive money or not. We started out with a pretty small amount of money. +

+ The initial donors — we were among them, but there were a number — contributed, best I recall, about $10 million. Launched the website early April 2020. Quickly inundated with, I think, four and a half thousand applications, which, given our promised 48-hour turnaround, was somewhat challenging. +

+ I should say this was myself. This was Silvana, my wife, and this was Tyler Cohen. So we had an immediate question as to, how do we actually run a philanthropic endeavor? And how do we stand it up in very short order? And he, through Mercatus and through Emergent Ventures, had some experience of very efficient and somewhat-scaled grant-giving. +

+ And so the three of us worked together to put it together over the course of a week or so. We proceeded over the course of, roughly speaking, the next year, slightly more, to make about 200 grants, eventually dispersing almost — or slightly over, actually — $50 million in total, to universities around the world, though primarily in the U.S. +

+ And you ask, kind of, what did we learn? A big surprise was how slowly other parts of the establishment mobilized. And various of the projects we funded or the labs we funded and so on — they’ve gone on to now do — none of them were directly implicated in the vaccine research project that ended up yielding so much fruit. So again, I don’t want to give Fast Grants too much credit. Eventually, the thing that really mattered, we had nothing to do with. +

+ But versus the projects, things like Saliva Direct, which was in the summer an early discovery that saliva tests work basically as well as the nasopharyngeal swabs we were all being subject to, or various discoveries around possible therapeutics, some of which are — still continue to go through clinical trials, and may still turn out to matter to a significant extent. And that 500 people are still dying in the U.S. per day from Covid, and — despite the existence of the vaccines and so on. +

+ So anyway, various discoveries ensued that I think will prove to be important. And the second thing we learned, which is not really related to Covid or the pandemic, but has certainly been significant for us, is — it just got us thinking more deeply and broadly about the questions of, how do scientists choose what to do? And what are the constraints they’re subject to as a practical and applied matter? +

+ And this gets back to all this discussion about both culture and institutions. And towards the end of Fast grants, we ran a survey of the grant recipients. And these are essentially all people who don’t normally — certainly don’t normally work on Covid. Covid didn’t exist. +

+ But they don’t even normally work on viruses, for the most part. These are basically kind of broadly drawn as a cross section across biology. And we just asked them, as a general matter in your regular research, if you could spend your grant money however you want, how much would you change your research agenda? +

+ So not an increase in the funding level, which tends to be what we discuss in as much as we’re discussing science policy across society. But much more specifically and narrowly, if you had complete autonomy in how you spend whatever grant money you’re getting, how much of your research agenda would change? And our intuition was that maybe a third of people would like to be doing something meaningfully different to what they actually are. +

+ But of these scientists, and these are really good scientists, four out of five told us that they would change their research agendas, quote, “a lot.” We gave them three options. Not much, or not at all, a little, and then a lot. Four out of five chose the maximum option on our survey. So I just find this incredibly thought-provoking. +

+ Basically, we seem to be in a situation where most of our top scientists aren’t doing what they think would be best for them to do. And we could say, no, our various committees and governing bodies and decision-making apparatus and so on, they know better. And I’m not saying it would be completely unreasonable for one to maintain that. But that would seem to be a very central question about the construction of our scientific apparatus. And I think that should give us some pause. +

+ EZRA KLEIN: There are a couple things there. One is that it is a consistent observation I have learning about new areas that there is a way we’re taught the thing works, or people think the thing works, and there’s this huge middle layer. Right? So in politics, which I know very well, and legislation, you have the “Schoolhouse Rock” version of how a bill becomes a law. And then it’s, like, a filibuster is how a bill becomes a law or does not become a law. +

+ We were talking about drug innovation earlier. I think the folk way people think it works is we make a discovery about a drug, and then, like, we make a drug out of it after some tests. But you talk to people who work on pharmaceuticals and just clinical trials. +

+ And in science — I think if you had asked me as a high schooler, had some science classes, I’d have told you something about the scientific method. And then you talk to a scientist, and it’s grants. Like, grants are how science works. Grants are the middle layer between — you are a scientist, and you can do some science. And grants are how the N.S.F. and the N.I.H. work. They’re how a lot of the universities work. There’s fund-raising. +

+ I mean, there are different ways that it happens. To make the question of “Are we doing science well?” a little bit more precise, I think one version of that question is, “Are we doing grants well?” And I think that question is more tractable. People don’t feel as defensive about it. +

+ But I’ve talked to a lot of scientists in the course of my work. I’ve covered health care for my entire career. And I don’t know any who think we’re doing grants well. I don’t know any who will not complain to you for hours. +

+ And they may be wrong. And I do want to note — because they also just have somewhat different incentives. I mean, I was noting earlier, and I think it’s very real. The government, particularly when it gives out grants, needs to worry about the reputational cost of the grant. If the grant goes wrong, if not enough of the grants pay out into useful research. If Rand Paul can stand up in Senate and make what you did sounds silly, these things really end up mattering. And so you get a process that is optimizing for a lot of different things. +

+ But the question of whether or not we do grants well ends up being really, really, really important in every country that does major capital science that I know of, and is just not the main question for a bunch of different reasons we ask. +

+ PATRICK COLLISON: Yeah. Another question we asked in our survey was how much time they spend on the grants. I think to some extent, this is perhaps — at least, of those who’ve spent some amount of time interacting with scientists, kind of more broadly known than perhaps the finding with respect to how they do — or the degree to which they can choose what they work on. But we found that — or they reported to us that they spend on the order of 40 percent of their time on grant administration. +

+ And even if one were to maintain that the decision-making apparatus around what scientists do is somehow efficient, I think it is a very tenuous position to also try to argue that 40 percent of the best scientist’s time is optimally allocated towards grant applications, authorship and administration. And we’re not talking about an inconsequential 40 percent here. +

+ I mean, this is 40 percent of the time of this super-elite 10,000, 100,000, whatever it is, some relatively finite number of people. And we’ve chosen to take and to redeploy almost half of their time in service of technocratic, bureaucratic undertaking. And getting back again to this point about people perhaps falsely assuming that things have been more inter-temporally consistent than they have, that percentage has increased very substantially over the last couple of decades as the overall edifice of science has grown, and as the kind of acceptance rates and the various thresholds for various grants has become more exacting. +

+ EZRA KLEIN: How we allocate people’s time is really important. But also, just how we allocate talent is really important. And it brings me to something you said that I wanted to ask you about. This was in response to a question about whether big tech companies are hogging all the talent in society. +

+ And you said, quote, “I don’t think that the ambitious upstarts who go into high speed rail in America, anyway, are going to have a great time or have much success in convincing their friends to follow them. And I suspect that for various reasons, too many domains look somewhat like high speed rail.” And so you go on to say that there’s a view that the internet is a frontier of last resort, and that you don’t think that’s totally wrong. +

+ So tell me about that. Tell me about the idea of the internet as a frontier of last resort. But behind that, this idea that other frontiers where talented people might want to go and make their mark on society have closed. +

+ PATRICK COLLISON: You’re familiar with and you’ve probably written about the Stephen Teles idea of kludgeocracy. And I kind of like the term “kludgeocracy,” because rather than making some of the inhibitions that people might encounter in pursuing something like high speed rail, rather than casting those as being deliberate, the valence is more that it’s this kind of emergent, inadvertent and kind of complicated phenomena that nobody perhaps particularly wants or chose. +

+ And I think the case of California’s high speed rail is quite striking, where — you’ve written about this and kind of similar projects and the New York subway expansion and so on. And congestion pricing and so on. But it’s striking where it’s not actually obviously a question of first order political will. +

+ Like, we’re willing to fund the high speed rail in California. We’re clearly willing to invest in building the subway expansion in New York. But somehow, somewhere between that first order decision and desire and our actual ability to kind of instantiate it, something really goes wrong. +

+ And you contrast that with stories of — in the case of, say, California, Henry Kaiser and these various other early part of the 20th century operators in the physical realm. And for a variety of reasons, but mostly prosaic state and county-level complications and things that would extend the time horizon of one’s project, it has simply become meaningfully less-appealing for those people to undertake these initiatives. I mean, Foster City, not too far from where we are now, that’s named after the eponymous Mr. Foster. +

+ He was a developer. He decided, well, with reclaimed wetlands, I’m going to build a city. California is growing quickly. The Bay Area is a — kind of propitious and will be a long-term successful area. And I think this place simply needs more housing. +

+ And he, with that kind of founder energy, was able to give birth and rise to the city that now bears his name. I haven’t met anybody pitching me on a similar city on the shores of the Bay in the last couple of years. But I would imagine that were one to adopt that ambition today and to propose that maybe the San Jose Marsh wetlands should themselves be an expansion of San Jose, I don’t think one would get very far. +

+ And in fact, even for much more sort of limited things, like additional runways or runway expansions at S.F.O., even they have now been stymied for decades at this point. That ability to translate that into something enunciated has dissipated and deteriorated. And then I think the kind of individual version is, and if I want to be that heroic solar farm entrepreneur or railway magnate, that my practical ability to do so has been meaningfully curtailed. +

+ EZRA KLEIN: Yeah. I mean, that’s what I’m getting at here a little bit, which is talent really matters for a society. Where the most talented people go really matters for society. And a lot of those people want to go somewhere where they can have a really big effect. +

+ I think there’s been a huge rush to digital land because you can build on digital land. You can build quickly. I mean, it’s interesting to some of the dynamics we’re talking about, the temporal dynamics we’re talking about, that you see this dynamic even within the tech world. +

+ There was a while where it was really exciting to go join Facebook, go join Google, go join one of the big companies. Because you could do so much. And that’s still, to some degree, true. But they got really big. +

+ And so crypto got — whatever you think of crypto, one thing that is exciting about it to people is the idea that it’s open land. That you can go in there and have a really big effect on it. Build something new just with a couple of friends that might change the whole direction of the field. +

+ And on some level, it’s always going to be harder for, say, putting high speed rail through the middle of California. Right? I mean, just building things in the world is just going to be tougher. +

+ But on the other hand, if you make building things in the world too hard, if you make grants too difficult — if you — I know a lot of doctors who their advice to young people is don’t become a doctor. And it always breaks my heart a little bit. We need really great people to be doctors. +

+ And their point is not, don’t go heal sick people. Their point is, being a doctor is too hard now. The amount of time you spend dealing with insurance agencies and malpractice insurance and boards, and this and that, it’s just too much administration. +

+ When industries become very complicated to operate in, you want to select for people who are good at operating complicated industries, which may be different than the people who are good at moving really fast and changing things dramatically. But two, you kind of subtly bias where different kinds of people in your society go. I think in China, if you want to change a lot, you still probably go into infrastructure construction, among other things. Right? +

+ The idea that you might be a genius rail mind, in China, that’s great. There’s probably a lot of rail you can make. That’s not true here. And the point is not to make too much of the rail example, but to make a lot of the idea that talent flows towards where it can have an effect and people can live the kinds of heroic lives they want to lead. And if we have subtly pushed a lot of people into maybe not the right — not the socially optimal directions, that over time will have a pretty big effect on a society. +

+ PATRICK COLLISON: I think a constant is that some number of ambitious young people will want to do something, as you say, heroic. And yeah, I think maybe two things have changed. For one, for whatever reason, our predisposition to putting those people in positions of authority has diminished. And then secondly, even if placed, their ability to actually execute, again for various reasons, has been attenuated. +

+ I’ve been reading about the university founders and presidents and those associated with some of the great US research institutions. And one thing that is striking is how many of them were so young when placed in those positions of authority. You know, Daniel Coit Gilman at Johns Hopkins, or William Rainey Harper at the University of Chicago. I think he was 32 when he was appointed president of the University of Chicago. Even in the recent past. +

+ So my dad was in the first year of the University of Limerick in Ireland. Or at the time, it was called N.I.H.E. It kind of acquired university status later in its life. And the Irish guy who founded it and was really the dynamo behind it, I think he was 29 when he was put in charge of that project. +

+ And I think it was in 1970 or ’71 that he was charged with this mission. But as recently as 1970 in Ireland, we were willing to put a 29-year-old — I mean, that’s a person meaningfully younger than me in charge of the project of overseeing the creation of a major new research institution. And I don’t know that I have compelling or confident observations to offer in terms of the etiology underlying these changes. But I think the changes themselves are important, or at least we should assume they’re important if we come from a place of humility, where this is what has worked in the past. Enabling these ambitious young people who are willing to contemplate spending multiple decades in pursuit of some ambitious and idiosyncratic vision. +

+ And maybe we’re more enlightened now. Maybe we figured out how to get all the same innovation and all the same breakthroughs without unleashing that force. But I guess my starting point, at least, would be, well, we should — before getting super confident in that or before really being deliberate about it, I think we should give some kind of credit and credence to the prescription and the methodology that’s worked heretofore. +

+ EZRA KLEIN: And before books, let me end on this. We’ve talked a lot about scientific slowdown, about technological slowdown. But let’s say in the next 15-year time frame, what are the three technological or scientific possibilities you’re most excited by? If in 20 — I guess it’d be 2037, we’re having a conversation about how dumb this conversation was because it was right on the cusp of so much incredible stuff happening, what do you think is likely to be on that list? +

+ PATRICK COLLISON: I don’t know that I’ve super non-consensus answers. I think that there are fundamental a priori reasons to believe that the rate of progress in biology could increase substantially over the years, and to your question, kind of decades to come. So if in 2037 we are enormously impressed and struck by the discontinuity there, that would not shock me. +

+ Clearly, over the past couple of years, there’s been acceleration in progress in A.I. And kind of far for me to try to point estimate for kind of where that is in 2037. But I would be surprised if that is not somewhere on that list. +

+ And then I think there’s something about education in the broadest sense that feels to me like a very significant, and hopefully very positive change happening in the world right now. Maybe best embodied by YouTube. But also by Twitter and by blogs and Substacks and even Zoom and kind of the growing ease of being in some kind of cultural proximity to people one aspires to emulating, or following in the footsteps of, or otherwise kind of being more like. +

+ And to the extent that one believes my story about the significance of sociology, and culture, and mentorship, and the kind of delicate transmission of tacit knowledge, it has until very recently only been possible for that to happen to a meaningful extent through physical co-location. And the fact that we’ve now thrown open those doors to such an extent feels to me like a really compelling and plausibly transformative change. And if it were the case in 2037 that we have multiplied by 20 the number of people who can — who have the initial mental models and understanding to become successful entrepreneurs, or successful scientists, or successful writers, or successful in whatever one might choose one’s domain to be, again, I think that would not be shocking. And I think it’s a pretty hopeful fact about the world. +

+ EZRA KLEIN: And then always our final question. What are the three books you’d recommend to the audience? +

+ PATRICK COLLISON: Well, I’m right now reading “Revolution and Empire,” which is a book about Edmund Burke. And it is just fabulous. I very highly recommend it. +

+ Edmund Burke, Ireland’s foremost political philosopher. And I’m embarrassed to say that I have known less about him than I feel like I ought to have. So I recommend that very highly. +

+ And the autobiography by Warren Weaver, who I mentioned, at Rockefeller. I can’t remember if it’s called “Scene of Change” or “Scene of the Action.” But it’s Warren Weaver’s autobiography. +

+ That’s not a great book in the sense that you don’t read it — you don’t find it to be a vivid, compelling page-turner. And your mind is not blown on every page. But I find myself thinking back to it quite a lot and having various parts of it sort of ricochet to my mind. +

+ And then it all depends on what people are interested in and all the rest. And you should read the things you like. But I have on my desk at home right now “A Widening Sphere,” which is a history of M.I.T. And I was re-reading it recently. And my — +

+ EZRA KLEIN: Who doesn’t re-read the histories of M.I.T.? +

+ PATRICK COLLISON: [LAUGHS] Well, William Barton Rogers, the founder, was the son of an Irishman, and started M.I.T. substantially with his brother. And I find it very inspiring, I guess back to what we were saying earlier, how motivated he was and they were by a kind of broad-based desire for societal betterment. Like, M.I.T. didn’t inadvertently end up being a significant contribution to American prosperity and ingenuity and welfare. He was really immersed in that milieu. +

+ He paid a lot of attention to some of the cultural dynamics we were describing in England, and the Darwins. And the early writing on M.I.T., if you go and just read the first two pages of the founding manifesto, it wasn’t utopian in some kind of implausibly lofty sense. But it was somebody who knew they weren’t founding a run of the mill nth technical college. +

+ And I feel like it’s easy to get cynical always. It’s easy to assume that the things that really worked out worked out through happenstance, as opposed to optimism and ambition. But yeah, I find the history of MIT to be a kind of inspiring reminder that sometimes these implausible, lofty, ambitious, long-term initiatives can work out much better than one would hope. +

+ [MUSIC PLAYING] +

+ EZRA KLEIN: Patrick Collison, thank you very much. +

+ PATRICK COLLISON: Thanks for having me. +

+ [MUSIC PLAYING] +

+ EZRA KLEIN: “The Ezra Klein Show” is produced by Annie Galvin and Rogé Karma. Fact-checking by Michelle Harris, Mary Marge Locker and Kate Sinclair. Original music by Isaac Jones. +

+ Mixing by Sonia Herrero, Isaac Jones and Carole Sabouraud. Audience strategy by Shannon Busta. Special thanks to Kristin Lin and Kristina Samulewski. +

+ [MUSIC PLAYING] +

\ No newline at end of file diff --git a/packages/readabilityjs/test/test-pages/nytimes.com/distiller.html b/packages/readabilityjs/test/test-pages/nytimes.com/distiller.html new file mode 100644 index 000000000..2078c23a1 --- /dev/null +++ b/packages/readabilityjs/test/test-pages/nytimes.com/distiller.html @@ -0,0 +1,216 @@ +

+ The social media company went public in 2013. But Elon + Musk is taking it private as part of his acquisition of + the firm. Here’s what that means. +

+ + + Elon Musk plans to change Twitter from a public to a privately held company.
Credit... Jason Henry for The New York Times
+ +

+ Elon Musk closes his purchase of Twitter and fires + several top executives. Follow for the latest news updates. +

+ SAN FRANCISCO — In August 2018, Elon Musk called + Michael Dell + for advice. +

+ Mr. Musk, who was + trying to take Tesla, his electric car company, + private, quizzed Mr. Dell, who had done that very thing with + his eponymous computer company in 2013, about the process and the best lawyers to use for + the complicated transaction. +

+ “I was really asking him about — did he find being + private was good?” Mr. Musk recalled in a + deposition + about the Tesla effort. “Did he regret going private?” +

+ Mr. Dell warned Mr. Musk that it was “a very difficult + process” but that he was glad to have done it, + according to court filings. +

+ Now, Mr. Musk, who did not end up taking Tesla + private, is doing so with Twitter. As part of his $44 + billion acquisition of the social media service, which + closed on Thursday, he is delisting the company’s + stock and taking it out of the hands of public + shareholders. +

+ Making Twitter a private company gives Mr. Musk some + advantages. Unlike publicly traded companies, + privately held firms do not have to make quarterly + public disclosures about their performance. They are + also subject to less regulatory scrutiny and can be + more tightly controlled by an owner. That means Mr. + Musk can make over Twitter — including tweaking the + platform’s content rules, its finances and its + priorities — without having to consider the worries of + the investing public. +

+ “It’s hard to run a public company if you think you + should be the one running it and you’re not open to + other views from people, like stockholders,” said + Brian J.M. Quinn, a professor at Boston College Law + School. “By taking it private, you feel like you have + a lot more flexibility.” +

+ Here’s how Twitter is set to change as a private + company under Mr. Musk. +

+ How is Mr. Musk taking Twitter private? +

+ As part of buying Twitter, Mr. Musk is merging the + social media company with + X Holdings, a corporate entity that he established in Delaware + to handle the deal. X is buying out all of Twitter’s + stock and will control the service, and Mr. Musk will + control the holding company. +

+ Image +
+ Credit...Lucas Jackson/Reuters +

+ What happens to Twitter’s stock? +

+ Twitter will be delisted from the New York Stock + Exchange and its shares will no longer trade on public + markets as of Nov. 8, according to + a securities filing. In September, Twitter’s shareholders approved the + company’s sale to Mr. Musk and agreed to sell their + stock to him for $54.20 a share. Investors will be + able to claim the cash value of their shares. +

+ What happens to Twitter’s board of directors? +

+ With the deal’s completion, + Twitter’s board of directors + will dissolve and its nine members will no longer + preside over the company’s operations. Mr. Musk will + most likely appoint a new board made up of friends and + investors who helped fund the acquisition. The new + board will be responsible for plotting Twitter’s + trajectory as a private company. +

+ “It will still be required by law to have a board of + directors, and that would probably include Elon Musk + and some of the other big equity investors in the + company,” said Eric Talley, a professor at Columbia + Law School. “I expect Mr. Musk will run it as a + somewhat friendly dictatorship.” +

+ What happens to Twitter’s top executives? +

+ Mr. Musk has already started cleaning house, with + several of Twitter’s top executives + getting fired on Thursday. +

+ The executives who were fired include + Parag Agrawal, Twitter’s chief executive, who has clashed publicly + and privately with Mr. Musk. When Mr. Musk complained + this year that Twitter had an unchecked spam problem, + Mr. Agrawal tweeted to rebut his claims. Mr. Musk + responded with a poop emoji. +

+ At another point, Mr. Agrawal texted Mr. Musk, telling + the billionaire that his criticisms were harming + Twitter, according to a + court filing. +

+ “This is a waste of time,” Mr. Musk retorted. +

+ Other executives who were fired include Ned Segal, + Twitter’s chief financial officer; Vijaya Gadde, the + top legal and policy executive; and Sean Edgett, the + general counsel. +

+ Under the merger agreement, Mr. Agrawal was + potentially set to receive a golden parachute worth + about $60 million, with Mr. Segal to receive $46 + million, while Ms. Gadde would receive about $20 + million. It was not immediately clear whether Mr. Musk + intended to make the payments. +

+ Image +
+ Credit...Kevin Dietsch/Getty Images +

+ What about Twitter’s employees? +

+ Twitter has about 7,500 employees. Some of them + have been jittery + for months about the company’s sale to Mr. Musk. Many + could face layoffs or job changes as their new owner + takes over. +

+ Their compensation is also set to change. Employees + typically receive stock options in the company. But, + with the delisting of Twitter’s stock, employees are + set to be cashed out for shares they already have and + to be paid with cash bonuses going forward, instead of + the stock options they were scheduled to receive, + according to the merger agreement. Some + employees have worried + that Mr. Musk may not honor the agreement. +

+ “Most of these employees have been in a public company + and are used to public option grants, which are + liquid,” Mr. Quinn said. “They will have to come up + with some other Silicon Valley-friendly method of + keeping people around.” +

+ What financial pressures will Twitter face as a + private company? +

+ By going private, Twitter will avoid some public + scrutiny since it will no longer be required to make + quarterly disclosures about the health of its + business. This will give Mr. Musk some flexibility as + he changes Twitter. +

+ But he will face pressure from the banks that lent him + $12.5 billion for the deal to begin repaying his debt. + The cost of repaying those loans could run as high as + $1 billion a year, financial analysts said. +

+ “He has less public pressure, but he has a lot of + private pressure from the banks to make the payments,” + Mr. Quinn said of Mr. Musk. “Like almost every other + private-equity take-private, he’s going to need a + manager who is very focused on operations, being lean + and being able to pay the bills on a day-to-day + basis.” +

+ Mr. Musk also took about $7.1 billion from equity + investors to push the deal through. He may also face + pressure from those investors, who might expect him to + take Twitter public again at some point so that they + can recoup their investment. +

+ In some take-private deals, owners have opted to sell + branches of their companies to pay their debts. Mr. + Musk could choose to do the same at Twitter. +

+ “It’s conceivable that some aspects of Twitter could + potentially be carved off, sold off or spun off to + raise money that could go toward paying the debt,” Mr. + Talley said. “Twitter kind of is pared down to its + core mission right now. They would have to get a + little creative.” +

Advertisement

\ No newline at end of file diff --git a/packages/readabilityjs/test/test-pages/nytimes/distiller.html b/packages/readabilityjs/test/test-pages/nytimes/distiller.html new file mode 100644 index 000000000..9ec163047 --- /dev/null +++ b/packages/readabilityjs/test/test-pages/nytimes/distiller.html @@ -0,0 +1,75 @@ +

+ Critics see the recent behavior of Brazil’s president — polarizing in the best of times — as an unnerving sign of a flailing leader. His strategy, if there is one, is difficult to discern. +

President Jair Bolsonaro of Brazil during a news conference on Wednesday.
Credit... Joedson Alves/EPA, via Shutterstock
+ +

+ RIO DE JANEIRO — He has sneered at the Covid-19 pandemic, even as it led Brazil’s health care system to collapse. He has ridiculed opposition lawmakers, who are gunning for his impeachment. +

+ His main rival is back in the political arena, threatening a re-election bid. +

+ And this week he ordered a sweeping cabinet shake-up and removed the heads of the armed forces — a strong base of support — with no public explanation. +

+ Even for a polarizing leader who often appears to act on gut instinct, the recent moves by President Jair Bolsonaro of Brazil have confounded and unnerved many in Latin America’s largest country, where the coronavirus is killing people at a record rate. +

+ Brazilian lawmakers on Wednesday presented a new initiative to impeach Mr. Bolsonaro, calling his dismissal of the military commanders the day before a dangerous and destabilizing action. +

+ The forcing out of the military commanders, which followed the replacement of roughly one-third of his cabinet, created consternation and bewilderment in political circles. There were no clear signs that the personnel changes represent a strategic shift for the government as it navigates the deadliest phase of the Covid-19 pandemic, which has killed more than 317,000 Brazilians. +

  • Thanks for reading The Times.

+ With hospitals overloaded, a slow vaccination campaign and growing unemployment, Mr. Bolsonaro is under enormous pressure to make bold policy changes. But politicians and analysts said they were struggling to make sense of his latest moves. +

People over 71 and their relatives standing in a line for vaccinations on Wednesday in Rio de Janiero.
Credit... Dado Galdieri for The New York Times

+ “I don’t see a strategic plan behind all this,” said Senator Kátia Abreu, a powerful lawmaker who heads the foreign affairs committee. “I see a lot of improvisation, impulsive actions that generate a crisis out of the blue.” +

+ Mr. Bolsonaro is facing an array of overlapping challenges as he struggles to shore up his electoral base ahead of a campaign for re-election next year. +

+ Unemployment rose to 14 percent in recent months, from the 11.6 percent jobless rate Mr. Bolsonaro inherited when he took office in January 2019. A formidable political adversary, former President Luiz Inácio Lula da Silva, re-emerged on the political stage this month after the courts vacated corruption cases against him, which restored his right to run for office. +

+ Your Coronavirus Tracker: We’ll send you the latest data for places you care about each day. +

+ The president confronts daunting economic obstacles. While he was able to retain considerable political support last year by spending billions on a pandemic welfare program, continuing to keep the politically popular assistance aid flowing, while meeting other fiscal obligations, would burst spending caps codified by law. +

+ At the same time, Brazil’s Covid-19 crisis has turned the country into an international pariah. Experts fear that the spread of the country’s more-contagious strain will accelerate across the globe, and that as transmission grows, new variants may yet emerge. +

Health care workers intubating a Covid-19 patient this month at a hospital in Porto Alegre, Brazil.
Credit... Mauricio Lima for The New York Times

+ Maurício Santoro, a political science professor at Rio de Janeiro State University, said Mr. Bolsonaro’s recent moves appear to reflect a desire to surround himself with loyal and subservient officials. But he said the fealty of Mr. Bolsonaro’s new subordinates does not appear to strengthen his hand in Congress or expand his base ahead of next year’s presidential election. +

+ “There is no political coherence,” Mr. Santoro said. “This is not a shift toward moderation, or an attempt to build bridges” in Congress, he added. +

+ On Wednesday, a day after Brazil recorded 3,780 daily deaths, a record, Mr. Bolsonaro renewed his attacks on lockdowns and other rigid measures that health experts have said are necessary to arrest the spread of the virus. +

+ “We’re not going to fix this problem by staying at home,” Mr. Bolsonaro said. “No nation can sustain itself for long with that kind of policy.” +

+ Mr. Bolsonaro offered no new information about the events that led him to dismiss the commanders of the Army, the Air Force and the Navy on Tuesday. The three stepped down abruptly a day after the president fired his defense secretary, a retired general. +

+ In their latest effort to start impeachment proceedings against Mr. Bolsonaro, a handful of lawmakers on Wednesday accused the president of having attempted to turn the armed forces into a political instrument. But the circumstances that led to the removal of the three men remained unclear, with neither the president nor the commanders offering an explanation. +

Supporters of Mr. Bolsonaro on Wednesday in Rio de Janiero.
Credit... Silvia Izquierdo/Associated Press

+ Lawmakers and analysts speculated that the dismissals followed a profound disagreement over the role of the armed forces in Mr. Bolsonaro’s government. The president has recently vowed that “my army” would not be deployed to enforce quarantines or lockdowns. +

+ In his resignation letter, the outgoing defense minister, Fernando Azevedo e Silva, said he had preserved the armed forces as “institutions of the state.” The remark was interpreted to mean that he had resisted efforts to politicize the military — a sensitive issue in a country that was governed by repressive military governments for more than two decades following the 1964 coup. +

+ “The question that looms large, and must be answered, is: What was the order given to the generals that they did not feel they could follow?” said Ms. Abreu, the senator, who was not a proponent of the latest impeachment initiative. “The president has an obligation to explain this on live television to calm people down.” +

+ In a sign of how deeply the pushing out of the military chiefs shook the political establishment, six presumptive presidential candidates for 2022 issued a joint statement warning that three decades after the end of the dictatorship, “Brazil’s democracy is threatened.” +

+ The statement added: “There is no shortage of examples of the way authoritarianism can emerge from the shadows if societies are careless and fail to speak up in defense of democratic values.” +

+ The new defense minister, Walter Souza Braga Netto, a former Army general who left active duty last year, rattled critics of the government by issuing a statement about the anniversary of the 1964 coup — the anniversary was Wednesday — saying the date should be “celebrated.” +

+ But later in the day, he called today’s armed forces a bedrock of Brazil’s democracy. “On this historic day, I reaffirm that the most valuable asset of a nation is the preservation of democracy and the freedom of its people,” he said during a ceremony in which the three new chiefs of the armed forces were announced. +

+ Lawmakers have called for Mr. Bolsonaro’s impeachment dozens of times since last year, but they have failed to garner broad support. Arthur Lira, the new leader of the House of Representatives, last week put the president on notice as he decried the government’s handling of the pandemic. +

+ Mr. Lira warned that the “political remedies in Congress are well known and all of them are bitter.” Some, he added in a clear reference to impeachment, are “fatal.” +

+ A group of lawmakers last week warned Mr. Bolsonaro in a letter that the 2021 budget, as currently drafted, would exceed fiscal limits established in 2016 to rein in public spending and attract foreign investment. Exceeding the cap, which economists say appears all but inevitable now, would open a new avenue for the president’s impeachment. +

+ “We’re on a path of fiscal irresponsibility and that creates a serious legal problem,” said Zeina Latif, an economist. +

+ But analysts and leading lawmakers say there is little appetite for a new impeachment, given how the ouster of President Dilma Rousseff in 2016 proved so politically disruptive and divisive. +

+ Monica de Bolle, a senior fellow at the Peterson Institute for International Economics, said powerful blocks of lawmakers are likely to use Mr. Bolsonaro’s isolation to extract concessions. +

+ “It’s in their best interest to leave him in office and get the things they always wanted,” she said. She predicted the president’s popularity would fall ahead of next year’s presidential election. “Next year they wash their hands of Bolsonaro.” +

+ Ernesto Londoño reported from Rio de Janeiro, and Letícia Casado from Brasília. +

+ Advertisement +

\ No newline at end of file diff --git a/packages/readabilityjs/test/test-pages/omnivore_getting_started/distiller.html b/packages/readabilityjs/test/test-pages/omnivore_getting_started/distiller.html new file mode 100644 index 000000000..051c576e5 --- /dev/null +++ b/packages/readabilityjs/test/test-pages/omnivore_getting_started/distiller.html @@ -0,0 +1,158 @@ +
Omnivore
+ Updates from the Omnivore team +
+ Let me read it first +
+ Already a subscriber? Sign in +

+ This guide will show you how to use Omnivore’s basic functions and advanced features, divided into four main activities: +

  • + Saving +

  • + Reading +

  • + Organizing +

  • + Integrations +

+ The Library is the center of your Omnivore experience, where you can quickly access any links you have saved. Saved links remain in your Library forever unless you delete them. +

+ There are five ways to save links to pages or articles that you wish to read later: +

  • + Saving from Your Omnivore Library +

  • + Saving from a Browser  +

  • + Saving from a Phone or Tablet (iOS or Android) +

  • + Newsletter Subscriptions via Email +

  • + Saving PDFs from a Mac
    +

+ 1. In the upper right corner of your Library, tap the Add Link button.
+ 2. Enter the URL you wish to save and tap Add Link.
+ 3. The link will appear in your Library the next time you refresh it.
+

+ 1. Download and install the Omnivore extension for your browser: +

+ Once the mobile app is installed: +

  1. + In your browser, navigate to the page you wish to save and tap the Share button. +

  2. + Tap the Omnivore icon in the Share menu. +

  3. + The link will appear in your Library the next time you refresh it. +

+ 1. On the Omnivore website or app, tap your photo, initial, or avatar in the top right corner to access the profile menu. Select Emails from the menu. +

+ 2. Tap Create a New Email Address to add a new email address (ex: username-123abc@inbox.omnivore.app) to the list. +

+ 3. Click the Copy icon next to the email address. +

+ 4. Navigate to the signup page for the newsletter you wish to subscribe to.
+ 5. Paste the Omnivore email address into the signup form. +

+ 6. New newsletters will be automatically delivered to your Omnivore inbox. +

  1. + Install the Mac App.  +

  2. + On your Mac, locate the PDF you wish to save and right-click or ctrl+click on the file name. +

  3. + Select Share from the menu and choose Omnivore. +

  4. + The link will appear in your Library the next time you refresh it. +

+ Click any link saved in your Library to enter the Reader view.  +

+ Omnivore formats pages for easy reading and highlighting, removing ads and clutter for distraction-free reading. The text-focused view also makes articles smaller and quicker to load. +

+ While reading, you can: +

  • + Change Formatting +

  • + Highlight Text +

  • + Add Notes +

  • + View All Saved Highlights and Notes +

  • + Track Reading Progress +

  1. + Theme: Tap your photo, initial, or avatar  in the top right corner to access the profile menu. Select the white or black thumbnail to choose the Light or Dark theme. +

  2. + Text Formatting: Tap the Aa icon to adjust the text size, font, margins, and line spacing. +

  1. + Select the text you wish to highlight. +

  2. + Tap the Highlight button. +

  3. + The text will appear highlighted next time you view the article. +

  1. + Highlight a section of text where you wish to add a note. +

  2. + Tap the Note button, type your note, and tap Save. +

  3. + The Note icon will appear next time you view this article. +

  1. + Tap the Highlight/Note icon to see a list of all the highlighted text and notes you have added to this page. +

  2. + To remove a note or highlight, select it from the list and tap the Trash icon. +

+ Omnivore automatically keeps track of your reading progress across your different devices so you can easily pick up where you left off. A progress bar will appear at the top of each link in your Library after you have started reading. +

+ By default, the Library inbox displays all links you have saved. To manage your list and keep your reading organized, Omnivore provides the following actions:  +

  • + Archiving +

  • + Labels +

  • + Search +

  • + Filters +

  1. + Tap the Menu icon next to the link you wish to archive (on the mobile app, long press the link to open the menu). +

  2. + Select Archive. +

  3. + The link will disappear from the default Library view, but will show up if you select the Archived filter (see Filters below). +

  1. + Tap the Menu icon next to any link and select Set Labels. +

  2. + Select an existing label from the list or tap Edit Labels to create a new one. +

  3. + The label will appear next to the link in your Library. Tap it to view all links with the same label. +

  4. + Omnivore mobile app only: tap Labels to see a complete list of all labels you have used; tap one to view all links with the same label +

  5. + Note: Omnivore will automatically assign some labels, such as “Newsletters.” +

  1. + To search through all your saved links, enter a keyword or phrase in the search bar.  +

  2. + You can combine keywords with labels and filters to focus your search even further. Learn more about advanced search. +

  1. + Use the Filters menu to refine your Library view (some filters may be visible by default). +

  2. + Select Read Later to view a list of all your non-archived links except Newsletters. +

  3. + Select Highlights to view the text selections you have highlighted in all your saved pages.  +

  4. + Select Today to view a list of links you saved today. +

  5. + Select Newsletters to view links saved via your newsletter subscriptions. +

+ Omnivore allows integrations with knowledge bases and note-taking apps including: +

  • + Logseq +

  • + Webhooks +

+ With Omnivore's Logseq plugin you can sync all your saved articles, highlights, and notes into Logseq, a popular knowledge base. For information on setting up and using the Logseq plugin, please refer to this helpful Omnivore for Logseq Plugin Guide. +

+ Omnivore can trigger webhooks when you save a link or add highlights to a page you are reading. This example shows webhooks being used to write all saved links to a Google Sheets spreadsheet stored on a Google Drive. +

+ Subscribe to Omnivore +

+ Updates from the Omnivore team +

+ TopNew +
\ No newline at end of file diff --git a/packages/readabilityjs/test/test-pages/ottawacitizen.com/distiller.html b/packages/readabilityjs/test/test-pages/ottawacitizen.com/distiller.html new file mode 100644 index 000000000..26eaa8b5d --- /dev/null +++ b/packages/readabilityjs/test/test-pages/ottawacitizen.com/distiller.html @@ -0,0 +1,55 @@ +
The façade of the headquarters of the Department of National Defence in Ottawa.
The façade of the headquarters of the Department of National Defence in Ottawa. Photo by Adrian Wyld / THE CANADIAN PRESS
+ Upcoming all-electric Jeep might be called ‘Jeepster’ Back to video +
+ Upcoming all-electric Jeep might be called ‘Jeepster’ +
+ X +

+ Neither the Department of National Defence nor the office of Defence Minister Anita Anand would comment on the report published Saturday that noted a few dozen commandos from NATO countries, including Britain, France, Canada and Lithuania, had been working inside Ukraine. The United States withdrew its own 150 military instructors before the war began in February. +

+ But the New York Times, citing three U.S. officials, reported that special forces from the NATO countries either remained or had gone in and out of Ukraine since then, training and advising Ukrainian troops and providing an on-the-ground conduit for weapons and other aid. +

+ U.S. special forces have also established a coalition planning cell in Germany to co-ordinate military assistance to Ukrainian commandos and other Ukrainian troops, the New York Times reported. The cell, which has grown to 20 nations, is modelled after a similar structure the U.S. and its NATO allies used in Afghanistan. +

+ Advertisement 3 +

+ Story continues below +

+ Article content +

+ U.S. Army Secretary Christine E. Wormuth has confirmed the cell provides intelligence and co-ordination for the flow of NATO weapons into Ukraine, allowing such convoys to avoid Russian attacks. +

+ Both CTV and Global News reported in late January that Canadian special forces had been sent to Ukraine, but National Defence did not comment on that deployment. +

+ The Liberal government is strongly supportive of Ukraine. It is still in discussions with South Korea to purchase 100,000 artillery shells for Ukraine in a deal that could cost Canadian taxpayers several hundred million dollars. +

+ The Liberal government already spent $98 million to buy 20,000 similar artillery rounds from the U.S. for Ukraine. In addition, it has provided Ukraine with Canadian Forces M777 artillery guns that can use the 155-millimetre ammunition. +

+ Advertisement 4 +

+ Story continues below +

+ Article content +

+ Besides artillery, Canada has shipped Carl Gustaf anti-tank systems, grenades and drone cameras to Ukraine. In February, it also sent a shipment that included .50-calibre sniper rifles equipped with silencers, 60-millimetre mortars, grenade launchers, pistols, ammunition, thermal-imaging binoculars, cameras, scopes and medical supplies. The equipment was enough to equip a force of between 500 and 600 personnel. The shipment also included C6 and C9 machine guns, which are used by the Canadian Forces. +

+ NATO nations have been sending large amounts of weapons to Ukraine since the Feb. 24 invasion of that country by Russia. Some NATO officials view the war as an opportunity to either force regime change in Russia or to seriously weaken that country militarily. +

+ Advertisement 5 +

+ Story continues below +

+ Article content +

+ On May 9, Lithuanian Foreign Minister Gabrielius Landsbergis told the Associated Press that removing Russian President Vladimir Putin from power was the only way to protect the West and its allies from future threats. Landsbergis noted that not only Putin must be removed, but also all members of his inner circle. +

+ U.S. defence secretary Lloyd Austin, a retired four-star U.S. general, said earlier this year that, with enough equipment, Ukraine could defeat Russia. Austin told reporters the U.S. wanted to see “Russia weakened to the degree that it can’t do the kinds of things that it has done in invading Ukraine.” +

+ In addition, NATO’s push to send billions of dollars in weapons to Ukraine has meant a windfall for mainly American defence firms as the U.S., Canada, and other nations will eventually have to replenish their stocks. New orders have already been placed by the U.S. Pentagon for missiles and other equipment. The Pentagon has estimated it will eventually provide Ukraine with $20 billion in military aid. +

+ Canadian Chief of the Defence Staff Gen. Wayne Eyre has called on defence companies to go to a war-time production footing so more weapons can be acquired. +

+ Share this article in your social network +

+ Latest National Stories +

\ No newline at end of file diff --git a/packages/readabilityjs/test/test-pages/people/distiller.html b/packages/readabilityjs/test/test-pages/people/distiller.html new file mode 100644 index 000000000..e5b7b51e1 --- /dev/null +++ b/packages/readabilityjs/test/test-pages/people/distiller.html @@ -0,0 +1,35 @@ +

+ The unidentified actors took photos with roughly 50 children at a holiday celebration in Ludowici, Georgia, according to officials +

+ By +
+ December 14, 2020 11:31 PM +
+ Santa Claus +
+ | Credit: getty images +

+ Two people who recently played Santa Claus and Mrs. Claus at a holiday celebration in Georgia have tested positive for COVID-19, possibly exposing dozens to the novel coronavirus. +

+ The unidentified pair were diagnosed with coronavirus on Saturday, two days after appearing in a Christmas parade in Ludowici and taking photos with roughly 50 children following a tree-lighting ceremony, according to officials. +

+ Robert Parker, the chairman of the Long County Board of Commissioners, said in a statement on its website that the duo "were not displaying any symptoms at the time of the event." +

+ "While this event was not put on by the City of Ludowici or the Long County Board of Commissioners, it was well attended by our public officials and I believe I speak for the majority of them in saying that we still stand by the decision of the Chamber to move forward with these holiday traditions and to bring some sense of normalcy to these trying times," he said. +

+ Parker added that his own kids took photos with Santa and said "no children were forced to attend the events and that choice was left solely in the hands of their parent." +

+ "I have personally known both 'Santa' and 'Mrs. Claus' my entire life and I can assure everyone that they would have never knowingly done anything to place any children in danger," he continued. "They have both filled these roles for many years, and bringing joy to children during the holidays is one of the most important parts of their lives." +

+ Officials have asked those who believe they were exposed to follow proper COVID-19 quarantine guidelines as recommended by the Centers for Disease Control and Prevention, which dictates that people who had contact with someone diagnosed with coronavirus isolate at home for 14 days. +

+ Dana Green, a parent coordinator for Long County Schools System, told local news station WJCL that students who were exposed have been advised to remain at home until after the holiday break. +

+ A Dec. 9 post shared on the Long County School System's Facebook said that the community has been "experiencing an increase in Long County students and staff who are currently diagnosed with COVID-19 and/or are quarantined due to being identified as a direct contact for a positive person." +

+ However, the notice said that all schools will remain open through to Dec. 18, though virtual learning is available for students. +

+ As of Monday, there have been more than 16,556,900 cases of COVID-19 in the United States and at least 300,800 deaths from coronavirus-related illnesses, according to a New York Times database. +

+ As information about the coronavirus pandemic rapidly changes, PEOPLE is committed to providing the most recent data in our coverage. Some of the information in this story may have changed after publication. For the latest on COVID-19, readers are encouraged to use online resources from CDC, WHO, and local public health departments. PEOPLE has partnered with GoFundMe to raise money for the COVID-19 Relief Fund, a GoFundMe.org fundraiser to support everything from frontline responders to families in need, as well as organizations helping communities. For more information or to donate, click here. +

\ No newline at end of file diff --git a/packages/readabilityjs/test/test-pages/rootsofprogress/distiller.html b/packages/readabilityjs/test/test-pages/rootsofprogress/distiller.html new file mode 100644 index 000000000..e7ec56081 --- /dev/null +++ b/packages/readabilityjs/test/test-pages/rootsofprogress/distiller.html @@ -0,0 +1,121 @@ +

+ A review of Where Is My Flying Car? by J. Storrs Hall +

+ Suppose you were to reach into the mirror universe, where everything is inverted, and pull out a book that is the exact opposite of Robert Gordon’s The Rise and Fall of American Growth. +

+ Instead of being written by a scholarly economic historian steeped in the study of the past, it would be written by an engineer who has spent his career on futuristic technology. Instead of coming from a prestigious academic press, it would be self-published, with misformatted tables and a cover featuring garish three-dimensional lettering. Instead of sticking to extremely conservative predictions about future technologies, it would speculate audaciously about the limits of the possible, from nanotech to cold fusion. Instead of a sober survey of economic history in one country and time period, it would range widely through engineering, physics and philosophy, exploring the power-to-weight ratio of jet turbines in one chapter, and describing the rise of the counterculture in the next. And instead of proclaiming the death of innovation and the end of growth, it would paint a bold vision of an ambitious technological future. +

+ That book has leapt out of the mirror universe and into an Amazon Kindle edition (priced at 𝜋 dollars): Where Is My Flying Car? A Memoir of Future Past, by J. Storrs Hall. +

+ Hall sets out to tackle the title question: why don’t we have flying cars yet? And indeed, several chapters in the book are devoted to deep dives on the history, engineering, and economics of flying cars. But to fully answer the question, Hall must go much broader and deeper, because he quickly concludes that the barriers to flying cars are not technological or economic—they are cultural and political. To explain the flying car gap is to explain the Great Stagnation itself. +

+ Bold futures +

+ The most valuable thing I took away from the book was an awareness of some powerful technological possibilities. +

+ Flying cars +

+ Before reading the book, I had assumed that the flying car was one of those ideas that sounds good on its face, but turns out not to work or be interesting in practice. Maybe they’re inherently too hard to fly, too dangerous, or just not all that valuable. This book changed my mind, by pointing out a simple analogy: today’s system of flight has all of the inconveniences of railroads, over a century ago. Airplanes are large, mass-transit vehicles that travel only defined, scheduled routes between a small number of stations. This creates two problems for travelers. First is the “three vehicles” problem: you have to get from your origin to the nearest station, and then from your arrival station to your actual destination, changing vehicles each time (and hauling your luggage). Second is the inconvenience of schedule: having to be on time to catch the train or plane, compared to the personal vehicle that is ready immediately whenever you want it. This is driven home when you remember that a 90-minute flight from, say, San Francisco to Los Angeles actually takes you half a day, after travel to and from the airport plus delays at ticketing, security and boarding. +

+ The book points out that the major value in a flying car (as with supersonic) would not be in taking the same trips you do now, only a bit faster. Instead, it would be in taking the trips you don’t take now, because they’re too inconvenient. A flying car would shrink your world, expanding the radius of what you would consider for a commute, a shopping trip, a visit to friends, a business meeting, or a weekend vacation. Indeed, Hall cites literature from travel studies finding that people in all societies travel on average about an hour a day, whether walking barefoot or driving on the highway. And he points out that increasing the effective radius for each of those trips increases the effective area open to you quadratically (doubling your travel radius means four times as many destinations). +

+ Hall did extensive research and analysis for the book, even learning to fly a private aircraft himself. He recounts the history of flying car research and development, which began much earlier and has had many more credible attempts than I realized. He catalogs design approaches, including convertibles (vehicles that convert between flying and driving) and VTOL (vertical take-off and landing). He models engineering tradeoffs and travel times. And he concludes that there is no technological or economic reason why we can’t have flying cars with existing technology—indeed, why we couldn’t have had them already, if sustained work on them had continued past the 1970s. +

+ Nanotechnology +

+ Hall’s degrees are in computer science, but much of his career has been in nanotech, which was surprisingly prominent in the book. He makes clear that he’s not talking about mere nanoscale materials, but rather true nanotech, as envisioned by Feynman in the ’60s and advanced by Eric Drexler in the ’90s: atomically precise manufacturing, placing each atom one at a time exactly where you want it, giving you complete control over the structure of matter. In Hall’s telling, while this technology is obviously a ways off, the physics is sound and many of the basic principles have been worked out. +

+ The potential capabilities of mature nanotech are mind-blowing. The incredible speed alone would dramatically lower the price of literally every physical product. Hall estimates that the entire capital stock of the US—“every single building, factory, highway, railroad, bridge, airplane, train, automobile, truck, and ship”—could be rebuilt in a week. And nanotech would allow materials with extreme properties, such as the strength of diamond, to be used for everyday manufacturing and construction. +

+ The possibilities are straight out of science fiction. The “space pier”, a set of towers a hundred kilometers tall with a magnetic accelerator to shoot payloads into orbit, saving the fuel required to escape Earth’s gravity well and bringing down launch costs by three orders of magnitude. Or the “Weather Machine”, a fleet of quintillions of centimeter-sized balloons floating in the stratosphere, made of nanometer-thick diamond, with remote-controlled mirrors that can reflect light or allow it to pass through, forming a “programmable greenhouse gas” that can regulate temperature and direct solar energy. And of course, affordable flying cars. +

+ Energy, energy, energy +

+ One of the clearest indications of stagnation is the flatlining of energy usage. Because the growth in this metric was mentioned in the autobiography of Henry Adams (grandson of John Quincy Adams), Hall calls the long-term trend of about 7% annual growth in energy usage per capita the “Henry Adams Curve”. In the late 20th century, we fell off of it: +

+ Some techno-optimists, such as Andy McAfee, celebrate the flatlining and even peaking of resource usage curves, saying that we are getting “more from less”. Hall reminds us that more is more. All else being equal, energy efficiency is great. But there’s no reason to believe that flatlining or declining resource usage is optimal for progress. A large part of progress is harnessing ever-more resources and putting them to productive use. And indeed, we’re going to need lots more energy if we’re ever going to get nanotech manufacturing, regular space travel, and of course flying cars. In fact, a good explanation for technological stagnation is that the only technological revolution of the last 50 years, computing, was the only one that didn’t need more power than could be provided by the technology of the 1970s. +

+ Where will all this energy come from? It could come from solar: the amount of power reaching the Earth from the Sun is some 10,000 times greater than the current power requirements of humanity. Of course, it’s hard to harness in practice, owing to cloud cover and pesky inconveniences such as nighttime, but that’s nothing a well-placed fleet of a quintillion remote-controlled aerostats in the stratosphere couldn’t handle. +

+ But the majority of the energy discussion in the book focuses on the amazing potential of nuclear. The upshot is that we ought to have nuclear-powered everything. Nuclear homes with local, compact reactors—they don’t need to be on the grid. Nuclear cars, whether flying or ground. Even nuclear batteries—I was shocked to learn that certain designs of nuclear batteries were actually manufactured decades ago and used safely in implantable pacemakers. +

+ The main benefit, of course, is the insane energy density of nuclear fuel: just over a pound of enriched uranium has as much energy as 10,000 gallons of gasoline or over 100,000 pounds of anthracite coal. With nuclear batteries, no device would ever need to be recharged; with nuclear engines and generators, “your ground car and your home’s power unit will be refueled upon annual maintenance.” This fuel efficiency makes the economics of nuclear look almost identical to that of renewables: the fuel is practically free, compared to the fixed cost of infrastructure: “A wind turbine uses up more lubricating oil than a nuclear plant uses uranium, per kilowatt-hour generated.” +

+ The book describes several potential engineering approaches for nuclear power, not just the established fission plants based on uranium-235 that are in operation today, but everything up through speculative possibilities such as “chainless reactors” that bombard fissionable materials with high-energy neutrons, avoiding any nuclear chain reaction. Hall says that even cold fusion—er, sorry, I mean “low-energy nuclear reactions” (LENR)—deserves more research: although it might still turn out to be an unexploitable phenomenon or even an experimental artifact, there’s something going on that we don’t yet understand. Three chapters are dedicated to nuclear power; my main takeaway is that the variety of possibilities, and the scope and magnitude of the potential here, is breathtaking and underappreciated. +

+ The need for energy is fundamental to the economy, and yet a remarkable feature of our culture is the opposition to almost any form of energy—a pathology that Hall dubs “ergophobia”. (More on this below.) +

+ Level 5 +

+ Putting together all this and more, Hall summarizes his vision for the future as a “Second Atomic Age” based on nuclear, nanotech, and artificial intelligence. It’s a vision of continued exponential or even super-exponential progress, a world in which we see improvement in the world of atoms as fast as we’ve recently seen improvement only in the world of bits. +

+ Hall cites the global development advocate Hans Rosling, who classified the world population into four levels of income, on a logarithmic scale from $1/day (extreme poverty) to $64/day (which gets you electricity, a car, a washing machine, etc.). Using this scale, he says (emphasis added): +

+ The miracle of the Industrial Revolution is now easily stated: In 1800, 85% of the world’s population was at Level 1. Today, only 9% is. Over the past half century, the bulk of humanity moved up out of Level 1 to erase the rich-poor gap and make the world wealth distribution roughly bell-shaped. +

+ The average American moved from Level 2 in 1800, to level 3 in 1900, to Level 4 in 2000. +

+ We can state the Great Stagnation story nearly as simply: There is no level 5. +

+ Where Is My Flying Car? paints a vivid picture of what Level 5 would look like, and why we should keep working to get there. +

+ The roots of stagnation +

+ So, why aren’t we on Level 5 yet? What caused the Great Stagnation? What flatlined the Henry Adams Curve? Why don’t we have nanotech manufacturing and nuclear-powered everything? And where is my flying car? +

+ Hall blames a number of political and cultural factors: +

+ Centralized funding +

+ He starts with a case study on nanotech. True nanotech, he says, was killed by federal funding. Well, not by federal funding directly, but by a storm of academic politics that followed predictably from the $500 million National Nanotech Initiative kicked off under President Clinton. With a new pot of money on the table, and with academic funding being largely a zero-sum game, researchers in adjacent fields responded in two ways. First, they rebranded whatever they were doing “nanotech”, even projects such as nanoscale materials science that are unrelated to the original vision of atomically precise manufacturing. Second, they aggressively attacked that original vision. The result was that all the funding and credibility for true nanotech evaporated. +

+ Hall cites a passage from Machiavelli written in the 1500s that describes how politically dangerous it is to attempt to introduce an innovation: all of those who will be the losers if you succeed are galvanized against you, whereas those who would be the winners are much less motivated, given how speculative and uncertain the new innovation is. Seeing sixteenth-century social theory perfectly describe modern academic politics, Hall dubs this “The Machiavelli Effect.” And he cites other instances: cold fusion research, he says, was killed by a similar process. +

+ He concludes that “the increasing centralization and bureaucratization of science and research funding” is a major culprit: +

+ Centralized funding of an intellectual elite makes it easier for cadres, cliques, and the politically skilled to gain control of a field, and they by their nature are resistant to new, outside, non-Ptolemaic ideas. The ivory tower has a moat full of crocodiles. +

+ It is at the least suspicious, one must admit, that the major runup in civilian federal funding for research pretty nearly coincides with the recent period of technological slowdown. +

+ The burden of regulation +

+ Hall quotes a post on a message board suggesting that even if you had built a flying car and were ready to take to the air, you’d be shot down by the FAA, the mayor, the news media, the insurance company, and your neighbors. An even greater regulatory burden applies to nuclear power, which Hall blames for the skyrocketing cost of power plants in the US: +

+ In addition to the direct friction this burden places on innovation, it’s also a drain on human capital: +

+ How much of a drain? +

+ According to a study conducted by Tillinghast-Towers Perrin, the cost of the U.S. tort system consumes about two percent of GDP, on average. If we assume this mostly started around 1980 when lawyers skyrocketed and the airplane industry was destroyed, the long-run compound-interest effect on the economy as a whole is startling: without it our economy today would be twice the size it actually is. This is the closest we can come to measuring the effect of taking more than a million of the country’s most talented and motivated people and put them to work making arguments and filing briefs against each other so their efforts mostly cancel out, instead of inventing, developing, and manufacturing things which could have made life better. +

+ The counterculture +

+ Through maybe the 1950s, visions of the future, although varied, were optimistic. People believed in progress and saw technology as taking us forward to a better world. In the span of a generation, that changed, with the shift becoming prominent by the late 1960s. A “counterculture” arose which did not believe in technology or progress: indeed, a major element of the counterculture was the environmentalist movement, much of which saw technology and industry as actively destroying the Earth. +

+ In H. G. Wells’s The Time Machine, the “Eloi” were a weak, dissolute race of useless people who contribute nothing to society (a parody of the idle rich of 19th-century England). Hall calls the activists of the counterculture the “Eloi Agonistes”, and blames them for “ergophobia” and for excessive regulation: +

+ Unlike a century ago, today for everyone who is working on advancing technological progress, there is someone else who fervently believes that they are saving the planet by stopping them. +

+ Just as much as legal compliance and litigation, social activism is a drain on human capital: +

+ … simply the diversion of so many of the most talented and motivated members of the last several generations from productive pursuits to expensive virtue signaling is one of the main causes of the technological slowdown and the Great Stagnation. If your neighbor is Saving the Planet, it seems somehow less valuable merely to keep clean water running in the mains, or fill potholes, or build bridges. Eloi Agonistes have stolen the respect and gratitude that the people who are actually doing valuable work should be getting. +

+ The shift in values was reflected in, and reinforced by, a shift in science fiction towards the dystopian: +

+ Science fiction has a long and valuable history of providing us with visions of a better world. Verne, Wells, Burroughs, Gernsback—even Bellamy—much less Campbell, Doc Smith, van Vogt, Heinlein, Asimov, Garrett, Piper, Niven, and Pournelle, provided people with places and lives they could imagine and aspire to create. Science fiction since the Sixties has signally failed in that regard; we have been fed, by and large, a diet of Chicken Little soup in a pot of message, ladled out over leg of Frankenstein. +

+ Where did the Eloi Agonistes come from, and why did they rise when they did? Hall suggests a couple of related factors. One, the success of industrial civilization at meeting everyone’s basic needs for food, clothing and shelter pushed people up Maslow’s Hierarchy to seek self-actualization, which they did in the form of social activism. Two, the closing of the frontier meant the loss of a world in which people had to contend directly with nature and reality: +

+ After a long period of sustained social interaction, many forms of self-deception will become baked into the culture, and major social institutions will become in large part vehicles for virtue signalling…. But on the frontier, where a majority of one’s efforts are not in competition with others but directly against nature, self-deception is considerably less valuable. A culture with a substantial frontier is one with at least a countervailing force against the cancerous overgrowth of largely virtue-signalling, cost-diseased institutions. +

+ Personally, I don’t think these explanations tell the whole story. If people needed self-actualization, why choose anti-technology crusades? Why not self-actualize through invention, or art? I think we need to find an explanation not only for the form of people’s behavior, but for the content. Deirdre McCloskey suggests that the intellectual class had turned against capitalism and industry as early as 1848 (and Ayn Rand traces the intellectual roots to the late 1700s, blaming Immanuel Kant for killing the Enlightenment). This remains an open question for me. +

+ There are many writers with optimistic visions of the future. However, the goals I most often hear are all the negation of negatives: cure cancer, eliminate poverty, stop climate change. +

+ This is good, but it is not enough. We should not only cure disease and let everyone live to what is now considered old age—we should cure aging itself and extend human lifespan indefinitely. We should not seek to merely sustain current per-capita energy usage—we should get back on the Henry Adams Curve and increase it. We should not only seek to avoid worsening the climate—we should seek to actively control and optimize it for human ends. We should not merely get the whole world up to Level 4—we should be striving for Level 5. +

+ Aiming only for the former, as some so-called techno-optimists do, is a poor sort of optimism. It is actually calling for very limited progress, followed by stagnation. It is complacency with the status quo, content with bringing the whole world up to the current best standard of living, but not increasing it. In this context, I found Where Is My Flying Car? refreshing. Hall unabashedly calls for unlimited progress in every dimension. +

+ My only significant criticism (well, other than the malformatted data tables) is that the content isn’t tightly organized; the chapters jump around a lot. And there are a number of very deep dives and long digressions on detailed technical topics; I mostly enjoyed these, but if you’re not into them, feel free to skim. +

+ Overall, though, I found the book captivating and it has become one of my favorite books on stagnation and progress. Recommended for all my readers. +

\ No newline at end of file diff --git a/packages/readabilityjs/test/test-pages/samcurry/distiller.html b/packages/readabilityjs/test/test-pages/samcurry/distiller.html new file mode 100644 index 000000000..689409f48 --- /dev/null +++ b/packages/readabilityjs/test/test-pages/samcurry/distiller.html @@ -0,0 +1,564 @@ +

+ During our engagement, we found a variety of vulnerabilities in core portions of their infrastructure that would've allowed an attacker to fully compromise both customer and employee applications, launch a worm capable of automatically taking over a victim's iCloud account, retrieve source code for internal Apple projects, fully compromise an industrial control warehouse software used by Apple, and take over the sessions of Apple employees with the capability of accessing management tools and sensitive resources. +

+ There were a total of 55 vulnerabilities discovered with 11 critical severity, 29 high severity, 13 medium severity, and 2 low severity reports. These severities were assessed by us for summarization purposes and are dependent on a mix of CVSS and our understanding of the business related impact. +

+ As of October 6th, 2020, the vast majority of these findings have been fixed and credited. They were typically remediated within 1-2 business days (with some being fixed in as little as 4-6 hours). +

+ Introduction +

+ While scrolling through Twitter sometime around July I noticed a blog post being shared where a researcher was awarded $100,000 from Apple for discovering an authentication bypass that allowed them to arbitrarily access any Apple customer account. This was surprising to me as I previously understood that Apple's bug bounty program only awarded security vulnerabilities affecting their physical products and did not payout for issues affecting their web assets. +

+ After finishing the article, I did a quick Google search and found their program page where it detailed Apple was willing to pay for vulnerabilities "with significant impact to users" regardless of whether or not the asset was explicitly listed in scope. +

+ This caught my attention as an interesting opportunity to investigate a new program which appeared to have a wide scope and fun functionality. At the time I had never worked on the Apple bug bounty program so I didn't really have any idea what to expect but decided why not try my luck and see what I could find. +

+ In order to make the project more fun I sent a few messages to hackers I'd worked with in the past and asked if they'd like to work together on the program. Even though there was no guarantee regarding payouts nor an understanding of how the program worked, everyone said yes, and we began hacking on Apple. +

+ Reconnaissance +

+ The first step in hacking Apple was figuring out what to actually target. Both Ben and Tanner were the experts here, so they began figuring out what all Apple owned that was accessible to us. All of the results from their scanning were indexed in a dashboard that included the HTTP status code, headers, response body, and screenshot of the accessible web servers under the various domains owned by Apple that we’d refer to over the engagement. +

+ To be brief: Apple's infrastructure is massive. +

+ They own the entire 17.0.0.0/8 IP range, which includes 25,000 web servers with 10,000 of them under apple.com, another 7,000 unique domains, and to top it all off, their own TLD (dot apple). Our time was primarily spent on the 17.0.0.0/8 IP range, .apple.com, and .icloud.com since that was where the interesting functionality appeared to be. +

+ After making a listing of all of the web servers, we began running directory brute forcing on the more interesting ones. +

+ Some of the immediate findings from the automated scanning were... +

  • VPN servers affected by Cisco CVE-2020-3452 Local File Read 1day (x22)
  • Leaked Spotify access token within an error message on a broken page

+ The information obtained by these processes were useful in understanding how authorization/authentication worked across Apple, what customer/employee applications existed, what integration/development tools were used, and various observable behaviors like web servers consuming certain cookies or redirecting to certain applications. +

+ After all of the scans were completed and we felt we had a general understanding of the Apple infrastructure, we began targeting individual web servers that felt instinctively more likely to be vulnerable than others. +

+ This began a series of findings that continued throughout our engagement and progressively increased our understanding of Apple’s program. +

+ Vulnerabilities Discovered +

+ Vulnerability Write-Ups +

+ We can’t write about all the vulnerabilities we discovered, but here is a sample of some of the more interesting vulnerabilities. +

+ Full Compromise of Apple Distinguished Educators Program via Authentication and Authorization Bypass +

+ One of the first services we spent time hacking was the “Apple Distinguished Educators” site. This was an invitation-only Jive forum where users could authenticate using their Apple account. Something interesting about this forum was that some of the core Jive functionality to register to the app was ported through a custom middleware page built by Apple in order to connect their authentication system (IDMSA) to the underlying Jive forum which normally used username/password authentication. +

+ This was built to allow users to easily use their already existing Apple account to authenticate to the forum and not have to deal with creating an additional user account. You would simply use the “Sign In With Apple” and be logged into the forum. +

+ The landing page for users who were not allowed to access the forum was an application portal where you provided information about yourself that was assessed by the forum moderators for approval. +

+ When you submitted an application to use the forum, you supplied nearly all of the values of your account as if you were registering to the Jive forum normally. This would allow the Jive forum to know who you were based on your IDMSA cookie since it tied your email address belonging to your Apple account to the forum. +

+ One of the values that was hidden on the page within the application to register to use the forum was a “password” field with the value “###INvALID#%!3”. When you submitted your application that included your username, first and last name, email address, and employer, you were also submitting a “password” value which was then secretly tied to your account sourced from a hidden input field on the page. + 

<div class="j-form-row">
+<input id="password" type="hidden" value="###INvALID#%!3">
+<div id="jive-pw-strength">
+...

+ After observing the hidden default password field, we immediately had the idea to find a way to manually authenticate to the application and access an approved account for the forum instead of attempting to login using the “Sign In With Apple” system. We investigated this because the password was the same for each one of us on our separate registrations. +

+ If anyone had applied using this system and there existed functionality where you could manually authenticate, you could simply login to their account using the default password and completely bypass the "Sign In With Apple" login. +

+ From a quick glance, it did not appear that you could manually authenticate, but after a few Google searches we identified a “cs_login” endpoint which was meant for logging in with a username and password to Jive applications. When we manually formed the test HTTP request to authenticate to the Apple Distinguished Developers application, we found that it attempted to authenticate us by displaying an incorrect password error. When we used our own accounts that we had previously applied with, the application errored out and did not allow us to authenticate as we were not yet approved. We would have to find the username of an already approved member if we wanted to authenticate. +

+ At this point, we loaded the HTTP request into Burp Suite’s intruder and attempted to brute force usernames between 1 and 3 characters via the login and default password. +

+ After about two minutes we received a 302 response indicating a successful login to a user with a 3 character username using the default password we found earlier. We were in! From this point, our next goal was to authenticate as someone with elevated permissions. We took a few screenshots of our access and clicked the “Users” list to view which users were administrators. We logged into the first account we saw on the list in an attempt to prove we could achieve remote code execution via the administrative functionality, however, there were still a few roadblocks ahead. +

+ When attempting to browse to “/admin/” (the Jive administrator console) as the admin account, the application redirected to login as if we were not yet authenticated. This was strange, as it was custom behavior for the Jive application and none of us had observed this before. Our guess was that Apple had restricted the administration console based on IP address to make sure that there was never a full compromise of the application. +

+ One of the first things we tried was using the X-Forwarded-For header to bypass the hypothetical restriction, but sadly that failed. The next thing we tried was to load a different form of “/admin/” in-case the application had path specific blacklists for accessing the administrator console. +

+ After just a few more HTTP requests, we figured out that “GET /admin;/” would allow an attacker to access the administration console. We automated this bypass by setting up a Burp Suite rule which automatically changed “GET/POST /admin/” to “GET/POST /admin;/” in our HTTP requests. +

+ When we finally navigated and loaded the administration console, it was immediately clear that something wasn’t right. We did not have access to the normal functionality that would demonstrate remote code execution (there was no templating, plugin upload, nor the standard administrative debugging functionality). +

+ At this point, we stopped to think about where we were and realized that the account we authenticated to may not be the “core” administrator of the application. We went ahead and authenticated to 2-3 more accounts before finally authenticating as the core administrator and seeing functionality that would allow for remote code execution. +

+ An attacker could (1) bypass the authentication by manually authenticating using a hidden default login functionality, then (2) access the administration console via sending a modified HTTP path in the request, and finally (3) completely compromise the application by using the one of many “baked in RCE” functionalities like plugin upload, templating, or file management. +

+ Overall, this would've allowed an attacker to... +

  • Execute arbitrary commands on the ade.apple.com webserver
  • Access the internal LDAP service for managing user accounts
  • Access the majority of Apple's internal network

+ At this point, we finished the report and submitted everything. +

+ Full Compromise of DELMIA Apriso Application via Authentication Bypass +

+ Something we had thought about a lot while hacking Apple was whether or not they had any accessible services relating to the manufacturing and distribution of their products. As it turns out, there was an application called "DELMIA Apriso" which was a third-party "Global Manufacturing Suite" which provided what appeared to be various warehouse solutions. +

+ Sadly, there did not appear to be much available interaction for the technology as you could only "login" and "reset password" from the available interfaces. +

+ After attempting to find vulnerabilities on the limited number of pages, something strange happened: we were authenticated as a user called "Apple No Password User" based on a bar which appeared in the upper right portion of the site. +

+ What had happened was that, by clicking "Reset Password", we were temporarily authenticated as a user who had "Permission" to use the page. +

+ The application's authentication model worked whereas users had specific permissions to use specific pages. The "reset password" page counted as a page itself, so in order to let us use it, the application automatically logged us into an account that was capable of using the page. +

+ We attempted a variety of things in order to elevate our permissions but didn't seem to get anywhere for a long time. After a while, we sent an HTTP request to an OAuth endpoint in an attempt to generate an authorization bearer that we could use to explore the API. This was successful! +

+ Our user account, even though its permissions were intended to be limited to authorization and resetting our password, could generate a bearer which had permission to access the API version of the application. +

+ We were now able to explore the API and hopefully find some permission issue which would allow us to compromise some portion of the application. Luckily, during our recon process, we found a list of API requests for the application. +

+ We sadly did not have access to the majority of the API calls, but some sections like "Operations" disclosed a massive number of available functionalities. +

+ If you hit the "/Apriso/HttpServices/api/platform/1/Operations" endpoint, it would return a list of nearly 5,000 different API calls. None of these required authentication beyond the initial authorization bearer we initially sent. The operations disclosed here included things like... +

  • Creating and modifying shipments
  • Creating and modifying employee paydays
  • Creating and modifying inventory information
  • Validating employee badges
  • Hundreds of warehouse related operations

+ The one that we paid most attention to was "APL_CreateEmployee_SO". +

+ You could send a GET request to the specific operations and receive the expected parameters using the following format: + 

GET /Apriso/HttpServices/api/platform/1/Operations/operation HTTP/1.1
+Host: colormasters.apple.com

+ With the following HTTP response: + 

{
+  "InputTypes": {
+    "OrderNo": "Char",
+    "OrderType": "Integer",
+    "OprSequenceNo": "Char",
+    "Comments": "Char",
+    "strStatus": "Char",
+    "UserName": "Char"
+  },
+  "OutputTypes": {},
+  "OperationCode": "APL_Redacted",
+  "OperationRevision": "APL.I.1.4"
+}

+ It took a bit of time, but after a while we realized that in order to actually call the API you had to send a POST request with JSON data in the following format: + 

{
+  "inputs": {
+    "param": "value"
+  }
+}

+ The above format (after the fact) appears very simple and easy to understand, but at the time of hacking we had absolutely no idea how to form this call. I had even tried emailing the company who provided the software asking how you were supposed to form these API calls, but they wouldn't respond to my email because I didn't have a subscription to the service. +

+ I'd spent nearly 6 hours trying to figure out how to form the above API call, but after we figured it out, it was very much smooth sailing. The "create employee" function required various parameters that relied on UUIDs, but we were able to retrieve these via the other "Operations" and fill them in as we went along. +

+ About two hours more, we finally formed the following API request: + 

POST /Apriso/HttpServices/api/platform/1/Operations/redacted HTTP/1.1
+Host: colormasters.apple.com
+Authorization: Bearer redacted
+Connection: close
+Content-Type: application/json
+Content-Length: 380
+{
+  "inputs": {
+    "Name": "Samuel Curry",
+    "EmployeeNo": "redacted",
+    "LoginName": "yourloginname123",
+    "Password": "yourpassword123",
+    "LanguageID": "redacted",
+    "AddressID": "redacted",
+    "ContactID": "redacted",
+    "DefaultFacility": "redacted",
+    "Department": "",
+    "DefaultMenuItemID": "redacted",
+    "RoleName": "redacted",
+    "WorkCenter": ""
+  }
+}

+ After we sent this API call, we could now authenticate as a global administrator to the application. This gave us full oversight to the warehouse management software and probably RCE via some accepted functionality. +

+ There were hundreds of different functionalities that would've caused massive information disclosure and been capable of disrupting what appeared to be a somewhat crucial application used for inventory and warehouse management. +

+ Wormable Stored Cross-Site Scripting Vulnerabilities Allow Attacker to Steal iCloud Data through a Modified Email +

+ One of the core parts of Apple’s infrastructure is their iCloud platform. This website works as an automatic storage mechanism for photos, videos, documents, and app related data for Apple products. Additionally, this platform provides services like Mail and Find my iPhone. +

+ The mail service is a full email platform where users can send and receive emails similar to Gmail and Yahoo. Additionally, there is a mail app on both iOS and Mac which is installed by default on the products. The mail service is hosted on “www.icloud.com” alongside all of the other services like file and document storage. +

+ This meant, from an attackers perspective, that any cross-site scripting vulnerability would allow an attacker to retrieve whatever information they wanted to from the iCloud service. We began to look for any cross-site scripting issues at this point. +

+ The way the mail application works is very straightforward. When the service receives an email and a user opens it, the data is processed into a JSON blob which is sanitized and picked apart by JavaScript and then displayed to the user. +

+ This means that there is no server side processing of the emails in terms of content sanitation, and that all of the actual functionality to render and process the mail body is within the JavaScript where it’s done client side. This isn’t necessarily a bad thing, but simplifies the process of identifying XSS by understanding what specifically we’ll need to break within the source code. +

+ Stored XSS via Style Tag Confusion +

+ When testing this functionality one of the things I eventually messed with was the “<style>” tag. This tag is interesting as the DOM will only cancel this element with an end “</style>” tag. This means that if we wrote “<style><script>alert(1)</script></style>” and it was fully rendered in the DOM, there would be no alert prompt as the content of the tag is strictly CSS and the script tag was stuffed within the tag and not beyond the closing tag. +

+ From a sanitization perspective, the only things Apple would need to worry about here would be an ending style tag, or if there was sensitive information on the page, CSS injection via import chaining. +

+ I decided to focus on trying to break out of the style tag without Apple realizing it since it would be a very straightforward stored XSS if achievable. +

+ I played around with this for a while trying various permutations and eventually observed something interesting: when you had two style tags within the email, the contents of the style tags would be concatenated together into one style tag. This meant that if we could get “</sty” into the first tag and “le>” into the second tag, it would be possible to trick the application into thinking our tag was still open when it really wasn’t. +

+ I sent the following payload to test if it would work: + 

<style></sty</style>
+<style>le><script>alert(1)</script></style>

+ The email popped up in my inbox. I clicked it. There was an alert prompt! It had worked! +

+ The DOM of the page included the following: + 

<style></style><script>alert(1)</script></style>

+ Since the mail application is hosted on “www.icloud.com” this meant that we had browser permissions to retrieve the HTTP responses for the corresponding APIs for the iCloud service (if we could sneak in the JavaScript to reach out to them). +

+ An explanation of the above payload is as follows: +

+ At this point we decided the coolest proof of concept would be something which steals all of the victim’s personal information (photos, calendar information, and documents) then forwards the same exploit to all of their contacts. +

+ We built a neat PoC which would return the photo URLs from the iCloud API, stick them into image tags, and then append a list of contacts for the user account underneath them. This demonstrated that it was possible to retrieve the values, but in order to exfiltrate them we would have to bypass a CSP which meant no easy outbound HTTP requests to anything but “.apple.com” and a few other domains. +

+ Luckily for us, the service is a mail client. We can simply use JavaScript to invoke an email to ourselves, attach the iCloud photo URLs and contacts, then fire away all of the victim’s signed iCloud photo and document URLs. +

+ The following video demonstrates a proof of concept whereas a victim’s photos are stolen. In a full exploitation scenario performed by a malicious party, an attacker could silently steal all of the victim’s photos, videos, and documents, then forward the modified email to the victim’s contact list and worm the cross-site scripting payload against the iCloud mail service. +

+ Stored XSS via Hyperlink Confusion +

+ Later on I found a second cross-site scripting vulnerability affecting mail in a similar fashion. +

+ One thing I’ll always check with these sorts of semi-HTML applications is how they handle hyperlinks. It seems intuitive to automatically turn an unmarked URL into a hyperlink, but it can get messy if it isn’t being sanitized properly or is combined with other functionalities. This is a common place to look for XSS due to the reliance on regex, innerHTML, and all of the accepted elements you can add alongside the URL. +

+ The second piece of interesting functionality for this XSS is the total removal of certain tags like “<script>” and “<iframe>”. This one is neat because certain things will rely on characters like space, tabs, and new lines whereas the void left by the removed tag can provide those characters without telling the JavaScript parser. These indifferences allow for attackers to confuse the application and sneak in malicious characters which can invoke XSS. +

+ I played around with both of these functionalities for a while (automatic hyperlinking and the total removal of certain tags) until deciding to combine the two and attempt to see how they behaved together. To my surprise, the following string broke the hyperlinking functionality and confused the DOM: + 

https://www.domain.com/abc#<script></script>https://domain.com/abc

+ After sending the above by itself within an email, the content was parsed to the following: + 

<a href="https://www.domain.com/abc#<a href=" https:="" www.domain.com="" abc="&quot;" rel="noopener noreferrer">https://www.domain.com/abc</a>

+ This was very interesting to see initially, but exploiting it would be a bit harder. It is easy to define the attributes within the tag (e.g. src, onmouseover, onclick, etc.) but providing the values would be difficult as we still had to match the URL regex so it wouldn’t escape the automatic hyperlinking functionality. The payload that eventually worked without sending single quotes, double quotes, parenthesis, spaces, or backticks was the following: + 

https://www.icloud.com/mail/#<script></script>https://www.icloud.com/onmouseover=location=/javascript:alert%28document.domain%29/.source;//

+ The payload produced this in the DOM: + 

<a href="https://www.icloud.com/mail#<a href=" https:="" www.icloud.com="" onmouseover="location=/javascript:alert%28document.domain%29/.source;//&quot;">https://www.icloud.com/onmouseover=location=/javascript:alert%28document.domain%29/.source;//</a>

+ And gave us this beautiful alert prompt: +

+ This payload was from a CTF solution by @Blaklis_. I had originally thought it might be an unexploitable XSS, but there seems to always be a solution somewhere for edge case XSS. +

+ My best explanation here is that (1) when loading the initial URL the characters within the “<script></script>” were acceptable within the automatic hyperlinking process and didn’t break it, then (2) the removal of the script tags created a space or some sort of void which reset the automatic hyperlinking functionality without closing the initial hyperlinking functionality, and lastly (3) the second hyperlink added the additional quote that was used to both break out of the href and create the onmouseover event handler. +

+ The impact for the second XSS was the same as the first one, except for this one the user would have to trigger the onmouseover event handler via putting their mouse somewhere within the email body, but this part could be simplified to trigger more easily by making the hyperlink of the entire email. +

+ Overall, an attacker could've abused this to... +

  • Create a worm that has the capability to silently exfiltrate/modify iCloud account information including photos and videos
  • Silently execute arbitrary HTML and JavaScript within the victim's browser

+ Command Injection in Author’s ePublisher +

+ A major feature of Apple is the ability to upload and sell books, movies, tv shows, and songs. The files you upload get propagated to various Apple services such as iTunes where people can download or purchase them. This seemed like a good vector for customer XSS and blind XSS against employees. +

+ In order to upload files, we first had to apply for access to the service on iTunes Connect. +

+ We ran into an interesting problem where we did not have access to an iPad or iPhone, but we kept on looking for ways to use this service still. After some investigating, we discovered a tool called Transporter. +

+ Transporter is a Java app that can be used to interact with a jsonrpc API for bulk uploading files utilizing a few different file services. +

+ At the same time, we were also looking through the iTunes Connect Book help docs and we found a page that explained a few different ways to upload books including an online web service: https://itunespartner.apple.com/books/articles/submit-your-ebook-2717 +

+ This led us to the following service, Apple Books for Authors. +

+ This service only has a couple of features: +

  • Sign-in / Register
  • Upload images for book cover
  • Upload book ePub file
  • Upload book Sample ePub file

+ The first thing we did was download sample epub files and upload them. Funny enough, the first epub file we grabbed was an epub version 1 format with invalid xhtml. The publish tool spit out a huge wall of text of errors to let us know why it failed to upload/validate. +

+ HTTP Request: + 

POST /api/v1/validate/epub HTTP/1.1
+Host: authors.apple.com
+{"epubKey":"2020_8_11/10f7f9ad-2a8a-44aa-9eec-8e48468de1d8_sample.epub","providerId":"BrettBuerhaus2096637541"}

+ HTTP Response: + 

[2020-08-11 21:49:59 UTC] <main> DBG-X:   parameter TransporterArguments = -m validateRawAssets -assetFile /tmp/10f7f9ad-2a8a-44aa-9eec-8e48468de1d8_sample.epub -dsToken **hidden value** -DDataCenters=contentdelivery.itunes.apple.com -Dtransporter.client=BooksPortal -Dcom.apple.transporter.updater.disable=true -verbose eXtreme -Dcom.transporter.client.version=1.0 -itc_provider BrettBuerhaus2096637541

+ As you can probably guess at this point, all we had to do was a simple command injection on the provderId JSON value. +

+ We intercepted the request on the next upload and replaced it with: + 

"providerId":"BrettBuerhaus2096637541||test123"

+ And we got the following output: + 

/bin/sh: 1: test123: not found

+ The following is a screenshot showing the output of "ls /": +

+ Overall, an attacker could've abused this to... +

  • Execute arbitrary commands on the authors.apple.com webserver
  • Access Apple's internal network

+ This was a good exercise in making sure you fully explore what you are testing. A lot of the big names in recon research talk about creating mind maps and this is an example of that. We started with iTunes Connect, started exploring Books, and continued to branch out until we fully understood what services exist around that single feature. +

+ It also is a good reminder that you need to find as much information as possible before you start going down rabbit-holes while testing. Without exploring the help docs, you may have missed the web epub app entirely as it is a single link on one page. +

+ Full Response SSRF on iCloud allows Attacker to Retrieve Apple Source Code +

+ The most elusive bug while hacking on Apple was full response SSRF. We found nearly a dozen blind or semi-blind SSRFs, but had a terribly hard time trying to find any way to retrieve the response. This was incredibly frustrating as during our recon process we found tons of references to what appeared to be awesome internal applications for source code management, user management, information lookup, and customer support. +

+ It wasn’t until the end of our engagement when we finally stumbled upon one which seemed to have a great deal of internal network access. +

+ During testing the iCloud application we noticed that you could open up certain attachments from the iCloud mail application in the iCloud pages application via the “Open in Pages” functionality. When you submitted the form to do this, it sent an HTTP request containing a URL parameter which included the URL of the mail file attachment in the request. If you attempted to modify this URL to something arbitrary, the request would fail and give a “400 Bad Request” error. The process would create a “job” where the response of the HTTP request was converted into an Apple Pages document, then opened in a new tab. +

+ It seemed to only allow URLs from the “p37-mailws.icloud.com” domain, would not convert pages with anything but a 200 OK HTTP response, and would additionally be a bit hard to test as the conversion process was done through multiple HTTP requests and a job queue. +

+ What worked to exploit this was appending “@ourdomain.com” after the white-listed domain which would point the request at our domain. The process would convert the raw HTML to an Apple pages file then display it to us in a new window. This was a bit annoying to fuzz with, so Brett ended up throwing together a python script to automate the process. +

+ Our proof of concept for this report was demonstrating we could read and access Apple’s internal maven repository. We did not access any source code nor was this ever exploited by other actors. +

+ If the file was too large to be saved to a Pages file, it would instead be stored to the drive in a downloadable zip file which would allow us to extract large files like jars and zips. +

+ We had found the internal maven URL disclosed in a Github repository. +

+ There were many other internal applications we could’ve pulled from, but since we demonstrated access to the Maven repository with source code access we reported the issue right away. +

+ Overall, an attacker could've abused this to... +

  • Read the various iOS source code files within the maven repository
  • Access anything else available within Apple's internal network
  • Fully compromise a victim's session via a cross-site scripting vulnerability due to the disclosed HTTP only cookies within the HTTP request

+ The full process that had to be followed when scripting this is as follows: +

+ Nova Admin Debug Panel Access via REST Error Leak +

+ While going through a list of all Apple subdomains one at a time, we discovered some interesting functionality from "concierge.apple.com", "s.apple.com", and "events.apple.com". +

+ With a little bit of Google dorking, we found that a specific request to "s.apple.com" would take you to "events.apple.com" with an authentication token. +

+ HTTP Request: + 

GET /dQ{REDACTED}fE HTTP/1.1
+Host: s.apple.com

+ HTTP Response: + 

HTTP/1.1 200
+Server: Apple
+Location: https://events.apple.com/content/events/retail_nso/ae/en/applecampathome.html?token=fh{REDACTED}VHUba&a=1&l=e

+ Performing our standard recon techniques, we grabbed the JavaScript files and started looking for endpoints and API routes. +

+ Discovering a /services/public/account endpoint, we started to play around with it. We quickly discovered that passing in an invalid marketCode parameter resulted in the server returning a REST exception error. +

+ HTTP Request: +

+ HTTP Response: +

+ From the error message we can see the server is forwarding an API request to the following location: + 

https://nova-admin.corp.apple.com/services/locations/searchLocation?locationName=t&rtm=1

+ We can also see that it leaked some request/response headers including a nova-admin cookie and an authorization token that the server is sending to make requests to nova-admin.corp.apple.com API requests. +

+ Also interesting is that the /services/ endpoint is similar to the /services/public/ API endpoints for the events app. We could not hit the endpoints on the event app and we did not have access to nova-admin.corp.apple.com. Going back to our recon data, we noticed that there is a nova.apple.com. +

+ Attempting to use the acquired auth token and cookie, we noted that the credentials were valid as we were no longer being redirected to idsmac auth, but it was still 403 forbidden. +

+ With a little bit of fuzzing, we discovered that we were able to hit /services/debug.func.php. +

+ Even though it was not a website with PHP extensions, it appeared adding any extension to the debug route would bypass the route restrictions they built since the authorization was separate from the functionality itself. +

+ HTTP Request: +

+ HTTP Response: +

+ This portal contained dozens of options, also contained several hundred configuration parameters and values. +

+ One of the values also contained an AWS secret key, another contained server crontabs. Having the ability to update these values was enough to prove command injection. +

+ Overall, an attacker could've abused this to... +

  • Execute arbitrary commands on the nova.apple.com webserver
  • Access Apple's internal network

+ At this point, we decided we had proven enough impact and stopped. The full flow from above is as follows: +

+ AWS Secret Keys via PhantomJS iTune Banners and Book Title XSS +

+ We discovered the iTunes banner maker website a few weeks prior to finding this vulnerability. It was not until we added a book via iTunes Connect did we discover an interesting feature on the banner maker. +

+ There are multiple banner image formats based on the height and width specified. We discovered that the "300x250" banner image included the book name. +

+ We also noticed that it was vulnerable to Cross-Site Scripting because the book name was underlined with our injected "<u>" element which we had set whilst registering the book on iTunes connect. +

+ Image URL: + 

https://banners.itunes.apple.com/bannerimages/banner.png?pr=itunes&t=catalog_black&c=us&l=en-US&id=1527342866&w=300&h=250&store=books&cache=false

+ Earlier we had already discovered that there was path traversal and parameter injection in a few of the request parameters such as "pr". For example: + 

https://banners.itunes.apple.com/bannerimages/banner.png?pr=itunes/../../&t=catalog_black&c=us&l=en-US&id=1527342866&w=300&h=250&store=books&cache=false

+ Results in a picture of the AWS S3 error page: +

+ From here we made the assumption that they were using a headerless browser client to take screenshots of HTML files inside of an S3 bucket. So the next step was to create a book with <script src=””> in the name to start playing around with the XSS in the image generation process. +

+ The first thing we noticed was in the request log when it hit our server: + 

54.210.212.22 - - [21/Aug/2020:15:54:07 +0000] "GET /imgapple.js?_=1598025246686 HTTP/1.1" 404 3901 "http://apple-itunes-banner-builder-templates-html-stage.s3-website-us-east-1.amazonaws.com/itunes/catalog_white/index.html?pr=itunes&t=catalog_white&c=us&l=en-US&id=1528672619&w=300&h=250&store=books&cache=false" "Mozilla/5.0 (Unknown; Linux x86_64) AppleWebKit/538.1 (KHTML, like Gecko) PhantomJS/2.1.1 Safari/538.1"

+ This is the S3 bucket / image it was hitting to generate the picture: + 

http://apple-itunes-banner-builder-templates-html-stage.s3-website-us-east-1.amazonaws.com/itunes/catalog_white/index.html?pr=itunes&t=catalog_white&c=us&l=en-US&id=1528672619&w=300&h=250&store=books&cache=false

+ And this is the User-Agent: + 

PhantomJS/2.1.1

+ Luckily for us, Brett had actually exploited exactly this a few years prior: +

+ The first thing was to write our JS XSS payload to perform Server-Side Request Forgery attacks. A good method to do this and render data is with the <iframe> element. +

+ Since we know this on AWS, we attempt to hit AWS metadata URI: + 

https://banners.itunes.apple.com/bannerimages/banner.png?pr=itunes&t=catalog_black&c=us&l=en-US&id=1528672619%26cachebust=12345%26url=http://169.254.169.254/latest/meta-data/identity-credentials/ec2/security-credentials/ec2-instance%26&w=800&h=800&store=books&cache=false

+ This rendered a new banner image with the full AWS secret keys for an ec2 and iam role: +

+ Most of Apple’s interesting infrastructure appears to be in the /8 IP CIDR they own dubbed “Applenet,” but they do have quite a bit of hosts and services in AWS ec2/S3. We knew the SSRF would not be super interesting with the recon we performed as most of the interesting corp targets are in Applenet and not AWS. +

+ Overall, an attacker could've abused this to... +

  • Read contents from Apple's internal Amazon Web Services environment
  • Access and use the AWS ec2 keys discloses within the internal metadata page

+ Heap Dump on Apple eSign Allows Attacker to Compromise Various External Employee Management Tools +

+ During our initial recon phase collecting sub-domains and discovering the Apple public-facing surface, we found a bunch of “esign” servers. +

  • https://esign-app-prod.corp.apple.com/
  • https://esign-corpapp-prod.corp.apple.com/
  • https://esign-internal.apple.com
  • https://esign-service-prod.corp.apple.com
  • https://esign-signature-prod.corp.apple.com
  • https://esign-viewer-prod.corp.apple.com/
  • https://esign.apple.com/

+ Upon loading the subdomain, you’re immediately redirected to a /viewer folder. When you go through the Apple idmsa authentication flow, you are returned to an “you are not authorized” error. +

+ We do not have access to any links or interesting js files from this page, so we tried some basic wordlists to see if we could find endpoints for the application. From here we discovered that /viewer/actuator responded with all of the actuator endpoints including mapping and heapdump. +

+ We were unable to make changes by sending state-changing requests to actuator in an attempt for Remote Code Execution, so we had to find an alternative route for proving impact. +

+ The mappings exposed all the web routes to us, including additional folders at the root of the host that had additional actuator heapdumps in them. It was at this point that we realized the actuator endpoints were vulnerable in each app folder on all esign subdomains. From here we grabbed a heapdump from ensign-internal. +

+ We loaded the heapdump using Eclipse Memory Analyzer and exported all the strings out to csv to sift with grep. +

+ From there we learned that the application’s authentication cookie is “acack”. We searched for acack in the heapdump until we found a valid session cookie. At this point we were certain that it was an Apple employee token and not a customer, otherwise we would not have tested it. Upon loading it, we were able to access the application. +

+ There’s not much we can show, but here’s a snippet showing the authenticated view of the page: +

+ This gave us access to 50+ application endpoints, including some admin endpoints such as “setProxy” that would likely have been easily escalated to an internal SSRF or RCE. We also noticed that the acack cookie was authenticating us to other applications as well. +

+ Having proven sufficient impact we stopped here and reported it. +

+ Actuators exposing heapdumps public-facing are nothing new and it’s a relatively low-hanging finding that most wordlists will catch. It’s important to remember that just because you aren’t finding them commonly, they’re still out there and on big targets just waiting to be found by an attacker. +

+ XML External Entity processing to Blind SSRF on Java Management API +

+ During testing, we discovered an API with multiple unauthenticated functions that all consumed "application/xml" after finding an exposed "application.wadl" on one of the many 17.0.0.0/8 hosts. +

+ An application.wadl file defines the endpoints used by this service. This was a test instance of a service that is normally locked down and inaccessible. +

+ We were able to use a blind XXE payload to demonstrate a blind SSRF. +

+ Sadly, we were not able to fully exploit this to read files on this machine or get a response back from an SSRF due to the Java version used on this machine (fully patched, preventing a 2 stage blind XXE payload). Additionally we did not know the expected XML format structure (preventing a non-blind XXE exploit). +

+ This vulnerability was interesting as Apple is heavily XML dependent and it felt like we would’ve found more instances of XXE with how many requests we’d seen using it. It was surprising exploiting this one because to achieve blind XXE as it was very straightforward compared to all of the complicated ways we’d tried to identify it over time. +

+ If we were to ever successfully exploit this to achieve local file read or full response SSRF, it would likely be through finding the proper XML format for the API itself in order to reflect the file contents directly versus achieve blind exfiltration. +

+ Overall, an attacker could've abused this to... +

  • Obtain what are essentially keys to various internal and external employee applications
  • Disclose various secrets (database credentials, OAuth secrets, private keys) from the various esign.apple.com applications

+ GBI Vertica SQL Injection and Exposed GSF API +

+ Our initial recon efforts involved capturing screenshots of all Apple owned domains and IP addresses that responded with an HTTP banner. We found a couple servers that looked like this: +

+ From here we started to mess around with some of the applications such as "/depReports". We could authenticate to them and access some data via API requests to an API on the "/gsf/" route. All of the applications that we accessed on this host routed requests through that GSF service. +

+ The request looked like the following: + 

POST /gsf/partShipment/businessareas/AppleCare/subjectareas/acservice/services/batch HTTP/1.1
+Host: gbiportal-apps-external-msc.apple.com
+{
+    "executionType": "parallel",
+    "requests": [{
+        "queryName": "redacted",
+        "filters": {
+            "redacted_id": ["redacted"],
+            "redacted": [""]
+        }
+    }, {
+        "queryName": "redacted",
+        "filters": {
+            "redacted_id": ["redacted"],
+            "redacted": [""],
+            "redacted": [""],
+            "redacted": [""],
+            "redacted": [""],
+            "redacted": [""],
+            "redacted": ["service_notification_number"],
+            "redacted": ["desc"]
+        }
+    }, {
+        "queryName": "redacted",
+        "filters": {
+            "redacted_id": ["redacted"],
+            "redacted": [""],
+            "redacted": [""],
+            "redacted": [""],
+            "redacted": [""],
+            "redacted": [""],
+            "redacted": ["service_notification_number"],
+            "redacted": ["desc"],
+            "redacted": ["100"],
+            "redacted": ["0"]
+        }
+    }]
+}
+

+ You can see almost immediately that there are some really strong indicators here that they are interacting with SQL. Keywords: query, limit, offset, column names, filter, etc. Making one tiny change to see what happens, we got the following: +

+ (Heavily redacted, covering up the query error that includes column names, table name, database name, etc). The important bit is: + 

exception is java.sql.SQLException: java.sql.SQLSyntaxErrorException: [Vertica][VJDBC](4856) ERROR: Syntax error at or near \"adesc\""}]}]}

+ We eventually got a union injection working. Some important parts were the extra "*/*/" closing comments in limit due to stacking queries. We also had to use /**/ between FROM and table as vSQL has some protections built into it against SQL injection. +

+ There is no vSQLMap, so a lot of manual effort went into getting a working injection: +

+ Once we got it working, we decided to script it out to make it easier. We uploaded a gist of it on Github here: +

+ If anyone is interested in Vertica SQL injection, I highly recommend checking out their SQL docs. There are some interesting functions that could be leveraged to take the injection further, e.g. +

+ If configured with AWS keys, you can use the SQL injection to pull AWS secret keys off of the server. In our case, this wasn’t configured for AWS so we were not able to do that. +

+ We had enough information to report the SQL injection at this point. We decided to explore the "/gsf/" API a bit more as we figured they might ACL off this host and it would no longer be public-facing. +

+ It was quickly apparent that the GSF API had access to the “GSF” module that exposed a lot of information about GSF applets. This included some API endpoints for pulling cluster data, application data, and possibly even deploying new clusters and applications. +

+ We speculate at this point we would have been able to deploy internal APIs to the public-facing "/gsf/ "in this cluster giving us access to sensitive data. However, we didn’t prove it out due to the risk. We reported it and stopped here. +

+ Overall, an attacker could've abused this to... +

  • Likely compromise the various internal applications via the publicly exposed GSF portal
  • Execute arbitrary Vertica SQL queries and extract database information

+ Various IDOR Vulnerabilities +

+ Throughout testing on Apple we discovered a variety of IDORs affecting different parts of Apple. The first one was found within the app store connect application that was used to manage apps on the app store. +

+ App Store Connect +

+ After signing up for the developer service, the first thing we did was explore the App Store Connect application which let developers manage their apps that they had or planned to release to the app store. +

+ Hidden behind a few hyperlinks from the settings page was a setting to enable the Game Center for the application. This would allow you to create leader-boards and manage locales for your application. If you enabled this, you were redirected to a more older looking page which used a new set of identifiers to manage the new game center/locale settings you can add to your app. +

+ There was an "itemId" parameter being sent in the URL which was a numeric value that defined which app settings you were modifying. By modifying the number, we could access and modify the leader-boards of any app. This would allow an attacker to deface or remove entirely the game center settings from the app. +

+ Overall, an attacker could've abused this to... +

  • View and modify metadata of any apps on the app store
  • Change data within any application's Game Center information page

+ iCloud Find my Friends IDOR +

+ The iCloud service has a functionality where parents can create and manage child accounts through their primarily Apple account. This behavior was super interesting because of the parent/child relationship and permission model within the application. +

+ If a parent created or added a child account they would have immediate access to perform certain actions against the sub-account like checking the location of the child’s device, limiting device usage, and viewing stored photos and videos. The user management was primarily done through the iOS app which you weren’t able to intercept without finding an SSL pinning bypass, so we decided to look at the other applications like Find my Friends and Photos which integrated sub-functionality for the parent/child relationship. +

+ There were functionalities under “Find my Friends” where you could select your family members then click “Share My Location” and, since it was a trusted relationship between family members, immediately share your location with the family member without them having to accept the request and without the ability to remove your shared presence from their app. Luckily enough for us, this HTTP request to perform the action was interceptable and we could see what the arguments looked like. +

+ The “dsIds” JSON parameter was used as an array of user IDs to share your location. Within the HTTP response, the family members email was returned. I went ahead and modified this value to another user’s ID and to my surprise received their email. +

+ This IDOR would allow us to enumerate the core identifier for Apple accounts to retrieve customer emails and irrevocably share our location with the victim user IDs in which we could send notifications and request access to their location. Since the parameter was sent via a JSON array, you could request hundreds of identifiers at a time and easily enumerate a massive amount of user IDs belonging to Apple customers. +

+ Overall, an attacker could've abused this to... +

  • Retrieve any Apple users email address via an incremental numeric identifier permanently tied to their account
  • Associate the attacker's Apple account with the victim's so that the attacker can send them notifications, show their own location within the victim's phone, and not be deleted from their Find my Friends page

+ Support Case IDOR +

+ One of the more challenging parts of figuring out what to hack on was intercepting the iOS HTTP traffic. There were a lot of really interesting apps and APIs on the actual device, but many of the domains belonging to Apple were SSL pinned and none of us had a strong enough mobile background nor the significant amount of time required to pick apart the actual iOS device. +

+ People have achieved this in the past and been able to intercept all of the HTTP traffic, but luckily for us, a huge portion of the traffic was still interceptable within various apps if you set up your proxy in a certain way. +

+ The way in which we did this was setting up the Burp proxy, installing the certificate, then connecting to the WiFi which had the Burp Proxy enabled whenever we got to a page that we wanted to try to intercept. An example of this would be the failure to load the core App Store while proxying all HTTP requests, but ability to load the app store while not proxying, navigating to the correct sub-page you want to intercept, then enabling the proxy at that point. +

+ This allowed us to capture many API calls for the Apple owned apps that were installed by default on the iPhone. One of these was a support app for scheduling support or speaking with a live chat agent. +

+ From intercepting these, there were a few very obvious IDORs from multiple endpoints that would reveal metadata about support case details. You were able to retrieve information about the victim’s support case details (what support they had requested, their device serial number, their user ID) and additionally a token that appeared to be used when requesting live chat with support agents. +

+ Overall, an attacker could've abused this to... +

  • Leak support case metadata like device serial number and support information for all apple support cases

+ IDOR on mfi.apple.com +

+ Another application that we spent a lot of time on was “mfi.apple.com”. This service was designed for employees of companies that produced third party electronic accessories that interfaced with the iPhone to retrieve documentation and support for their manufacturing process. +

+ After filling out the application, we observed an HTTP request being sent to “getReview.action” with a numeric ID parameter. We went ahead and incremented the parameter via minus one and observed we could access another company's application. +

+ The application returned nearly every value provided for the company application including names, emails, addresses, and invitation keys which could be used to join the company. A simple estimate based on our most recent ID and the base ID indicated around 50,000 different retrievable applications through this vulnerability. +

+ Overall, an attacker could've abused this to... +

  • Leak the entire account information for anyone who has applied to use Apple's MFi portal

+ Various Blind XSS Vulnerabilities +

+ With nearly every application encountered we made sure to spray as many blind XSS payloads as possible. This lead to some very interesting vulnerabilities affecting applications like... +

  • Employee session access within an internal app for managing Apple Maps address information
  • Employee session access within an internal app for managing Apple Books publisher information
  • Employee session access within an internal app for managing Apple Store and customer support tickets

+ These findings were very typical blind XSS as we'd found them by submitting payloads within an address field, Apple Books book submission title, and lastly our first and last name. +

+ The internal applications were very interesting and all appeared to have a comfortable level of access since they fired within the context of Apple employee management tools. We were able to act on the behalf of someone who was expected to be logged in from either a VPN or an on-site location to manage user and system information. +

+ The following screenshots show the redacted panels that we were able to exfiltrate via HTML5 DOM screenshots through XSS hunter: +

+ Since the applications were internal and we weren't actual attackers we stopped here at each finding. These applications would've allowed an attacker to at the very least exfiltrate a large amount of sensitive information regarding internal Apple logistics and employees/users. +

+ Conclusion +

+ When we first started this project we had no idea we'd spend a little bit over three months working towards it's completion. This was originally meant to be a side project that we'd work on every once in a while, but with all of the extra free time with the pandemic we each ended up putting a few hundred hours into it. +

+ Overall, Apple was very responsive to our reports. The turn around for our more critical reports was only four hours between time of submission and time of remediation. +

+ Since no-one really knew much about their bug bounty program, we were pretty much going into unchartered territory with such a large time investment. Apple has had an interesting history working with security researchers, but it appears that their vulnerability disclosure program is a massive step in the right direction to working with hackers in securing assets and allowing those interested to find and report vulnerabilities. +

+ Writing this blog post has been an interesting process as we were a bit unsure how to actually go about doing it. To be honest, each bug we found could've probably been turned into a full writeup with how much random information there was. The authentication system Apple uses was fairly complex and to reference it with 1-2 sentences felt as if we were cheating someone out of information. The same thing could be said about many elements within Apple's infrastructure like iCloud, the Apple store, and the Developer platform. +

+ As of now, October 8th, we have received 32 payments totaling $288,500 for various vulnerabilities. +

+ However, it appears that Apple does payments in batches and will likely pay for more of the issues in the following months. +

+ We've obtained permission from the Apple security team (product-security@apple.com) to publish this and are doing so under their discretion. All of the vulnerabilities disclosed here have been fixed and re-tested. Please do not disclose information pertaining to Apple's security without their permission. +

+ Supporting Resources +

+ Addendum +

+ Huge thanks to the following people: +

  • Kane Gamble
  • Nick Wright (@FaIIenGhouI) for the cover photo
  • Jack Cable
  • Dan Ritter
  • Nathanial Lattimer
  • Jasraj Bedi
  • Justin Rhinehart
  • Everyone on the Apple product security team who have been incredibly responsive and involved with our submissions

+ If you made it this far, thanks for reading! Feel free to reach out with any questions or comments at admin(at)samcurry.net or using the contact form. +

\ No newline at end of file diff --git a/packages/readabilityjs/test/test-pages/sciencedirect/distiller.html b/packages/readabilityjs/test/test-pages/sciencedirect/distiller.html new file mode 100644 index 000000000..864b5a0bc --- /dev/null +++ b/packages/readabilityjs/test/test-pages/sciencedirect/distiller.html @@ -0,0 +1,9 @@ +
Elsevier

+ Abstract +

+ The “Weak Garden of Eden” model for the origin and dispersal of modern humans (Harpendinget al., 1993) posits that modern humans spread into separate regions from a restricted source, around 100 ka (thousand years ago), then passed through population bottlenecks. Around 50 ka, dramatic growth occurred within dispersed populations that were genetically isolated from each other. Population growth began earliest in Africa and later in Eurasia and is hypothesized to have been caused by the invention and spread of a more efficient Later Stone Age/Upper Paleolithic technology, which developed in equatorial Africa. +

+ Climatic and geological evidence suggest an alternative hypothesis for Late Pleistocene population bottlenecks and releases. The last glacial period was preceded by one thousand years of the coldest temperatures of the Later Pleistocene (∼71–70 ka), apparently caused by the eruption of Toba, Sumatra. Toba was the largest known explosive eruption of the Quaternary. Toba's volcanic winter could have decimated most modern human populations, especially outside of isolated tropical refugia. Release from the bottleneck could have occurred either at the end of this hypercold phase, or 10,000 years later, at the transition from cold oxygen isotope stage 4 to warmer stage 3. The largest populations surviving through the bottleneck should have been found in the largest tropical refugia, and thus in equatorial Africa. High genetic diversity in modern Africans may thus reflect a less severe bottleneck rather than earlier population growth. +

+ Volcanic winter may have reduced populations to levels low enough for founder effects, genetic drift and local adaptations to produce rapid population differentiation. If Toba caused the bottlenecks, then modern human races may have differentiated abruptly, only 70 thousand years ago. +

\ No newline at end of file diff --git a/packages/readabilityjs/test/test-pages/slowboring/distiller.html b/packages/readabilityjs/test/test-pages/slowboring/distiller.html new file mode 100644 index 000000000..f321d5d95 --- /dev/null +++ b/packages/readabilityjs/test/test-pages/slowboring/distiller.html @@ -0,0 +1,23 @@ +

+ Because conservatives agree with Musk about the importance of making social media a welcoming place for transphobic jokes, they haven’t been nearly as quick to condemn him telling the Financial Times that Taiwan ought to become a Beijing-ruled Special Administrative Region. But when it comes to a random celebrity like John Cena, conservatives understand the basic dynamic perfectly well — money talks. +

+ But this applies to corporate entities like Apple and Mercedes-Benz even more than it does to individuals like Cena. An individual always has the option of voluntarily earning less money in order to take a principled stand on human rights. If Apple executives started “doing the right thing” in a way that seriously impaired the company’s stock price, they’d be vulnerable to activist investor campaigns and potentially even litigation. +

+ This strikes me as a fairly profound policy problem. Part of the promise of the old bipartisan consensus in favor of trade with China was that economic integration would help spur the export of American speech norms. Not only has that not worked out, but the practical consequences have been the opposite. I’m sure that if you asked Intel executives whether it’s good that China is perpetrating a genocide and using forced labor in Xinjiang, the vast majority of them would say, in private, that it’s actually bad. But in practice, Intel has apologized for past statements about Xinjiang. Because for all the virtues of capitalism and the for-profit business corporation, it also has at its core an amorality that can produce really bad results when it interfaces with a large despotic regime. +

+ The global electronics industry is famously exposed to the China market. But as we saw with Mercedes Benz, China is a really big deal for automakers as well. Tesla is setting sales records in China, which is obviously only possible because the Chinese government lets Tesla sell cars there. +

+ Like many global manufacturing companies, Tesla also builds things in China — including what Musk projects will be the company’s largest factory in the world. +

+ There’s nothing particularly untoward or suspect about this. Many more people live in East Asia than live in North America, so it’s natural that over the long term any manufacturer of large objects is going to want to have their biggest production facilities in Asia. +

+ But Tesla has received unusually generous treatment from the Chinese government. Part of China’s strategy for industrializing has been to say to foreign companies operating in key sectors that if they want to be in the China market, they need to operate as a joint venture with a Chinese-owned firm. It’s become pretty clear that this is a mechanism for Chinese companies to learn foreign technology and business methods, and ultimately pull Chinese-owned companies up the value chain. Foreign companies don’t like it, but in most cases the lure of the China market is just too good to resist. Telsa, though, managed to become the first automaker to receive an exemption from the joint venture requirement. +

+ That’s a huge coup for Musk and a testament both to the quality of Tesla’s products and also to Musk’s shrewdness and skill as a businessman. But without being a hater, I have to say that “shrewdness and skill at making Xi Jinping like you and want to give you favorable treatment” is a mixed ethical bag. The New York Times is blocked in China as part of their censorship regime. So is Substack. It would be better for their businesses if A.G. Sulzberger and Chris Best were as skilled as Musk at getting Xi to do special favors for their companies. But achieving that goal would require them to fundamentally compromise the integrity of editorial businesses in a way that I think would outweigh the benefits. More fundamentally, as a non-corporate person, I would be thrilled to have Chinese readers but also profoundly embarrassed if this site managed to become popular in China without getting censored. That would suggest I’m doing a really bad job of standing up for core values that Americans on the left and right broadly agree on. +

+ Ten years ago, Tesla was a very small company that Mitt Romney denounced as the kind of “loser” that was dependent on unwise subsidies from the Obama administration. At the time, liberals were generally enthusiastic about the idea of an electric vehicle startup. And to the extent that Musk had any known political views, he seemed like a pretty mainstream Democrat. +

+ He also had very normal geopolitical opinions like “foreign dictatorships are bad” that a person who doesn’t have extensive Chinese business interests is free to express on Twitter (which, of course, is banned in China). +

+ But as Musk’s business grew, he became more China-friendly in his public statements. Initially this took the relatively sensible form adopted by many who need to come up with something nice to say about the PRC: praise for their extensive infrastructure buildout. This is something I’ve done myself — it’s incredibly impressive that China built mid-sized metro systems in a couple of dozen cities over the course of time that it took the United States to build three miles of subway on the Upper East Side. +

\ No newline at end of file diff --git a/packages/readabilityjs/test/test-pages/spakhm/distiller.html b/packages/readabilityjs/test/test-pages/spakhm/distiller.html new file mode 100644 index 000000000..c7a752649 --- /dev/null +++ b/packages/readabilityjs/test/test-pages/spakhm/distiller.html @@ -0,0 +1,67 @@ +

+ What career advice do you get as an employee of a growing company? Do great work, keep yourself challenged, focus on learning at every step, find mentors, mentor others, crush your OKRs. Every few quarters you go through a performance review that identifies your strengths and weaknesses, and sets you up for learning and growth. Every week you have one on ones with your boss where you get continuous course corrections on your performance. +

+ But it's easy to see it's mostly social fiction. Following the overt mechanisms of advancement is neither sufficient nor necessary. It may even be actively harmful— almost everyone who does great work and takes performance reviews seriously toils in relative obscurity. Look at the people who occupy positions of considerable authority in your organization. Did any of them get there by following some middle manager's feedback? +

+ In college I had friends who studied biology, so I hung out in the life sciences building a lot. They had a beautiful little forest nearby. We'd often go on walks there and enjoy the serenity. One time I went with a trained botanist. Over the next hour, as she talked about what she was seeing, the forest transformed from an idyllic little haven to a savage battlefield. Trees would fight their enemies with ruthless efficiency for every square inch of precious access to the sun. Bushes would be relegated to the leftovers, and moss and grass on the forest floor were forced to subsist on even less sunlight than that. Young, light-green plants would tirelessly invade the territory of their older, dark-green rivals, never ceasing the effort to starve them of energy. Seeds and spores would invent ingenious mechanisms to cover a wider and wider area than their opponents. +

+ At first, when you start working at a rapidly growing company, what you see is smart, idealistic, driven people working together to accomplish a goal greater than themselves. When you leave, unless you are willfully blind or exceptionally naive, what you see is a ruthless political arena— a modern day Game of Thrones, where machinations take place over email, and battles are won and lost over cups of light roast coffee. +

+ Opportunists +

+ What are the dynamics of this arena invisible to an untrained eye? The first thing to realize is that the coveted advantages of a rapidly growing company— money, growth, publicity, status— come with a trade off. Precisely because the company has all these things, it will attract people who seek them, and the more successful the company, the stronger the attraction. +

+ Don't all of us seek wealth and status? Yes, but it doesn't always manifest in quite the same way. A hacker who works at a unicorn and contributes patches to xmonad in his spare time may want wealth and status, but he also has firmly entrenched and far-reaching principles. He may care about his text editor, or his programming language, or the API naming scheme. His principles may be advantageous, or silly, or counterproductive. But he has them. +

+ The kinds of opportunists who are attracted solely to wealth and status have no principles at all beyond accumulation of these two objects. It isn't that they don't have taste or good judgement. They do— that's why they got hired in the first place. But if they ever had a compulsion to express their sense of taste, it's long ago been subordinated to their primary and only concern: climbing the corporate ladder. +

+ When you first encounter this mode of being, it may be so far outside of your normal range of experience you'll have trouble processing it. Marx thought that to be fulfilled, humans must feel a connection to the end result of their work. For example, a carpenter feels satisfaction when he finishes a chair or a table. But in an industrialized society people no longer feel this connection, which robs them of the fulfillment. He called this phenomenon "estranged labor". One way to think about people who are attracted purely to wealth and status is that under these same conditions they don't feel estranged. They've either eradicated this feeling in themselves long ago, or never felt it in the first place. +

+ Can you build a successful organization that keeps the unprincipled out? No. As the company grows more successful, so does the allure. The organization starts getting constantly bombarded by world class actors who specialize in slipping past the founders's defenses. And since at higher rungs much of the job is recruiting, the new hire becomes a Trojan horse. As soon as they're in, they open the door to dozens of cronies who diffuse into the company. It's like putting a drop of ink in a glass of water-- there is no undo. +

+ Exploit +

+ How would you advance through the hierarchy if advancement were your only principle? You'd find vulnerabilities in the system and attack them, just like a black hat hacker would. Ideally the attack vectors would be inherent to a growing company— something that can't be patched. One such vulnerability is attracting opportunists. You could try to patch it, but ultimately it's inseparable from success. What are the others? +

+ Here is one of my favorite quotes by Jeff Bezos because of how it inadvertently captures an inherent vulnerability of growing companies: +

+ When somebody congratulates Amazon on a good quarter, I say thank you. But what I'm thinking to myself is— those quarterly results were actually pretty much fully baked about 3 years ago. Today I'm working on a quarter that is going to happen in 2020. Not next quarter. Next quarter for all practical purposes is done already and it has probably been done for a couple of years." +

+ — Jeff Bezos, 2017 +

+ The key to corporate opportunism is all there, in this quote. When a normal person reads it, he thinks "wow! Amazon is really thinking long term!" which is perhaps how Jeff intended it. But when an opportunist reads it, he thinks "wow! I can do anything I want for three years and it won't show up in the metrics!" +

+ Company metrics have momentum and lag. Nearly all political behavior exploits these two properties. A metric in motion tends to remain in motion. Changing that requires good decisions, a lot of luck, and application of enormous force. And observing a metric is like observing light from a distant star— you're observing the work done in the past. So opportunists don't worry about any of that. The winning strategy is to ignore company metrics completely and move between projects every eighteen months so that nobody notices. +

+ Wouldn't people notice anyway? Rank and file employees will, but not the management. In a fast growing company things change very quickly. There is a hurricane of activity. New projects and teams are constantly spun up, products launched, reorgs, valuations, office moves, new hires, funding rounds, customers lost and gained, offsites, PR victories and scandals— nobody can keep track of all of this. You can't remember what things looked like two weeks ago, let alone last quarter. Two years ago? Forget it! So long as you looked good at the time, nobody will remember what you did in your last role. And even if they do, people tend to attribute everything to the present. If you screwed up a project, switched roles, and it's only now apparent that the project is in trouble, the blame will naturally fall on its current leader, not on you. +

+ So your job isn't to make good decisions to improve company metrics. It's to look good. That means you must understand basic tenets of human nature and learn to manipulate them. +

+ Rituals +

+ First, the fact that metrics have momentum means the company is always in one of two modes. When the metrics are up and to the right, the company is in peace mode. When they start to stall, the company is in war mode. A rapidly growing company is by definition in peace mode, and you must act the appropriate part. When the money is flowing, everything is expanding and everyone is happy, nobody wants to hear doomsayers, even if the prophecies will eventually come true. It doesn't matter if the company "doesn't have a culture of blame"— bad news always implies there is someone to blame. So in the upper echelons the flow of negative information is controlled extremely carefully. +

+ Of course you can't seem like you never deliver bad news either. It's a matter of packaging. Bad news is never individualized. If you ever carelessly imply that some team may be doing something poorly, you will make yourself a target for every opportunist in the company. Instead, bad news is always delivered in general terms and always as a matter of slight adjustments to a continuous variable, never as a binary switch. +

+ For example, "the codebase is such a mess this team can't ship anything until we spend three months refactoring" is really bad delivery. An opportunist would say "we need to work toward paying off the technical debt". See? Codebase quality is just another analog knob to tweak. No big deal— it's something we should be doing in the future. And it's the collective "we". Nobody in particular is experiencing this problem. +

+ Second, you must understand that KPIs, OKRs, meetings, and company values are just performative rituals. Don't naively assume they contain actionable information. Look carefully— no successful opportunist takes them at face value. If you do take these rituals at face value, you will interfere with other people's reputation, career plans, and flow of money. That's unwise, because the powers that be will start getting negative feedback about you from multiple seemingly unrelated sources. That's not conducive to a successful career. It also explains why following traditional career advice rarely gets you promoted beyond middle management. Most good work doesn't look good. Sometimes it interferes with other people and tanks your career. But mostly it doesn't look bad either. It just doesn't look like much of anything. +

+ Third, while you shouldn't take performative rituals at face value, you must still perform them— enthusiastically and with gusto. In countries like Japan, Russia, and parts of Eastern Europe, to successfully work with someone you must drink with them. Drinking is a trust building ritual and a mechanism to signal that you are part of the in-group. No drinking, no promotion. In corporate America the lion's share of this signaling is done through proselytizing management technologies. Going to meetings, talking about KPIs, OKRs, collaboration, Agile— all are shibboleth to signal you're part of the in-group, in the same way drinking is an in-group signal in other parts of the world. No shibboleth, no promotion. +

+ Covert +

+ So far we've talked about how you should act, but what should you actually do? Since the overt objectives are merely performative, you need to determine your actual objectives. Fortunately they're the same in every growing company, so you don't need to do any detective work to discover what they are. Your primary objective is simple— headcount growth ("do a lot with a little" is another shibboleth, don't take it literally). Figure out the baseline headcount growth rate for the company, and grow your team at least as much. Any less, and you'll be left in the dust. More is better— superstars always grow their headcount considerably faster than baseline. +

+ Don't worry about what all these people will work on. The devil will conveniently find work for idle hands. Expanding companies never have a shortage of business opportunities, projects, and tasks. The work will invent itself. Your job isn't the work— it's to grow headcount and make it appear well-managed. That latter part is nearly entirely performative. Look at other teams that are considered well-managed and emulate the trappings. Usually that means lots of meetings, cross-functional collaboration, document writing, one on ones, performance reviews, and agile. You will need to ship things, but not that much. By the time any of this gets objectively evaluated, you'll be happily working in a different role, and someone else will deal with the objective metrics. +

+ While nearly all your objectives are performative (feigning excitement, generating meetings and documents) or covert (headcount growth), there is one objective you must take at face value. You cannot allow acute failures in your area of responsibility. Low product quality is fine because you can always use the analog knob trick for that. "Latency spikes are an important issue, we need to budget engineering time to lower spike incidence." Then have a team work on latency for two quarters. But if you get a couple of major security breaches, you're done. This is why there are fewer opportunists in sensitive areas like security and infrastructure. +

+ Before I puzzled all this out, I spent years directly asking folks for their reasoning. "Why did you decide to double the team, don't you see it will only make us slower? Why schedule all these meetings, don't you see they have negative utility? And why simultaneously talk about needing fewer meetings?" Every time I asked, the questions were met with bewilderment. "Of course we need to schedule meetings. And of course we need fewer of them. What's unclear about that?" +

+ At first I thought the bewilderment was feigned. But over time, as I asked more and more people and always got the same answer, I decided it isn't feigned at all. Corporate apparatchiks are completely genuine when they want to reduce meeting load as they schedule more meetings. It's easy to hold contradictory beliefs as long as your paycheck depends on them, so most people learn to compartmentalize. Feigning sincerity in a convincing way is extremely difficult. So if you take any of this career advice, it's best to learn to candidly believe it. +

+ TL;DR: +

+ In summary, an opportunist's career advice is: ignore OKRs, switch projects well before the consequences of your decisions can be measured, act happy and easy-going, package bad news as appeals for slow systemic adjustments, don't make anyone look bad, perform rituals with enthusiasm, grow headcount faster than baseline, let work invent itself, follow management fashions, avoid acute failures, believe this sincerely. +

\ No newline at end of file diff --git a/packages/readabilityjs/test/test-pages/stackoverflow/distiller.html b/packages/readabilityjs/test/test-pages/stackoverflow/distiller.html new file mode 100644 index 000000000..ba551f68b --- /dev/null +++ b/packages/readabilityjs/test/test-pages/stackoverflow/distiller.html @@ -0,0 +1,1182 @@ +

+ What is the use of the yield keyword in Python? What does it do? +

+ For example, I'm trying to understand this code1: + 

def _get_child_candidates(self, distance, min_dist, max_dist):
+    if self._leftchild and distance - max_dist < self._median:
+        yield self._leftchild
+    if self._rightchild and distance + max_dist >= self._median:
+        yield self._rightchild  
+

+ And this is the caller: + 

result, candidates = [], [self]
+while candidates:
+    node = candidates.pop()
+    distance = node._get_dist(obj)
+    if distance <= max_dist and distance >= min_dist:
+        result.extend(node._values)
+    candidates.extend(node._get_child_candidates(distance, min_dist, max_dist))
+return result
+

+ What happens when the method _get_child_candidates is called? Is a list returned? A single element? Is it called again? When will subsequent calls stop? +

+ 1. This piece of code was written by Jochen Schulz (jrschulz), who made a great Python library for metric spaces. This is the link to the complete source: Module mspace. +
Kai - Kazuya Ito's user avatar
9 + +
+ This answer is useful + +
+ 2426 +
+ This answer is not useful + +
+ Save this answer. + +
+ Show activity on this post. + +

+ Shortcut to understanding yield +

+ When you see a function with yield statements, apply this easy trick to understand what will happen: +

  1. Insert a line result = [] at the start of the function.
  2. Replace each yield expr with result.append(expr).
  3. Insert a line return result at the bottom of the function.
  4. Yay - no more yield statements! Read and figure out the code.
  5. Compare function to the original definition.

+ This trick may give you an idea of the logic behind the function, but what actually happens with yield is significantly different than what happens in the list-based approach. In many cases, the yield approach will be a lot more memory efficient and faster too. In other cases, this trick will get you stuck in an infinite loop, even though the original function works just fine. Read on to learn more... +

+ Don't confuse your Iterables, Iterators, and Generators +

+ First, the iterator protocol - when you write + 

for x in mylist:
+    ...loop body...
+

+ Python performs the following two steps: +

  1. + Gets an iterator for mylist: +

    +

    + Call iter(mylist) -> this returns an object with a next() method (or __next__() in Python 3). +

    +

    + [This is the step most people forget to tell you about] +

  2. + Uses the iterator to loop over items: +

    +

    + Keep calling the next() method on the iterator returned from step 1. The return value from next() is assigned to x and the loop body is executed. If an exception StopIteration is raised from within next(), it means there are no more values in the iterator and the loop is exited. +

+ The truth is Python performs the above two steps anytime it wants to loop over the contents of an object - so it could be a for loop, but it could also be code like otherlist.extend(mylist) (where otherlist is a Python list). +

+ Here mylist is an iterable because it implements the iterator protocol. In a user-defined class, you can implement the __iter__() method to make instances of your class iterable. This method should return an iterator. An iterator is an object with a next() method. It is possible to implement both __iter__() and next() on the same class, and have __iter__() return self. This will work for simple cases, but not when you want two iterators looping over the same object at the same time. +

+ So that's the iterator protocol, many objects implement this protocol: +

  1. Built-in lists, dictionaries, tuples, sets, and files.
  2. User-defined classes that implement __iter__().
  3. Generators.

+ Note that a for loop doesn't know what kind of object it's dealing with - it just follows the iterator protocol, and is happy to get item after item as it calls next(). Built-in lists return their items one by one, dictionaries return the keys one by one, files return the lines one by one, etc. And generators return... well that's where yield comes in: + 

def f123():
+    yield 1
+    yield 2
+    yield 3
+
+for item in f123():
+    print item
+

+ Instead of yield statements, if you had three return statements in f123() only the first would get executed, and the function would exit. But f123() is no ordinary function. When f123() is called, it does not return any of the values in the yield statements! It returns a generator object. Also, the function does not really exit - it goes into a suspended state. When the for loop tries to loop over the generator object, the function resumes from its suspended state at the very next line after the yield it previously returned from, executes the next line of code, in this case, a yield statement, and returns that as the next item. This happens until the function exits, at which point the generator raises StopIteration, and the loop exits. +

+ So the generator object is sort of like an adapter - at one end it exhibits the iterator protocol, by exposing __iter__() and next() methods to keep the for loop happy. At the other end, however, it runs the function just enough to get the next value out of it, and puts it back in suspended mode. +

+ Why Use Generators? +

+ Usually, you can write code that doesn't use generators but implements the same logic. One option is to use the temporary list 'trick' I mentioned before. That will not work in all cases, for e.g. if you have infinite loops, or it may make inefficient use of memory when you have a really long list. The other approach is to implement a new iterable class SomethingIter that keeps the state in instance members and performs the next logical step in its next() (or __next__() in Python 3) method. Depending on the logic, the code inside the next() method may end up looking very complex and prone to bugs. Here generators provide a clean and easy solution. +

3 + +
+ This answer is useful + +
+ 720 +
+ This answer is not useful + +
+ Save this answer. + +
+ Show activity on this post. + +

+ Think of it this way: +

+ An iterator is just a fancy sounding term for an object that has a next() method. So a yield-ed function ends up being something like this: +

+ Original version: + 

def some_function():
+    for i in xrange(4):
+        yield i
+
+for i in some_function():
+    print i
+

+ This is basically what the Python interpreter does with the above code: + 

class it:
+    def __init__(self):
+        
+        self.count = -1
+
+    
+    
+    
+    def __iter__(self):
+        return self
+
+    
+    
+    def next(self):
+        self.count += 1
+        if self.count < 4:
+            return self.count
+        else:
+            
+            
+            
+            raise StopIteration
+
+def some_func():
+    return it()
+
+for i in some_func():
+    print i
+

+ For more insight as to what's happening behind the scenes, the for loop can be rewritten to this: + 

iterator = some_func()
+try:
+    while 1:
+        print iterator.next()
+except StopIteration:
+    pass
+

+ Does that make more sense or just confuse you more? :) +

+ I should note that this is an oversimplification for illustrative purposes. :) +

4 + +
+ This answer is useful + +
+ 610 +
+ This answer is not useful + +
+ Save this answer. + +
+ Show activity on this post. + +

+ The yield keyword is reduced to two simple facts: +

  1. If the compiler detects the yield keyword anywhere inside a function, that function no longer returns via the return statement. Instead, it immediately returns a lazy "pending list" object called a generator
  2. A generator is iterable. What is an iterable? It's anything like a list or set or range or dict-view, with a built-in protocol for visiting each element in a certain order.

+ In a nutshell: Most commonly, a generator is a lazy, incrementally-pending list, and yield statements allow you to use function notation to program the list values the generator should incrementally spit out. Furthermore, advanced usage lets you use generators as coroutines (see below). + 

generator = myYieldingFunction(...)  
+x = list(generator)  
+
+   generator
+       v
+[x[0], ..., ???]
+
+         generator
+             v
+[x[0], x[1], ..., ???]
+
+               generator
+                   v
+[x[0], x[1], x[2], ..., ???]
+
+                       StopIteration exception
+[x[0], x[1], x[2]]     done
+

+ Basically, whenever the yield statement is encountered, the function pauses and saves its state, then emits "the next return value in the 'list'" according to the python iterator protocol (to some syntactic construct like a for-loop that repeatedly calls next() and catches a StopIteration exception, etc.). You might have encountered generators with generator expressions; generator functions are more powerful because you can pass arguments back into the paused generator function, using them to implement coroutines. More on that later. +

+ Basic Example ('list') +

+ Let's define a function makeRange that's just like Python's range. Calling makeRange(n) RETURNS A GENERATOR: + 

def makeRange(n):
+    
+    i = 0
+    while i < n:
+        yield i
+        i += 1
+
+>>> makeRange(5)
+<generator object makeRange at 0x19e4aa0>
+

+ To force the generator to immediately return its pending values, you can pass it into list() (just like you could any iterable): + 

>>> list(makeRange(5))
+[0, 1, 2, 3, 4]
+

+ Comparing example to "just returning a list" +

+ The above example can be thought of as merely creating a list which you append to and return: + 


+def makeRange(n):                
+    """return [0,1,2,...,n-1]""" 
+    TO_RETURN = []               
+    i = 0                        
+    while i < n:                 
+        TO_RETURN += [i]         
+        i += 1                   
+    return TO_RETURN             
+
+>>> makeRange(5)
+[0, 1, 2, 3, 4]
+

+ There is one major difference, though; see the last section. +

+ How you might use generators +

+ An iterable is the last part of a list comprehension, and all generators are iterable, so they're often used like so: + 


+>>> [x+10 for x in makeRange(5)]
+[10, 11, 12, 13, 14]
+

+ To get a better feel for generators, you can play around with the itertools module (be sure to use chain.from_iterable rather than chain when warranted). For example, you might even use generators to implement infinitely-long lazy lists like itertools.count(). You could implement your own def enumerate(iterable): zip(count(), iterable), or alternatively do so with the yield keyword in a while-loop. +

+ Please note: generators can actually be used for many more things, such as implementing coroutines or non-deterministic programming or other elegant things. However, the "lazy lists" viewpoint I present here is the most common use you will find. +

+ Behind the scenes +

+ This is how the "Python iteration protocol" works. That is, what is going on when you do list(makeRange(5)). This is what I describe earlier as a "lazy, incremental list". + 

>>> x=iter(range(5))
+>>> next(x)  
+0
+>>> next(x)
+1
+>>> next(x)
+2
+>>> next(x)
+3
+>>> next(x)
+4
+>>> next(x)
+Traceback (most recent call last):
+  File "<stdin>", line 1, in <module>
+StopIteration
+

+ The built-in function next() just calls the objects .__next__() function, which is a part of the "iteration protocol" and is found on all iterators. You can manually use the next() function (and other parts of the iteration protocol) to implement fancy things, usually at the expense of readability, so try to avoid doing that... +

+ Coroutines +

+ Coroutine example: + 

def interactiveProcedure():
+    userResponse = yield makeQuestionWebpage()
+    print('user response:', userResponse)
+    yield 'success'
+
+coroutine = interactiveProcedure()
+webFormData = next(coroutine)  
+userResponse = serveWebForm(webFormData)
+
+
+
+successStatus = coroutine.send(userResponse)
+

+ A coroutine (generators which generally accept input via the yield keyword e.g. nextInput = yield nextOutput, as a form of two-way communication) is basically a computation which is allowed to pause itself and request input (e.g. to what it should do next). When the coroutine pauses itself (when the running coroutine eventually hits a yield keyword), the computation is paused and control is inverted (yielded) back to the 'calling' function (the frame which requested the next value of the computation). The paused generator/coroutine remains paused until another invoking function (possibly a different function/context) requests the next value to unpause it (usually passing input data to direct the paused logic interior to the coroutine's code). +

+ You can think of python coroutines as lazy incrementally-pending lists, where the next element doesn't just depend on the previous computation, but also on input you may opt to inject during the generation process. +

+ Minutiae +

+ Normally, most people would not care about the following distinctions and probably want to stop reading here. +

+ In Python-speak, an iterable is any object which "understands the concept of a for-loop" like a list [1,2,3], and an iterator is a specific instance of the requested for-loop like [1,2,3].__iter__(). A generator is exactly the same as any iterator, except for the way it was written (with function syntax). +

+ When you request an iterator from a list, it creates a new iterator. However, when you request an iterator from an iterator (which you would rarely do), it just gives you a copy of itself. +

+ Thus, in the unlikely event that you are failing to do something like this... + 

> x = myRange(5)
+> list(x)
+[0, 1, 2, 3, 4]
+> list(x)
+[]
+

+ ... then remember that a generator is an iterator; that is, it is one-time-use. If you want to reuse it, you should call myRange(...) again. If you need to use the result twice, convert the result to a list and store it in a variable x = list(myRange(5)). Those who absolutely need to clone a generator (for example, who are doing terrifyingly hackish metaprogramming) can use itertools.tee (still works in Python 3) if absolutely necessary, since the copyable iterator Python PEP standards proposal has been deferred. +

0 + +
+ This answer is useful + +
+ 527 +
+ This answer is not useful + +
+ Save this answer. + +
+ Show activity on this post. + +

+ What does the yield keyword do in Python? +

+ Answer Outline/Summary +

  • A function with yield, when called, returns a Generator.
  • Generators are iterators because they implement the iterator protocol, so you can iterate over them.
  • A generator can also be sent information, making it conceptually a coroutine.
  • In Python 3, you can delegate from one generator to another in both directions with yield from.
  • (Appendix critiques a couple of answers, including the top one, and discusses the use of return in a generator.)

+ Generators: +

+ yield is only legal inside of a function definition, and the inclusion of yield in a function definition makes it return a generator. +

+ The idea for generators comes from other languages (see footnote 1) with varying implementations. In Python's Generators, the execution of the code is frozen at the point of the yield. When the generator is called (methods are discussed below) execution resumes and then freezes at the next yield. +

+ yield provides an easy way of implementing the iterator protocol, defined by the following two methods: __iter__ and __next__. Both of those methods make an object an iterator that you could type-check with the Iterator Abstract Base Class from the collections module. + 

def func():
+    yield 'I am'
+    yield 'a generator!'
+

+ Let's do some introspection: + 

>>> type(func)                 
+<type 'function'>
+>>> gen = func()
+>>> type(gen)                  
+<type 'generator'>
+>>> hasattr(gen, '__iter__')   
+True
+>>> hasattr(gen, '__next__')   
+True                           
+

+ The generator type is a sub-type of iterator: + 

from types import GeneratorType
+from collections.abc import Iterator
+
+>>> issubclass(GeneratorType, Iterator)
+True
+

+ And if necessary, we can type-check like this: + 

>>> isinstance(gen, GeneratorType)
+True
+>>> isinstance(gen, Iterator)
+True
+

+ A feature of an Iterator is that once exhausted, you can't reuse or reset it: + 

>>> list(gen)
+['I am', 'a generator!']
+>>> list(gen)
+[]
+

+ You'll have to make another if you want to use its functionality again (see footnote 2): + 

>>> list(func())
+['I am', 'a generator!']
+

+ One can yield data programmatically, for example: + 

def func(an_iterable):
+    for item in an_iterable:
+        yield item
+

+ The above simple generator is also equivalent to the below - as of Python 3.3 you can use yield from: + 

def func(an_iterable):
+    yield from an_iterable
+

+ However, yield from also allows for delegation to subgenerators, which will be explained in the following section on cooperative delegation with sub-coroutines. +

+ Coroutines: +

+ yield forms an expression that allows data to be sent into the generator (see footnote 3) +

+ Here is an example, take note of the received variable, which will point to the data that is sent to the generator: + 

def bank_account(deposited, interest_rate):
+    while True:
+        calculated_interest = interest_rate * deposited 
+        received = yield calculated_interest
+        if received:
+            deposited += received
+
+
+>>> my_account = bank_account(1000, .05)
+

+ First, we must queue up the generator with the builtin function, next. It will call the appropriate next or __next__ method, depending on the version of Python you are using: + 

>>> first_year_interest = next(my_account)
+>>> first_year_interest
+50.0
+

+ And now we can send data into the generator. (Sending None is the same as calling next.) : + 

>>> next_year_interest = my_account.send(first_year_interest + 1000)
+>>> next_year_interest
+102.5
+

+ Cooperative Delegation to Sub-Coroutine with yield from +

+ Now, recall that yield from is available in Python 3. This allows us to delegate coroutines to a subcoroutine: + 


+def money_manager(expected_rate):
+    
+    under_management = yield                   
+    while True:
+        try:
+            additional_investment = yield expected_rate * under_management 
+            if additional_investment:
+                under_management += additional_investment
+        except GeneratorExit:
+            '''TODO: write function to send unclaimed funds to state'''
+            raise
+        finally:
+            '''TODO: write function to mail tax info to client'''
+        
+
+def investment_account(deposited, manager):
+    '''very simple model of an investment account that delegates to a manager'''
+    
+    next(manager)      
+    
+    manager.send(deposited)
+    try:
+        yield from manager
+    except GeneratorExit:
+        return manager.close()  
+

+ And now we can delegate functionality to a sub-generator and it can be used by a generator just as above: + 

my_manager = money_manager(.06)
+my_account = investment_account(1000, my_manager)
+first_year_return = next(my_account) 
+

+ Now simulate adding another 1,000 to the account plus the return on the account (60.0): + 

next_year_return = my_account.send(first_year_return + 1000)
+next_year_return 
+

+ You can read more about the precise semantics of yield from in PEP 380. +

+ Other Methods: close and throw +

+ The close method raises GeneratorExit at the point the function execution was frozen. This will also be called by __del__ so you can put any cleanup code where you handle the GeneratorExit: + 

my_account.close()
+

+ You can also throw an exception which can be handled in the generator or propagated back to the user: + 

import sys
+try:
+    raise ValueError
+except:
+    my_manager.throw(*sys.exc_info())
+

+ Raises: + 

Traceback (most recent call last):
+  File "<stdin>", line 4, in <module>
+  File "<stdin>", line 6, in money_manager
+  File "<stdin>", line 2, in <module>
+ValueError
+

+ Conclusion +

+ I believe I have covered all aspects of the following question: +

+ What does the yield keyword do in Python? +

+ It turns out that yield does a lot. I'm sure I could add even more thorough examples to this. If you want more or have some constructive criticism, let me know by commenting below. +

+ Appendix: +

+ Critique of the Top/Accepted Answer** +

  • It is confused on what makes an iterable, just using a list as an example. See my references above, but in summary: an iterable has an __iter__ method returning an iterator. An iterator additionally provides a .__next__ method, which is implicitly called by for loops until it raises StopIteration, and once it does raise StopIteration, it will continue to do so.
  • It then uses a generator expression to describe what a generator is. Since a generator expression is simply a convenient way to create an iterator, it only confuses the matter, and we still have not yet gotten to the yield part.
  • In Controlling a generator exhaustion he calls the .next method (which only works in Python 2), when instead he should use the builtin function, next. Calling next(obj) would be an appropriate layer of indirection, because his code does not work in Python 3.
  • Itertools? This was not relevant to what yield does at all.
  • No discussion of the methods that yield provides along with the new functionality yield from in Python 3.

+ The top/accepted answer is a very incomplete answer. +

+ Critique of answer suggesting yield in a generator expression or comprehension. +

+ The grammar currently allows any expression in a list comprehension. + 

expr_stmt: testlist_star_expr (annassign | augassign (yield_expr|testlist) |
+                     ('=' (yield_expr|testlist_star_expr))*)
+...
+yield_expr: 'yield' [yield_arg]
+yield_arg: 'from' test | testlist
+

+ Since yield is an expression, it has been touted by some as interesting to use it in comprehensions or generator expression - in spite of citing no particularly good use-case. +

+ The CPython core developers are discussing deprecating its allowance. Here's a relevant post from the mailing list: +

+ On 30 January 2017 at 19:05, Brett Cannon wrote: +

+ On Sun, 29 Jan 2017 at 16:39 Craig Rodrigues wrote: +

+ I'm OK with either approach. Leaving things the way they are in Python 3 is no good, IMHO. +

+ My vote is it be a SyntaxError since you're not getting what you expect from the syntax. +

+ I'd agree that's a sensible place for us to end up, as any code relying on the current behaviour is really too clever to be maintainable. +

+ In terms of getting there, we'll likely want: +

  • SyntaxWarning or DeprecationWarning in 3.7
  • Py3k warning in 2.7.x
  • SyntaxError in 3.8

+ Cheers, Nick. +

+ -- Nick Coghlan | ncoghlan at gmail.com | Brisbane, Australia +

+ Further, there is an outstanding issue (10544) which seems to be pointing in the direction of this never being a good idea (PyPy, a Python implementation written in Python, is already raising syntax warnings.) +

+ Bottom line, until the developers of CPython tell us otherwise: Don't put yield in a generator expression or comprehension. +

+ 356k85 gold badges395 silver badges329 bronze badges +
0 + +
+ This answer is useful + +
+ 432 +
+ This answer is not useful + +
+ Save this answer. + +
+ Show activity on this post. + +

+ yield is just like return - it returns whatever you tell it to (as a generator). The difference is that the next time you call the generator, execution starts from the last call to the yield statement. Unlike return, the stack frame is not cleaned up when a yield occurs, however control is transferred back to the caller, so its state will resume the next time the function is called. +

+ In the case of your code, the function get_child_candidates is acting like an iterator so that when you extend your list, it adds one element at a time to the new list. +

+ list.extend calls an iterator until it's exhausted. In the case of the code sample you posted, it would be much clearer to just return a tuple and append that to the list. +

0 + +
+ This answer is useful + +
+ 275 +
+ This answer is not useful + +
+ Save this answer. + +
+ Show activity on this post. + +

+ TL;DR +

+ Instead of this: + 

def square_list(n):
+    the_list = []                         
+    for x in range(n):
+        y = x * x
+        the_list.append(y)                
+    return the_list                       
+

+ do this: + 

def square_yield(n):
+    for x in range(n):
+        y = x * x
+        yield y                           
+

+ Whenever you find yourself building a list from scratch, yield each piece instead. +

+ This was my first "aha" moment with yield. +

+ yield is a sugary way to say +

+ build a series of stuff +

+ Same behavior: + 

>>> for square in square_list(4):
+...     print(square)
+...
+0
+1
+4
+9
+>>> for square in square_yield(4):
+...     print(square)
+...
+0
+1
+4
+9
+

+ Different behavior: +

+ Yield is single-pass: you can only iterate through once. When a function has a yield in it we call it a generator function. And an iterator is what it returns. Those terms are revealing. We lose the convenience of a container, but gain the power of a series that's computed as needed, and arbitrarily long. +

+ Yield is lazy, it puts off computation. A function with a yield in it doesn't actually execute at all when you call it. It returns an iterator object that remembers where it left off. Each time you call next() on the iterator (this happens in a for-loop) execution inches forward to the next yield. return raises StopIteration and ends the series (this is the natural end of a for-loop). +

+ Yield is versatile. Data doesn't have to be stored all together, it can be made available one at a time. It can be infinite. + 

>>> def squares_all_of_them():
+...     x = 0
+...     while True:
+...         yield x * x
+...         x += 1
+...
+>>> squares = squares_all_of_them()
+>>> for _ in range(4):
+...     print(next(squares))
+...
+0
+1
+4
+9
+

+ If you need multiple passes and the series isn't too long, just call list() on it: + 

>>> list(square_yield(4))
+[0, 1, 4, 9]
+

+ Brilliant choice of the word yield because both meanings apply: +

+ yield — produce or provide (as in agriculture) +

+ ...provide the next data in the series. +

+ yield — give way or relinquish (as in political power) +

+ ...relinquish CPU execution until the iterator advances. +

0 + +
+ This answer is useful + +
+ 236 +
+ This answer is not useful + +
+ Save this answer. + +
+ Show activity on this post. + +

+ Yield gives you a generator. + 

def get_odd_numbers(i):
+    return range(1, i, 2)
+def yield_odd_numbers(i):
+    for x in range(1, i, 2):
+       yield x
+foo = get_odd_numbers(10)
+bar = yield_odd_numbers(10)
+foo
+[1, 3, 5, 7, 9]
+bar
+<generator object yield_odd_numbers at 0x1029c6f50>
+bar.next()
+1
+bar.next()
+3
+bar.next()
+5
+

+ As you can see, in the first case foo holds the entire list in memory at once. It's not a big deal for a list with 5 elements, but what if you want a list of 5 million? Not only is this a huge memory eater, it also costs a lot of time to build at the time that the function is called. +

+ In the second case, bar just gives you a generator. A generator is an iterable--which means you can use it in a for loop, etc, but each value can only be accessed once. All the values are also not stored in memory at the same time; the generator object "remembers" where it was in the looping the last time you called it--this way, if you're using an iterable to (say) count to 50 billion, you don't have to count to 50 billion all at once and store the 50 billion numbers to count through. +

+ Again, this is a pretty contrived example, you probably would use itertools if you really wanted to count to 50 billion. :) +

+ This is the most simple use case of generators. As you said, it can be used to write efficient permutations, using yield to push things up through the call stack instead of using some sort of stack variable. Generators can also be used for specialized tree traversal, and all manner of other things. +

1 + +
+ This answer is useful + +
+ 235 +
+ This answer is not useful + +
+ Save this answer. + +
+ Show activity on this post. + +

+ It's returning a generator. I'm not particularly familiar with Python, but I believe it's the same kind of thing as C#'s iterator blocks if you're familiar with those. +

+ The key idea is that the compiler/interpreter/whatever does some trickery so that as far as the caller is concerned, they can keep calling next() and it will keep returning values - as if the generator method was paused. Now obviously you can't really "pause" a method, so the compiler builds a state machine for you to remember where you currently are and what the local variables etc look like. This is much easier than writing an iterator yourself. +

0 + +
+ This answer is useful + +
+ 207 +
+ This answer is not useful + +
+ Save this answer. + +
+ Show activity on this post. + +

+ There is one type of answer that I don't feel has been given yet, among the many great answers that describe how to use generators. Here is the programming language theory answer: +

+ The yield statement in Python returns a generator. A generator in Python is a function that returns continuations (and specifically a type of coroutine, but continuations represent the more general mechanism to understand what is going on). +

+ Continuations in programming languages theory are a much more fundamental kind of computation, but they are not often used, because they are extremely hard to reason about and also very difficult to implement. But the idea of what a continuation is, is straightforward: it is the state of a computation that has not yet finished. In this state, the current values of variables, the operations that have yet to be performed, and so on, are saved. Then at some point later in the program the continuation can be invoked, such that the program's variables are reset to that state and the operations that were saved are carried out. +

+ Continuations, in this more general form, can be implemented in two ways. In the call/cc way, the program's stack is literally saved and then when the continuation is invoked, the stack is restored. +

+ In continuation passing style (CPS), continuations are just normal functions (only in languages where functions are first class) which the programmer explicitly manages and passes around to subroutines. In this style, program state is represented by closures (and the variables that happen to be encoded in them) rather than variables that reside somewhere on the stack. Functions that manage control flow accept continuation as arguments (in some variations of CPS, functions may accept multiple continuations) and manipulate control flow by invoking them by simply calling them and returning afterwards. A very simple example of continuation passing style is as follows: + 

def save_file(filename):
+  def write_file_continuation():
+    write_stuff_to_file(filename)
+
+  check_if_file_exists_and_user_wants_to_overwrite(write_file_continuation)
+

+ In this (very simplistic) example, the programmer saves the operation of actually writing the file into a continuation (which can potentially be a very complex operation with many details to write out), and then passes that continuation (i.e, as a first-class closure) to another operator which does some more processing, and then calls it if necessary. (I use this design pattern a lot in actual GUI programming, either because it saves me lines of code or, more importantly, to manage control flow after GUI events trigger.) +

+ The rest of this post will, without loss of generality, conceptualize continuations as CPS, because it is a hell of a lot easier to understand and read. +

+ Now let's talk about generators in Python. Generators are a specific subtype of continuation. Whereas continuations are able in general to save the state of a computation (i.e., the program's call stack), generators are only able to save the state of iteration over an iterator. Although, this definition is slightly misleading for certain use cases of generators. For instance: + 

def f():
+  while True:
+    yield 4
+

+ This is clearly a reasonable iterable whose behavior is well defined -- each time the generator iterates over it, it returns 4 (and does so forever). But it isn't probably the prototypical type of iterable that comes to mind when thinking of iterators (i.e., for x in collection: do_something(x)). This example illustrates the power of generators: if anything is an iterator, a generator can save the state of its iteration. +

+ To reiterate: Continuations can save the state of a program's stack and generators can save the state of iteration. This means that continuations are more a lot powerful than generators, but also that generators are a lot, lot easier. They are easier for the language designer to implement, and they are easier for the programmer to use (if you have some time to burn, try to read and understand this page about continuations and call/cc). +

+ But you could easily implement (and conceptualize) generators as a simple, specific case of continuation passing style: +

+ Whenever yield is called, it tells the function to return a continuation. When the function is called again, it starts from wherever it left off. So, in pseudo-pseudocode (i.e., not pseudocode, but not code) the generator's next method is basically as follows: + 

class Generator():
+  def __init__(self,iterable,generatorfun):
+    self.next_continuation = lambda:generatorfun(iterable)
+
+  def next(self):
+    value, next_continuation = self.next_continuation()
+    self.next_continuation = next_continuation
+    return value
+

+ where the yield keyword is actually syntactic sugar for the real generator function, basically something like: + 

def generatorfun(iterable):
+  if len(iterable) == 0:
+    raise StopIteration
+  else:
+    return (iterable[0], lambda:generatorfun(iterable[1:]))
+

+ Remember that this is just pseudocode and the actual implementation of generators in Python is more complex. But as an exercise to understand what is going on, try to use continuation passing style to implement generator objects without use of the yield keyword. +

0 + +
+ This answer is useful + +
+ 207 +
+ This answer is not useful + +
+ Save this answer. + +
+ Show activity on this post. + +

+ Here is an example in plain language. I will provide a correspondence between high-level human concepts to low-level Python concepts. +

+ I want to operate on a sequence of numbers, but I don't want to bother my self with the creation of that sequence, I want only to focus on the operation I want to do. So, I do the following: +

  • I call you and tell you that I want a sequence of numbers which are calculated in a specific way, and I let you know what the algorithm is.
    + This step corresponds to defining the generator function, i.e. the function containing a yield.
  • Sometime later, I tell you, "OK, get ready to tell me the sequence of numbers".
    + This step corresponds to calling the generator function which returns a generator object. Note that you don't tell me any numbers yet; you just grab your paper and pencil.
  • I ask you, "tell me the next number", and you tell me the first number; after that, you wait for me to ask you for the next number. It's your job to remember where you were, what numbers you have already said, and what is the next number. I don't care about the details.
    + This step corresponds to calling next(generator) on the generator object.
    + (In Python 2, .next was a method of the generator object; in Python 3, it is named .__next__, but the proper way to call it is using the builtin next() function just like len() and .__len__)
  • … repeat previous step, until…
  • eventually, you might come to an end. You don't tell me a number; you just shout, "hold your horses! I'm done! No more numbers!"
    + This step corresponds to the generator object ending its job, and raising a StopIteration exception.
    + The generator function does not need to raise the exception. It's raised automatically when the function ends or issues a return.

+ This is what a generator does (a function that contains a yield); it starts executing on the first next(), pauses whenever it does a yield, and when asked for the next() value it continues from the point it was last. It fits perfectly by design with the iterator protocol of Python, which describes how to sequentially request values. +

+ The most famous user of the iterator protocol is the for command in Python. So, whenever you do a: + 

for item in sequence:
+

+ it doesn't matter if sequence is a list, a string, a dictionary or a generator object like described above; the result is the same: you read items off a sequence one by one. +

+ Note that defining a function which contains a yield keyword is not the only way to create a generator; it's just the easiest way to create one. +

+ For more accurate information, read about iterator types, the yield statement and generators in the Python documentation. +

0 + +
+ This answer is useful + +
+ 162 +
+ This answer is not useful + +
+ Save this answer. + +
+ Show activity on this post. + +

+ While a lot of answers show why you'd use a yield to create a generator, there are more uses for yield. It's quite easy to make a coroutine, which enables the passing of information between two blocks of code. I won't repeat any of the fine examples that have already been given about using yield to create a generator. +

+ To help understand what a yield does in the following code, you can use your finger to trace the cycle through any code that has a yield. Every time your finger hits the yield, you have to wait for a next or a send to be entered. When a next is called, you trace through the code until you hit the yield… the code on the right of the yield is evaluated and returned to the caller… then you wait. When next is called again, you perform another loop through the code. However, you'll note that in a coroutine, yield can also be used with a send… which will send a value from the caller into the yielding function. If a send is given, then yield receives the value sent, and spits it out the left hand side… then the trace through the code progresses until you hit the yield again (returning the value at the end, as if next was called). +

+ For example: + 

>>> def coroutine():
+...     i = -1
+...     while True:
+...         i += 1
+...         val = (yield i)
+...         print("Received %s" % val)
+...
+>>> sequence = coroutine()
+>>> sequence.next()
+0
+>>> sequence.next()
+Received None
+1
+>>> sequence.send('hello')
+Received hello
+2
+>>> sequence.close()
+
+ answered Feb 4, 2014 at 2:27 +
Mike McKerns's user avatar
1 + +
+ This answer is useful + +
+ 133 +
+ This answer is not useful + +
+ Save this answer. + +
+ Show activity on this post. + +

+ From a programming viewpoint, the iterators are implemented as thunks. +

+ To implement iterators, generators, and thread pools for concurrent execution, etc. as thunks, one uses messages sent to a closure object, which has a dispatcher, and the dispatcher answers to "messages". +

+ "next" is a message sent to a closure, created by the "iter" call. +

+ There are lots of ways to implement this computation. I used mutation, but it is possible to do this kind of computation without mutation, by returning the current value and the next yielder (making it referential transparent). Racket uses a sequence of transformations of the initial program in some intermediary languages, one of such rewriting making the yield operator to be transformed in some language with simpler operators. +

+ Here is a demonstration of how yield could be rewritten, which uses the structure of R6RS, but the semantics is identical to Python's. It's the same model of computation, and only a change in syntax is required to rewrite it using yield of Python. + 

Welcome to Racket v6.5.0.3.
+
+-> (define gen
+     (lambda (l)
+       (define yield
+         (lambda ()
+           (if (null? l)
+               'END
+               (let ((v (car l)))
+                 (set! l (cdr l))
+                 v))))
+       (lambda(m)
+         (case m
+           ('yield (yield))
+           ('init  (lambda (data)
+                     (set! l data)
+                     'OK))))))
+-> (define stream (gen '(1 2 3)))
+-> (stream 'yield)
+1
+-> (stream 'yield)
+2
+-> (stream 'yield)
+3
+-> (stream 'yield)
+'END
+-> ((stream 'init) '(a b))
+'OK
+-> (stream 'yield)
+'a
+-> (stream 'yield)
+'b
+-> (stream 'yield)
+'END
+-> (stream 'yield)
+'END
+->
+
0 + +
+ This answer is useful + +
+ 116 +
+ This answer is not useful + +
+ Save this answer. + +
+ Show activity on this post. + +

+ I was going to post "read page 19 of Beazley's 'Python: Essential Reference' for a quick description of generators", but so many others have posted good descriptions already. +

+ Also, note that yield can be used in coroutines as the dual of their use in generator functions. Although it isn't the same use as your code snippet, (yield) can be used as an expression in a function. When a caller sends a value to the method using the send() method, then the coroutine will execute until the next (yield) statement is encountered. +

+ Generators and coroutines are a cool way to set up data-flow type applications. I thought it would be worthwhile knowing about the other use of the yield statement in functions. +

+ answered Jan 28, 2013 at 1:37 +
johnzachary's user avatar
0 + +
+ This answer is useful + +
+ 100 +
+ This answer is not useful + +
+ Save this answer. + +
+ Show activity on this post. + +

+ Here is a simple example: + 

def isPrimeNumber(n):
+    print "isPrimeNumber({}) call".format(n)
+    if n==1:
+        return False
+    for x in range(2,n):
+        if n % x == 0:
+            return False
+    return True
+
+def primes (n=1):
+    while(True):
+        print "loop step ---------------- {}".format(n)
+        if isPrimeNumber(n): yield n
+        n += 1
+
+for n in primes():
+    if n> 10:break
+    print "wiriting result {}".format(n)
+

+ Output: + 

loop step ---------------- 1
+isPrimeNumber(1) call
+loop step ---------------- 2
+isPrimeNumber(2) call
+loop step ---------------- 3
+isPrimeNumber(3) call
+wiriting result 3
+loop step ---------------- 4
+isPrimeNumber(4) call
+loop step ---------------- 5
+isPrimeNumber(5) call
+wiriting result 5
+loop step ---------------- 6
+isPrimeNumber(6) call
+loop step ---------------- 7
+isPrimeNumber(7) call
+wiriting result 7
+loop step ---------------- 8
+isPrimeNumber(8) call
+loop step ---------------- 9
+isPrimeNumber(9) call
+loop step ---------------- 10
+isPrimeNumber(10) call
+loop step ---------------- 11
+isPrimeNumber(11) call
+

+ I am not a Python developer, but it looks to me yield holds the position of program flow and the next loop start from "yield" position. It seems like it is waiting at that position, and just before that, returning a value outside, and next time continues to work. +

+ It seems to be an interesting and nice ability :D +

1 + +
+ This answer is useful + +
+ 86 +
+ This answer is not useful + +
+ Save this answer. + +
+ Show activity on this post. + +

+ Here is a mental image of what yield does. +

+ I like to think of a thread as having a stack (even when it's not implemented that way). +

+ When a normal function is called, it puts its local variables on the stack, does some computation, then clears the stack and returns. The values of its local variables are never seen again. +

+ With a yield function, when its code begins to run (i.e. after the function is called, returning a generator object, whose next() method is then invoked), it similarly puts its local variables onto the stack and computes for a while. But then, when it hits the yield statement, before clearing its part of the stack and returning, it takes a snapshot of its local variables and stores them in the generator object. It also writes down the place where it's currently up to in its code (i.e. the particular yield statement). +

+ So it's a kind of a frozen function that the generator is hanging onto. +

+ When next() is called subsequently, it retrieves the function's belongings onto the stack and re-animates it. The function continues to compute from where it left off, oblivious to the fact that it had just spent an eternity in cold storage. +

+ Compare the following examples: + 

def normalFunction():
+    return
+    if False:
+        pass
+
+def yielderFunction():
+    return
+    if False:
+        yield 12
+

+ When we call the second function, it behaves very differently to the first. The yield statement might be unreachable, but if it's present anywhere, it changes the nature of what we're dealing with. + 

>>> yielderFunction()
+<generator object yielderFunction at 0x07742D28>
+

+ Calling yielderFunction() doesn't run its code, but makes a generator out of the code. (Maybe it's a good idea to name such things with the yielder prefix for readability.) + 

>>> gen = yielderFunction()
+>>> dir(gen)
+['__class__',
+ ...
+ '__iter__',    
+ ...            
+ 'close',
+ 'gi_code',
+ 'gi_frame',
+ 'gi_running',
+ 'next',        
+ 'send',
+ 'throw']
+

+ The gi_code and gi_frame fields are where the frozen state is stored. Exploring them with dir(..), we can confirm that our mental model above is credible. +

2 + +
+ This answer is useful + +
+ 74 +
+ This answer is not useful + +
+ Save this answer. + +
+ Show activity on this post. + +

+ Like every answer suggests, yield is used for creating a sequence generator. It's used for generating some sequence dynamically. For example, while reading a file line by line on a network, you can use the yield function as follows: + 

def getNextLines():
+   while con.isOpen():
+       yield con.read()
+

+ You can use it in your code as follows: + 

for line in getNextLines():
+    doSomeThing(line)
+

+ Execution Control Transfer gotcha +

+ The execution control will be transferred from getNextLines() to the for loop when yield is executed. Thus, every time getNextLines() is invoked, execution begins from the point where it was paused last time. +

+ Thus in short, a function with the following code + 

def simpleYield():
+    yield "first time"
+    yield "second time"
+    yield "third time"
+    yield "Now some useful value {}".format(12)
+
+for i in simpleYield():
+    print i
+

+ will print + 

"first time"
+"second time"
+"third time"
+"Now some useful value 12"
+
+ 10.2k4 gold badges61 silver badges94 bronze badges +
0 + +
+ This answer is useful + +
+ 72 +
+ This answer is not useful + +
+ Save this answer. + +
+ Show activity on this post. + +

+ (My below answer only speaks from the perspective of using Python generator, not the underlying implementation of generator mechanism, which involves some tricks of stack and heap manipulation.) +

+ When yield is used instead of a return in a python function, that function is turned into something special called generator function. That function will return an object of generator type. The yield keyword is a flag to notify the python compiler to treat such function specially. Normal functions will terminate once some value is returned from it. But with the help of the compiler, the generator function can be thought of as resumable. That is, the execution context will be restored and the execution will continue from last run. Until you explicitly call return, which will raise a StopIteration exception (which is also part of the iterator protocol), or reach the end of the function. I found a lot of references about generator but this one from the functional programming perspective is the most digestable. +

+ (Now I want to talk about the rationale behind generator, and the iterator based on my own understanding. I hope this can help you grasp the essential motivation of iterator and generator. Such concept shows up in other languages as well such as C#.) +

+ As I understand, when we want to process a bunch of data, we usually first store the data somewhere and then process it one by one. But this naive approach is problematic. If the data volume is huge, it's expensive to store them as a whole beforehand. So instead of storing the data itself directly, why not store some kind of metadata indirectly, i.e. the logic how the data is computed. +

+ There are 2 approaches to wrap such metadata. +

  1. The OO approach, we wrap the metadata as a class. This is the so-called iterator who implements the iterator protocol (i.e. the __next__(), and __iter__() methods). This is also the commonly seen iterator design pattern.
  2. The functional approach, we wrap the metadata as a function. This is the so-called generator function. But under the hood, the returned generator object still IS-A iterator because it also implements the iterator protocol.

+ Either way, an iterator is created, i.e. some object that can give you the data you want. The OO approach may be a bit complex. Anyway, which one to use is up to you. +

0 + +
+ This answer is useful + +
+ 71 +
+ This answer is not useful + +
+ Save this answer. + +
+ Show activity on this post. + +

+ In summary, the yield statement transforms your function into a factory that produces a special object called a generator which wraps around the body of your original function. When the generator is iterated, it executes your function until it reaches the next yield then suspends execution and evaluates to the value passed to yield. It repeats this process on each iteration until the path of execution exits the function. For instance, + 

def simple_generator():
+    yield 'one'
+    yield 'two'
+    yield 'three'
+
+for i in simple_generator():
+    print i
+

+ simply outputs + 

one
+two
+three
+

+ The power comes from using the generator with a loop that calculates a sequence, the generator executes the loop stopping each time to 'yield' the next result of the calculation, in this way it calculates a list on the fly, the benefit being the memory saved for especially large calculations +

+ Say you wanted to create a your own range function that produces an iterable range of numbers, you could do it like so, + 

def myRangeNaive(i):
+    n = 0
+    range = []
+    while n < i:
+        range.append(n)
+        n = n + 1
+    return range
+

+ and use it like this; + 

for i in myRangeNaive(10):
+    print i
+

+ But this is inefficient because +

  • You create an array that you only use once (this wastes memory)
  • This code actually loops over that array twice! :(

+ Luckily Guido and his team were generous enough to develop generators so we could just do this; + 

def myRangeSmart(i):
+    n = 0
+    while n < i:
+       yield n
+       n = n + 1
+    return
+
+for i in myRangeSmart(10):
+    print i
+

+ Now upon each iteration a function on the generator called next() executes the function until it either reaches a 'yield' statement in which it stops and 'yields' the value or reaches the end of the function. In this case on the first call, next() executes up to the yield statement and yield 'n', on the next call it will execute the increment statement, jump back to the 'while', evaluate it, and if true, it will stop and yield 'n' again, it will continue that way until the while condition returns false and the generator jumps to the end of the function. +

\ No newline at end of file diff --git a/packages/readabilityjs/test/test-pages/stratechery/distiller.html b/packages/readabilityjs/test/test-pages/stratechery/distiller.html new file mode 100644 index 000000000..b2fe45655 --- /dev/null +++ b/packages/readabilityjs/test/test-pages/stratechery/distiller.html @@ -0,0 +1,85 @@ +

+ Let’s start with the caveats: no, Tesla did not sell 276,000 Model 3’s in three days; +

1 that is the number of fully refundable pre-orders that required a deposit of “only” $1,000. +
2 And yes, Tesla has a history of delivering cars late and with a higher price than expected. Moreover, given the fact that Tesla only delivered just over 50,000 cars last year, no matter how quickly Tesla scales it will almost certainly be years before this first week of reservations is fulfilled, and even then Tesla will only control a fraction of the car market. +

+ With that out of the way, can we marvel at what Tesla and CEO Elon Musk have accomplished? Nearly 300,000 people have willingly parted with $1,000 despite the fact they will not have a chance to purchase a car for years; an astounding 115,000 of them sent in their deposit before they even knew what the car looked like. A friend got in line to make his reservation at 6:45am and there were 123 people in front of him. This is, no matter how you measure it, a phenomenon that is nearly unprecedented; the only possible comparison is Apple and its iPhone. +

+ Long lines and fans committed to ordering new products sight-unseen are not the only things Tesla and Apple have in common: both companies have been doubted for allegedly not understanding Disruption Theory; both, though, are proving that Disruption Theory does not have all the answers, particularly when it comes to consumer markets. +

+ The iPhone and Disruption +

+ Back in 2013 I wrote What Clayton Christensen Got Wrong, specifically about Apple and the iPhone: +

+ Christensen’s theory is based on examples drawn from buying decisions made by businesses, not consumers. The reason this matters is that the theory of low-end disruption presumes: +

  • Buyers are rational
  • Every attribute that matters can be documented and measured
  • Modular providers can become “good enough” on all the attributes that matter to the buyers

+ All three of the assumptions fail in the consumer market, and this, ultimately, is why Christensen’s theory fails as well… +

+ My conclusion was that the iPhone was, contrary to the then-conventional wisdom, not likely to suffer from low-end disruption, and not only has that proven to be correct, Apple has in fact expanded its global marketshare. And now, with the iPhone SE, Apple is expanding the high end to a price point accessible to customers in developing markets who very much want an iPhone but simply don’t have the means to afford top-of-the-line prices. +

+ It’s this latter point — that a high-end approach can drive growth at lower price points — that seems particularly pertinent to the Model 3. +

+ Tesla’s Master Plan +

+ During Thursday’s Model 3 introduction Musk referenced Tesla’s “secret master plan,” which he laid out in a blog post back in 2006. Musk wrote: +

+ The initial product of Tesla Motors is a high performance electric sports car called the Tesla Roadster. However, some readers may not be aware of the fact that our long term plan is to build a wide range of models, including affordably priced family cars…Critical to making that happen is an electric car without compromises, which is why the Tesla Roadster is designed to beat a gasoline sports car like a Porsche or Ferrari in a head to head showdown. Then, over and above that fact, it has twice the energy efficiency of a Prius… +

+ The strategy of Tesla is to enter at the high end of the market, where customers are prepared to pay a premium, and then drive down market as fast as possible to higher unit volume and lower prices with each successive model…The second model will be a sporty four door family car at roughly half the $89k price point of the Tesla Roadster and the third model will be even more affordable. +

+ In keeping with a fast growing technology company, all free cash flow is plowed back into R&D to drive down the costs and bring the follow on products to market as fast as possible. When someone buys the Tesla Roadster sports car, they are actually helping pay for development of the low cost family car. +

+ Leaving aside the fact that the “sporty four door family car” costs $25,000 more than Musk’s promise, +

3 the strategy seems to have worked: Tesla, with a detour to build the Model X crossover along the way, has moved down the cost curve culminating in the announcement of the affordable Model 3. +

+ A closer look, though, suggests that Musk painted too rosy a picture when it comes to the Master Plan’s funding: while Tesla’s cars have, from 2009 on, been sold at a profit, the company has still lost significant amounts of money every year. The reality is the company’s significant research and development costs have been paid for by issuing stock and incurring debt, not the profits of high-end models. +

+ Low End Dogma +

+ Given Tesla’s finances, it’s tempting to ask why the company didn’t simply start with the low-end; indeed, researchers at Christensen’s Forum for Growth and Innovation argued last year that a better approach to the electric vehicle market would be exactly that. From a Harvard Business Review article entitled Tesla’s Not as Disruptive as You Might Think: +

+ It [is] clear that Tesla is not a disrupter. It’s a classic “sustaining innovation” — a product that, according to Christensen’s definition, offers incrementally better performance at a higher price…because it’s a sustaining innovation, theory predicts that competitors will emerge. Our analysis concludes that a competitive response won’t happen until Tesla expands outside its current niche of people who prefer electric vehicles to gas-powered cars — but if it expands by creating more variety (such as SUVs) and more-affordable vehicles, competition will be fierce. +

+ Instead the research team suggested the better route for electric vehicles would be “neighborhood electric vehicles”, which are pretty much the exact opposite of a Tesla: the article describes them as “a low-speed vehicle that resembles a souped-up golf cart.” +

+ Don’t feel bad if you haven’t heard of neighborhood electric vehicles: Global Electric Motorcars, featured in the article’s sidebar, have only sold 50,000 vehicles in 17 years, despite the fact they cost around a tenth as much as a Model S. Tesla, meanwhile, sold over 50,000 Model S’s last year alone; its growth rate relative to its competitors was especially impressive. This table about large luxury vehicles in the U.S. is from the company’s February 2016 letter to investors: +

Screen Shot 2016-04-05 at 9.08.36 PM

+ One would think the other car companies in this table would be incentivized to respond, no? Yet Tesla is not being out-competed, and they sure as heck aren’t selling glorified golf carts: +

+ + Video Player +

+ The truth is that the HBS Growth and Innovation Forum team is right: Tesla is not disruptive. Rather, their error was a repeat of the mistake Christensen made with the iPhone; first, they don’t understand why people buy Teslas, and two, they assume that disruption is the only viable strategy to enter a new market. +

+ The Power of Best +

+ When it comes to the iPhone I have argued that Apple’s smartphone was, relative to the phones on the market, Obsoletive: the iPhone effectively reduced the phones that came before it to apps on a general purpose computer, justifying a higher price even as it made cheaper incumbents obsolete. +

+ This doesn’t quite work for Tesla: at the end of the day a Model S is still doing the same job as a traditional BMW or Mercedes-Benz. It just does it better: a Model S accelerates faster, it has more storage, it has innovative features like limited auto-pilot and a huge touch-screen interface, and you don’t have to stop at the gas station. Most importantly, though, it is a Tesla. +

+ The real payoff of Musk’s “Master Plan” is the fact that Tesla means something: yes, it stands for sustainability and caring for the environment, but more important is that Tesla also means amazing performance and Silicon Valley cool. To be sure, Tesla’s focus on the high end has helped them move down the cost curve, but it was Musk’s insistence on making “An electric car without compromises” that ultimately led to 276,000 people reserving a Model 3, many without even seeing the car: after all, it’s a Tesla. +

+ The electric car business has taken the form of an old-fashioned race for a prize…but now it looks pretty clear who the winner will be. And it ain’t Tesla. +

+ General Motors first unveiled the Chevy Bolt as a concept car in January 2015, billing it as a vehicle that would offer 200 miles of range for just $30,000 (after a $7,500 federal tax credit). Barring any unforeseen delays, the first Bolts will roll off the production line at GM’s Orion Assembly facility in Michigan by the end of 2016. As Pam Fletcher, GM’s executive chief engineer for electric vehicles, recently put it to me with a confident grin: “Who wants to be second?” +

+ Good for GM, but I’m afraid the company — and Wired — missed the plot; as the article notes an optimistic goal for the Bolt is 50,000 units a year, and I’d bet the under: at the end of the day the company is still selling a relatively slow and ugly Chevrolet. Brand and reputation matters far more than being “first” to a product category where every model on the market has fallen short of expectations — except for Tesla. +

+ To that end, the significance of electric to Tesla is that the radical rethinking of a car made possible by a new drivetrain gave Tesla the opportunity to make the best car: there was a clean slate. More than that, Tesla’s lack of car-making experience was actually an advantage: the company’s mission, internal incentives, and bottom line were all dependent on getting electric right. +

+ Again the iPhone is a useful comparison: people contend that Microsoft lost mobile to Apple, but the reality is that smartphones required a radical rethinking of the general purpose computer: there was a clean slate. More than that, Microsoft was fundamentally handicapped by the fact Windows was so successful on PCs: the company could never align their mission, incentives, and bottom line like Apple could. +

+ To be sure, what Tesla and Apple have accomplished is not easy, and ongoing success is not guaranteed, particularly at lower price points. I think, though, Tesla, like Apple before them, has more control of their destiny than it may appear. Writing about Apple in an article called Best I said: +

+ As nearly every other consumer industry has shown, as long as there is a clear delineation between the top-of-the-line and everything else, some segment of the user base will pay a premium for the best. That’s the key to Apple’s future: they don’t need completely new products every other year (or half-decade); they just need to keep creating the best stuff in their categories. Easy, right? +

+ It is, in fact, devilishly hard; indeed Apple’s software quality has, in the eyes of many observers, declined the last few years. The company, though, has been synonymous with “best” for so long that they have time to get it right, and millions of new customers who can’t wait to buy their first iPhone; Tesla will likely receive similar grace when and if the Model 3 comes in late and over its promised price. After all, it will still be a Tesla. +

  1. + Elon Musk has promised an update on first week reservations tomorrow ↩ +

  2. + Obviously $1,000 is a fairly substantial amount of money; it is, though, a mere 3% of the promised base price of $35,000 (before any applicable tax credits)  ↩ +

  3. + Although the Model S was originally available for $59,900, “only” $15,000 more than Musk’s promise; then again, the Tesla Roadster ended up costing $20,000 more too ↩ +

  4. + Yes, smartphones were disruptive to PCs, as Christensen later acknowledged; this, though, does not explain why Nokia and Blackberry in particular were so devastated ↩ +

\ No newline at end of file diff --git a/packages/readabilityjs/test/test-pages/substack-email/distiller.html b/packages/readabilityjs/test/test-pages/substack-email/distiller.html new file mode 100644 index 000000000..069e77bcb --- /dev/null +++ b/packages/readabilityjs/test/test-pages/substack-email/distiller.html @@ -0,0 +1,89 @@ +

+ #39: New corona regs, Kotti cops, electric + cars

+ and startups bringing in more money than our brains can comprehend.

+ + CommentShare
+ Surprising no one who has ever been at Kotti. Photo by Montecruz Photo licensed under CC BY 2.0. +

Hello 20 + Percent,

We + agreed to confine my wife to my son’s room. She was included in the + democratic process because she’s the one with corona. And now every time she heads out + for food, to use the restroom or to just change clothes, we go through a complex tetris of + opening windows, dawning masks and maintaining social distancing. A weird thing in your own + house. But we’ve survived lice, moths in the larder and even black mold - Berlin + things. So we’ll get through this too - so far it’s been little more + than a serious cold for her (YMMV). Since we’re all + boostered, it will be interesting to see who is also felled by the virus. I’m fine for + now (knocks on wooden floor).

But it’s + also why this is a touch late today.

Enjoy the + weekend. You’ve earned it!

And a double + enjoy-the-weekend to those who have backed us on Patreon. You can’t + imagine how much of a boost that has given us. Thanks!

Andrew

+ The Berlin corona stats for Friday, January 14

Fully + vaccinated: 73.2% (72.2% Tuesday)
Received booster: + 43.3%
New cases in one day: +7,238 (5,976 + Tuesday)
Total deaths: 4,063 (+20 over Tuesday) +

🔴 + 7-day Covid-19 incidence (cases per 100,000): 949.8 (737.2 + Tuesday)
🔴 7-day hospitalization incidence (also per 100,000): 11.2 + (10.6 Tuesday)
🟡 Covid-19 ICU + patient occupancy: 18.8% (19.1% Tuesday)

+ New corona rules today (Friday)

Why make it + easy when you can make it more complex? Corona regulations are generally passed by the + federal government and then have to be passed again at the state level, often with + inexplicable local flourishes, before becoming law. This creates a Germany-wide quilt of + differing regulations that few understand. This week is no different though Berlin has + decided to stretch the introduction over two days. Starting today in Berlin, people + in contact with an infected person no longer have to go into quarantine if they’ve + received a booster vaccination or belong to the new “fresh” + category - “freshly” vaccinated or recovered. “Fresh” + means within the past three months. Quarantine for other contact persons now ends after 10 + days, rather than 14, and people can test out (rapid antibody or PCR) after 7 days. Is + anyone enforcing this?

+ New corona rules Saturday (tomorrow)

FFP2 + masks are now mandatory on public transport, so keep the surgical masks until the + next time politicians change their minds (the only constant in this pandemic). Bars and + restaurants are also 2G plus, which means recently recovered or vaccinated PLUS a same-day + negative test result or proof of a booster vaccination. This applies to any other indoor + events with more than 10 people as well.

+ Kotti police station

You know when + an aging celebrity dies and you think, Oh, I thought they were already dead? Same feeling + when you hear they’re going to open a police station at Kotti. You + mean there wasn’t one already? The fuzz want to build a new 300-square-meter cop shop + on Reichenberger Strasse that would be home to 65 officers, according to Berliner + Zeitung. Criminals don’t have to worry - if they’re just planning now it’ll + be years until it happens. In the final quarter of 2021, police reported 2,309 + crimes at the public transport hub, or 152 more than in the third quarter. Most + crimes fell but drug-related offenses jumped to 758 from 454.

+ Police raid political artist’s atelier

Officers + specialized in political crime Thursday searched the apartment and studio of an artist + affiliated with the Center for Political Beauty, according to RBB24. The + activist artists last summer created a fake flyer distribution company and convinced the + right-wing AfD to hire them to distribute 5 million election-related flyers. The artists + then destroyed the pamphlets. The artist is reportedly accused of falsifying documents that + could be used to secure official documents. The center decried the raid on Twitter and its + attorney claimed it was illegal.

+ Electric and hybrid autos increasingly popular

Almost + half, or 48.6 percent, of all cars newly registered in Berlin last year were either electric + or hybrid, an improvement over the 29.9 percent of 2020 and well above the 11.9 + percent of 2019 before the pandemic. Berliners are also buying fewer cars - just 60,000 new + cars were registered in the Haupstadt last year, compared with 65,900 in 2020 and + 87,000 in 2019.

+ Record funding year for Berlin startups

Proof big + money doesn’t care about labor: Gorillas, the grocery delivery service better + known for how it treats its employees, reeled in €861 million in fresh capital + last year, the biggest pile of cash ever amassed by a Berlin startup, according + to consultancy EY. 2021 was a record for the city’s startups, bringing in + €10.5 billion in fresh money, €7 billion more than in 2020. Where’s + our cut?

+ Factoid

+ His Meister’s Voice? If you get that, you’re old. Photo by Dixtuin, licensed under CC BY 2.0

Since it was + set up in the early 1960s, Hansa Studios near Potsdamer Platz has drawn countless + musicians to Berlin, from Andrew Lloyd Webber to ZZ Top. You’ve probably + heard about David Bowie’s late-night sessions in the vocal booth recording iconic + tracks like Heroes. But did you know that U2 recorded One in the majestic + Meisterhalle, known as the Big Hall by the Wall? Or that when Depeche Mode made + Construction Time Again at Hansa, hordes of fans hung out on the long stairway to the studio + in the hope of glimpsing their idols? - Andrew Blackman, who recently toured the + studio

\ No newline at end of file diff --git a/packages/readabilityjs/test/test-pages/substack-michaelshellenberger/distiller.html b/packages/readabilityjs/test/test-pages/substack-michaelshellenberger/distiller.html new file mode 100644 index 000000000..9d00e05dc --- /dev/null +++ b/packages/readabilityjs/test/test-pages/substack-michaelshellenberger/distiller.html @@ -0,0 +1,32 @@ +
+ + CommentShare

+ For decades, people have claimed that homelessness is just a + housing problem. Sure, many also have substance use and mental + illness issues. But if we just give homeless people their own + own studio apartments, and decriminalize public camping, + drugs, and shoplifting, the problem will go away, many + claimed. +

+ That hasn’t happened. Instead, the open drug scenes have + worsened. Nationally, drug overdoses and poisonings increased + from 17,000 in 2000 to 108,000 in 2021. And California, which + pioneered the “Housing First”/decriminalization + approach saw its homeless population increase 31% between 2011 + and 2020, even as homelessness declined 18% in the rest of the + country. +

+ I debunked the lies about homelessness in + San Fransicko, in hundreds of articles, and on dozens + of TV and podcast appearances. But when it comes to educating + the public, nothing has been more impactful than the video + interviews of homeless people that I’ve conducted over + the last few months with my friend Leighton Woodhouse, a + documentary filmmaker, as part of my run to become governor of + California. +

+ Now Leighton has assembled those interviews into a two-minute + 20 second video we’ve posted on Twitter. It’s a + must-watch. It’s only been on-line for a few hours, and + over 130,000 people have seen it. +

Twitter avatar for @ShellenbergerMD
\ No newline at end of file diff --git a/packages/readabilityjs/test/test-pages/sydney.com/distiller.html b/packages/readabilityjs/test/test-pages/sydney.com/distiller.html new file mode 100644 index 000000000..309c39c6f --- /dev/null +++ b/packages/readabilityjs/test/test-pages/sydney.com/distiller.html @@ -0,0 +1,145 @@ +

+ From stunning views of Sydney Harbour to mesmerising sandstone cliffs and vibrant culture, a walk is an amazing (and affordable) way to experience the true beauty of Sydney. Take in historic sites, fascinating wildlife, world-famous beaches and lots more with our picks for the best walks in Sydney. +

+ Whether you’re after a peaceful amble or something a bit more intensive to get your blood pumping, you’ll find a top Sydney stroll right here. +

+ Glebe Foreshore Walk +

Blackwattle Bay in Glebe

+ Region: Inner City +

+ The Glebe Foreshore walk starts at Bicentennial Park and ends at the Sydney Fish Markets, with 27 hectares of beautiful parkland and reserves to enjoy along the way. The spacious walking path runs alongside Rozelle Bay and Blackwater Bay, making for fantastic views of the bays and the Sydney CBD, Sydney Harbour, and Anzac Bridge. It’s a gorgeous morning walk in particular, or if you’re not much of an early riser, it’s just as lovely during the afternoon or sunset. +

+ Who’s it for? Anyone looking for a short, peaceful Sydney walk with beautiful bay views and verdant, wide open parks. It’s also dog-friendly, with sections of the walk and its parks being off-leash, so this is a great walk to enjoy with your furry friend. +

+ Tip: Come hungry so that you can reward yourself with fresh seafood and hot chips at the Sydney Fish Markets at the end of it. +

  • How long is the walk? 2.2km one way, 30min to 1hr
  • Difficulty? Easy – a flat walk for all fitness levels
  • Great for: Sydney Harbour, ocean and parkland views, nature, picnics, historical sites, dogs
  • Walk Highlights: Rozelle Bay, Blackwattle Bay Park, Sydney Fish Market

+ Barangaroo Foreshore & Reserve +

Couple enjoying a walk around Nawi Cove in Barangaroo, Sydney City

+ Region: Inner City +

+ The main path starts at the Hickson Road entrance and encircles Barangaroo Reserve before finishing at the Towns Place roundabout on the other side, but there are several paths branching through the Reserve that you can (and should) stop to wander through along the way. The Reserve itself is a stunning, multi-level park with over 75,000 native plants and trees within its lush gardens. It provides views of Darling Harbour, the Harbour Bridge, Luna Park and Goat Island, as well as plenty of quiet, shady spots to sit and relax. The walk is also a great chance to get familiar with Sydney’s trendiest new precinct and all of the modern restaurants, shops and architecture it has to offer. +

+ Who’s it for? Everybody. A family and dog-friendly stroll suitable for people of all fitness levels, you can make it an all-day picnic or just stop by to see it the next time you’re in the city. It’s also ideal for solo walkers who want a relaxing stroll with quiet spots to sit and enjoy the scenery. +

+ Tip: You’ll make the most of it if you stop to explore the park and take in the incredible views of Sydney Harbour – head up to Stargazer’s Lawn for a great viewpoint. +

  • How long is the walk? 2km one way, 1-2hr
  • Difficulty? Easy – mostly flat for all fitness levels
  • Great for: Sydney Harbour, ocean and parkland views, nature, picnics, historical sites, cycling
  • Walk Highlights: Darling Harbour, Harbour Bridge, Barangaroo Reserve, Stargazer’s Lawn, The Cutaway

+ Hermitage Foreshore Walk +

Scenic harbour views from the Hermitage Foreshore Track, Vaucluse

+ Region: Eastern Suburbs +

+ If you’re after a well-shaded coastal walk packed with family-friendly beaches and historical sites, try this one. Start at Nielsen Park and finish up at Bayview Hill Road in Rose Bay. This trail includes five beaches where you can have a swim, picnic, take a pit stop, or watch the sunset. Not to mention views of multi-million dollar mansions, including the famous historic Strickland House, and the beautiful Shark Island. By the time you’re finished with this walk, your photo storage will be completely full. +

+ Who’s it for? This is a great walk if you’re visiting during the hot summers and are travelling in a group. If you’re after something that’s relatively relaxed but still helps you get in some steps, this is perfect for you. +

+ Tip: Milk Beach is our pick for the best beach along the walk for a quick stop, but we encourage you to try them all out and decide for yourself! +

+ Manly to Spit Walk +

views of Sydney Harbour from Dobroyd Head - Balgowlah Heights

+ Region: Northern Beaches +

+ For a longer walk with spectacular Sydney Harbour views, this is the track for you. Start near Manly Wharf and work your way to the stunning Spit Bridge. This harbourside trail passes through Sydney Harbour National Park, a mix of subtropical bushland and a series of short tracks crossing between stunning beaches, bays and inlets in North Sydney. Take in historical sites such as the Grotto Point Lighthouse and admire Aboriginal rock engravings of fish, boomerangs and kangaroos. You could alternatively start at Spit Bridge and end in Manly so that you can explore the beaches, shops, and restaurants for the rest of the day. +

+ Who’s it for? The walk is long with steep and uneven steps along the way, so it’s not one to take little kids on. Though it’s suitable for people of most fitness levels, you’ll need to be ready to walk for a good few hours. The views are worth it, but you’ll definitely have to work for it. +

+ Tip: Bring a swimming costume in case you want to stop and swim at one of the secret beaches along the way, and don’t forget to pack water, snacks, a hat and sunscreen. +

+ Bundeena to Wedding Cake Rock +

Bundeena to Wedding Cake Rock

+ Region: Sydney’s South +

+ If you want to witness the spectacular views of the Coast Track, but aren’t quite ready for the entire 26km hike, this trail is an ideal option. Start at Beachcomber Avenue in Bundeena and make your way to Wedding Cake Rock. Picture mesmerising sandstone cliffs, coastal lookouts, and seasonal flowers that will lead you to this treasured Sydney beauty. Come during late June or early July and you might be able to spot some whales! +

+ Who’s it for? We’d recommend tackling this walk if you have at least a moderate level of fitness. It’s not too long, but the track gets steep and rocky, and it crosses through coastal heathland and bushes, so it can be demanding for the less experienced. +

+ Tip: There’s limited phone reception here, and it’s best to bring water and snacks with you and use the toilet before you set. Also please remember that Wedding Cake Rock is highly unstable, so stay behind the fence – the risk is definitely not worth the photo. +

  • How long is the walk? 3.5km one way, about 2hr 30min
  • Difficulty? Medium/Difficult – walking through the bush, some steep sandstone stairs and up hills
  • Great for: A decent workout, sightseeing, bird watching, whale watching
  • Walk Highlights: Wedding Cake Rock, Royal National Park

+ Botany Bay Coastal Walk +

Cape Solander Kurnell, Kamay Botany Bay National Park

+ Region: Southeast Sydney +

+ If you love exploring cliff tops, rock formations, and ocean views but you’ve already tried most of the Harbour and beach walks, try the Botany Bay Coastal Walk. Begin at the end of Polo Street in Kurnell and keep walking until you reach Cronulla Station. This coastal walk features views of small sand dunes, hanging swamps, Boat Harbour Aquatic Reserve and the Cape Baily Lighthouse. Once you’re finished, take a dip in one of Cronulla’s beautiful pools or beaches. +

+ Who’s it for? Another longer, more rugged track, this isn’t so much a casual stroll as it is a real nature walk. It’s a good few hours and you’ll traverse some rougher track along the way, so we’d suggest it for people with at least a moderate level of fitness. +

+ Tip: If four hours sounds a bit too long for you, you can always tackle a smaller section – we’re partial to the track from Cape Solander to the Cape Baily Lighthouse. +

+ Bondi to Coogee Walk +

Bondi to Coogee Walk, Sydney

+ Region: Eastern Suburbs +

+ The quintessential Sydney experience, no list of the best walks in the city is complete without the Bondi to Coogee walk. Here you’ll see Sydney’s most iconic attractions, starting at the famous Bondi Beach, passing the ever-Instagrammable Bondi Icebergs swimming club, and following the paved path that leads to the stunning Coogee Beach. You can make a full day of it by stopping at some of the many gorgeous beaches, cliff-tops, natural rock pools, and cafes along the way. If you’re around from late June to early July, it’s also a great opportunity to try to spot some whales. +

+ Who’s it for? Everyone! It’s an ideal walk to enjoy with friends or take a date for the day. Keep in mind that although it can make for an amazing family day out, if you have younger children you might be better off going part-way (we like Bondi to Bronte), unless you’re willing to do a bit of carrying! +

+ Tip: Stop off at Gordon’s Bay along the way. It’s usually less crowded than the main beaches, and features an underwater nature trail that’s perfect for snorkelling. +

+ Garigal National Park Cascades Walk +

Pipeline and Bungaroo tracks to Stepping Stones Crossing, Garigal National Park

+ Region: North Shore +

+ Looking to avoid the crowds or try something a little different to your usual oceanside walk? Start from Acron Oval in St Ives and traverse dense bushland through Garigal National Park. You’ll pass an abundance of native Australian trees and flowers on your way to the rocky cascades, a beautiful set of small waterfalls and rock pools. Look out for some of the park’s wildlife, such as burrowing frogs, Rosenberg’s goannas and kookaburras. There are also plenty of sites with Aboriginal cave art and rock engravings for you to marvel at. +

+ Who’s it for? Nature lovers who are looking to do some bushwalking in the wilderness, away from the crowds. It’s not very long and can be undertaken by people of most fitness levels without any bushwalking experience, but it’s still considered a moderately challengingwalk. +

+ Tip: Bungaroo Pool used to be a public swimming hole. While we wouldn’t suggest swimming there now, you should keep an eye out for the inscription of ‘Ladies’ in the rock to mark the old ladies’ changing area from the early 1900s. +

+ Bradleys Head to Chowder Bay +

Sydney Harbour from Bradleys Head, Mosman

+ Region: North Shore +

+ Lace up your runners for a trail with a stunning view of the city and the Sydney Opera House. Start at Taronga Zoo Wharf and make your way around the coast to Chowder Bay. You’ll walk through an old angophora forest filled with wildlife and ancient trees, and then find yourself on a track that eventually opens up to the Clifton Gardens Reserve and Chowder Bay. While walking, you’ll have extraordinary views of the harbour and Sydney Opera House, as well as South Head, Rose Bay, Shark Island and Bradleys Head. +

+ Who’s it for? City walkers who are looking for a mid-length stroll around Sydney. There are some stairs, but it’s a fairly easy walk with many places to stop along the way, so it’s suitable for people of all fitness levels. +

+ Tip: There’s not much parking at Bradley’s Head, so if you’re driving then definitely start from the Taronga Zoo end. +

+ Fairfax Walk at North Head +

Bradleys Head to Chowder Bay walk, Sydney Harbour National Park

+ Region: Northern Beaches +

+ If you’re looking for a walk that’s short and sweet but still delivers top-notch views, give Fairfax Walk at North Head a try. This walk starts at the end of North Head Scenic Drive and then circles around the top of North Head. You’ll get stunning ocean views as well as a glimpse into the hustle and bustle happening at the CBD. This is another one for the late June-July and August-October whale watching aficionados. +

+ Who’s it for? The walk is short and relaxed, so it’s suitable for everyone. It’s a particularly great choice if you’re travelling with children or need wheelchair accessibility. +

+ Tip: Finished the walk but don’t feel ready to call it a day? The track connects to the Manly Scenic Walkway where you can continue walking to your heart’s content. +

+ Watsons Bay to Hornby Lighthouse +

Hornby Lighthouse, South Head of Sydney Harbour

+ Region: Eastern Suburbs +

+ Pick this walk if you’re curious about Sydney’s rich history and want to soak up some ocean views while you’re at it. Beginning at Watsons Bay ferry terminal, you’ll follow a sidewalk along the beach until you pass through beautiful Camp Cove. From there, take the cobblestone pathways that will lead you to the historic Hornby Lighthouse. You’ll also stumble across a lightkeeper’s cottage, old military cannons and sandstone gun emplacements. While at the lighthouse, you’ll be able to see stunning views of Sydney Harbour, Middle Head and North Head, and you may also see some whales if you’re lucky! +

+ Who’s it for? Another easier walk that’s suitable for all ages and fitness levels, with a paved walkway and some stairs. +

+ Tip: Linger around Watsons Bay for a while before you depart and try one of its amazing restaurants, or lounge around the park. +

+ Western Escarpment Walking Track +

Western Escarpment walking track, Malabar Headland National Park

+ Region: Eastern Suburbs +

+ To explore Sydney’s lush bushlands, follow the Western Escarpment Track. Start your journey at Pioneers Park in Malabar where you’ll make your way through an island of preserved bushland that will lead you to 360-degree views over Malabar Headland National Park, Maroubra Beach and Botany Bay. Along the way, there are plenty of lookouts and seating for you to soak up the views. You’ll also encounter some of Sydney’s beautiful wildlife, such as red wattlebirds, new holland honeyeaters and blue fairy-wrens. +

+ Who’s it for? People who want both the coastal and bushwalking experience in one short, sweet walk. However, it’s not ideal for parents with prams, since the track is a bit rough. +

+ Tip: Look up and, if you’re lucky, you might catch a glimpse of sea eagles or kestrels circling above. +

  • How long is the walk? 1km one way, about 20-30min
  • Difficulty? Easy/Medium – short well-formed track, short steep hills and occasional steps
  • Great for: Nature lovers, bird watching, ocean views, whale watching, short walk
  • Highlights: Malabar Headland National Park, Maroubra Beach, Botany Bay

+ Anzac Bridge to Pyrmont Bridge +

Sunset over ANZAC Bridge in Pyrmont

+ Region: Inner West +

+ For a scenic walk that will lead you into the city, give Anzac Bridge to Pyrmont Bridge a go. Start your walk at the Anzac Bridge that will lead you to Blackwattle Bay where you can experience the Sydney Fish Market. After you’ve sampled the wares of the Sydney Fish Market continue to Darling Harbour across the Pyrmont Bridge. You’ll get amazing views of the city and Cockle Bay and at the end of it will end up right in the heart of Sydney’s CBD where you can access wonderful shopping, eats, pubs, and much more. +

+ Who’s it for? Everybody. Try this one for great views of the harbour along a pleasant stroll that will lead you comfortably into the centre of the city. +

+ Tip: Go early in the morning for a spectacular sunrise and a full day to explore the city. If you decide to stop by the Fish Markets, try the sushi doughnuts! +

+ Aboriginal Heritage Walk, Ku-ring-gai Chase National Park +

West Head - Best Walks in Sydney

+ Region: North Shore +

+ This walk through Ku-ring-gai Chase National Park is arguably the best in Sydney for getting out in nature while also learning more about the stories and heritage of the local Aboriginal people. Start from the Resolute picnic area at the end of West Head Road, close to the edge of the peninsula. Follow the looping trail past the sacred Red Hands Cave to the breathtaking West Head Lookout and then the secluded Resolute Beach for a truly magical walking experience. Along the way you can get a closer look at traditional Aboriginal engravings, artworks and even a historic occupation shelter. +

+ Who’s it for? When you’re looking for a stunning nature walk, but also want to know more about the rich Aboriginal culture and history of the area. It can be difficult, so it’s best for people with at least a moderate level of fitness or some bushwalking experience. Bring sun protection, swimmers, food and plenty of water. +

+ Tip: Just try this one. Really. +

\ No newline at end of file diff --git a/packages/readabilityjs/test/test-pages/techcrunch/distiller.html b/packages/readabilityjs/test/test-pages/techcrunch/distiller.html new file mode 100644 index 000000000..6189217dc --- /dev/null +++ b/packages/readabilityjs/test/test-pages/techcrunch/distiller.html @@ -0,0 +1 @@ +

The saga of FTX, formerly one of the world’s largest crypto exchanges that fell rapidly into bankruptcy, took a new turn today after Vox published a series of messages with its former CEO Sam Bankman-Fried. The erstwhile executive, known in the crypto world as SBF, discussed regulators, ethics and bankruptcy regrets, amongst other issues that have become the de jure conversation in tech since FTX itself immolated.

“Everyone goes around pretending that perception reflects reality, it doesn’t,” SBF said in a Twitter conversation with reporter Kelsey Piper. “Some of this decade’s greatest heroes will never be known, and some of its most beloved people are basically shams.”

In the notes, shared in screenshot form by the publication, SBF spoke harshly of regulators, saying that they “make everything worse” and that “they don’t protect customers at all.” Given that SBF’s former company will soon face at least the American Congress, the approach and tone are notable.

His take on regulators is predicated, later messages make clear, on his view that their methods of control are too simplified — “just ‘do more business’ vs ‘do less business’ and ‘put up more moats’ vs ‘put up fewer moats’” — which doesn’t distinguish “between good and bad” in his estimation.

The Vox interview spent a good chunk of its time discussing ethics and philanthropy, an unsurprising choice given that SBF was a well-known person in the “effective altruism” movement, a method of helping others that focuses on what is practical. SBF was also an active political donor until recently, further keeping him in the media limelight.

Back on the matters most pertinent to TechCrunch, while discussing his own activities, SBF wrote that he “didn’t want to do sketchy stuff [as] there are huge negative effects from it,” adding in a following message that he “didn’t mean to.” Last week, SBF officially stepped down as chief executive of FTX while Enron wind-down veteran John J. Ray III was appointed as the new CEO.

In response to SBF’s public statements, although we’re not exactly sure which ones as there are many, Ray published a statement saying that “Mr. Bankman-Fried has no ongoing role at FTX…and does not speak on their behalf.”

Later in the conversation with Vox, SBF brought up CZ, the well-known leader of Binance, the largest crypto exchange in the world. CZ and SBF’s dueling Twitter accounts up until, and after, the FTX meltdown centered the attention of the world on their different business approaches, and leverage.

“A month ago CZ was a walking example of ‘don’t do unethical shit or your money is worthless,’” SBF Wrote, “now he’s a hero,” later asking if the shift in his view of market perception of CZ was due to his being virtuous, or simply having had the “bigger balance sheet,” leading to CZ winning and not SBF. CZ’s comments about FTX’s native token FTT are viewed by some as a precipitating event in the collapse of the latter exchange; precisely where blame lies is not yet entirely clear, so grains of salt, please.

Interestingly enough, Bankman-Fried tells Vox that his “biggest single fuckup [was] the one thing everyone told” him to do: file for Chapter 11 bankruptcy. He thinks if he hadn’t filed for bankruptcy, “withdrawals would be opening up in a month with customers fully whole.”

He adds: “But instead I filed, and the people in charge of it are trying to burn it all to the ground out of shame.” So Vox inquired whether he was suggesting he should’ve just kept trying to raise the $8 billion lifeline. SBF added that he might still get there, but with way more “collateral damage.”

Damage is correct. The impact is still being felt; at the other firms in the crypto trading and investing business or the smaller individuals and businesses that had assets on the platform (pre-bankruptcy). The fall-out even hurts early-stage entrepreneurs, with MIT Media Lab canceling its fellowship that was originally backed by FTX Future Fund.

There are entire chapters, if not volumes to come. And thankfully for those of us observing, and reporting, SBF continues to talk.

Want to hear from the best and brightest in crypto? Attend TechCrunch Sessions: Crypto on November 17 in Miami. Get your tickets here.
\ No newline at end of file diff --git a/packages/readabilityjs/test/test-pages/thevaluable.dev/distiller.html b/packages/readabilityjs/test/test-pages/thevaluable.dev/distiller.html new file mode 100644 index 000000000..5d7dfa50b --- /dev/null +++ b/packages/readabilityjs/test/test-pages/thevaluable.dev/distiller.html @@ -0,0 +1,423 @@ +
Is inheritance that evil?

+ “You used inheritance in your code! Are you crazy? It’s forbidden! It’s clearly written in the Laws and Mantras of Good Software Practice Everyone Must Follow™ hanged in the toilets!” +

+ Your thoughts about the nice Youtube video full of cute dogs you saw yesterday stop abruptly. Anxious, you look on your right: Dave, your colleague developer, is yelling at Davina, your desk neighbor. All three of you are working for the fantastic company MegaCorpMoneyMaker, the famous e-commerce which can sell ice to penguins. +

+ “That’s true, Dave, you’re right. I used inheritance”, begins Davina. “No need to scream. You could have written it during the code review.” +

+ Dave, disarmed by her calm and her honesty, replies: “I… that’s true but… well… I wanted to make an example! We should ban the Demon of Inheritance from the surface of Earth. It will destroy our codebase, our companies, our jobs, and our lives.” He’s now addressing the whole open-space. “Inheritance is evil! It has always been, and it will always be. Composition will save us all!”. +

+ Inheritance is considered as a “pillar of OOP” in many articles, books, and other resources on software development. But, at the same time, many developers, like Dave, will recommend not using it in every possible context. Why is that? +

+ We’ll try to answer this question throughout this article. In particular, we’ll see: +

  • The properties of this concept we call inheritance.
  • A brief history of inheritance to understand where it comes from.
  • How powerful single inheritance can be.
  • Composition versus single inheritance.
  • The legacy of inheritance.
  • Concrete use of inheritance.
  • How modern languages (Rust and Golang) implement inheritance.

+ The few examples of this article are written in PHP. Don’t worry if you don’t know it, it’s very easy to understand (when you don’t do complicated stuff with it). I don’t follow the Perfect Formatting the PHP Grandmasters follow, so don’t feel sad about that. +

+ Dave and Davina are ready. Secure yourself and let’s go! +

+ What’s Inheritance? +

+ After Dave finishes his speech, Davina begins to explain her point of view. +

+ “I don’t think inheritance is bad in every situation.” +

+ She pauses, considering. “First, To be sure we understand each other, let’s decompose the concept of inheritance. We might discover some misconceptions and learn from each other.” +

+ “Misconceptions…” repeat Dave, doubtful. “It’s perfectly clear for me, but go ahead. Let’s see if you understand it”. +

+ Defining Inheritance +

+ Davina remembers an interesting definition of inheritance encapsulating some core ideas: +

+ A class may inherit - use by default - the fields and methods of its superclass. Inheritance is transitive, so a class may inherit from another class which inherits from another class, and so on, up to a base class (typically Object, possibly implicit/absent). Subclasses may override some methods and/or fields to alter the default behavior. +

+ Let’s decompose this definition: +

  • superclass: a class which has at least one subclass.
  • subclass: a class which is a descendant of at least one superclass.
    • It can add new methods or properties, or use methods or properties of the superclass.
    • It can modify (or override) some methods or properties of the superclass.
  • base class : a superclass which is not a subclass.

+ In many common programming languages, inheritance has also these two properties you can use together: +

  • Multilevel inheritance: a subclass can be a superclass of other subclass(es).
  • Hierarchical inheritance: a superclass can have more than one subclass.

+ Finally, inheritance systems can have one of the following properties but not both: +

  • Single inheritance: a subclass can only have one superclass.
  • Multiple inheritance: a subclass can have more than one superclass.

+ At that point, both Davina and Dave agree to speak mostly about single inheritance. Most programming languages won’t allow you to use multiple inheritance anyway. More on that later. +

+ Since inheritance is a hierarchy of superclasses and subclasses, it’s easier to show them than to describe them. Davina draws with the precision of a beaver and the eye of an eagle the following: +

Single, multiple, hierarchical, and multilevel inheritance

+ Inheritance for Code Reuse +

+ “What can we already see with what we have here?” asks Davina. +

+ You answer this one: “We can see that a given subclass inherit all the properties and behaviors of all of its superclasses. It means that the leaves of our tree (the superclasses without subclasses) concentrate all the behaviors of all their parents”. +

+ Imagine if humanity could conceive children with all the knowledge of each of their ancestors. How smart would they be? Well, maybe not as much as you think. The processing power of our brain is limited, that’s why we have difficulty to put in our head all the details of a complex codebase. As a result, it’s likely that our children would be lost in an ocean of knowledge. +

+ Similarly, the more levels of hierarchy you have in your inheritance tree, the more complex the class inheriting from all this knowledge will be. We will have difficulties to think and reason about them if we need to maintain or modify them. +

+ Inheritance for Specialization +

+ For now, we only talked about inheritance as a way to inherit the implementation of superclasses in subclasses. That’s not all: many programming languages allow you to substitute a superclass by one of its subclass thanks to polymorphism. The subclasses can be seen as specialization of their superclasses. +

+ This is a very important piece of the inheritance puzzle we’re lying here. Let’s take this exciting example: + 

<?php declare(strict_types=1);
+
+class Parser
+{
+    public function count(string $filepath) {
+        printf("I'm counting lines of %s pretty hard!", $filepath);
+    }
+}
+
+class JSONParser extends Parser {
+    public function parse(string $filepath) {
+        printf("I'm parsing some JSON file %s pretty hard!", $filepath);
+    }
+}
+
+class Shipment
+{
+    private Parser $parser;
+
+    public function __construct(Parser $parser) {
+        $this->parser = $parser;
+    }
+
+    public function count(string $filepath) {
+        $this->parser->count($filepath);
+    }
+}
+
+$shipment = new Shipment(new JSONParser());
+$shipment->count("/my/superb/shipment.json");
+

+ What’s happening here? +

  1. We feed to the object Shipment a new JSONParser.
  2. Even if Shipment expect an object of type Parser, you can give JSONParser instead because it’s a subclass of Parser. That’s good old polymorphism here!
  3. If you run this code, the surprising output “I’m counting lines of /my/superb/shipment.json pretty hard!” is displayed before your amazed eyes. I know, it feels like Christmas.

+ Dave, who begins to get bored, shoot: “I know what’s next. You’ll speak about the superclass Animal and its subclasses, Dog and Platypus. I’m not stupid! I’m a Senior Web Developer for 18.3 years now! The CTO told me that I was a Ninja of the Crown this morning and…”. +

+ With a gracious and meaningful movement of her left hand, Davina stop Dave in his enthusiastic show-off. Her eyes light up and her calm but determined voice begins to fill the entire open space. You’re still following the conversation, as well as other colleagues who began, curious, to gather around your desks. The tension begins to rise. +

+ “Not at all”, begins Davina. “OOP was never meant to represent objects or living creatures surrounding our daily life, like a dog, a car, or a coffee machine. It was invented to solve specific problems linked to software development. In particular, it was designed because we can’t reason about complex systems with our limited brainpower. Inheritance is part of this process of problem solving, and we should analyse it in this context.” +

+ She continues. “How many times did you create a Cat class, or a Dog class? You didn’t, not even once. You implemented abstract concepts, like a login or a parser, which have nothing to do with these “real-life” examples. Even if some of your classes are loosely related to real life objects, like a Shipment class, you shouldn’t even use inheritance for those, as we’ll see later.” +

+ … you should be wary of attaching too much importance to the notion that object-oriented systems are directly deduced from the “real world”. +

+ The open space is now silent. Everybody begins to realize the lies which was taught to them all these years. +

+ Davina goes on. “These real-life examples confuse you: you have the impress that you understand inheritance because animals and dogs are familiar. It’s only a mere illusion. Inheritance can bring a lot of complexity.” +

+ Dave shrugs. Suddenly, Davina stands up, addressing what is now her audience. “Let’s see where inheritance come from and why it was invented. I’ll tell you now The Story of Inheritance.”. +

+ You see by the window a lightning streaking across the sky. A storm is coming. +

+ A Brief History of Inheritance +

Once upon a time in the land of inheritance

+ Inheritance has been a hot subject since the creation of the OOP paradigm itself. The first programming languages implementing it was Simula in the 60s. Simula created also most of the concepts we take for granted in OOP, like classes and objects. +

+ The next big step for OOP and inheritance was Smalltalk, a programming language created by Alan Kay and his team at Xerox Park. It’s interesting to note that the first implementation of Smalltalk didn’t include inheritance. From Alan Kay himself: +

+ I didn’t like the way Simula I or Simula 67 did inheritance (though I thought Nygaard and Dahl were just tremendous thinkers and designers). So I decided to leave out inheritance as a built-in feature until I understood it better. +

+ Dan Ingalls was part of Alan Kay team and ended up implementing five generations of Smalltalk environments. He liked inheritance, and implemented it in every version of Smalltalk following the first one. It’s where the disagreement with inheritance began in Software Development; this debate continues today. +

+ What problem Dan Ingalls tried to solve with inheritance? Code reuse. He wanted a mechanism which could help programmers not repeating the same code in different classes. He wanted a system making the knowledge codified in a codebase more general. +

+ Another language designer was heavily influenced by Simula: Bjarne Stroustrup, who created “C with classes”. The goal was to design and reason about complex system more easily. The first implementations of the language were copying many ideas from Simula, including inheritance, while being different from Smalltalk. +

+ “C with classes” became C++. As time passed, the language gained a lot of traction: more and more programmers were using it. Smalltalk, after a huge success, began its descent into the Pit of Forgotten Languages. At the end of the 80s, C++ was the only language implementing multiple inheritance, something many believed impossible to achieve. +

+ But C++ wasn’t the only attempt to extend C with classes. Apple Computer had its own version called Objective-C. The language was extended by Steve’s Job team at NeXT at the beginning of the 90s, and, among other things, they implemented a construct sharing similar properties with single and multiple inheritance. Similar, but not identical: only the interface was inherited, not the implementation. They called this new construct a protocol. +

+ Java, in the middle of the 90s, implemented exactly the same thing, with a different name: interface. I hate this name, it’s too easy to confuse this “interface” with the more general idea of interface (ways from the outside of a construct to act on its inside). That’s why I call this Java “interface” the interface construct in my articles. +

+ Davina pauses, drink a bit of water, and adds: “When you read in a random tutorial that a class Dog and a class Platypus inherit from a class Animal, do they speak about code reuse?” +

+ You begin to imagine the “code” an Animal could have. Are we in the Matrix? +

+ “Not at all. You speak about specialization: a Cat is a specialized form of Animal, although in this context it doesn’t really make sense either. These examples are just lame. Anyway, inheritance is a concept which can bring many powerful features, and that’s its main problem, as we’ll see below. That’s why it was discussed for so long and the concept was ultimately dumbed down”. +

+ How Powerful Is Single Inheritance? +

Inheritance is as powerful as Hulk!

+ To understand the drawbacks of single inheritance your colleagues, friends, and dogs are complaining about, let’s decompose first what we can precisely do with single inheritance of implementation: +

  1. You can add some behavior in a subclass.
  2. You can override (modify) the behavior of any superclass in a subclass.

+ Let’s see what the possible benefits and drawbacks of these two approaches. +

+ Adding Behavior In a Subclass +

+ Here’s a slightly modified version of our legendary parser: + 

<?php declare(strict_types=1);
+
+class Parser
+{
+    public function count(string $filepath) {
+        printf("I'm counting lines of %s pretty hard!", $filepath);
+    }
+}
+
+
+class JSONParser extends Parser {
+    public function parse(string $filepath) {
+        printf("I'm parsing some JSON file %s pretty hard!", $filepath);
+    }
+}
+
+class Shipment
+{
+    private JSONParser $parser;
+
+    public function __construct(JSONParser $parser) {
+        $this->parser = $parser;
+    }
+
+    public function count(string $filepath) {
+        $this->parser->count($filepath);
+    }
+
+    public function import(string $filepath) {
+        $this->parser->parse($filepath);
+    }
+
+}
+
+$shipment = new Shipment(new JSONParser());
+$shipment->count("/my/superb/shipment.json");
+$shipment->import("/my/superb/shipment.json");
+

+ The class JSONParser inherit the implementation of Parser and add its own method parse. +

+ How our system supports changes in this example? If we modify the behavior of the method count from Parser, every subclass inheriting from Parser will get the change too. In that sense, inheritance breaks encapsulation between the superclasses and their subclasses. +

+ In languages with inheritance, a data abstraction implementation (i.e., a class) has two kinds of users. There are the “outsiders” who simply use the objects by calling the operations. But in addition there are the “insiders.” These are the subclasses, which are typically permitted to violate encapsulation. +

+ Let’s imagine that Parser has two subclasses, and these subclasses have two more subclasses. You end up with a hierarchy on 3 levels. It doesn’t seem that much of a stretch, but you still end up with seven classes in total. If you modify the base class Parser, six other classes will be affected! +

+ We didn’t create a huge inheritance tree here, but changing a superclass has a rippling effect in the whole hierarchy. +

+ Hierarchical systems seem to have the property that something considered as an undivided entity on one level, is considered as a composite object on the next lower level of greater detail. +

+ Don’t get me wrong: it can be beneficial if you want that each change of a superclass affects every subclass at every level below. Actually, it’s the main reason why inheritance was invented at the first place: being able to make the code more general and reusing it easily. But you need to be sure that your classes are very cohesive, that is, the change of any superclass needs to affect every subclass. +

+ If your classes are not cohesive, you’ll have all the drawbacks of tight coupled classes in your face: nobody will know if changing a superclass will either cover them with glory and fame or crash the entire system. Your codebase, while growing in complexity, will become impossible to reason about, because you don’t have enough brainpower to build in your head an accurate mental model of all the effects of a change. In short, you’ll end up with one of the problem the OOP paradigm tried to solve at the first place. +

+ Overriding Behavior in a Subclass +

+ Let’s continue further by adding a new element in our inheritance soup: overriding. Here’s another simple example: + 

<?php declare(strict_types=1);
+
+class Parser
+{
+    public function count(string $filepath) {
+        printf("I'm counting lines of %s pretty hard!\n", $filepath);
+    }
+}
+
+
+class JSONParser extends Parser {
+    public function count(string $filepath) {
+        printf("I'm counting JSON objects from file %s pretty hard!\n", $filepath);
+    }
+}
+
+class Shipment
+{
+    private Parser $parser;
+
+    public function __construct(Parser $parser) {
+        $this->parser = $parser;
+    }
+
+    public function count(string $filepath) {
+        $this->parser->count($filepath);
+    }
+}
+
+$shipment = new Shipment(new Parser());
+$shipment->count("/my/superb/shipment");
+
+

+ We override here the method count in our subclass JSONParser. Now, our class Shipment will “work” if we pass to our new Shipment both Parser or JSONParser, in the sense that no type error will be thrown. +

+ But will it works as intended? What parser to use when we want an instance of Shipment? A Parser? A JSONParser? +

+ We could look at the implementation of Shipment, then at the implementation of both JSONParser and Parser to decide what behavior we need depending on the context. But we create objects and abstract behavior not to look at their implementations. It frees us some precious brain power to think about the part of the system we want to modify. +

+ Mixing overriding and polymorphism is a recipe for disasters. In an inheritance tree with more classes and more overidding, you need to know what superclass override what behavior, if the subclass override what the superclass overrided, and so on. +

+ This confusion between inheritance of implementation and specialization led to the Liskov Substitution Principle (LSP). +

+ The Liskov Substitution Principle +

The LSP is about substituting a parent class with its base class

+ Barbara Liskov was a researcher who won the Turing Award for her work on abstract data types. When she was asked to talk at the keynote of OOPSLA in 1987, she looked at the papers about inheritance hierarchies and how developers were using them. She was pretty disappointed. +

+ This keynote led to the paper Data abstraction and hierarchy. From there, some began to speak about a “Liskov Substitution Principle”, often quoting the following: +

+ If for each object o1 of type S there is an object o2 of type T such that for all programs P defined in terms of T, the behavior of P is unchanged when o1 is substituted for o2, then S is a subtype of T. +

+ Who doesn’t like good old academic writing full of S, T, o2, and friends? Here’s a clearer way to define the same idea: +

+ Objects of subtypes should behave like those of supertypes if used via supertype methods. +

+ It’s with this paper that the concept of subtyping was born. +

+ This solves the problem of substitution of subclasses. According to Liskov, if you want to use polymorphism with inheritance, you need to have proper subtyping. How? By following this rule: when you substitute one superclass by its subclass, the behavior of the whole system should be unchanged. +

+ Said differently: don’t override anything. +

+ With this definition of the LSP, and to come back to our example above, there won’t be any doubt about the consequences substituting Parser by its subclass JSONParser. We are sure it won’t have unexpected results because JSONParser doesn’t override any behavior of its superclass. Said differently, you would always use the same method parse from Parser whatever the subclass of Parser you use. +

+ But between Barbara Liskov’s first statement and now, the LSP changed. From “the behavior should stay the same”, we know think that “the behavior shouldn’t break the application”. This last definition is more ambiguous: how do we know that our system still behave correctly? What does it mean? Do we have every possible tests to ensure that it’s the case? If our system doesn’t break but doesn’t follow the specifications either, is it a violation of the LSP? +

+ “But wait!” interrupts suddenly Dave. “This is not the Liskov Substitution Principle! This is not how it’s defined in the Holy SOLID Principles!” +

+ Davina sigh. “The SOLID principles should be the D principle. The last one is the only one we can still save. The others are misinterpretations of important ideas when they’re not bad ideas. The LSP is a misinterpretation: the definition given by Robert Martin has not much to do with the definition given by Barbara Liskov.”. +

+ Let’s look at Martin definition of the LSP: +

+ All implementations of interfaces are subtypes of an interface. +

+ From the example above, the class JSONParser implements the same interface as Parser, but one method count count the number of lines and the other count the number of JSON objects. Interface substitution is not what Barbara Liskov was speaking about, and it won’t save your codebase if you try to mix specialization and overriding. +

+ Often, developers don’t like strong behavioral subtyping as Barbara Liskov defined it. That’s why the principle was transformed over time. Often, inheritance is used to override the implementation of a superclass. On that regard, it’s interesting to note that inheritance is only interesting for Liskov in the context of subtyping; she doesn’t see any value to use it for inheriting implementation. +

+ Why? Because inheritance is not the only solution for code reuse. Many prefer using composition. +

+ Composition vs Single Inheritance +

+ While the open space is still silent, you begin to feel the tension dropping. The magic word has been pronounced: composition. While inheritance is a demon which tries to eat companies and their employees, composition is the solution to every possible disaster. +

+ Composition, Delegation, or Aggregation? +

+ “Are we speaking about composition, delegation, or aggregation here?” asks Dave. “What’s the difference?” ask another colleague. +

+ Dave, with a smile, begins his explanation: “Look at the examples we were speaking about. A JSONParser is a parser, so inheritance makes sense in that case. But, for example, a Shipment has a parser, that’s why we used composition”. +

+ You intervene: “does it make sense to say that a shipment has a parser? No, here we’re speaking about delegation: the shipment use a parser, it delegates a task to a parser object. That’s all.” +

+ “Really?” begins another colleague. “Are you sure a JSON parser is a parser, or does it has the behavior of a parser?” Another colleague takes part of the conversation: “No! Our Shipment own a Parser, so we’re speaking about aggregation here!” +

+ Outside, the thunder growl again. Everybody begins to speak at the same time, throwing at each other is-a, has-a, part-of, add-to, and other pair of very short words you can link with a hyphen. +

+ Davina listens to the conversation carefully, and when everybody calms down, she gives her opinion: “I see these ‘is-a’ or ‘has-a’ tricks all over the Internet. I also saw many developers defining different flavors of composition, delegation, aggregation, and whatnot. At the end, our problem are often so specific they don’t fit any of these definitions. They are useless in practice. Don’t use them”. +

+ She continues. “Using natural language tricks (like is-a or has-a) to decide what solution we should apply to a problem is ambiguous, as we just witnessed. It’s one of the reason why Mathematical notation was invented at the first place: to avoid the ambiguity of natural language. My advice: don’t use these tricks to decide what solution you should use.” +

+ To understand what Davina is speaking about, let’s get back to our class Shipment: + 

<?php
+class Shipment
+{
+    private JSONParser $parser;
+
+    public function __construct(JSONParser $parser)
+    {
+        $this->parser = $parser;
+    }
+
+    public function import(string $filepath)
+    {
+        $this->parser->parse($filepath);
+    }
+}
+

+ When we want to create an instance of Shipment, we need to inject an object of type JSONParser. It doesn’t matter if it’s called aggregation, composition, or delegation. At the end, it boils down to the same simple mechanism: injecting an object into another one. +

+ Is Composition Better Than Inheritance? +

+ We all know the Mantra of Composition, the one which will bless your codebase with the benediction of The Hasa and the Partof Gods. If you don’t know it yet, I’m sure you’ll hear it a good hundred of times in your career: +

+ Favor object composition over class inheritance. +

+ This is from the book Design Patterns, written by the Gang of Four. With a name like that, I’m not sure if they were trying to force some general-but-specific (admire the paradox) solutions to our poor codebases or if their real goals were to create the mafia of software developers. One thing is certain: this book gave to beginners the perfect pretext to show how smart they are by instantly changing a healthy codebase into a legacy mess full of Singleton and Abstract Factories. +

+ Like many, I’m no innocent: I’ve chanted the Mantra of Composition for years, without looking at the Mantra in its context. But context is important. +

+ So, what our godfathers Gang of Four are saying just after enlightening the world with their Mantra? +

+ You should be able to get all the functionality you need just by assembling existing components through object composition. But this is rarely the case, because the set of available components is never quite rich enough in practice. Reuse by inheritance makes it easier to make new components that can be composed with old ones. Inheritance and object composition thus work together. +

+ According to this book, we should favor composition not because inheritance is evil, but because nobody uses it correctly. Well, hopefully we understand it better now. +

+ The Benefits of Composition +

+ Composition is very useful indeed. Let’s imagine that we inject object A into object B. Here are the benefits: +

  • If Object A is properly encapsulated, nobody will destroy anything by changing object’s A inner implementation. Object B can suffer the change, but nothing else down the road.
  • You can use only part of the object’s implementation. Object B doesn’t automatically inherit from the whole object A.

+ To get back to our Shipment example, this means that the object JSONParser we inject is tightly coupled to the class Shipment, but this coupling stop there. If you instantiate Shipment later and you change the implementation of JSONParser, the class Shipment might need to change, and that’s all. +

+ As we saw with inheritance, the problem of tight coupling (or the benefit of cohesion) will affect every layer down the inheritance tree. +

+ The Drawbacks of Composition +

+ Composition is not the best solution when you want to use many objects or objects with a lot of behavior. +

+ Let’s say that you want to use 10 methods from 3 different objects and you want to add some implementation on top: you’ll need to inject your 3 objects, create 10 methods in your new class wrapping the 10 methods of the objects injected, and add more methods to take care of your new functionality. +

+ “But I could directly call the object JSONParser from an instance of Shipment”, cut Dave. +

+ “That’s true, answers Davina. But it would break the encapsulation of our objects Shipment in that case. It means that everything using the object Shipment would be tightly coupled to the object JSONParser. Encapsulation is broken.” +

+ Composition doesn’t bring you the benefits of subtyping either. When you inject an object to a class, you’ll need to inject a precise object if your language has some sort of type checking, and nothing else. On that regards, it constrains you (which can be a good thing!). If your language doesn’t have any type checking and you can just give any object to the constructor of your class, it doesn’t mean that it will work as intended. The problem stays the same. +

+ At least, an inheritance hierarchy can indicate what object you can use instead of another and, if it follows a strict form of LSP, nothing should break. +

+ “That’s wrong!”, shoot Dave, suddenly. “What about the interface construct? I love these, and you can do some good polymorphism with them without using the Demon of Inheritance!” +

+ “You’re right, answers Davina. But using interface constructs is only using another form of inheritance.” +

+ Single and Multiple Inheritance Dumbed Down +

Over the years, inheritance has seen less powerful implementations.

+ As we saw, inheritance is very powerful, because you can mix reuse of implementation and subtyping in a hierarchy tree as deep as you want it to. This power is its biggest problem: many developers, not knowing all the implications we saw above, have a tendency to misuse inheritance, tightly coupling everything in huge inheritance hierarchies, which led to the Mantra of Composition. That’s why many gave up on single inheritance. +

+ But multiple inheritance is even more flawed: the possibility for a subclass to have more than one superclass is making everything very ambiguous. As an example, you can look at the diamond problem. Additionally, multiple inheritance is very complex to implement in a programming language. +

+ That’s why the designers of Objective-C and Java restricted inheritance with the protocol and the interface construct respectively. The benefits? +

  1. A class is forced to use the interface given by a protocol, which guarantee that each subtype has the same interface (but doesn’t guarantee that the substitution will work!).
  2. There is no multilevel inheritance anymore.
  3. No more inheritance of implementation: the protocol is only an interface.
  4. Multiple inheritance of interfaces is much easier to manage than multiple inheritance of implementations.

+ Please note that even Java includes a limited form of multiple inheritance: inheritance of interfaces.” +

+ Concrete Use of Inheritance +

+ We saw already some potential use of inheritance, but can we be more concrete? Over the years I’ve come up with this set of rules: +

  1. Never use inheritance for classes which are related to the business domain.
  2. Sometimes use inheritance for reusing implementation of classes bringing some mechanics.
  3. If you use subtypes, always try to respect the original strictness of LSP as much as possible.
  4. If you use the interface constructs for subtyping, keep in mind that the implementation of the interface can still break everything.

+ These rules are from my experience. Don’t use them as Mantras working in every situation. We should experiment carefully with them and use our brain to see if the technical solutions fit the problem you have. +

+ Inheritance and Domain Objects +

+ Davina explain further: “When I speak about domain objects, I mean all the objects which are related to the business we work for. In our present case, in MegaCorpMoneyMaker, it would be classes like Shipment, Order, or Product.” +

+ Introducing any form of hard coupling or premature abstractions with these objects is always dangerous. They are the representation of real world constructs, and since the real world change in unpredictable manners, these objects will change in unpredictable manners too. Keep them isolated as much as possible from the mechanical parts of your system. +

+ Inheritance and Mechanics +

+ You ask Davina: “what do you mean by mechanics”? +

+ “These classes are everything which are not domain objects. For example, our classes to parse files represent some mechanisms: they don’t represent anything from our business, they’re just general constructs to parse some files. Objects of this sort are often more general and can be applied in many more contexts than our precise business domain.” +

+ For example, it’s not very likely that the world will come up tomorrow with a different definition of stacks. That’s why the object Stack won’t need many changes overtime. +

+ Anything representing mathematical constructs are good examples too. After all, Mathematics try to be as disconnected as possible from the real world. It’s when you try to use mathematical concepts on the real world that everything begins to break. That’s what we call applied Mathematics. +

+ If we think about it, inheritance create a hierarchy where its elements are not encapsulated with each others, but the hierarchy itself is encapsulated from its outside. We create a new construct doing so, an aggregation of objects. In that case, inheritance can be useful if you have to codify a general and coherent set of ideas where the properties and behaviors of the different objects will rarely change, or when the whole hierarchy needs to change when one of its member change predictably. +

+ Modern Languages and Inheritance +

+ Modern languages often take the decision to implement inheritance differently from “older” languages like Java, Python, Ruby, or PHP. They try hard to differentiate subtyping and inheritance of implementation. For example, in Rust’s documentation: +

+ If a language must have inheritance to be an object-oriented language, then Rust is not one. There is no way to define a struct that inherits the parent struct’s fields and method implementations. +

+ But Rust implement some form of inheritance I didn’t cover here: traits. If you need some polymorphism, Rust give you generic programming like many other languages. +

+ Another example: Golang. You can’t do any inheritance of implementation, only composition is allowed. You can also use interface constructs if you want some inheritance of interface. +

+ Inheritance Is Not Evil +

Inheritance is not evil, it's just too powerful.

+ What did we see in this article? +

  • Inheritance create a hierarchical construct composed of superclasses, subclasses, and base classes.
  • Many common programming languages allow us to create a tree with infinite depth (multilevel inheritance) and width (hierarchical inheritance).
  • Your system can become hard to maintain if you mix two of the Three Power Gems of Inheritance in the same soup:
    • Inheritance of implementation.
    • Substituting superclasses with their subclasses (subtyping if it follows some rules).
    • Multilevel inheritance.
  • Most of the time, composition seems to be the best alternative to inheritance of implementation.
  • Everything is tightly coupled in an inheritance hierarchy, but this blurb of classes is still encapsulated from the outside.
  • Inheritance was a crude concept defined at the beginning of OOP and later refined with, for example, the interface construct.
  • Inheritance can be useful for the part of your system which won’t change too much (mechanical part).

+ If you need to retain one thing from all of that: don’t use DRY, or inheritance, or composition before you understand clearly what’s the problem you’re trying to solve and its context. These concepts should be used when you refactor you code; consider the first writing as a messy draft and, in that spirit, defer all the important decisions making your design hard to change as much as you can. +

+ What should be together and what should not (cohesion) is one of these important decision. What should be under a layer of indirection using interface constructs is another. +

+ If you think it’s a good idea to use an inheritance hierarchy, begin with a small one and see how it behaves in your system overtime. +

+ Davina concludes: +

+ “The concept of inheritance was refined over the years and gave us the constructs we use today, like the interface construct. In that sense, inheritance is definitely a pillar of the OOP paradigm. But it’s true that mixing features which are not necessarily orthogonal make inheritance difficult to harness in many programming languages.” +

+ Everybody is silent now. The storm outside stopped. Dave is thinking hard, like everybody in the open space of MegaCorpMoneyMaker. +

+ At the end, you should always read the documentation of the programming languages you’re using to see exactly how they implement inheritance. You’ll now be able to guess why the language’s designers made their decisions and how you can use their implementation of inheritance effectively. +

\ No newline at end of file diff --git a/packages/readabilityjs/test/test-pages/vanityfair/distiller.html b/packages/readabilityjs/test/test-pages/vanityfair/distiller.html new file mode 100644 index 000000000..3088075cc --- /dev/null +++ b/packages/readabilityjs/test/test-pages/vanityfair/distiller.html @@ -0,0 +1,98 @@ +
+ Levin Report +
+ The president said Tuesday it’s “terrific” we’re 15% of the way to herd immunity. +
+ +
Image may contain Mark Meadows Human Person Audience Crowd Suit Coat Clothing Overcoat Apparel and Tie
By BRENDAN SMIALOWSKI/AFP/Getty Images.

+ Earlier this week, we learned that Pfizer and Moderna, the two drug manufacturers with the most promising COVID-19 vaccines, declined invitations from the White House to attend Donald Trump’s “vaccine summit.” The companies presumably chose to do so in an effort to avoid their drugs being politicized, or being associated with the Trump administration in any way whatsoever. (A third factor may have been a desire not to contract the coronavirus, which at this point is basically a 50-50 chance for anyone who visits the White House.) And now that the event has concluded, you can probably see where they were coming from! +

+ While another, saner, more humanoid president might have kicked things off by congratulating the nation’s hardworking scientists and offering some words of hope during these extremely scary times, Trump chose to begin by playing a deceptively edited video trashing Anthony Fauci, among others: +

+ The event…was the typical Trumpian mixture of political theatrics, brags, grievances and accolades doled out based on fealty to his preferred narrative…. The gathering opened with a video castigating media pundits, [Joe] Biden, and Fauci for offering more cautious assessments of the vaccine development timeline. +

+ But many of the remarks the video highlighted were misleadingly portrayed. Fauci, for instance, was shown saying a vaccine arrival “could be January, could be later,” a statement that is largely true, especially for the majority of the population. Biden was also shown saying there is “no prospect” for the majority of the public to get a vaccine before the middle of 2021, a timeline that mostly tracks with the tentative rollout schedule. +

+ Later, Trump claimed that before he came on the scene, practically no vaccines had ever been created, telling the group assembled: “Before Operation Warp Speed, the typical timeframe for development and approval, as you know, could be infinity.” +

+ https://twitter.com/atrupar/status/1336387395994980353 +

+ Then, in the midst of his “vaccine summit,” Trump suggested that if enough people die from COVID-19, we won’t need vaccines at all. +

+ https://twitter.com/bhrenton/status/1336392398650413057 +

+ To be clear, people contracting a virus is not “a very powerful vaccine,” or any kind of vaccine, period. Also, the estimate for how many people would have to die for the U.S. to achieve herd immunity is approximately 2.13 million, according to The Washington Post. So there’s that. +

+ And hey, it wouldn‘t have been a Trump event in 2020 if it didn’t involve the president of the United States claiming the election was stolen from him and seemingly asking people to help him stage a coup, saying just before he exited the room: “If somebody has the courage, I know who the next administration will be.” +

+ https://twitter.com/atrupar/status/1336393524359663617 +

+ It’s truly a mystery why Pfizer and Moderna wouldn’t want to be part of that. +

+ Advertisement + +

+ If you would like to receive the Levin Report in your inbox daily, click here to subscribe. +

+ Report: Trump is approaching pardons the same way he approaches kissing women he finds attractive +

+ In that he apparently plans to dispense them without getting consent first, even on people who don’t want them. Per Axios: +

+ President Trump isn’t just accepting pardon requests but blindly discussing them “like Christmas gifts” to people who haven’t even asked, sources with direct knowledge of the conversations told Axios…. Trump recently told one adviser he was going to pardon “every person who ever talked to me,” suggesting an even larger pardon blitz to come. As with most Trump conversations, the adviser wasn’t sure how seriously to take the president—although Trump gave no indication he was joking. +

+ The big picture: The president relishes his unilateral authority to issue get-out-of-jail-free cards. Lately, though, he's been soliciting recipients, asking friends and advisers who they think he should pardon. Trump has also interrupted conversations to spontaneously suggest that he add the person he's speaking with to his pardon list, these sources said. +

+ According to Jonathan Swan, “the offers haven’t always been welcome,” with one source apparently feeling awkward because “the president was clearly trying to be helpful but the adviser didn’t believe they had committed any crimes.” Anyway, Merry Christmas! +

+ Nothing to see here, just a data scientist who accused Florida of covering up the extent of the pandemic having her home raided at gunpoint +

+ The Florida Department of Law Enforcement executed a search warrant Monday morning at the home of data scientist Rebekah Jones, who was fired by the state Department of Health in May. The agency is investigating whether Jones accessed a state government messaging system without authorization to urge employees to speak out about coronavirus deaths, according to an affidavit by an agent working on the case. Jones told CNN that she hadn't improperly accessed any state messaging system and that she lost access to her government computer accounts after she was removed from her position. +

+ About 10 officers with guns drawn showed up to her Tallahassee home around 8:30 a.m., Jones said. A video taken from a camera in her house, which she posted on social media, showed an officer pointing a gun up a stairwell as Jones told him her two children were upstairs. Jones said that the officer was pointing his gun at her 2-year-old daughter, 11-year-old son and her husband, who she said were in the stairwell, although the video doesn’t make that clear. Officers also “pointed a gun six inches from my face” and took all of her computers, her phone and several hard drives and thumb drives, Jones said. +

+ Advertisement +

+ According to CNN, an investigator with Jones’s former department alleged that an unauthorized individual had illegally accessed an emergency management system and sent a text message to government officials last month urging them to speak out about the pandemic. “It’s time to speak up before another 17,000 people are dead,” the message said, according to the affidavit. “You know this is wrong. You don’t have to be part of this. Be a hero. Speak out before it’s too late.” According to the investigator, the IP address of the message was allegedly connected to Jones’s house. But Jones told CNN’s Chris Cuomo Monday night that she didn’t send the message, saying, “I’m not a hacker” and adding that the language in the message was “not the way I talk.” She noted the message contained errors, notably that it was under by roughly 430 deaths. “I would never round down 430 deaths,” she said. +

+ Jones also told Cuomo she believes the raid on her house was orchestrated by Florida governor and Trump ally Ron DeSantis, who she’s accused of mishandling the public health crisis. “This is what happens when you challenge powerful and corrupt people,” Jones said. “If he thinks this is going to scare me into silence, he’s wrong.” (DeSantis’s spokesperson told CNN that “the governor’s office had no involvement, no knowledge, no nothing, of this investigation.” +

+ The people of Minnesota have a choice to make +

+ And apparently it’s not going to be an easy one: +

+ https://twitter.com/CallaghanPeter/status/1336402006148337664 +

+ Elsewhere! +

+ Supreme Court denies Trump allies’ bid to overturn Pa. election results (Washington Post) +

+ Biden lays out plan to combat COVID in first 100 days, including requiring masks on interstate buses, trains (Washington Post) +

+ Andrew Yang calling NYC elected officials to gauge mayoral run (Politico) +

+ He Pretended to Be Trump’s Family. Then Trump Fell for It. (NYT) +

+ Nigella Lawson’s pronunciation of “microwave” drives internet wild (NYP) +

+ Widowed otters move in together after meeting on dating app (NYP) +

+ More Great Stories From Vanity Fair        +

+ — Mary Trump Thinks Her Uncle’s Postpresidency Woes Are Just Beginning
+ — There’s a Wave of COVID Patients Who Don’t Believe It’s Real
+ — Doug Band: Confessions of a Clintonworld Exile
+ — Will Rupert Murdoch Spring for a Postpresidential Fox Gig?
+ — Ivanka Desperately Tries to Rehab Her Image on Her Way Out
+ — After Remaking CNN and Antagonizing Trump, Jeff Zucker Eyes the Exits
+ — With COVID Vaccines Approaching, Is the FDA Ready to Inspect Where They’re Made?
+ — From the Archive: Probing the Nightmare Reality of Randy Quaid and His Wife, Evi
+ — Not a subscriber? Join Vanity Fair to receive full access to VF.com and the complete online archive now. +

+ Bess Levin is a politics correspondent at Vanity Fair. +

+ Get the Hive Newsletter +

+ The freshest-and most essential-updates from Washington, Wall Street, and Silicon Valley. +
+ Read More +
\ No newline at end of file diff --git a/packages/readabilityjs/test/test-pages/variety/distiller.html b/packages/readabilityjs/test/test-pages/variety/distiller.html new file mode 100644 index 000000000..40e693db9 --- /dev/null +++ b/packages/readabilityjs/test/test-pages/variety/distiller.html @@ -0,0 +1,37 @@ +
Trilith
Courtesy Trilith

+ Imagine that you’re a makeup artist or a production manager on a Marvel movie. You’re working long hours and barely have time to shop for groceries or get in any exercise, let alone do yardwork or socialize. But what if your house was a pleasant 10-minute stroll from the studio, during which you passed a craft beer bar and outdoor concert stage? You’re too tired when you get home to think about grocery shopping, but that’s OK, because the delivery robot has just rolled up in front of your house with your weekly order. +

+ That’s the almost-too-utopian promise of Trilith, the town portion of Trilith Studios (formerly Pinewood Atlanta), the homebase for Marvel’s Georgia productions. Pinewood’s (now Trilith) stake in the studios and town was bought out by River’s Rock Trust in 2020. River’s Rock is the independently managed trust of Chick-fil-A principal Dan T. Cathy and his family. +

+ ‘Pleasure’ Trailer: Neon’s NSFW Porn Drama Is One of the Most Shocking Films of 2022 +

+ The wholesome suburban development might not appeal to everyone, but its innovative, environmentally conscious design shows that new home developments don’t have to be generic cookie-cutter boxes. +

+ Originally called Pinewood Forest, the four-year-old residential portion of Trilith was modeled after the building styles and village feel of European cities, plopped into the middle of Georgia countryside 13 miles from the Atlanta airport. But it’s not a quaint replica town, nor is it aiming to rope in fervent fans in the same way as Disney’s housing developments. +

+ Instead, Trilith is a new town for about 5,000 people, created using the principles of New Urbanism, a concept that means neighborhoods should be walkable and compact, with varied building types and less emphasis on cars — basically the polar opposite of the vast majority of tract house developments across the country. Not everyone who lives at Trilith works in entertainment — about 30% of single-family home residents and about 70% of apartment dwellers have some association with the productions at Trilith Studios, according to the developer. +

+ Designed by architects and town planner Lew Oliver Inc., some of Trilith’s houses and condos are reinterpretations of traditional European and British-style designs, while others sport more modern facades. They’re built closer together than the typical tract house to make the neighborhood more walkable, most with no yards to speak of, but plenty of pocket parks — a boon to busy production workers. Garages are in back so cars are de-prioritized, and the energy-efficient homes are powered by geothermal energy from underground. +

+ “We’ve got a waiting list,” says Trilith Development president Rob Parker. “If you come from the film industry, it’s easy living.” The single-family homes are priced around $700,000 and up — more than twice what homes go for in surrounding neighborhoods, but reasonable by L.A. or N.Y. standards. +

+ Sorry, Marvel fanatics — you won’t find any streets named after the Avengers or other corny tie-ins, though you might stumble across “WandaVision” being shot on your street. “There’s about a dozen different looks” for shooting in the town, says Parker, who points out that Wakanda was built in the open area down the road. Meanwhile, “Family Feud” shoots on the town side. Trilith village includes coffee shops, restaurants, a multiplex, fitness facilities, retail shops and community gathering spaces, along with 51% of the 200-acre town devoted to green spaces. +

Lazy loaded image
Trilith’s small lake is a popular spot for dog-walking. Courtesy Trilith

+ Some production workers relocating from New York or Los Angeles find the convenience worth the higher price. +

+ “I feel like I’m actually living here,” says stunt woman Shauna Galligan, an early Trilith home buyer. Galligan, who worked on “Avengers: Infinity War” and “Avengers: Endgame,” sold her house in Los Angeles and says enough industry people have now moved to the neighborhood — an actress, an assistant director, a director — that “we could pretty much film an entire movie with the people we have now.” +

+ “I have three dogs, we live at the lake, watching the sun go down,” she says. Galligan likes remembering shooting films like “Thor: Ragnarok” and “Guardians of the Galaxy” when she looks out at the stages across the lake. +

+ Galligan says the higher home prices didn’t dissuade her, because “It’s got that style you wouldn’t see anywhere else. I was blown away by the architecture and the details.” +

+ Coby Lefkowitz, a New York-based developer who makes a daily practice of tweeting about new buildings that are both useful and beautiful, recently highlighted Trilith for showing a level of thoughtfulness that’s not often seen in new construction. “We can still build great places in America,” he contends, “There’s a large trend right now of creating more walkable places.” Though the suburban Atlanta area isn’t normally known for its walkability, Lefkowitz applauds Trilith’s parks, trails and interior walkways, as well as thoughtful home and commercial building design. “There’s a lot of variety, which is really important,” he says. +

Lazy loaded image
A micro-home area at Trilith has a European village feel. Courtesy Trilith

+ Another thing that’s fairly revolutionary for new housing developments, Lefkowitz points out, is that Trilith isn’t just single-family homes — apartments, a hotel and “microhomes” (basically small condos with community courtyards) — make the community accessible to a wider range of incomes and work arrangements. +

+ Lefkowitz sees only a few drawbacks to Trilith’s grand vision. The town isn’t integrated into a broader transportation system, he points out, since suburban Atlanta doesn’t have a deep public transportation system to begin with. And despite a variety of housing types, none of them are categorized as affordable housing. That means that while actors, catering managers and crew members could potentially afford to live there, it’s unlikely that the maintenance crew would be able to. But those considerations aren’t as big an issue in Georgia, he says, where lower-priced housing is generally more available. Trilith isn’t perfect, but “It’s still doing more than other developments,” he says. +

+ While Chick-fil-A has distanced itself from making donations to conservative organizations lately, Georgia’s stance on political issues including abortion and voting rights continues to create tensions for entertainment companies working in the state. But Marvel’s productions at Trilith Studios show no signs of slowing down and the town will likely continue to grow. +

+ Parker says Trilith has enough land to double the size of the town. “It’s very attractive to a lot of different people,” he says. +

\ No newline at end of file diff --git a/packages/readabilityjs/test/test-pages/youtube-embed/distiller.html b/packages/readabilityjs/test/test-pages/youtube-embed/distiller.html new file mode 100644 index 000000000..281c6866c --- /dev/null +++ b/packages/readabilityjs/test/test-pages/youtube-embed/distiller.html @@ -0,0 +1 @@ +
\ No newline at end of file diff --git a/packages/readabilityjs/test/view.html b/packages/readabilityjs/test/view.html new file mode 100644 index 000000000..6575cc16a --- /dev/null +++ b/packages/readabilityjs/test/view.html @@ -0,0 +1,1114 @@ + + + + + + + + + + + + + +
+ + + +
+ + diff --git a/packages/readabilityjs/test/view.py b/packages/readabilityjs/test/view.py new file mode 100644 index 000000000..c8bd7bf2a --- /dev/null +++ b/packages/readabilityjs/test/view.py @@ -0,0 +1,49 @@ +#! /usr/bin/env python3 + +from os import listdir +from os.path import isfile, join + +testdirs = [f for f in listdir( + "test-pages") if not isfile(join("test-pages", f))] + +print(""" + + + + + + + + + + + + + + + +
+ + + +
+ + + +""")