diff --git a/android/Omnivore/app/src/main/graphql/CreateHighlight.graphql b/android/Omnivore/app/src/main/graphql/CreateHighlight.graphql new file mode 100644 index 000000000..d4417067d --- /dev/null +++ b/android/Omnivore/app/src/main/graphql/CreateHighlight.graphql @@ -0,0 +1,13 @@ +mutation CreateHighlight($input: CreateHighlightInput!) { + createHighlight(input: $input) { + ... on CreateHighlightSuccess { + highlight { + ...HighlightFields + } + } + + ... on CreateHighlightError { + errorCodes + } + } +} diff --git a/android/Omnivore/app/src/main/graphql/DeleteHighlight.graphql b/android/Omnivore/app/src/main/graphql/DeleteHighlight.graphql new file mode 100644 index 000000000..e61011a3e --- /dev/null +++ b/android/Omnivore/app/src/main/graphql/DeleteHighlight.graphql @@ -0,0 +1,12 @@ +mutation DeleteHighlight($highlightId: ID!) { + deleteHighlight(highlightId: $highlightId) { + ... on DeleteHighlightSuccess { + highlight { + id + } + } + ... on DeleteHighlightError { + errorCodes + } + } +} diff --git a/android/Omnivore/app/src/main/graphql/ReadingProgressMutation.graphql b/android/Omnivore/app/src/main/graphql/ReadingProgressMutation.graphql new file mode 100644 index 000000000..80a30eea1 --- /dev/null +++ b/android/Omnivore/app/src/main/graphql/ReadingProgressMutation.graphql @@ -0,0 +1,14 @@ +mutation SaveArticleReadingProgress($input: SaveArticleReadingProgressInput!) { + saveArticleReadingProgress(input: $input) { + ... on SaveArticleReadingProgressSuccess { + updatedArticle { + id + readingProgressPercent + readingProgressAnchorIndex + } + } + ... on SaveArticleReadingProgressError { + errorCodes + } + } +} diff --git a/android/Omnivore/app/src/main/graphql/UpdateHighlight.graphql b/android/Omnivore/app/src/main/graphql/UpdateHighlight.graphql new file mode 100644 index 000000000..18f93dcea --- /dev/null +++ b/android/Omnivore/app/src/main/graphql/UpdateHighlight.graphql @@ -0,0 +1,13 @@ +mutation UpdateHighlight($input: UpdateHighlightInput!) { + updateHighlight(input: $input) { + ... on UpdateHighlightSuccess { + highlight { + id + } + } + + ... on UpdateHighlightError { + errorCodes + } + } +} diff --git a/android/Omnivore/app/src/main/java/app/omnivore/omnivore/networking/HighlightMutations.kt b/android/Omnivore/app/src/main/java/app/omnivore/omnivore/networking/HighlightMutations.kt new file mode 100644 index 000000000..8c418d63e --- /dev/null +++ b/android/Omnivore/app/src/main/java/app/omnivore/omnivore/networking/HighlightMutations.kt @@ -0,0 +1,36 @@ +package app.omnivore.omnivore.networking + +import android.util.Log +import app.omnivore.omnivore.graphql.generated.CreateHighlightMutation +import app.omnivore.omnivore.graphql.generated.type.CreateHighlightInput +import com.apollographql.apollo3.api.Optional +import com.google.gson.Gson + +data class CreateHighlightParams( + val shortId: String?, + val highlightID: String?, + val quote: String?, + val patch: String?, + val articleId: String?, + val `annotation`: String? +) { + fun asCreateHighlightInput() = CreateHighlightInput( + annotation = Optional.presentIfNotNull(`annotation`), + articleId = articleId ?: "", + id = highlightID ?: "", + patch = patch ?: "", + quote = quote ?: "", + shortId = shortId ?: "" + ) +} + +suspend fun Networker.createHighlight(jsonString: String): Boolean { + val input = Gson().fromJson(jsonString, CreateHighlightParams::class.java).asCreateHighlightInput() + + Log.d("Loggo", "created highlight input: $input") + + val result = authenticatedApolloClient().mutation(CreateHighlightMutation(input)).execute() + + val highlight = result.data?.createHighlight?.onCreateHighlightSuccess?.highlight + return highlight != null +} diff --git a/android/Omnivore/app/src/main/java/app/omnivore/omnivore/networking/ReadingProgressMutations.kt b/android/Omnivore/app/src/main/java/app/omnivore/omnivore/networking/ReadingProgressMutations.kt new file mode 100644 index 000000000..f6670efe3 --- /dev/null +++ b/android/Omnivore/app/src/main/java/app/omnivore/omnivore/networking/ReadingProgressMutations.kt @@ -0,0 +1,36 @@ +package app.omnivore.omnivore.networking + +import app.omnivore.omnivore.graphql.generated.SaveArticleReadingProgressMutation +import app.omnivore.omnivore.graphql.generated.type.SaveArticleReadingProgressInput + + +import android.util.Log +import com.google.gson.Gson + +data class ReadingProgressParams( + val id: String?, + val readingProgressPercent: Double?, + val readingProgressAnchorIndex: Int? +) { + fun asSaveReadingProgressInput() = SaveArticleReadingProgressInput( + id = id ?: "", + readingProgressPercent = readingProgressPercent ?: 0.0, + readingProgressAnchorIndex = readingProgressAnchorIndex ?: 0 + ) +} + +suspend fun Networker.updateReadingProgress(jsonString: String): Boolean { + val input = Gson().fromJson(jsonString, ReadingProgressParams::class.java).asSaveReadingProgressInput() + + Log.d("Loggo", "created reading progress input: $input") + + val result = authenticatedApolloClient() + .mutation(SaveArticleReadingProgressMutation(input)) + .execute() + + val articleID = result.data?.saveArticleReadingProgress?.onSaveArticleReadingProgressSuccess?.updatedArticle?.id + + Log.d("Loggo", "updated article with id: $articleID") + + return articleID != null +} diff --git a/android/Omnivore/app/src/main/java/app/omnivore/omnivore/networking/SearchQuery.kt b/android/Omnivore/app/src/main/java/app/omnivore/omnivore/networking/SearchQuery.kt index a658a3aea..7086c123e 100644 --- a/android/Omnivore/app/src/main/java/app/omnivore/omnivore/networking/SearchQuery.kt +++ b/android/Omnivore/app/src/main/java/app/omnivore/omnivore/networking/SearchQuery.kt @@ -22,7 +22,7 @@ suspend fun Networker.search( ) ).execute() - val cursor = result.data?.search?.onSearchSuccess?.pageInfo?.endCursor + val newCursor = result.data?.search?.onSearchSuccess?.pageInfo?.endCursor val itemList = result.data?.search?.onSearchSuccess?.edges ?: listOf() val items = itemList.map { @@ -49,5 +49,5 @@ suspend fun Networker.search( ) } - return SearchQueryResponse(cursor, items) + return SearchQueryResponse(newCursor, items) } diff --git a/android/Omnivore/app/src/main/java/app/omnivore/omnivore/ui/reader/AnnotationEditView.kt b/android/Omnivore/app/src/main/java/app/omnivore/omnivore/ui/reader/AnnotationEditView.kt new file mode 100644 index 000000000..e308250e2 --- /dev/null +++ b/android/Omnivore/app/src/main/java/app/omnivore/omnivore/ui/reader/AnnotationEditView.kt @@ -0,0 +1,68 @@ +package app.omnivore.omnivore.ui.reader + +import androidx.compose.foundation.background +import androidx.compose.foundation.layout.* +import androidx.compose.foundation.shape.RoundedCornerShape +import androidx.compose.material3.* +import androidx.compose.runtime.Composable +import androidx.compose.runtime.mutableStateOf +import androidx.compose.runtime.remember +import androidx.compose.ui.Alignment +import androidx.compose.ui.Modifier +import androidx.compose.ui.draw.clip +import androidx.compose.ui.unit.dp + +// TODO: better layout and styling for this view +@OptIn(ExperimentalMaterial3Api::class) +@Composable +fun AnnotationEditView( + initialAnnotation: String, + onSave: (String) -> Unit, + onCancel: () -> Unit, +) { + val annotation = remember { mutableStateOf(initialAnnotation) } + + Column( + modifier = Modifier + .clip(RoundedCornerShape(4.dp)) + .background(MaterialTheme.colorScheme.background) + .padding(8.dp), + ) { + Column( + modifier = Modifier.padding(16.dp), + ) { + Text(text = "Note") + + Spacer(modifier = Modifier.height(8.dp)) + + TextField( + value = annotation.value, + onValueChange = { annotation.value = it } + ) + } + + Spacer(modifier = Modifier.height(8.dp)) + + Row( + modifier = Modifier.align(Alignment.End) + ) { + Button( + onClick = { + onCancel() + } + ) { + Text("Cancel") + } + + Spacer(modifier = Modifier.width(8.dp)) + + Button( + onClick = { + onSave(annotation.value) + } + ) { + Text("Save") + } + } + } +} diff --git a/android/Omnivore/app/src/main/java/app/omnivore/omnivore/ui/reader/WebReader.kt b/android/Omnivore/app/src/main/java/app/omnivore/omnivore/ui/reader/WebReader.kt index 98401f013..c141a78ef 100644 --- a/android/Omnivore/app/src/main/java/app/omnivore/omnivore/ui/reader/WebReader.kt +++ b/android/Omnivore/app/src/main/java/app/omnivore/omnivore/ui/reader/WebReader.kt @@ -8,12 +8,17 @@ import android.view.* import android.webkit.JavascriptInterface import android.webkit.WebView import android.webkit.WebViewClient +import androidx.compose.foundation.layout.Box import androidx.compose.material3.Text import androidx.compose.runtime.Composable import androidx.compose.runtime.getValue import androidx.compose.runtime.livedata.observeAsState +import androidx.compose.runtime.mutableStateOf +import androidx.compose.runtime.remember import androidx.compose.ui.viewinterop.AndroidView import app.omnivore.omnivore.R +import app.omnivore.omnivore.networking.ReadingProgressParams +import com.google.gson.Gson import org.json.JSONObject @@ -36,6 +41,11 @@ fun WebReaderLoadingContainer(slug: String, webReaderViewModel: WebReaderViewMod @SuppressLint("SetJavaScriptEnabled") @Composable fun WebReader(params: WebReaderParams, webReaderViewModel: WebReaderViewModel) { + // TODO: maybe handle cases where js can be queued up? + val javascriptToExecute = remember { mutableStateOf(null) } + + val annotation: String? by webReaderViewModel.annotationLiveData.observeAsState(null) + WebView.setWebContentsDebuggingEnabled(true) val webReaderContent = WebReaderContent( @@ -51,39 +61,81 @@ fun WebReader(params: WebReaderParams, webReaderViewModel: WebReaderViewModel) { val styledContent = webReaderContent.styledContent() - AndroidView(factory = { - OmnivoreWebView(it).apply { - layoutParams = ViewGroup.LayoutParams( - ViewGroup.LayoutParams.MATCH_PARENT, - ViewGroup.LayoutParams.MATCH_PARENT + Box { + AndroidView(factory = { + OmnivoreWebView(it).apply { + layoutParams = ViewGroup.LayoutParams( + ViewGroup.LayoutParams.MATCH_PARENT, + ViewGroup.LayoutParams.MATCH_PARENT + ) + + settings.javaScriptEnabled = true + settings.allowContentAccess = true + settings.allowFileAccess = true + settings.domStorageEnabled = true + + webViewClient = object : WebViewClient() { + } + + val javascriptInterface = AndroidWebKitMessenger { actionID, json -> + when (actionID) { + "existingHighlightTap" -> { + isExistingHighlightSelected = true + actionTapCoordinates = Gson().fromJson(json, ActionTapCoordinates::class.java) + Log.d("Loggo", "receive existing highlight tap action: $actionTapCoordinates") + startActionMode(null, ActionMode.TYPE_PRIMARY) + } + else -> { + webReaderViewModel.handleIncomingWebMessage(actionID, json) + } + } + } + + addJavascriptInterface(javascriptInterface, "AndroidWebKitMessenger") + + loadDataWithBaseURL( + "file:///android_asset/", + styledContent, + "text/html; charset=utf-8", + "utf-8", + null + ) + } + }, update = { + if (javascriptToExecute.value != null) { + it.evaluateJavascript(javascriptToExecute.value!!, null) + } + }) + + if (annotation != null) { + AnnotationEditView( + initialAnnotation = annotation!!, + onSave = { + val script = "var event = new Event('saveAnnotation');event.annotation = '$it';document.dispatchEvent(event);" + javascriptToExecute.value = script + webReaderViewModel.cancelAnnotationEdit() + }, + onCancel = { + webReaderViewModel.cancelAnnotationEdit() + } ) - - settings.javaScriptEnabled = true - settings.allowContentAccess = true - settings.allowFileAccess = true - settings.domStorageEnabled = true - - webViewClient = object : WebViewClient() { - } - - val javascriptInterface = AndroidWebKitMessenger { actionID, json -> - webReaderViewModel.handleIncomingWebMessage(actionID, json) - } - - addJavascriptInterface(javascriptInterface, "AndroidWebKitMessenger") - loadDataWithBaseURL("file:///android_asset/", styledContent, "text/html; charset=utf-8", "utf-8", null); - } - }, update = { - it.loadDataWithBaseURL("file:///android_asset/", styledContent, "text/html; charset=utf-8", "utf-8", null); - }) + } } class OmnivoreWebView(context: Context) : WebView(context) { + var isExistingHighlightSelected = false + var actionTapCoordinates: ActionTapCoordinates? = null + private val actionModeCallback = object : ActionMode.Callback2() { // Called when the action mode is created; startActionMode() was called override fun onCreateActionMode(mode: ActionMode, menu: Menu): Boolean { - mode.menuInflater.inflate(R.menu.text_selection_menu, menu) + if (isExistingHighlightSelected) { + mode.menuInflater.inflate(R.menu.highlight_selection_menu, menu) + isExistingHighlightSelected = false + } else { + mode.menuInflater.inflate(R.menu.text_selection_menu, menu) + } return true } @@ -97,7 +149,8 @@ class OmnivoreWebView(context: Context) : WebView(context) { override fun onActionItemClicked(mode: ActionMode, item: MenuItem): Boolean { return when (item.itemId) { R.id.annotate -> { - Log.d("Loggo", "Annotate action selected") + val script = "var event = new Event('annotate');document.dispatchEvent(event);" + evaluateJavascript(script, null) mode.finish() true } @@ -108,6 +161,13 @@ class OmnivoreWebView(context: Context) : WebView(context) { mode.finish() true } + R.id.delete -> { + val script = "var event = new Event('remove');document.dispatchEvent(event);" + evaluateJavascript(script, null) + clearFocus() + mode.finish() + true + } else -> { Log.d("Loggo", "${item.itemId} selected") false @@ -117,35 +177,44 @@ class OmnivoreWebView(context: Context) : WebView(context) { // Called when the user exits the action mode override fun onDestroyActionMode(mode: ActionMode) { -// actionMode = null + Log.d("Loggo", "destroying menu: $mode") + isExistingHighlightSelected = false + actionTapCoordinates = null } override fun onGetContentRect(mode: ActionMode?, view: View?, outRect: Rect?) { + Log.d("Loggo", "outRect: $outRect, View: $view") outRect?.set(left, top, right, bottom) } } - private var currentActionModeCallback: ActionMode.Callback? = actionModeCallback - override fun startActionMode(callback: ActionMode.Callback?): ActionMode { - return super.startActionMode(currentActionModeCallback) + return super.startActionMode(actionModeCallback) } override fun startActionModeForChild( originalView: View?, callback: ActionMode.Callback? ): ActionMode { - return super.startActionModeForChild(originalView, currentActionModeCallback) + return super.startActionModeForChild(originalView, actionModeCallback) } override fun startActionMode(callback: ActionMode.Callback?, type: Int): ActionMode { - return super.startActionMode(currentActionModeCallback, type) + Log.d("Loggo", "startActionMode:type called") + return super.startActionMode(actionModeCallback, type) } } -class AndroidWebKitMessenger(val messageHandler: (String, JSONObject) -> Unit) { +class AndroidWebKitMessenger(val messageHandler: (String, String) -> Unit) { @JavascriptInterface fun handleIdentifiableMessage(actionID: String, jsonString: String) { - messageHandler(actionID, JSONObject(jsonString)) + messageHandler(actionID, jsonString) } } + +data class ActionTapCoordinates( + val rectX: Double, + val rectY: Double, + val rectWidth: Double, + val rectHeight: Double, +) diff --git a/android/Omnivore/app/src/main/java/app/omnivore/omnivore/ui/reader/WebReaderViewModel.kt b/android/Omnivore/app/src/main/java/app/omnivore/omnivore/ui/reader/WebReaderViewModel.kt index f05943dcb..cb0908ed5 100644 --- a/android/Omnivore/app/src/main/java/app/omnivore/omnivore/ui/reader/WebReaderViewModel.kt +++ b/android/Omnivore/app/src/main/java/app/omnivore/omnivore/ui/reader/WebReaderViewModel.kt @@ -6,8 +6,7 @@ import androidx.lifecycle.ViewModel import androidx.lifecycle.viewModelScope import app.omnivore.omnivore.DatastoreRepository import app.omnivore.omnivore.models.LinkedItem -import app.omnivore.omnivore.networking.Networker -import app.omnivore.omnivore.networking.linkedItem +import app.omnivore.omnivore.networking.* import com.google.gson.Gson import dagger.hilt.android.lifecycle.HiltViewModel import kotlinx.coroutines.launch @@ -19,12 +18,17 @@ data class WebReaderParams( val articleContent: ArticleContent ) +data class AnnotationWebViewMessage( + val annotation: String? +) + @HiltViewModel class WebReaderViewModel @Inject constructor( private val datastoreRepo: DatastoreRepository, private val networker: Networker ): ViewModel() { val webReaderParamsLiveData = MutableLiveData(null) + val annotationLiveData = MutableLiveData(null) fun loadItem(slug: String) { viewModelScope.launch { @@ -45,37 +49,50 @@ class WebReaderViewModel @Inject constructor( } } - fun handleIncomingWebMessage(actionID: String, json: JSONObject) { + fun handleIncomingWebMessage(actionID: String, jsonString: String) { when (actionID) { "createHighlight" -> { - Log.d("Loggo", "receive create highlight action: $json") + viewModelScope.launch { + val isHighlightSynced = networker.createHighlight(jsonString) + Log.d("Network", "isHighlightSynced = $isHighlightSynced") + } } "deleteHighlight" -> { // { highlightId } - Log.d("Loggo", "receive delete highlight action: $json") + Log.d("Loggo", "receive delete highlight action: $jsonString") } "updateHighlight" -> { - Log.d("Loggo", "receive update highlight action: $json") + Log.d("Loggo", "receive update highlight action: $jsonString") } "articleReadingProgress" -> { - Log.d("Loggo", "received article reading progress action: $json") + viewModelScope.launch { + val isReadingProgressSynced = networker.updateReadingProgress(jsonString) + Log.d("Network", "isReadingProgressSynced = $isReadingProgressSynced") + } } "annotate" -> { - Log.d("Loggo", "received annotate action: $json") - } - "existingHighlightTap" -> { - Log.d("Loggo", "receive existing highlight tap action: $json") + viewModelScope.launch { + val annotation = Gson() + .fromJson(jsonString, AnnotationWebViewMessage::class.java) + .annotation ?: "" + annotationLiveData.value = annotation + } } "shareHighlight" -> { // unimplemented } else -> { - Log.d("Loggo", "receive unrecognized action of $actionID with json: $json") + Log.d("Loggo", "receive unrecognized action of $actionID with json: $jsonString") } } } fun reset() { webReaderParamsLiveData.value = null + annotationLiveData.value = null + } + + fun cancelAnnotationEdit() { + annotationLiveData.value = null } } diff --git a/android/Omnivore/app/src/main/res/menu/highlight_selection_menu.xml b/android/Omnivore/app/src/main/res/menu/highlight_selection_menu.xml new file mode 100644 index 000000000..4cc6d8400 --- /dev/null +++ b/android/Omnivore/app/src/main/res/menu/highlight_selection_menu.xml @@ -0,0 +1,15 @@ + + + + + + + + diff --git a/android/Omnivore/app/src/main/res/values/strings.xml b/android/Omnivore/app/src/main/res/values/strings.xml index 7bb14ba15..b6ed5d009 100644 --- a/android/Omnivore/app/src/main/res/values/strings.xml +++ b/android/Omnivore/app/src/main/res/values/strings.xml @@ -5,4 +5,5 @@ Save articles and read them later in our distraction-free reader. Highlight Annotate + Delete diff --git a/apple/Omnivore.xcodeproj/project.pbxproj b/apple/Omnivore.xcodeproj/project.pbxproj index 7b20a9b8c..24594bc28 100644 --- a/apple/Omnivore.xcodeproj/project.pbxproj +++ b/apple/Omnivore.xcodeproj/project.pbxproj @@ -1233,7 +1233,7 @@ CODE_SIGN_IDENTITY = "Apple Development"; CODE_SIGN_STYLE = Automatic; COMBINE_HIDPI_IMAGES = YES; - CURRENT_PROJECT_VERSION = 70; + CURRENT_PROJECT_VERSION = 75; DEVELOPMENT_TEAM = QJF2XZ86HB; ENABLE_HARDENED_RUNTIME = YES; INFOPLIST_FILE = InfoPlists/ShareExtensionMac.plist; @@ -1243,7 +1243,7 @@ "@executable_path/../../../../Frameworks", ); MACOSX_DEPLOYMENT_TARGET = 12.0; - MARKETING_VERSION = 1.14.0; + MARKETING_VERSION = 1.15.0; MTL_ENABLE_DEBUG_INFO = INCLUDE_SOURCE; MTL_FAST_MATH = YES; PRODUCT_BUNDLE_IDENTIFIER = "app.omnivore.app.ShareExtension-Mac"; @@ -1265,7 +1265,7 @@ CODE_SIGN_IDENTITY = "Apple Development"; CODE_SIGN_STYLE = Automatic; COMBINE_HIDPI_IMAGES = YES; - CURRENT_PROJECT_VERSION = 70; + CURRENT_PROJECT_VERSION = 75; DEVELOPMENT_TEAM = QJF2XZ86HB; ENABLE_HARDENED_RUNTIME = YES; INFOPLIST_FILE = InfoPlists/ShareExtensionMac.plist; @@ -1275,7 +1275,7 @@ "@executable_path/../../../../Frameworks", ); MACOSX_DEPLOYMENT_TARGET = 12.0; - MARKETING_VERSION = 1.14.0; + MARKETING_VERSION = 1.15.0; MTL_FAST_MATH = YES; PRODUCT_BUNDLE_IDENTIFIER = "app.omnivore.app.ShareExtension-Mac"; PRODUCT_NAME = "$(TARGET_NAME)"; @@ -1347,7 +1347,7 @@ CODE_SIGN_IDENTITY = "Apple Development"; CODE_SIGN_STYLE = Automatic; COMBINE_HIDPI_IMAGES = YES; - CURRENT_PROJECT_VERSION = 70; + CURRENT_PROJECT_VERSION = 75; DEVELOPMENT_ASSET_PATHS = ""; DEVELOPMENT_TEAM = QJF2XZ86HB; ENABLE_HARDENED_RUNTIME = YES; @@ -1358,7 +1358,7 @@ "@executable_path/../Frameworks", ); MACOSX_DEPLOYMENT_TARGET = 12.0; - MARKETING_VERSION = 1.14.0; + MARKETING_VERSION = 1.15.0; MTL_ENABLE_DEBUG_INFO = INCLUDE_SOURCE; MTL_FAST_MATH = YES; PRODUCT_BUNDLE_IDENTIFIER = app.omnivore.app; @@ -1381,7 +1381,7 @@ CODE_SIGN_IDENTITY = "Apple Development"; CODE_SIGN_STYLE = Automatic; COMBINE_HIDPI_IMAGES = YES; - CURRENT_PROJECT_VERSION = 70; + CURRENT_PROJECT_VERSION = 75; DEVELOPMENT_ASSET_PATHS = ""; DEVELOPMENT_TEAM = QJF2XZ86HB; ENABLE_HARDENED_RUNTIME = YES; @@ -1392,7 +1392,7 @@ "@executable_path/../Frameworks", ); MACOSX_DEPLOYMENT_TARGET = 12.0; - MARKETING_VERSION = 1.14.0; + MARKETING_VERSION = 1.15.0; MTL_FAST_MATH = YES; PRODUCT_BUNDLE_IDENTIFIER = app.omnivore.app; PRODUCT_NAME = "$(TARGET_NAME)"; @@ -1447,7 +1447,7 @@ "$(inherited)", "@executable_path/Frameworks", ); - MARKETING_VERSION = 1.14.0; + MARKETING_VERSION = 1.15.0; PRODUCT_BUNDLE_IDENTIFIER = app.omnivore.app; PRODUCT_NAME = Omnivore; PROVISIONING_PROFILE_SPECIFIER = ""; @@ -1479,7 +1479,7 @@ "@executable_path/Frameworks", "@executable_path/../../Frameworks", ); - MARKETING_VERSION = 1.14.0; + MARKETING_VERSION = 1.15.0; MTL_ENABLE_DEBUG_INFO = INCLUDE_SOURCE; MTL_FAST_MATH = YES; OTHER_LDFLAGS = ( @@ -1518,7 +1518,7 @@ "@executable_path/Frameworks", "@executable_path/../../Frameworks", ); - MARKETING_VERSION = 1.14.0; + MARKETING_VERSION = 1.15.0; MTL_FAST_MATH = YES; OTHER_LDFLAGS = ( "-framework", @@ -1544,7 +1544,7 @@ CODE_SIGN_ENTITLEMENTS = "Entitlements/SafariExtension-Mac.entitlements"; CODE_SIGN_IDENTITY = "Apple Development"; CODE_SIGN_STYLE = Automatic; - CURRENT_PROJECT_VERSION = 70; + CURRENT_PROJECT_VERSION = 75; DEVELOPMENT_TEAM = QJF2XZ86HB; ENABLE_HARDENED_RUNTIME = YES; GENERATE_INFOPLIST_FILE = YES; @@ -1557,7 +1557,7 @@ "@executable_path/../../../../Frameworks", ); MACOSX_DEPLOYMENT_TARGET = 12.0; - MARKETING_VERSION = 1.14.0; + MARKETING_VERSION = 1.15.0; MTL_ENABLE_DEBUG_INFO = INCLUDE_SOURCE; MTL_FAST_MATH = YES; OTHER_LDFLAGS = ( @@ -1583,7 +1583,7 @@ CODE_SIGN_ENTITLEMENTS = "Entitlements/SafariExtension-Mac.entitlements"; CODE_SIGN_IDENTITY = "Apple Development"; CODE_SIGN_STYLE = Automatic; - CURRENT_PROJECT_VERSION = 70; + CURRENT_PROJECT_VERSION = 75; DEVELOPMENT_TEAM = QJF2XZ86HB; ENABLE_HARDENED_RUNTIME = YES; GENERATE_INFOPLIST_FILE = YES; @@ -1596,7 +1596,7 @@ "@executable_path/../../../../Frameworks", ); MACOSX_DEPLOYMENT_TARGET = 12.0; - MARKETING_VERSION = 1.14.0; + MARKETING_VERSION = 1.15.0; MTL_FAST_MATH = YES; OTHER_LDFLAGS = ( "-framework", @@ -1683,7 +1683,7 @@ "@executable_path/Frameworks", "@executable_path/../../Frameworks", ); - MARKETING_VERSION = 1.14.0; + MARKETING_VERSION = 1.15.0; PRODUCT_BUNDLE_IDENTIFIER = "app.omnivore.app.share-extension"; PRODUCT_NAME = ShareExtension; PROVISIONING_PROFILE_SPECIFIER = ""; @@ -1738,7 +1738,7 @@ "$(inherited)", "@executable_path/Frameworks", ); - MARKETING_VERSION = 1.14.0; + MARKETING_VERSION = 1.15.0; PRODUCT_BUNDLE_IDENTIFIER = app.omnivore.app; PRODUCT_NAME = Omnivore; PROVISIONING_PROFILE_SPECIFIER = ""; @@ -1767,7 +1767,7 @@ "@executable_path/Frameworks", "@executable_path/../../Frameworks", ); - MARKETING_VERSION = 1.14.0; + MARKETING_VERSION = 1.15.0; PRODUCT_BUNDLE_IDENTIFIER = "app.omnivore.app.share-extension"; PRODUCT_NAME = ShareExtension; PROVISIONING_PROFILE_SPECIFIER = ""; diff --git a/apple/OmnivoreKit/Sources/App/PDFSupport/PDFViewer.swift b/apple/OmnivoreKit/Sources/App/PDFSupport/PDFViewer.swift index 1316f4dce..923229a92 100644 --- a/apple/OmnivoreKit/Sources/App/PDFSupport/PDFViewer.swift +++ b/apple/OmnivoreKit/Sources/App/PDFSupport/PDFViewer.swift @@ -39,6 +39,7 @@ import Utils .useParentNavigationBar(true) .updateConfiguration { builder in builder.textSelectionShouldSnapToWord = true + builder.shouldAskForAnnotationUsername = false } .updateControllerConfiguration { controller in // Store config state so we only run this update closure once diff --git a/apple/OmnivoreKit/Sources/App/Views/AudioPlayer/MiniPlayer.swift b/apple/OmnivoreKit/Sources/App/Views/AudioPlayer/MiniPlayer.swift index d17521415..2879b5366 100644 --- a/apple/OmnivoreKit/Sources/App/Views/AudioPlayer/MiniPlayer.swift +++ b/apple/OmnivoreKit/Sources/App/Views/AudioPlayer/MiniPlayer.swift @@ -192,11 +192,11 @@ public struct MiniPlayer: View { if !expanded { Text(itemAudioProperties.title) - .font(expanded ? .appTitle : .appCallout) + .font(.appCallout) .lineSpacing(1.25) .foregroundColor(.appGrayTextContrast) .fixedSize(horizontal: false, vertical: false) - .frame(maxWidth: .infinity, alignment: expanded ? .center : .leading) + .frame(maxWidth: .infinity, alignment: .leading) .matchedGeometryEffect(id: "ArticleTitle", in: animation) playPauseButtonItem @@ -210,55 +210,26 @@ public struct MiniPlayer: View { Spacer() if expanded { - Text(itemAudioProperties.title) - .lineLimit(1) - .font(expanded ? .appTitle : .appCallout) - .lineSpacing(1.25) + Marquee(text: itemAudioProperties.title, font: UIFont(name: "Inter-Regular", size: 22)!) .foregroundColor(.appGrayTextContrast) - .frame(maxWidth: .infinity, alignment: expanded ? .center : .leading) - .matchedGeometryEffect(id: "ArticleTitle", in: animation) .onTapGesture { viewArticle() } - HStack { - Spacer() - if let byline = itemAudioProperties.byline { - Text(byline) - .lineLimit(1) - .font(.appCallout) - .lineSpacing(1.25) - .foregroundColor(.appGrayText) - .frame(alignment: .trailing) - } - Spacer() + if let byline = itemAudioProperties.byline { + Marquee(text: byline, font: UIFont(name: "Inter-Regular", size: 16)!) + .foregroundColor(.appGrayText) } - Slider(value: $audioController.timeElapsed, - in: 0 ... self.audioController.duration, - onEditingChanged: { scrubStarted in - if scrubStarted { - self.audioController.scrubState = .scrubStarted - } else { - self.audioController.scrubState = .scrubEnded(self.audioController.timeElapsed) - } - }) - .accentColor(.appCtaYellow) - .introspectSlider { slider in - // Make the thumb a little smaller than the default and give it the CTA color - // for some reason this doesn't work on my iPad though. - let tintColor = UIColor(Color.appCtaYellow) - - let image = UIImage(systemName: "circle.fill", - withConfiguration: UIImage.SymbolConfiguration(scale: .small))? - .withTintColor(tintColor) - .withRenderingMode(.alwaysOriginal) - - slider.setThumbImage(image, for: .selected) - slider.setThumbImage(image, for: .normal) - - slider.minimumTrackTintColor = tintColor - } + ScrubberView(value: $audioController.timeElapsed, + minValue: 0, maxValue: self.audioController.duration, + onEditingChanged: { scrubStarted in + if scrubStarted { + self.audioController.scrubState = .scrubStarted + } else { + self.audioController.scrubState = .scrubEnded(self.audioController.timeElapsed) + } + }) HStack { Text(audioController.timeElapsedString ?? "0:00") @@ -333,7 +304,7 @@ public struct MiniPlayer: View { withAnimation(.easeIn(duration: 0.08)) { expanded = true } }.sheet(isPresented: $showVoiceSheet) { NavigationView { - TextToSpeechVoiceSelectionView(forLanguage: audioController.currentVoiceLanguage) + TextToSpeechVoiceSelectionView(forLanguage: audioController.currentVoiceLanguage, showLanguageChanger: true) .navigationBarTitle("Voice") .navigationBarTitleDisplayMode(.inline) .navigationBarItems(leading: Button(action: { self.showVoiceSheet = false }) { diff --git a/apple/OmnivoreKit/Sources/App/Views/AudioPlayer/ScrubberView.swift b/apple/OmnivoreKit/Sources/App/Views/AudioPlayer/ScrubberView.swift new file mode 100644 index 000000000..edff16b09 --- /dev/null +++ b/apple/OmnivoreKit/Sources/App/Views/AudioPlayer/ScrubberView.swift @@ -0,0 +1,73 @@ +// +// ScrubberView.swift +// +// +// Created by Jackson Harper on 9/27/22. +// + +import Foundation +import SwiftUI + +struct ScrubberView: UIViewRepresentable { + typealias UIViewType = UISlider + + @Binding var value: Double + var minValue: Double + var maxValue: Double + var onEditingChanged: (Bool) -> Void + + init(value: Binding, minValue: Double, maxValue: Double, onEditingChanged: @escaping (Bool) -> Void) { + self._value = value + self.minValue = minValue + self.maxValue = maxValue + self.onEditingChanged = onEditingChanged + } + + func makeUIView(context: Context) -> UISlider { + let slider = UISlider(frame: .zero) + slider.maximumValue = Float(minValue) + slider.maximumValue = Float(maxValue) + + let tintColor = UIColor(Color.appCtaYellow) + + let image = UIImage(systemName: "circle.fill", + withConfiguration: UIImage.SymbolConfiguration(scale: .small))? + .withTintColor(tintColor) + .withRenderingMode(.alwaysOriginal) + + slider.setThumbImage(image, for: .selected) + slider.setThumbImage(image, for: .normal) + + slider.minimumTrackTintColor = tintColor + slider.addTarget(context.coordinator, + action: #selector(Coordinator.valueChanged(_:)), + for: .valueChanged) + + return slider + } + + func updateUIView(_ uiView: UISlider, context _: Context) { + uiView.value = Float(value) + } + + func makeCoordinator() -> Coordinator { + let coordinator = Coordinator(value: $value, onEditingChanged: onEditingChanged) + return coordinator + } + + class Coordinator: NSObject { + var value: Binding + var onEditingChanged: (Bool) -> Void + + init(value: Binding, onEditingChanged: @escaping (Bool) -> Void) { + self.value = value + self.onEditingChanged = onEditingChanged + super.init() + } + + @objc func valueChanged(_ sender: UISlider) { + value.wrappedValue = Double(sender.value) + onEditingChanged(sender.isTracking) + } + } +} diff --git a/apple/OmnivoreKit/Sources/App/Views/Labels/MarqueTextView.swift b/apple/OmnivoreKit/Sources/App/Views/Labels/MarqueTextView.swift new file mode 100644 index 000000000..333f79e60 --- /dev/null +++ b/apple/OmnivoreKit/Sources/App/Views/Labels/MarqueTextView.swift @@ -0,0 +1,204 @@ +import SwiftUI + +// Mostly from: https://kavsoft.dev/swiftui_3.0_marquee_text_animation with some customizations + +struct Marquee: View { + var text: String + var font: UIFont + + // Storing Text Size + @State var storedSize: CGSize = .zero + @State var offset: CGFloat = 0 + @State var animatedText: String = "" + + var animationSpeed: Double = 0.03 + var delayTime: Double = 3.0 + + var body: some View { + // Since it scrolls horizontal using ScrollView + GeometryReader { proxy in + + let size = proxy.size + + let condition = textSize(text: text).width < (size.width - 50) + + ScrollView(condition ? .init() : .horizontal, showsIndicators: false) { + HStack(alignment: .center) { + Spacer(minLength: 0) + Text(condition ? text : animatedText) + .font(Font(font)) + .offset(x: condition ? 0 : offset) + .padding(.horizontal, 15) + Spacer(minLength: 0) + } + } + .frame(maxWidth: .infinity, maxHeight: .infinity, alignment: .center) + } + .frame(height: storedSize.height) + .overlay(content: { + HStack { + let color: Color = .systemBackground + + LinearGradient(colors: [color, color.opacity(0.7), color.opacity(0.5), color.opacity(0.3)], startPoint: .leading, endPoint: .trailing) + .frame(width: 8) + + Spacer() + + LinearGradient(colors: [color, color.opacity(0.7), color.opacity(0.5), color.opacity(0.3)].reversed(), startPoint: .leading, endPoint: .trailing) + .frame(width: 8) + } + }) + .disabled(true) + .onAppear { + startAnimation(text: text) + } + .onReceive(Timer.publish(every: (animationSpeed * storedSize.width) + delayTime, + on: .main, + in: .default).autoconnect() + ) { _ in + offset = 0 + withAnimation(.linear(duration: animationSpeed * storedSize.width).delay(delayTime)) { + offset = -storedSize.width + } + } + .onChange(of: text) { newValue in + animatedText = "" + offset = 0 + startAnimation(text: newValue) + } + } + + func startAnimation(text: String) { + // Double the text with some spacing so that we can create a continuous loop + animatedText.append(text) + (1 ... 15).forEach { _ in + animatedText.append(" ") + } + storedSize = textSize(text: animatedText) + animatedText.append(text) + + let timing: Double = (animationSpeed * storedSize.width) + withAnimation(.linear(duration: timing).delay(delayTime)) { + offset = -storedSize.width + } + } + + func textSize(text: String) -> CGSize { + let attributes = [NSAttributedString.Key.font: font] + + let size = (text as NSString).size(withAttributes: attributes) + + return size + } +} + +// Old version: +// +// struct MarqueTextView: View { +// let font: Font +// +// @State var text: String +// @State private var intrinsicSize: CGSize = .zero +// @State private var truncatedSize: CGSize = .zero +// +// @State private var shouldAnimate: Bool = false +// @State private var animationOffset: Double = 0.0 +// +// var body: some View { +// GeometryReader { geo in +// ScrollView(.horizontal, showsIndicators: false) { +// HStack(alignment: .center) { +// Spacer(minLength: 0) +// Text(text) +// .font(font) +// .lineLimit(1) +// .lineSpacing(1.25) +// .offset(x: animationOffset) +// .readSize { size in +// truncatedSize = size +// intrinsicSize = geo.size +// +// shouldAnimate = textSize().width > intrinsicSize.width +// } +// Spacer(minLength: 0) +// } +// .frame(width: max(geo.size.width, textSize().width + 10)) +// } +// .frame(maxWidth: .infinity, alignment: .center) +// .disabled(true) +// .onChange(of: shouldAnimate) { _ in +// +// let baseText = text +// text.append(" ") +// let initialSize = textSize() +// +// print("starting animation, truncatedSize: ", truncatedSize, "geo width: ", geo.size) +// if shouldAnimate { +// DispatchQueue.main.asyncAfter(deadline: .now() + .milliseconds(500)) { +// withAnimation(.linear(duration: 0.05 * truncatedSize.width)) { +// animationOffset = -truncatedSize.width +// } +// } +// } +// } +// .onReceive(Timer.publish(every: 0.05 * truncatedSize.width + 0.5, on: .main, in: .default).autoconnect()) { _ in +// if shouldAnimate { +// animationOffset = 0 +// withAnimation(.linear(duration: 0.05 * truncatedSize.width)) { +// animationOffset = -truncatedSize.width +// } +// } +// } +// } +// } +// +// func textSize() -> CGSize { +// let attributes = [NSAttributedString.Key.font: UIFont(name: "Inter-Regular", size: 16)!] +// return (text as NSString).size(withAttributes: attributes) +// } +// } +// +//// text() +//// .lineLimit(lineLimit) +//// .offset(x: animationOffset) +//// .readSize { size in +//// truncatedSize = size +//// shouldAnimate = truncatedSize != intrinsicSize +//// print("trunvatedSize: ", truncatedSize, "intrinsicSize: ", intrinsicSize) +//// } +//// .background( +//// text() +//// .fixedSize(horizontal: false, vertical: true) +//// .hidden() +//// .readSize { size in +//// intrinsicSize = size +//// shouldAnimate = truncatedSize != intrinsicSize +//// } +//// ) +//// .onChange(of: shouldAnimate, perform: { _ in +//// print("starting animation") +//// DispatchQueue.main.asyncAfter(deadline: .now() + .milliseconds(500)) { +//// withAnimation(.linear(duration: 0.2 * intrinsicSize.width)) { +//// animationOffset = intrinsicSize.width +//// } +//// } +//// }) +//// } +//// } +// +// extension View { +// func readSize(onChange: @escaping (CGSize) -> Void) -> some View { +// background( +// GeometryReader { geometryProxy in +// Color.clear +// .preference(key: SizePreferenceKey.self, value: geometryProxy.size) +// } +// ) +// .onPreferenceChange(SizePreferenceKey.self, perform: onChange) +// } +// } +// +// struct SizePreferenceKey: PreferenceKey { +// static var defaultValue: CGSize = .zero +// static func reduce(value _: inout CGSize, nextValue _: () -> CGSize) {} +// } diff --git a/apple/OmnivoreKit/Sources/App/Views/Profile/TextToSpeechView.swift b/apple/OmnivoreKit/Sources/App/Views/Profile/TextToSpeechView.swift index 82dbc0869..ea5fadfc0 100644 --- a/apple/OmnivoreKit/Sources/App/Views/Profile/TextToSpeechView.swift +++ b/apple/OmnivoreKit/Sources/App/Views/Profile/TextToSpeechView.swift @@ -30,7 +30,7 @@ struct TextToSpeechView: View { private var innerBody: some View { Section("Voices") { ForEach(Voices.Languages, id: \.key) { language in - NavigationLink(destination: TextToSpeechVoiceSelectionView(forLanguage: language)) { + NavigationLink(destination: TextToSpeechVoiceSelectionView(forLanguage: language, showLanguageChanger: false)) { Text(language.name) } } diff --git a/apple/OmnivoreKit/Sources/App/Views/Profile/TextToSpeechVoiceSelectionView.swift b/apple/OmnivoreKit/Sources/App/Views/Profile/TextToSpeechVoiceSelectionView.swift index 1d6e3e11c..ad6ce2ba3 100644 --- a/apple/OmnivoreKit/Sources/App/Views/Profile/TextToSpeechVoiceSelectionView.swift +++ b/apple/OmnivoreKit/Sources/App/Views/Profile/TextToSpeechVoiceSelectionView.swift @@ -6,18 +6,22 @@ import Views struct TextToSpeechVoiceSelectionView: View { @EnvironmentObject var audioController: AudioController let language: VoiceLanguage + let showLanguageChanger: Bool - init(forLanguage: VoiceLanguage) { + init(forLanguage: VoiceLanguage, showLanguageChanger: Bool) { self.language = forLanguage + self.showLanguageChanger = showLanguageChanger } var body: some View { Group { #if os(iOS) Form { - Section("Language") { - NavigationLink(destination: TextToSpeechLanguageView().navigationTitle("Language")) { - Text(audioController.currentVoiceLanguage.name) + if showLanguageChanger { + Section("Language") { + NavigationLink(destination: TextToSpeechLanguageView().navigationTitle("Language")) { + Text(audioController.currentVoiceLanguage.name) + } } } innerBody diff --git a/apple/OmnivoreKit/Sources/App/Views/WelcomeView.swift b/apple/OmnivoreKit/Sources/App/Views/WelcomeView.swift index 89b7aee19..6e3baa145 100644 --- a/apple/OmnivoreKit/Sources/App/Views/WelcomeView.swift +++ b/apple/OmnivoreKit/Sources/App/Views/WelcomeView.swift @@ -33,7 +33,7 @@ struct WelcomeView: View { var headlineText: some View { Group { - Text("Never miss a great read.") + Text("Read-it-later for serious readers.") } .font(.appLargeTitle) } @@ -153,6 +153,7 @@ struct WelcomeView: View { action: { showEmailLoginModal = true }, label: { Text("Continue with Email") + .font(.appHeadline) .foregroundColor(.appGrayTextContrast) .underline() } diff --git a/apple/OmnivoreKit/Sources/Services/AudioSession/AudioController.swift b/apple/OmnivoreKit/Sources/Services/AudioSession/AudioController.swift index 61cd7133a..f34b5e935 100644 --- a/apple/OmnivoreKit/Sources/Services/AudioSession/AudioController.swift +++ b/apple/OmnivoreKit/Sources/Services/AudioSession/AudioController.swift @@ -390,7 +390,7 @@ public class AudioController: NSObject, ObservableObject, AVAudioPlayerDelegate } } - @AppStorage(UserDefaultKey.textToSpeechPreloadEnabled.rawValue) public var preloadEnabled = true + @AppStorage(UserDefaultKey.textToSpeechPreloadEnabled.rawValue) public var preloadEnabled = false public var currentVoiceLanguage: VoiceLanguage { Voices.Languages.first(where: { $0.key == currentLanguage }) ?? Voices.English diff --git a/apple/OmnivoreKit/Sources/Services/DataService/Mutations/DeleteHighlight.swift b/apple/OmnivoreKit/Sources/Services/DataService/Mutations/DeleteHighlight.swift index ff9a95b87..286e0524b 100644 --- a/apple/OmnivoreKit/Sources/Services/DataService/Mutations/DeleteHighlight.swift +++ b/apple/OmnivoreKit/Sources/Services/DataService/Mutations/DeleteHighlight.swift @@ -5,20 +5,30 @@ import SwiftGraphQL public extension DataService { func deleteHighlight(highlightID: String) { - if let highlight = Highlight.lookup(byID: highlightID, inContext: backgroundContext) { + if let highlight = Highlight.lookup(byID: highlightID, inContext: viewContext) { deleteHighlight(objectID: highlight.objectID) } } private func deleteHighlight(objectID: NSManagedObjectID) { // Update CoreData - backgroundContext.perform { [weak self] in - guard let self = self else { return } - guard let highlight = self.backgroundContext.object(with: objectID) as? Highlight else { return } - highlight.remove(inContext: self.backgroundContext) + viewContext.performAndWait { + guard let highlight = viewContext.object(with: objectID) as? Highlight else { return } + highlight.serverSyncStatus = Int64(ServerSyncStatus.needsDeletion.rawValue) - // Send update to server - self.syncHighlightDeletion(highlightID: highlight.unwrappedID, objectID: objectID) + do { + try viewContext.save() + logger.debug("Highlight succesfully marked for deletion") + } catch { + viewContext.rollback() + logger.debug("Failed to mark Highlight for deletion: \(error.localizedDescription)") + } + } + + // Send update to server + backgroundContext.perform { [weak self] in + guard let highlight = self?.backgroundContext.object(with: objectID) as? Highlight else { return } + self?.syncHighlightDeletion(highlightID: highlight.unwrappedID, objectID: objectID) } } diff --git a/apple/OmnivoreKit/Sources/Services/DataService/Mutations/RemoveLabelPublisher.swift b/apple/OmnivoreKit/Sources/Services/DataService/Mutations/RemoveLabelPublisher.swift index fc7c02c2b..9df043f40 100644 --- a/apple/OmnivoreKit/Sources/Services/DataService/Mutations/RemoveLabelPublisher.swift +++ b/apple/OmnivoreKit/Sources/Services/DataService/Mutations/RemoveLabelPublisher.swift @@ -5,14 +5,21 @@ import SwiftGraphQL extension DataService { public func removeLabel(labelID: String, name: String) { // Update CoreData - backgroundContext.perform { [weak self] in - guard let self = self else { return } - guard let label = LinkedItemLabel.lookup(byID: labelID, inContext: self.backgroundContext) else { return } - label.remove(inContext: self.backgroundContext) + viewContext.performAndWait { + guard let label = LinkedItemLabel.lookup(byID: labelID, inContext: self.viewContext) else { return } + label.serverSyncStatus = Int64(ServerSyncStatus.needsDeletion.rawValue) - // Send update to server - self.syncLabelDeletion(labelID: labelID, labelName: name) + do { + try viewContext.save() + logger.debug("Label succesfully marked for deletion") + } catch { + viewContext.rollback() + logger.debug("Failed to mark Label for deletion: \(error.localizedDescription)") + } } + + // Send update to server + syncLabelDeletion(labelID: labelID, labelName: name) } func syncLabelDeletion(labelID: String, labelName _: String) { diff --git a/apple/OmnivoreKit/Sources/Services/DataService/Mutations/RemoveLink.swift b/apple/OmnivoreKit/Sources/Services/DataService/Mutations/RemoveLink.swift index d6a3e3894..5fd1bbfca 100644 --- a/apple/OmnivoreKit/Sources/Services/DataService/Mutations/RemoveLink.swift +++ b/apple/OmnivoreKit/Sources/Services/DataService/Mutations/RemoveLink.swift @@ -3,16 +3,26 @@ import Foundation import Models import SwiftGraphQL -extension DataService { - public func removeLink(objectID: NSManagedObjectID) { +public extension DataService { + func removeLink(objectID: NSManagedObjectID) { // Update CoreData - backgroundContext.perform { [weak self] in - guard let self = self else { return } - guard let linkedItem = self.backgroundContext.object(with: objectID) as? LinkedItem else { return } - linkedItem.remove(inContext: self.backgroundContext) + viewContext.performAndWait { + guard let linkedItem = viewContext.object(with: objectID) as? LinkedItem else { return } + linkedItem.serverSyncStatus = Int64(ServerSyncStatus.needsDeletion.rawValue) - // Send update to server - self.syncLinkDeletion(itemID: linkedItem.unwrappedID, objectID: objectID) + do { + try viewContext.save() + logger.debug("LinkedItem succesfully marked for deletion") + } catch { + viewContext.rollback() + logger.debug("Failed to mark LinkedItem for deletion: \(error.localizedDescription)") + } + } + + // Send update to server + backgroundContext.perform { [weak self] in + guard let linkedItem = self?.backgroundContext.object(with: objectID) as? LinkedItem else { return } + self?.syncLinkDeletion(itemID: linkedItem.unwrappedID, objectID: objectID) } } diff --git a/apple/OmnivoreKit/Sources/Views/Fonts.swift b/apple/OmnivoreKit/Sources/Views/Fonts.swift index bf32124e6..40385127c 100644 --- a/apple/OmnivoreKit/Sources/Views/Fonts.swift +++ b/apple/OmnivoreKit/Sources/Views/Fonts.swift @@ -6,7 +6,7 @@ import SwiftUI public extension Font { /// 34pt, Inter-Regular static var appLargeTitle: Font { - .customFont(InterFont.regular.rawValue, size: 34, relativeTo: .largeTitle) + .customFont(InterFont.bold.rawValue, size: 34, relativeTo: .largeTitle) } /// 28pt, Inter-Regular diff --git a/packages/api/src/elastic/types.ts b/packages/api/src/elastic/types.ts index 33d00b038..065770aa6 100644 --- a/packages/api/src/elastic/types.ts +++ b/packages/api/src/elastic/types.ts @@ -247,6 +247,8 @@ export interface SearchItem { labels?: Label[] highlights?: Highlight[] wordsCount?: number + siteName?: string + siteIcon?: string } const keys = ['_id', 'url', 'slug', 'userId', 'uploadFileId', 'state'] as const diff --git a/packages/api/src/entity/subscription.ts b/packages/api/src/entity/subscription.ts index d6bbf9cb5..d0f89b4db 100644 --- a/packages/api/src/entity/subscription.ts +++ b/packages/api/src/entity/subscription.ts @@ -43,6 +43,9 @@ export class Subscription { @Column('text', { nullable: true }) unsubscribeHttpUrl?: string + @Column('text', { nullable: true }) + icon?: string + @CreateDateColumn() createdAt!: Date diff --git a/packages/api/src/generated/graphql.ts b/packages/api/src/generated/graphql.ts index 600243e98..24f683035 100644 --- a/packages/api/src/generated/graphql.ts +++ b/packages/api/src/generated/graphql.ts @@ -1644,6 +1644,7 @@ export type SearchItem = { readingProgressPercent: Scalars['Float']; savedAt: Scalars['Date']; shortId?: Maybe; + siteIcon?: Maybe; siteName?: Maybe; slug: Scalars['String']; state?: Maybe; @@ -1987,6 +1988,7 @@ export type Subscription = { __typename?: 'Subscription'; createdAt: Scalars['Date']; description?: Maybe; + icon?: Maybe; id: Scalars['ID']; name: Scalars['String']; newsletterEmail: Scalars['String']; @@ -4159,6 +4161,7 @@ export type SearchItemResolvers; savedAt?: Resolver; shortId?: Resolver, ParentType, ContextType>; + siteIcon?: Resolver, ParentType, ContextType>; siteName?: Resolver, ParentType, ContextType>; slug?: Resolver; state?: Resolver, ParentType, ContextType>; @@ -4368,6 +4371,7 @@ export type SubscribeSuccessResolvers = { createdAt?: SubscriptionResolver; description?: SubscriptionResolver, "description", ParentType, ContextType>; + icon?: SubscriptionResolver, "icon", ParentType, ContextType>; id?: SubscriptionResolver; name?: SubscriptionResolver; newsletterEmail?: SubscriptionResolver; diff --git a/packages/api/src/generated/schema.graphql b/packages/api/src/generated/schema.graphql index a0d0e1161..a15f5c497 100644 --- a/packages/api/src/generated/schema.graphql +++ b/packages/api/src/generated/schema.graphql @@ -1170,6 +1170,7 @@ type SearchItem { readingProgressPercent: Float! savedAt: Date! shortId: String + siteIcon: String siteName: String slug: String! state: ArticleSavingRequestStatus @@ -1485,6 +1486,7 @@ type SubscribeSuccess { type Subscription { createdAt: Date! description: String + icon: String id: ID! name: String! newsletterEmail: String! diff --git a/packages/api/src/resolvers/article/index.ts b/packages/api/src/resolvers/article/index.ts index e9b214102..40b77af12 100644 --- a/packages/api/src/resolvers/article/index.ts +++ b/packages/api/src/resolvers/article/index.ts @@ -899,6 +899,7 @@ export const searchResolver = authorized< publishedAt: validatedDate(r.publishedAt), ownedByViewer: r.userId === claims.uid, pageType: r.pageType || PageType.Highlights, + siteIcon: r.siteIcon && createImageProxyUrl(r.siteIcon, 32, 32), } as SearchItem, cursor: endCursor, } diff --git a/packages/api/src/resolvers/subscriptions/index.ts b/packages/api/src/resolvers/subscriptions/index.ts index 6ee982f6d..eb416b868 100644 --- a/packages/api/src/resolvers/subscriptions/index.ts +++ b/packages/api/src/resolvers/subscriptions/index.ts @@ -23,6 +23,7 @@ import { User } from '../../entity/user' import { Subscription } from '../../entity/subscription' import { getSubscribeHandler, unsubscribe } from '../../services/subscriptions' import { ILike } from 'typeorm' +import { createImageProxyUrl } from '../../utils/imageproxy' export const subscriptionsResolver = authorized< SubscriptionsSuccess, @@ -57,7 +58,10 @@ export const subscriptionsResolver = authorized< }) return { - subscriptions, + subscriptions: subscriptions.map((s) => ({ + ...s, + icon: s.icon && createImageProxyUrl(s.icon, 32, 32), + })), } } catch (error) { log.error(error) diff --git a/packages/api/src/routers/svc/emails.ts b/packages/api/src/routers/svc/emails.ts index e8ccc8c05..6aae07d74 100644 --- a/packages/api/src/routers/svc/emails.ts +++ b/packages/api/src/routers/svc/emails.ts @@ -8,14 +8,11 @@ import { analytics } from '../../utils/analytics' import { getNewsletterEmail } from '../../services/newsletters' import { env } from '../../env' import { - findNewsletterUrl, generateUniqueUrl, getTitleFromEmailSubject, isProbablyArticle, - isProbablyNewsletter, parseEmailAddress, } from '../../utils/parser' -import { saveNewsletterEmail } from '../../services/save_newsletter_email' import { saveEmail } from '../../services/save_email' import { buildLogger } from '../../utils/logger' @@ -80,25 +77,6 @@ export function emailsServiceRouter() { const ctx = { pubsub: createPubSubClient(), uid: user.id } const parsedFrom = parseEmailAddress(data.from) - if (await isProbablyNewsletter(data.html)) { - logger.info('handling as newsletter', data) - await saveNewsletterEmail( - { - email: data.to, - title: data.subject, - content: data.html, - author: parsedFrom.name, - url: (await findNewsletterUrl(data.html)) || generateUniqueUrl(), - unsubMailTo: data.unsubMailTo, - unsubHttpUrl: data.unsubHttpUrl, - newsletterEmail, - }, - ctx - ) - res.status(200).send('Newsletter') - return - } - if ( await isProbablyArticle( data.forwardedFrom || parsedFrom.address, diff --git a/packages/api/src/schema.ts b/packages/api/src/schema.ts index dc0936431..c2e6f3805 100755 --- a/packages/api/src/schema.ts +++ b/packages/api/src/schema.ts @@ -1495,6 +1495,7 @@ const schema = gql` readAt: Date savedAt: Date! highlights: [Highlight!] + siteIcon: String } type SearchItemEdge { @@ -1530,6 +1531,7 @@ const schema = gql` status: SubscriptionStatus! unsubscribeMailTo: String unsubscribeHttpUrl: String + icon: String createdAt: Date! updatedAt: Date! } diff --git a/packages/api/src/services/save_email.ts b/packages/api/src/services/save_email.ts index 100858745..da623b7a0 100644 --- a/packages/api/src/services/save_email.ts +++ b/packages/api/src/services/save_email.ts @@ -70,6 +70,7 @@ export const saveEmail = async ( readingProgressPercent: 0, subscription: input.author, state: ArticleSavingRequestStatus.Succeeded, + siteIcon: parseResult.parsedContent?.siteIcon, } const page = await getPageByParam({ diff --git a/packages/api/src/services/save_newsletter_email.ts b/packages/api/src/services/save_newsletter_email.ts index ecf5177dd..84daf9a40 100644 --- a/packages/api/src/services/save_newsletter_email.ts +++ b/packages/api/src/services/save_newsletter_email.ts @@ -12,6 +12,8 @@ import { Page } from '../elastic/types' import { addLabelToPage } from './labels' import { saveSubscription } from './subscriptions' import { NewsletterEmail } from '../entity/newsletter_email' +import { fetchFavicon } from '../utils/parser' +import { updatePage } from '../elastic/pages' interface NewsletterMessage { email: string @@ -33,7 +35,6 @@ export const saveNewsletterEmail = async ( // get user from newsletter email const newsletterEmail = data.newsletterEmail || (await getNewsletterEmail(data.email)) - if (!newsletterEmail) { console.log('newsletter email not found', data.email) return false @@ -54,7 +55,6 @@ export const saveNewsletterEmail = async ( pubsub: createPubSubClient(), uid: newsletterEmail.user.id, } - const input: SaveEmailInput = { url: data.url, originalContent: data.content, @@ -63,22 +63,31 @@ export const saveNewsletterEmail = async ( unsubMailTo: data.unsubMailTo, unsubHttpUrl: data.unsubHttpUrl, } - const page = await saveEmail(saveCtx, input) if (!page) { console.log('newsletter not created:', input) return false } + if (!page.siteIcon) { + // fetch favicon if not already set + const favicon = await fetchFavicon(page.url) + if (favicon) { + page.siteIcon = favicon + await updatePage(page.id, { siteIcon: favicon }, saveCtx) + } + } + // creates or updates subscription - const subscription = await saveSubscription( - newsletterEmail.user.id, - data.author, - newsletterEmail.address, - data.unsubMailTo, - data.unsubHttpUrl - ) - console.log('subscription', subscription) + const subscription = await saveSubscription({ + userId: newsletterEmail.user.id, + name: data.author, + newsletterEmail: newsletterEmail.address, + unsubscribeMailTo: data.unsubMailTo, + unsubscribeHttpUrl: data.unsubHttpUrl, + icon: page.siteIcon, + }) + console.log('subscription saved', subscription) // adds newsletters label to page const result = await addLabelToPage(saveCtx, page.id, { diff --git a/packages/api/src/services/subscriptions.ts b/packages/api/src/services/subscriptions.ts index f0d7997ac..7ac233e83 100644 --- a/packages/api/src/services/subscriptions.ts +++ b/packages/api/src/services/subscriptions.ts @@ -6,6 +6,15 @@ import axios from 'axios' import { NewsletterEmail } from '../entity/newsletter_email' import { createNewsletterEmail } from './newsletters' +interface SaveSubscriptionInput { + userId: string + name: string + newsletterEmail: string + unsubscribeMailTo?: string + unsubscribeHttpUrl?: string + icon?: string +} + const sendUnsubscribeEmail = async ( unsubscribeMailTo: string, newsletterEmail: string @@ -30,13 +39,14 @@ const sendUnsubscribeHttpRequest = async (url: string): Promise => { } } -export const saveSubscription = async ( - userId: string, - name: string, - newsletterEmail: string, - unsubscribeMailTo?: string, - unsubscribeHttpUrl?: string -): Promise => { +export const saveSubscription = async ({ + userId, + name, + newsletterEmail, + unsubscribeMailTo, + unsubscribeHttpUrl, + icon, +}: SaveSubscriptionInput): Promise => { const subscription = await getRepository(Subscription).findOneBy({ name, user: { id: userId }, @@ -46,6 +56,7 @@ export const saveSubscription = async ( // if subscription already exists, updates updatedAt subscription.status = SubscriptionStatus.Active subscription.newsletterEmail = newsletterEmail + icon && (subscription.icon = icon) unsubscribeMailTo && (subscription.unsubscribeMailTo = unsubscribeMailTo) unsubscribeHttpUrl && (subscription.unsubscribeHttpUrl = unsubscribeHttpUrl) return getRepository(Subscription).save(subscription) @@ -59,6 +70,7 @@ export const saveSubscription = async ( status: SubscriptionStatus.Active, unsubscribeHttpUrl, unsubscribeMailTo, + icon, }) } diff --git a/packages/api/src/textToSpeech.d.ts b/packages/api/src/textToSpeech.d.ts deleted file mode 100644 index 7cf74d63b..000000000 --- a/packages/api/src/textToSpeech.d.ts +++ /dev/null @@ -1,31 +0,0 @@ -declare module '@omnivore/text-to-speech-handler' { - export function htmlToSpeechFile(htmlInput: HtmlInput): SpeechFile - - export interface HtmlInput { - title?: string - content: string - options: SSMLOptions - } - - export interface SSMLOptions { - primaryVoice?: string - secondaryVoice?: string - rate?: string - language?: string - } - - interface Utterance { - idx: string - wordOffset: number - wordCount: number - voice?: string - text: string - } - - export interface SpeechFile { - wordCount: number - language: string - defaultVoice: string - utterances: Utterance[] - } -} diff --git a/packages/api/src/utils/parser.ts b/packages/api/src/utils/parser.ts index 01e97d0bd..3d6242264 100644 --- a/packages/api/src/utils/parser.ts +++ b/packages/api/src/utils/parser.ts @@ -450,150 +450,6 @@ export const parseUrlMetadata = async ( } } -// Attempt to determine if an HTML blob is a newsletter -// based on it's contents. -// TODO: when we consolidate the handlers we could include this -// as a utility method on each one. -export const isProbablyNewsletter = async (html: string): Promise => { - const dom = parseHTML(html).document - const domCopy = parseHTML(dom.documentElement.outerHTML).document - const article = await new Readability(domCopy, { - debug: false, - keepTables: true, - }).parse() - - if (!article || !article.content) { - return false - } - - // substack newsletter emails have tables with a *post-meta class - if (dom.querySelector('table[class$="post-meta"]')) { - return true - } - - // If the article has a header link, and substack icons its probably a newsletter - const href = findNewsletterHeaderHref(dom) - const heartIcon = dom.querySelector( - 'table tbody td span a img[src*="HeartIcon"]' - ) - const recommendIcon = dom.querySelector( - 'table tbody td span a img[src*="RecommendIconRounded"]' - ) - if (href && (heartIcon || recommendIcon)) { - return true - } - - // Check if this is a beehiiv.net newsletter - if (dom.querySelectorAll('img[src*="beehiiv.net"]').length > 0) { - const beehiivUrl = beehiivNewsletterHref(dom) - if (beehiivUrl) { - return true - } - } - - // Check if this is a newsletter from revue - if ( - dom.querySelectorAll('img[src*="getrevue.co"], img[src*="revue.email"]') - .length > 0 - ) { - const getrevueUrl = revueNewsletterHref(dom) - if (getrevueUrl) { - return true - } - } - - // Check if this is a convertkit.com newsletter - return ( - dom.querySelectorAll( - 'img[src*="convertkit.com"], img[src*="convertkit-mail.com"]' - ).length > 0 - ) -} - -const beehiivNewsletterHref = (dom: Document): string | undefined => { - const readOnline = dom.querySelectorAll('table tr td div a[class*="link"]') - let res: string | undefined = undefined - readOnline.forEach((e) => { - if (e.textContent === 'Read Online') { - res = e.getAttribute('href') || undefined - } - }) - return res -} - -const convertkitNewsletterHref = (dom: Document): string | undefined => { - const readOnline = dom.querySelectorAll('table tr td a') - let res: string | undefined = undefined - readOnline.forEach((e) => { - if (e.textContent === 'View this email in your browser') { - res = e.getAttribute('href') || undefined - } - }) - return res -} - -const revueNewsletterHref = (dom: Document): string | undefined => { - const viewOnline = dom.querySelectorAll('table tr td a[target="_blank"]') - let res: string | undefined = undefined - viewOnline.forEach((e) => { - if (e.textContent === 'View online') { - res = e.getAttribute('href') || undefined - } - }) - return res -} - -const findNewsletterHeaderHref = (dom: Document): string | undefined => { - // Substack header links - const postLink = dom.querySelector('h1 a ') - if (postLink) { - return postLink.getAttribute('href') || undefined - } - - // Check if this is a beehiiv.net newsletter - const beehiiv = beehiivNewsletterHref(dom) - if (beehiiv) { - return beehiiv - } - - // Check if this is a revue newsletter - const revue = revueNewsletterHref(dom) - if (revue) { - return revue - } - - // Check if this is a convertkit.com newsletter - const convertkitUrl = convertkitNewsletterHref(dom) - if (convertkitUrl) { - return convertkitUrl - } - - return undefined -} - -// Given an HTML blob tries to find a URL to use for -// a canonical URL. -export const findNewsletterUrl = async ( - html: string -): Promise => { - const dom = parseHTML(html).document - - // Check if this is a substack newsletter - const href = findNewsletterHeaderHref(dom) - if (href) { - // Try to make a HEAD request so we get the redirected URL, since these - // will usually be behind tracking url redirects - return axios({ - method: 'HEAD', - url: href, - }) - .then((res) => res.request.res.responseUrl as string | undefined) - .catch((e) => href) - } - - return undefined -} - export const isProbablyArticle = async ( email: string, subject: string @@ -621,3 +477,18 @@ export const parseEmailAddress = (from: string): addressparser.EmailAddress => { } return { name: '', address: from } } + +export const fetchFavicon = async ( + url: string +): Promise => { + try { + // get the correct url if it's a redirect + const response = await axios.head(url, { timeout: 5000 }) + const realUrl = response.request.res.responseUrl + const domain = new URL(realUrl).hostname + return `https://api.faviconkit.com/${domain}/32` + } catch (e) { + console.log('Error fetching favicon', e) + return undefined + } +} diff --git a/packages/api/test/resolvers/article.test.ts b/packages/api/test/resolvers/article.test.ts index f9175d599..d64a23d4e 100644 --- a/packages/api/test/resolvers/article.test.ts +++ b/packages/api/test/resolvers/article.test.ts @@ -10,7 +10,11 @@ import { expect } from 'chai' import 'mocha' import { User } from '../../src/entity/user' import chaiString from 'chai-string' -import { UpdateReason, UploadFileStatus } from '../../src/generated/graphql' +import { + SyncUpdatedItemEdge, + UpdateReason, + UploadFileStatus, +} from '../../src/generated/graphql' import { ArticleSavingRequestStatus, Highlight, @@ -1009,7 +1013,7 @@ describe('Article API', () => { } // set the since to be the timestamp before deletion - since = pages[4].createdAt.toISOString() + since = pages[4].updatedAt!.toISOString() // Delete some pages for (let i = 0; i < 3; i++) { @@ -1033,7 +1037,11 @@ describe('Article API', () => { authToken ).expect(200) - expect(res.body.data.updatesSince.edges.length).to.eql(3) + expect( + res.body.data.updatesSince.edges.filter( + (e: SyncUpdatedItemEdge) => e.updateReason === UpdateReason.Deleted + ).length + ).to.eql(3) expect(res.body.data.updatesSince.edges[0].itemID).to.eq( deletedPages[0].id ) diff --git a/packages/api/test/routers/emails.test.ts b/packages/api/test/routers/emails.test.ts index 26220d656..2f78483a7 100644 --- a/packages/api/test/routers/emails.test.ts +++ b/packages/api/test/routers/emails.test.ts @@ -52,35 +52,8 @@ describe('Emails Router', () => { sinon.restore() }) - context('when email is a newsletter', () => { - before(() => { - sinon.replace(parser, 'isProbablyNewsletter', sinon.fake.resolves(true)) - }) - - it('saves the email as a newsletter', async () => { - const data = { - message: { - data: Buffer.from( - JSON.stringify({ from, to, subject, html }) - ).toString('base64'), - publishTime: new Date().toISOString(), - }, - } - const res = await request - .post(`/svc/pubsub/emails/forward?token=${token}`) - .send(data) - .expect(200) - expect(res.text).to.eql('Newsletter') - }) - }) - context('when email is an article', () => { before(() => { - sinon.replace( - parser, - 'isProbablyNewsletter', - sinon.fake.resolves(false) - ) sinon.replace(parser, 'isProbablyArticle', sinon.fake.resolves(true)) }) @@ -103,11 +76,6 @@ describe('Emails Router', () => { context('when email is a regular email', () => { before(() => { - sinon.replace( - parser, - 'isProbablyNewsletter', - sinon.fake.resolves(false) - ) sinon.replace(parser, 'isProbablyArticle', sinon.fake.resolves(false)) }) diff --git a/packages/api/test/utils/parser.test.ts b/packages/api/test/utils/parser.test.ts index 83bd23294..356dfde21 100644 --- a/packages/api/test/utils/parser.test.ts +++ b/packages/api/test/utils/parser.test.ts @@ -4,11 +4,8 @@ import { expect } from 'chai' import 'chai/register-should' import fs from 'fs' import { - findNewsletterUrl, - generateUniqueUrl, getTitleFromEmailSubject, isProbablyArticle, - isProbablyNewsletter, parseEmailAddress, parsePageMetadata, parsePreparedContent, @@ -24,69 +21,6 @@ const load = (path: string): string => { return fs.readFileSync(path, 'utf8') } -describe('isProbablyNewsletter', () => { - it('returns true for substack newsletter', async () => { - const html = load('./test/utils/data/substack-forwarded-newsletter.html') - await expect(isProbablyNewsletter(html)).to.eventually.be.true - }) - it('returns true for private forwarded substack newsletter', async () => { - const html = load( - './test/utils/data/substack-private-forwarded-newsletter.html' - ) - await expect(isProbablyNewsletter(html)).to.eventually.be.true - }) - it('returns false for substack welcome email', async () => { - const html = load('./test/utils/data/substack-forwarded-welcome-email.html') - await expect(isProbablyNewsletter(html)).to.eventually.be.false - }) - it('returns true for beehiiv.com newsletter', async () => { - const html = load('./test/utils/data/beehiiv-newsletter.html') - await expect(isProbablyNewsletter(html)).to.eventually.be.true - }) -}) - -describe('findNewsletterUrl', async () => { - it('gets the URL from the header if it is a substack newsletter', async () => { - nock('https://email.mg2.substack.com') - .head( - '/c/eJxNkk2TojAQhn-N3KTyQfg4cGDGchdnYcsZx9K5UCE0EMVAkTiKv36iHnarupNUd7rfVJ4W3EDTj1M89No496Uw0wCxgovuwBgYnbOGsZBVjDHzKPWYU8VehUMWOlIX9Qhw4rKLzXgGZziXnRTcyF7dK0iIGMVOG_OS1aTmKPRDilgVhTQUPCQIcE0x-MFTmJ8rCUpA3KtuenR2urg1ZtAzmszI0tq_Z7m66y-ilQo0uAqMTQ7WRX8auJKg56blZg7WB-iHDuYEBzO6NP0R1IwuYFphQbbTjnTH9NBfs80nym4Zyj8uUvyKbtUyGr5eUz9fNDQ7JCxfJDo9dW1lY9lmj_JNivPbGmf2Pt_lN9tDit9b-WeTetni85Z9pDpVOd7L1E_Vy7egayNO23ZP34eSeLJeux1b0rer_xaZ7ykS78nuSjMY-nL98rparNZNcv07JCjN06_EkTFBxBqOUMACErnELUNMSxTUjLDQZwzcqa4bRjCfeejUEFefS224OLr2S5wxPtij7lVrs80d2CNseRV2P52VNFMBipcdVE-U5jkRD7hFAwpGOylVwU2Mfc9qBh7DoR89yVnWXhgQFHnIsbpVb6tU_B-hH_2yzWY' - ) - .reply(302, undefined, { - Location: - 'https://newsletter.slowchinese.net/p/companies-that-eat-people-217', - }) - .get('/p/companies-that-eat-people-217') - .reply(200, '') - const html = load('./test/utils/data/substack-forwarded-newsletter.html') - const url = await findNewsletterUrl(html) - // Not sure if the redirects from substack expire, this test could eventually fail - expect(url).to.startWith( - 'https://newsletter.slowchinese.net/p/companies-that-eat-people-217' - ) - }) - it('gets the URL from the header if it is a beehiiv newsletter', async () => { - nock('https://u23463625.ct.sendgrid.net') - .head( - '/ss/c/AX1lEgEQaxtvFxLaVo0GBo_geajNrlI1TGeIcmMViR3pL3fEDZnbbkoeKcaY62QZk0KPFudUiUXc_uMLerV4nA/3k5/3TFZmreTR0qKSCgowABnVg/h30/zzLik7UXd1H_n4oyd5W8Xu639AYQQB2UXz-CsssSnno' - ) - .reply(302, undefined, { - Location: 'https://www.milkroad.com/p/talked-guy-spent-30m-beeple', - }) - .get('/p/talked-guy-spent-30m-beeple') - .reply(200, '') - const html = load('./test/utils/data/beehiiv-newsletter.html') - const url = await findNewsletterUrl(html) - expect(url).to.startWith( - 'https://www.milkroad.com/p/talked-guy-spent-30m-beeple' - ) - }) - it('returns undefined if it is not a newsletter', async () => { - const html = load('./test/utils/data/substack-forwarded-welcome-email.html') - const url = await findNewsletterUrl(html) - expect(url).to.be.undefined - }) -}) - describe('parseMetadata', async () => { it('gets author, title, image, description', async () => { const html = load('./test/utils/data/substack-post.html') @@ -164,15 +98,6 @@ describe('isProbablyArticle', () => { }) }) -describe('generateUniqueUrl', () => { - it('generates a unique URL', () => { - const url1 = generateUniqueUrl() - const url2 = generateUniqueUrl() - - expect(url1).to.not.eql(url2) - }) -}) - describe('getTitleFromEmailSubject', () => { it('returns the title from the email subject', () => { const title = 'test subject' diff --git a/packages/content-fetch/apple-news-handler.js b/packages/content-fetch/apple-news-handler.js deleted file mode 100644 index 0759dec23..000000000 --- a/packages/content-fetch/apple-news-handler.js +++ /dev/null @@ -1,36 +0,0 @@ -/* eslint-disable no-undef */ -/* eslint-disable no-empty */ -/* eslint-disable @typescript-eslint/explicit-function-return-type */ -/* eslint-disable @typescript-eslint/no-var-requires */ -/* eslint-disable @typescript-eslint/no-require-imports */ -require('dotenv').config(); -const Url = require('url'); -const axios = require('axios'); -const { promisify } = require('util'); -const { DateTime } = require('luxon'); -const os = require('os'); -const { Cipher } = require('crypto'); -const { parseHTML } = require('linkedom'); - -exports.appleNewsHandler = { - - shouldPrehandle: (url, env) => { - const u = new URL(url); - if (u.hostname === 'apple.news') { - return true; - } - return false - }, - - prehandle: async (url, env) => { - const MOBILE_USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.61 Safari/537.36' - const response = await axios.get(url, { headers: { 'User-Agent': MOBILE_USER_AGENT } } ); - const data = response.data; - - const dom = parseHTML(data).document; - - // make sure its a valid URL by wrapping in new URL - const u = new URL(dom.querySelector('span.click-here').parentNode.href); - return { url: u.href }; - } -} diff --git a/packages/content-fetch/bloomberg-handler.js b/packages/content-fetch/bloomberg-handler.js deleted file mode 100644 index d79a568bb..000000000 --- a/packages/content-fetch/bloomberg-handler.js +++ /dev/null @@ -1,39 +0,0 @@ -/* eslint-disable no-undef */ -/* eslint-disable no-empty */ -/* eslint-disable @typescript-eslint/explicit-function-return-type */ -/* eslint-disable @typescript-eslint/no-var-requires */ -/* eslint-disable @typescript-eslint/no-require-imports */ -require('dotenv').config(); -const axios = require('axios'); -const os = require('os'); -const { parseHTML } = require('linkedom'); - -exports.bloombergHandler = { - - shouldPrehandle: (url, env) => { - const BLOOMBERG_URL_MATCH = - /https?:\/\/(www\.)?bloomberg.[a-zA-Z0-9()]{1,6}\b([-a-zA-Z0-9()@:%_\+.~#?&//=]*)/ - return BLOOMBERG_URL_MATCH.test(url.toString()) - }, - - prehandle: async (url, env) => { - console.log('prehandling bloomberg url', url) - - try { - const response = await axios.get('https://app.scrapingbee.com/api/v1', { - params: { - 'api_key': process.env.SCRAPINGBEE_API_KEY, - 'url': url, - 'return_page_source': true, - 'block_ads': true, - 'block_resources': false, - } - }) - const dom = parseHTML(response.data).document; - return { title: dom.title, content: dom.querySelector('body').innerHTML, url: url } - } catch (error) { - console.error('error prehandling bloomberg url', error) - throw error - } - } -} diff --git a/packages/content-fetch/derstandard-handler.js b/packages/content-fetch/derstandard-handler.js deleted file mode 100644 index a44db6f2a..000000000 --- a/packages/content-fetch/derstandard-handler.js +++ /dev/null @@ -1,35 +0,0 @@ -/* eslint-disable no-undef */ -/* eslint-disable no-empty */ -/* eslint-disable @typescript-eslint/explicit-function-return-type */ -/* eslint-disable @typescript-eslint/no-var-requires */ -/* eslint-disable @typescript-eslint/no-require-imports */ -require('dotenv').config(); -const axios = require('axios'); -const { parseHTML } = require('linkedom'); - -exports.derstandardHandler = { - shouldPrehandle: (url, env) => { - const u = new URL(url); - return u.hostname === 'www.derstandard.at'; - }, - - prehandle: async (url, env) => { - const response = await axios.get(url, { - // set cookie to give consent to get the article - headers: { - 'cookie': `DSGVO_ZUSAGE_V1=true; consentUUID=2bacb9c1-1e80-4be0-9f7b-ee987cf4e7b0_6` - }, - }); - const content = response.data; - - var title = undefined; - const dom = parseHTML(content).document; - const titleElement = dom.querySelector('.article-title') - if (!titleElement) { - title = titleElement.textContent - titleElement.remove() - } - - return { content: dom.body.outerHTML, title: title }; - } -} diff --git a/packages/content-fetch/fetch-content.js b/packages/content-fetch/fetch-content.js index 01dbe199c..0c3118544 100644 --- a/packages/content-fetch/fetch-content.js +++ b/packages/content-fetch/fetch-content.js @@ -9,16 +9,10 @@ const puppeteer = require('puppeteer-core'); const axios = require('axios'); const jwt = require('jsonwebtoken'); const { promisify } = require('util'); +const { parseHTML } = require('linkedom'); +const { preHandleContent } = require('@omnivore/content-handler'); + const signToken = promisify(jwt.sign); -const { appleNewsHandler } = require('./apple-news-handler'); -const { twitterHandler } = require('./twitter-handler'); -const { youtubeHandler } = require('./youtube-handler'); -const { tDotCoHandler } = require('./t-dot-co-handler'); -const { pdfHandler } = require('./pdf-handler'); -const { mediumHandler } = require('./medium-handler'); -const { derstandardHandler } = require('./derstandard-handler'); -const { imageHandler } = require('./image-handler'); -const { scrapingBeeHandler } = require('./scrapingBee-handler') const MOBILE_USER_AGENT = 'Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/93.0.4577.62 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)' const DESKTOP_USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 11_6_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4372.0 Safari/537.36' @@ -29,8 +23,6 @@ const NON_SCRIPT_HOSTS= ['medium.com', 'fastcompany.com']; const ALLOWED_CONTENT_TYPES = ['text/html', 'application/octet-stream', 'text/plain', 'application/pdf']; -const { parseHTML } = require('linkedom'); - // Add stealth plugin to hide puppeteer usage // const StealthPlugin = require('puppeteer-extra-plugin-stealth'); // puppeteer.use(StealthPlugin()); @@ -207,19 +199,6 @@ const saveUploadedPdf = async (userId, url, uploadFileId, articleSavingRequestId ); }; -const handlers = { - 'pdf': pdfHandler, - 'apple-news': appleNewsHandler, - 'twitter': twitterHandler, - 'youtube': youtubeHandler, - 't-dot-co': tDotCoHandler, - 'medium': mediumHandler, - 'derstandard': derstandardHandler, - 'image': imageHandler, - 'scrapingBee': scrapingBeeHandler, -}; - - async function fetchContent(req, res) { functionStartTime = Date.now(); @@ -246,61 +225,19 @@ async function fetchContent(req, res) { return res.sendStatus(400); } - // if (!userId || !articleSavingRequestId) { - // Object.assign(logRecord, { invalidParams: true, body: req.body, query: req.query }); - // console.log(`Invalid parameters`, logRecord); - // return res.sendStatus(400); - // } - - // Before we run the regular handlers we check to see if we need tp - // pre-resolve the URL. TODO: This should probably happen recursively, - // so URLs can be pre-resolved, handled, pre-resolved, handled, etc. - for (const [key, handler] of Object.entries(handlers)) { - if (handler.shouldResolve && handler.shouldResolve(url)) { - try { - url = await handler.resolve(url); - validateUrlString(url); - } catch (err) { - console.log('error resolving url with handler', key, err); - } - break; - } - } - - // Before we fetch the page we check the handlers, to see if they want - // to perform a prefetch action that can modify our requests. - // enumerate the handlers and see if any of them want to handle the request - const handler = Object.keys(handlers).find(key => { - try { - return handlers[key].shouldPrehandle(url) - } catch (e) { - console.log('error with handler: ', key, e); - } - return false; - }); - - var title = undefined; - var content = undefined; - var contentType = undefined; - - if (handler) { - try { - // The only handler we have now can modify the URL, but in the - // future maybe we let it modify content. In that case - // we might exit the request early. - console.log('pre-handling url with handler: ', handler); - - const result = await handlers[handler].prehandle(url); - if (result && result.url) { - url = result.url - validateUrlString(url); - } - if (result && result.title) { title = result.title } - if (result && result.content) { content = result.content } - if (result && result.contentType) { contentType = result.contentType } - } catch (e) { - console.log('error with handler: ', handler, e); + // pre handle url with custom handlers + let title, content, contentType; + try { + const result = await preHandleContent(url); + if (result && result.url) { + url = result.url + validateUrlString(url); } + if (result && result.title) { title = result.title } + if (result && result.content) { content = result.content } + if (result && result.contentType) { contentType = result.contentType } + } catch (e) { + console.log('error with handler: ', e); } let context, page, finalUrl; diff --git a/packages/content-fetch/image-handler.js b/packages/content-fetch/image-handler.js deleted file mode 100644 index 59f132afc..000000000 --- a/packages/content-fetch/image-handler.js +++ /dev/null @@ -1,34 +0,0 @@ -/* eslint-disable no-undef */ -/* eslint-disable no-empty */ -/* eslint-disable @typescript-eslint/explicit-function-return-type */ -/* eslint-disable @typescript-eslint/no-var-requires */ -/* eslint-disable @typescript-eslint/no-require-imports */ -require('dotenv').config(); - - -exports.imageHandler = { - shouldPrehandle: (url, env) => { - const IMAGE_URL_PATTERN = - /(https?:\/\/.*\.(?:jpg|jpeg|png|webp))/i - return IMAGE_URL_PATTERN.test(url.toString()) - }, - - prehandle: async (url, env) => { - const title = url.toString().split('/').pop(); - const content = ` - - - ${title} - - - - -
- ${title} -
- - ` - - return { title, content }; - } -} diff --git a/packages/content-fetch/medium-handler.js b/packages/content-fetch/medium-handler.js deleted file mode 100644 index e6a605a0e..000000000 --- a/packages/content-fetch/medium-handler.js +++ /dev/null @@ -1,29 +0,0 @@ -/* eslint-disable no-undef */ -/* eslint-disable no-empty */ -/* eslint-disable @typescript-eslint/explicit-function-return-type */ -/* eslint-disable @typescript-eslint/no-var-requires */ -/* eslint-disable @typescript-eslint/no-require-imports */ -require('dotenv').config(); -const axios = require('axios'); -const os = require('os'); - -exports.mediumHandler = { - - shouldPrehandle: (url, env) => { - const u = new URL(url); - return u.hostname.endsWith('medium.com') - }, - - prehandle: async (url, env) => { - console.log('prehandling medium url', url) - - try { - const res = new URL(url); - res.searchParams.delete('source'); - return { url: res.toString() } - } catch (error) { - console.error('error prehandling medium url', error) - throw error - } - } -} diff --git a/packages/content-fetch/package.json b/packages/content-fetch/package.json index f58675a74..3df85a237 100644 --- a/packages/content-fetch/package.json +++ b/packages/content-fetch/package.json @@ -11,7 +11,8 @@ "linkedom": "^0.14.9", "luxon": "^2.3.1", "puppeteer-core": "^16.1.0", - "underscore": "^1.13.4" + "underscore": "^1.13.4", + "@omnivore/content-handler": "1.0.0" }, "scripts": { "start": "node app.js", diff --git a/packages/content-fetch/pdf-handler.js b/packages/content-fetch/pdf-handler.js deleted file mode 100644 index 1260db287..000000000 --- a/packages/content-fetch/pdf-handler.js +++ /dev/null @@ -1,21 +0,0 @@ -/* eslint-disable no-undef */ -/* eslint-disable no-empty */ -/* eslint-disable @typescript-eslint/explicit-function-return-type */ -/* eslint-disable @typescript-eslint/no-var-requires */ -/* eslint-disable @typescript-eslint/no-require-imports */ -require('dotenv').config(); -const Url = require('url'); - - -exports.pdfHandler = { - - shouldPrehandle: (url, env) => { - const u = Url.parse(url) - const path = u.path.replace(u.search, '') - return path.endsWith('.pdf') - }, - - prehandle: async (url, env) => { - return { contentType: 'application/pdf' }; - } -} diff --git a/packages/content-fetch/scrapingBee-handler.js b/packages/content-fetch/scrapingBee-handler.js deleted file mode 100644 index 6563fca44..000000000 --- a/packages/content-fetch/scrapingBee-handler.js +++ /dev/null @@ -1,44 +0,0 @@ -/* eslint-disable no-undef */ -/* eslint-disable no-empty */ -/* eslint-disable @typescript-eslint/explicit-function-return-type */ -/* eslint-disable @typescript-eslint/no-var-requires */ -/* eslint-disable @typescript-eslint/no-require-imports */ -require('dotenv').config(); -const axios = require('axios'); -const { parseHTML } = require('linkedom'); - -const os = require('os'); - -exports.scrapingBeeHandler = { - - shouldPrehandle: (url, env) => { - const u = new URL(url); - const hostnames = [ - 'nytimes.com', - 'news.google.com', - ] - - return hostnames.some((h) => u.hostname.endsWith(h)) - }, - - prehandle: async (url, env) => { - console.log('prehandling url with scrapingbee', url) - - try { - const response = await axios.get('https://app.scrapingbee.com/api/v1', { - params: { - 'api_key': process.env.SCRAPINGBEE_API_KEY, - 'url': url, - 'return_page_source': true, - 'block_ads': true, - 'block_resources': false, - } - }) - const dom = parseHTML(response.data).document; - return { title: dom.title, content: response.data, url: url } - } catch (error) { - console.error('error prehandling url w/scrapingbee', error) - throw error - } - } -} diff --git a/packages/content-fetch/t-dot-co-handler.js b/packages/content-fetch/t-dot-co-handler.js deleted file mode 100644 index 170f97fb7..000000000 --- a/packages/content-fetch/t-dot-co-handler.js +++ /dev/null @@ -1,31 +0,0 @@ -/* eslint-disable no-undef */ -/* eslint-disable no-empty */ -/* eslint-disable @typescript-eslint/explicit-function-return-type */ -/* eslint-disable @typescript-eslint/no-var-requires */ -/* eslint-disable @typescript-eslint/no-require-imports */ -require('dotenv').config(); -const axios = require('axios'); -const Url = require('url'); - - -exports.tDotCoHandler = { - - shouldResolve: function (url, env) { - const T_DOT_CO_URL_MATCH = /^https:\/\/(?:www\.)?t\.co\/.*$/; - return T_DOT_CO_URL_MATCH.test(url); - }, - - resolve: async function(url, env) { - return await axios.get(url, { maxRedirects: 0, validateStatus: null }) - .then(res => { - return Url.parse(res.headers.location).href; - }).catch((err) => { - console.log('err with t.co url', err); - return undefined; - }); - }, - - shouldPrehandle: (url, env) => { - return false - }, -} diff --git a/packages/content-fetch/test/apple-news-handler.test.js b/packages/content-fetch/test/apple-news-handler.test.js deleted file mode 100644 index 4531d720e..000000000 --- a/packages/content-fetch/test/apple-news-handler.test.js +++ /dev/null @@ -1,9 +0,0 @@ -const { expect } = require('chai') -const { appleNewsHandler } = require('../apple-news-handler') - -describe('open a simple web page', () => { - it('should return a response', async () => { - const response = await appleNewsHandler.prehandle('https://apple.news/AxjzaZaPvSn23b67LhXI5EQ') - console.log('response', response) - }) -}) diff --git a/packages/content-fetch/test/babel-register.js b/packages/content-fetch/test/babel-register.js new file mode 100644 index 000000000..a6f65f60a --- /dev/null +++ b/packages/content-fetch/test/babel-register.js @@ -0,0 +1,3 @@ +const register = require('@babel/register').default + +register({ extensions: ['.ts', '.tsx', '.js', '.jsx'] }) diff --git a/packages/content-fetch/test/stub.test.ts b/packages/content-fetch/test/stub.test.ts new file mode 100644 index 000000000..173ca4917 --- /dev/null +++ b/packages/content-fetch/test/stub.test.ts @@ -0,0 +1,13 @@ +import 'mocha' +import * as chai from 'chai' +import { expect } from 'chai' +import 'chai/register-should' +import chaiString from 'chai-string' + +chai.use(chaiString) + +describe('Stub test', () => { + it('should pass', () => { + expect(true).to.be.true + }) +}) diff --git a/packages/content-fetch/test/youtube-handler.test.js b/packages/content-fetch/test/youtube-handler.test.js deleted file mode 100644 index d34643773..000000000 --- a/packages/content-fetch/test/youtube-handler.test.js +++ /dev/null @@ -1,12 +0,0 @@ -const { expect } = require('chai') -const { getYoutubeVideoId } = require('../youtube-handler') - -describe('getYoutubeVideoId', () => { - it('should parse video id out of a URL', async () => { - expect('BnSUk0je6oo').to.eq(getYoutubeVideoId('https://www.youtube.com/watch?v=BnSUk0je6oo&t=269s')); - expect('vFD2gu007dc').to.eq(getYoutubeVideoId('https://www.youtube.com/watch?v=vFD2gu007dc&list=RDvFD2gu007dc&start_radio=1')); - expect('vFD2gu007dc').to.eq(getYoutubeVideoId('https://youtu.be/vFD2gu007dc')); - expect('BMFVCnbRaV4').to.eq(getYoutubeVideoId('https://youtube.com/watch?v=BMFVCnbRaV4&feature=share')); - expect('cg9b4RC87LI').to.eq(getYoutubeVideoId('https://youtu.be/cg9b4RC87LI?t=116')); - }) -}) diff --git a/packages/content-fetch/twitter-handler.js b/packages/content-fetch/twitter-handler.js deleted file mode 100644 index 7ae93072c..000000000 --- a/packages/content-fetch/twitter-handler.js +++ /dev/null @@ -1,172 +0,0 @@ -/* eslint-disable no-undef */ -/* eslint-disable no-empty */ -/* eslint-disable @typescript-eslint/explicit-function-return-type */ -/* eslint-disable @typescript-eslint/no-var-requires */ -/* eslint-disable @typescript-eslint/no-require-imports */ -require('dotenv').config(); -const axios = require('axios'); -const { DateTime } = require('luxon'); -const _ = require('underscore'); - -const TWITTER_BEARER_TOKEN = process.env.TWITTER_BEARER_TOKEN; -const TWITTER_URL_MATCH = /twitter\.com\/(?:#!\/)?(\w+)\/status(?:es)?\/(\d+)(?:\/.*)?/ - -const embeddedTweet = async (url) => { - - const BASE_ENDPOINT = 'https://publish.twitter.com/oembed' - - const apiUrl = new URL(BASE_ENDPOINT) - apiUrl.searchParams.append('url', url); - apiUrl.searchParams.append('omit_script', true); - apiUrl.searchParams.append('dnt', true); - - return await axios.get(apiUrl.toString(), { - headers: { - Authorization: `Bearer ${TWITTER_BEARER_TOKEN}`, - redirect: "follow", - }, - }); -}; - -const getTweetFields = () => { - const TWEET_FIELDS = - "&tweet.fields=attachments,author_id,conversation_id,created_at," + - "entities,geo,in_reply_to_user_id,lang,possibly_sensitive,public_metrics,referenced_tweets," + - "source,withheld"; - const EXPANSIONS = "&expansions=author_id,attachments.media_keys"; - const USER_FIELDS = - "&user.fields=created_at,description,entities,location,pinned_tweet_id,profile_image_url,protected,public_metrics,url,verified,withheld"; - const MEDIA_FIELDS = - "&media.fields=duration_ms,height,preview_image_url,url,media_key,public_metrics,width"; - - return `${TWEET_FIELDS}${EXPANSIONS}${USER_FIELDS}${MEDIA_FIELDS}`; -} - -const getTweetById = async (id) => { - const BASE_ENDPOINT = "https://api.twitter.com/2/tweets/"; - const apiUrl = new URL(BASE_ENDPOINT + id + '?' + getTweetFields()) - - return await axios.get(apiUrl.toString(), { - headers: { - Authorization: `Bearer ${TWITTER_BEARER_TOKEN}`, - redirect: "follow", - }, - }); -}; - -const getUserByUsername = async (username) => { - const BASE_ENDPOINT = "https://api.twitter.com/2/users/by/username/"; - - const apiUrl = new URL(BASE_ENDPOINT + username) - apiUrl.searchParams.append('user.fields', 'profile_image_url'); - - return await axios.get(apiUrl.toString(), { - headers: { - Authorization: `Bearer ${TWITTER_BEARER_TOKEN}`, - redirect: "follow", - }, - }); -}; - -const titleForTweet = (tweet) => { - return `${tweet.data.author_name} on Twitter` -}; - -const titleForAuthor = (author) => { - return `${author.name} on Twitter` -}; - -const usernameFromStatusUrl = (url) => { - const match = url.toString().match(TWITTER_URL_MATCH) - return match[1] -}; - -const tweetIdFromStatusUrl = (url) => { - const match = url.toString().match(TWITTER_URL_MATCH) - return match[2] -}; - -const formatTimestamp = (timestamp) => { - return DateTime.fromJSDate(new Date(timestamp)).toLocaleString(DateTime.DATETIME_FULL); -}; - -exports.twitterHandler = { - - shouldPrehandle: (url, env) => { - return TWITTER_BEARER_TOKEN && TWITTER_URL_MATCH.test(url.toString()) - }, - - // version of the handler that uses the oembed API - // This isn't great as it doesn't work well with our - // readability API. But could potentially give a more consistent - // look to the tweets - // prehandle: async (url, env) => { - // const oeTweet = await embeddedTweet(url) - // const dom = new JSDOM(oeTweet.data.html); - // const bq = dom.window.document.querySelector('blockquote') - // console.log('blockquote:', bq); - - // const title = titleForTweet(oeTweet) - // return { title, content: '
' + bq.innerHTML + '
', url: oeTweet.data.url }; - // } - - prehandle: async (url, env) => { - console.log('prehandling twitter url', url) - - const tweetId = tweetIdFromStatusUrl(url) - const tweetData = (await getTweetById(tweetId)).data; - const authorId = tweetData.data.author_id; - const author = tweetData.includes.users.filter(u => u.id = authorId)[0]; - // escape html entities in title - const title = _.escape(titleForAuthor(author)) - const authorImage = author.profile_image_url.replace('_normal', '_400x400') - - let text = tweetData.data.text; - if (tweetData.data.entities && tweetData.data.entities.urls) { - for (let urlObj of tweetData.data.entities.urls) { - text = text.replace( - urlObj.url, - `${urlObj.display_url}` - ); - } - } - - const front = ` -
-

${text}

- ` - - var includesHtml = ''; - if (tweetData.includes.media) { - includesHtml = tweetData.includes.media.map(m => { - const linkUrl = m.type == 'photo' ? m.url : url; - const previewUrl = m.type == 'photo' ? m.url : m.preview_image_url; - const mediaOpen = ` - - - - ` - return mediaOpen - }).join('\n'); - } - - const back = ` - — ${author.username} ${author.name} ${formatTimestamp(tweetData.data.created_at)} -
- ` - const content = ` - - - - - - - - ${front} - ${includesHtml} - ${back} - ` - - return { content, url, title }; - } -} diff --git a/packages/content-handler/.eslintignore b/packages/content-handler/.eslintignore new file mode 100644 index 000000000..c2658d7d1 --- /dev/null +++ b/packages/content-handler/.eslintignore @@ -0,0 +1 @@ +node_modules/ diff --git a/packages/content-handler/.eslintrc b/packages/content-handler/.eslintrc new file mode 100644 index 000000000..e006282a6 --- /dev/null +++ b/packages/content-handler/.eslintrc @@ -0,0 +1,6 @@ +{ + "extends": "../../.eslintrc", + "parserOptions": { + "project": "tsconfig.json" + } +} \ No newline at end of file diff --git a/packages/content-handler/.gitignore b/packages/content-handler/.gitignore new file mode 100644 index 000000000..0ae7e5c9e --- /dev/null +++ b/packages/content-handler/.gitignore @@ -0,0 +1,2 @@ +node_modules +/lib diff --git a/packages/content-handler/.npmignore b/packages/content-handler/.npmignore new file mode 100644 index 000000000..b5e2b8569 --- /dev/null +++ b/packages/content-handler/.npmignore @@ -0,0 +1,7 @@ +/test/ +src +tsconfig.json +.eslintrc +.eslintignore +.gitignore +mocha-config.json diff --git a/packages/content-handler/mocha-config.json b/packages/content-handler/mocha-config.json new file mode 100644 index 000000000..44d1d24c1 --- /dev/null +++ b/packages/content-handler/mocha-config.json @@ -0,0 +1,5 @@ +{ + "extension": ["ts"], + "spec": "test/**/*.test.ts", + "require": "test/babel-register.js" + } \ No newline at end of file diff --git a/packages/content-handler/package.json b/packages/content-handler/package.json new file mode 100644 index 000000000..e4021b3e4 --- /dev/null +++ b/packages/content-handler/package.json @@ -0,0 +1,34 @@ +{ + "name": "@omnivore/content-handler", + "version": "1.0.0", + "description": "A standalone version of content handler to parse and format each type of content", + "main": "build/src/index.js", + "types": "build/src/index.d.ts", + "files": [ + "build/src" + ], + "license": "Apache-2.0", + "scripts": { + "test": "yarn mocha -r ts-node/register --config mocha-config.json", + "lint": "eslint src --ext ts,js,tsx,jsx", + "compile": "tsc", + "build": "tsc" + }, + "devDependencies": { + "chai": "^4.3.6", + "chai-as-promised": "^7.1.1", + "chai-string": "^1.5.0", + "eslint-plugin-prettier": "^4.0.0", + "mocha": "^10.0.0", + "nock": "^13.2.9" + }, + "dependencies": { + "addressparser": "^1.0.1", + "axios": "^0.27.2", + "linkedom": "^0.14.16", + "luxon": "^3.0.4", + "rfc2047": "^4.0.1", + "underscore": "^1.13.6", + "uuid": "^9.0.0" + } +} diff --git a/packages/content-handler/src/content-handler.ts b/packages/content-handler/src/content-handler.ts new file mode 100644 index 000000000..22216fabe --- /dev/null +++ b/packages/content-handler/src/content-handler.ts @@ -0,0 +1,175 @@ +import addressparser from 'addressparser' +import rfc2047 from 'rfc2047' +import { v4 as uuid } from 'uuid' +import { parseHTML } from 'linkedom' +import axios from 'axios' + +interface Unsubscribe { + mailTo?: string + httpUrl?: string +} + +export interface NewsletterInput { + postHeader: string + from: string + unSubHeader: string + email: string + html: string + title: string +} + +export interface NewsletterResult { + email: string + content: string + url: string + title: string + author: string + unsubMailTo?: string + unsubHttpUrl?: string +} + +export interface PreHandleResult { + url?: string + title?: string + content?: string + contentType?: string + dom?: Document +} + +export const FAKE_URL_PREFIX = 'https://omnivore.app/no_url?q=' +export const generateUniqueUrl = () => FAKE_URL_PREFIX + uuid() + +export abstract class ContentHandler { + protected senderRegex: RegExp + protected urlRegex: RegExp + name: string + + protected constructor() { + this.senderRegex = new RegExp(/NEWSLETTER_SENDER_REGEX/) + this.urlRegex = new RegExp(/NEWSLETTER_URL_REGEX/) + this.name = 'Handler name' + } + + shouldResolve(url: string): boolean { + return false + } + + async resolve(url: string): Promise { + return Promise.resolve(url) + } + + shouldPreHandle(url: string, dom?: Document): boolean { + return false + } + + async preHandle(url: string, dom?: Document): Promise { + return Promise.resolve({ url, dom }) + } + + async isNewsletter(input: { + postHeader: string + from: string + unSubHeader: string + html?: string + }): Promise { + const re = new RegExp(this.senderRegex) + return Promise.resolve( + re.test(input.from) && (!!input.postHeader || !!input.unSubHeader) + ) + } + + findNewsletterHeaderHref(dom: Document): string | undefined { + return undefined + } + + // Given an HTML blob tries to find a URL to use for + // a canonical URL. + async findNewsletterUrl(html: string): Promise { + const dom = parseHTML(html).document + + // Check if this is a substack newsletter + const href = this.findNewsletterHeaderHref(dom) + if (href) { + // Try to make a HEAD request, so we get the redirected URL, since these + // will usually be behind tracking url redirects + try { + const response = await axios.head(href, { timeout: 5000 }) + return Promise.resolve( + // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access + response.request.res.responseUrl as string | undefined + ) + } catch (e) { + console.log('error making HEAD request', e) + return Promise.resolve(href) + } + } + + return Promise.resolve(undefined) + } + + async parseNewsletterUrl( + _postHeader: string, + html: string + ): Promise { + // get newsletter url from html + const matches = html.match(this.urlRegex) + if (matches) { + return Promise.resolve(matches[1]) + } + return Promise.resolve(undefined) + } + + parseAuthor(from: string): string { + // get author name from email + // e.g. 'Jackson Harper from Omnivore App ' + // or 'Mike Allen ' + const parsed = addressparser(from) + if (parsed.length > 0) { + return parsed[0].name + } + return from + } + + parseUnsubscribe(unSubHeader: string): Unsubscribe { + // parse list-unsubscribe header + // e.g. List-Unsubscribe: , + const decoded = rfc2047.decode(unSubHeader) + return { + mailTo: decoded.match(/<(https?:\/\/[^>]*)>/)?.[1], + httpUrl: decoded.match(/]*)>/)?.[1], + } + } + + async handleNewsletter({ + email, + html, + postHeader, + title, + from, + unSubHeader, + }: NewsletterInput): Promise { + console.log('handleNewsletter', email, postHeader, title, from) + + if (!email || !html || !title || !from) { + console.log('invalid newsletter email') + throw new Error('invalid newsletter email') + } + + // fallback to default url if newsletter url does not exist + // assign a random uuid to the default url to avoid duplicate url + const url = + (await this.parseNewsletterUrl(postHeader, html)) || generateUniqueUrl() + const author = this.parseAuthor(from) + const unsubscribe = this.parseUnsubscribe(unSubHeader) + + return { + email, + content: html, + url, + title, + author, + unsubMailTo: unsubscribe.mailTo || '', + unsubHttpUrl: unsubscribe.httpUrl || '', + } + } +} diff --git a/packages/content-handler/src/index.ts b/packages/content-handler/src/index.ts new file mode 100644 index 000000000..e41c811c4 --- /dev/null +++ b/packages/content-handler/src/index.ts @@ -0,0 +1,116 @@ +import { AppleNewsHandler } from './websites/apple-news-handler' +import { BloombergHandler } from './websites/bloomberg-handler' +import { DerstandardHandler } from './websites/derstandard-handler' +import { ImageHandler } from './websites/image-handler' +import { MediumHandler } from './websites/medium-handler' +import { PdfHandler } from './websites/pdf-handler' +import { ScrapingBeeHandler } from './websites/scrapingBee-handler' +import { TDotCoHandler } from './websites/t-dot-co-handler' +import { TwitterHandler } from './websites/twitter-handler' +import { YoutubeHandler } from './websites/youtube-handler' +import { WikipediaHandler } from './websites/wikipedia-handler' +import { + ContentHandler, + NewsletterInput, + NewsletterResult, + PreHandleResult, +} from './content-handler' +import { SubstackHandler } from './newsletters/substack-handler' +import { AxiosHandler } from './newsletters/axios-handler' +import { GolangHandler } from './newsletters/golang-handler' +import { MorningBrewHandler } from './newsletters/morning-brew-handler' +import { BloombergNewsletterHandler } from './newsletters/bloomberg-newsletter-handler' +import { BeehiivHandler } from './newsletters/beehiiv-handler' +import { ConvertkitHandler } from './newsletters/convertkit-handler' +import { RevueHandler } from './newsletters/revue-handler' + +const validateUrlString = (url: string) => { + const u = new URL(url) + // Make sure the URL is http or https + if (u.protocol !== 'http:' && u.protocol !== 'https:') { + throw new Error('Invalid URL protocol check failed') + } + // Make sure the domain is not localhost + if (u.hostname === 'localhost' || u.hostname === '0.0.0.0') { + throw new Error('Invalid URL is localhost') + } + // Make sure the domain is not a private IP + if (/^(10|172\.16|192\.168)\..*/.test(u.hostname)) { + throw new Error('Invalid URL is private ip') + } +} + +const contentHandlers: ContentHandler[] = [ + new AppleNewsHandler(), + new BloombergHandler(), + new DerstandardHandler(), + new ImageHandler(), + new MediumHandler(), + new PdfHandler(), + new ScrapingBeeHandler(), + new TDotCoHandler(), + new TwitterHandler(), + new YoutubeHandler(), + new WikipediaHandler(), +] + +const newsletterHandlers: ContentHandler[] = [ + new AxiosHandler(), + new BloombergNewsletterHandler(), + new GolangHandler(), + new SubstackHandler(), + new MorningBrewHandler(), + new SubstackHandler(), + new BeehiivHandler(), + new ConvertkitHandler(), + new RevueHandler(), +] + +export const preHandleContent = async ( + url: string, + dom?: Document +): Promise => { + // Before we run the regular handlers we check to see if we need tp + // pre-resolve the URL. TODO: This should probably happen recursively, + // so URLs can be pre-resolved, handled, pre-resolved, handled, etc. + for (const handler of contentHandlers) { + if (handler.shouldResolve(url)) { + try { + const resolvedUrl = await handler.resolve(url) + if (resolvedUrl && validateUrlString(resolvedUrl)) { + url = resolvedUrl + } + } catch (err) { + console.log('error resolving url with handler', handler.name, err) + } + break + } + } + // Before we fetch the page we check the handlers, to see if they want + // to perform a prefetch action that can modify our requests. + // enumerate the handlers and see if any of them want to handle the request + for (const handler of contentHandlers) { + if (handler.shouldPreHandle(url, dom)) { + console.log('preHandleContent', handler.name, url) + return handler.preHandle(url, dom) + } + } + return undefined +} + +export const handleNewsletter = async ( + input: NewsletterInput +): Promise => { + for (const handler of newsletterHandlers) { + if (await handler.isNewsletter(input)) { + return handler.handleNewsletter(input) + } + } + + return undefined +} + +module.exports = { + preHandleContent, + handleNewsletter, +} diff --git a/packages/content-handler/src/newsletters/axios-handler.ts b/packages/content-handler/src/newsletters/axios-handler.ts new file mode 100644 index 000000000..cd783c30e --- /dev/null +++ b/packages/content-handler/src/newsletters/axios-handler.ts @@ -0,0 +1,46 @@ +import { ContentHandler, PreHandleResult } from '../content-handler' + +export class AxiosHandler extends ContentHandler { + constructor() { + super() + this.senderRegex = /<.+@axios.com>/ + this.urlRegex = /View in browser at (.*)<\/a>/ + this.name = 'axios' + } + + shouldPreHandle(url: string, dom?: Document): boolean { + const host = this.name + '.com' + // check if url ends with axios.com + return new URL(url).hostname.endsWith(host) + } + + async preHandle(url: string, dom: Document): Promise { + const body = dom.querySelector('table') + + let isFooter = false + // this removes ads and replaces table with a div + body?.querySelectorAll('table').forEach((el) => { + // remove the footer and the ads + if (!el.textContent || el.textContent.length < 20 || isFooter) { + el.remove() + } else { + // removes the first few rows of the table (the header) + // remove the last two rows of the table (they are ads) + el.querySelectorAll('tr').forEach((tr, i) => { + if (i <= 7 || i >= el.querySelectorAll('tr').length - 2) { + console.log('removing', tr) + tr.remove() + } + }) + // replace the table with a div + const div = dom.createElement('div') + div.innerHTML = el.innerHTML + el.parentNode?.replaceChild(div, el) + // set the isFooter flag to true because the next table is the footer + isFooter = true + } + }) + + return Promise.resolve({ dom }) + } +} diff --git a/packages/content-handler/src/newsletters/beehiiv-handler.ts b/packages/content-handler/src/newsletters/beehiiv-handler.ts new file mode 100644 index 000000000..0a50c1920 --- /dev/null +++ b/packages/content-handler/src/newsletters/beehiiv-handler.ts @@ -0,0 +1,43 @@ +import { ContentHandler } from '../content-handler' +import { parseHTML } from 'linkedom' + +export class BeehiivHandler extends ContentHandler { + constructor() { + super() + this.name = 'beehiiv' + } + + findNewsletterHeaderHref(dom: Document): string | undefined { + const readOnline = dom.querySelectorAll('table tr td div a[class*="link"]') + let res: string | undefined = undefined + readOnline.forEach((e) => { + if (e.textContent === 'Read Online') { + res = e.getAttribute('href') || undefined + } + }) + return res + } + + async isNewsletter(input: { + postHeader: string + from: string + unSubHeader: string + html: string + }): Promise { + const dom = parseHTML(input.html).document + if (dom.querySelectorAll('img[src*="beehiiv.net"]').length > 0) { + const beehiivUrl = this.findNewsletterHeaderHref(dom) + if (beehiivUrl) { + return Promise.resolve(true) + } + } + return false + } + + async parseNewsletterUrl( + postHeader: string, + html: string + ): Promise { + return this.findNewsletterUrl(html) + } +} diff --git a/packages/content-handler/src/newsletters/bloomberg-newsletter-handler.ts b/packages/content-handler/src/newsletters/bloomberg-newsletter-handler.ts new file mode 100644 index 000000000..a5f84f076 --- /dev/null +++ b/packages/content-handler/src/newsletters/bloomberg-newsletter-handler.ts @@ -0,0 +1,37 @@ +import { ContentHandler, PreHandleResult } from '../content-handler' + +export class BloombergNewsletterHandler extends ContentHandler { + constructor() { + super() + this.senderRegex = /<.+@mail.bloomberg.*.com>/ + this.urlRegex = / { + const body = dom.querySelector('.wrapper') + + // this removes header + body?.querySelector('.sailthru-variables')?.remove() + body?.querySelector('.preview-text')?.remove() + body?.querySelector('.logo-wrapper')?.remove() + body?.querySelector('.by-the-number-wrapper')?.remove() + // this removes footer + body?.querySelector('.quote-box-wrapper')?.remove() + body?.querySelector('.header-wrapper')?.remove() + body?.querySelector('.component-wrapper')?.remove() + body?.querySelector('.footer')?.remove() + + return Promise.resolve({ dom }) + } +} diff --git a/packages/content-handler/src/newsletters/convertkit-handler.ts b/packages/content-handler/src/newsletters/convertkit-handler.ts new file mode 100644 index 000000000..72e65f5da --- /dev/null +++ b/packages/content-handler/src/newsletters/convertkit-handler.ts @@ -0,0 +1,41 @@ +import { ContentHandler } from '../content-handler' +import { parseHTML } from 'linkedom' + +export class ConvertkitHandler extends ContentHandler { + constructor() { + super() + this.name = 'convertkit' + } + + findNewsletterHeaderHref(dom: Document): string | undefined { + const readOnline = dom.querySelectorAll('table tr td a') + let res: string | undefined = undefined + readOnline.forEach((e) => { + if (e.textContent === 'View this email in your browser') { + res = e.getAttribute('href') || undefined + } + }) + return res + } + + async isNewsletter(input: { + postHeader: string + from: string + unSubHeader: string + html: string + }): Promise { + const dom = parseHTML(input.html).document + return Promise.resolve( + dom.querySelectorAll( + 'img[src*="convertkit.com"], img[src*="convertkit-mail.com"]' + ).length > 0 + ) + } + + async parseNewsletterUrl( + postHeader: string, + html: string + ): Promise { + return this.findNewsletterUrl(html) + } +} diff --git a/packages/content-handler/src/newsletters/golang-handler.ts b/packages/content-handler/src/newsletters/golang-handler.ts new file mode 100644 index 000000000..7d4724004 --- /dev/null +++ b/packages/content-handler/src/newsletters/golang-handler.ts @@ -0,0 +1,27 @@ +import { ContentHandler, PreHandleResult } from '../content-handler' + +export class GolangHandler extends ContentHandler { + constructor() { + super() + this.senderRegex = /<.+@golangweekly.com>/ + this.urlRegex = /Read on the Web<\/a>/ + this.name = 'golangweekly' + } + + shouldPreHandle(url: string, dom?: Document): boolean { + const host = this.name + '.com' + // check if url ends with golangweekly.com + return new URL(url).hostname.endsWith(host) + } + + async preHandle(url: string, dom: Document): Promise { + const body = dom.querySelector('body') + + // this removes the "Subscribe" button + body?.querySelector('.el-splitbar')?.remove() + // this removes the title + body?.querySelector('.el-masthead')?.remove() + + return Promise.resolve({ dom }) + } +} diff --git a/packages/content-handler/src/newsletters/morning-brew-handler.ts b/packages/content-handler/src/newsletters/morning-brew-handler.ts new file mode 100644 index 000000000..f187ac0dc --- /dev/null +++ b/packages/content-handler/src/newsletters/morning-brew-handler.ts @@ -0,0 +1,35 @@ +import { ContentHandler, PreHandleResult } from '../content-handler' + +export class MorningBrewHandler extends ContentHandler { + constructor() { + super() + this.senderRegex = /Morning Brew / + this.urlRegex = /View Online<\/a>/ + this.name = 'morningbrew' + } + + shouldPreHandle(url: string, dom?: Document): boolean { + const host = this.name + '.com' + // check if url ends with morningbrew.com + return new URL(url).hostname.endsWith(host) + } + + async preHandle(url: string, dom: Document): Promise { + // retain the width of the cells in the table of market info + dom.querySelectorAll('.markets-arrow-cell').forEach((td) => { + const table = td.closest('table') + if (table) { + const bubbleTable = table.querySelector('.markets-bubble') + if (bubbleTable) { + // replace the nested table with the text + const e = bubbleTable.querySelector('.markets-table-text') + e && bubbleTable.parentNode?.replaceChild(e, bubbleTable) + } + // set custom class for the table + table.className = 'morning-brew-markets' + } + }) + + return Promise.resolve({ dom }) + } +} diff --git a/packages/content-handler/src/newsletters/revue-handler.ts b/packages/content-handler/src/newsletters/revue-handler.ts new file mode 100644 index 000000000..d8c8f911c --- /dev/null +++ b/packages/content-handler/src/newsletters/revue-handler.ts @@ -0,0 +1,46 @@ +import { ContentHandler } from '../content-handler' +import { parseHTML } from 'linkedom' + +export class RevueHandler extends ContentHandler { + constructor() { + super() + this.name = 'revue' + } + + findNewsletterHeaderHref(dom: Document): string | undefined { + const viewOnline = dom.querySelectorAll('table tr td a[target="_blank"]') + let res: string | undefined = undefined + viewOnline.forEach((e) => { + if (e.textContent === 'View online') { + res = e.getAttribute('href') || undefined + } + }) + return res + } + + async isNewsletter(input: { + postHeader: string + from: string + unSubHeader: string + html: string + }): Promise { + const dom = parseHTML(input.html).document + if ( + dom.querySelectorAll('img[src*="getrevue.co"], img[src*="revue.email"]') + .length > 0 + ) { + const getrevueUrl = this.findNewsletterHeaderHref(dom) + if (getrevueUrl) { + return Promise.resolve(true) + } + } + return false + } + + async parseNewsletterUrl( + postHeader: string, + html: string + ): Promise { + return this.findNewsletterUrl(html) + } +} diff --git a/packages/content-handler/src/newsletters/substack-handler.ts b/packages/content-handler/src/newsletters/substack-handler.ts new file mode 100644 index 000000000..164068623 --- /dev/null +++ b/packages/content-handler/src/newsletters/substack-handler.ts @@ -0,0 +1,90 @@ +import addressparser from 'addressparser' +import { ContentHandler, PreHandleResult } from '../content-handler' +import { parseHTML } from 'linkedom' + +export class SubstackHandler extends ContentHandler { + constructor() { + super() + this.name = 'substack' + } + + shouldPreHandle(url: string, dom: Document): boolean { + const host = this.name + '.com' + // check if url ends with substack.com + // or has a profile image hosted at substack.com + return ( + new URL(url).hostname.endsWith(host) || + !!dom + .querySelector('.email-body img') + ?.getAttribute('src') + ?.includes(host) + ) + } + + async preHandle(url: string, dom: Document): Promise { + const body = dom.querySelector('.email-body-container') + + // this removes header and profile avatar + body?.querySelector('.header')?.remove() + body?.querySelector('.preamble')?.remove() + body?.querySelector('.meta-author-wrap')?.remove() + // this removes meta button + body?.querySelector('.post-meta')?.remove() + // this removes footer + body?.querySelector('.post-cta')?.remove() + body?.querySelector('.container-border')?.remove() + body?.querySelector('.footer')?.remove() + + return Promise.resolve(dom) + } + + findNewsletterHeaderHref(dom: Document): string | undefined { + // Substack header links + const postLink = dom.querySelector('h1 a ') + if (postLink) { + return postLink.getAttribute('href') || undefined + } + + return undefined + } + + async isNewsletter({ + postHeader, + html, + }: { + postHeader: string + from: string + unSubHeader: string + html: string + }): Promise { + if (postHeader) { + return Promise.resolve(true) + } + const dom = parseHTML(html).document + // substack newsletter emails have tables with a *post-meta class + if (dom.querySelector('table[class$="post-meta"]')) { + return true + } + // If the article has a header link, and substack icons its probably a newsletter + const href = this.findNewsletterHeaderHref(dom) + const heartIcon = dom.querySelector( + 'table tbody td span a img[src*="HeartIcon"]' + ) + const recommendIcon = dom.querySelector( + 'table tbody td span a img[src*="RecommendIconRounded"]' + ) + return Promise.resolve(!!(href && (heartIcon || recommendIcon))) + } + + async parseNewsletterUrl( + postHeader: string, + html: string + ): Promise { + // raw SubStack newsletter url is like + // we need to get the real url from the raw url + if (postHeader && addressparser(postHeader).length > 0) { + return Promise.resolve(addressparser(postHeader)[0].name) + } + return this.findNewsletterUrl(html) + } +} diff --git a/packages/content-handler/src/websites/apple-news-handler.ts b/packages/content-handler/src/websites/apple-news-handler.ts new file mode 100644 index 000000000..0b4026fb6 --- /dev/null +++ b/packages/content-handler/src/websites/apple-news-handler.ts @@ -0,0 +1,31 @@ +import axios from 'axios' +import { parseHTML } from 'linkedom' +import { ContentHandler, PreHandleResult } from '../content-handler' + +export class AppleNewsHandler extends ContentHandler { + constructor() { + super() + this.name = 'Apple News' + } + + shouldPreHandle(url: string, dom?: Document): boolean { + const u = new URL(url) + return u.hostname === 'apple.news' + } + + async preHandle(url: string, document?: Document): Promise { + const MOBILE_USER_AGENT = + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.61 Safari/537.36' + const response = await axios.get(url, { + headers: { 'User-Agent': MOBILE_USER_AGENT }, + }) + const data = response.data as string + const dom = parseHTML(data).document + // make sure it's a valid URL by wrapping in new URL + const href = dom + .querySelector('span.click-here') + ?.parentElement?.getAttribute('href') + const u = href ? new URL(href) : undefined + return { url: u?.href } + } +} diff --git a/packages/content-handler/src/websites/bloomberg-handler.ts b/packages/content-handler/src/websites/bloomberg-handler.ts new file mode 100644 index 000000000..a867a3503 --- /dev/null +++ b/packages/content-handler/src/websites/bloomberg-handler.ts @@ -0,0 +1,41 @@ +import axios from 'axios' +import { parseHTML } from 'linkedom' +import { ContentHandler, PreHandleResult } from '../content-handler' + +export class BloombergHandler extends ContentHandler { + constructor() { + super() + this.name = 'Bloomberg' + } + + shouldPreHandle(url: string, dom?: Document): boolean { + const BLOOMBERG_URL_MATCH = + /https?:\/\/(www\.)?bloomberg.[a-zA-Z0-9()]{1,6}\b([-a-zA-Z0-9()@:%_+.~#?&/=]*)/ + return BLOOMBERG_URL_MATCH.test(url.toString()) + } + + async preHandle(url: string, document?: Document): Promise { + console.log('prehandling bloomberg url', url) + + try { + const response = await axios.get('https://app.scrapingbee.com/api/v1', { + params: { + api_key: process.env.SCRAPINGBEE_API_KEY, + url: url, + return_page_source: true, + block_ads: true, + block_resources: false, + }, + }) + const dom = parseHTML(response.data).document + return { + title: dom.title, + content: dom.querySelector('body')?.innerHTML, + url: url, + } + } catch (error) { + console.error('error prehandling bloomberg url', error) + throw error + } + } +} diff --git a/packages/content-handler/src/websites/derstandard-handler.ts b/packages/content-handler/src/websites/derstandard-handler.ts new file mode 100644 index 000000000..28742a3e5 --- /dev/null +++ b/packages/content-handler/src/websites/derstandard-handler.ts @@ -0,0 +1,34 @@ +import { ContentHandler, PreHandleResult } from '../content-handler' +import axios from 'axios' +import { parseHTML } from 'linkedom' + +export class DerstandardHandler extends ContentHandler { + constructor() { + super() + this.name = 'Derstandard' + } + + shouldPreHandle(url: string, dom?: Document): boolean { + const u = new URL(url) + return u.hostname === 'www.derstandard.at' + } + + async preHandle(url: string, document?: Document): Promise { + const response = await axios.get(url, { + // set cookie to give consent to get the article + headers: { + cookie: `DSGVO_ZUSAGE_V1=true; consentUUID=2bacb9c1-1e80-4be0-9f7b-ee987cf4e7b0_6`, + }, + }) + const content = response.data as string + + const dom = parseHTML(content).document + const titleElement = dom.querySelector('.article-title') + titleElement && titleElement.remove() + + return { + content: dom.body.outerHTML, + title: titleElement?.textContent || undefined, + } + } +} diff --git a/packages/content-handler/src/websites/image-handler.ts b/packages/content-handler/src/websites/image-handler.ts new file mode 100644 index 000000000..068a1cc66 --- /dev/null +++ b/packages/content-handler/src/websites/image-handler.ts @@ -0,0 +1,32 @@ +import { ContentHandler, PreHandleResult } from '../content-handler' + +export class ImageHandler extends ContentHandler { + constructor() { + super() + this.name = 'Image' + } + + shouldPreHandle(url: string, dom?: Document): boolean { + const IMAGE_URL_PATTERN = /(https?:\/\/.*\.(?:jpg|jpeg|png|webp))/i + return IMAGE_URL_PATTERN.test(url.toString()) + } + + async preHandle(url: string, document?: Document): Promise { + const title = url.toString().split('/').pop() || 'Image' + const content = ` + + + ${title} + + + + +
+ ${title} +
+ + ` + + return Promise.resolve({ title, content }) + } +} diff --git a/packages/content-handler/src/websites/medium-handler.ts b/packages/content-handler/src/websites/medium-handler.ts new file mode 100644 index 000000000..211a30c37 --- /dev/null +++ b/packages/content-handler/src/websites/medium-handler.ts @@ -0,0 +1,26 @@ +import { ContentHandler, PreHandleResult } from '../content-handler' + +export class MediumHandler extends ContentHandler { + constructor() { + super() + this.name = 'Medium' + } + + shouldPreHandle(url: string, dom?: Document): boolean { + const u = new URL(url) + return u.hostname.endsWith('medium.com') + } + + async preHandle(url: string, document?: Document): Promise { + console.log('prehandling medium url', url) + + try { + const res = new URL(url) + res.searchParams.delete('source') + return Promise.resolve({ url: res.toString() }) + } catch (error) { + console.error('error prehandling medium url', error) + throw error + } + } +} diff --git a/packages/content-handler/src/websites/pdf-handler.ts b/packages/content-handler/src/websites/pdf-handler.ts new file mode 100644 index 000000000..4c4ef748d --- /dev/null +++ b/packages/content-handler/src/websites/pdf-handler.ts @@ -0,0 +1,18 @@ +import { ContentHandler, PreHandleResult } from '../content-handler' + +export class PdfHandler extends ContentHandler { + constructor() { + super() + this.name = 'PDF' + } + + shouldPreHandle(url: string, dom?: Document): boolean { + const u = new URL(url) + const path = u.pathname.replace(u.search, '') + return path.endsWith('.pdf') + } + + async preHandle(_url: string, document?: Document): Promise { + return Promise.resolve({ contentType: 'application/pdf' }) + } +} diff --git a/packages/content-handler/src/websites/scrapingBee-handler.ts b/packages/content-handler/src/websites/scrapingBee-handler.ts new file mode 100644 index 000000000..4c04d00e8 --- /dev/null +++ b/packages/content-handler/src/websites/scrapingBee-handler.ts @@ -0,0 +1,38 @@ +import { ContentHandler, PreHandleResult } from '../content-handler' +import axios from 'axios' +import { parseHTML } from 'linkedom' + +export class ScrapingBeeHandler extends ContentHandler { + constructor() { + super() + this.name = 'ScrapingBee' + } + + shouldPreHandle(url: string, dom?: Document): boolean { + const u = new URL(url) + const hostnames = ['nytimes.com', 'news.google.com'] + + return hostnames.some((h) => u.hostname.endsWith(h)) + } + + async preHandle(url: string, document?: Document): Promise { + console.log('prehandling url with scrapingbee', url) + + try { + const response = await axios.get('https://app.scrapingbee.com/api/v1', { + params: { + api_key: process.env.SCRAPINGBEE_API_KEY, + url: url, + return_page_source: true, + block_ads: true, + block_resources: false, + }, + }) + const dom = parseHTML(response.data).document + return { title: dom.title, content: response.data as string, url: url } + } catch (error) { + console.error('error prehandling url w/scrapingbee', error) + throw error + } + } +} diff --git a/packages/content-handler/src/websites/t-dot-co-handler.ts b/packages/content-handler/src/websites/t-dot-co-handler.ts new file mode 100644 index 000000000..277a8c087 --- /dev/null +++ b/packages/content-handler/src/websites/t-dot-co-handler.ts @@ -0,0 +1,26 @@ +import { ContentHandler } from '../content-handler' +import axios from 'axios' + +export class TDotCoHandler extends ContentHandler { + constructor() { + super() + this.name = 't.co' + } + + shouldResolve(url: string): boolean { + const T_DOT_CO_URL_MATCH = /^https:\/\/(?:www\.)?t\.co\/.*$/ + return T_DOT_CO_URL_MATCH.test(url) + } + + async resolve(url: string) { + return axios + .get(url, { maxRedirects: 0, validateStatus: null }) + .then((res) => { + return new URL(res.headers.location).href + }) + .catch((err) => { + console.log('err with t.co url', err) + return undefined + }) + } +} diff --git a/packages/content-handler/src/websites/twitter-handler.ts b/packages/content-handler/src/websites/twitter-handler.ts new file mode 100644 index 000000000..ddd37e45c --- /dev/null +++ b/packages/content-handler/src/websites/twitter-handler.ts @@ -0,0 +1,167 @@ +import { ContentHandler, PreHandleResult } from '../content-handler' +import axios from 'axios' +import { DateTime } from 'luxon' +import _ from 'underscore' + +const TWITTER_BEARER_TOKEN = process.env.TWITTER_BEARER_TOKEN +const TWITTER_URL_MATCH = + /twitter\.com\/(?:#!\/)?(\w+)\/status(?:es)?\/(\d+)(?:\/.*)?/ + +const getTweetFields = () => { + const TWEET_FIELDS = + '&tweet.fields=attachments,author_id,conversation_id,created_at,' + + 'entities,geo,in_reply_to_user_id,lang,possibly_sensitive,public_metrics,referenced_tweets,' + + 'source,withheld' + const EXPANSIONS = '&expansions=author_id,attachments.media_keys' + const USER_FIELDS = + '&user.fields=created_at,description,entities,location,pinned_tweet_id,profile_image_url,protected,public_metrics,url,verified,withheld' + const MEDIA_FIELDS = + '&media.fields=duration_ms,height,preview_image_url,url,media_key,public_metrics,width' + + return `${TWEET_FIELDS}${EXPANSIONS}${USER_FIELDS}${MEDIA_FIELDS}` +} + +const getTweetById = async (id: string) => { + const BASE_ENDPOINT = 'https://api.twitter.com/2/tweets/' + const apiUrl = new URL(BASE_ENDPOINT + id + '?' + getTweetFields()) + + if (!TWITTER_BEARER_TOKEN) { + throw new Error('No Twitter bearer token found') + } + + return axios.get(apiUrl.toString(), { + headers: { + Authorization: `Bearer ${TWITTER_BEARER_TOKEN}`, + redirect: 'follow', + }, + }) +} + +const titleForAuthor = (author: { name: string }) => { + return `${author.name} on Twitter` +} + +const tweetIdFromStatusUrl = (url: string): string | undefined => { + const match = url.toString().match(TWITTER_URL_MATCH) + return match?.[2] +} + +const formatTimestamp = (timestamp: string) => { + return DateTime.fromJSDate(new Date(timestamp)).toLocaleString( + DateTime.DATETIME_FULL + ) +} + +export class TwitterHandler extends ContentHandler { + constructor() { + super() + this.name = 'Twitter' + } + + shouldPreHandle(url: string, dom?: Document): boolean { + return !!TWITTER_BEARER_TOKEN && TWITTER_URL_MATCH.test(url.toString()) + } + + async preHandle(url: string, document?: Document): Promise { + console.log('prehandling twitter url', url) + + const tweetId = tweetIdFromStatusUrl(url) + if (!tweetId) { + throw new Error('could not find tweet id in url') + } + const tweetData = (await getTweetById(tweetId)).data as { + data: { + author_id: string + text: string + entities: { + urls: [ + { + url: string + expanded_url: string + display_url: string + } + ] + } + created_at: string + } + includes: { + users: [ + { + id: string + name: string + profile_image_url: string + username: string + } + ] + media: [ + { + preview_image_url: string + type: string + url: string + } + ] + } + } + const authorId = tweetData.data.author_id + const author = tweetData.includes.users.filter((u) => (u.id = authorId))[0] + // escape html entities in title + const title = _.escape(titleForAuthor(author)) + const authorImage = author.profile_image_url.replace('_normal', '_400x400') + + let text = tweetData.data.text + if (tweetData.data.entities && tweetData.data.entities.urls) { + for (const urlObj of tweetData.data.entities.urls) { + text = text.replace( + urlObj.url, + `${urlObj.display_url}` + ) + } + } + + const front = ` +
+

${text}

+ ` + + let includesHtml = '' + if (tweetData.includes.media) { + includesHtml = tweetData.includes.media + .map((m) => { + const linkUrl = m.type == 'photo' ? m.url : url + const previewUrl = m.type == 'photo' ? m.url : m.preview_image_url + const mediaOpen = ` + + + + ` + return mediaOpen + }) + .join('\n') + } + + const back = ` + — ${ + author.username + } ${author.name} ${formatTimestamp( + tweetData.data.created_at + )} +
+ ` + const content = ` + + + + + + + + ${front} + ${includesHtml} + ${back} + ` + + return { content, url, title } + } +} diff --git a/packages/content-handler/src/websites/wikipedia-handler.ts b/packages/content-handler/src/websites/wikipedia-handler.ts new file mode 100644 index 000000000..8c3a176fd --- /dev/null +++ b/packages/content-handler/src/websites/wikipedia-handler.ts @@ -0,0 +1,20 @@ +import { ContentHandler, PreHandleResult } from '../content-handler' + +export class WikipediaHandler extends ContentHandler { + constructor() { + super() + this.name = 'wikipedia' + } + + shouldPreHandle(url: string, dom?: Document): boolean { + return new URL(url).hostname.endsWith('wikipedia.org') + } + + async preHandle(url: string, dom: Document): Promise { + // This removes the [edit] anchors from wikipedia pages + dom.querySelectorAll('.mw-editsection').forEach((e) => e.remove()) + // this removes the sidebar + dom.querySelector('.infobox')?.remove() + return Promise.resolve({ dom }) + } +} diff --git a/packages/content-fetch/youtube-handler.js b/packages/content-handler/src/websites/youtube-handler.ts similarity index 55% rename from packages/content-fetch/youtube-handler.js rename to packages/content-handler/src/websites/youtube-handler.ts index e1866428a..4cdb7ee98 100644 --- a/packages/content-fetch/youtube-handler.js +++ b/packages/content-handler/src/websites/youtube-handler.ts @@ -1,18 +1,13 @@ -/* eslint-disable no-undef */ -/* eslint-disable no-empty */ -/* eslint-disable @typescript-eslint/explicit-function-return-type */ -/* eslint-disable @typescript-eslint/no-var-requires */ -/* eslint-disable @typescript-eslint/no-require-imports */ -require('dotenv').config(); -const axios = require('axios'); -const _ = require('underscore'); +import { ContentHandler, PreHandleResult } from '../content-handler' +import axios from 'axios' +import _ from 'underscore' const YOUTUBE_URL_MATCH = /^((?:https?:)?\/\/)?((?:www|m)\.)?((?:youtube\.com|youtu.be))(\/(?:[\w-]+\?v=|embed\/|v\/)?)([\w-]+)(\S+)?$/ -function getYoutubeVideoId(url) { - const u = new URL(url); - const videoId = u.searchParams.get('v'); +export const getYoutubeVideoId = (url: string) => { + const u = new URL(url) + const videoId = u.searchParams.get('v') if (!videoId) { const match = url.toString().match(YOUTUBE_URL_MATCH) if (match === null || match.length < 6 || !match[5]) { @@ -22,28 +17,41 @@ function getYoutubeVideoId(url) { } return videoId } -exports.getYoutubeVideoId = getYoutubeVideoId -exports.youtubeHandler = { - shouldPrehandle: (url, env) => { +export class YoutubeHandler extends ContentHandler { + constructor() { + super() + this.name = 'Youtube' + } + + shouldPreHandle(url: string, dom?: Document): boolean { return YOUTUBE_URL_MATCH.test(url.toString()) - }, + } - prehandle: async (url, env) => { + async preHandle(url: string, document?: Document): Promise { const videoId = getYoutubeVideoId(url) if (!videoId) { return {} } - const oembedUrl = `https://www.youtube.com/oembed?format=json&url=` + encodeURIComponent(`https://www.youtube.com/watch?v=${videoId}`) - const oembed = (await axios.get(oembedUrl.toString())).data; + const oembedUrl = + `https://www.youtube.com/oembed?format=json&url=` + + encodeURIComponent(`https://www.youtube.com/watch?v=${videoId}`) + const oembed = (await axios.get(oembedUrl.toString())).data as { + title: string + width: number + height: number + thumbnail_url: string + author_name: string + author_url: string + } // escape html entities in title - const title = _.escape(oembed.title); - const ratio = oembed.width / oembed.height; - const thumbnail = oembed.thumbnail_url; - const height = 350; - const width = height * ratio; - const authorName = _.escape(oembed.author_name); + const title = _.escape(oembed.title) + const ratio = oembed.width / oembed.height + const thumbnail = oembed.thumbnail_url + const height = 350 + const width = height * ratio + const authorName = _.escape(oembed.author_name) const content = ` @@ -63,6 +71,6 @@ exports.youtubeHandler = { console.log('got video id', videoId) - return { content, title: 'Youtube Content' }; + return { content, title: 'Youtube Content' } } } diff --git a/packages/content-handler/test/apple-news-handler.test.ts b/packages/content-handler/test/apple-news-handler.test.ts new file mode 100644 index 000000000..1584f9e28 --- /dev/null +++ b/packages/content-handler/test/apple-news-handler.test.ts @@ -0,0 +1,10 @@ +import { AppleNewsHandler } from '../src/websites/apple-news-handler' + +describe('open a simple web page', () => { + it('should return a response', async () => { + const response = await new AppleNewsHandler().preHandle( + 'https://apple.news/AxjzaZaPvSn23b67LhXI5EQ' + ) + console.log('response', response) + }) +}) diff --git a/packages/content-handler/test/babel-register.js b/packages/content-handler/test/babel-register.js new file mode 100644 index 000000000..a6f65f60a --- /dev/null +++ b/packages/content-handler/test/babel-register.js @@ -0,0 +1,3 @@ +const register = require('@babel/register').default + +register({ extensions: ['.ts', '.tsx', '.js', '.jsx'] }) diff --git a/packages/api/test/utils/data/beehiiv-newsletter.html b/packages/content-handler/test/data/beehiiv-newsletter.html similarity index 100% rename from packages/api/test/utils/data/beehiiv-newsletter.html rename to packages/content-handler/test/data/beehiiv-newsletter.html diff --git a/packages/api/test/utils/data/substack-forwarded-newsletter.html b/packages/content-handler/test/data/substack-forwarded-newsletter.html similarity index 100% rename from packages/api/test/utils/data/substack-forwarded-newsletter.html rename to packages/content-handler/test/data/substack-forwarded-newsletter.html diff --git a/packages/api/test/utils/data/substack-forwarded-welcome-email.html b/packages/content-handler/test/data/substack-forwarded-welcome-email.html similarity index 100% rename from packages/api/test/utils/data/substack-forwarded-welcome-email.html rename to packages/content-handler/test/data/substack-forwarded-welcome-email.html diff --git a/packages/api/test/utils/data/substack-private-forwarded-newsletter.html b/packages/content-handler/test/data/substack-private-forwarded-newsletter.html similarity index 100% rename from packages/api/test/utils/data/substack-private-forwarded-newsletter.html rename to packages/content-handler/test/data/substack-private-forwarded-newsletter.html diff --git a/packages/content-handler/test/newsletter.test.ts b/packages/content-handler/test/newsletter.test.ts new file mode 100644 index 000000000..dd3b7941c --- /dev/null +++ b/packages/content-handler/test/newsletter.test.ts @@ -0,0 +1,191 @@ +import 'mocha' +import * as chai from 'chai' +import { expect } from 'chai' +import chaiAsPromised from 'chai-as-promised' +import chaiString from 'chai-string' +import { SubstackHandler } from '../src/newsletters/substack-handler' +import { AxiosHandler } from '../src/newsletters/axios-handler' +import { BloombergNewsletterHandler } from '../src/newsletters/bloomberg-newsletter-handler' +import { GolangHandler } from '../src/newsletters/golang-handler' +import { MorningBrewHandler } from '../src/newsletters/morning-brew-handler' +import nock from 'nock' +import { generateUniqueUrl } from '../src/content-handler' +import fs from 'fs' +import { BeehiivHandler } from '../src/newsletters/beehiiv-handler' + +chai.use(chaiAsPromised) +chai.use(chaiString) + +const load = (path: string): string => { + return fs.readFileSync(path, 'utf8') +} + +describe('Newsletter email test', () => { + describe('#getNewsletterUrl()', () => { + it('returns url when email is from SubStack', async () => { + const rawUrl = '' + + await expect( + new SubstackHandler().parseNewsletterUrl(rawUrl, '') + ).to.eventually.equal('https://hongbo130.substack.com/p/tldr') + }) + + it('returns url when email is from Axios', async () => { + const url = 'https://axios.com/blog/the-best-way-to-build-a-web-app' + const html = `View in browser at ${url}` + + await expect( + new AxiosHandler().parseNewsletterUrl('', html) + ).to.eventually.equal(url) + }) + + it('returns url when email is from Bloomberg', async () => { + const url = 'https://www.bloomberg.com/news/google-is-now-a-partner' + const html = ` + + View in browser + + ` + + await expect( + new BloombergNewsletterHandler().parseNewsletterUrl('', html) + ).to.eventually.equal(url) + }) + + it('returns url when email is from Golang Weekly', async () => { + const url = 'https://www.golangweekly.com/first' + const html = ` + Read on the Web + ` + + await expect( + new GolangHandler().parseNewsletterUrl('', html) + ).to.eventually.equal(url) + }) + + it('returns url when email is from Morning Brew', async () => { + const url = 'https://www.morningbrew.com/daily/issues/first' + const html = ` + View Online + ` + + await expect( + new MorningBrewHandler().parseNewsletterUrl('', html) + ).to.eventually.equal(url) + }) + }) + + describe('get author from email address', () => { + it('returns author when email is from Substack', () => { + const from = 'Jackson Harper from Omnivore App ' + expect(new AxiosHandler().parseAuthor(from)).to.equal( + 'Jackson Harper from Omnivore App' + ) + }) + + it('returns author when email is from Axios', () => { + const from = 'Mike Allen ' + expect(new AxiosHandler().parseAuthor(from)).to.equal('Mike Allen') + }) + }) + + describe('isProbablyNewsletter', () => { + it('returns true for substack newsletter', async () => { + const html = load('./test/data/substack-forwarded-newsletter.html') + await expect( + new SubstackHandler().isNewsletter({ + html, + postHeader: '', + from: '', + unSubHeader: '', + }) + ).to.eventually.be.true + }) + it('returns true for private forwarded substack newsletter', async () => { + const html = load( + './test/data/substack-private-forwarded-newsletter.html' + ) + await expect( + new SubstackHandler().isNewsletter({ + html, + postHeader: '', + from: '', + unSubHeader: '', + }) + ).to.eventually.be.true + }) + it('returns false for substack welcome email', async () => { + const html = load('./test/data/substack-forwarded-welcome-email.html') + await expect( + new SubstackHandler().isNewsletter({ + html, + postHeader: '', + from: '', + unSubHeader: '', + }) + ).to.eventually.be.false + }) + it('returns true for beehiiv.com newsletter', async () => { + const html = load('./test/data/beehiiv-newsletter.html') + await expect( + new BeehiivHandler().isNewsletter({ + html, + postHeader: '', + from: '', + unSubHeader: '', + }) + ).to.eventually.be.true + }) + }) + + describe('findNewsletterUrl', async () => { + it('gets the URL from the header if it is a substack newsletter', async () => { + nock('https://email.mg2.substack.com') + .head( + '/c/eJxNkk2TojAQhn-N3KTyQfg4cGDGchdnYcsZx9K5UCE0EMVAkTiKv36iHnarupNUd7rfVJ4W3EDTj1M89No496Uw0wCxgovuwBgYnbOGsZBVjDHzKPWYU8VehUMWOlIX9Qhw4rKLzXgGZziXnRTcyF7dK0iIGMVOG_OS1aTmKPRDilgVhTQUPCQIcE0x-MFTmJ8rCUpA3KtuenR2urg1ZtAzmszI0tq_Z7m66y-ilQo0uAqMTQ7WRX8auJKg56blZg7WB-iHDuYEBzO6NP0R1IwuYFphQbbTjnTH9NBfs80nym4Zyj8uUvyKbtUyGr5eUz9fNDQ7JCxfJDo9dW1lY9lmj_JNivPbGmf2Pt_lN9tDit9b-WeTetni85Z9pDpVOd7L1E_Vy7egayNO23ZP34eSeLJeux1b0rer_xaZ7ykS78nuSjMY-nL98rparNZNcv07JCjN06_EkTFBxBqOUMACErnELUNMSxTUjLDQZwzcqa4bRjCfeejUEFefS224OLr2S5wxPtij7lVrs80d2CNseRV2P52VNFMBipcdVE-U5jkRD7hFAwpGOylVwU2Mfc9qBh7DoR89yVnWXhgQFHnIsbpVb6tU_B-hH_2yzWY' + ) + .reply(302, undefined, { + Location: + 'https://newsletter.slowchinese.net/p/companies-that-eat-people-217', + }) + .get('/p/companies-that-eat-people-217') + .reply(200, '') + const html = load('./test/data/substack-forwarded-newsletter.html') + const url = await new SubstackHandler().findNewsletterUrl(html) + // Not sure if the redirects from substack expire, this test could eventually fail + expect(url).to.startWith( + 'https://newsletter.slowchinese.net/p/companies-that-eat-people-217' + ) + }).timeout(10000) + it('gets the URL from the header if it is a beehiiv newsletter', async () => { + nock('https://u23463625.ct.sendgrid.net') + .head( + '/ss/c/AX1lEgEQaxtvFxLaVo0GBo_geajNrlI1TGeIcmMViR3pL3fEDZnbbkoeKcaY62QZk0KPFudUiUXc_uMLerV4nA/3k5/3TFZmreTR0qKSCgowABnVg/h30/zzLik7UXd1H_n4oyd5W8Xu639AYQQB2UXz-CsssSnno' + ) + .reply(302, undefined, { + Location: 'https://www.milkroad.com/p/talked-guy-spent-30m-beeple', + }) + .get('/p/talked-guy-spent-30m-beeple') + .reply(200, '') + const html = load('./test/data/beehiiv-newsletter.html') + const url = await new BeehiivHandler().findNewsletterUrl(html) + expect(url).to.startWith( + 'https://www.milkroad.com/p/talked-guy-spent-30m-beeple' + ) + }) + it('returns undefined if it is not a newsletter', async () => { + const html = load('./test/data/substack-forwarded-welcome-email.html') + const url = await new SubstackHandler().findNewsletterUrl(html) + expect(url).to.be.undefined + }) + }) + + describe('generateUniqueUrl', () => { + it('generates a unique URL', () => { + const url1 = generateUniqueUrl() + const url2 = generateUniqueUrl() + + expect(url1).to.not.eql(url2) + }) + }) +}) diff --git a/packages/content-handler/test/youtube-handler.test.ts b/packages/content-handler/test/youtube-handler.test.ts new file mode 100644 index 000000000..beb4d3a66 --- /dev/null +++ b/packages/content-handler/test/youtube-handler.test.ts @@ -0,0 +1,25 @@ +import { expect } from 'chai' +import 'mocha' +import { getYoutubeVideoId } from '../src/websites/youtube-handler' + +describe('getYoutubeVideoId', () => { + it('should parse video id out of a URL', async () => { + expect('BnSUk0je6oo').to.eq( + getYoutubeVideoId('https://www.youtube.com/watch?v=BnSUk0je6oo&t=269s') + ) + expect('vFD2gu007dc').to.eq( + getYoutubeVideoId( + 'https://www.youtube.com/watch?v=vFD2gu007dc&list=RDvFD2gu007dc&start_radio=1' + ) + ) + expect('vFD2gu007dc').to.eq( + getYoutubeVideoId('https://youtu.be/vFD2gu007dc') + ) + expect('BMFVCnbRaV4').to.eq( + getYoutubeVideoId('https://youtube.com/watch?v=BMFVCnbRaV4&feature=share') + ) + expect('cg9b4RC87LI').to.eq( + getYoutubeVideoId('https://youtu.be/cg9b4RC87LI?t=116') + ) + }) +}) diff --git a/packages/content-handler/tsconfig.json b/packages/content-handler/tsconfig.json new file mode 100644 index 000000000..aeb8d2c3a --- /dev/null +++ b/packages/content-handler/tsconfig.json @@ -0,0 +1,10 @@ +{ + "extends": "@tsconfig/node14/tsconfig.json", + "compilerOptions": { + "rootDir": ".", + "declaration": true, + "outDir": "build", + "lib": ["dom"] + }, + "include": ["src"] +} diff --git a/packages/db/migrations/0096.do.subscriptions_icon.sql b/packages/db/migrations/0096.do.subscriptions_icon.sql new file mode 100755 index 000000000..7f668b023 --- /dev/null +++ b/packages/db/migrations/0096.do.subscriptions_icon.sql @@ -0,0 +1,9 @@ +-- Type: DO +-- Name: subscriptions_icon +-- Description: Add icon field to subscriptions table + +BEGIN; + +ALTER TABLE omnivore.subscriptions ADD COLUMN icon text; + +COMMIT; diff --git a/packages/db/migrations/0096.undo.subscriptions_icon.sql b/packages/db/migrations/0096.undo.subscriptions_icon.sql new file mode 100755 index 000000000..eb095c17f --- /dev/null +++ b/packages/db/migrations/0096.undo.subscriptions_icon.sql @@ -0,0 +1,9 @@ +-- Type: UNDO +-- Name: subscriptions_icon +-- Description: Add icon field to subscriptions table + +BEGIN; + +ALTER TABLE omnivore.subscriptions DROP COLUMN IF EXISTS icon; + +COMMIT; diff --git a/packages/inbound-email-handler/package.json b/packages/inbound-email-handler/package.json index f3048a3b2..ed22a170c 100644 --- a/packages/inbound-email-handler/package.json +++ b/packages/inbound-email-handler/package.json @@ -31,6 +31,7 @@ "@google-cloud/pubsub": "^2.18.4", "@sendgrid/client": "^7.6.0", "@sentry/serverless": "^6.16.1", + "@omnivore/content-handler": "1.0.0", "addressparser": "^1.0.1", "axios": "^0.27.2", "jsonwebtoken": "^8.5.1", diff --git a/packages/inbound-email-handler/src/axios-handler.ts b/packages/inbound-email-handler/src/axios-handler.ts deleted file mode 100644 index 706d11047..000000000 --- a/packages/inbound-email-handler/src/axios-handler.ts +++ /dev/null @@ -1,10 +0,0 @@ -import { NewsletterHandler } from './newsletter' - -export class AxiosHandler extends NewsletterHandler { - constructor() { - super() - this.senderRegex = /<.+@axios.com>/ - this.urlRegex = /View in browser at (.*)<\/a>/ - this.defaultUrl = 'https://axios.com' - } -} diff --git a/packages/inbound-email-handler/src/bloomberg-handler.ts b/packages/inbound-email-handler/src/bloomberg-handler.ts deleted file mode 100644 index 3239ab176..000000000 --- a/packages/inbound-email-handler/src/bloomberg-handler.ts +++ /dev/null @@ -1,10 +0,0 @@ -import { NewsletterHandler } from './newsletter' - -export class BloombergHandler extends NewsletterHandler { - constructor() { - super() - this.senderRegex = /<.+@mail.bloomberg.*.com>/ - this.urlRegex = // - this.urlRegex = /Read on the Web<\/a>/ - this.defaultUrl = 'https://golangweekly.com' - } -} diff --git a/packages/inbound-email-handler/src/index.ts b/packages/inbound-email-handler/src/index.ts index ee11511dc..67d38694d 100644 --- a/packages/inbound-email-handler/src/index.ts +++ b/packages/inbound-email-handler/src/index.ts @@ -9,35 +9,27 @@ import * as multipart from 'parse-multipart-data' import { handleConfirmation, isConfirmationEmail, - NewsletterHandler, parseUnsubscribe, } from './newsletter' import { PubSub } from '@google-cloud/pubsub' import { handlePdfAttachment } from './pdf' -import { SubstackHandler } from './substack-handler' -import { AxiosHandler } from './axios-handler' -import { BloombergHandler } from './bloomberg-handler' -import { GolangHandler } from './golang-handler' -import { MorningBrewHandler } from './morning-brew-handler' +import { handleNewsletter } from '@omnivore/content-handler' +const NEWSLETTER_EMAIL_RECEIVED_TOPIC = 'newsletterEmailReceived' const NON_NEWSLETTER_EMAIL_TOPIC = 'nonNewsletterEmailReceived' const pubsub = new PubSub() -const NEWSLETTER_HANDLERS = [ - new SubstackHandler(), - new AxiosHandler(), - new BloombergHandler(), - new GolangHandler(), - new MorningBrewHandler(), -] -export const getNewsletterHandler = ( - postHeader: string, - from: string, - unSubHeader: string -): NewsletterHandler | undefined => { - return NEWSLETTER_HANDLERS.find((h) => { - return h.isNewsletter(postHeader, from, unSubHeader) - }) +export const publishMessage = async ( + topic: string, + message: any +): Promise => { + return pubsub + .topic(topic) + .publishMessage({ json: message }) + .catch((err) => { + console.log('error publishing message:', err) + return undefined + }) } export const inboundEmailHandler = Sentry.GCPFunction.wrapHttpFunction( @@ -86,23 +78,20 @@ export const inboundEmailHandler = Sentry.GCPFunction.wrapHttpFunction( try { // check if it is a confirmation email or forwarding newsletter - const newsletterHandler = getNewsletterHandler( - postHeader, + const newsletterMessage = await handleNewsletter({ from, - unSubHeader - ) - - if (newsletterHandler) { - console.log('handleNewsletter', from, to) - await newsletterHandler.handleNewsletter( - to, - html, - postHeader, - subject, - from, - unSubHeader + html, + postHeader, + unSubHeader, + email: to, + title: subject, + }) + if (newsletterMessage) { + await publishMessage( + NEWSLETTER_EMAIL_RECEIVED_TOPIC, + newsletterMessage ) - return res.send('ok') + return res.status(200).send('newsletter received') } console.log('non-newsletter email from', from, 'to', to) diff --git a/packages/inbound-email-handler/src/morning-brew-handler.ts b/packages/inbound-email-handler/src/morning-brew-handler.ts deleted file mode 100644 index 6f5478c63..000000000 --- a/packages/inbound-email-handler/src/morning-brew-handler.ts +++ /dev/null @@ -1,10 +0,0 @@ -import { NewsletterHandler } from './newsletter' - -export class MorningBrewHandler extends NewsletterHandler { - constructor() { - super() - this.senderRegex = /Morning Brew / - this.urlRegex = /View Online<\/a>/ - this.defaultUrl = 'https://www.morningbrew.com' - } -} diff --git a/packages/inbound-email-handler/src/newsletter.ts b/packages/inbound-email-handler/src/newsletter.ts index cf031e905..1f3294541 100644 --- a/packages/inbound-email-handler/src/newsletter.ts +++ b/packages/inbound-email-handler/src/newsletter.ts @@ -1,15 +1,12 @@ -import { PubSub } from '@google-cloud/pubsub' -import { v4 as uuidv4 } from 'uuid' import addressparser from 'addressparser' import rfc2047 from 'rfc2047' +import { publishMessage } from './index' interface Unsubscribe { mailTo?: string httpUrl?: string } -const pubsub = new PubSub() -const NEWSLETTER_EMAIL_RECEIVED_TOPIC = 'newsletterEmailReceived' const EMAIL_CONFIRMATION_CODE_RECEIVED_TOPIC = 'emailConfirmationCodeReceived' const CONFIRMATION_EMAIL_SENDER_ADDRESS = 'forwarding-noreply@google.com' // check unicode parentheses too @@ -35,73 +32,6 @@ const parseAddress = (address: string): string => { return '' } -export class NewsletterHandler { - protected senderRegex = /NEWSLETTER_SENDER_REGEX/ - protected urlRegex = /NEWSLETTER_URL_REGEX/ - protected defaultUrl = 'NEWSLETTER_DEFAULT_URL' - - isNewsletter(postHeader: string, from: string, unSubHeader: string): boolean { - // Axios newsletter is from - const re = new RegExp(this.senderRegex) - return re.test(from) && (!!postHeader || !!unSubHeader) - } - - parseNewsletterUrl(_postHeader: string, html: string): string | undefined { - // get newsletter url from html - const matches = html.match(this.urlRegex) - if (matches) { - return matches[1] - } - return undefined - } - - parseAuthor(from: string): string { - // get author name from email - // e.g. 'Jackson Harper from Omnivore App ' - // or 'Mike Allen ' - const parsed = addressparser(from) - if (parsed.length > 0) { - return parsed[0].name - } - return from - } - - async handleNewsletter( - email: string, - html: string, - postHeader: string, - title: string, - from: string, - unSubHeader: string - ): Promise { - console.log('handleNewsletter', email, postHeader, title, from) - - if (!email || !html || !title || !from) { - console.log('invalid newsletter email') - throw new Error('invalid newsletter email') - } - - // fallback to default url if newsletter url does not exist - // assign a random uuid to the default url to avoid duplicate url - const url = - this.parseNewsletterUrl(postHeader, html) || - `${this.defaultUrl}?source=newsletters&id=${uuidv4()}` - const author = this.parseAuthor(from) - const unsubscribe = parseUnsubscribe(unSubHeader) - const message = { - email, - content: html, - url, - title, - author, - unsubMailTo: unsubscribe.mailTo || '', - unsubHttpUrl: unsubscribe.httpUrl || '', - } - - return publishMessage(NEWSLETTER_EMAIL_RECEIVED_TOPIC, message) - } -} - export const handleConfirmation = async (email: string, subject: string) => { console.log('confirmation email', email, subject) @@ -136,16 +66,3 @@ export const isConfirmationEmail = (from: string, subject: string): boolean => { CONFIRMATION_CODE_PATTERN.test(subject) ) } - -const publishMessage = async ( - topic: string, - message: Record -): Promise => { - return pubsub - .topic(topic) - .publishMessage({ json: message }) - .catch((err) => { - console.log('error publishing message:', err) - return undefined - }) -} diff --git a/packages/inbound-email-handler/src/substack-handler.ts b/packages/inbound-email-handler/src/substack-handler.ts deleted file mode 100644 index 10160b76e..000000000 --- a/packages/inbound-email-handler/src/substack-handler.ts +++ /dev/null @@ -1,25 +0,0 @@ -import { NewsletterHandler } from './newsletter' -import addressparser from 'addressparser' - -export class SubstackHandler extends NewsletterHandler { - constructor() { - super() - this.defaultUrl = 'https://www.substack.com' - } - - parseNewsletterUrl(postHeader: string, _html: string): string | undefined { - // raw SubStack newsletter url is like - // we need to get the real url from the raw url - return addressparser(postHeader).length > 0 - ? addressparser(postHeader)[0].name - : undefined - } - - isNewsletter( - postHeader: string, - _from: string, - _unSubHeader: string - ): boolean { - return !!postHeader - } -} diff --git a/packages/inbound-email-handler/test/newsletter.test.ts b/packages/inbound-email-handler/test/newsletter.test.ts index 8c20d9747..253c294ab 100644 --- a/packages/inbound-email-handler/test/newsletter.test.ts +++ b/packages/inbound-email-handler/test/newsletter.test.ts @@ -2,15 +2,8 @@ import { expect } from 'chai' import { getConfirmationCode, isConfirmationEmail, - NewsletterHandler, parseUnsubscribe, } from '../src/newsletter' -import { SubstackHandler } from '../src/substack-handler' -import { AxiosHandler } from '../src/axios-handler' -import { BloombergHandler } from '../src/bloomberg-handler' -import { GolangHandler } from '../src/golang-handler' -import { getNewsletterHandler } from '../src' -import { MorningBrewHandler } from '../src/morning-brew-handler' describe('Confirmation email test', () => { describe('#isConfirmationEmail()', () => { @@ -54,126 +47,6 @@ describe('Confirmation email test', () => { }) describe('Newsletter email test', () => { - describe('#getNewsletterHandler()', () => { - it('returns SubstackHandler when email is from SubStack', () => { - const rawUrl = '' - - expect(getNewsletterHandler(rawUrl, '', '')).to.be.instanceof( - SubstackHandler - ) - }) - - it('returns AxiosHandler when email is from Axios', () => { - const from = 'Mike Allen ' - const unSubRawUrl = - '' - - expect(getNewsletterHandler('', from, unSubRawUrl)).to.be.instanceof( - AxiosHandler - ) - }) - - context('when email is from Bloomberg', () => { - it('should return BloombergHandler when email is from Bloomberg Business', () => { - const from = 'From: Bloomberg ' - const unSubRawUrl = '' - - expect(getNewsletterHandler('', from, unSubRawUrl)).to.be.instanceof( - BloombergHandler - ) - }) - - it('should return BloombergHandler when email is from Bloomberg View', () => { - const from = 'From: Bloomberg ' - const unSubRawUrl = '' - - expect(getNewsletterHandler('', from, unSubRawUrl)).to.be.instanceof( - BloombergHandler - ) - }) - }) - - it('should return GolangHandler when email is from Golang Weekly', () => { - const from = 'Golang Weekly ' - const unSubRawUrl = '' - - expect(getNewsletterHandler('', from, unSubRawUrl)).to.be.instanceof( - GolangHandler - ) - }) - - it('should return MorningBrewHandler when email is from Morning Brew', () => { - const from = 'Morning Brew ' - const unSubRawUrl = '' - - expect(getNewsletterHandler('', from, unSubRawUrl)).to.be.instanceof( - MorningBrewHandler - ) - }) - }) - - describe('#getNewsletterUrl()', () => { - it('returns url when email is from SubStack', () => { - const rawUrl = '' - - expect(new SubstackHandler().parseNewsletterUrl(rawUrl, '')).to.equal( - 'https://hongbo130.substack.com/p/tldr' - ) - }) - - it('returns url when email is from Axios', () => { - const url = 'https://axios.com/blog/the-best-way-to-build-a-web-app' - const html = `View in browser at ${url}` - - expect(new AxiosHandler().parseNewsletterUrl('', html)).to.equal(url) - }) - - it('returns url when email is from Bloomberg', () => { - const url = 'https://www.bloomberg.com/news/google-is-now-a-partner' - const html = ` - - View in browser - - ` - - expect(new BloombergHandler().parseNewsletterUrl('', html)).to.equal(url) - }) - - it('returns url when email is from Golang Weekly', () => { - const url = 'https://www.golangweekly.com/first' - const html = ` - Read on the Web - ` - - expect(new GolangHandler().parseNewsletterUrl('', html)).to.equal(url) - }) - - it('returns url when email is from Morning Brew', () => { - const url = 'https://www.morningbrew.com/daily/issues/first' - const html = ` - View Online - ` - - expect(new MorningBrewHandler().parseNewsletterUrl('', html)).to.equal( - url - ) - }) - }) - - describe('get author from email address', () => { - it('returns author when email is from Substack', () => { - const from = 'Jackson Harper from Omnivore App ' - expect(new NewsletterHandler().parseAuthor(from)).to.equal( - 'Jackson Harper from Omnivore App' - ) - }) - - it('returns author when email is from Axios', () => { - const from = 'Mike Allen ' - expect(new NewsletterHandler().parseAuthor(from)).to.equal('Mike Allen') - }) - }) - describe('get unsubscribe from header', () => { const mailTo = 'unsub@omnivore.com' const httpUrl = 'https://omnivore.com/unsubscribe' diff --git a/packages/puppeteer-parse/apple-news-handler.js b/packages/puppeteer-parse/apple-news-handler.js deleted file mode 100644 index 0759dec23..000000000 --- a/packages/puppeteer-parse/apple-news-handler.js +++ /dev/null @@ -1,36 +0,0 @@ -/* eslint-disable no-undef */ -/* eslint-disable no-empty */ -/* eslint-disable @typescript-eslint/explicit-function-return-type */ -/* eslint-disable @typescript-eslint/no-var-requires */ -/* eslint-disable @typescript-eslint/no-require-imports */ -require('dotenv').config(); -const Url = require('url'); -const axios = require('axios'); -const { promisify } = require('util'); -const { DateTime } = require('luxon'); -const os = require('os'); -const { Cipher } = require('crypto'); -const { parseHTML } = require('linkedom'); - -exports.appleNewsHandler = { - - shouldPrehandle: (url, env) => { - const u = new URL(url); - if (u.hostname === 'apple.news') { - return true; - } - return false - }, - - prehandle: async (url, env) => { - const MOBILE_USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.61 Safari/537.36' - const response = await axios.get(url, { headers: { 'User-Agent': MOBILE_USER_AGENT } } ); - const data = response.data; - - const dom = parseHTML(data).document; - - // make sure its a valid URL by wrapping in new URL - const u = new URL(dom.querySelector('span.click-here').parentNode.href); - return { url: u.href }; - } -} diff --git a/packages/puppeteer-parse/bloomberg-handler.js b/packages/puppeteer-parse/bloomberg-handler.js deleted file mode 100644 index d79a568bb..000000000 --- a/packages/puppeteer-parse/bloomberg-handler.js +++ /dev/null @@ -1,39 +0,0 @@ -/* eslint-disable no-undef */ -/* eslint-disable no-empty */ -/* eslint-disable @typescript-eslint/explicit-function-return-type */ -/* eslint-disable @typescript-eslint/no-var-requires */ -/* eslint-disable @typescript-eslint/no-require-imports */ -require('dotenv').config(); -const axios = require('axios'); -const os = require('os'); -const { parseHTML } = require('linkedom'); - -exports.bloombergHandler = { - - shouldPrehandle: (url, env) => { - const BLOOMBERG_URL_MATCH = - /https?:\/\/(www\.)?bloomberg.[a-zA-Z0-9()]{1,6}\b([-a-zA-Z0-9()@:%_\+.~#?&//=]*)/ - return BLOOMBERG_URL_MATCH.test(url.toString()) - }, - - prehandle: async (url, env) => { - console.log('prehandling bloomberg url', url) - - try { - const response = await axios.get('https://app.scrapingbee.com/api/v1', { - params: { - 'api_key': process.env.SCRAPINGBEE_API_KEY, - 'url': url, - 'return_page_source': true, - 'block_ads': true, - 'block_resources': false, - } - }) - const dom = parseHTML(response.data).document; - return { title: dom.title, content: dom.querySelector('body').innerHTML, url: url } - } catch (error) { - console.error('error prehandling bloomberg url', error) - throw error - } - } -} diff --git a/packages/puppeteer-parse/derstandard-handler.js b/packages/puppeteer-parse/derstandard-handler.js deleted file mode 100644 index a44db6f2a..000000000 --- a/packages/puppeteer-parse/derstandard-handler.js +++ /dev/null @@ -1,35 +0,0 @@ -/* eslint-disable no-undef */ -/* eslint-disable no-empty */ -/* eslint-disable @typescript-eslint/explicit-function-return-type */ -/* eslint-disable @typescript-eslint/no-var-requires */ -/* eslint-disable @typescript-eslint/no-require-imports */ -require('dotenv').config(); -const axios = require('axios'); -const { parseHTML } = require('linkedom'); - -exports.derstandardHandler = { - shouldPrehandle: (url, env) => { - const u = new URL(url); - return u.hostname === 'www.derstandard.at'; - }, - - prehandle: async (url, env) => { - const response = await axios.get(url, { - // set cookie to give consent to get the article - headers: { - 'cookie': `DSGVO_ZUSAGE_V1=true; consentUUID=2bacb9c1-1e80-4be0-9f7b-ee987cf4e7b0_6` - }, - }); - const content = response.data; - - var title = undefined; - const dom = parseHTML(content).document; - const titleElement = dom.querySelector('.article-title') - if (!titleElement) { - title = titleElement.textContent - titleElement.remove() - } - - return { content: dom.body.outerHTML, title: title }; - } -} diff --git a/packages/puppeteer-parse/image-handler.js b/packages/puppeteer-parse/image-handler.js deleted file mode 100644 index 59f132afc..000000000 --- a/packages/puppeteer-parse/image-handler.js +++ /dev/null @@ -1,34 +0,0 @@ -/* eslint-disable no-undef */ -/* eslint-disable no-empty */ -/* eslint-disable @typescript-eslint/explicit-function-return-type */ -/* eslint-disable @typescript-eslint/no-var-requires */ -/* eslint-disable @typescript-eslint/no-require-imports */ -require('dotenv').config(); - - -exports.imageHandler = { - shouldPrehandle: (url, env) => { - const IMAGE_URL_PATTERN = - /(https?:\/\/.*\.(?:jpg|jpeg|png|webp))/i - return IMAGE_URL_PATTERN.test(url.toString()) - }, - - prehandle: async (url, env) => { - const title = url.toString().split('/').pop(); - const content = ` - - - ${title} - - - - -
- ${title} -
- - ` - - return { title, content }; - } -} diff --git a/packages/puppeteer-parse/index.js b/packages/puppeteer-parse/index.js index 23e743f57..e158f06d7 100644 --- a/packages/puppeteer-parse/index.js +++ b/packages/puppeteer-parse/index.js @@ -15,18 +15,10 @@ const { DateTime } = require('luxon'); const os = require('os'); const Sentry = require('@sentry/serverless'); const { Storage } = require('@google-cloud/storage'); -const { appleNewsHandler } = require('./apple-news-handler'); -const { twitterHandler } = require('./twitter-handler'); -const { youtubeHandler } = require('./youtube-handler'); -const { tDotCoHandler } = require('./t-dot-co-handler'); -const { pdfHandler } = require('./pdf-handler'); -const { mediumHandler } = require('./medium-handler'); -const { derstandardHandler } = require('./derstandard-handler'); -const { imageHandler } = require('./image-handler'); -const { scrappingBeeHandler } = require('./scrapingBee-handler'); const chromium = require('chrome-aws-lambda'); const puppeteer = require('puppeteer-core'); +const { preHandleContent } = require("@omnivore/content-handler"); // Add stealth plugin to hide puppeteer usage // const StealthPlugin = require('puppeteer-extra-plugin-stealth'); @@ -257,18 +249,6 @@ const saveUploadedPdf = async (userId, url, uploadFileId, articleSavingRequestId ); }; -const handlers = { - 'pdf': pdfHandler, - 'apple-news': appleNewsHandler, - 'twitter': twitterHandler, - 'youtube': youtubeHandler, - 't-dot-co': tDotCoHandler, - 'medium': mediumHandler, - 'derstandard': derstandardHandler, - 'image': imageHandler, - 'scrappingBee': scrappingBeeHandler, -}; - /** * Cloud Function entry point, HTTP trigger. * Loads the requested URL via Puppeteer, captures page content and sends it to backend @@ -309,61 +289,19 @@ exports.puppeteer = Sentry.GCPFunction.wrapHttpFunction(async (req, res) => { return res.sendStatus(400); } - // if (!userId || !articleSavingRequestId) { - // Object.assign(logRecord, { invalidParams: true, body: req.body, query: req.query }); - // logger.error(`Invalid parameters`, logRecord); - // return res.sendStatus(400); - // } - - // Before we run the regular handlers we check to see if we need tp - // pre-resolve the URL. TODO: This should probably happen recursively, - // so URLs can be pre-resolved, handled, pre-resolved, handled, etc. - for (const [key, handler] of Object.entries(handlers)) { - if (handler.shouldResolve && handler.shouldResolve(url)) { - try { - url = await handler.resolve(url); - validateUrlString(url); - } catch (err) { - console.log('error resolving url with handler', key, err); - } - break; - } - } - - // Before we fetch the page we check the handlers, to see if they want - // to perform a prefetch action that can modify our requests. - // enumerate the handlers and see if any of them want to handle the request - const handler = Object.keys(handlers).find(key => { - try { - return handlers[key].shouldPrehandle(url) - } catch (e) { - console.log('error with handler: ', key, e); - } - return false; - }); - - var title = undefined; - var content = undefined; - var contentType = undefined; - - if (handler) { - try { - // The only handler we have now can modify the URL, but in the - // future maybe we let it modify content. In that case - // we might exit the request early. - console.log('pre-handling url with handler: ', handler); - - const result = await handlers[handler].prehandle(url); - if (result && result.url) { - url = result.url - validateUrlString(url); - } - if (result && result.title) { title = result.title } - if (result && result.content) { content = result.content } - if (result && result.contentType) { contentType = result.contentType } - } catch (e) { - console.log('error with handler: ', handler, e); + // pre handle url with custom handlers + let title, content, contentType; + try { + const result = await preHandleContent(url); + if (result && result.url) { + url = result.url + validateUrlString(url); } + if (result && result.title) { title = result.title } + if (result && result.content) { content = result.content } + if (result && result.contentType) { contentType = result.contentType } + } catch (e) { + console.log('error with handler: ', e); } var context, page, finalUrl; diff --git a/packages/puppeteer-parse/medium-handler.js b/packages/puppeteer-parse/medium-handler.js deleted file mode 100644 index 8d0443447..000000000 --- a/packages/puppeteer-parse/medium-handler.js +++ /dev/null @@ -1,31 +0,0 @@ -/* eslint-disable no-undef */ -/* eslint-disable no-empty */ -/* eslint-disable @typescript-eslint/explicit-function-return-type */ -/* eslint-disable @typescript-eslint/no-var-requires */ -/* eslint-disable @typescript-eslint/no-require-imports */ -require('dotenv').config(); -const axios = require('axios'); -const os = require('os'); - -exports.mediumHandler = { - - shouldPrehandle: (url, env) => { - const MEDIUM_URL_MATCH = - /https?:\/\/(www\.)?medium.[a-zA-Z0-9()]{1,6}\b([-a-zA-Z0-9()@:%_\+.~#?&//=]*)/ - const res = MEDIUM_URL_MATCH.test(url.toString()) - return res - }, - - prehandle: async (url, env) => { - console.log('prehandling medium url', url) - - try { - const res = new URL(url); - res.searchParams.delete('source'); - return { url: res } - } catch (error) { - console.error('error prehandling medium url', error) - throw error - } - } -} diff --git a/packages/puppeteer-parse/pdf-handler.js b/packages/puppeteer-parse/pdf-handler.js deleted file mode 100644 index 1260db287..000000000 --- a/packages/puppeteer-parse/pdf-handler.js +++ /dev/null @@ -1,21 +0,0 @@ -/* eslint-disable no-undef */ -/* eslint-disable no-empty */ -/* eslint-disable @typescript-eslint/explicit-function-return-type */ -/* eslint-disable @typescript-eslint/no-var-requires */ -/* eslint-disable @typescript-eslint/no-require-imports */ -require('dotenv').config(); -const Url = require('url'); - - -exports.pdfHandler = { - - shouldPrehandle: (url, env) => { - const u = Url.parse(url) - const path = u.path.replace(u.search, '') - return path.endsWith('.pdf') - }, - - prehandle: async (url, env) => { - return { contentType: 'application/pdf' }; - } -} diff --git a/packages/puppeteer-parse/scrapingBee-handler.js b/packages/puppeteer-parse/scrapingBee-handler.js deleted file mode 100644 index 6563fca44..000000000 --- a/packages/puppeteer-parse/scrapingBee-handler.js +++ /dev/null @@ -1,44 +0,0 @@ -/* eslint-disable no-undef */ -/* eslint-disable no-empty */ -/* eslint-disable @typescript-eslint/explicit-function-return-type */ -/* eslint-disable @typescript-eslint/no-var-requires */ -/* eslint-disable @typescript-eslint/no-require-imports */ -require('dotenv').config(); -const axios = require('axios'); -const { parseHTML } = require('linkedom'); - -const os = require('os'); - -exports.scrapingBeeHandler = { - - shouldPrehandle: (url, env) => { - const u = new URL(url); - const hostnames = [ - 'nytimes.com', - 'news.google.com', - ] - - return hostnames.some((h) => u.hostname.endsWith(h)) - }, - - prehandle: async (url, env) => { - console.log('prehandling url with scrapingbee', url) - - try { - const response = await axios.get('https://app.scrapingbee.com/api/v1', { - params: { - 'api_key': process.env.SCRAPINGBEE_API_KEY, - 'url': url, - 'return_page_source': true, - 'block_ads': true, - 'block_resources': false, - } - }) - const dom = parseHTML(response.data).document; - return { title: dom.title, content: response.data, url: url } - } catch (error) { - console.error('error prehandling url w/scrapingbee', error) - throw error - } - } -} diff --git a/packages/puppeteer-parse/t-dot-co-handler.js b/packages/puppeteer-parse/t-dot-co-handler.js deleted file mode 100644 index cbbfb304a..000000000 --- a/packages/puppeteer-parse/t-dot-co-handler.js +++ /dev/null @@ -1,32 +0,0 @@ -/* eslint-disable no-undef */ -/* eslint-disable no-empty */ -/* eslint-disable @typescript-eslint/explicit-function-return-type */ -/* eslint-disable @typescript-eslint/no-var-requires */ -/* eslint-disable @typescript-eslint/no-require-imports */ -require('dotenv').config(); -const axios = require('axios'); -const Url = require('url'); - - -exports.tDotCoHandler = { - - shouldResolve: function (url, env) { - const T_DOT_CO_URL_MATCH = /^https:\/\/(?:www\.)?t\.co\/.*$/; - console.log('should preresolve?', T_DOT_CO_URL_MATCH.test(url), url) - return T_DOT_CO_URL_MATCH.test(url); - }, - - resolve: async function(url, env) { - return await axios.get(url, { maxRedirects: 0, validateStatus: null }) - .then(res => { - return Url.parse(res.headers.location).href; - }).catch((err) => { - console.log('err with t.co url', err); - return undefined; - }); - }, - - shouldPrehandle: (url, env) => { - return false - }, -} diff --git a/packages/puppeteer-parse/test/apple-news-handler.test.js b/packages/puppeteer-parse/test/apple-news-handler.test.js deleted file mode 100644 index 4531d720e..000000000 --- a/packages/puppeteer-parse/test/apple-news-handler.test.js +++ /dev/null @@ -1,9 +0,0 @@ -const { expect } = require('chai') -const { appleNewsHandler } = require('../apple-news-handler') - -describe('open a simple web page', () => { - it('should return a response', async () => { - const response = await appleNewsHandler.prehandle('https://apple.news/AxjzaZaPvSn23b67LhXI5EQ') - console.log('response', response) - }) -}) diff --git a/packages/puppeteer-parse/test/babel-register.js b/packages/puppeteer-parse/test/babel-register.js new file mode 100644 index 000000000..a6f65f60a --- /dev/null +++ b/packages/puppeteer-parse/test/babel-register.js @@ -0,0 +1,3 @@ +const register = require('@babel/register').default + +register({ extensions: ['.ts', '.tsx', '.js', '.jsx'] }) diff --git a/packages/puppeteer-parse/test/stub.test.ts b/packages/puppeteer-parse/test/stub.test.ts new file mode 100644 index 000000000..173ca4917 --- /dev/null +++ b/packages/puppeteer-parse/test/stub.test.ts @@ -0,0 +1,13 @@ +import 'mocha' +import * as chai from 'chai' +import { expect } from 'chai' +import 'chai/register-should' +import chaiString from 'chai-string' + +chai.use(chaiString) + +describe('Stub test', () => { + it('should pass', () => { + expect(true).to.be.true + }) +}) diff --git a/packages/puppeteer-parse/test/youtube-handler.test.js b/packages/puppeteer-parse/test/youtube-handler.test.js deleted file mode 100644 index d34643773..000000000 --- a/packages/puppeteer-parse/test/youtube-handler.test.js +++ /dev/null @@ -1,12 +0,0 @@ -const { expect } = require('chai') -const { getYoutubeVideoId } = require('../youtube-handler') - -describe('getYoutubeVideoId', () => { - it('should parse video id out of a URL', async () => { - expect('BnSUk0je6oo').to.eq(getYoutubeVideoId('https://www.youtube.com/watch?v=BnSUk0je6oo&t=269s')); - expect('vFD2gu007dc').to.eq(getYoutubeVideoId('https://www.youtube.com/watch?v=vFD2gu007dc&list=RDvFD2gu007dc&start_radio=1')); - expect('vFD2gu007dc').to.eq(getYoutubeVideoId('https://youtu.be/vFD2gu007dc')); - expect('BMFVCnbRaV4').to.eq(getYoutubeVideoId('https://youtube.com/watch?v=BMFVCnbRaV4&feature=share')); - expect('cg9b4RC87LI').to.eq(getYoutubeVideoId('https://youtu.be/cg9b4RC87LI?t=116')); - }) -}) diff --git a/packages/puppeteer-parse/twitter-handler.js b/packages/puppeteer-parse/twitter-handler.js deleted file mode 100644 index fe68e4782..000000000 --- a/packages/puppeteer-parse/twitter-handler.js +++ /dev/null @@ -1,171 +0,0 @@ -/* eslint-disable no-undef */ -/* eslint-disable no-empty */ -/* eslint-disable @typescript-eslint/explicit-function-return-type */ -/* eslint-disable @typescript-eslint/no-var-requires */ -/* eslint-disable @typescript-eslint/no-require-imports */ -require('dotenv').config(); -const axios = require('axios'); -const { DateTime } = require('luxon'); -const _ = require("underscore"); - -const TWITTER_BEARER_TOKEN = process.env.TWITTER_BEARER_TOKEN; -const TWITTER_URL_MATCH = /twitter\.com\/(?:#!\/)?(\w+)\/status(?:es)?\/(\d+)(?:\/.*)?/ - -const embeddedTweet = async (url) => { - - const BASE_ENDPOINT = 'https://publish.twitter.com/oembed' - - const apiUrl = new URL(BASE_ENDPOINT) - apiUrl.searchParams.append('url', url); - apiUrl.searchParams.append('omit_script', true); - apiUrl.searchParams.append('dnt', true); - - return await axios.get(apiUrl.toString(), { - headers: { - Authorization: `Bearer ${TWITTER_BEARER_TOKEN}`, - redirect: "follow", - }, - }); -}; - -const getTweetFields = () => { - const TWEET_FIELDS = - "&tweet.fields=attachments,author_id,conversation_id,created_at," + - "entities,geo,in_reply_to_user_id,lang,possibly_sensitive,public_metrics,referenced_tweets," + - "source,withheld"; - const EXPANSIONS = "&expansions=author_id,attachments.media_keys"; - const USER_FIELDS = - "&user.fields=created_at,description,entities,location,pinned_tweet_id,profile_image_url,protected,public_metrics,url,verified,withheld"; - const MEDIA_FIELDS = - "&media.fields=duration_ms,height,preview_image_url,url,media_key,public_metrics,width"; - - return `${TWEET_FIELDS}${EXPANSIONS}${USER_FIELDS}${MEDIA_FIELDS}`; -} - -const getTweetById = async (id) => { - const BASE_ENDPOINT = "https://api.twitter.com/2/tweets/"; - const apiUrl = new URL(BASE_ENDPOINT + id + '?' + getTweetFields()) - - return await axios.get(apiUrl.toString(), { - headers: { - Authorization: `Bearer ${TWITTER_BEARER_TOKEN}`, - redirect: "follow", - }, - }); -}; - -const getUserByUsername = async (username) => { - const BASE_ENDPOINT = "https://api.twitter.com/2/users/by/username/"; - - const apiUrl = new URL(BASE_ENDPOINT + username) - apiUrl.searchParams.append('user.fields', 'profile_image_url'); - - return await axios.get(apiUrl.toString(), { - headers: { - Authorization: `Bearer ${TWITTER_BEARER_TOKEN}`, - redirect: "follow", - }, - }); -}; - -const titleForTweet = (tweet) => { - return `${tweet.data.author_name} on Twitter` -}; - -const titleForAuthor = (author) => { - return `${author.name} on Twitter` -}; - -const usernameFromStatusUrl = (url) => { - const match = url.toString().match(TWITTER_URL_MATCH) - return match[1] -}; - -const tweetIdFromStatusUrl = (url) => { - const match = url.toString().match(TWITTER_URL_MATCH) - return match[2] -}; - -const formatTimestamp = (timestamp) => { - return DateTime.fromJSDate(new Date(timestamp)).toLocaleString(DateTime.DATETIME_FULL); -}; - -exports.twitterHandler = { - - shouldPrehandle: (url, env) => { - return TWITTER_BEARER_TOKEN && TWITTER_URL_MATCH.test(url.toString()) - }, - - // version of the handler that uses the oembed API - // This isn't great as it doesn't work well with our - // readability API. But could potentially give a more consistent - // look to the tweets - // prehandle: async (url, env) => { - // const oeTweet = await embeddedTweet(url) - // const dom = new JSDOM(oeTweet.data.html); - // const bq = dom.window.document.querySelector('blockquote') - // console.log('blockquote:', bq); - - // const title = titleForTweet(oeTweet) - // return { title, content: '
' + bq.innerHTML + '
', url: oeTweet.data.url }; - // } - - prehandle: async (url, env) => { - console.log('prehandling twitter url', url) - - const tweetId = tweetIdFromStatusUrl(url) - const tweetData = (await getTweetById(tweetId)).data; - const authorId = tweetData.data.author_id; - const author = tweetData.includes.users.filter(u => u.id = authorId)[0]; - const title = _.escape(titleForAuthor(author)) - const authorImage = author.profile_image_url.replace('_normal', '_400x400') - - let text = tweetData.data.text; - if (tweetData.data.entities && tweetData.data.entities.urls) { - for (let urlObj of tweetData.data.entities.urls) { - text = text.replace( - urlObj.url, - `${urlObj.display_url}` - ); - } - } - - const front = ` -
-

${text}

- ` - - var includesHtml = ''; - if (tweetData.includes.media) { - includesHtml = tweetData.includes.media.map(m => { - const linkUrl = m.type == 'photo' ? m.url : url; - const previewUrl = m.type == 'photo' ? m.url : m.preview_image_url; - const mediaOpen = ` - - - - ` - return mediaOpen - }).join('\n'); - } - - const back = ` - — ${author.username} ${author.name} ${formatTimestamp(tweetData.data.created_at)} -
- ` - const content = ` - - - - - - - - ${front} - ${includesHtml} - ${back} - ` - - return { content, url, title }; - } -} diff --git a/packages/puppeteer-parse/youtube-handler.js b/packages/puppeteer-parse/youtube-handler.js deleted file mode 100644 index 68dfc5af6..000000000 --- a/packages/puppeteer-parse/youtube-handler.js +++ /dev/null @@ -1,67 +0,0 @@ -/* eslint-disable no-undef */ -/* eslint-disable no-empty */ -/* eslint-disable @typescript-eslint/explicit-function-return-type */ -/* eslint-disable @typescript-eslint/no-var-requires */ -/* eslint-disable @typescript-eslint/no-require-imports */ -require('dotenv').config(); -const axios = require('axios'); -const _ = require("underscore"); - -const YOUTUBE_URL_MATCH = - /^((?:https?:)?\/\/)?((?:www|m)\.)?((?:youtube\.com|youtu.be))(\/(?:[\w-]+\?v=|embed\/|v\/)?)([\w-]+)(\S+)?$/ - -function getYoutubeVideoId(url) { - const u = new URL(url); - const videoId = u.searchParams.get('v'); - if (!videoId) { - const match = url.toString().match(YOUTUBE_URL_MATCH) - if (match === null || match.length < 6 || !match[5]) { - return undefined - } - return match[5] - } - return videoId -} -exports.getYoutubeVideoId = getYoutubeVideoId - -exports.youtubeHandler = { - shouldPrehandle: (url, env) => { - return YOUTUBE_URL_MATCH.test(url.toString()) - }, - - prehandle: async (url, env) => { - const videoId = getYoutubeVideoId(url) - if (!videoId) { - return {} - } - - const oembedUrl = `https://www.youtube.com/oembed?format=json&url=` + encodeURIComponent(`https://www.youtube.com/watch?v=${videoId}`) - const oembed = (await axios.get(oembedUrl.toString())).data; - const title = _.escape(oembed.title); - const ratio = oembed.width / oembed.height; - const thumbnail = oembed.thumbnail_url; - const height = 350; - const width = height * ratio; - const authorName = _.escape(oembed.author_name); - - const content = ` - - ${title} - - - - - - - - -

${title}

- - - ` - - console.log('got video id', videoId) - - return { content, title: 'Youtube Content' }; - } -} diff --git a/packages/readabilityjs/Readability.js b/packages/readabilityjs/Readability.js index 1c653ef36..4afd4b095 100644 --- a/packages/readabilityjs/Readability.js +++ b/packages/readabilityjs/Readability.js @@ -171,7 +171,7 @@ Readability.prototype = { // Readability-readerable.js. Please keep both copies in sync. articleNegativeLookBehindCandidates: /breadcrumbs|breadcrumb|utils|trilist/i, articleNegativeLookAheadCandidates: /outstream(.?)_|sub(.?)_|m_|omeda-promo-|in-article-advert|block-ad-.*/i, - unlikelyCandidates: /\bad\b|ai2html|banner|breadcrumbs|breadcrumb|combx|comment|community|cover-wrap|disqus|extra|footer|gdpr|header|legends|menu|related|remark|replies|rss|shoutbox|sidebar|skyscraper|social|sponsor|supplemental|ad-break|agegate|pagination|pager(?!ow)|popup|yom-remote|copyright|keywords|outline|infinite-list|beta|recirculation|site-index|hide-for-print|post-end-share-cta|post-end-cta-full|post-footer|main-navigation|programtic-ads|outstream_article|hfeed|comment-holder|back-to-top|show-up-next|onward-journey|topic-tracker|list-nav|block-ad-entity|adSpecs/i, + unlikelyCandidates: /\bad\b|ai2html|banner|breadcrumbs|breadcrumb|combx|comment|community|cover-wrap|disqus|extra|footer|gdpr|header|legends|menu|related|remark|replies|rss|shoutbox|sidebar|skyscraper|social|sponsor|supplemental|ad-break|agegate|pagination|pager(?!ow)|popup|yom-remote|copyright|keywords|outline|infinite-list|beta|recirculation|site-index|hide-for-print|post-end-share-cta|post-end-cta-full|post-footer|main-navigation|programtic-ads|outstream_article|hfeed|comment-holder|back-to-top|show-up-next|onward-journey|topic-tracker|list-nav|block-ad-entity|adSpecs|gift-article-button|modal-title|in-story-masthead|share-tools|standard-dock|expanded-dock|margins-h/i, // okMaybeItsACandidate: /and|article(?!-breadcrumb)|body|column|content|main|shadow|post-header/i, get okMaybeItsACandidate() { return new RegExp(`and|(?", + "siteName": null, + "publishedDate": null, + "language": "English", + "readerable": true +} diff --git a/packages/readabilityjs/test/test-pages/newsletters/bloomberg/expected.html b/packages/readabilityjs/test/test-pages/newsletters/bloomberg/expected.html new file mode 100644 index 000000000..dca4b0c10 --- /dev/null +++ b/packages/readabilityjs/test/test-pages/newsletters/bloomberg/expected.html @@ -0,0 +1,98 @@ +
+
+ + + + + + + + + + + + + + + +
+ + +

/>

+

The two-year Covid crisis has been a roller coaster for just about every market, but few have had a crazier time than crude oil.

+

At the start of the pandemic, crude oil cratered—at one point even sliding into negative territory. Today, it’s nearing $100.

+

The price swings have shocked motorists, investors, CEOs and OPEC+ ministers alike. An entire industry has gone from being written off as a wounded dinosaur to become a key player in the recovery.

+

This is the story of crude’s collapse, and what its comeback means for the global economy.

+

Read The Big Take.

+ + +
+ + + + +
+ + +

From today's story

+ + + + +
+ + “Eighteen months ago, we were in a global apocalypse for the energy sector, and now you’re talking about out-sized returns." + + + +

CEO, Diamondback Energy +

+ + + + on the wild prices swings in the oil market. + +
+ + +
+ + + + +
+ + +

+ ICYMI +

+ +

Read more from The Big Take.

+ + +
+ + + + + + + + + + + +
+
\ No newline at end of file diff --git a/packages/readabilityjs/test/test-pages/newsletters/bloomberg/source.html b/packages/readabilityjs/test/test-pages/newsletters/bloomberg/source.html new file mode 100644 index 000000000..e7bf56d82 --- /dev/null +++ b/packages/readabilityjs/test/test-pages/newsletters/bloomberg/source.html @@ -0,0 +1,768 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+
+
+
+ Oil's recovery from -$40 to nearly $100 a barrel isn't just a crazy + comeback story. +
+
+ + + + + + + + +
+ + + + +
+
+ /> +
+

+ The two-year + Covid crisis + has been a roller coaster for just about every market, but few + have had a crazier time than crude oil. +

+

+ At the start of the pandemic, crude oil cratered—at one point + even sliding into negative territory. Today, it’s nearing + $100. +

+

+ The price swings have shocked motorists, investors, CEOs and + OPEC+ ministers alike. An entire industry has gone from being + written off as a wounded dinosaur to become a key player in + the recovery. +

+

+ This is the story of crude’s collapse, and what its comeback + means for the global economy. +

+

+ Read + The Big Take. +

+
+
+ + + + + + + +
+

+ From today's story +

+
+ + + + + + + + + + +
+ “Eighteen months ago, we were in a global apocalypse for + the energy sector, and now you’re talking about out-sized + returns." +
+ Travis Stice + +
+ CEO, Diamondback Energy +
+
+ on the wild prices swings in the oil market. +
+
+
+ + + + +
+

+ ICYMI +

+ +

+ Read more from + The Big Take. +

+
+
+ + + + + + + + +
+ + + + +
+ + + + +
+ You received this message because you are subscribed to + Bloomberg's newsletter. If a friend forwarded you this + message, + sign up here + to get it in your inbox. +
+ + + + + +
+ Unsubscribe +
+ Bloomberg.com +
+ Contact Us + 		+ Bloomberg L.P.
731 Lexington Avenue,
New York, + NY 10022 +
+
+
+ + + + + diff --git a/packages/readabilityjs/test/test-pages/newsletters/bloomberg/url.txt b/packages/readabilityjs/test/test-pages/newsletters/bloomberg/url.txt new file mode 100644 index 000000000..0f007a3d4 --- /dev/null +++ b/packages/readabilityjs/test/test-pages/newsletters/bloomberg/url.txt @@ -0,0 +1 @@ +https://www.bloomberg.com/news/newsletters/2022-02-17/the-big-take-why-oil-prices-are-surging-around-the-world diff --git a/packages/readabilityjs/test/test-pages/newsletters/golang-weekly/expected-metadata.json b/packages/readabilityjs/test/test-pages/newsletters/golang-weekly/expected-metadata.json new file mode 100644 index 000000000..f81217cd7 --- /dev/null +++ b/packages/readabilityjs/test/test-pages/newsletters/golang-weekly/expected-metadata.json @@ -0,0 +1,10 @@ +{ + "title": "Go's first commit was in 1972?", + "byline": null, + "dir": null, + "excerpt": "Go’s Version Control History\n — Did you know the first commit in the Go repository is from\n 1972? Or is it..? Russ starts there and walks us through\n relevant commits, revision control tool changes, and pranks\n will be enjoyable to any gopher.", + "siteName": null, + "publishedDate": null, + "language": "English", + "readerable": true +} diff --git a/packages/readabilityjs/test/test-pages/newsletters/golang-weekly/expected.html b/packages/readabilityjs/test/test-pages/newsletters/golang-weekly/expected.html new file mode 100644 index 000000000..5bcd91375 --- /dev/null +++ b/packages/readabilityjs/test/test-pages/newsletters/golang-weekly/expected.html @@ -0,0 +1,142 @@ +
+
+ + +
+ + +
+ + + + + +
+
+ + +

+ Go’s Version Control History — Did you know the first commit in the Go repository is from 1972? Or is it..? Russ starts there and walks us through relevant commits, revision control tool changes, and pranks will be enjoyable to any gopher. +

+

Russ Cox

+ + +
+
+ + +

+ Go 1.18 Release Candidate 1: The Release Notes — There’s no official blog post but the first release candidate of Go 1.18 is now out (if you want to try it out, follow the instructions in this golang-announce post) so it’s a good time to skim the release notes and prepare for the final release any week now (and hopefully not five minutes after we send this newsletter…) +

+

Go Team

+ + +
+
+ + + +

+ Build Video for Go That Just Works — Mux is an API-first platform that makes it easy to build video into your apps. Live and on-demand video stream beautifully to any device, plus analytics are built-in so you can track engagement. +

+

Mux sponsor +

+ + +
+
+ + +

+ ▶ The Other Features in Go 1.18 — Had quite enough of hearing about generics and fuzz testing? No more. Michael Matloob and Daniel Martí join Mat Ryer on Go Time to talk about anything else Go 1.18 has to offer, such as workspaces. + (59 minutes.) +

+

Go Time Podcast podcast +

+ + +
+
+ + +

+ In brief: +

+ + + +
+
+ + + +
+ + +

+ Find a Job Through Hired — Create a profile on Hired to connect with hiring managers at growing startups and Fortune 500 companies. It's free for job-seekers.
Hired +

+ + +
+ + +
+
+ + +

+ File-Driven Testing in Go — If you’re familiar with table-driven tests, this is just the next step along that path (pun fully intended). +

+

Eli Bendersky

+ + +
+
+ + + + + +
+
+ + +

+ fq: Like jq But for Binary Formats — This is quite a neat idea. It’s a Go-powered tool (that is, admittedly, ‘early in development’) for working with non-text formats, such as graphics, audio, archives, etc. It’d be neat to see this improve and there’s even a list of to-dos if you want to get involved. +

+

Mattias Wadman

+ + +
+
+ + + + + +
+
+ + +

+ TCG: Terminal Cell Graphics Library — An interesting way to render monochrome graphics in the terminal by way of using special Unicode block symbols. You can, however, work at ‘pixel’ level, making it quite flexible for certain kinds of use case. The only big downside? You have to use a special font in your terminal to make it work. +

+

Sergey Mudrik

+ + +
+
+ + +
+
\ No newline at end of file diff --git a/packages/readabilityjs/test/test-pages/newsletters/golang-weekly/source.html b/packages/readabilityjs/test/test-pages/newsletters/golang-weekly/source.html new file mode 100644 index 000000000..637e30ad3 --- /dev/null +++ b/packages/readabilityjs/test/test-pages/newsletters/golang-weekly/source.html @@ -0,0 +1,2468 @@ + + + + + + + + + + + + +
+ Plus an interesting monochrome graphics library for the terminal. | +
+ + + + + +
+
+ + + + + + +
+

+ #​400 — February 18, 2022 +

+ 		+

+ Unsubscribe + | + Read on the Web +

+
+ + + + + +
+

+ Go Weekly +

+
+ + + + +
+ +
+ + + + + +
+

+ Go’s Version Control History + — Did you know the first commit in the Go repository is from + 1972? Or is it..? Russ starts there and walks us through + relevant commits, revision control tool changes, and pranks + will be enjoyable to any gopher. +

+

+ Russ Cox +

+
+ + + + + +
+

+ Go 1.18 Release Candidate 1: The Release Notes + — There’s no official blog post but the first + release candidate of Go 1.18 is now out (if you + want to try it out, follow the instructions in + this golang-announce post) so it’s a good time to skim the release notes and prepare + for the final release any week now (and hopefully not five + minutes after we send this newsletter…) +

+

+ Go Team +

+
+ + + + + +
+ +

+ Build Video for Go That Just Works + — Mux is an API-first platform that makes it easy to build + video into your apps. Live and on-demand video stream + beautifully to any device, plus analytics are built-in so + you can track engagement. +

+

+ Mux + sponsor +

+
+ + + + + +
+

+ ▶ + The Other Features in Go 1.18 + — Had quite enough of hearing about generics and fuzz + testing? No more. Michael Matloob and Daniel Martí join Mat + Ryer on Go Time to talk about + anything else Go 1.18 has to offer, such as + workspaces. + (59 minutes.) +

+

+ Go Time Podcast + podcast +

+
+ + + + +
+

+ In brief: +

+ +
+ + + + +
+ + + + + +
+

+ Jobs +

+
+ + + + + +
+

+ Golang Engineers — 100% Remote (North/South + America & Europe) + — We’ve got several opportunities for Go devs (some + working directly with Bill Kennedy!) and would love to + hear from those looking for new challenges in + distributed systems projects.
Ardan Labs +

+
+ + + + + +
+

+ Backend Engineer | Remote within CET (-3/+3 + hours) | Full-Time + — Europe's leading business finance solution. You will + help us simplify everything from everyday banking and + financing, to bookkeeping and spend management. +
Qonto +

+
+ + + + + +
+

+ Find a Job Through Hired + — Create a profile on Hired to connect with hiring + managers at growing startups and Fortune 500 + companies. It's free for job-seekers.
Hired +

+
+ + + + +
+
+ + + + +
+ + + + + +
+

+ Continuous Building and Deployment of Go Apps with + Google Cloud Build + — Cloud66 uses Google Cloud Build (Google’s CI/CD service) + to build its Go apps and this is how it all works. +

+

+ Khash Sajadi (Cloud66) +

+
+ + + + + +
+

+ File-Driven Testing in Go + — If you’re familiar with table-driven tests, this is just + the next step along that path (pun fully intended). +

+

+ Eli Bendersky +

+
+ + + + + +
+

+ ▶ + Mastering Your Error Domain: Graceful Error Handling in + Go + — A 20 minute talk (followed by Q&A) given at FOSDEM + 2022 about the errors.As helper added in Go + 1.13 and using it to improve both how you handle errors and + think about their role in your apps. +

+

+ Carl Johnson +

+
+ + + + + +
+

+ Moving Pinterest’s iOS Builds to Autoscaled EC2 Mac +

+

+ Buildkite + sponsor +

+
+ + + + + +
+

+ GoF Design Patterns That Still Make Sense in Go + — While there are people who think the classic book on + patterns is obsolete, there are plenty of patterns used in + Go today, and your code could probably use them. +

+

+ Maurício Linhares +

+
+ + + + + +
+

+ How To Use Dates and Times in Go + — A comprehensive, introductory tutorial from DigitalOcean’s + article writing program. +

+

+ Kristin Davidson +

+
+ + + + +
+

+ 🛠 Code & Tools +

+
+ + + + +
+ +
+ + + + +
+ + + + + +
+

+ fq: Like + jq + But for Binary Formats + — This is quite a neat idea. It’s a Go-powered tool (that + is, admittedly, ‘early in development’) for working with + non-text formats, such as graphics, audio, archives, etc. + It’d be neat to see this improve and there’s even a list of + to-dos + if you want to get involved. +

+

+ Mattias Wadman +

+
+ + + + + +
+

+ Bubble Tea 0.20.0: A Powerful Elm-Inspired TUI + Framework + — Based on the + Elm architecture, this is aimed at building slick terminal applications. +

+

+ Charm +

+
+ + + + + +
+

+ gRPC UI: An Interactive Web UI for Working with gRPC + — Bills itself as being + “sort of like + Postman” + but for gRPC APIs instead of REST. +

+

+ Engineering at FullStory +

+
+ + + + + +
+

+ spew: A Deep Pretty Printer for Go Data Structures +
Spew +

+
+ + + + + +
+

+ ko 0.10: Build and Deploy Go Apps on Kubernetes +
Google +

+
+ + + + +
+

+ 😎 A Cool One.. +

+
+ + + + +
+ +
+ + + + +
+ + + + + +
+

+ TCG: Terminal Cell Graphics Library + — An interesting way to render monochrome graphics in the + terminal by way of using special Unicode block symbols. You + can, however, work at ‘pixel’ level, making it quite + flexible for certain kinds of use case. The only big + downside? + You have to use a special font in your terminal to make + it work. +

+

+ Sergey Mudrik +

+
+ + + + +
+
+
+ + + o + + diff --git a/packages/readabilityjs/test/test-pages/newsletters/golang-weekly/url.txt b/packages/readabilityjs/test/test-pages/newsletters/golang-weekly/url.txt new file mode 100644 index 000000000..ed715b2ad --- /dev/null +++ b/packages/readabilityjs/test/test-pages/newsletters/golang-weekly/url.txt @@ -0,0 +1 @@ +https://u25184427.ct.sendgrid.net/ls/click?upn=MnmHBiCwIPe9TmIJeskmAyPFobFFs-2BGvrG-2FfiZ7tbf8z3z427iXCWAarenlr8M-2B4ToT4_vVXscVLXlj5UtQe3aqo5RMTdTq2PepdZjP86UOmA8nzP4yaWf257iDuG6iOHUre6y7-2Fu8NxRNvthorp-2B5Q69zvqHLE7uHn34H2Ir1kq5RNMc-2BBlyDAcOfjKFBN908nyKEgt1bPmTioC7r-2BIAIjoauPeiSUt6qjN0mIQv-2F-2BOL-2BEpvE7LwaAOU2f8jAKTnjcPlAZxiEmpyxj7VQdLHO15dKw-3D-3D diff --git a/packages/readabilityjs/test/test-pages/newsletters/milk-road/expected-metadata.json b/packages/readabilityjs/test/test-pages/newsletters/milk-road/expected-metadata.json new file mode 100644 index 000000000..826c7e523 --- /dev/null +++ b/packages/readabilityjs/test/test-pages/newsletters/milk-road/expected-metadata.json @@ -0,0 +1,10 @@ +{ + "title": "🥛 How my friend found millions NFT dumpster diving", + "byline": null, + "dir": null, + "excerpt": "GM. This is the Milk Road, the email that\n teaches you all about crypto in less time than\n it takes to choose a Netflix show.", + "siteName": null, + "publishedDate": null, + "language": "English", + "readerable": false +} diff --git a/packages/readabilityjs/test/test-pages/newsletters/milk-road/expected.html b/packages/readabilityjs/test/test-pages/newsletters/milk-road/expected.html new file mode 100644 index 000000000..b8dccd9a8 --- /dev/null +++ b/packages/readabilityjs/test/test-pages/newsletters/milk-road/expected.html @@ -0,0 +1,654 @@ +
+
+ + + + +
+ + + + + + + + + + + + +
+ + +
+ + + + + +
+ + +
+
+ + +
+ + + + + +
+ + +
+
+ + +

GM. This is the Milk Road, the email that teaches you all about crypto in less time than it takes to choose a Netflix show.

+ + +
+
+ + +

Today, we found out that inflation rose 8.5% in March (a new 40 year-high) and we’ve officially hit the “Extreme Fear” zone…

+ + +
+
+ + +
+ + + + + +
+ + +
+
+ + +

I’m not here to tell you what prices are gonna do following the US’s inflation report…but i am here to tell you that I love you. 

+ + +
+
+ + +

Yeah, I love you for giving me the next 176 seconds of your day to read this.

+ + +
+
+ + +

Today's estimated read time: 2 minutes &  56 seconds +

+ + +
+
+ + +

Let's get into the good stuff: +

+ + +
+
+ + +
    +
  • +  ⬇️ Crypto assets saw a big outflow last week +
    • +
  • +  🤑 My friend just found millions of dollars +
    • +
  • +  🗞️ Quick news nuggets +
    • +
  • +  🤣 A great Meme +
    • +
+ + +
+
+ + +
+ + + + + +
+ + +
+
+ + +

+ CRYPTO ASSETS SAW AN OUTFLOW OF $134M LAST WEEK +

+ + +
+
+ + +
+ + + + + +
+ + +
+
+ + +

I like looking at the inflows & outflows of money.

+ + +
+
+ + +

When times are good, like 2 weeks ago, we see lots of new capital coming into crypto ($450M combined in 2 weeks). This drives the price up. Because, duh. 

+ + +
+
+ + +

Last week, the flow flipped. Like when your shower all of a sudden goes ice cold because someone turned on the dishwasher. 

+ + +
+
+ + +

In the last 7d, we have a net outflow of $134M. This could be investors taking profits ahead of the US inflation data hitting or something else entirely.

+ + +
+
+ + +

Money in, money out, potato, po-tah-toh. 

+ + +
+
+ + +

I don’t get too caught up in the price movements. Instead, I try to get a fundamental understanding of why I believe in something over the long term.  +

+ + +
+
+ + +

Kyle Samani, the co-founder of Multicoin (one of the big crypto VC funds) said something similar yesterday on Twitter. He said he tries to distill every crypto deal into a 1-line reason-to-believe. 

+ + +
+
+ + +

For example, here are the 1-liners for his big investments : +

+ + +
+
+ + +

LayerZero - bridges are fucking complicated, focus on simplicity

+ + +
+
+ + +

Helium - radically reduced cost structure to build physical network of WAPs

+ + +
+
+ + +

Solana - technical scalability creates social scalability

+ + +
+
+ + +

Fractal - NFT gaming gonna be huge, led by the best conceivable for that market

+ + +
+
+ + +

They are intentionally reduced to be as simple as possible. You can’t always reduce it down into a catchy 1-liner, but it’s the thought that counts. 

+ + +
+
+ + +

P.S. Here’s your investment thesis for why you invest your time reading the Milk Road: +

+ + +
+
+ + +

“Because I can laugh every day while increasing my crypto IQ by 15 points”  +

+ + +
+
+ + +
+ + + + + +
+ + +
+
+ + +

+ MY FRIEND JUST "FOUND" MILLIONS +

+ + +
+
+ + +
+ + + + + +
+ + +
+
+ + +

What happened? XCopy is one of the most popular NFT artists in the world. And someone just “found” 100 of his old works in a junk pile (and is going to make millions off it).

+ + +
+
+ + +

Check this out: My friend Gianni went through all of Xcopy’s contracts on Etherscan and discovered that XCopy minted 100 of his first ETH NFTs on a marketplace called “RareBits” that went out of business.

+ + +
+
+ + +

He reverse engineered the unverified contract and bought them all for ~6.9e total and is now the proud owner of some of XCopy’s earliest public work and most likely will be able to sell these for millions of dollars.

+ + +
+
+ + +

Check out one of the actual NFTs he grabbed:

+ + +
+
+ + +
+ + + + + +
+ + +
+
+ + +

1/ Rarebits is a great example of how “timing is a bitch” in startups. Super smart team working on an NFT marketplace. Shut down in 2019 right before NFTs started taking off. 

+ + +
+
+ + +

2/ This is a cool example of “NFT ArchAeology” (digging up valuable treasures on the blockchain) 

+ + +
+
+ + +

3/ NFTs are “provably true.” So there is no dispute that these are authentic pieces of art by XCOPY. 

+ + +
+
+ + +
+ + + + + +
+ + +
+
+ + +

+ 6 PIECES O'NEWS NUGGETS +

+ + +
+
+ + +

Twitter Beef of the Week: Do Kwon vs. Jack Niewold about his LUNA criticisms. If you missed it, check out the thread.

+ + +
+
+ + +
+ + + + + +
+ + +
+
+ + +

+ Today's Milk Road is brought to you by LEX 🏢 +

+ + +
+
+ + +

What they do: make it easy to invest in real estate +

+ + +
+
+ + +

What asset class has created more millionaires than any other? 

+ + +
+
+ + +

Today’s sponsor, LEX, has a really cool angle for investing in real estate. 

+ + +
+
+ + +

LEX does an “IPO” for a building, so you can directly invest in marquee commercial real estate. You can build a portfolio of buildings you want to invest in. Each building has a ticker, just like stocks. 

+ + +
+
+ + +

As a shareholder, you can get paid dividends flowing from the rent paid by the tenants. You can also earn tax advantaged passive income and trade without lockups.

+ + +
+
+ + +

Check out LEX’s live assets in New York City and upcoming IPO in Seattle. 

+ + +
+
+ + +
+ + + + + +
+ + +
+
+ + +

+ MEME OF THE DAY +

+ + +
+
+ + +

Share Milk Road

+
+ + +

You currently have 0 referrals, only 1 away from receiving An Inside Look At What The Crypto Whales Are Betting On.

+ + +
+ + +
+
+ + +
+ + + + + +
+ + +
+
+ + +
+ + + + + +
+ + +
+
+ + +
+ + + + + +
+ + +
+
+ + +
+ + + + + +
+ + +
+
+ + +

What'd you think of today's email? +

+ + +
+ +
+ + +
+ + + + + +
+ + +
+ + + + +
+ + + + + + + + + [endif]--> + +
+ + +
+ + + + + + +

228 Park Ave S, #29976, New York, New York 10003

+ + + + + + +
+ + +
+ + + +
+ + +
+ + + + +
+
\ No newline at end of file diff --git a/packages/readabilityjs/test/test-pages/newsletters/milk-road/source.html b/packages/readabilityjs/test/test-pages/newsletters/milk-road/source.html new file mode 100644 index 000000000..6aab0d85b --- /dev/null +++ b/packages/readabilityjs/test/test-pages/newsletters/milk-road/source.html @@ -0,0 +1,4775 @@ + + + + + + + + + + + 🥛 How my friend found millions NFT dumpster diving + + + + + +
+ PLUS a ton of money was just pulled out of crypto +
+
+ +
+ + + + + + +
+
+ +
+ + diff --git a/packages/readabilityjs/test/test-pages/newsletters/milk-road/url.txt b/packages/readabilityjs/test/test-pages/newsletters/milk-road/url.txt new file mode 100644 index 000000000..ca8929364 --- /dev/null +++ b/packages/readabilityjs/test/test-pages/newsletters/milk-road/url.txt @@ -0,0 +1 @@ +https://www.milkroad.com/p/friend-found-millions-nft-dumpster-diving diff --git a/packages/readabilityjs/test/test-pages/newsletters/money-stuff/expected-metadata.json b/packages/readabilityjs/test/test-pages/newsletters/money-stuff/expected-metadata.json new file mode 100644 index 000000000..3fa7ff56f --- /dev/null +++ b/packages/readabilityjs/test/test-pages/newsletters/money-stuff/expected-metadata.json @@ -0,0 +1,10 @@ +{ + "title": "Money Stuff", + "byline": null, + "dir": null, + "excerpt": "I\n \n wrote last Thursday\n about a speech that Gary Gensler, the chair of the US\n Securities and Exchange Commission, gave about securities\n regulation and crypto. My basic point was that Gensler wants\n the SEC to have jurisdiction over basically all of crypto,\n because basically every crypto token is a security, but that\n he does not seem to have any interest in writing new rules to\n accommodate the crypto market. Gensler’s approach would put\n the SEC in charge of crypto, and then more or less ban crypto,\n and I am not sure that is a winning position for him to take.", + "siteName": null, + "publishedDate": null, + "language": "English", + "readerable": true +} diff --git a/packages/readabilityjs/test/test-pages/newsletters/money-stuff/expected.html b/packages/readabilityjs/test/test-pages/newsletters/money-stuff/expected.html new file mode 100644 index 000000000..00ee72511 --- /dev/null +++ b/packages/readabilityjs/test/test-pages/newsletters/money-stuff/expected.html @@ -0,0 +1,203 @@ +
+
+ + + + + + + + + + + + + + + +
+ + +
+ + +

Crypto rules

+ + +
+

I wrote last Thursday about a speech that Gary Gensler, the chair of the US Securities and Exchange Commission, gave about securities regulation and crypto. My basic point was that Gensler wants the SEC to have jurisdiction over basically all of crypto, because basically every crypto token is a security, but that he does not seem to have any interest in writing new rules to accommodate the crypto market. Gensler’s approach would put the SEC in charge of crypto, and then more or less ban crypto, and I am not sure that is a winning position for him to take.

+

Today I want to come back to one bit of Gensler’s speech that I think represents an important philosophical disconnect between the SEC and the crypto world. Gensler said:

+
+

I’ve asked the SEC staff to work directly with entrepreneurs to get their tokens registered and regulated, where appropriate, as securities. ...

+

Given the nature of crypto investments, I recognize that it may be appropriate to be flexible in applying existing disclosure requirements. Tailored disclosures exist elsewhere — for example, asset-backed securities disclosure differs from that for equities.

+
+

What I said on Thursday was that the SEC does not seem to have actually been doing any of that tailoring. I wrote:

+
+

The SEC has been suing crypto projects for illegally issuing securities for about five years now, but in that time it has not issued any rules, or proposed any rules, or put anything on its rulemaking agenda, about adapting the securities disclosure rules to crypto projects.

+
+

But arguably that slightly misrepresents what Gensler said. He didn’t say “I have asked the staff to write rules that will let entrepreneurs register tokens.” He said “I’ve asked the SEC staff to work directly with entrepreneurs to get their tokens registered.” Gensler’s paradigm is not that the SEC will write rules, and you can read them and follow them and register your tokens. The paradigm is that you walk into the SEC’s office and say “here’s a token, can you help me figure out how to register it,” and they do. There was no suggestion of new rules, but of good customer service in adapting, interpreting or perhaps waiving old rules.

+

Now. One objection to this might be that it’s empirically untrue; certainly lots of crypto entrepreneurs think that the SEC is very unhelpful in helping them figure out how to comply with existing rules. The stereotype is that, if you walk into the SEC to ask about doing a compliant crypto thing, you either get told not to do it at all, or you find a path to doing it legally but you have to pay a big fine first. The incentives are bad.

+

But here I want to focus on a different objection. The objection that I want to make here is that Gensler’s offer — “come talk to us and maybe we can be flexible to adapt the rules and figure out a way to register your tokens” — is not what crypto people want, even if he really means it.

+

The ethos of crypto is about decentralization and public code. You can read the Ethereum white paper and related standards and learn about how Ethereum works and then go off and build a decentralized options exchange or a lending platform or a pyramid scheme or whatever else you want on Ethereum. You don’t have to set up a meeting with Vitalik Buterin and get his approval; you just do it. The requirements are open and public, and if you meet them you can do what you want.

+

This is something that crypto people take seriously. People talk about “permissionless innovation,” and about how easy it is to build new businesses in crypto compared to the legacy financial system. If you are a young person with an idea for a crypto product, you can code it up and make it work with existing blockchains and protocols. Everyone has the same access to the blockchain as everyone else; everything is set up, by default, to work for everybody. If you want to run a stock trading fund, you can spend months negotiating credit terms with a prime broker and getting set up for access to the stock exchanges. If you want to run a crypto trading fund, you can trade on decentralized exchanges and get leveraged from decentralized lending platforms. Everything just sort of plugs in and works, because it is built on a model of trustless blockchains and open access and public code.

+

In principle securities regulation could be similarly open, and in practice, locally, in most places, it is. There are rules about, for instance, when an activist shareholder has to disclose her position in a company and what information she needs to include; you can read those rules (or hire a lawyer to read them), and follow them, and that’s it. You don’t need to meet with the SEC to negotiate that disclosure; you just follow the publicly available, reasonably clear rules. (Or you don’t if you’re Elon Musk, but that’s another issue.) 

+

But there are also a lot of places in securities law where the rules are a little bit vague and you are operating a little bit on the cutting edge and the best practice is to pick up the phone and call the SEC staff and say “hey what do you think about this?” Sometimes this is fairly formalized: The SEC staff issues “no-action letters” (you send them a letter saying “is it okay if we do this,” and they send back a letter saying “if you do that, we probably won’t sue you,” which is almost as good as them saying “yes”) and “telephone interpretations” (you call them up and ask “is it okay if we do this,” they say “sure seems fine” or “no that’s bad,” and then they write down the question and answer so other people with the same question don’t have to ask it again). These are places where the rules are unclear, or they are clear but applying them as written would create bad results, so the solution is to ask the SEC “is it okay if we do this” and they just tell you.

+

Sometimes it’s less formal. Your lawyer calls an SEC lawyer and has an informal chat about the issues raised by whatever you’ve got cooking, and the SEC staff raises some concerns, and you work to address those concerns, and eventually the SEC staffers say “yeah this seems fine now” and you do it. And, as you’d expect, these sorts of informal contacts tend to work better for certain sorts of people. If you are a big firm who can hire good lawyers (perhaps ones who used to work at the SEC), that’s good. If you are a big incumbent who has a reputation for knowing what you’re doing, and a lot to lose if you mess up, that’s good. If you’re a couple of 20-somethings with no track record, it might be hard to get the SEC to take you seriously, and they might be suspicious of what you’re up to.

+

Many things in crypto are (1) on the cutting edge of securities regulation and (2) done by a couple of 20-somethings with no track record. So the offer of “come in and chat with the SEC” is less appealing to them than it would be to, you know, Goldman Sachs Group Inc.

+

Crypto people want rules! I don’t mean that they want to be regulated; I mean, specifically, that they want rules. They want published, objectively specified, open and transparent rules, so that everyone is on a level playing field to do crypto stuff that complies with whatever the rules are. I don’t mean that everyone in crypto wants that: Informal regulation favors the well-capitalized and the incumbent, and if you’re a big crypto firm on good terms with the SEC (which might be an empty set) you might prefer informality and opacity. I just mean that, philosophically, crypto people should want open transparent rules for permissionless innovation. That is how the crypto system is designed to work, and they should want the securities laws to work the same way. And in fact Coinbase Global Inc., which is maybe the closest thing the US crypto industry has to a big regulated incumbent, has sent the SEC a petition asking it to make rules for crypto securities. 

+

Temperamentally I do not think the SEC likes this, and I think that Gensler means what he says about “working directly with entrepreneurs,” and I think that this is a reasoned choice. Look at how crypto often works in practice. People write smart contracts with immutable public code, and then other people hack them to steal their money. That could be the SEC! If you are the SEC, and crypto people say “please write clear transparent rules so we know what is and isn’t allowed,” you might hear that as “please write clear transparent rules so that we can game them.” This would be a reasonable lesson for the SEC to take from (1) the history of crypto’s “code is law” philosophy ending in hacks, (2) the history of crypto firms ignoring the US securities laws, and for that matter (3) the history of traditional finance firms trying to game the SEC’s rules. Crypto is a wholly new area for US securities regulation, and if you try to write all the rules from scratch in one go you will get things wrong. And then people will ruthlessly exploit whatever you get wrong.

+

For the SEC, having the rules develop informally by a process of collaboration makes sense. Someone comes in and says “can we do X?” You meet them. You ask them questions. You look them in the eye. You look at their backgrounds and their backers and get a sense of whether they are good people. (The fact that they came to you suggests that they want to be compliant; people who just read the rules on their own might be dodging your scrutiny.) They tell you what they’re doing and you evaluate it and you say “sure yeah that seems fine, for you.” They go off and do it and you see if it works. If it works out okay, then you are a little bit more generous to the next person who walks into your office looking to do something similar. If it works out terribly, then you walk it back. You proceed incrementally, by trial and error, evaluating each request not just on how well it complies with the specific written rules but on what you think about the project, its promoters and their motivations. If some big regulated public company shows up at your office with a bunch of former SEC lawyers and asks to do a thing, you might let them. If two scruffy 20-somethings show up at your office with no lawyers at all and ask to do the same thing, you might not. These choices might be totally rational as a matter of investor protection and incremental development of the rules in a new area. 

+

Philosophically I sympathize with the crypto industry here: There should be clear rules that are open and available to everyone. Practically I am pretty sympathetic to the SEC. But mostly I just want to point out that there is a disconnect. And if your vision of crypto is about disrupting the traditional financial system, then this might look like the SEC protecting the traditional system from disruption. “Just come in and talk to us,” the SEC says, but you might hear that as “you can’t do anything in crypto without talking to us first.” 

+
+ + + + + + + + + + + + + +
+
+ + + + + + + + + + + + + +
+
+ + +

Twitter vote

+ + +
+

Twitter Inc.’s shareholders are voting today on whether to sell the company to Elon Musk at $54.20 per share. Twitter closed yesterday at $41.41 per share. There is not much suspense here. If you have stock that is worth $41.41 per share, and someone wants to buy it from you at $54.20, you should let him. Twitter is easily going to get its votes. + [1]  The Wall Street Journal reports:

+
+

Early votes show investors approving the deal by a wide margin, the people said, though there is always a chance that the results could change as shareholders can alter their votes through a meeting scheduled for Tuesday at 1 p.m. Eastern time.

+
+

I do not actually think there’s much chance that the results could change. If you are a Twitter shareholder, what could possibly happen between now and 1 p.m. that would make you not want to cash out at $54.20?

+

There are a few complications. One is that news is definitely happening about Twitter today. Peiter “Mudge” Zatko, Twitter’s former head of security who has turned whistle-blower, testified in Congress today about how bad Twitter’s security is. But nothing that he says is going to make Twitter shareholders less likely to vote for the deal. The worse Twitter is, the more excited you should be about getting $54.20 for your Twitter shares. If Zatko showed up at this hearing and said “actually Twitter’s security is great and they’ve discovered cold fusion” then I guess you should vote to keep your shares; in a world where Twitter is worth much more than $54.20 on its own, the vote will probably fail. But he didn’t say that.

+

Another complication is that, of course, voting to sell to Musk at $54.20 doesn’t mean it’ll actually happen. Musk has terminated the deal (three times!) because he claims that Twitter has breached some conditions and so he doesn’t have to actually buy it; a Delaware court will decide if he’s right about any of those things. I tend to think that he’s wrong and will have to close, but I don’t have especially huge confidence in that belief, and the market-implied odds aren’t that great, which is why the stock is trading at $41.41. Still, if you are a Twitter shareholder, you have to vote yes on the deal, because if everyone votes no then the deal is definitely dead; if shareholders don’t approve the deal, that gives Musk a fourth and unassailable reason for terminating it. + [2]  The shareholders voting to close the deal is a necessary but not sufficient condition to the deal closing. Which is why they’ll vote yes.

+

A third complication is that Twitter’s biggest shareholder is, uh, Elon Musk, + [3]  and he’s trying to get out of the deal. Could he vote his 9.5% stake in Twitter against the deal, thus preventing it from closing? Well. The merger agreement (section 6.2(d)) says that he has to vote in favor of the deal, but he claims to have terminated the agreement (three times!) so perhaps he no longer feels bound by that, and it is a bit awkward for him to vote yes on a deal that he wants to get out of. What will he do? Eh, it doesn’t really matter; I’m pretty sure that Twitter is going to get a huge majority and won’t actually need Musk’s votes.

+
+ + +

Twitter whistle-blower

+ + +
+

Surely the highest-variance aspect of the Twitter vs. Musk saga is Zatko’s whistle-blower complaint. If Zatko can make a compelling case that Twitter is horribly bad — that its information security is so bad that it violates the law, that it has fraudulently concealed its problems, etc. — then that is probably Musk’s best argument to get out of the deal: Twitter is doing fraud, it has suffered a material adverse effect, etc. If Zatko is just a run-of-the-mill paranoid security researcher who is aggrieved about being fired and making mountains out of molehills, then his complaint will quickly be kicked out of court and won’t affect the Musk deal. Zatko’s credibility — whether he’s telling the truth, and also whether he is exaggerating or underselling the importance of Twitter’s problems — is a key input into your evaluation of Twitter’s stock value. The more credible he is, the less likely it is that Twitter will get $54.20 per share, and the less Twitter will be worth without Musk’s deal.

+

So if you are a hedge fund, or an expert-network firm working on behalf of hedge funds, you obviously want to know how credible he is. You might, for instance, want to talk to some of his old coworkers to get a feel for him. You might offer to pay them a lot of money for a one-hour phone call, because you might have a lot of money riding on the Twitter deal, which means specifically that you have a lot of money riding on your evaluation of Zatko’s credibility.

+

At the New Yorker, Ronan Farrow has a story on “The Search for Dirt on the Twitter Whistle-Blower”:

+
+

The dozens of e-mails and LinkedIn messages received by people in Zatko’s professional orbit appeared to be mostly from research-and-advisory companies, part of a burgeoning industry whose clients include investment firms and individuals jockeying for financial advantage through information. At least six research outfits—Gerson Lehrman Group (G.L.G.), AlphaSights, Mosaic Research Management, Ridgetop Research, Coleman Research Group, and Guidepoint—approached former colleagues of Zatko’s at Stripe, Google, and the Pentagon research agency DARPA. All offered to pay for information, sometimes noting that the compensation would be high or apparently unrestricted. At least two investment firms, Farallon Capital Management L.L.C. and Pentwater Capital Management L.P., also sought information from individuals close to Zatko.

+
+

I have to say that Farrow, and Zatko’s former coworkers, seem a lot more shocked by this than I am. Yes, right now, for a series of weird reasons, information about whether Peiter Zatko is or is not a good guy is incredibly valuable to hedge funds, and they will pay “high or apparently unrestricted” amounts of money for some informal chats with his former colleagues about their impressions of the guy. Sometimes that is how financial markets work. You get paid for incorporating information into prices. 

+

That said I particularly enjoyed this reaction:

+
+

Two members of Musk’s team, who asked not to be named, owing to the sensitivity of the ongoing litigation, said that they also had no connection to the inquiries. “There’s a lot of hedge funds currently betting that the deal flows. And so they’re doing everything they possibly can to undermine that not happening,” one of them told me. “It’s obviously wrong. You can’t discredit a witness, as opposed to listening to what he has to say and taking seriously these security threats. . . . That should be the priority, not making a buck.”

+
+

Yeah no of course, right, Elon Musk’s priority in evaluating Zatko’s complaint is solely about “taking seriously these security threats”; he has no economic interest at all in Zatko’s credibility and is just dispassionately following the truth wherever it leads.

+
+ + +

People are worried about bond market liquidity

+ + +
+

This theme is having a nice little comeback:

+
+

Pacific Investment Management Co. is advocating a radical solution to fix the liquidity woes plaguing the world of bonds: The entire $23.7 trillion Treasury market should move to a model where investors can transact directly with each other -- reducing their unhealthy dependence on balance-sheet-constrained banks.

+

Among other suggestions, a report from the nearly $2 trillion asset manager urges Janet Yellen’s Treasury Department and other regulators to help create alternative avenues that would allow traders to find buyers and sellers when the primary dealers who normally handle large orders are unable to do so.

+

“We would like the entire Treasury market to move to all-to-all trading -- a platform where asset managers, dealers, and non-bank liquidity providers are able to trade on a level playing field, with equal access to information,” wrote Pimco’s Libby Cantrill, Tim Crowley, Jerry Woytash, Jerome Schneider and Rick Chan. “The vast majority of the bond market, including most parts of the Treasury market, liquidity remains intermediated, making the market more fragile, less liquid, and more susceptible to shocks.”

+
+

Here is the report. When I was young and naive, I thought that “all-to-all trading” meant that big asset managers like Pimco would want to sell bonds, and big asset managers like BlackRock Inc. would want to buy bonds, and they would meet on some sort of exchange platform and trade bonds with each other. But the stock market is all-to-all, and it’s mostly big asset managers trading stocks with intermediaries: High-frequency traders buy from the sellers and sell to the buyers. I suppose it’s more electronic and competitive — the HFTs are largely “non-bank liquidity providers” — but still, it’s not easy to get rid of middlemen.

+
+ + +

How M&A happens

+ + +
+

Last week Anthony Scaramucci’s SkyBridge Capital announced that Sam Bankman-Fried’s FTX Ventures would acquire 30% of SkyBridge; the deal apparently also includes an option for FTX to buy 85% of SkyBridge. From the outside it is not hard to guess at the motivations of the principals. Bankman-Fried has lots of money and has been an opportunistic acquirer in a crypto bear market; buying SkyBridge presumably gives him some more mainstream distribution for crypto products. Scaramucci has had a rough year and needs the money; the Financial Times reports:

+
+

Scaramucci said that the FTX deal was a product of poor performance in a poor market. SkyBridge, which has $2.8bn in assets under management, is down 25 per cent this year, he said.

+

“Bear markets suck,” he added. “If I was doing super-well right now — our performance is mediocre, lacklustre — who knows if we would be doing the transaction.”

+
+

But there was also another motivation. The FT article goes on:

+
+

Scaramucci said the transaction was decided over a two-hour lunch at a hotel in the Bahamas, where Bankman-Fried is based. Scaramucci was with his family on a Disney cruise that had docked in the islands.

+

He said he proposed lunch to discuss the possibility of a partnership, as well as to avoid going to a water park with his children.

+
+

What percentage of mergers and acquisitions do you think are driven by people trying to avoid spending time with their children? 

+
+ + +

Things happen

+ + +
+

US Inflation Tops Forecasts, Cementing Odds of Big Fed Hike. Congresspeople just love trading individual stocks. U.S. Banks Lost a Record $370 Billion in Deposits Last Quarter. How Wall Street stormed the music business. Wall Street-Backed Crypto Exchange EDX Markets Is Set for November Debut. KKR Makes Piece of PE Fund Available on Public Blockchain. Fidelity Weighs Bitcoin Trading on Brokerage Platform. SEC Charges VMware with Misleading Investors by Obscuring Financial Performance. Tippee Pleads Guilty In First Ever Cryptocurrency Insider Trading Case. Beware Ryan Cohen, the Meme-Stock King. The Billionaire Hedge Fund Manager Who Wants to Build NFL Rosters.

+

+ If you'd like to get Money Stuff in handy email form, right in your inbox, please subscribe at this link. Or you can subscribe to Money Stuff and other great Bloomberg newsletters here. Thanks! +

+

[1] This is different from Digital World Acquisition Corp., the special purpose acquisition company that has a deal to buy Donald Trump’s social media company and can’t get the votes to extend the deadline to complete that deal, for a couple of reasons. The main one is probably that Twitter is owned by index funds, merger arbitrageurs and other institutions, while DWAC is mainly owned by retail Trump enthusiasts, who tend not to vote. But also voting on a merger is slightly more salient than voting on a necessary extension to complete that merger. “Do you want $54.20” is a simple question; “do you want to delay a year to have a good chance of getting $25-ish of value rather than getting $10.20 next week” is more confusing.

+

[2] See section 8.1(b)(iii) of the merger agreement, which unlike some of Musk’s other termination rights is not qualified by the requirement that *he* not be in breach of his obligations.

+

[3] The link in that sentence goes to Twitter’s merger proxy, which lists Vanguard Group as the biggest shareholder, a footnote cites to an April 8 Vanguard filing for Vanguard’s holdings. Bloomberg’s HDS page shows Vanguard disposing of some shares after that filing but before the record date for the Twitter meeting, leaving Musk as the biggest shareholder. Either way it’s close though.

+ + +
+ + + + + + + + +
+ + + +
+

+ Like getting this newsletter? Subscribe to Bloomberg.com for unlimited access to trusted, data-driven journalism and subscriber-only insights. +

+

+ Before it’s here, it’s on the Bloomberg Terminal. Find out more about how the Terminal delivers information and analysis that financial professionals can’t find anywhere else. Learn more. +

+
+ + + +
+ + + +
+
\ No newline at end of file diff --git a/packages/readabilityjs/test/test-pages/newsletters/money-stuff/source.html b/packages/readabilityjs/test/test-pages/newsletters/money-stuff/source.html new file mode 100644 index 000000000..64eae21a5 --- /dev/null +++ b/packages/readabilityjs/test/test-pages/newsletters/money-stuff/source.html @@ -0,0 +1,2174 @@ + + + + + + Money Stuff + + + + + + + + + + + + + + + + + + + + + + + + + +
+
+
+
+ I wrote last Thursday about a speech that Gary Gensler, the chair of + the US Securities and Exchange Commission, gave about securities + regula +
+
+ + + + +
+ + + +
+ + + + + + + + + + + + + +
+ + + + +
+ + + + +
+

+ Crypto rules +

+
+

+ I + + wrote last Thursday + about a speech that Gary Gensler, the chair of the US + Securities and Exchange Commission, gave about securities + regulation and crypto. My basic point was that Gensler wants + the SEC to have jurisdiction over basically all of crypto, + because basically every crypto token is a security, but that + he does not seem to have any interest in writing new rules to + accommodate the crypto market. Gensler’s approach would put + the SEC in charge of crypto, and then more or less ban crypto, + and I am not sure that is a winning position for him to take. +

+

+ Today I want to come back to one bit of + + Gensler’s speech + that I think represents an important philosophical disconnect + between the SEC and the crypto world. Gensler said: +

+
+

+ I’ve asked the SEC staff to work directly with entrepreneurs + to get their tokens registered and regulated, where + appropriate, as securities. ... +

+

+ Given the nature of crypto investments, I recognize that it + may be appropriate to be flexible in applying existing + disclosure requirements. Tailored disclosures exist + elsewhere — for example, asset-backed securities disclosure + differs from that for equities. +

+
+

+ What I said on Thursday was that the SEC does not seem to have + actually been doing any of that tailoring. I wrote: +

+
+

+ The SEC has been suing crypto projects for illegally issuing + securities for about five years now, but in that time it has + not issued any rules, or proposed any rules, or put anything + on its rulemaking agenda, about adapting the securities + disclosure rules to crypto projects. +

+
+

+ But arguably that slightly misrepresents what Gensler said. He + didn’t say “I have asked the staff to + write rules that will let entrepreneurs register + tokens.” He said “I’ve asked the SEC staff to + work directly with entrepreneurs to get their tokens + registered.” Gensler’s paradigm is not that the SEC will write + rules, and you can read them and follow them and register your + tokens. The paradigm is that you walk into the SEC’s office + and say “here’s a token, can you help me figure out how to + register it,” and they do. There was no suggestion of new + rules, but of good customer service in adapting, interpreting + or perhaps waiving old rules. +

+

+ Now. One objection to this might be that it’s empirically + untrue; certainly lots of crypto entrepreneurs think that the + SEC is very unhelpful in helping them figure out how + to comply with existing rules. The stereotype is that, if you + walk into the SEC to ask about doing a compliant crypto thing, + you either get + + told not to do it at all, or you find a path to doing it legally but you have to pay a big fine first. The incentives are bad. +

+

+ But here I want to focus on a different objection. The + objection that I want to make here is that Gensler’s offer + — “come talk to us and maybe we can be flexible to adapt the + rules and figure out a way to register your tokens” — is not + what crypto people want, even if he really means it. +

+

+ The ethos of crypto is about decentralization and public code. + You can read the Ethereum white paper and related standards + and learn about how Ethereum works and then go off and build a + decentralized options exchange or a lending platform or a + pyramid scheme or whatever else you want on Ethereum. You + don’t have to set up a meeting with Vitalik Buterin and get + his approval; you just do it. The requirements are open and + public, and if you meet them you can do what you want. +

+

+ This is something that crypto people take seriously. People + talk about “permissionless innovation,” and about how easy it + is to build new businesses in crypto compared to the legacy + financial system. If you are a young person with an idea for a + crypto product, you can code it up and make it work with + existing blockchains and protocols. Everyone has the same + access to the blockchain as everyone else; everything is set + up, by default, to work for everybody. If you want to run a + stock trading fund, you can spend months negotiating credit + terms with a prime broker and getting set up for access to the + stock exchanges. If you want to run a crypto trading fund, you + can trade on decentralized exchanges and get leveraged from + decentralized lending platforms. Everything just sort of plugs + in and works, because it is built on a model of trustless + blockchains and open access and public code. +

+

+ In principle securities regulation could be similarly open, + and in practice, locally, in most places, it is. There are + rules about, for instance, when an activist shareholder has to + disclose her position in a company and what information she + needs to include; you can read those rules (or hire a lawyer + to read them), and follow them, and that’s it. You don’t need + to meet with the SEC to negotiate that disclosure; you just + follow the publicly available, reasonably clear rules. (Or + + you don’t if you’re Elon Musk, but that’s another issue.)  +

+

+ But there are also a lot of places in securities law where the + rules are a little bit vague and you are operating a little + bit on the cutting edge and the best practice is to pick up + the phone and call the SEC staff and say “hey what do you + think about this?” Sometimes this is fairly formalized: The + SEC staff issues “no-action letters” (you send them a letter saying “is it okay if we do this,” + and they send back a letter saying “if you do that, we + probably won’t sue you,” which is almost as good as them + saying “yes”) and “telephone interpretations” (you call them up and ask “is it okay if we do this,” they + say “sure seems fine” or “no that’s bad,” and then they write + down the question and answer so other people with the same + question don’t have to ask it again). These are places where + the rules are unclear, or they are clear but applying + them as written would create bad results, so the solution is + to ask the SEC “is it okay if we do this” and they just tell + you. +

+

+ Sometimes it’s less formal. Your lawyer calls an SEC lawyer + and has an informal chat about the issues raised by whatever + you’ve got cooking, and the SEC staff raises some concerns, + and you work to address those concerns, and eventually the SEC + staffers say “yeah this seems fine now” and you do it. And, as + you’d expect, these sorts of informal contacts tend to work + better for certain sorts of people. If you are a big firm who + can hire good lawyers (perhaps ones who used to work at the + SEC), that’s good. If you are a big incumbent who has a + reputation for knowing what you’re doing, and a lot to lose if + you mess up, that’s good. If you’re a couple of 20-somethings + with no track record, it might be hard to get the SEC to take + you seriously, and they might be suspicious of what you’re up + to. +

+

+ Many things in crypto are (1) on the cutting edge of + securities regulation and (2) done by a couple of + 20-somethings with no track record. So the offer of “come in + and chat with the SEC” is less appealing to them than it would + be to, you know, Goldman Sachs Group Inc. +

+

+ Crypto people want rules! I don’t mean that they want to be + regulated; I mean, specifically, that they + want rules. They want published, objectively + specified, open and transparent rules, so that everyone is on + a level playing field to do crypto stuff that complies with + whatever the rules are. I don’t mean that everyone in + crypto wants that: Informal regulation favors the + well-capitalized and the incumbent, and if you’re a big crypto + firm on good terms with the SEC (which might be an empty set) + you might prefer informality and opacity. I just mean that, + philosophically, crypto people should want open + transparent rules for permissionless innovation. That is how + the crypto system is designed to work, and they should want + the securities laws to work the same way. And in fact Coinbase + Global Inc., which is maybe the closest thing the US crypto + industry has to a big regulated incumbent, has + sent the SEC a petition + asking it to make rules for crypto securities.  +

+

+ Temperamentally I do not think the SEC likes this, and I think + that Gensler means what he says about “working directly with + entrepreneurs,” and I think that this is a reasoned choice. + Look at how crypto often works in practice. People write smart + contracts with immutable public code, and then other people + + hack them + to steal their money. That could be the SEC! If you are the + SEC, and crypto people say “please write clear transparent + rules so we know what is and isn’t allowed,” you might hear + that as “please write clear transparent rules so that we can + game them.” This would be a reasonable lesson for the SEC to + take from (1) the history of crypto’s “code is law” philosophy + ending in hacks, (2) the history of crypto firms ignoring the + US securities laws, and for that matter (3) the history of + traditional finance firms trying to game the SEC’s rules. + Crypto is a wholly new area for US securities regulation, and + if you try to write all the rules from scratch in one go you + will get things wrong. And then people will ruthlessly exploit + whatever you get wrong. +

+

+ For the SEC, having the rules develop informally by a process + of collaboration makes sense. Someone comes in and says “can + we do X?” You meet them. You ask them questions. You look them + in the eye. You look at their backgrounds and their backers + and get a sense of whether they are good people. (The + fact that they came to you suggests that they want to + be compliant; people who just read the rules on their own + might be dodging your scrutiny.) They tell you what they’re + doing and you evaluate it and you say “sure yeah that seems + fine, for you.” They go off and do it and you see if it works. + If it works out okay, then you are a little bit more generous + to the next person who walks into your office looking to do + something similar. If it works out terribly, then you walk it + back. You proceed incrementally, by trial and error, + evaluating each request not just on how well it complies with + the specific written rules but on what you think about the + project, its promoters and their motivations. If some big + regulated public company shows up at your office with a bunch + of former SEC lawyers and asks to do a thing, you might let + them. If two scruffy 20-somethings show up at your office with + no lawyers at all and ask to do the same thing, you might not. + These choices might be totally rational as a matter of + investor protection and incremental development of the rules + in a new area.  +

+

+ Philosophically I sympathize with the crypto industry here: + There should be clear rules that are open and available to + everyone. Practically I am pretty sympathetic to the SEC. But + mostly I just want to point out that there is a disconnect. + And if your vision of crypto is about disrupting the + traditional financial system, then this might look like the + SEC protecting the traditional system from disruption. “Just + come in and talk to us,” the SEC says, but you might hear that + as “you can’t do anything in crypto without talking to us + first.”  +

+ + + + + + + +
+ + + +
+ +
+ + + + +
+ + + +
+
+ +
+ + + + + + + +
+ + + +
+ +
+ + + + +
+ + + +
+
+ +
+ + + + +
+

+ Twitter vote +

+
+

+ Twitter Inc.’s shareholders are voting today on whether to + sell the company to Elon Musk at $54.20 per share. Twitter + closed yesterday at $41.41 per share. There is not much + suspense here. If you have stock that is worth $41.41 per + share, and someone wants to buy it from you at $54.20, you + should let him. Twitter is easily going to get its votes. + [1]  The + + Wall Street Journal reports: +

+
+

+ Early votes show investors approving the deal by a wide + margin, the people said, though there is always a chance + that the results could change as shareholders can alter + their votes through a meeting scheduled for Tuesday at 1 + p.m. Eastern time. +

+
+

+ I do not actually think there’s much chance that the results + could change. If you are a Twitter shareholder, what could + possibly happen between now and 1 p.m. that would make + you not want to cash out at $54.20? +

+

+ There are a few complications. One is that news is definitely + happening about Twitter today. Peiter “Mudge” Zatko, Twitter’s + former head of security who has turned whistle-blower, + testified in Congress today about how bad Twitter’s security is. But nothing that + he says is going to make Twitter + shareholders less likely to vote for the deal. The + worse Twitter is, the more excited you should be about getting + $54.20 for your Twitter shares. If Zatko showed up at this + hearing and said “actually Twitter’s security is great and + they’ve discovered cold fusion” then I guess you should vote + to keep your shares; in a world where Twitter is worth much + more than $54.20 on its own, the vote will probably fail. But + he didn’t say that. +

+

+ Another complication is that, of course, voting to sell to + Musk at $54.20 doesn’t mean it’ll actually happen. Musk has + terminated the deal (three times!) because he claims that Twitter has breached some conditions + and so he doesn’t have to actually buy it; a Delaware court + will decide if he’s right about any of those things. I tend to + think that he’s wrong and will have to close, but I don’t have + especially huge confidence in that belief, and the + market-implied odds aren’t that great, which is why the stock + is trading at $41.41. Still, if you are a Twitter shareholder, + you have to vote yes on the deal, because if everyone votes no + then the deal is definitely dead; if shareholders + don’t approve the deal, that gives Musk a fourth and + unassailable reason for terminating it. + [2]  The shareholders voting to close the deal is a necessary but + not sufficient condition to the deal closing. Which is why + they’ll vote yes. +

+

+ A third complication is that Twitter’s + + biggest shareholder + is, uh, Elon Musk, + [3]  and he’s trying to get out of the deal. Could he vote his + 9.5% stake in Twitter against the deal, thus + preventing it from closing? Well. The merger agreement (section 6.2(d)) says that he has to vote in favor of the deal, but he + claims to have terminated the agreement (three times!) so + perhaps he no longer feels bound by that, and it is a bit + awkward for him to vote yes on a deal that he wants to get out + of. What will he do? Eh, it doesn’t really matter; I’m pretty + sure that Twitter is going to get a huge majority and won’t + actually need Musk’s votes. +

+ + + + +
+ + + +
+ + + + +
+

+ Twitter whistle-blower +

+
+

+ Surely the + + highest-variance aspect + of the Twitter vs. Musk saga is Zatko’s whistle-blower + complaint. If Zatko can make a compelling case that Twitter is + horribly bad — that its information security is so bad that it + violates the law, that it has fraudulently concealed its + problems, etc. — then that is probably Musk’s best argument to + get out of the deal: Twitter is doing fraud, it has suffered a + material adverse effect, etc. If Zatko is just a + run-of-the-mill paranoid security researcher who is aggrieved + about being fired and making mountains out of molehills, then + his complaint will quickly be kicked out of court and won’t + affect the Musk deal. Zatko’s credibility — whether he’s + telling the truth, and also whether he is exaggerating or + underselling the importance of Twitter’s problems — + is a key input into your evaluation of Twitter’s stock value. + The more credible he is, the less likely it is that Twitter + will get $54.20 per share, and the less Twitter will be worth + without Musk’s deal. +

+

+ So if you are a hedge fund, or an expert-network firm working + on behalf of hedge funds, you obviously want to know how + credible he is. You might, for instance, want to talk to some + of his old coworkers to get a feel for him. You might offer to + pay them a lot of money for a one-hour phone call, because you + might have a lot of money riding on the Twitter deal, which + means specifically that you have a lot of money riding on your + evaluation of Zatko’s credibility. +

+

+ At the New Yorker, Ronan Farrow has a story on “The Search for Dirt on the Twitter Whistle-Blower”: +

+
+

+ The dozens of e-mails and LinkedIn messages received by + people in Zatko’s professional orbit appeared to be mostly + from research-and-advisory companies, part of a burgeoning + industry whose clients include investment firms and + individuals jockeying for financial advantage through + information. At least six research outfits—Gerson Lehrman + Group (G.L.G.), AlphaSights, Mosaic Research Management, + Ridgetop Research, Coleman Research Group, and + Guidepoint—approached former colleagues of Zatko’s at + Stripe, Google, and the Pentagon research agency DARPA. All + offered to pay for information, sometimes noting that the + compensation would be high or apparently unrestricted. At + least two investment firms, Farallon Capital Management + L.L.C. and Pentwater Capital Management L.P., also sought + information from individuals close to Zatko. +

+
+

+ I have to say that Farrow, and Zatko’s former coworkers, seem + a lot more shocked by this than I am. Yes, right now, for a + series of weird reasons, information about whether + Peiter Zatko is or is not a good guy is incredibly valuable to + hedge funds, and they will pay “high or apparently + unrestricted” amounts of money for some informal chats with + his former colleagues about their impressions of the guy. + Sometimes that is how financial markets work. You get paid for + incorporating information into prices.  +

+

+ That said I particularly enjoyed this reaction: +

+
+

+ Two members of Musk’s team, who asked not to be named, owing + to the sensitivity of the ongoing litigation, said that they + also had no connection to the inquiries. “There’s a lot of + hedge funds currently betting that the deal flows. And so + they’re doing everything they possibly can to undermine that + not happening,” one of them told me. “It’s obviously wrong. + You can’t discredit a witness, as opposed to listening to + what he has to say and taking seriously these security + threats. . . . That should be the priority, not making a + buck.” +

+
+

+ Yeah no of course, right, Elon Musk’s priority in evaluating + Zatko’s complaint is solely about “taking seriously these + security threats”; he has no economic interest at all in + Zatko’s credibility and is just dispassionately following the + truth wherever it leads. +

+ + + + +
+

+ People are worried about bond market liquidity +

+
+

+ This theme is + + having a nice little comeback: +

+
+

+ Pacific Investment Management Co. is advocating a radical + solution to fix the liquidity woes plaguing the world of + bonds: The entire $23.7 trillion Treasury market should move + to a model where investors can transact directly with each + other -- reducing their unhealthy dependence on + balance-sheet-constrained banks. +

+

+ Among other suggestions, a report from the nearly $2 + trillion asset manager urges Janet Yellen’s Treasury + Department and other regulators to help create alternative + avenues that would allow traders to find buyers and sellers + when the primary dealers who normally handle large orders + are unable to do so. +

+

+ “We would like the entire Treasury market to move to + all-to-all trading -- a platform where asset managers, + dealers, and non-bank liquidity providers are able to trade + on a level playing field, with equal access to information,” + wrote Pimco’s Libby Cantrill, Tim Crowley, Jerry Woytash, + Jerome Schneider and Rick Chan. “The vast majority of the + bond market, including most parts of the Treasury market, + liquidity remains intermediated, making the market more + fragile, less liquid, and more susceptible to shocks.” +

+
+

+ Here is + + the report. When I was young and naive, I thought that “all-to-all + trading” meant that big asset managers like Pimco would want + to sell bonds, and big asset managers like BlackRock Inc. + would want to buy bonds, and they would meet on some sort of + exchange platform and trade bonds with each other. But the + stock market is all-to-all, and it’s mostly big asset managers + trading stocks with intermediaries: High-frequency traders buy + from the sellers and sell to the buyers. I suppose it’s more + electronic and competitive — the HFTs are largely “non-bank + liquidity providers” — but still, it’s not easy to get rid of + middlemen. +

+ + + + +
+

+ How M&A happens +

+
+

+ Last week Anthony Scaramucci’s SkyBridge Capital + + announced + that Sam Bankman-Fried’s FTX Ventures would acquire 30% of + SkyBridge; the deal apparently also includes an option for FTX + to buy 85% of SkyBridge. From the outside it is not hard to + guess at the motivations of the principals. Bankman-Fried has + lots of money and has been an opportunistic acquirer in a + crypto bear market; buying SkyBridge presumably gives him some + more mainstream distribution for crypto products. Scaramucci + has had a rough year and needs the money; + + the Financial Times reports: +

+
+

+ Scaramucci said that the FTX deal was a product of poor + performance in a poor market. SkyBridge, which has $2.8bn in + assets under management, is down 25 per cent this year, he + said. +

+

+ “Bear markets suck,” he added. “If I was doing super-well + right now — our performance is mediocre, lacklustre — who + knows if we would be doing the transaction.” +

+
+

+ But there was also another motivation. The FT article goes on: +

+
+

+ Scaramucci said the transaction was decided over a two-hour + lunch at a hotel in the Bahamas, where Bankman-Fried is + based. Scaramucci was with his family on a Disney cruise + that had docked in the islands. +

+

+ He said he proposed lunch to discuss the possibility of a + partnership, as well as to avoid going to a water park with + his children. +

+
+

+ What percentage of mergers and acquisitions do you think are + driven by people trying to avoid spending time with their + children?  +

+ + + + +
+

+ Things happen +

+
+

+ US + + Inflation + Tops Forecasts, Cementing Odds of Big Fed Hike. Congresspeople + just love + + trading individual stocks. U.S. Banks Lost a Record + + $370 Billion in Deposits + Last Quarter. How Wall Street stormed + + the music business. Wall Street-Backed Crypto Exchange + + EDX Markets + Is Set for November Debut. KKR Makes Piece of PE Fund + Available + + on Public Blockchain. Fidelity Weighs + + Bitcoin Trading + on Brokerage Platform. SEC Charges + + VMware + with Misleading Investors by Obscuring Financial Performance. + Tippee Pleads Guilty In First Ever + + Cryptocurrency Insider Trading Case. Beware + + Ryan Cohen, the Meme-Stock King. The Billionaire Hedge Fund Manager Who + Wants to Build NFL Rosters. +

+

+ If you'd like to get Money Stuff in handy email form, + right in your inbox, please subscribe at this link. Or you can subscribe to Money Stuff and other great + Bloomberg newsletters + here. Thanks! +

+
+

+ [1] This is different from Digital World Acquisition Corp., + the special purpose acquisition company that has a deal to + buy Donald Trump’s social media company and + + can’t get the votes + to extend the deadline to complete that deal, for a couple + of reasons. The main one is probably that Twitter is owned + by index funds, merger arbitrageurs and other institutions, + while DWAC is mainly owned by retail Trump enthusiasts, who + tend not to vote. But also voting on a merger is slightly + more salient than voting on a necessary extension to + complete that merger. “Do you want $54.20” is a simple + question; “do you want to delay a year to have a good chance + of getting $25-ish of value rather than getting $10.20 next + week” is more confusing. +

+
+
+

+ [2] See section 8.1(b)(iii) of + + the merger agreement, which unlike some of Musk’s other termination rights is + not qualified by the requirement that *he* not be in breach + of his obligations. +

+
+
+

+ [3] The link in that sentence goes to Twitter’s merger + proxy, which lists Vanguard Group as the biggest + shareholder, a footnote cites to an April 8 Vanguard filing + for Vanguard’s holdings. Bloomberg’s HDS page shows Vanguard + disposing of some shares after that filing but before the + record date for the Twitter meeting, leaving Musk as the + biggest shareholder. Either way it’s close though. +

+
+
+
+ + + + + + + +
+ + + + +
+ + + +
+ + + + + + +
+
+
+

+ Like getting this newsletter? + Subscribe to Bloomberg.com + for unlimited access to trusted, data-driven journalism + and subscriber-only insights. +

+

+ Before it’s here, it’s on the Bloomberg Terminal. Find + out more about how the Terminal delivers information and + analysis that financial professionals can’t find + anywhere else. Learn more. +

+
+
+ + + + + + + + + + + + + +
+ You received this message because you are subscribed to + Bloomberg's Money Stuff newsletter. +
+ Unsubscribe + | + Bloomberg.com + | + Contact Us +
+ + + + + + +
+ + + + + | + + + + +
+
+ Bloomberg L.P. 731 Lexington, New York, NY, 10022 +
+
+ + + + + + + + + + +
+ + + + + + + + + +
+ + + + diff --git a/packages/readabilityjs/test/test-pages/newsletters/money-stuff/url.txt b/packages/readabilityjs/test/test-pages/newsletters/money-stuff/url.txt new file mode 100644 index 000000000..a5ad7feb9 --- /dev/null +++ b/packages/readabilityjs/test/test-pages/newsletters/money-stuff/url.txt @@ -0,0 +1 @@ +https://www.bloomberg.com/opinion/articles/2022-09-13/crypto-wants-some-sec-rules diff --git a/packages/readabilityjs/test/test-pages/newsletters/substack/expected-metadata.json b/packages/readabilityjs/test/test-pages/newsletters/substack/expected-metadata.json new file mode 100644 index 000000000..7cdc14db4 --- /dev/null +++ b/packages/readabilityjs/test/test-pages/newsletters/substack/expected-metadata.json @@ -0,0 +1,10 @@ +{ + "title": "Tic's Weekly Thoughts", + "byline": "Tic Toc Trading", + "dir": null, + "excerpt": "Traders-", + "siteName": null, + "publishedDate": "2001-01-28T16:00:00.000Z", + "language": "English", + "readerable": true +} diff --git a/packages/readabilityjs/test/test-pages/newsletters/substack/expected.html b/packages/readabilityjs/test/test-pages/newsletters/substack/expected.html new file mode 100644 index 000000000..6180c673c --- /dev/null +++ b/packages/readabilityjs/test/test-pages/newsletters/substack/expected.html @@ -0,0 +1,281 @@ +
+
+
+

Parts Unknown...

+
+
+

Traders-

+

I was bullish this week at 4200/4300 with the view the market may balance between 4200 and 4500 for quite some time and I was quite right in this assumption as the market made multiple attempts to take out 4200 and failed each time.

+

This week’s installment of the Weekly Plan I will try to figure out if this assumption still stands, various factors that support this assumptions as well as the factors which may be indicating that there may be more sell off coming ahead.

+

+ Note this is a preview post from my substack where I do a longer form analysis about 5 times a week. Click the link below to become a part of my emails to get a copy whenever they are published. I will never-ever spam you. That is a promise. +

+
+
+
+ + + + + + + + + +
+
NFP on Friday
+
+
+

+ Image above is from "The Week" +

+

Without much ado, let us dive into the chart of S&P500 Emini (Chart A). This is the 5 auctions of this past week, each graphically representing the frustrations of sellers as they tried to break the lows. Each one of these levels and my context was shared in 5 Daily newsletters this past week. Posts are sent every day around 4 PM, after market close.

+

From a 10000 feet view, this chart below shows me while the lows were fought bitterly for and won by the bulls, the bulls are not necessarily out of the danger here yet. Read on to find out why I think so.

+
+
+
+ + + + + + + + + +
+
+ Chart A Emini Daily Profiles, 5 Days +
+
+
+

More than half of the Nasdaq stocks have been cut in half in market cap. Majority, as many as 75-80% are now trading below their 200 DMA . These stats are not exactly the cheerleaders of strong bull markets, if any thing they showcase the carnage that has been done and portends possibly more.

+

See below chart B for the 5 Day Auction in erstwhile momentum sweetheart $ARKK. Cathie and her ETF has fallen from graces and is now in the dumps. I was bear on this at 125 and is now cut in half, last traded lows around 65 bucks.

+

Can it rise from here? This profile chart certainly seems to indicate that to me. But not without a fight, and a little help from the FED.

+
+
+
+ + + + + + + + + +
+
+ Chart B ARKK Daily Auction +
+
+
+

Fundamentals and earnings aside, S&P500 is impacted by nothing more than geopolitics and FED liquidity or lack thereof. 10 year yields are a very good indicator of FED tightening or easing. And I use TLT quite a bit as a gauge of this.

+

TLT has been stubborn to trade below that vaunted 140 all this week, rallying on Friday, and taking the equities with it. This chart C certainly suggests there may be more juice to this.

+
+
+
+ + + + + + + + + +
+
Chart C TLT Auctions
+
+
+

Last but not the least, here is the actual ten year bond yields.. Still elevated but down from the highs. I have a theory about where these end up for most of the year, read on more to find out.

+
+
+
+ + + + + + + + + +
+
Chart D 10 year yields cool off the highs
+
+
+

Friday was a very good day for the equities. Capping off a tumultuous week. It reinforced my opinion that prices below 4200 will be harder to achieve per my trade plans from last week. I turned out to be right and we closed right above the 200 DMA at 4425. Here is the link to my trade plan from last week, in case you have not read it yet: PAST WEEKLY PLAN +

+

Looking ahead, I am cautiously bullish into dips. At this point in time, I do not see evidence that suggests there will be earth shattering moves either on the downside or to the upside.

+

My current thinking is that we could balance here between 4200-4500 for few more days before a break higher into 4600-4700. Remember my thinking is not informed by any charts or technical analysis but it is heavily influenced by the order flow. These are the actual orders that hit the tape every day. Order flow can and does change at any time due to macro factors or sudden events. Therefore my opinion can change at any time as a result. However, if we assume the current factors stay in homeostasis, then I have no reason to suspect 4200-4500 thesis is not intact this coming week.

+

+ These are the main factors which I think support bullish action: +

+
    +
  1. +

    + Seasonality: this is the tax season. There is a natural tailwind for stocks due to various type of tax events, whether that is 401, IRA contributions or rebalancing. +

    +
    2. +
  2. +

    + Money Velocity: while CPI has run rampant, the money supply has been in the dumps. This suggests the inflation problem is more demand driven than systemic. Think of money velocity as how many times the same dollar bill changes hands. When the same dollar bill goes from person to person or business to business, several times, it creates more money velocity and IMO those type of movement create persistent inflation problems like we had in the 1980s. +

    +
    +
    +
    + + + + + + + + + +
    +
    M2 Velocity
    +
    +
    +
    3. +
  3. +

    + Current inflation situation is due in most part to destruction of supply channels. Demand is there but for how long is any one’s guess. I see as more and more coronavirus restrictions are taken off, the supply may overwhelm the system, driving down the prices. That may be next month, that may be months from now. However looking at mid terms and political scene, I think that is closer than you think. +

    +
    4. +
  4. +

    + Valuations: Valuations of some of the bellwether stocks like GOOGL, have already been cut down quite a bit! It is trading at a forward p/e of 19 and I thought at 2500 it was ridiculous! So there is that… yes more sell may come but once GOOGL starts trading at 2300-2500 you oughta think, “man this is silly!” +

    +
    5. +
  5. +

    + Ten year yields have come off the highs. I think they will find a balance between 1.7-2 % and that will not be too extreme for the equities. +

    +
    6. +
+

+ Then there are factors which are potentially bearish: +

+
    +
  1. +

    + FED FED FED: despite the uncertainty around inflation, Powell was adamant about pulling liquidity out of the system . FED is the ultimate LP (liquidity provider) . No liquidity = no stock market. Less liquidity = lesser stock market. I think the way a lot of funds read him is he wants lower stock prices. Lower stock prices are deflationary by nature . So even though the inflation may cool down, if enough people believe Powell wants lower stock market that becomes a self fulfilling prophecy. +

    +
    2. +
  2. +

    + Technical damage: S&P500 is within an inch of 200 DMA. It may take lot more than one or two closed above 4410 for calm to return. +

    +
    3. +
+

+ Key events next week: +

+
    +
  • +

    Monday: Chicago PMI and FED Speak.

    +
    • +
  • +

    Tuesday: JOLTS and ISM

    +
    • +
  • +

    Wednesday: ADP pre NFP

    +
    • +
  • +

    Friday: Non FARM Payroll Report (NFP) AND Wage Inflation numbers.

    +
    • +
+

The theme of these events for me will be to see if we are beginning to see the inflation numbers come down or are they still surprising to the upside. Same for wage inflation numbers. With regards to actual job growth, I think we are now in a phase of market where good news is bad for stocks and bad news is good for stocks. So any miss in the NFP number may be perceived to be good for stocks .

+

Expectation is 166 K jobs added.

+

With this context and background, here is how I am technically preparing for next week’s trading:

+

I suspect Friday’s late rally was driven by short squeeze. If so, we may find sellers here at 4430-4454. Key level for me for the week ahead is 4360.

+
    +
  1. +

    On Monday if we open or offer below 4360, I think more softness may develop, testing the lows at 4288/4300. I will validate this with the Tic TOP indicator and TRIN. See this link if you have not yet viewed Tic TOP script: Trend Trading using Tic TOP Indicator +

    +
    2. +
  2. +

    Break of 4288 will become a bearish event for me and may target recent swing lows at 4210.

    +
    3. +
  3. +

    In an unlikely event of an open or bids above 4411 on Monday AM, I will be bullish for a test of 4450-4456. Validated with Tic TOP indicator.

    +
    4. +
  4. +

    Any openings or prints between 4360-4411 may be balance trades for me, in anticipation of the jobs report on Friday.

    +
    5. +
+

Remember levels are static . Context and order flow is dynamic. Always validated with other things like TRIN, TICK, Tic TOP, etc

+

Earnings next week:

+

There are tremendous earnings next week with GOOG, AMZN, FB, XOM being a few of them..

+

Keeping in line with my prior analysis of the general market conditions, these stocks while attractive, may find some selling action as well.

+

AMZN

+

Amazon specifically, last traded a high of 2900. This stock BTW which I shared at 2700 before a 200 point zipper, if this drops into 2500-2621 on earnings induced swoon may be a buy for me.

+

HD

+

Home Depot which was my TOP stock in 2021 has been a victim of recent sell as well. I did not notice earlier it had fallen to the 350 lows recently and if it revisits those lows, I want to be in. Last traded 366.

+

XOM

+

This stock shared by me at 60 has been on a tear and could be headed a bit higher after earnings as it makes a climactic high. Last traded 75, in my opinion this may test 82-84 if 68/69 held.

+

FB

+

FaceBook has run into some execution issues especially with their desperate foray into Meta and Crypto NFT space. I do not know if this is temporary glitch or systemic issue with leadership/execution. However, I am on alert to see if this stock falls below 274/280 on earnings (last traded 301). If it does , I do want to dip my toes in it and see if it holds.

+
+
+
+ + + + + + + + + +
+
Chart E FB Monthly Auction
+
+
+

Subscribers get my earnings analysis before and after key events. Stay tuned as more actionable ideas develop for me.

+

To Summarize:

+
    +
  1. +

    I was bullish on S&P500 at the lows last week and was proven to be right as the market staged an impressive 200 point rally off the lows.

    +
    2. +
  2. +

    While longer term bullish for a test of 4700, I do not think the market is out of the woods yet as may chop around due to technical and lack of clarity on a few important data prints.

    +
    3. +
  3. +

    That clarity may come this week with flailing inflation and falling NFP numbers. Market paradigm may shift to “Bad news is good news”. Do not get shafted when the paradigm shifts. Markets are forward looking, they do not make next moves on yesterdays news.

    +
    4. +
  4. +

    + Investor Tic is liking the sale being offered on Big Tech names like GOOG, TSLA, AMZN, FB, HD and will buy more if they fall more. Investor Tic time frame is very long (10 years +) with the money he does not need neither today nor a year from now. +

    +
    5. +
  5. +

    + Trader Tic expects more volatility. He thinks one more dip before we really firm up on shifting paradigm. But must validate with Tic provided tools like TICK, TRIN, and TIC TOP indicators. Trader Tic shares his levels and thoughts BEFORE market opens, every day. Subscribe to get Trader Tic’s thoughts. +

    +

    + Subscribe now +

    +
    6. +
+

Have an awesome week ahead. Feel free to share this preview of the newsletter to help any one else who may need it.

+

~ Tic

+

+ Share +

+

+ Share Tic Toc Newsletter +

+

+ Disclaimer: This newsletter is not trading or investment advice, but for general informational purposes only. This newsletter represents my personal opinions which I am sharing publicly as my personal blog. Futures, stocks, bonds trading of any kind involves a lot of risk. No guarantee of any profit whatsoever is made. In fact, you may lose everything you have. So be very careful. I guarantee no profit whatsoever, You assume the entire cost and risk of any trading or investing activities you choose to undertake. You are solely responsible for making your own investment decisions. Owners/authors of this newsletter, its representatives, its principals, its moderators and its members, are NOT registered as securities broker-dealers or investment advisors either with the U.S. Securities and Exchange Commission, CFTC or with any other securities/regulatory authority. Consult with a registered investment advisor, broker-dealer, and/or financial advisor. Reading and using this newsletter or any of my publications, you are agreeing to these terms. Any screenshots used here are the courtesy of Ninja Trader, Think or Swim and/or Jigsaw. I am just an end user, they own all copyrights to their products. +

+
+
+

+ This is the Free once-a-week post from Orderflow. Feel free to share it. For up-to 5 posts a week, become a paying subscriber. This is my personal opinion about current market affairs and is not financial advice. +

+
+
+
\ No newline at end of file diff --git a/packages/readabilityjs/test/test-pages/newsletters/substack/source.html b/packages/readabilityjs/test/test-pages/newsletters/substack/source.html new file mode 100644 index 000000000..70dc5fa64 --- /dev/null +++ b/packages/readabilityjs/test/test-pages/newsletters/substack/source.html @@ -0,0 +1,3156 @@ + + + Tic's Weekly Thoughts + + + + +
+ Parts + Unknown... ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ ‌ +
+ + + + + + + + + + + + + + + diff --git a/packages/readabilityjs/test/test-pages/newsletters/substack/url.txt b/packages/readabilityjs/test/test-pages/newsletters/substack/url.txt new file mode 100644 index 000000000..e1e68b163 --- /dev/null +++ b/packages/readabilityjs/test/test-pages/newsletters/substack/url.txt @@ -0,0 +1 @@ +https://email.mg2.substack.com/c/eJxVkk2PozAMhn9NuRWRhJLmkMPOdLrL7MBMtx-a7gVB4kJaCIiEduHXbzo9jWTZku3XlvxY5BbKth951xrr3V1mxw64hpupwVrovcFAnynJKV5QSjHyJA8lWi6WnjLZqQdoclVz2w_gdUNRK5Fb1eq7YkFQGCGv4hJhQaBggYxYTgWjIqCRwEguTyyHoHgszgepQAvgcIV-bDV4Na-s7cyM_JjhtTOrhG2F7XOpdOmboTA2FxdftI0rdo8GM78BXOpxbqt2KCvr1GvbXkDPyArGVyTwYfzE9SU-tyQ570k6iTHZ3pT4ySa5Zt3f5zhKV2KRrGKSrvYmbupKulyyOwbJtCHp-TK9u_78M53cDCV-HdTbbj8luw1OtrGJdYqOKo5i_XQVZGNFc6iO5E9X4FCdNr743VyFfqs-XvP1P5jPRdptovHl_elj-TKKJCmfu6Z-295Mf_QUxwHGAcIMUYxR4GMfANHoxIKACXwSBPvNVZdF1EWzMGhK_O0mXs_N6N-GqnDF8k7pK-sgZS42g1Z2zEDnRQ3ywc8-3uCLaFaCht69h8xyy1EUkpBGjLDFkjxwOcAhZQGm4cJza2XrVJp_Q_QfomLN7g diff --git a/packages/readabilityjs/test/test-pages/nytimes-podcasts/expected-metadata.json b/packages/readabilityjs/test/test-pages/nytimes-podcasts/expected-metadata.json new file mode 100644 index 000000000..9f428f123 --- /dev/null +++ b/packages/readabilityjs/test/test-pages/nytimes-podcasts/expected-metadata.json @@ -0,0 +1,12 @@ +{ + "title": "Transcript: Ezra Klein Interviews Patrick Collison", + "byline": "The New York Times", + "dir": null, + "excerpt": "The Sept. 27, 2022 episode of “The Ezra Klein Show”", + "siteName": null, + "siteIcon": "/vi-assets/static-assets/favicon-d2483f10ef688e6f89e23806b9700298.ico", + "previewImage": "https://static01.nyt.com/newsgraphics/images/icons/defaultPromoCrop.png", + "publishedDate": "2022-09-27T16:25:17.221Z", + "language": "English", + "readerable": true +} diff --git a/packages/readabilityjs/test/test-pages/nytimes-podcasts/expected.html b/packages/readabilityjs/test/test-pages/nytimes-podcasts/expected.html new file mode 100644 index 000000000..8d79daf8c --- /dev/null +++ b/packages/readabilityjs/test/test-pages/nytimes-podcasts/expected.html @@ -0,0 +1,354 @@ +
+
+
+
+
+
+

The Ezra Klein Show

+

+

+
+
+
+

Every Tuesday and Friday, Ezra Klein invites you into a conversation about something that matters, like today’s episode with Patrick Collison. Listen wherever you get your podcasts.

+

Transcripts of our episodes are made available as soon as possible. They are not fully edited for grammar or spelling.

+
+
+

EZRA KLEIN: I’m Ezra Klein. This is “The Ezra Klein Show.”

+

This is a great conversation today. But it’s a tricky one to introduce, because the guest I have — I’m not having him on for the thing he’s best known for. So Patrick Collison — by day, co-founder and C.E.O. of the multibillion-dollar payments company, Stripe; by night, by weekend, I think, one of the most important thinkers now in Silicon Valley — certainly, one of the most quietly influential, someone who is forging and traversing an intellectual path that a lot of other people are now following.

+
+
+

And it’s this second incarnation and role that I’m really interviewing him in today — the soft power side, I guess, of Patrick Collison. Collison’s work here centers around this question of progress. The argument is that human progress is much more precious and rare and fragile than we realize.

+

We maybe take it for granted. We live in this time when things have been changing, atop decades and decades, even centuries and centuries, even millennia now, when things have kept changing. But for most of human history, that was not true. It was not true.

+

There just was no market rapid advance in human living standards. It’s only in the past 10,000 years, and then practically in the past few hundred — just an eye-blink in the time human beings have been on Earth — that things kept changing, usually for the better. And the question is, why?

+

And Collison’s particular meta question is, given the clear fragility of forward motion here, given how rare it has proven to be — and so how easy it might be to lose — why isn’t the question of the conditions of progress more central? Why isn’t the study of progress in a wide multidisciplinary way a more common and central discipline?

+

Collison has written a few influential essays here, with the economist Tyler Cowen. He called for the inauguration of a discipline — they call it progress studies — and that now has people studying it. There’s people creating journals for it, creating syllabi and podcasts and books around the topic. It’s one of the more singularly successful calls for a research direction I have seen.

+
+
+

Separately, in a piece co-authored with the scientist, Michael Nielsen, Collison and Nielsen argued that, though it is hard to measure, it seems like the rate of scientific progress is slowing down, and that’s particularly true if you account for how much more we’re putting into science, in terms of money, of people, of time and technology.

+

Now, these ideas are not original to Collison. The point is not that nobody studied human progress before this or worried about the pace of scientific research. He wouldn’t claim that. It wouldn’t be true. But he is playing a distinctive role in their framing and their popularization, and in creating and funding a community around them.

+

And what I see in my travels here is that it is working. Something is burbling here. But I can’t find many big pieces where Collison really lays out his worldview. There are a couple essays, tweets, interviews, but he’s not been primarily writing this down.

+

What he has been doing is funding it through Fast Grants, which has been successful, but more than that, intellectually influential effort to show you can give out scientific grants quickly and with very little overhead, through the Arc Institute, a big biotech organization he’s creating to push a researcher-first approach to biotech, and through giving a bit of money, and a bit of time, and a bit of prestige, and a bit of networking to a lot of different projects that circle these questions.

+

He’s got this funny quality of being nowhere in particular, but also somehow, almost everywhere, if you’re interested in these questions. So what I wanted to do in this conversation was try to get as close as I could to the Patrick Collison worldview, the underlying theory of the case here that animates his thinking his funding, and the ways in which he’s trying to nudge the culture he’s a part of, or the ways in which he’s trying to actively create a culture he doesn’t yet see.

+

As always, my email — ezrakleinshow@nytimes.com.

+

Patrick Collison, welcome to the show.

+

PATRICK COLLISON: Great to be back.

+

EZRA KLEIN: So you’ve made the argument that science — all science — is slowing down, that we’re putting more money and more people into research, and we’re getting less and less out of it. Tell me about that.

+
+
+

PATRICK COLLISON: Well, I want to separate two things. There’s a question as to whether science in its totality is slowing down, in terms of the absolute returns from it. I think that might be true. You can maybe divide up the first half of the 20th century and the second half and so on, and sort of try to compare one with the other.

+

And we had general relativity and quantum mechanics and various other major breakthroughs in the first half. You can ask the question of, well, did we have as many in the second half? But in the second half, we did have the discovery of D.N.A. and molecular biology and lots of other things.

+

So I don’t know that I would claim a total slowdown. The thing that I think is clearer and should be very concerning to us is, as you look at the number of scientists engaged in the pursuit of science, and if you look at the total amount that we’re spending, and as you look at the total output, as coarsely measured by things like papers and number of journals, all of those metrics have grown by, depending on the number, let’s say, between 20 and 100x between 1950 and, say, 2010.

+

And if you look at it on a per-capita basis, or a per-unit-of-work basis, now used to divide all those total outcomes by a factor of 50, and it seems like if you imagine yourself as the median scientist, you’re meaningfully less likely to produce anything like as consequential a breakthrough as you would have, say, in 1920. And so Michael Nielsen and I, in order to try to put slightly more rigor on that question — we went and we surveyed a bunch of scientists across a number of universities in a number of different disciplines, and we presented them with different Nobel Prize-winning breakthroughs.

+

And we tried to compute an approximate ordering of their significance in the eyes of these scientists. And the thing that would kind of have to be true — for the per-capita impact, we remain in constant — is we’d have to be discovering much more important things in the latter half of the 20th century in order to compensate for, to make it worthwhile, for us to be investing this 50-fold greater effort.

+

And we didn’t find that. In physics, in the estimation of physicists, there was a kind of flat-to-declining trend. It’s not super obvious which way it points, but in as much as there’s a trend visible, it’s probably slightly downwards. And in other fields, it was maybe similarly equivocal, perhaps a slight increase, visible in some, but importantly, in no fields that it looked like we’re on this crazy, exponentially improving trajectory, which is what you would have to have for this per-capita phenomenon to not be present.

+

And I think that should be something we’re interested in for multiple reasons. One, because presumably, as a society, we’re interested in just how much more scientific progress and technological progress and so forth, how much more innovation is there going to be over the next 10 years or the next 50 years or the next century. But also, because there’s kind of two possibilities. One possibility is, fundamentally, we’re running out of low-hanging fruit, and it’s just going to be harder to do this stuff.

+
+
+

And in as much as we’re setting investment or making investment decisions around to what degree should be pursuing the stuff, I guess it’s important to know what we think the returns should be. Or the other possibility is, somehow, we’re doing it suboptimally. Something changed, and we were pursuing this process of discovery more effectively in the past, and presumably, for inadvertent reasons, something went wrong, and now, we’re just less efficient at it.

+

But either explanation — and it doesn’t necessarily have to be fully binary — but either explanation is important, and either explanation, I think, has prescriptions for what we should do going forward.

+

EZRA KLEIN: Let me start with the low-hanging-fruit explanation, which I think is a more popular one. And you have — in the piece you did on this with Michael Nielsen, the sad, but in the very academic way, very funny quote from the physicist Paul Dirac, who says of the 1920s, there was a time when, quote, “Even second-rate physicists could make first-rate discoveries,” which I just kind of love.

+

But the theory there is you can only make a lot of the big discoveries once. You discover quantum mechanics once. You discover the atom once. And most of them have just been made, so what you have now is more complicated, smaller, requires much larger teams of people, much more complicated experiments, with much more infrastructure.

+

So we’re just structurally in a period where it’s going to get harder and harder and harder to make big gains. Do you believe that?

+

PATRICK COLLISON: I think it’s possible, but even though it’s intuitively compelling on some level, I’m not sure that it’s true. It’s probably true to at least some degree for some particular research direction, right? We go after discovering the various subatomic particles, and initially, without too much difficulty, we discover the electron or whatever.

+

And by the time we’ve discovered the nth quark, it’s now gotten super hard, and even with ever-larger particle accelerators, we’re not necessarily making breakthroughs of the same magnitude. So I think it’s pretty true for a given direction. But obviously, the question is, well, to what degree is progress in any area opening up other directions, right?

+
+
+

And so I mean, you mentioned the Dirac quote and, say, physics in the early part of the 20th century. Those discoveries opened up new techniques and investigation methodologies and so on, that then gave rise to molecular biology in the ’50s, ’60s and ’70s. And so there’s kind of a combinatorial benefit, where discoveries over here or discoveries over there might unlock opportunities and major breakthroughs in areas that we could not have foreseen in advance.

+

There are lots of, quote unquote, “low-hanging-fruit discoveries” made in computers and computer science in the ’70s, ’80s, and ’90s. Maybe we’re even still in that regime, right? We’re still making some pretty fundamental breakthroughs. And of course, again, those, quote, “low-hanging discoveries” would not have been possible without a lot of this optimization and discovery in other fields.

+

And so I think it’s probably true for a given research direction, but the relevant question for society is, is it true in aggregate. And there, it’s much less clear to me that it is.

+

EZRA KLEIN: I want to read something provocative you said in an interview with the economist Noah Smith. And you said, quote, “Most systems get worse in at least certain ways as they scale. The idea that science could have gotten worse in significant ways sometimes sounds strange to people. Like, we’re doing so much more. How could that be bad? But I think that misses the many examples of sensitivity of scientific processes to institutions and culture. Swiss nationals have won more than 10 times more science Nobels per capita than Italians have. 10 times. And yet, they’re neighbors. And Italy certainly isn’t lacking in scientific tradition — Fermi, Galileo, the oldest university in Europe, et cetera. The ‘how’ of science just really matters.”

+

And this seems, to me, to be where your exploration really goes. So tell me what you think might have gone wrong in the “how” of science.

+

PATRICK COLLISON: So I think this point about the sensitivity of scientific outcomes to the specifics of the institutions and the cultures is very important and probably underappreciated. At the beginning of the 20th century, not only was the U.S. not a scientific powerhouse, but it barely had a presence in frontier research, whatsoever.

+

To become a credible researcher in the U.S. in 1900, you almost certainly had to go and spend time in, most likely, Germany, and failing that, in France or England — you know, what have you. And by 1900, the U.S. was already a pretty prosperous place, and it had a well-educated society, as societies went.

+
+
+

And yet, somehow — and it had universities, right? I mean, Harvard was hundreds of years old by that time. And so it checked many of the ostensible boxes, and yet, the sum total of the U.S.’ research output as of 1900 was still de minimis.

+

When James Conant, who was later president of Harvard for 20 years — when he went to Germany as a chemist, which was his original training, in the 1920s, he recounts how dispirited he was by what he found there and how far ahead of Harvard German research was, as of the early 20th century. And then, for a variety of reasons, all sorts of cultural, institutional funding — various transformations happened. And of course, by the latter half of the 20th century, the U.S. was the unquestioned leader at the frontier of scientific progress.

+

If you look backwards, you see where that locus has been, where the most successful and fertile scientific grounds have been — it has repeatedly moved. As we just said, maybe the 19th century, it was Germany.

+

Before that, in the 18th century, it was plausibly France. They had a couple of these really successful École Polytechnique and Grande École and so on. And so as a kind of first-order empirical matter, we can just notice, huh, this really seems to matter — and then, the example you just gave of the divergence between Switzerland and Italy.

+

And so then, if we kind of accept that, and we try to ask ourselves, well, specifically, what are the mechanisms? You know, what’s actually going on? It’s hard for me to say. It seems like the transmission of research culture by individual researchers matters a great deal.

+

And you see these kinds of pockets of the cultural transmission repeatedly crop up, where Gerty and Carl Cori — you probably haven’t heard of — they ran a little biology lab in Missouri, and no fewer than six of their trainees, of students they trained, went on themselves again to win Nobel Prizes.

+

And if we tell ourselves a standard kind of mechanistic story as to, well, it’s the funding level, it’s how much are we investing in science, or it’s something about whether there’s an institution in the courser sense, that can possibly be amenable to it, it’s very hard to explain these eddies where you see these pockets of excellence really produce these outsized returns. So I think it’s a complicated question.

+
+
+

I think all of aggregate culture, funding, institutional characteristics, and so on all contribute to it. But if I had to isolate a single variable, it seems to me that the research culture set by specific people and the tacit knowledge transmitted through direct experience is probably the number-one thing.

+

EZRA KLEIN: This, I think, is where I sometimes fall into my own pessimism on this. Because I want to believe, as you do, that we can double the rate of scientific advance, maybe even go further than that. But I think the prediction — if I’m putting this on institutions, on culture, on pockets of transmission and mentorship — I think the prediction I would make is then, even if you believe, say, that America had a great 20th century, but its institutions have become sclerotic, and we’ve slowed down, and everything is piled in lawsuits and review boards now, somewhere else that didn’t have that, that has a different culture, that has different institutions, would be pulling way ahead.

+

So you might think, well, China will be pulling way ahead. And you’ve noted this in some places. We’re getting a lot of peer-reviewed research out of China — huge number of citations out of China. We’re not seeing them dominate the big breakthrough advances of the era.

+

It doesn’t seem like Europe is lapping us. And so if you think this slowdown is somewhat global, then that seems to me to militate against questions of individual institutions, cultures, how different labs work, because there is so much variation that you should have some of these labs that are doing it right, some of these places that haven’t piled on a little bit too much bureaucracy. But I don’t think we really see that.

+

PATRICK COLLISON: This diagnosis of these phenomena to cultural, institutional, mentorship-related, interpersonal dynamics, and your observation that it’s not obviously the case, that there are other places we can pointed that are doing it so much better — for me, my takeaway is that, well, successful cultures are a pretty narrow path. Homo sapiens emerged 200,000 years ago.

+

And as far as we can tell, for the first 190,000 years of our genesis, we think we were largely biologically equivalent to the people we are today. But as best we can tell, there was some kind of cultural capital that those people lacked for a very extended period of time before human societies in somewhat recognizable modern form started to emerge — agriculture, all the rest.

+

And in a similar vein, we had many billions of lives and centuries elapsed before the Industrial Revolution., and before we started to put together many of the input ingredients or enough of the input ingredients that we can get sustained improvement in standards of living and ongoing economic growth and progress. And so your point about, well, as I look around, I don’t see anything or anywhere that’s obviously better, I agree with that.

+
+
+

But again, my takeaway is that that’s what makes the question of how do we improve or how can we do somewhat better so urgent and pressing, where it’s many things have to go right. It’s not easy to be even as good as — or to get to a place where things are as good as they are today. What we have is very precious. And I think the threads and the themes that you’ve been pulling on of late — all of these dynamics underscore their importance.

+

EZRA KLEIN: I think that’s a good bridge to progress studies as an idea. And I want to have people hold in their heads that idea that progress is very narrow, that it is a very narrow bridge that we have walked on for a very short period of time. But let’s try to define it.

+

When you say progress here, what are you actually talking about? Is it just shorthand for economic growth or G.D.P.? What is progress?

+

PATRICK COLLISON: Well, I don’t know that I would claim to put forth some kind of definitive definition. And I think, to some extent, our intuitions around it are probably broadly correct. And so it might not matter to define it super precisely and finely.

+

For, me it is something along the lines of our success in realizing a liberal, pluralistic and prosperous society, and a sense among people that their offspring can and probably will do better than they themselves have, and that more broadly, the future will be better than the past, and that we’re at least making incremental progress towards embodying values and morals that we collectively think we can be proud of.

+

But I don’t think anything that novel in that. I don’t think my conception of progress would differ that materially from some kind of average aggregate over any other group of people in the country.

+

EZRA KLEIN: I do think there’s something interesting, though, which is that if you look at eras that I think progress-studies-type people and economic-growth people and historians of economic growth study most closely, actually, some of the periods where people feel a lot of rapid progress don’t fit that at all. You have, say, the Industrial Revolution, where life spans and lifestyle get worse for a lot of the people.

+
+
+

I don’t think one will look at that period as unbelievably pluralistic. You have a lot of periods of war when you have very, very, very rapid technological progress, but it happens in context of much more martial societies. So there is an interesting tension, at least in periods — and some of them quite long, actually — where you can have fairly rapid economic progress, but it comes at a cost that I think isn’t always acknowledged, but is an important thing to think about.

+

PATRICK COLLISON: Yeah. So I don’t think you could point to some of these periods in the past and say that they definitively embody to the extent that we would fully aspire to some of these broader traits and characteristics. But I think the question is more, what are they doing as — you have to judge it relative to the baseline that preceded them.

+

And I don’t know that the 18th century in the U.K. is some ideal as a society. But if you compare it to the 16th century in the U.K., the ideals and ideas of natural rights and religious tolerance and so on — they were somewhat better embodied by the 18th century than they had just a couple of centuries previously.

+

And similarly, in the U.S., say, during either war or the ’30s or whatever, again, it’s not like that was any kind of perfect society, but assessed relative to the society of 1830, I think it compares relatively favorably. And I think it’s not a coincidence that Adam Smith — his first book, of course, was on ethics and morals and trying to instill better general ideals and behaviors across a society.

+

And maybe after that, he then argued for and laid many of the foundations of what we would recognize as modern economics. So I don’t think it’s perfect. But on average, I think the correlation is positive.

+

EZRA KLEIN: So let’s talk about the Industrial Revolution for a little bit here. I think a lot of people locate a takeoff in human living standards — it continues to this day — there. And it’s strange in a way, right? “There” is a very geographically contiguous spot. It’s the U.K. — England, actually, I should say, at that point.

+

And there is a moment in time that probably could have come at another moment in time, depending on how human history plays out in the counterfactual. I know that you have an interest in the theories of why then, why there. How do you work your way through them? What do you think is persuasive for why then, why there?

+
+
+

PATRICK COLLISON: Well, you know, again, I caveat. With all of these topics we’re discussing through this podcast, maybe the first-order banner for all of them should be, I don’t know, these are my best guesses, and I think it’s important that all of us were pretty humble in the claims and the assertions and the beliefs that we hold.

+

Recently, I’ve been reading a bunch of Irish and Scottish writers around then. It’s very interesting, because for both the Irish and the Scots, there was a sort of a pressing and kind of obvious question where England was much more prosperous than they were or we were. And there’s no super obvious explanation for that. There wasn’t an obvious climatic or natural resource endowment that England benefited from that was lacking in Ireland or Scotland.

+

It wasn’t like England was actually a vastly larger polity. The orders of magnitude were comparable. And Bishop Berkeley wrote this book, “The Querist.” He was asking these questions directly, just like, what’s going on? What’s wrong with Ireland? You know, why can’t we do this?

+

And then, you have the Act of Union in 1707, uniting Scotland and England — and sort of similarly, of all these Scottish thinkers being like, all right, we’re now literally the same country. Why are we so much more impoverished? And then, if you shift to England, there’s Joel Mokyr and — you’ve read his work — and more recently, people like Anton Howes.

+

And in a similar vein, they go back to — I mean, the word, improvement, came from Francis Bacon, or it was kind of popularized as a concept by Francis Bacon. But that’s noteworthy, right? Like, that was not a pervasive broad concept in the 15th century.

+

I mean, literally, the word, improvement, in this broader societal context, came from word, “translated,” at the beginning of the 17th century. And the ultimate conclusion that these historians and scholars and analysts of the Industrial Revolution come to — and I think it’s a correct one — is somehow, whether it’s through Bacon or Newton or various of the tinkerers who produced some of the earliest technological breakthroughs, that somehow, this improving mind-set became pervasive.

+

You had societies explicitly — like the Hartlib Circle or the Lunar Society, or the Select Society, and the club, and so on — all these societies explicitly devoted to figuring out ways to advance the state of affairs that prevailed. And these societies were comprised of many of the leading people and thinkers and so on of the day.

+
+
+

And it seems maybe a bit satisfyingly squishy to attribute it to something so hard to pin down. But as you run through all the possible other explanations, it’s differences in IP law. It’s difference in the Malthusian conditions. It’s difference in the prevalence of coal, you know, et cetera, et cetera. Through various cross-sectional analyses, you can exclude most of these in looking at all of Ireland, Scotland, and England.

+

It really does seem to me that differences in the mind-set and in the culture are where you have to net out. And that’s not to say maybe that it’s fully sufficient. There might be other preconditions that are important. And then, maybe as a last thing to say, it is striking to me that many of these kind of original 18th-century economic writers and thinkers — and again, the kind of people we look to as the founders of much of the discipline — that they themselves were kind of centrally preoccupied with this.

+

And yeah, they were in favor of free trade and specialization and human labor and lots of these concepts that we’re now very familiar with, but they really thought that general mind-set played a big role, too.

+

EZRA KLEIN: So let’s talk about Joel Mokyr ideas for a minute. So Mokyr is an economic historian. People should read his book, “The Culture of Growth,” which is really fascinating. He argues, as you’re saying, that in this period, this mind-set that we can increase the store of usable knowledge, and then use it to alter nature, to better the human condition, takes hold.

+

That’s a new mind-set. It’s different than religious ideas of the past. It’s different than cultural ideas of the present. And that, plus a bunch of other things, particularly the republic of letters, the way people are writing letters back and forth, kind of combine into a culture that is able to grow.

+

But one of the things that I really take from his work, that sits in my head, is he believes it’s all very contingent. He really believes it might have not happened. But the other is that I think it opens up this question that as a tech person, I’m curious to hear your thoughts on, which is, he really believes — Mokyr really believes — that there is a communications infrastructure that arises at that time, that has a kind of culture of generosity and argument and honesty in it, and is built on writing letters slowly to one another, and then copying those letters over to other people.

+

And that culture is really good for intellectual advancement. I think one of the promises of the internet and the age we live in is, it’s all faster. We can write to people immediately. Things we write can go viral and be seen by 5 million people all of a sudden.

+
+
+

And that was going to speed up economic growth really, really rapidly. And I would say, you don’t see that. So I’m curious how you think about communication cultures here and what you think for all the advantages of ours we might not have.

+

PATRICK COLLISON: I mean, I think it’s hard to say in aggregate. I feel it’s pretty likely that the effects are very heterogeneous across different populations. And you’ve made the case that you think Twitter is bad for journalism and for journalists.

+

And I guess you live this yourself with your now mostly inactive Twitter account, I guess, apart from announcements. And I think in the case of the internet, that it’s almost certainly a tremendously large gain that billions of people now have access to educational materials. And some of the otherwise hard-to-communicate tacit knowledge — that things like YouTube videos now made legible and available.

+

And I think it’s true that there are various gravity equations that we see across different disciplines. I mean, in economies themselves, in trade, where you rapidly decline in propensities to trade as countries get further from each other — but you have versions of this in academic disciplines as well, where geographic distance correlates inversely with likelihood of the exchange of ideas and so on.

+

And I think it’s clearly the case that the sort of reaction surface area has increased substantially by the internet there and represents a kind of efficiency gain for people looking to exchange in ideas. Many of the companies that Stripe works with are remote companies, and they might employ people across myriad countries, and that’s a kind of communication and efficiency gain that would certainly not otherwise be achievable.

+

I think it’s worth recognizing that the aggregate amount of G.D.P. that we are creating or gaining every year is so much larger now than — I mean, the percentage might be the same. But the total amount of stuff happening, or the increasing amount of stuff happening, is so much larger now than it was 100 or 200 or 300 years ago.

+

And so for all of those reasons, I think we should give superior communication technologies and faster communication technologies a significant amount of credit, even though the ways in which those are manifests might be hard to measure and somewhat prosaic.

+
+
+

Take my mom, for example. My mom works with a hospital in Minnesota. Our youngest brother has a physical disability. And in the course of that, she trained herself in treatment for cerebral palsy, this condition, and she wrote a book about it, and she did a master’s in this. And now, she’s trying to improve treatment for this condition throughout Ireland, in the U.S. and other countries as well.

+

She’s a retired Irish mother who spends some of her year living in the U.S. near her sons, spends the rest of her year living in Ireland, working at a hospital in Minnesota, who just got a proposal to have her book translated into German a couple of days ago. And that’s a relatively prosaic story, but literally, millions of these stories exist in kind of aggregate form around the world.

+

To circle back to the initial thrust of your question, though, I think it’s at least possible that the internet is bad for civic discourse. I’m not saying it is, but it’s certainly in the realm of plausibility — and that perhaps both things are true, where there’s some kind of iceberg where there are these enormous welfare gains that are not that legible, not that visible, lie beneath the surface, and then certain of the most visible manifestations, like what we see on cable news or what we see written in the papers — perhaps that is worse, and perhaps, slightly more structural judiciousness would be desirable there.

+

EZRA KLEIN: I want to try to flip that and suggest that — because I’m going to push some counter ideas on why we maybe don’t see as much progress as we wish we did. But one is that I think possibly, very large welfare losses lie beneath the surface. And beneath the surface of stories like the one you just told about your mother, I think we all have stories of ways or people for whom the internet has unlocked a possibility.

+

I mean, my whole career is built on the internet. I was an early blogger. I got rejected from my student newspaper. And if there was no blogging, like, god knows what would have happened to me. [LAUGHS] I mean, nothing too terrible, probably, but I wouldn’t have the career I have today.

+

And at the same time, I think that the group of people who, by luck or by temperament, proved very, very good at using the internet, to some degree, distracts from the many, many, many people for whom the internet is fundamentally a distraction machine, or for whom the internet is creating, because of what we built on it. You know, shorter attention spans — how many people would have had an idea, sitting in a room by themselves, or taking a walk, that they never have now, because they never have to have a moment where they’re thinking alone?

+

And so one thing that I think we’re all loathe to do is we’ll talk a lot about how it’s weird that we have so much more knowledge, but productivity isn’t increasing faster. It’s weird that we have so much more rapid communication between researchers, but science isn’t advancing faster. And then, the idea that maybe there are things happening to us that makes us less able to use that increasing stock of knowledge well, or makes us less able to collaborate in a useful way, I think, gets dismissed rather quickly.

+
+
+

But I don’t think it’s totally implausible. Now, I don’t want to say, like, the greatest technology we ever had was letter-writing. Obviously, the greatest technology we ever had was blogging in the early aughts when I became a blogger. And whatever happened in your 20s is, like, as good as it was ever going to get.

+

But I do wonder about these questions. And I think something Mokyr is right to put a lot of attention on is communicative cultures. Communication is how we collaborate. And if communication is in any way getting worse, it’s going to have pretty big macro effects.

+

The other thing is if you believe these cultures matter, weirdly, as big as we’re getting, the internet allows a certain disciplines culture to stretch boundaries and borders in time in a way that it would have been harder. I suspect that labs were more different 50 years ago than they are today.

+

The countries and the disciplines of researchers and the cultures of researchers in countries or cities are more different from each other 50 years ago than today, which is great if we have the best of all cultures today, but it’s not that great if you actually think variation is really important.

+

PATRICK COLLISON: Let’s wrap up there. So first, I agree, as a basic matter, that there are welfare losses occurring across society that we should be worried about, and probably everybody listening to this is familiar with the Stephen Pinker case for optimism, and rather than focusing in the headlines, you zoom out, look at these long-term time series. And once one does that, things seem a lot more encouraging, whether you look at it by income or life expectancy or infant mortality or choose your metric.

+

Something that’s been striking to me of late is if you change the x-axis on those time series, and look at many of those phenomena and trends over a much shorter window, the valence changes substantially, and life expectancy in the U.S. is now, in fact, declining. According to C.D.C. data, 54 percent of teenage girls now report persistent feelings of sadness and hopelessness. And you could say, well, teenagers were never stereotyped as the most cheerful lot, but we do have some degree of longitudinal data here, and that number is up from being in the 20s as recently as 2009.

+

½ the population now is either prediabetic or diabetic — again, according to the C.D.C. Basically, point is, when we look at more recent windows, I think there are plenty of aggregate, emergent, complicated outcomes and phenomena that should give us concern. On the degree to which we should attribute the diagnosis to the internet or to our kind of communication media more broadly, it’s less clear to me in that — not saying it’s not true, but presumably, the life expectancy one is not — or at least if it is, the mechanism has to be very complicated.

+
+
+

There are a bunch of other health-related ones. So take, for example, say, the incidence of diabetes or pre-diabetes. And you could say, OK, fine, all those things might be true, but they’re totally different. I guess the question I wonder about is, well, we know that lots of basic biological outcomes are correlated with mental states and so on.

+

And so to what degree is there some more nuanced and complicated relationship there? But I think it’s a fair question, and I wonder a lot about it myself.

+

EZRA KLEIN: Let me ask you about how you think, over the long period here, about the relationship between technology and equity or egalitarianism. And something specific is in my mind. I flicked earlier at the way the Industrial Revolution, for an extended period of time, seems to have reduced a lot of people’s living standards. And it wasn’t till later you had changes in redistribution in labor unions and labor protections that the amount of material prosperity that was generating created more broad-based prosperity, particularly at a very high level.

+

I don’t know that you can sustain that kind of thing today. We have much more a small-d democratic culture. If things aren’t working for people, it’s much easier for them to organize and be heard.

+

I think there’s a much more direct and complicated relationship now between whether or not people feel benefited by technology, and whether or not they are going to accept the conditions and the risks of rapid technological advance. But I’m curious, from your vantage point, how you see that both kind of historically and currently.

+

PATRICK COLLISON: I agree with that. I worry a lot about the basic stability of a society that does not successfully generate and make sufficiently broadly accessible the benefits of economic growth. The world simply has too little prosperity. And if it is not the case that people in the U.S. or people in any country — if they either feel like things aren’t progressing, or if they feel like maybe somewhere distant from them, things are progressing but they personally will never be able to benefit from it, I think we put ourselves in a very dangerous and likely unstable equilibrium.

+

And if you go back to — well, you don’t have to go back very far in history to see, obviously, plenty of instances where this kind of instability brought the whole house of cards down. And so as a consequence of that, I worry a lot about, how do we simply make sure that — or one of the small things we each individually can do to try to make sure that society is generating enough economic gain and enough broadly experienced welfare gain that the whole compact can be maintained?

+
+
+

And again, I don’t think there’s a ready neat kind of singular answer to that. Maybe Stripe as part of our small little contribution in one little fissure. But I think the central question you’re getting at is super important.

+

EZRA KLEIN: And one of the questions I wonder about there — we’ve talked about the way progress has been very geographically lumpy, let’s call it, right? There’s a lot that happens in very small places, and it ends up affecting the whole world. Obviously, then, the gains of progress sometimes have that quality, too.

+

And I do think of one of the politically destabilizing effects of the past, let’s call it, 30 or 40 years of digital progress, is being the concentrations of wealth. We just used to have a lot more spread. Even putting the questions of rising inequality aside, just where rich people were was different.

+

And so where they were giving a lot of money to the local hospital was more spread out, say, across the country or in other countries across the land. But here, even as the internet is supposed to democratize distance, and in many ways, has — I mean, telework is not a fake phenomenon. It has really concentrated the wealth of that to, literally, where we’re sitting, but to New York. There’s a lot of money now in Austin.

+

And then, on top of that, you often have barriers of entry, in terms of how many homes can be bought. So it’s not even like people can move to the place where all the economic opportunity is happening. And I do think that creates some of the skepticism you see of technology.

+

I don’t think a lot of people’s — I think people are really excited about a lot of the goods they’ve gotten from it. But in this kind of macro political sense, as you’re saying, in a period of a lot of change, a lot of folks with real backing in the data don’t feel life has gotten better at the macro level.

+

Life expectancy, happiness, political stability — it’s not like you can look around and say, well, I got this computer in my pocket, and everything else is going great, too. It’s like, I got this computer in my pocket, and what it keeps telling me is that everything is going to hell. Now, maybe it’s telling me that a little bit too much, but there is validity to the narrative.

+
+
+

PATRICK COLLISON: Yeah. So again, vehement in agreement on the sort of central importance of making sure that improvements in the standard of living are actually broadly realized across the society. That, too, I think, could serve as a manifesto for some of these Progress Studies ideas.

+

On the internet in particular, or on technology and the technology sector and so forth, I think it’s complicated and difficult to try to sort of fully collapse or linearize it or something, where on the one hand, you have some of these concentration dynamics you identify. At the same time, of course, it is also a tremendous and incredible dispersal agent in making some of those possibilities and opportunities be more broadly available.

+

I think it’s dangerous to take an excessively U.S.-centric perspective here. If you interact with or look at survey data, or otherwise try to assess what’s the sentiment of people in Poland, what’s the sentiment of people in India, or what’s the sentiment of people in Indonesia, they view the internet extremely positively. And I think correctly so, where their opportunities for advancement would be substantially curtailed in the absence of much of what the internet makes possible.

+

I think in places like the U.S., or actually, even at home in Ireland, some of this story is complicated by lots of other things, but including — and I think, substantially — dynamics around housing policy, where there’s an extensive literature showing that, for example, a very substantial moderating effect on the increase in wealth inequality that would otherwise have happened, or income inequality, was geographic reallocation and people responding rationally to, OK, things in New York are going great, or things in California are going great.

+

And if you look at the rate of increase of the Californian population, say, through the 1960s, that was a tremendously potent mechanism for us redistributing some of the economic gains that were being realized at the time. And of course, now, we have this crazy position, where California is losing population at the same time where the market caps of these companies and the profits of these companies are increasing very rapidly.

+

But as one assesses that dynamic and tries to ask the question of, well, why aren’t these gains being better or more broadly distributed, it’s certainly not clear to me that the answer even lies in the realm of technology qua technology. And I think it’s certainly more broadly, again, some of these considerations like geographic allocation.

+

EZRA KLEIN: Let me ask one more question on the geographic dimension, and then I’ll move on to it. And it’s on my mind, in part because when I try to think about progress, when I try to think about what inventions and innovations are coming really quickly, I actually see a bunch here. And I’ll use A.I. as an example.

+
+
+

I’ve met people who are trying to automate a bunch of legal contracts. It makes a ton of sense. People pay a lot all over the country — to some degree, all over the world — to get fairly basic legal contracts drawn up — wills and real estate documents and merger agreements and all kinds of — from the small to the large.

+

If you imagine that getting really effectively automated, though —

+

you think about Saint Louis, Missouri, where some of the people who are important pillars of the community work in law firms there, and what they do is contracts. They do estate planning and all the things that people have to do in contracts. And if it actually does get concentrated to really, really great contracting firms in the Bay Area or in New York, on the one hand, the democratizing potential will really be realized. Those contracts will get cheaper.

+

There’s also a theory in crypto of smart contracts. And on the other hand, you really will have a lot of that — the gains of that, economically, going to smaller areas and aggregated across a bunch of different domains. So graphic design, in all kinds of areas of the country — midlevel graphic designers get paid to make logos for local businesses.

+

It’s pretty clear they’re going to be able to do that really, really easily on things like DALL-E pretty fast. So you can imagine a lot of that area getting wiped out. And you kind of run through a couple of these. And before you get to really unbelievable and sci-fi-like dimensions of artificial intelligence, you just have a thing that is going to democratize a lot of capabilities in a way that’s going to put the money for those capabilities both a little bit back into the pockets of the people who need them, and then a lot into the people who run the best A.I. rigs and is going to have a really weird geographically destabilizing effect.

+

And that paradox of the internet both democratizing geography, and then concentrating wealth and capital in very small areas is, to me, a central challenge. Because if you get that wrong, if it goes too much in the concentration area, I think we’re going to lose a lot of the political stability we need here. But you’re more on top of these technological advances than I am. Do you think the trends there are going to play out differently than I’m worried they will?

+

PATRICK COLLISON: First, yeah, it’s not — I don’t think it’s foreordained whether or not these are going to be centralized technologies. I don’t know. And on the one hand, there’s, I think, an obvious feature we can contemplate, where there are only three A.I. models, and they are rooted in the hegemons, the citadels of Silicon Valley technology, and we all are digital serfs who are subsistence-farming on their gains.

+

I think there’s also a very plausible story where these technologies prove substantially less defensible than we might have expected, and where, instead, they have this enormously decentralizing effect. Because otherwise, economies of scale that only large firms could benefit from can now be realized and pursued, even by massively smaller firms. And if we look at the recent history of A.I. — I don’t think any clear story there, but it does feel to me that it has been more biased towards the second story than the first.

+
+
+

There are now multiple companies with large language models. There are a number of very successful open-source A.I. efforts. Actually, there was a really cool example from Replit, which is a service — it’s a programming I.D. in the browser, used by kids learning to code, but also increasingly used by people who are pursuing serious programming.

+

And they recently released a GitHub copilot-like technology, where it will kind of autocomplete your code in the editor, and where you can do some pretty cool things. Like, you can highlight a block of code and ask it to be explained, and it’ll turn code into natural language, into English, and say, hey, here’s what this code is doing.

+

Anyway, they wrote a blog post about how they built this, and they describe how it was built by one guy over the course of a couple of weeks. And maybe that’s only the case in the early days of this AI technology. I mean, in early computer games, the first games were built by a single heroic person, and now, it’s these gigantic studios and enormous CapEx budgets.

+

And so I think the fact that this is the case today doesn’t mean that it will remain the case through time. But my takeaway is that at least not foreordained that AI or any of these other technologies will be centralizing forces. And then, the other thing to observe is that when we talk about these being centralizing, I think there’s a question as to, do we look at it in relative or absolute terms?

+

And my contention would be that, both from a moral standpoint, but maybe more importantly from kind of a political-economy standpoint, what will matter is whether, on an absolute basis, people feel like they are realizing opportunities, their lives are improving, that things are getting better, that their kids will be in a better situation and so forth. And exactly how much value is realized by the companies themselves doesn’t actually matter that much, compared to that former question.

+

And whether A.W.S. or whether any of these organizations has super high or super low profit margins, I don’t know is nearly as important as what is the actual effect on these communities and individuals across the society. And so in as much as one means — by centralizing, one means a large share of the profits, I think it is probably a more useful framing to look at it instead in terms of absolutes, and in particular, the absolute surplus generated by the users.

+

[MUSIC PLAYING]

+

EZRA KLEIN: What have you come to believe about the relationship between progress and war?

+
+
+

PATRICK COLLISON: I am somewhat skeptical that war is as conducive to breakthroughs as we might intuitively conclude, or as is sometimes claimed. You’re probably familiar with Alexander Field’s work on the ’30s here. And his basic claim is, the productivity gains we often attribute to the Second World War in the U.S. — like, those foundations actually were laid in the ’30s, and then the first half of the ’40s were a period of decreasing productivity as we massively, inefficiently reallocated our economic resources for the purposes of winning the war, which was probably a good thing to do, but inefficient in narrow economic terms.

+

And he has a new book coming out, I think, next month, that sort of extends this argument into the ’50s.

+

And as one takes stock of the scientific breakthroughs — and so Stripe Press recently republished Vannevar Bush’s memoir, where he takes stock of this.

+

And obviously, you have, say, the Manhattan Project, and that’s a big deal, certainly. But it doesn’t feel to me that had the Manhattan Project not occurred, that peaceful development of nuclear technology would have been massively stymied. Maybe it would have taken another 10 years, but it was already happening to some meaningful extent.

+

And then, as you take stock of all the other breakthroughs that took place in the U.S. during the Second World War, there were some meaningful stuff like blood plasma and blood transfusions. There was some significant breakthroughs there. Some of the first antimalarial medications, radar, the proximity fuse, which I’m not sure is all that useful outside of military applications.

+

But by the time you get down to invention 6 on the list, I don’t know that as you compare that list to, again, some counterfactual of what would otherwise have ensued, that it looks radically better as you take stock of the Cold War and the enormous fraction of our economic resources and human capital that were devoted towards us, that the gains necessarily look that impressive. And so again, it’s super hard to judge. You don’t have proper controls and so on. But I’m skeptical.

+

EZRA KLEIN: Let me take the other side. So there’s a question of, during war, how much did we invent during World War II. And that’s a question of how much the threat of war or the competition with an adversary ends up charging up innovation and convinces us to put resources, both in terms of people and in terms of money, and maybe in terms of institutions, into projects we wouldn’t otherwise have done.

+
+
+

I think there’s an argument, at least, that we went to the moon because of the Soviet Union. It would not have done that for some time. Probably would have eventually done it, but also, who knows?

+

As Derek Thompson, who I’m working on a lot of these ideas with, likes to point out, the Apollo Project was unpopular. It was not something that commanded wide popular support.

+

Even now, if you look at the CHIPS Act that passed, it passed, with all that spending on semiconductor research and other kinds of next-generation technologies, under the framework of, let’s compete more effectively with China. If you look at all the things Darpa has done or been part of, the fact that “defense” is the first word in the Darpa acronym, I think, is meaningful.

+

There’s something about what threat persuades societies to do, and persuades them to do technologically or what risks it allows otherwise-more-cautious governments to take, or what failures they could justify that allows them to have big successes. Something there doesn’t seem to small to me.

+

And maybe it’s my political side, where I so often see scientific funding justified in Congress in terms of countries we’re competing with or are adversaries with. And I see what the defense industry can do that other institutions cannot, because they don’t get a lot of political blowback. But I don’t know.

+

I worry a little bit about how much we seem to need the threat of another to accelerate things. And in a small way, maybe, we see what the pandemic — where we were willing to move much, much quicker on things like mRNA technology than I think we would have outside of it.

+

PATRICK COLLISON: Yeah. So I think it’s certainly true that the crisis can cause the discontinuous shifts that have large effects, which in your example, say, are probably super beneficial. I don’t know. If you take Darpa as an example, it started as Arpa, as a more open-ended research institution and set of programs, and then with the Vietnam War, had the D pretended to it.

+
+
+

And we decided, in the face of threat, to make it more applied, to take more seriously its translational and kind of, quote unquote, “competition-oriented mandate.” And I think that was bad for Darpa. And the internet, which arose under Arpa — it’s hard to think of innovations of similar magnitudes that then occurred in then-Darpa’s subsequent, say, two decades.

+

If you take, say, U.S. science in general, the war — the Second World War — to some extent, the first, but much more so the second — precipitated an enormous centralization of U.S. science in its aftermath. Because we really marshaled together all of the — or a significant fraction of the scientific capacity of the U.S. in service of the war effort.

+

For, example the 50 percent overhead, the fraction of government grants that goes to universities — that was chosen in the early days of the coordination of the war effort, and has now become a kind of a pillar of academic and research funding in the U.S. And in the aftermath of the war, we sort have this question of OK, we’ve kind of pulled everything together. Now, what do we do?

+

And that became, in various ways, the N.I.H. and the N.S.F. and so on. I mean, the N.I.H. predated it, but the growth of the N.I.H. really occurred after the war. And the federal government, shortly thereafter, for the first time, became the majority funder of US science. And all that centralization — and I mean, you pointed out the benefits of variety and of experimentation and of heterogeneity, and having some degree of institutional and structural diversity and so on, I totally agree with all of that.

+

And I think all of that was very meaningfully curtailed by, again, the aftershocks of some of the threats that we faced during the war. And if you think about the things that we’re maybe happiest about having happened — the founding of the major new U.S. research universities in the latter parts of the 19th century or the revolution in health care and kind of medical practice that first happened at Johns Hopkins, and then kind of codified in the Flexner Report, or the great industrial research labs of Bell and Park and so on — or excuse me — Xerox — they didn’t obviously come from a place of fear or a threat.

+

They came from a place of hope and optimism and opportunity. And maybe there are some inventions that you’re more likely to get to from some of these external pressures. But yeah, if you gave me a dial, and I can kind of turn up or down the threat or fear index of society, it’s not super obvious to me that one would want to turn it up if what one cared about was the aggregate rate of progress.

+

EZRA KLEIN: That’s a good bridge, I think, to the question of institutions. And I take one of the main concerns of yours, of progress studies, as being around institutional slowdown. You have this idea that we don’t meta-maintain institutions very well.

+
+
+

They start in one place, and then over time, they crust over, and we don’t really know what to do with that. Give me a little bit of your thinking there.

+

PATRICK COLLISON: I think institutions, the cultures they instill and act as kind of coordination points and training sites for — those of enormous consequence — I think much of the success of the U.S. and of various other Western countries has, in substantial part, been attributable to successful institutions. Most people would accept, I think, that there is, to some extent, consistent trends that tend to happen with institutions through time.

+

Just maybe most basically, the problem that gives rise to an institution in the first place is probably a pretty real and significant problem. No one would have taken the time to found the institution if it wasn’t. And there can be some degree of drift there, where we don’t necessarily decommission the institution once the problem has subsided or abated.

+

And then, you tend to attract a certain kind of person in the early days of an institution — people who are slightly less status and reputation and procedure-oriented, because a new institution almost never has that. And then, through time, the sort of collective or the mission-oriented incentives of the institution can kind of drift somewhat from the individual incentives that particular people are subject to.

+

I think all this stuff exists. And the thing that I observe, or that I just find myself thinking about is, we’ve had eras of institution formation in the U.S. It has not been kind of a constant rate through time. And the New Deal maybe, and say, the 30 years afterwards, and the Great Society — we bookend it with those start and endpoints.

+

That was a period of tremendously active institution construction and formation in the U.S., Darpa being — or Arpa originally being a good example, and indeed, NASA. And I guess I find myself wondering, one, if we didn’t have any of these institutions — and I’m not saying we should get rid of them. But if we didn’t have them, what institutions would we found today, first, and how high in the list would NASA be, for example?

+

And then, secondly, in as much as we accept that some of these institutional dynamics exist, like the fact that sclerosis as an emergent property arises, what do we do about that? And lots of people have told us it’s pretty — doesn’t need a lot of teasing apart to see it as one compares NASA and SpaceX and the respective budgets, and the respective achievements, and so forth, I think it’s hard to not at least wonder about their respective efficiencies.

+
+
+

And say, if society could only have SpaceX or NASA, which one would we choose, and what should we conclude from that, and to what extent do those phenomena generalize elsewhere? I don’t have answers to these questions. But I find that in the political discourse — not that anybody is celebrating that, but in the discourse, it’s very easy to get, I think, very wrapped up in questions of optimal funding levels, and should this number be 10 percent or 50 percent or higher or whatever, whereas to me, a lot of our satisfaction with the outcomes seems to hinge on deeper questions about the nature of the institution.

+

I don’t know that the problem or benefit, or anything good or bad about NASA is attributable to the budget, per se. It seems more, kind of, resonant in some of these deeper cultural questions. And then, in the recent pandemic, or in the — I don’t know. I was going to say, ongoing pandemic. But I guess as of two days ago, with the President’s verdict, it is now over.

+

EZRA KLEIN: It’s over. Congratulations, everybody.

+

PATRICK COLLISON: Exactly. But anyway, I think that was maybe a vivid demonstration of many of these dynamics, where I don’t know this any of the story about the institutional response to the pandemic should be primarily one of funding. I think it’s much more about the dispositions and the attitudes and the cultural biases of entities like the N.I.H. and the F.D.A. and the C.D.C.

+

EZRA KLEIN: I find the NASA SpaceX example an interesting and provocative one. Because on the one hand, I think what you’re saying is completely true. And where a lot of the NASA programs and projects have gone in recent decades, is just — it’s sad. It’s just a sad story.

+

And on the other hand, the idea that you — the thought experiment of choosing between NASA and SpaceX — the thing that it immediately asks is, well, you can’t. Because without NASA, there is no SpaceX. And one way the private sector handles a lot of these questions — I mean, I’m always struck by how much of the way biotech research works is that big pharmaceutical companies acquire small biotech firms that have made a breakthrough or have come up with a very promising candidate. This is kind of an accepted thing that the big companies — they do a fair amount of research, but a major, major innovation transmission there is small groups do more, quicker, and they’re just going to buy them. And the NASA SpaceX example has a little bit of that dynamic to it, although with a different mechanism of financing. And I don’t know. I wonder if there aren’t deeper lessons there.

+

PATRICK COLLISON: Yeah, I don’t mean here in the NASA example — like, I don’t think reducing it to a simple binary of this-or-that is correct. It’s more, what should we make of the differences in these two organizations? And given those observations or beliefs, what do we then think an efficient outcome might look like?

+

And do we think that where we are today — this prevailing status quo — is optimal? Or are there other things we can do better? And maybe an important thing to say within all of this is, to the extent that these are all kind of inevitably determined outcomes, maybe it doesn’t really matter if we think things would be better or worse.

+
+
+

We’re going to end up in the same place, regardless. But I think for all of these, it’s super contingent. And various aspects of both funding decisions and, kind of, the precepts and methodologies of the N.I.H., how we design I.P. law, how we regulate and require and run clinical trials — there are tons of individual contingent decisions that we kind of have collectively made that give rise to the biotech and to the pharma ecosystem.

+

And certainly, in the case of space, you know, like, it doesn’t have to be this way other. Universes, no pun intended, are possible. I think perhaps the thing that people underappreciated with science in the U.S. is, it has been very different in the not-too-distant past.

+

Peer review is a relatively recent invention. Modern journals are a relatively recent invention. As I mentioned, the federal government being the primary funder of basic research is a relatively recent invention.

+

And molecular biology was, in significant part, a thesis by Warren Weaver at the Rockefeller Foundation. There’s a thing here, and we should aggressively pursue it. But importantly, it was not — it required an institution, an organization, that was not part of the standard apparatus, for want of a better term.

+

And the Broad Institute, over the last 25 years, has been enormously successful in the field of genomics and functional genomics and CRISPR, et cetera. And the Broad Institute is itself a kind of structural innovation, breaking somewhat from the more traditional prevailing university model. And so I think the fact that so many of our successes are associated with some degree of structural and institutional change should be somewhat thought-provoking for us.

+

EZRA KLEIN: You’ve been trying to work in the space of institution-building here, too. I want to talk about Fast Grants and about Arc a little bit. So let’s begin with Fast Grants. What is it, and what has it taught you?

+

PATRICK COLLISON: Well, it’s mostly “what was it.” In the early days of the pandemic — well, I should preface all of this by saying — well, I’ll reaffirm my preface that I don’t know, to every question. But more importantly here, I will say, my now-wife is herself a scientist. We’ve known each other since we were teenagers. We spend a lot of time talking about science in various forms.

+
+
+

EZRA KLEIN: You met — am I allowed to say this? You met at a science competition.

+

PATRICK COLLISON: That is true. We met at a science competition, 100 teenagers, and —

+

EZRA KLEIN: And she beat you.

+

PATRICK COLLISON: And yes. I was the runner-up, and she was the winner. I had created a programming language and a new dialect of lisp, and she had created a new treatment for urinary tract infections. And so I really don’t envy the judges for having to figure out what framework one should use to make all these comparisons and lots of other people.

+

EZRA KLEIN: You sound a little bitter, man.

+

PATRICK COLLISON: [CHUCKLES] I was gonna say, but no, we can all agree this the correct outcomes ensued. Anyway, so we were living together in March of 2020, holed up. And a number of her friends and colleagues were unsurprisingly with, I guess, a large fraction of all biology scientists, were trying to urgently repurpose their work to figure out, well, could they do something that would be somehow benefit to accelerating the end of the pandemic?

+

And by early April, so a couple of weeks into lockdown, when it was becoming apparent and striking to us, which was it is difficult for these people to get funding for their work. And that might sound a bit, kind of, surprising, because you think, well, don’t they have some degree of money already? And couldn’t they just go and just spend that?

+

But there are, obviously, significant rules around and restrictions around that which one can do with one’s grant money. This is money provided by the government for a purpose. And so it’s not like you can go and readily spend it on something totally unrelated.

+

And the money is administered by the university, and so you have to go through their proper procurement processes. Point is, lots of restrictions on scientists’ pecuniary ability to suddenly repurpose the research agendas. And we kind of thought, well — we assume maybe in the early weeks, that presumably various bodies — I don’t know who — some kind of amorphous other, some combination of C.D.C., F.D.A., N.I.H., philanthropies — whatever.

+

Somebody will come along and just give these scientists the obvious money that society clearly should, so they can go, and they can pursue these programs. Didn’t seem to be happening. I mean, to be fair, I don’t want to give us too much credit.

+
+
+

Various people were doing things right off the bat in various different places, but we just personally knew of lots of specific examples of really good scientists who were unable to make progress of their work to the extent that they would like. So we tried to set up what we thought would be a pretty small initiative, and called Fast Grants.

+

The basic idea would be, you send us some kind of proposal. And initially, within 48 hours, you would get a funding decision and either receive money or not. We started out with a pretty small amount of money.

+

The initial donors — we were among them, but there were a number — contributed, best I recall, about $10 million. Launched the website early April 2020. Quickly inundated with, I think, four and a half thousand applications, which, given our promised 48-hour turnaround, was somewhat challenging.

+

I should say this was myself. This was Silvana, my wife, and this was Tyler Cohen. So we had an immediate question as to, how do we actually run a philanthropic endeavor? And how do we stand it up in very short order? And he, through Mercatus and through Emergent Ventures, had some experience of very efficient and somewhat-scaled grant-giving.

+

And so the three of us worked together to put it together over the course of a week or so. We proceeded over the course of, roughly speaking, the next year, slightly more, to make about 200 grants, eventually dispersing almost — or slightly over, actually — $50 million in total, to universities around the world, though primarily in the U.S.

+

And you ask, kind of, what did we learn? A big surprise was how slowly other parts of the establishment mobilized. And various of the projects we funded or the labs we funded and so on — they’ve gone on to now do — none of them were directly implicated in the vaccine research project that ended up yielding so much fruit. So again, I don’t want to give Fast Grants too much credit. Eventually, the thing that really mattered, we had nothing to do with.

+

But versus the projects, things like Saliva Direct, which was in the summer an early discovery that saliva tests work basically as well as the nasopharyngeal swabs we were all being subject to, or various discoveries around possible therapeutics, some of which are — still continue to go through clinical trials, and may still turn out to matter to a significant extent. And that 500 people are still dying in the U.S. per day from Covid, and — despite the existence of the vaccines and so on.

+
+
+

So anyway, various discoveries ensued that I think will prove to be important. And the second thing we learned, which is not really related to Covid or the pandemic, but has certainly been significant for us, is — it just got us thinking more deeply and broadly about the questions of, how do scientists choose what to do? And what are the constraints they’re subject to as a practical and applied matter?

+

And this gets back to all this discussion about both culture and institutions. And towards the end of Fast grants, we ran a survey of the grant recipients. And these are essentially all people who don’t normally — certainly don’t normally work on Covid. Covid didn’t exist.

+

But they don’t even normally work on viruses, for the most part. These are basically kind of broadly drawn as a cross section across biology. And we just asked them, as a general matter in your regular research, if you could spend your grant money however you want, how much would you change your research agenda?

+

So not an increase in the funding level, which tends to be what we discuss in as much as we’re discussing science policy across society. But much more specifically and narrowly, if you had complete autonomy in how you spend whatever grant money you’re getting, how much of your research agenda would change? And our intuition was that maybe a third of people would like to be doing something meaningfully different to what they actually are.

+

But of these scientists, and these are really good scientists, four out of five told us that they would change their research agendas, quote, “a lot.” We gave them three options. Not much, or not at all, a little, and then a lot. Four out of five chose the maximum option on our survey. So I just find this incredibly thought-provoking.

+

Basically, we seem to be in a situation where most of our top scientists aren’t doing what they think would be best for them to do. And we could say, no, our various committees and governing bodies and decision-making apparatus and so on, they know better. And I’m not saying it would be completely unreasonable for one to maintain that. But that would seem to be a very central question about the construction of our scientific apparatus. And I think that should give us some pause.

+

EZRA KLEIN: There are a couple things there. One is that it is a consistent observation I have learning about new areas that there is a way we’re taught the thing works, or people think the thing works, and there’s this huge middle layer. Right? So in politics, which I know very well, and legislation, you have the “Schoolhouse Rock” version of how a bill becomes a law. And then it’s, like, a filibuster is how a bill becomes a law or does not become a law.

+
+
+

We were talking about drug innovation earlier. I think the folk way people think it works is we make a discovery about a drug, and then, like, we make a drug out of it after some tests. But you talk to people who work on pharmaceuticals and just clinical trials.

+

And in science — I think if you had asked me as a high schooler, had some science classes, I’d have told you something about the scientific method. And then you talk to a scientist, and it’s grants. Like, grants are how science works. Grants are the middle layer between — you are a scientist, and you can do some science. And grants are how the N.S.F. and the N.I.H. work. They’re how a lot of the universities work. There’s fund-raising.

+

I mean, there are different ways that it happens. To make the question of “Are we doing science well?” a little bit more precise, I think one version of that question is, “Are we doing grants well?” And I think that question is more tractable. People don’t feel as defensive about it.

+

But I’ve talked to a lot of scientists in the course of my work. I’ve covered health care for my entire career. And I don’t know any who think we’re doing grants well. I don’t know any who will not complain to you for hours.

+

And they may be wrong. And I do want to note — because they also just have somewhat different incentives. I mean, I was noting earlier, and I think it’s very real. The government, particularly when it gives out grants, needs to worry about the reputational cost of the grant. If the grant goes wrong, if not enough of the grants pay out into useful research. If Rand Paul can stand up in Senate and make what you did sounds silly, these things really end up mattering. And so you get a process that is optimizing for a lot of different things.

+

But the question of whether or not we do grants well ends up being really, really, really important in every country that does major capital science that I know of, and is just not the main question for a bunch of different reasons we ask.

+

PATRICK COLLISON: Yeah. Another question we asked in our survey was how much time they spend on the grants. I think to some extent, this is perhaps — at least, of those who’ve spent some amount of time interacting with scientists, kind of more broadly known than perhaps the finding with respect to how they do — or the degree to which they can choose what they work on. But we found that — or they reported to us that they spend on the order of 40 percent of their time on grant administration.

+
+
+

And even if one were to maintain that the decision-making apparatus around what scientists do is somehow efficient, I think it is a very tenuous position to also try to argue that 40 percent of the best scientist’s time is optimally allocated towards grant applications, authorship and administration. And we’re not talking about an inconsequential 40 percent here.

+

I mean, this is 40 percent of the time of this super-elite 10,000, 100,000, whatever it is, some relatively finite number of people. And we’ve chosen to take and to redeploy almost half of their time in service of technocratic, bureaucratic undertaking. And getting back again to this point about people perhaps falsely assuming that things have been more inter-temporally consistent than they have, that percentage has increased very substantially over the last couple of decades as the overall edifice of science has grown, and as the kind of acceptance rates and the various thresholds for various grants has become more exacting.

+

EZRA KLEIN: How we allocate people’s time is really important. But also, just how we allocate talent is really important. And it brings me to something you said that I wanted to ask you about. This was in response to a question about whether big tech companies are hogging all the talent in society.

+

And you said, quote, “I don’t think that the ambitious upstarts who go into high speed rail in America, anyway, are going to have a great time or have much success in convincing their friends to follow them. And I suspect that for various reasons, too many domains look somewhat like high speed rail.” And so you go on to say that there’s a view that the internet is a frontier of last resort, and that you don’t think that’s totally wrong.

+

So tell me about that. Tell me about the idea of the internet as a frontier of last resort. But behind that, this idea that other frontiers where talented people might want to go and make their mark on society have closed.

+

PATRICK COLLISON: You’re familiar with and you’ve probably written about the Stephen Teles idea of kludgeocracy. And I kind of like the term “kludgeocracy,” because rather than making some of the inhibitions that people might encounter in pursuing something like high speed rail, rather than casting those as being deliberate, the valence is more that it’s this kind of emergent, inadvertent and kind of complicated phenomena that nobody perhaps particularly wants or chose.

+

And I think the case of California’s high speed rail is quite striking, where — you’ve written about this and kind of similar projects and the New York subway expansion and so on. And congestion pricing and so on. But it’s striking where it’s not actually obviously a question of first order political will.

+
+
+

Like, we’re willing to fund the high speed rail in California. We’re clearly willing to invest in building the subway expansion in New York. But somehow, somewhere between that first order decision and desire and our actual ability to kind of instantiate it, something really goes wrong.

+

And you contrast that with stories of — in the case of, say, California, Henry Kaiser and these various other early part of the 20th century operators in the physical realm. And for a variety of reasons, but mostly prosaic state and county-level complications and things that would extend the time horizon of one’s project, it has simply become meaningfully less-appealing for those people to undertake these initiatives. I mean, Foster City, not too far from where we are now, that’s named after the eponymous Mr. Foster.

+

He was a developer. He decided, well, with reclaimed wetlands, I’m going to build a city. California is growing quickly. The Bay Area is a — kind of propitious and will be a long-term successful area. And I think this place simply needs more housing.

+

And he, with that kind of founder energy, was able to give birth and rise to the city that now bears his name. I haven’t met anybody pitching me on a similar city on the shores of the Bay in the last couple of years. But I would imagine that were one to adopt that ambition today and to propose that maybe the San Jose Marsh wetlands should themselves be an expansion of San Jose, I don’t think one would get very far.

+

And in fact, even for much more sort of limited things, like additional runways or runway expansions at S.F.O., even they have now been stymied for decades at this point. That ability to translate that into something enunciated has dissipated and deteriorated. And then I think the kind of individual version is, and if I want to be that heroic solar farm entrepreneur or railway magnate, that my practical ability to do so has been meaningfully curtailed.

+

EZRA KLEIN: Yeah. I mean, that’s what I’m getting at here a little bit, which is talent really matters for a society. Where the most talented people go really matters for society. And a lot of those people want to go somewhere where they can have a really big effect.

+

I think there’s been a huge rush to digital land because you can build on digital land. You can build quickly. I mean, it’s interesting to some of the dynamics we’re talking about, the temporal dynamics we’re talking about, that you see this dynamic even within the tech world.

+
+
+

There was a while where it was really exciting to go join Facebook, go join Google, go join one of the big companies. Because you could do so much. And that’s still, to some degree, true. But they got really big.

+

And so crypto got — whatever you think of crypto, one thing that is exciting about it to people is the idea that it’s open land. That you can go in there and have a really big effect on it. Build something new just with a couple of friends that might change the whole direction of the field.

+

And on some level, it’s always going to be harder for, say, putting high speed rail through the middle of California. Right? I mean, just building things in the world is just going to be tougher.

+

But on the other hand, if you make building things in the world too hard, if you make grants too difficult — if you — I know a lot of doctors who their advice to young people is don’t become a doctor. And it always breaks my heart a little bit. We need really great people to be doctors.

+

And their point is not, don’t go heal sick people. Their point is, being a doctor is too hard now. The amount of time you spend dealing with insurance agencies and malpractice insurance and boards, and this and that, it’s just too much administration.

+

When industries become very complicated to operate in, you want to select for people who are good at operating complicated industries, which may be different than the people who are good at moving really fast and changing things dramatically. But two, you kind of subtly bias where different kinds of people in your society go. I think in China, if you want to change a lot, you still probably go into infrastructure construction, among other things. Right?

+

The idea that you might be a genius rail mind, in China, that’s great. There’s probably a lot of rail you can make. That’s not true here. And the point is not to make too much of the rail example, but to make a lot of the idea that talent flows towards where it can have an effect and people can live the kinds of heroic lives they want to lead. And if we have subtly pushed a lot of people into maybe not the right — not the socially optimal directions, that over time will have a pretty big effect on a society.

+
+
+

PATRICK COLLISON: I think a constant is that some number of ambitious young people will want to do something, as you say, heroic. And yeah, I think maybe two things have changed. For one, for whatever reason, our predisposition to putting those people in positions of authority has diminished. And then secondly, even if placed, their ability to actually execute, again for various reasons, has been attenuated.

+

I’ve been reading about the university founders and presidents and those associated with some of the great US research institutions. And one thing that is striking is how many of them were so young when placed in those positions of authority. You know, Daniel Coit Gilman at Johns Hopkins, or William Rainey Harper at the University of Chicago. I think he was 32 when he was appointed president of the University of Chicago. Even in the recent past.

+

So my dad was in the first year of the University of Limerick in Ireland. Or at the time, it was called N.I.H.E. It kind of acquired university status later in its life. And the Irish guy who founded it and was really the dynamo behind it, I think he was 29 when he was put in charge of that project.

+

And I think it was in 1970 or ’71 that he was charged with this mission. But as recently as 1970 in Ireland, we were willing to put a 29-year-old — I mean, that’s a person meaningfully younger than me in charge of the project of overseeing the creation of a major new research institution. And I don’t know that I have compelling or confident observations to offer in terms of the etiology underlying these changes. But I think the changes themselves are important, or at least we should assume they’re important if we come from a place of humility, where this is what has worked in the past. Enabling these ambitious young people who are willing to contemplate spending multiple decades in pursuit of some ambitious and idiosyncratic vision.

+

And maybe we’re more enlightened now. Maybe we figured out how to get all the same innovation and all the same breakthroughs without unleashing that force. But I guess my starting point, at least, would be, well, we should — before getting super confident in that or before really being deliberate about it, I think we should give some kind of credit and credence to the prescription and the methodology that’s worked heretofore.

+

EZRA KLEIN: And before books, let me end on this. We’ve talked a lot about scientific slowdown, about technological slowdown. But let’s say in the next 15-year time frame, what are the three technological or scientific possibilities you’re most excited by? If in 20 — I guess it’d be 2037, we’re having a conversation about how dumb this conversation was because it was right on the cusp of so much incredible stuff happening, what do you think is likely to be on that list?

+

PATRICK COLLISON: I don’t know that I’ve super non-consensus answers. I think that there are fundamental a priori reasons to believe that the rate of progress in biology could increase substantially over the years, and to your question, kind of decades to come. So if in 2037 we are enormously impressed and struck by the discontinuity there, that would not shock me.

+
+
+

Clearly, over the past couple of years, there’s been acceleration in progress in A.I. And kind of far for me to try to point estimate for kind of where that is in 2037. But I would be surprised if that is not somewhere on that list.

+

And then I think there’s something about education in the broadest sense that feels to me like a very significant, and hopefully very positive change happening in the world right now. Maybe best embodied by YouTube. But also by Twitter and by blogs and Substacks and even Zoom and kind of the growing ease of being in some kind of cultural proximity to people one aspires to emulating, or following in the footsteps of, or otherwise kind of being more like.

+

And to the extent that one believes my story about the significance of sociology, and culture, and mentorship, and the kind of delicate transmission of tacit knowledge, it has until very recently only been possible for that to happen to a meaningful extent through physical co-location. And the fact that we’ve now thrown open those doors to such an extent feels to me like a really compelling and plausibly transformative change. And if it were the case in 2037 that we have multiplied by 20 the number of people who can — who have the initial mental models and understanding to become successful entrepreneurs, or successful scientists, or successful writers, or successful in whatever one might choose one’s domain to be, again, I think that would not be shocking. And I think it’s a pretty hopeful fact about the world.

+

EZRA KLEIN: And then always our final question. What are the three books you’d recommend to the audience?

+

PATRICK COLLISON: Well, I’m right now reading “Revolution and Empire,” which is a book about Edmund Burke. And it is just fabulous. I very highly recommend it.

+

Edmund Burke, Ireland’s foremost political philosopher. And I’m embarrassed to say that I have known less about him than I feel like I ought to have. So I recommend that very highly.

+

And the autobiography by Warren Weaver, who I mentioned, at Rockefeller. I can’t remember if it’s called “Scene of Change” or “Scene of the Action.” But it’s Warren Weaver’s autobiography.

+
+
+

That’s not a great book in the sense that you don’t read it — you don’t find it to be a vivid, compelling page-turner. And your mind is not blown on every page. But I find myself thinking back to it quite a lot and having various parts of it sort of ricochet to my mind.

+

And then it all depends on what people are interested in and all the rest. And you should read the things you like. But I have on my desk at home right now “A Widening Sphere,” which is a history of M.I.T. And I was re-reading it recently. And my —

+

EZRA KLEIN: Who doesn’t re-read the histories of M.I.T.?

+

PATRICK COLLISON: [LAUGHS] Well, William Barton Rogers, the founder, was the son of an Irishman, and started M.I.T. substantially with his brother. And I find it very inspiring, I guess back to what we were saying earlier, how motivated he was and they were by a kind of broad-based desire for societal betterment. Like, M.I.T. didn’t inadvertently end up being a significant contribution to American prosperity and ingenuity and welfare. He was really immersed in that milieu.

+

He paid a lot of attention to some of the cultural dynamics we were describing in England, and the Darwins. And the early writing on M.I.T., if you go and just read the first two pages of the founding manifesto, it wasn’t utopian in some kind of implausibly lofty sense. But it was somebody who knew they weren’t founding a run of the mill nth technical college.

+

And I feel like it’s easy to get cynical always. It’s easy to assume that the things that really worked out worked out through happenstance, as opposed to optimism and ambition. But yeah, I find the history of MIT to be a kind of inspiring reminder that sometimes these implausible, lofty, ambitious, long-term initiatives can work out much better than one would hope.

+

[MUSIC PLAYING]

+

EZRA KLEIN: Patrick Collison, thank you very much.

+

PATRICK COLLISON: Thanks for having me.

+

[MUSIC PLAYING]

+

EZRA KLEIN: “The Ezra Klein Show” is produced by Annie Galvin and Rogé Karma. Fact-checking by Michelle Harris, Mary Marge Locker and Kate Sinclair. Original music by Isaac Jones.

+
+
+

Mixing by Sonia Herrero, Isaac Jones and Carole Sabouraud. Audience strategy by Shannon Busta. Special thanks to Kristin Lin and Kristina Samulewski.

+

[MUSIC PLAYING]

+
+
+
+
+
+
+
\ No newline at end of file diff --git a/packages/readabilityjs/test/test-pages/nytimes-podcasts/source.html b/packages/readabilityjs/test/test-pages/nytimes-podcasts/source.html new file mode 100644 index 000000000..d777dda15 --- /dev/null +++ b/packages/readabilityjs/test/test-pages/nytimes-podcasts/source.html @@ -0,0 +1,5576 @@ + + + + + + Transcript: Ezra Klein Interviews Patrick Collison - The New York Times + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+
+
+
+
+
+
+
+
+ +
+
+ +
Skip to contentSkip to site index +
+
+ Podcasts +
+
+ + +
+
+ Subscribe for $0.50/weekLog in + + +
+
+ +
+
+
+
+
+
+
+
+
+ Podcasts|Transcript: Ezra Klein Interviews Patrick Collison +
+
+
+ + + +
https://www.nytimes.com/2022/09/27/podcasts/transcript-ezra-klein-interviews-patrick-collison.html +
+
+
+
    +
  • +
    +
    + +
    +
    +
    • +
  • +
    +
    + +
    +
    +
    • +
  • + +
    • +
+
+
+
+
+
+
+
+
+
+
+

+ Advertisement +

+
Continue reading the main story +
+
+ +
+
+
+
+
+ +
+

+ The Ezra Klein Show +

+
+
+

+ Transcript: Ezra Klein Interviews Patrick Collison +

+
+
+
+
+
    +
  • + +
    +
    + +
    +
    +
    • +
  • +
    +
    + +
    +
    +
    • +
  • + +
    • +
  • + +
    + +
    +
    • +
+
+
+
+
+ +
+
+
+
+
+
+
+
+ Real conversations. Ideas that matter. So many book recommendations.
+
+ Listen to “The Ezra Klein Show”: Apple Podcasts, Spotify, Pocket Casts, Google Podcasts, Stitcher, How to Listen +
+
+
+

+ Every Tuesday and Friday, Ezra Klein invites you into a conversation about something that matters, like today’s episode with Patrick Collison. Listen wherever you get your podcasts. +

+

+ Transcripts of our episodes are made available as soon as possible. They are not fully edited for grammar or spelling. +

+
+ +
+
+
+
+
+
+
+
+ The Ezra Klein Show Poster +
+
+
+
+

+ We Know Shockingly Little About What Makes Humanity Prosper +

Patrick Collison calls for a new “science of progress.” +
+
+
+
+
+ +
+
+
+
+ +
+
+
+ +
+ + + bars + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+
+
+ 0:00/1:33:01 +
+
+ -1:33:01 +
+
+
+
+
+
+
+

+ transcript +

+

+ We Know Shockingly Little About What Makes Humanity Prosper +

+

+ Patrick Collison calls for a new “science of progress.” +

+
+
+
+ ezra klein +
+
+

+ I’m Ezra Klein. This is “The Ezra Klein Show.” +

+

+ This is a great conversation today. But it’s a tricky one to introduce, because the guest I have — I’m not having him on for the thing he’s best known for. So Patrick Collison — by day, co-founder and C.E.O. of the multibillion-dollar payments company, Stripe; by night, by weekend, I think, one of the most important thinkers now in Silicon Valley — certainly, one of the most quietly influential, someone who is forging and traversing an intellectual path that a lot of other people are now following. +

+

+ And it’s this second incarnation and role that I’m really interviewing him in today — the soft power side, I guess, of Patrick Collison. Collison’s work here centers around this question of progress. The argument is that human progress is much more precious and rare and fragile than we realize. +

+

+ We maybe take it for granted. We live in this time when things have been changing, atop decades and decades, even centuries and centuries, even millennia now, when things have kept changing. But for most of human history, that was not true. It was not true. +

+

+ There just was no market rapid advance in human living standards. It’s only in the past 10,000 years, and then practically in the past few hundred — just an eye-blink in the time human beings have been on Earth — that things kept changing, usually for the better. And the question is, why? +

+

+ And Collison’s particular meta question is, given the clear fragility of forward motion here, given how rare it has proven to be — and so how easy it might be to lose — why isn’t the question of the conditions of progress more central? Why isn’t the study of progress in a wide multidisciplinary way a more common and central discipline? +

+

+ Collison has written a few influential essays here, with the economist Tyler Cowen. He called for the inauguration of a discipline — they call it progress studies — and that now has people studying it. There’s people creating journals for it, creating syllabi and podcasts and books around the topic. It’s one of the more singularly successful calls for a research direction I have seen. +

+

+ Separately, in a piece co-authored with the scientist, Michael Nielsen, Collison and Nielsen argued that, though it is hard to measure, it seems like the rate of scientific progress is slowing down, and that’s particularly true if you account for how much more we’re putting into science, in terms of money, of people, of time and technology. +

+

+ Now, these ideas are not original to Collison. The point is not that nobody studied human progress before this or worried about the pace of scientific research. He wouldn’t claim that. It wouldn’t be true. But he is playing a distinctive role in their framing and their popularization, and in creating and funding a community around them. +

+

+ And what I see in my travels here is that it is working. Something is burbling here. But I can’t find many big pieces where Collison really lays out his worldview. There are a couple essays, tweets, interviews, but he’s not been primarily writing this down. +

+

+ What he has been doing is funding it through Fast Grants, which has been successful, but more than that, intellectually influential effort to show you can give out scientific grants quickly and with very little overhead, through the Arc Institute, a big biotech organization he’s creating to push a researcher-first approach to biotech, and through giving a bit of money, and a bit of time, and a bit of prestige, and a bit of networking to a lot of different projects that circle these questions. +

+

+ He’s got this funny quality of being nowhere in particular, but also somehow, almost everywhere, if you’re interested in these questions. So what I wanted to do in this conversation was try to get as close as I could to the Patrick Collison worldview, the underlying theory of the case here that animates his thinking his funding, and the ways in which he’s trying to nudge the culture he’s a part of, or the ways in which he’s trying to actively create a culture he doesn’t yet see. +

+

+ As always, my email — ezrakleinshow@nytimes.com. +

+

+ Patrick Collison, welcome to the show. +

+
+
+ patrick collison +
+
+

+ Great to be back. +

+
+
+ ezra klein +
+
+

+ So you’ve made the argument that science — all science — is slowing down, that we’re putting more money and more people into research, and we’re getting less and less out of it. Tell me about that. +

+
+
+ patrick collison +
+
+

+ Well, I want to separate two things. There’s a question as to whether science in its totality is slowing down, in terms of the absolute returns from it. I think that might be true. You can maybe divide up the first half of the 20th century and the second half and so on, and sort of try to compare one with the other. +

+

+ And we had general relativity and quantum mechanics and various other major breakthroughs in the first half. You can ask the question of, well, did we have as many in the second half? But in the second half, we did have the discovery of D.N.A. and molecular biology and lots of other things. So I don’t know that I would claim a total slowdown. The thing that I think is clearer and should be very concerning to us is, as you look at the number of scientists engaged in the pursuit of science, and if you look at the total amount that we’re spending, and as you look at the total output, as coarsely measured by things like papers and number of journals, all of those metrics have grown by, depending on the number, let’s say, between 20 and 100x between 1950 and, say, 2010. +

+

+ And if you look at it on a per-capita basis, or a per-unit-of-work basis, now used to divide all those total outcomes by a factor of 50, and it seems like if you imagine yourself as the median scientist, you’re meaningfully less likely to produce anything like as consequential a breakthrough as you would have, say, in 1920. And so Michael Nielsen and I, in order to try to put slightly more rigor on that question — we went and we surveyed a bunch of scientists across a number of universities in a number of different disciplines, and we presented them with different Nobel Prize-winning breakthroughs. +

+

+ And we tried to compute an approximate ordering of their significance in the eyes of these scientists. And the thing that would kind of have to be true — for the per-capita impact, we remain in constant — is we’d have to be discovering much more important things in the latter half of the 20th century in order to compensate for, to make it worthwhile, for us to be investing this 50-fold greater effort. +

+

+ And we didn’t find that. In physics, in the estimation of physicists, there was a kind of flat-to-declining trend. It’s not super obvious which way it points, but in as much as there’s a trend visible, it’s probably slightly downwards. And in other fields, it was maybe similarly equivocal, perhaps a slight increase, visible in some, but importantly, in no fields that it looked like we’re on this crazy, exponentially improving trajectory, which is what you would have to have for this per-capita phenomenon to not be present. And I think that should be something we’re interested in for multiple reasons. One, because presumably, as a society, we’re interested in just how much more scientific progress and technological progress and so forth, how much more innovation is there going to be over the next 10 years or the next 50 years or the next century. But also, because there’s kind of two possibilities. One possibility is, fundamentally, we’re running out of low-hanging fruit, and it’s just going to be harder to do this stuff. +

+

+ And in as much as we’re setting investment or making investment decisions around to what degree should be pursuing the stuff, I guess it’s important to know what we think the returns should be. Or the other possibility is, somehow, we’re doing it suboptimally. Something changed, and we were pursuing this process of discovery more effectively in the past, and presumably, for inadvertent reasons, something went wrong, and now, we’re just less efficient at it. +

+

+ But either explanation — and it doesn’t necessarily have to be fully binary — but either explanation is important, and either explanation, I think, has prescriptions for what we should do going forward. +

+
+
+ ezra klein +
+
+

+ Let me start with the low-hanging-fruit explanation, which I think is a more popular one. And you have — in the piece you did on this with Michael Nielsen, the sad, but in the very academic way, very funny quote from the physicist Paul Dirac, who says of the 1920s, there was a time when, quote, “Even second-rate physicists could make first-rate discoveries,” which I just kind of love. +

+

+ But the theory there is you can only make a lot of the big discoveries once. You discover quantum mechanics once. You discover the atom once. And most of them have just been made, so what you have now is more complicated, smaller, requires much larger teams of people, much more complicated experiments, with much more infrastructure. +

+

+ So we’re just structurally in a period where it’s going to get harder and harder and harder to make big gains. Do you believe that? +

+
+
+ patrick collison +
+
+

+ I think it’s possible, but even though it’s intuitively compelling on some level, I’m not sure that it’s true. It’s probably true to at least some degree for some particular research direction, right? We go after discovering the various subatomic particles, and initially, without too much difficulty, we discover the electron or whatever. +

+

+ And by the time we’ve discovered the nth quark, it’s now gotten super hard, and even with ever-larger particle accelerators, we’re not necessarily making breakthroughs of the same magnitude. So I think it’s pretty true for a given direction. But obviously, the question is, well, to what degree is progress in any area opening up other directions, right? +

+

+ And so I mean, you mentioned the Dirac quote and, say, physics in the early part of the 20th century. Those discoveries opened up new techniques and investigation methodologies and so on, that then gave rise to molecular biology in the ‘50s, ‘60s and ‘70s. And so there’s kind of a combinatorial benefit, where discoveries over here or discoveries over there might unlock opportunities and major breakthroughs in areas that we could not have foreseen in advance. +

+

+ There are lots of, quote unquote, “low-hanging-fruit discoveries” made in computers and computer science in the ‘70s, ‘80s, and ‘90s. Maybe we’re even still in that regime, right? We’re still making some pretty fundamental breakthroughs. And of course, again, those, quote, “low-hanging discoveries” would not have been possible without a lot of this optimization and discovery in other fields. +

+

+ And so I think it’s probably true for a given research direction, but the relevant question for society is, is it true in aggregate. And there, it’s much less clear to me that it is. +

+
+
+ ezra klein +
+
+

+ I want to read something provocative you said in an interview with the economist Noah Smith. And you said, quote, “Most systems get worse in at least certain ways as they scale. The idea that science could have gotten worse in significant ways sometimes sounds strange to people. Like, we’re doing so much more. How could that be bad? But I think that misses the many examples of sensitivity of scientific processes to institutions and culture. Swiss nationals have won more than 10 times more science Nobels per capita than Italians have. 10 times. And yet, they’re neighbors. And Italy certainly isn’t lacking in scientific tradition — Fermi, Galileo, the oldest university in Europe, et cetera. The ‘how’ of science just really matters.” +

+

+ And this seems, to me, to be where your exploration really goes. So tell me what you think might have gone wrong in the “how” of science. +

+
+
+ patrick collison +
+
+

+ So I think this point about the sensitivity of scientific outcomes to the specifics of the institutions and the cultures is very important and probably underappreciated. At the beginning of the 20th century, not only was the U.S. not a scientific powerhouse, but it barely had a presence in frontier research, whatsoever. +

+

+ To become a credible researcher in the U.S. in 1900, you almost certainly had to go and spend time in, most likely, Germany, and failing that, in France or England — you know, what have you. And by 1900, the U.S. was already a pretty prosperous place, and it had a well-educated society, as societies went. +

+

+ And yet, somehow — and it had universities, right? I mean, Harvard was hundreds of years old by that time. And so it checked many of the ostensible boxes, and yet, the sum total of the U.S.’ research output as of 1900 was still de minimis. +

+

+ When James Conant, who was later president of Harvard for 20 years — when he went to Germany as a chemist, which was his original training, in the 1920s, he recounts how dispirited he was by what he found there and how far ahead of Harvard German research was, as of the early 20th century. And then, for a variety of reasons, all sorts of cultural, institutional funding — various transformations happened. And of course, by the latter half of the 20th century, the U.S. was the unquestioned leader at the frontier of scientific progress. +

+

+ If you look backwards, you see where that locus has been, where the most successful and fertile scientific grounds have been — it has repeatedly moved. As we just said, maybe the 19th century, it was Germany. +

+

+ Before that, in the 18th century, it was plausibly France. They had a couple of these really successful École Polytechnique and Grande École and so on. And so as a kind of first-order empirical matter, we can just notice, huh, this really seems to matter — and then, the example you just gave of the divergence between Switzerland and Italy. +

+

+ And so then, if we kind of accept that, and we try to ask ourselves, well, specifically, what are the mechanisms? You know, what’s actually going on? It’s hard for me to say. It seems like the transmission of research culture by individual researchers matters a great deal. +

+

+ And you see these kinds of pockets of the cultural transmission repeatedly crop up, where Gerty and Carl Cori — you probably haven’t heard of — they ran a little biology lab in Missouri, and no fewer than six of their trainees, of students they trained, went on themselves again to win Nobel Prizes. +

+

+ And if we tell ourselves a standard kind of mechanistic story as to, well, it’s the funding level, it’s how much are we investing in science, or it’s something about whether there’s an institution in the courser sense, that can possibly be amenable to it, it’s very hard to explain these eddies where you see these pockets of excellence really produce these outsized returns. So I think it’s a complicated question. +

+

+ I think all of aggregate culture, funding, institutional characteristics, and so on all contribute to it. But if I had to isolate a single variable, it seems to me that the research culture set by specific people and the tacit knowledge transmitted through direct experience is probably the number-one thing. +

+
+
+ ezra klein +
+
+

+ This, I think, is where I sometimes fall into my own pessimism on this. Because I want to believe, as you do, that we can double the rate of scientific advance, maybe even go further than that. But I think the prediction — if I’m putting this on institutions, on culture, on pockets of transmission and mentorship — I think the prediction I would make is then, even if you believe, say, that America had a great 20th century, but its institutions have become sclerotic, and we’ve slowed down, and everything is piled in lawsuits and review boards now, somewhere else that didn’t have that, that has a different culture, that has different institutions, would be pulling way ahead. +

+

+ So you might think, well, China will be pulling way ahead. And you’ve noted this in some places. We’re getting a lot of peer-reviewed research out of China — huge number of citations out of China. We’re not seeing them dominate the big breakthrough advances of the era. +

+

+ It doesn’t seem like Europe is lapping us. And so if you think this slowdown is somewhat global, then that seems to me to militate against questions of individual institutions, cultures, how different labs work, because there is so much variation that you should have some of these labs that are doing it right, some of these places that haven’t piled on a little bit too much bureaucracy. But I don’t think we really see that. +

+
+
+ patrick collison +
+
+

+ This diagnosis of these phenomena to cultural, institutional, mentorship-related, interpersonal dynamics, and your observation that it’s not obviously the case, that there are other places we can pointed that are doing it so much better — for me, my takeaway is that, well, successful cultures are a pretty narrow path. Homo sapiens emerged 200,000 years ago. +

+

+ And as far as we can tell, for the first 190,000 years of our genesis, we think we were largely biologically equivalent to the people we are today. But as best we can tell, there was some kind of cultural capital that those people lacked for a very extended period of time before human societies in somewhat recognizable modern form started to emerge — agriculture, all the rest. +

+

+ And in a similar vein, we had many billions of lives and centuries elapsed before the Industrial Revolution., and before we started to put together many of the input ingredients or enough of the input ingredients that we can get sustained improvement in standards of living and ongoing economic growth and progress. And so your point about, well, as I look around, I don’t see anything or anywhere that’s obviously better, I agree with that. +

+

+ But again, my takeaway is that that’s what makes the question of how do we improve or how can we do somewhat better so urgent and pressing, where it’s many things have to go right. It’s not easy to be even as good as — or to get to a place where things are as good as they are today. What we have is very precious. And I think the threads and the themes that you’ve been pulling on of late — all of these dynamics underscore their importance. +

+
+
+ ezra klein +
+
+

+ I think that’s a good bridge to progress studies as an idea. And I want to have people hold in their heads that idea that progress is very narrow, that it is a very narrow bridge that we have walked on for a very short period of time. But let’s try to define it. +

+

+ When you say progress here, what are you actually talking about? Is it just shorthand for economic growth or G.D.P.? What is progress? +

+
+
+ patrick collison +
+
+

+ Well, I don’t know that I would claim to put forth some kind of definitive definition. And I think, to some extent, our intuitions around it are probably broadly correct. And so it might not matter to define it super precisely and finely. +

+

+ For, me it is something along the lines of our success in realizing a liberal, pluralistic and prosperous society, and a sense among people that their offspring can and probably will do better than they themselves have, and that more broadly, the future will be better than the past, and that we’re at least making incremental progress towards embodying values and morals that we collectively think we can be proud of. +

+

+ But I don’t think anything that novel in that. I don’t think my conception of progress would differ that materially from some kind of average aggregate over any other group of people in the country. +

+
+
+ ezra klein +
+
+

+ I do think there’s something interesting, though, which is that if you look at eras that I think progress-studies-type people and economic-growth people and historians of economic growth study most closely, actually, some of the periods where people feel a lot of rapid progress don’t fit that at all. You have, say, the Industrial Revolution, where life spans and lifestyle get worse for a lot of the people. +

+

+ I don’t think one will look at that period as unbelievably pluralistic. You have a lot of periods of war when you have very, very, very rapid technological progress, but it happens in context of much more martial societies. So there is an interesting tension, at least in periods — and some of them quite long, actually — where you can have fairly rapid economic progress, but it comes at a cost that I think isn’t always acknowledged, but is an important thing to think about. +

+
+
+ patrick collison +
+
+

+ Yeah. So I don’t think you could point to some of these periods in the past and say that they definitively embody to the extent that we would fully aspire to some of these broader traits and characteristics. But I think the question is more, what are they doing as — you have to judge it relative to the baseline that preceded them. +

+

+ And I don’t know that the 18th century in the U.K. is some ideal as a society. But if you compare it to the 16th century in the U.K., the ideals and ideas of natural rights and religious tolerance and so on — they were somewhat better embodied by the 18th century than they had just a couple of centuries previously. +

+

+ And similarly, in the U.S., say, during either war or the ‘30s or whatever, again, it’s not like that was any kind of perfect society, but assessed relative to the society of 1830, I think it compares relatively favorably. And I think it’s not a coincidence that Adam Smith — his first book, of course, was on ethics and morals and trying to instill better general ideals and behaviors across a society. +

+

+ And maybe after that, he then argued for and laid many of the foundations of what we would recognize as modern economics. So I don’t think it’s perfect. But on average, I think the correlation is positive. +

+
+
+ ezra klein +
+
+

+ So let’s talk about the Industrial Revolution for a little bit here. I think a lot of people locate a takeoff in human living standards — it continues to this day — there. And it’s strange in a way, right? “There” is a very geographically contiguous spot. It’s the U.K. — England, actually, I should say, at that point. +

+

+ And there is a moment in time that probably could have come at another moment in time, depending on how human history plays out in the counterfactual. I know that you have an interest in the theories of why then, why there. How do you work your way through them? What do you think is persuasive for why then, why there? +

+
+
+ patrick collison +
+
+

+ Well, you know, again, I caveat. With all of these topics we’re discussing through this podcast, maybe the first-order banner for all of them should be, I don’t know, these are my best guesses, and I think it’s important that all of us were pretty humble in the claims and the assertions and the beliefs that we hold. +

+

+ Recently, I’ve been reading a bunch of Irish and Scottish writers around then. It’s very interesting, because for both the Irish and the Scots, there was a sort of a pressing and kind of obvious question where England was much more prosperous than they were or we were. And there’s no super obvious explanation for that. There wasn’t an obvious climatic or natural resource endowment that England benefited from that was lacking in Ireland or Scotland. +

+

+ It wasn’t like England was actually a vastly larger polity. The orders of magnitude were comparable. And Bishop Berkeley wrote this book, “The Querist.” He was asking these questions directly, just like, what’s going on? What’s wrong with Ireland? You know, why can’t we do this? +

+

+ And then, you have the Act of Union in 1707, uniting Scotland and England — and sort of similarly, of all these Scottish thinkers being like, all right, we’re now literally the same country. Why are we so much more impoverished? And then, if you shift to England, there’s Joel Mokyr and — you’ve read his work — and more recently, people like Anton Howes. +

+

+ And in a similar vein, they go back to — I mean, the word, improvement, came from Francis Bacon, or it was kind of popularized as a concept by Francis Bacon. But that’s noteworthy, right? Like, that was not a pervasive broad concept in the 15th century. +

+

+ I mean, literally, the word, improvement, in this broader societal context, came from word, “translated,” at the beginning of the 17th century. And the ultimate conclusion that these historians and scholars and analysts of the Industrial Revolution come to — and I think it’s a correct one — is somehow, whether it’s through Bacon or Newton or various of the tinkerers who produced some of the earliest technological breakthroughs, that somehow, this improving mind-set became pervasive. +

+

+ You had societies explicitly — like the Hartlib Circle or the Lunar Society, or the Select Society, and the club, and so on — all these societies explicitly devoted to figuring out ways to advance the state of affairs that prevailed. And these societies were comprised of many of the leading people and thinkers and so on of the day. +

+

+ And it seems maybe a bit satisfyingly squishy to attribute it to something so hard to pin down. But as you run through all the possible other explanations, it’s differences in IP law. It’s difference in the Malthusian conditions. It’s difference in the prevalence of coal, you know, et cetera, et cetera. Through various cross-sectional analyses, you can exclude most of these in looking at all of Ireland, Scotland, and England. +

+

+ It really does seem to me that differences in the mind-set and in the culture are where you have to net out. And that’s not to say maybe that it’s fully sufficient. There might be other preconditions that are important. And then, maybe as a last thing to say, it is striking to me that many of these kind of original 18th-century economic writers and thinkers — and again, the kind of people we look to as the founders of much of the discipline — that they themselves were kind of centrally preoccupied with this. +

+

+ And yeah, they were in favor of free trade and specialization and human labor and lots of these concepts that we’re now very familiar with, but they really thought that general mind-set played a big role, too. +

+
+
+ ezra klein +
+
+

+ So let’s talk about Joel Mokyr ideas for a minute. So Mokyr is an economic historian. People should read his book, “The Culture of Growth,” which is really fascinating. He argues, as you’re saying, that in this period, this mind-set that we can increase the store of usable knowledge, and then use it to alter nature, to better the human condition, takes hold. +

+

+ That’s a new mind-set. It’s different than religious ideas of the past. It’s different than cultural ideas of the present. And that, plus a bunch of other things, particularly the republic of letters, the way people are writing letters back and forth, kind of combine into a culture that is able to grow. +

+

+ But one of the things that I really take from his work, that sits in my head, is he believes it’s all very contingent. He really believes it might have not happened. But the other is that I think it opens up this question that as a tech person, I’m curious to hear your thoughts on, which is, he really believes — Mokyr really believes — that there is a communications infrastructure that arises at that time, that has a kind of culture of generosity and argument and honesty in it, and is built on writing letters slowly to one another, and then copying those letters over to other people. +

+

+ And that culture is really good for intellectual advancement. I think one of the promises of the internet and the age we live in is, it’s all faster. We can write to people immediately. Things we write can go viral and be seen by 5 million people all of a sudden. +

+

+ And that was going to speed up economic growth really, really rapidly. And I would say, you don’t see that. So I’m curious how you think about communication cultures here and what you think for all the advantages of ours we might not have. +

+
+
+ patrick collison +
+
+

+ I mean, I think it’s hard to say in aggregate. I feel it’s pretty likely that the effects are very heterogeneous across different populations. And you’ve made the case that you think Twitter is bad for journalism and for journalists. +

+

+ And I guess you live this yourself with your now mostly inactive Twitter account, I guess, apart from announcements. And I think in the case of the internet, that it’s almost certainly a tremendously large gain that billions of people now have access to educational materials. And some of the otherwise hard-to-communicate tacit knowledge — that things like YouTube videos now made legible and available. +

+

+ And I think it’s true that there are various gravity equations that we see across different disciplines. I mean, in economies themselves, in trade, where you rapidly decline in propensities to trade as countries get further from each other — but you have versions of this in academic disciplines as well, where geographic distance correlates inversely with likelihood of the exchange of ideas and so on. +

+

+ And I think it’s clearly the case that the sort of reaction surface area has increased substantially by the internet there and represents a kind of efficiency gain for people looking to exchange in ideas. Many of the companies that Stripe works with are remote companies, and they might employ people across myriad countries, and that’s a kind of communication and efficiency gain that would certainly not otherwise be achievable. +

+

+ I think it’s worth recognizing that the aggregate amount of G.D.P. that we are creating or gaining every year is so much larger now than — I mean, the percentage might be the same. But the total amount of stuff happening, or the increasing amount of stuff happening, is so much larger now than it was 100 or 200 or 300 years ago. +

+

+ And so for all of those reasons, I think we should give superior communication technologies and faster communication technologies a significant amount of credit, even though the ways in which those are manifests might be hard to measure and somewhat prosaic. +

+

+ Take my mom, for example. My mom works with a hospital in Minnesota. Our youngest brother has a physical disability. And in the course of that, she trained herself in treatment for cerebral palsy, this condition, and she wrote a book about it, and she did a master’s in this. And now, she’s trying to improve treatment for this condition throughout Ireland, in the U.S. and other countries as well. +

+

+ She’s a retired Irish mother who spends some of her year living in the U.S. near her sons, spends the rest of her year living in Ireland, working at a hospital in Minnesota, who just got a proposal to have her book translated into German a couple of days ago. And that’s a relatively prosaic story, but literally, millions of these stories exist in kind of aggregate form around the world. +

+

+ To circle back to the initial thrust of your question, though, I think it’s at least possible that the internet is bad for civic discourse. I’m not saying it is, but it’s certainly in the realm of plausibility — and that perhaps both things are true, where there’s some kind of iceberg where there are these enormous welfare gains that are not that legible, not that visible, lie beneath the surface, and then certain of the most visible manifestations, like what we see on cable news or what we see written in the papers — perhaps that is worse, and perhaps, slightly more structural judiciousness would be desirable there. +

+
+
+ ezra klein +
+
+

+ I want to try to flip that and suggest that — because I’m going to push some counter ideas on why we maybe don’t see as much progress as we wish we did. But one is that I think possibly, very large welfare losses lie beneath the surface. And beneath the surface of stories like the one you just told about your mother, I think we all have stories of ways or people for whom the internet has unlocked a possibility. +

+

+ I mean, my whole career is built on the internet. I was an early blogger. I got rejected from my student newspaper. And if there was no blogging, like, god knows what would have happened to me. [LAUGHS] I mean, nothing too terrible, probably, but I wouldn’t have the career I have today. +

+

+ And at the same time, I think that the group of people who, by luck or by temperament, proved very, very good at using the internet, to some degree, distracts from the many, many, many people for whom the internet is fundamentally a distraction machine, or for whom the internet is creating, because of what we built on it. You know, shorter attention spans — how many people would have had an idea, sitting in a room by themselves, or taking a walk, that they never have now, because they never have to have a moment where they’re thinking alone? +

+

+ And so one thing that I think we’re all loathe to do is we’ll talk a lot about how it’s weird that we have so much more knowledge, but productivity isn’t increasing faster. It’s weird that we have so much more rapid communication between researchers, but science isn’t advancing faster. And then, the idea that maybe there are things happening to us that makes us less able to use that increasing stock of knowledge well, or makes us less able to collaborate in a useful way, I think, gets dismissed rather quickly. +

+

+ But I don’t think it’s totally implausible. Now, I don’t want to say, like, the greatest technology we ever had was letter-writing. Obviously, the greatest technology we ever had was blogging in the early aughts when I became a blogger. And whatever happened in your 20s is, like, as good as it was ever going to get. +

+

+ But I do wonder about these questions. And I think something Mokyr is right to put a lot of attention on is communicative cultures. Communication is how we collaborate. And if communication is in any way getting worse, it’s going to have pretty big macro effects. +

+

+ The other thing is if you believe these cultures matter, weirdly, as big as we’re getting, the internet allows a certain disciplines culture to stretch boundaries and borders in time in a way that it would have been harder. I suspect that labs were more different 50 years ago than they are today. +

+

+ The countries and the disciplines of researchers and the cultures of researchers in countries or cities are more different from each other 50 years ago than today, which is great if we have the best of all cultures today, but it’s not that great if you actually think variation is really important. +

+
+
+ patrick collison +
+
+

+ Let’s wrap up there. So first, I agree, as a basic matter, that there are welfare losses occurring across society that we should be worried about, and probably everybody listening to this is familiar with the Stephen Pinker case for optimism, and rather than focusing in the headlines, you zoom out, look at these long-term time series. And once one does that, things seem a lot more encouraging, whether you look at it by income or life expectancy or infant mortality or choose your metric. +

+

+ Something that’s been striking to me of late is if you change the x-axis on those time series, and look at many of those phenomena and trends over a much shorter window, the valence changes substantially, and life expectancy in the U.S. is now, in fact, declining. According to C.D.C. data, 54 percent of teenage girls now report persistent feelings of sadness and hopelessness. And you could say, well, teenagers were never stereotyped as the most cheerful lot, but we do have some degree of longitudinal data here, and that number is up from being in the 20s as recently as 2009. +

+

+ 1/2 the population now is either prediabetic or diabetic — again, according to the C.D.C. Basically, point is, when we look at more recent windows, I think there are plenty of aggregate, emergent, complicated outcomes and phenomena that should give us concern. On the degree to which we should attribute the diagnosis to the internet or to our kind of communication media more broadly, it’s less clear to me in that — not saying it’s not true, but presumably, the life expectancy one is not — or at least if it is, the mechanism has to be very complicated. +

+

+ There are a bunch of other health-related ones. So take, for example, say, the incidence of diabetes or pre-diabetes. And you could say, OK, fine, all those things might be true, but they’re totally different. I guess the question I wonder about is, well, we know that lots of basic biological outcomes are correlated with mental states and so on. +

+

+ And so to what degree is there some more nuanced and complicated relationship there? But I think it’s a fair question, and I wonder a lot about it myself. +

+
+
+ ezra klein +
+
+

+ Let me ask you about how you think, over the long period here, about the relationship between technology and equity or egalitarianism. And something specific is in my mind. I flicked earlier at the way the Industrial Revolution, for an extended period of time, seems to have reduced a lot of people’s living standards. And it wasn’t till later you had changes in redistribution in labor unions and labor protections that the amount of material prosperity that was generating created more broad-based prosperity, particularly at a very high level. I don’t know that you can sustain that kind of thing today. We have much more a small-d democratic culture. If things aren’t working for people, it’s much easier for them to organize and be heard. +

+

+ I think there’s a much more direct and complicated relationship now between whether or not people feel benefited by technology, and whether or not they are going to accept the conditions and the risks of rapid technological advance. But I’m curious, from your vantage point, how you see that both kind of historically and currently. +

+
+
+ patrick collison +
+
+

+ I agree with that. I worry a lot about the basic stability of a society that does not successfully generate and make sufficiently broadly accessible the benefits of economic growth. The world simply has too little prosperity. And if it is not the case that people in the U.S. or people in any country — if they either feel like things aren’t progressing, or if they feel like maybe somewhere distant from them, things are progressing but they personally will never be able to benefit from it, I think we put ourselves in a very dangerous and likely unstable equilibrium. +

+

+ And if you go back to — well, you don’t have to go back very far in history to see, obviously, plenty of instances where this kind of instability brought the whole house of cards down. And so as a consequence of that, I worry a lot about, how do we simply make sure that — or one of the small things we each individually can do to try to make sure that society is generating enough economic gain and enough broadly experienced welfare gain that the whole compact can be maintained? +

+

+ And again, I don’t think there’s a ready neat kind of singular answer to that. Maybe Stripe as part of our small little contribution in one little fissure. But I think the central question you’re getting at is super important. +

+
+
+ ezra klein +
+
+

+ And one of the questions I wonder about there — we’ve talked about the way progress has been very geographically lumpy, let’s call it, right? There’s a lot that happens in very small places, and it ends up affecting the whole world. Obviously, then, the gains of progress sometimes have that quality, too. +

+

+ And I do think of one of the politically destabilizing effects of the past, let’s call it, 30 or 40 years of digital progress, is being the concentrations of wealth. We just used to have a lot more spread. Even putting the questions of rising inequality aside, just where rich people were was different. +

+

+ And so where they were giving a lot of money to the local hospital was more spread out, say, across the country or in other countries across the land. But here, even as the internet is supposed to democratize distance, and in many ways, has — I mean, telework is not a fake phenomenon. It has really concentrated the wealth of that to, literally, where we’re sitting, but to New York. There’s a lot of money now in Austin. +

+

+ And then, on top of that, you often have barriers of entry, in terms of how many homes can be bought. So it’s not even like people can move to the place where all the economic opportunity is happening. And I do think that creates some of the skepticism you see of technology. +

+

+ I don’t think a lot of people’s — I think people are really excited about a lot of the goods they’ve gotten from it. But in this kind of macro political sense, as you’re saying, in a period of a lot of change, a lot of folks with real backing in the data don’t feel life has gotten better at the macro level. +

+

+ Life expectancy, happiness, political stability — it’s not like you can look around and say, well, I got this computer in my pocket, and everything else is going great, too. It’s like, I got this computer in my pocket, and what it keeps telling me is that everything is going to hell. Now, maybe it’s telling me that a little bit too much, but there is validity to the narrative. +

+
+
+ patrick collison +
+
+

+ Yeah. So again, vehement in agreement on the sort of central importance of making sure that improvements in the standard of living are actually broadly realized across the society. That, too, I think, could serve as a manifesto for some of these Progress Studies ideas. +

+

+ On the internet in particular, or on technology and the technology sector and so forth, I think it’s complicated and difficult to try to sort of fully collapse or linearize it or something, where on the one hand, you have some of these concentration dynamics you identify. At the same time, of course, it is also a tremendous and incredible dispersal agent in making some of those possibilities and opportunities be more broadly available. +

+

+ I think it’s dangerous to take an excessively U.S.-centric perspective here. If you interact with or look at survey data, or otherwise try to assess what’s the sentiment of people in Poland, what’s the sentiment of people in India, or what’s the sentiment of people in Indonesia, they view the internet extremely positively. And I think correctly so, where their opportunities for advancement would be substantially curtailed in the absence of much of what the internet makes possible. +

+

+ I think in places like the U.S., or actually, even at home in Ireland, some of this story is complicated by lots of other things, but including — and I think, substantially — dynamics around housing policy, where there’s an extensive literature showing that, for example, a very substantial moderating effect on the increase in wealth inequality that would otherwise have happened, or income inequality, was geographic reallocation and people responding rationally to, OK, things in New York are going great, or things in California are going great. +

+

+ And if you look at the rate of increase of the Californian population, say, through the 1960s, that was a tremendously potent mechanism for us redistributing some of the economic gains that were being realized at the time. And of course, now, we have this crazy position, where California is losing population at the same time where the market caps of these companies and the profits of these companies are increasing very rapidly. +

+

+ But as one assesses that dynamic and tries to ask the question of, well, why aren’t these gains being better or more broadly distributed, it’s certainly not clear to me that the answer even lies in the realm of technology qua technology. And I think it’s certainly more broadly, again, some of these considerations like geographic allocation. +

+
+
+ ezra klein +
+
+

+ Let me ask one more question on the geographic dimension, and then I’ll move on to it. And it’s on my mind, in part because when I try to think about progress, when I try to think about what inventions and innovations are coming really quickly, I actually see a bunch here. And I’ll use A.I. as an example. +

+

+ I’ve met people who are trying to automate a bunch of legal contracts. It makes a ton of sense. People pay a lot all over the country — to some degree, all over the world — to get fairly basic legal contracts drawn up — wills and real estate documents and merger agreements and all kinds of — from the small to the large. +

+

+ If you imagine that getting really effectively automated, though — +

+

+ you think about Saint Louis, Missouri, where some of the people who are important pillars of the community work in law firms there, and what they do is contracts. They do estate planning and all the things that people have to do in contracts. And if it actually does get concentrated to really, really great contracting firms in the Bay Area or in New York, on the one hand, the democratizing potential will really be realized. Those contracts will get cheaper. +

+

+ There’s also a theory in crypto of smart contracts. And on the other hand, you really will have a lot of that — the gains of that, economically, going to smaller areas and aggregated across a bunch of different domains. So graphic design, in all kinds of areas of the country — midlevel graphic designers get paid to make logos for local businesses. +

+

+ It’s pretty clear they’re going to be able to do that really, really easily on things like DALL-E pretty fast. So you can imagine a lot of that area getting wiped out. And you kind of run through a couple of these. And before you get to really unbelievable and sci-fi-like dimensions of artificial intelligence, you just have a thing that is going to democratize a lot of capabilities in a way that’s going to put the money for those capabilities both a little bit back into the pockets of the people who need them, and then a lot into the people who run the best A.I. rigs and is going to have a really weird geographically destabilizing effect. +

+

+ And that paradox of the internet both democratizing geography, and then concentrating wealth and capital in very small areas is, to me, a central challenge. Because if you get that wrong, if it goes too much in the concentration area, I think we’re going to lose a lot of the political stability we need here. But you’re more on top of these technological advances than I am. Do you think the trends there are going to play out differently than I’m worried they will? +

+
+
+ patrick collison +
+
+

+ First, yeah, it’s not — I don’t think it’s foreordained whether or not these are going to be centralized technologies. I don’t know. And on the one hand, there’s, I think, an obvious feature we can contemplate, where there are only three A.I. models, and they are rooted in the hegemons, the citadels of Silicon Valley technology, and we all are digital serfs who are subsistence-farming on their gains. +

+

+ I think there’s also a very plausible story where these technologies prove substantially less defensible than we might have expected, and where, instead, they have this enormously decentralizing effect. Because otherwise, economies of scale that only large firms could benefit from can now be realized and pursued, even by massively smaller firms. And if we look at the recent history of A.I. — I don’t think any clear story there, but it does feel to me that it has been more biased towards the second story than the first. +

+

+ There are now multiple companies with large language models. There are a number of very successful open-source A.I. efforts. Actually, there was a really cool example from Replit, which is a service — it’s a programming I.D. in the browser, used by kids learning to code, but also increasingly used by people who are pursuing serious programming. +

+

+ And they recently released a GitHub copilot-like technology, where it will kind of autocomplete your code in the editor, and where you can do some pretty cool things. Like, you can highlight a block of code and ask it to be explained, and it’ll turn code into natural language, into English, and say, hey, here’s what this code is doing. +

+

+ Anyway, they wrote a blog post about how they built this, and they describe how it was built by one guy over the course of a couple of weeks. And maybe that’s only the case in the early days of this AI technology. I mean, in early computer games, the first games were built by a single heroic person, and now, it’s these gigantic studios and enormous CapEx budgets. +

+

+ And so I think the fact that this is the case today doesn’t mean that it will remain the case through time. But my takeaway is that at least not foreordained that AI or any of these other technologies will be centralizing forces. And then, the other thing to observe is that when we talk about these being centralizing, I think there’s a question as to, do we look at it in relative or absolute terms? +

+

+ And my contention would be that, both from a moral standpoint, but maybe more importantly from kind of a political-economy standpoint, what will matter is whether, on an absolute basis, people feel like they are realizing opportunities, their lives are improving, that things are getting better, that their kids will be in a better situation and so forth. And exactly how much value is realized by the companies themselves doesn’t actually matter that much, compared to that former question. +

+

+ And whether A.W.S. or whether any of these organizations has super high or super low profit margins, I don’t know is nearly as important as what is the actual effect on these communities and individuals across the society. And so in as much as one means — by centralizing, one means a large share of the profits, I think it is probably a more useful framing to look at it instead in terms of absolutes, and in particular, the absolute surplus generated by the users. +

+

+ [MUSIC PLAYING] +

+
+
+ ezra klein +
+
+

+ What have you come to believe about the relationship between progress and war? +

+
+
+ patrick collison +
+
+

+ I am somewhat skeptical that war is as conducive to breakthroughs as we might intuitively conclude, or as is sometimes claimed. You’re probably familiar with Alexander Field’s work on the ‘30s here. And his basic claim is, the productivity gains we often attribute to the Second World War in the U.S. — like, those foundations actually were laid in the ‘30s, and then the first half of the ‘40s were a period of decreasing productivity as we massively, inefficiently reallocated our economic resources for the purposes of winning the war, which was probably a good thing to do, but inefficient in narrow economic terms. +

+

+ And he has a new book coming out, I think, next month, that sort of extends this argument into the ‘50s. +

+

+ And as one takes stock of the scientific breakthroughs — and so Stripe Press recently republished Vannevar Bush’s memoir, where he takes stock of this. +

+

+ And obviously, you have, say, the Manhattan Project, and that’s a big deal, certainly. But it doesn’t feel to me that had the Manhattan Project not occurred, that peaceful development of nuclear technology would have been massively stymied. Maybe it would have taken another 10 years, but it was already happening to some meaningful extent. +

+

+ And then, as you take stock of all the other breakthroughs that took place in the U.S. during the Second World War, there were some meaningful stuff like blood plasma and blood transfusions. There was some significant breakthroughs there. Some of the first antimalarial medications, radar, the proximity fuse, which I’m not sure is all that useful outside of military applications. +

+

+ But by the time you get down to invention 6 on the list, I don’t know that as you compare that list to, again, some counterfactual of what would otherwise have ensued, that it looks radically better as you take stock of the Cold War and the enormous fraction of our economic resources and human capital that were devoted towards us, that the gains necessarily look that impressive. And so again, it’s super hard to judge. You don’t have proper controls and so on. But I’m skeptical. +

+
+
+ ezra klein +
+
+

+ Let me take the other side. So there’s a question of, during war, how much did we invent during World War II. And that’s a question of how much the threat of war or the competition with an adversary ends up charging up innovation and convinces us to put resources, both in terms of people and in terms of money, and maybe in terms of institutions, into projects we wouldn’t otherwise have done. I think there’s an argument, at least, that we went to the moon because of the Soviet Union. It would not have done that for some time. Probably would have eventually done it, but also, who knows? +

+

+ As Derek Thompson, who I’m working on a lot of these ideas with, likes to point out, the Apollo Project was unpopular. It was not something that commanded wide popular support. +

+

+ Even now, if you look at the CHIPS Act that passed, it passed, with all that spending on semiconductor research and other kinds of next-generation technologies, under the framework of, let’s compete more effectively with China. If you look at all the things Darpa has done or been part of, the fact that “defense” is the first word in the Darpa acronym, I think, is meaningful. +

+

+ There’s something about what threat persuades societies to do, and persuades them to do technologically or what risks it allows otherwise-more-cautious governments to take, or what failures they could justify that allows them to have big successes. Something there doesn’t seem to small to me. +

+

+ And maybe it’s my political side, where I so often see scientific funding justified in Congress in terms of countries we’re competing with or are adversaries with. And I see what the defense industry can do that other institutions cannot, because they don’t get a lot of political blowback. But I don’t know. +

+

+ I worry a little bit about how much we seem to need the threat of another to accelerate things. And in a small way, maybe, we see what the pandemic — where we were willing to move much, much quicker on things like mRNA technology than I think we would have outside of it. +

+
+
+ patrick collison +
+
+

+ Yeah. So I think it’s certainly true that the crisis can cause the discontinuous shifts that have large effects, which in your example, say, are probably super beneficial. I don’t know. If you take Darpa as an example, it started as Arpa, as a more open-ended research institution and set of programs, and then with the Vietnam War, had the D pretended to it. +

+

+ And we decided, in the face of threat, to make it more applied, to take more seriously its translational and kind of, quote unquote, “competition-oriented mandate.” And I think that was bad for Darpa. And the internet, which arose under Arpa — it’s hard to think of innovations of similar magnitudes that then occurred in then-Darpa’s subsequent, say, two decades. +

+

+ If you take, say, U.S. science in general, the war — the Second World War — to some extent, the first, but much more so the second — precipitated an enormous centralization of U.S. science in its aftermath. Because we really marshaled together all of the — or a significant fraction of the scientific capacity of the U.S. in service of the war effort. +

+

+ For, example the 50 percent overhead, the fraction of government grants that goes to universities — that was chosen in the early days of the coordination of the war effort, and has now become a kind of a pillar of academic and research funding in the U.S. And in the aftermath of the war, we sort have this question of OK, we’ve kind of pulled everything together. Now, what do we do? +

+

+ And that became, in various ways, the N.I.H. and the N.S.F. and so on. I mean, the N.I.H. predated it, but the growth of the N.I.H. really occurred after the war. And the federal government, shortly thereafter, for the first time, became the majority funder of US science. And all that centralization — and I mean, you pointed out the benefits of variety and of experimentation and of heterogeneity, and having some degree of institutional and structural diversity and so on, I totally agree with all of that. +

+

+ And I think all of that was very meaningfully curtailed by, again, the aftershocks of some of the threats that we faced during the war. And if you think about the things that we’re maybe happiest about having happened — the founding of the major new U.S. research universities in the latter parts of the 19th century or the revolution in health care and kind of medical practice that first happened at Johns Hopkins, and then kind of codified in the Flexner Report, or the great industrial research labs of Bell and Park and so on — or excuse me — Xerox — they didn’t obviously come from a place of fear or a threat. +

+

+ They came from a place of hope and optimism and opportunity. And maybe there are some inventions that you’re more likely to get to from some of these external pressures. But yeah, if you gave me a dial, and I can kind of turn up or down the threat or fear index of society, it’s not super obvious to me that one would want to turn it up if what one cared about was the aggregate rate of progress. +

+
+
+ ezra klein +
+
+

+ That’s a good bridge, I think, to the question of institutions. And I take one of the main concerns of yours, of progress studies, as being around institutional slowdown. You have this idea that we don’t meta-maintain institutions very well. +

+

+ They start in one place, and then over time, they crust over, and we don’t really know what to do with that. Give me a little bit of your thinking there. +

+
+
+ patrick collison +
+
+

+ I think institutions, the cultures they instill and act as kind of coordination points and training sites for — those of enormous consequence — I think much of the success of the U.S. and of various other Western countries has, in substantial part, been attributable to successful institutions. Most people would accept, I think, that there is, to some extent, consistent trends that tend to happen with institutions through time. +

+

+ Just maybe most basically, the problem that gives rise to an institution in the first place is probably a pretty real and significant problem. No one would have taken the time to found the institution if it wasn’t. And there can be some degree of drift there, where we don’t necessarily decommission the institution once the problem has subsided or abated. +

+

+ And then, you tend to attract a certain kind of person in the early days of an institution — people who are slightly less status and reputation and procedure-oriented, because a new institution almost never has that. And then, through time, the sort of collective or the mission-oriented incentives of the institution can kind of drift somewhat from the individual incentives that particular people are subject to. +

+

+ I think all this stuff exists. And the thing that I observe, or that I just find myself thinking about is, we’ve had eras of institution formation in the U.S. It has not been kind of a constant rate through time. And the New Deal maybe, and say, the 30 years afterwards, and the Great Society — we bookend it with those start and endpoints. +

+

+ That was a period of tremendously active institution construction and formation in the U.S., Darpa being — or Arpa originally being a good example, and indeed, NASA. And I guess I find myself wondering, one, if we didn’t have any of these institutions — and I’m not saying we should get rid of them. But if we didn’t have them, what institutions would we found today, first, and how high in the list would NASA be, for example? +

+

+ And then, secondly, in as much as we accept that some of these institutional dynamics exist, like the fact that sclerosis as an emergent property arises, what do we do about that? And lots of people have told us it’s pretty — doesn’t need a lot of teasing apart to see it as one compares NASA and SpaceX and the respective budgets, and the respective achievements, and so forth, I think it’s hard to not at least wonder about their respective efficiencies. +

+

+ And say, if society could only have SpaceX or NASA, which one would we choose, and what should we conclude from that, and to what extent do those phenomena generalize elsewhere? I don’t have answers to these questions. But I find that in the political discourse — not that anybody is celebrating that, but in the discourse, it’s very easy to get, I think, very wrapped up in questions of optimal funding levels, and should this number be 10 percent or 50 percent or higher or whatever, whereas to me, a lot of our satisfaction with the outcomes seems to hinge on deeper questions about the nature of the institution. +

+

+ I don’t know that the problem or benefit, or anything good or bad about NASA is attributable to the budget, per se. It seems more, kind of, resonant in some of these deeper cultural questions. And then, in the recent pandemic, or in the — I don’t know. I was going to say, ongoing pandemic. But I guess as of two days ago, with the President’s verdict, it is now over. +

+
+
+ ezra klein +
+
+

+ It’s over. Congratulations, everybody. +

+
+
+ patrick collison +
+
+

+ Exactly. But anyway, I think that was maybe a vivid demonstration of many of these dynamics, where I don’t know this any of the story about the institutional response to the pandemic should be primarily one of funding. I think it’s much more about the dispositions and the attitudes and the cultural biases of entities like the N.I.H. and the F.D.A. and the C.D.C. +

+
+
+ ezra klein +
+
+

+ I find the NASA SpaceX example an interesting and provocative one. Because on the one hand, I think what you’re saying is completely true. And where a lot of the NASA programs and projects have gone in recent decades, is just — it’s sad. It’s just a sad story. +

+

+ And on the other hand, the idea that you — the thought experiment of choosing between NASA and SpaceX — the thing that it immediately asks is, well, you can’t. Because without NASA, there is no SpaceX. And one way the private sector handles a lot of these questions — I mean, I’m always struck by how much of the way biotech research works is that big pharmaceutical companies acquire small biotech firms that have made a breakthrough or have come up with a very promising candidate. This is kind of an accepted thing that the big companies — they do a fair amount of research, but a major, major innovation transmission there is small groups do more, quicker, and they’re just going to buy them. And the NASA SpaceX example has a little bit of that dynamic to it, although with a different mechanism of financing. And I don’t know. I wonder if there aren’t deeper lessons there. +

+
+
+ patrick collison +
+
+

+ Yeah, I don’t mean here in the NASA example — like, I don’t think reducing it to a simple binary of this-or-that is correct. It’s more, what should we make of the differences in these two organizations? And given those observations or beliefs, what do we then think an efficient outcome might look like? +

+

+ And do we think that where we are today — this prevailing status quo — is optimal? Or are there other things we can do better? And maybe an important thing to say within all of this is, to the extent that these are all kind of inevitably determined outcomes, maybe it doesn’t really matter if we think things would be better or worse. We’re going to end up in the same place, regardless. But I think for all of these, it’s super contingent. And various aspects of both funding decisions and, kind of, the precepts and methodologies of the N.I.H., how we design I.P. law, how we regulate and require and run clinical trials — there are tons of individual contingent decisions that we kind of have collectively made that give rise to the biotech and to the pharma ecosystem. +

+

+ And certainly, in the case of space, you know, like, it doesn’t have to be this way other. Universes, no pun intended, are possible. I think perhaps the thing that people underappreciated with science in the U.S. is, it has been very different in the not-too-distant past. +

+

+ Peer review is a relatively recent invention. Modern journals are a relatively recent invention. As I mentioned, the federal government being the primary funder of basic research is a relatively recent invention. +

+

+ And molecular biology was, in significant part, a thesis by Warren Weaver at the Rockefeller Foundation. There’s a thing here, and we should aggressively pursue it. But importantly, it was not — it required an institution, an organization, that was not part of the standard apparatus, for want of a better term. +

+

+ And the Broad Institute, over the last 25 years, has been enormously successful in the field of genomics and functional genomics and CRISPR, et cetera. And the Broad Institute is itself a kind of structural innovation, breaking somewhat from the more traditional prevailing university model. And so I think the fact that so many of our successes are associated with some degree of structural and institutional change should be somewhat thought-provoking for us. +

+
+
+ ezra klein +
+
+

+ You’ve been trying to work in the space of institution-building here, too. I want to talk about Fast Grants and about Arc a little bit. So let’s begin with Fast Grants. What is it, and what has it taught you? +

+
+
+ patrick collison +
+
+

+ Well, it’s mostly “what was it.” In the early days of the pandemic — well, I should preface all of this by saying — well, I’ll reaffirm my preface that I don’t know, to every question. But more importantly here, I will say, my now-wife is herself a scientist. We’ve known each other since we were teenagers. We spend a lot of time talking about science in various forms. +

+
+
+ ezra klein +
+
+

+ You met — am I allowed to say this? You met at a science competition. +

+
+
+ patrick collison +
+
+

+ That is true. We met at a science competition, 100 teenagers, and — +

+
+
+ ezra klein +
+
+

+ And she beat you. +

+
+
+ patrick collison +
+
+

+ And yes. I was the runner-up, and she was the winner. I had created a programming language and a new dialect of lisp, and she had created a new treatment for urinary tract infections. And so I really don’t envy the judges for having to figure out what framework one should use to make all these comparisons and lots of other people. +

+
+
+ ezra klein +
+
+

+ You sound a little bitter, man. +

+
+
+ patrick collison +
+
+

+ [CHUCKLES] I was gonna say, but no, we can all agree this the correct outcomes ensued. Anyway, so we were living together in March of 2020, holed up. And a number of her friends and colleagues were unsurprisingly with, I guess, a large fraction of all biology scientists, were trying to urgently repurpose their work to figure out, well, could they do something that would be somehow benefit to accelerating the end of the pandemic? +

+

+ And by early April, so a couple of weeks into lockdown, when it was becoming apparent and striking to us, which was it is difficult for these people to get funding for their work. And that might sound a bit, kind of, surprising, because you think, well, don’t they have some degree of money already? And couldn’t they just go and just spend that? +

+

+ But there are, obviously, significant rules around and restrictions around that which one can do with one’s grant money. This is money provided by the government for a purpose. And so it’s not like you can go and readily spend it on something totally unrelated. +

+

+ And the money is administered by the university, and so you have to go through their proper procurement processes. Point is, lots of restrictions on scientists’ pecuniary ability to suddenly repurpose the research agendas. And we kind of thought, well — we assume maybe in the early weeks, that presumably various bodies — I don’t know who — some kind of amorphous other, some combination of C.D.C., F.D.A., N.I.H., philanthropies — whatever. +

+

+ Somebody will come along and just give these scientists the obvious money that society clearly should, so they can go, and they can pursue these programs. Didn’t seem to be happening. I mean, to be fair, I don’t want to give us too much credit. Various people were doing things right off the bat in various different places, but we just personally knew of lots of specific examples of really good scientists who were unable to make progress of their work to the extent that they would like. So we tried to set up what we thought would be a pretty small initiative, and called Fast Grants. +

+

+ The basic idea would be, you send us some kind of proposal. And initially, within 48 hours, you would get a funding decision and either receive money or not. We started out with a pretty small amount of money. +

+

+ The initial donors — we were among them, but there were a number — contributed, best I recall, about $10 million. Launched the website early April 2020. Quickly inundated with, I think, four and a half thousand applications, which, given our promised 48-hour turnaround, was somewhat challenging. +

+

+ I should say this was myself. This was Silvana, my wife, and this was Tyler Cohen. So we had an immediate question as to, how do we actually run a philanthropic endeavor? And how do we stand it up in very short order? And he, through Mercatus and through Emergent Ventures, had some experience of very efficient and somewhat-scaled grant-giving. And so the three of us worked together to put it together over the course of a week or so. We proceeded over the course of, roughly speaking, the next year, slightly more, to make about 200 grants, eventually dispersing almost — or slightly over, actually — $50 million in total, to universities around the world, though primarily in the U.S. +

+

+ And you ask, kind of, what did we learn? A big surprise was how slowly other parts of the establishment mobilized. And various of the projects we funded or the labs we funded and so on — they’ve gone on to now do — none of them were directly implicated in the vaccine research project that ended up yielding so much fruit. So again, I don’t want to give Fast Grants too much credit. Eventually, the thing that really mattered, we had nothing to do with. But versus the projects, things like Saliva Direct, which was in the summer an early discovery that saliva tests work basically as well as the nasopharyngeal swabs we were all being subject to, or various discoveries around possible therapeutics, some of which are — still continue to go through clinical trials, and may still turn out to matter to a significant extent. And that 500 people are still dying in the U.S. per day from Covid, and — despite the existence of the vaccines and so on. +

+

+ So anyway, various discoveries ensued that I think will prove to be important. And the second thing we learned, which is not really related to Covid or the pandemic, but has certainly been significant for us, is — it just got us thinking more deeply and broadly about the questions of, how do scientists choose what to do? And what are the constraints they’re subject to as a practical and applied matter? +

+

+ And this gets back to all this discussion about both culture and institutions. And towards the end of Fast grants, we ran a survey of the grant recipients. And these are essentially all people who don’t normally — certainly don’t normally work on Covid. Covid didn’t exist. But they don’t even normally work on viruses, for the most part. These are basically kind of broadly drawn as a cross section across biology. And we just asked them, as a general matter in your regular research, if you could spend your grant money however you want, how much would you change your research agenda? So not an increase in the funding level, which tends to be what we discuss in as much as we’re discussing science policy across society. But much more specifically and narrowly, if you had complete autonomy in how you spend whatever grant money you’re getting, how much of your research agenda would change? And our intuition was that maybe a third of people would like to be doing something meaningfully different to what they actually are. +

+

+ But of these scientists, and these are really good scientists, four out of five told us that they would change their research agendas, quote, “a lot.” We gave them three options. Not much, or not at all, a little, and then a lot. Four out of five chose the maximum option on our survey. So I just find this incredibly thought-provoking. +

+

+ Basically, we seem to be in a situation where most of our top scientists aren’t doing what they think would be best for them to do. And we could say, no, our various committees and governing bodies and decision-making apparatus and so on, they know better. And I’m not saying it would be completely unreasonable for one to maintain that. But that would seem to be a very central question about the construction of our scientific apparatus. And I think that should give us some pause. +

+
+
+ ezra klein +
+
+

+ There are a couple things there. One is that it is a consistent observation I have learning about new areas that there is a way we’re taught the thing works, or people think the thing works, and there’s this huge middle layer. Right? So in politics, which I know very well, and legislation, you have the “Schoolhouse Rock” version of how a bill becomes a law. And then it’s, like, a filibuster is how a bill becomes a law or does not become a law. +

+

+ We were talking about drug innovation earlier. I think the folk way people think it works is we make a discovery about a drug, and then, like, we make a drug out of it after some tests. But you talk to people who work on pharmaceuticals and just clinical trials. +

+

+ And in science — I think if you had asked me as a high schooler, had some science classes, I’d have told you something about the scientific method. And then you talk to a scientist, and it’s grants. Like, grants are how science works. Grants are the middle layer between — you are a scientist, and you can do some science. And grants are how the N.S.F. and the N.I.H. work. They’re how a lot of the universities work. There’s fund-raising. I mean, there are different ways that it happens. To make the question of “Are we doing science well?” a little bit more precise, I think one version of that question is, “Are we doing grants well?” And I think that question is more tractable. People don’t feel as defensive about it. +

+

+ But I’ve talked to a lot of scientists in the course of my work. I’ve covered health care for my entire career. And I don’t know any who think we’re doing grants well. I don’t know any who will not complain to you for hours. +

+

+ And they may be wrong. And I do want to note — because they also just have somewhat different incentives. I mean, I was noting earlier, and I think it’s very real. The government, particularly when it gives out grants, needs to worry about the reputational cost of the grant. If the grant goes wrong, if not enough of the grants pay out into useful research. If Rand Paul can stand up in Senate and make what you did sounds silly, these things really end up mattering. And so you get a process that is optimizing for a lot of different things. +

+

+ But the question of whether or not we do grants well ends up being really, really, really important in every country that does major capital science that I know of, and is just not the main question for a bunch of different reasons we ask. +

+
+
+ patrick collison +
+
+

+ Yeah. Another question we asked in our survey was how much time they spend on the grants. I think to some extent, this is perhaps — at least, of those who’ve spent some amount of time interacting with scientists, kind of more broadly known than perhaps the finding with respect to how they do — or the degree to which they can choose what they work on. But we found that — or they reported to us that they spend on the order of 40 percent of their time on grant administration. +

+

+ And even if one were to maintain that the decision-making apparatus around what scientists do is somehow efficient, I think it is a very tenuous position to also try to argue that 40 percent of the best scientist’s time is optimally allocated towards grant applications, authorship and administration. And we’re not talking about an inconsequential 40 percent here. +

+

+ I mean, this is 40 percent of the time of this super-elite 10,000, 100,000, whatever it is, some relatively finite number of people. And we’ve chosen to take and to redeploy almost half of their time in service of technocratic, bureaucratic undertaking. And getting back again to this point about people perhaps falsely assuming that things have been more inter-temporally consistent than they have, that percentage has increased very substantially over the last couple of decades as the overall edifice of science has grown, and as the kind of acceptance rates and the various thresholds for various grants has become more exacting. +

+
+
+ ezra klein +
+
+

+ How we allocate people’s time is really important. But also, just how we allocate talent is really important. And it brings me to something you said that I wanted to ask you about. This was in response to a question about whether big tech companies are hogging all the talent in society. +

+

+ And you said, quote, “I don’t think that the ambitious upstarts who go into high speed rail in America, anyway, are going to have a great time or have much success in convincing their friends to follow them. And I suspect that for various reasons, too many domains look somewhat like high speed rail.” And so you go on to say that there’s a view that the internet is a frontier of last resort, and that you don’t think that’s totally wrong. +

+

+ So tell me about that. Tell me about the idea of the internet as a frontier of last resort. But behind that, this idea that other frontiers where talented people might want to go and make their mark on society have closed. +

+
+
+ patrick collison +
+
+

+ You’re familiar with and you’ve probably written about the Stephen Teles idea of kludgeocracy. And I kind of like the term “kludgeocracy,” because rather than making some of the inhibitions that people might encounter in pursuing something like high speed rail, rather than casting those as being deliberate, the valence is more that it’s this kind of emergent, inadvertent and kind of complicated phenomena that nobody perhaps particularly wants or chose. +

+

+ And I think the case of California’s high speed rail is quite striking, where — you’ve written about this and kind of similar projects and the New York subway expansion and so on. And congestion pricing and so on. But it’s striking where it’s not actually obviously a question of first order political will. Like, we’re willing to fund the high speed rail in California. We’re clearly willing to invest in building the subway expansion in New York. But somehow, somewhere between that first order decision and desire and our actual ability to kind of instantiate it, something really goes wrong. +

+

+ And you contrast that with stories of — in the case of, say, California, Henry Kaiser and these various other early part of the 20th century operators in the physical realm. And for a variety of reasons, but mostly prosaic state and county-level complications and things that would extend the time horizon of one’s project, it has simply become meaningfully less-appealing for those people to undertake these initiatives. I mean, Foster City, not too far from where we are now, that’s named after the eponymous Mr. Foster. +

+

+ He was a developer. He decided, well, with reclaimed wetlands, I’m going to build a city. California is growing quickly. The Bay Area is a — kind of propitious and will be a long-term successful area. And I think this place simply needs more housing. +

+

+ And he, with that kind of founder energy, was able to give birth and rise to the city that now bears his name. I haven’t met anybody pitching me on a similar city on the shores of the Bay in the last couple of years. But I would imagine that were one to adopt that ambition today and to propose that maybe the San Jose Marsh wetlands should themselves be an expansion of San Jose, I don’t think one would get very far. +

+

+ And in fact, even for much more sort of limited things, like additional runways or runway expansions at S.F.O., even they have now been stymied for decades at this point. That ability to translate that into something enunciated has dissipated and deteriorated. And then I think the kind of individual version is, and if I want to be that heroic solar farm entrepreneur or railway magnate, that my practical ability to do so has been meaningfully curtailed. +

+
+
+ ezra klein +
+
+

+ Yeah. I mean, that’s what I’m getting at here a little bit, which is talent really matters for a society. Where the most talented people go really matters for society. And a lot of those people want to go somewhere where they can have a really big effect. I think there’s been a huge rush to digital land because you can build on digital land. You can build quickly. I mean, it’s interesting to some of the dynamics we’re talking about, the temporal dynamics we’re talking about, that you see this dynamic even within the tech world. +

+

+ There was a while where it was really exciting to go join Facebook, go join Google, go join one of the big companies. Because you could do so much. And that’s still, to some degree, true. But they got really big. And so crypto got — whatever you think of crypto, one thing that is exciting about it to people is the idea that it’s open land. That you can go in there and have a really big effect on it. Build something new just with a couple of friends that might change the whole direction of the field. +

+

+ And on some level, it’s always going to be harder for, say, putting high speed rail through the middle of California. Right? I mean, just building things in the world is just going to be tougher. +

+

+ But on the other hand, if you make building things in the world too hard, if you make grants too difficult — if you — I know a lot of doctors who their advice to young people is don’t become a doctor. And it always breaks my heart a little bit. We need really great people to be doctors. +

+

+ And their point is not, don’t go heal sick people. Their point is, being a doctor is too hard now. The amount of time you spend dealing with insurance agencies and malpractice insurance and boards, and this and that, it’s just too much administration. +

+

+ When industries become very complicated to operate in, you want to select for people who are good at operating complicated industries, which may be different than the people who are good at moving really fast and changing things dramatically. But two, you kind of subtly bias where different kinds of people in your society go. I think in China, if you want to change a lot, you still probably go into infrastructure construction, among other things. Right? +

+

+ The idea that you might be a genius rail mind, in China, that’s great. There’s probably a lot of rail you can make. That’s not true here. And the point is not to make too much of the rail example, but to make a lot of the idea that talent flows towards where it can have an effect and people can live the kinds of heroic lives they want to lead. And if we have subtly pushed a lot of people into maybe not the right — not the socially optimal directions, that over time will have a pretty big effect on a society. +

+
+
+ patrick collison +
+
+

+ I think a constant is that some number of ambitious young people will want to do something, as you say, heroic. And yeah, I think maybe two things have changed. For one, for whatever reason, our predisposition to putting those people in positions of authority has diminished. And then secondly, even if placed, their ability to actually execute, again for various reasons, has been attenuated. +

+

+ I’ve been reading about the university founders and presidents and those associated with some of the great US research institutions. And one thing that is striking is how many of them were so young when placed in those positions of authority. You know, Daniel Coit Gilman at Johns Hopkins, or William Rainey Harper at the University of Chicago. I think he was 32 when he was appointed president of the University of Chicago. Even in the recent past. +

+

+ So my dad was in the first year of the University of Limerick in Ireland. Or at the time, it was called N.I.H.E. It kind of acquired university status later in its life. And the Irish guy who founded it and was really the dynamo behind it, I think he was 29 when he was put in charge of that project. +

+

+ And I think it was in 1970 or ‘71 that he was charged with this mission. But as recently as 1970 in Ireland, we were willing to put a 29-year-old — I mean, that’s a person meaningfully younger than me in charge of the project of overseeing the creation of a major new research institution. And I don’t know that I have compelling or confident observations to offer in terms of the etiology underlying these changes. But I think the changes themselves are important, or at least we should assume they’re important if we come from a place of humility, where this is what has worked in the past. Enabling these ambitious young people who are willing to contemplate spending multiple decades in pursuit of some ambitious and idiosyncratic vision. +

+

+ And maybe we’re more enlightened now. Maybe we figured out how to get all the same innovation and all the same breakthroughs without unleashing that force. But I guess my starting point, at least, would be, well, we should — before getting super confident in that or before really being deliberate about it, I think we should give some kind of credit and credence to the prescription and the methodology that’s worked heretofore. +

+
+
+ ezra klein +
+
+

+ And before books, let me end on this. We’ve talked a lot about scientific slowdown, about technological slowdown. But let’s say in the next 15-year time frame, what are the three technological or scientific possibilities you’re most excited by? If in 20 — I guess it’d be 2037, we’re having a conversation about how dumb this conversation was because it was right on the cusp of so much incredible stuff happening, what do you think is likely to be on that list? +

+
+
+ patrick collison +
+
+

+ I don’t know that I’ve super non-consensus answers. I think that there are fundamental a priori reasons to believe that the rate of progress in biology could increase substantially over the years, and to your question, kind of decades to come. So if in 2037 we are enormously impressed and struck by the discontinuity there, that would not shock me. +

+

+ Clearly, over the past couple of years, there’s been acceleration in progress in A.I. And kind of far for me to try to point estimate for kind of where that is in 2037. But I would be surprised if that is not somewhere on that list. +

+

+ And then I think there’s something about education in the broadest sense that feels to me like a very significant, and hopefully very positive change happening in the world right now. Maybe best embodied by YouTube. But also by Twitter and by blogs and Substacks and even Zoom and kind of the growing ease of being in some kind of cultural proximity to people one aspires to emulating, or following in the footsteps of, or otherwise kind of being more like. +

+

+ And to the extent that one believes my story about the significance of sociology, and culture, and mentorship, and the kind of delicate transmission of tacit knowledge, it has until very recently only been possible for that to happen to a meaningful extent through physical co-location. And the fact that we’ve now thrown open those doors to such an extent feels to me like a really compelling and plausibly transformative change. And if it were the case in 2037 that we have multiplied by 20 the number of people who can — who have the initial mental models and understanding to become successful entrepreneurs, or successful scientists, or successful writers, or successful in whatever one might choose one’s domain to be, again, I think that would not be shocking. And I think it’s a pretty hopeful fact about the world. +

+
+
+ ezra klein +
+
+

+ And then always our final question. What are the three books you’d recommend to the audience? +

+
+
+ patrick collison +
+
+

+ Well, I’m right now reading “Revolution and Empire,” which is a book about Edmund Burke. And it is just fabulous. I very highly recommend it. +

+

+ Edmund Burke, Ireland’s foremost political philosopher. And I’m embarrassed to say that I have known less about him than I feel like I ought to have. So I recommend that very highly. +

+

+ And the autobiography by Warren Weaver, who I mentioned, at Rockefeller. I can’t remember if it’s called “Scene of Change” or “Scene of the Action.” But it’s Warren Weaver’s autobiography. +

+

+ That’s not a great book in the sense that you don’t read it — you don’t find it to be a vivid, compelling page-turner. And your mind is not blown on every page. But I find myself thinking back to it quite a lot and having various parts of it sort of ricochet to my mind. +

+

+ And then it all depends on what people are interested in and all the rest. And you should read the things you like. But I have on my desk at home right now “A Widening Sphere,” which is a history of M.I.T. And I was re-reading it recently. And my — +

+
+
+ ezra klein +
+
+

+ Who doesn’t re-read the histories of M.I.T.? +

+
+
+ patrick collison +
+
+

+ [LAUGHS] Well, William Barton Rogers, the founder, was the son of an Irishman, and started M.I.T. substantially with his brother. And I find it very inspiring, I guess back to what we were saying earlier, how motivated he was and they were by a kind of broad-based desire for societal betterment. Like, M.I.T. didn’t inadvertently end up being a significant contribution to American prosperity and ingenuity and welfare. He was really immersed in that milieu. +

+

+ He paid a lot of attention to some of the cultural dynamics we were describing in England, and the Darwins. And the early writing on M.I.T., if you go and just read the first two pages of the founding manifesto, it wasn’t utopian in some kind of implausibly lofty sense. But it was somebody who knew they weren’t founding a run of the mill nth technical college. +

+

+ And I feel like it’s easy to get cynical always. It’s easy to assume that the things that really worked out worked out through happenstance, as opposed to optimism and ambition. But yeah, I find the history of MIT to be a kind of inspiring reminder that sometimes these implausible, lofty, ambitious, long-term initiatives can work out much better than one would hope. +

+

+ [MUSIC PLAYING] +

+
+
+ ezra klein +
+
+

+ Patrick Collison, thank you very much. +

+
+
+ patrick collison +
+
+

+ Thanks for having me. [MUSIC PLAYING] +

+
+
+ ezra klein +
+
+

+ “The Ezra Klein Show” is produced by Annie Galvin and Rogé Karma. Fact-checking by Michelle Harris, Mary Marge Locker and Kate Sinclair. Original music by Isaac Jones. +

+

+ Mixing by Sonia Herrero, Isaac Jones and Carole Sabouraud. Audience strategy by Shannon Busta. Special thanks to Kristin Lin and Kristina Samulewski. +

+

+ [MUSIC PLAYING] +

+
+
+
+
+
+
+
+
+
+
+
+
+ +
+
+
+ +
+
+ +
+
+
+ Listen 1:33:01 +
+
+
+
+
+
+
+ +
+
+
+ +
+
+
+
+
+
+
+
+
+
+
+ +
+
+
+
+
+
+

+ EZRA KLEIN: I’m Ezra Klein. This is “The Ezra Klein Show.” +

+

+ This is a great conversation today. But it’s a tricky one to introduce, because the guest I have — I’m not having him on for the thing he’s best known for. So Patrick Collison — by day, co-founder and C.E.O. of the multibillion-dollar payments company, Stripe; by night, by weekend, I think, one of the most important thinkers now in Silicon Valley — certainly, one of the most quietly influential, someone who is forging and traversing an intellectual path that a lot of other people are now following. +

+
+ +
+
+
+
+

+ Advertisement +

+
Continue reading the main story +
+ +
+
+
+
+
+
+

+ And it’s this second incarnation and role that I’m really interviewing him in today — the soft power side, I guess, of Patrick Collison. Collison’s work here centers around this question of progress. The argument is that human progress is much more precious and rare and fragile than we realize. +

+

+ We maybe take it for granted. We live in this time when things have been changing, atop decades and decades, even centuries and centuries, even millennia now, when things have kept changing. But for most of human history, that was not true. It was not true. +

+
+
+
+
    +
  • Thanks for reading The Times. +
    • +
Subscribe to The Times +
+
+
+

+ There just was no market rapid advance in human living standards. It’s only in the past 10,000 years, and then practically in the past few hundred — just an eye-blink in the time human beings have been on Earth — that things kept changing, usually for the better. And the question is, why? +

+

+ And Collison’s particular meta question is, given the clear fragility of forward motion here, given how rare it has proven to be — and so how easy it might be to lose — why isn’t the question of the conditions of progress more central? Why isn’t the study of progress in a wide multidisciplinary way a more common and central discipline? +

+

+ Collison has written a few influential essays here, with the economist Tyler Cowen. He called for the inauguration of a discipline — they call it progress studies — and that now has people studying it. There’s people creating journals for it, creating syllabi and podcasts and books around the topic. It’s one of the more singularly successful calls for a research direction I have seen. +

+
+ +
+
+
+
+

+ Advertisement +

+
Continue reading the main story +
+ +
+
+
+
+
+
+

+ Separately, in a piece co-authored with the scientist, Michael Nielsen, Collison and Nielsen argued that, though it is hard to measure, it seems like the rate of scientific progress is slowing down, and that’s particularly true if you account for how much more we’re putting into science, in terms of money, of people, of time and technology. +

+

+ Now, these ideas are not original to Collison. The point is not that nobody studied human progress before this or worried about the pace of scientific research. He wouldn’t claim that. It wouldn’t be true. But he is playing a distinctive role in their framing and their popularization, and in creating and funding a community around them. +

+

+ And what I see in my travels here is that it is working. Something is burbling here. But I can’t find many big pieces where Collison really lays out his worldview. There are a couple essays, tweets, interviews, but he’s not been primarily writing this down. +

+

+ What he has been doing is funding it through Fast Grants, which has been successful, but more than that, intellectually influential effort to show you can give out scientific grants quickly and with very little overhead, through the Arc Institute, a big biotech organization he’s creating to push a researcher-first approach to biotech, and through giving a bit of money, and a bit of time, and a bit of prestige, and a bit of networking to a lot of different projects that circle these questions. +

+

+ He’s got this funny quality of being nowhere in particular, but also somehow, almost everywhere, if you’re interested in these questions. So what I wanted to do in this conversation was try to get as close as I could to the Patrick Collison worldview, the underlying theory of the case here that animates his thinking his funding, and the ways in which he’s trying to nudge the culture he’s a part of, or the ways in which he’s trying to actively create a culture he doesn’t yet see. +

+

+ As always, my email — ezrakleinshow@nytimes.com. +

+

+ Patrick Collison, welcome to the show. +

+

+ PATRICK COLLISON: Great to be back. +

+

+ EZRA KLEIN: So you’ve made the argument that science — all science — is slowing down, that we’re putting more money and more people into research, and we’re getting less and less out of it. Tell me about that. +

+
+ +
+
+
+
+

+ Advertisement +

+
Continue reading the main story +
+ +
+
+
+
+
+
+

+ PATRICK COLLISON: Well, I want to separate two things. There’s a question as to whether science in its totality is slowing down, in terms of the absolute returns from it. I think that might be true. You can maybe divide up the first half of the 20th century and the second half and so on, and sort of try to compare one with the other. +

+

+ And we had general relativity and quantum mechanics and various other major breakthroughs in the first half. You can ask the question of, well, did we have as many in the second half? But in the second half, we did have the discovery of D.N.A. and molecular biology and lots of other things. +

+

+ So I don’t know that I would claim a total slowdown. The thing that I think is clearer and should be very concerning to us is, as you look at the number of scientists engaged in the pursuit of science, and if you look at the total amount that we’re spending, and as you look at the total output, as coarsely measured by things like papers and number of journals, all of those metrics have grown by, depending on the number, let’s say, between 20 and 100x between 1950 and, say, 2010. +

+

+ And if you look at it on a per-capita basis, or a per-unit-of-work basis, now used to divide all those total outcomes by a factor of 50, and it seems like if you imagine yourself as the median scientist, you’re meaningfully less likely to produce anything like as consequential a breakthrough as you would have, say, in 1920. And so Michael Nielsen and I, in order to try to put slightly more rigor on that question — we went and we surveyed a bunch of scientists across a number of universities in a number of different disciplines, and we presented them with different Nobel Prize-winning breakthroughs. +

+

+ And we tried to compute an approximate ordering of their significance in the eyes of these scientists. And the thing that would kind of have to be true — for the per-capita impact, we remain in constant — is we’d have to be discovering much more important things in the latter half of the 20th century in order to compensate for, to make it worthwhile, for us to be investing this 50-fold greater effort. +

+

+ And we didn’t find that. In physics, in the estimation of physicists, there was a kind of flat-to-declining trend. It’s not super obvious which way it points, but in as much as there’s a trend visible, it’s probably slightly downwards. And in other fields, it was maybe similarly equivocal, perhaps a slight increase, visible in some, but importantly, in no fields that it looked like we’re on this crazy, exponentially improving trajectory, which is what you would have to have for this per-capita phenomenon to not be present. +

+

+ And I think that should be something we’re interested in for multiple reasons. One, because presumably, as a society, we’re interested in just how much more scientific progress and technological progress and so forth, how much more innovation is there going to be over the next 10 years or the next 50 years or the next century. But also, because there’s kind of two possibilities. One possibility is, fundamentally, we’re running out of low-hanging fruit, and it’s just going to be harder to do this stuff. +

+
+ +
+
+
+
+

+ Advertisement +

+
Continue reading the main story +
+ +
+
+
+
+
+
+

+ And in as much as we’re setting investment or making investment decisions around to what degree should be pursuing the stuff, I guess it’s important to know what we think the returns should be. Or the other possibility is, somehow, we’re doing it suboptimally. Something changed, and we were pursuing this process of discovery more effectively in the past, and presumably, for inadvertent reasons, something went wrong, and now, we’re just less efficient at it. +

+

+ But either explanation — and it doesn’t necessarily have to be fully binary — but either explanation is important, and either explanation, I think, has prescriptions for what we should do going forward. +

+

+ EZRA KLEIN: Let me start with the low-hanging-fruit explanation, which I think is a more popular one. And you have — in the piece you did on this with Michael Nielsen, the sad, but in the very academic way, very funny quote from the physicist Paul Dirac, who says of the 1920s, there was a time when, quote, “Even second-rate physicists could make first-rate discoveries,” which I just kind of love. +

+

+ But the theory there is you can only make a lot of the big discoveries once. You discover quantum mechanics once. You discover the atom once. And most of them have just been made, so what you have now is more complicated, smaller, requires much larger teams of people, much more complicated experiments, with much more infrastructure. +

+

+ So we’re just structurally in a period where it’s going to get harder and harder and harder to make big gains. Do you believe that? +

+

+ PATRICK COLLISON: I think it’s possible, but even though it’s intuitively compelling on some level, I’m not sure that it’s true. It’s probably true to at least some degree for some particular research direction, right? We go after discovering the various subatomic particles, and initially, without too much difficulty, we discover the electron or whatever. +

+

+ And by the time we’ve discovered the nth quark, it’s now gotten super hard, and even with ever-larger particle accelerators, we’re not necessarily making breakthroughs of the same magnitude. So I think it’s pretty true for a given direction. But obviously, the question is, well, to what degree is progress in any area opening up other directions, right? +

+
+ +
+
+
+
+

+ Advertisement +

+
Continue reading the main story +
+ +
+
+
+
+
+
+

+ And so I mean, you mentioned the Dirac quote and, say, physics in the early part of the 20th century. Those discoveries opened up new techniques and investigation methodologies and so on, that then gave rise to molecular biology in the ’50s, ’60s and ’70s. And so there’s kind of a combinatorial benefit, where discoveries over here or discoveries over there might unlock opportunities and major breakthroughs in areas that we could not have foreseen in advance. +

+

+ There are lots of, quote unquote, “low-hanging-fruit discoveries” made in computers and computer science in the ’70s, ’80s, and ’90s. Maybe we’re even still in that regime, right? We’re still making some pretty fundamental breakthroughs. And of course, again, those, quote, “low-hanging discoveries” would not have been possible without a lot of this optimization and discovery in other fields. +

+

+ And so I think it’s probably true for a given research direction, but the relevant question for society is, is it true in aggregate. And there, it’s much less clear to me that it is. +

+

+ EZRA KLEIN: I want to read something provocative you said in an interview with the economist Noah Smith. And you said, quote, “Most systems get worse in at least certain ways as they scale. The idea that science could have gotten worse in significant ways sometimes sounds strange to people. Like, we’re doing so much more. How could that be bad? But I think that misses the many examples of sensitivity of scientific processes to institutions and culture. Swiss nationals have won more than 10 times more science Nobels per capita than Italians have. 10 times. And yet, they’re neighbors. And Italy certainly isn’t lacking in scientific tradition — Fermi, Galileo, the oldest university in Europe, et cetera. The ‘how’ of science just really matters.” +

+

+ And this seems, to me, to be where your exploration really goes. So tell me what you think might have gone wrong in the “how” of science. +

+

+ PATRICK COLLISON: So I think this point about the sensitivity of scientific outcomes to the specifics of the institutions and the cultures is very important and probably underappreciated. At the beginning of the 20th century, not only was the U.S. not a scientific powerhouse, but it barely had a presence in frontier research, whatsoever. +

+

+ To become a credible researcher in the U.S. in 1900, you almost certainly had to go and spend time in, most likely, Germany, and failing that, in France or England — you know, what have you. And by 1900, the U.S. was already a pretty prosperous place, and it had a well-educated society, as societies went. +

+
+ +
+
+
+
+

+ Advertisement +

+
Continue reading the main story +
+ +
+
+
+
+
+
+

+ And yet, somehow — and it had universities, right? I mean, Harvard was hundreds of years old by that time. And so it checked many of the ostensible boxes, and yet, the sum total of the U.S.’ research output as of 1900 was still de minimis. +

+

+ When James Conant, who was later president of Harvard for 20 years — when he went to Germany as a chemist, which was his original training, in the 1920s, he recounts how dispirited he was by what he found there and how far ahead of Harvard German research was, as of the early 20th century. And then, for a variety of reasons, all sorts of cultural, institutional funding — various transformations happened. And of course, by the latter half of the 20th century, the U.S. was the unquestioned leader at the frontier of scientific progress. +

+

+ If you look backwards, you see where that locus has been, where the most successful and fertile scientific grounds have been — it has repeatedly moved. As we just said, maybe the 19th century, it was Germany. +

+

+ Before that, in the 18th century, it was plausibly France. They had a couple of these really successful École Polytechnique and Grande École and so on. And so as a kind of first-order empirical matter, we can just notice, huh, this really seems to matter — and then, the example you just gave of the divergence between Switzerland and Italy. +

+

+ And so then, if we kind of accept that, and we try to ask ourselves, well, specifically, what are the mechanisms? You know, what’s actually going on? It’s hard for me to say. It seems like the transmission of research culture by individual researchers matters a great deal. +

+

+ And you see these kinds of pockets of the cultural transmission repeatedly crop up, where Gerty and Carl Cori — you probably haven’t heard of — they ran a little biology lab in Missouri, and no fewer than six of their trainees, of students they trained, went on themselves again to win Nobel Prizes. +

+

+ And if we tell ourselves a standard kind of mechanistic story as to, well, it’s the funding level, it’s how much are we investing in science, or it’s something about whether there’s an institution in the courser sense, that can possibly be amenable to it, it’s very hard to explain these eddies where you see these pockets of excellence really produce these outsized returns. So I think it’s a complicated question. +

+
+ +
+
+
+
+

+ Advertisement +

+
Continue reading the main story +
+ +
+
+
+
+
+
+

+ I think all of aggregate culture, funding, institutional characteristics, and so on all contribute to it. But if I had to isolate a single variable, it seems to me that the research culture set by specific people and the tacit knowledge transmitted through direct experience is probably the number-one thing. +

+

+ EZRA KLEIN: This, I think, is where I sometimes fall into my own pessimism on this. Because I want to believe, as you do, that we can double the rate of scientific advance, maybe even go further than that. But I think the prediction — if I’m putting this on institutions, on culture, on pockets of transmission and mentorship — I think the prediction I would make is then, even if you believe, say, that America had a great 20th century, but its institutions have become sclerotic, and we’ve slowed down, and everything is piled in lawsuits and review boards now, somewhere else that didn’t have that, that has a different culture, that has different institutions, would be pulling way ahead. +

+

+ So you might think, well, China will be pulling way ahead. And you’ve noted this in some places. We’re getting a lot of peer-reviewed research out of China — huge number of citations out of China. We’re not seeing them dominate the big breakthrough advances of the era. +

+

+ It doesn’t seem like Europe is lapping us. And so if you think this slowdown is somewhat global, then that seems to me to militate against questions of individual institutions, cultures, how different labs work, because there is so much variation that you should have some of these labs that are doing it right, some of these places that haven’t piled on a little bit too much bureaucracy. But I don’t think we really see that. +

+

+ PATRICK COLLISON: This diagnosis of these phenomena to cultural, institutional, mentorship-related, interpersonal dynamics, and your observation that it’s not obviously the case, that there are other places we can pointed that are doing it so much better — for me, my takeaway is that, well, successful cultures are a pretty narrow path. Homo sapiens emerged 200,000 years ago. +

+

+ And as far as we can tell, for the first 190,000 years of our genesis, we think we were largely biologically equivalent to the people we are today. But as best we can tell, there was some kind of cultural capital that those people lacked for a very extended period of time before human societies in somewhat recognizable modern form started to emerge — agriculture, all the rest. +

+

+ And in a similar vein, we had many billions of lives and centuries elapsed before the Industrial Revolution., and before we started to put together many of the input ingredients or enough of the input ingredients that we can get sustained improvement in standards of living and ongoing economic growth and progress. And so your point about, well, as I look around, I don’t see anything or anywhere that’s obviously better, I agree with that. +

+
+ +
+
+
+
+

+ Advertisement +

+
Continue reading the main story +
+ +
+
+
+
+
+
+

+ But again, my takeaway is that that’s what makes the question of how do we improve or how can we do somewhat better so urgent and pressing, where it’s many things have to go right. It’s not easy to be even as good as — or to get to a place where things are as good as they are today. What we have is very precious. And I think the threads and the themes that you’ve been pulling on of late — all of these dynamics underscore their importance. +

+

+ EZRA KLEIN: I think that’s a good bridge to progress studies as an idea. And I want to have people hold in their heads that idea that progress is very narrow, that it is a very narrow bridge that we have walked on for a very short period of time. But let’s try to define it. +

+

+ When you say progress here, what are you actually talking about? Is it just shorthand for economic growth or G.D.P.? What is progress? +

+

+ PATRICK COLLISON: Well, I don’t know that I would claim to put forth some kind of definitive definition. And I think, to some extent, our intuitions around it are probably broadly correct. And so it might not matter to define it super precisely and finely. +

+

+ For, me it is something along the lines of our success in realizing a liberal, pluralistic and prosperous society, and a sense among people that their offspring can and probably will do better than they themselves have, and that more broadly, the future will be better than the past, and that we’re at least making incremental progress towards embodying values and morals that we collectively think we can be proud of. +

+

+ But I don’t think anything that novel in that. I don’t think my conception of progress would differ that materially from some kind of average aggregate over any other group of people in the country. +

+

+ EZRA KLEIN: I do think there’s something interesting, though, which is that if you look at eras that I think progress-studies-type people and economic-growth people and historians of economic growth study most closely, actually, some of the periods where people feel a lot of rapid progress don’t fit that at all. You have, say, the Industrial Revolution, where life spans and lifestyle get worse for a lot of the people. +

+
+ +
+
+
+
+

+ Advertisement +

+
Continue reading the main story +
+ +
+
+
+
+
+
+

+ I don’t think one will look at that period as unbelievably pluralistic. You have a lot of periods of war when you have very, very, very rapid technological progress, but it happens in context of much more martial societies. So there is an interesting tension, at least in periods — and some of them quite long, actually — where you can have fairly rapid economic progress, but it comes at a cost that I think isn’t always acknowledged, but is an important thing to think about. +

+

+ PATRICK COLLISON: Yeah. So I don’t think you could point to some of these periods in the past and say that they definitively embody to the extent that we would fully aspire to some of these broader traits and characteristics. But I think the question is more, what are they doing as — you have to judge it relative to the baseline that preceded them. +

+

+ And I don’t know that the 18th century in the U.K. is some ideal as a society. But if you compare it to the 16th century in the U.K., the ideals and ideas of natural rights and religious tolerance and so on — they were somewhat better embodied by the 18th century than they had just a couple of centuries previously. +

+

+ And similarly, in the U.S., say, during either war or the ’30s or whatever, again, it’s not like that was any kind of perfect society, but assessed relative to the society of 1830, I think it compares relatively favorably. And I think it’s not a coincidence that Adam Smith — his first book, of course, was on ethics and morals and trying to instill better general ideals and behaviors across a society. +

+

+ And maybe after that, he then argued for and laid many of the foundations of what we would recognize as modern economics. So I don’t think it’s perfect. But on average, I think the correlation is positive. +

+

+ EZRA KLEIN: So let’s talk about the Industrial Revolution for a little bit here. I think a lot of people locate a takeoff in human living standards — it continues to this day — there. And it’s strange in a way, right? “There” is a very geographically contiguous spot. It’s the U.K. — England, actually, I should say, at that point. +

+

+ And there is a moment in time that probably could have come at another moment in time, depending on how human history plays out in the counterfactual. I know that you have an interest in the theories of why then, why there. How do you work your way through them? What do you think is persuasive for why then, why there? +

+
+ +
+
+
+
+

+ Advertisement +

+
Continue reading the main story +
+ +
+
+
+
+
+
+

+ PATRICK COLLISON: Well, you know, again, I caveat. With all of these topics we’re discussing through this podcast, maybe the first-order banner for all of them should be, I don’t know, these are my best guesses, and I think it’s important that all of us were pretty humble in the claims and the assertions and the beliefs that we hold. +

+

+ Recently, I’ve been reading a bunch of Irish and Scottish writers around then. It’s very interesting, because for both the Irish and the Scots, there was a sort of a pressing and kind of obvious question where England was much more prosperous than they were or we were. And there’s no super obvious explanation for that. There wasn’t an obvious climatic or natural resource endowment that England benefited from that was lacking in Ireland or Scotland. +

+

+ It wasn’t like England was actually a vastly larger polity. The orders of magnitude were comparable. And Bishop Berkeley wrote this book, “The Querist.” He was asking these questions directly, just like, what’s going on? What’s wrong with Ireland? You know, why can’t we do this? +

+

+ And then, you have the Act of Union in 1707, uniting Scotland and England — and sort of similarly, of all these Scottish thinkers being like, all right, we’re now literally the same country. Why are we so much more impoverished? And then, if you shift to England, there’s Joel Mokyr and — you’ve read his work — and more recently, people like Anton Howes. +

+

+ And in a similar vein, they go back to — I mean, the word, improvement, came from Francis Bacon, or it was kind of popularized as a concept by Francis Bacon. But that’s noteworthy, right? Like, that was not a pervasive broad concept in the 15th century. +

+

+ I mean, literally, the word, improvement, in this broader societal context, came from word, “translated,” at the beginning of the 17th century. And the ultimate conclusion that these historians and scholars and analysts of the Industrial Revolution come to — and I think it’s a correct one — is somehow, whether it’s through Bacon or Newton or various of the tinkerers who produced some of the earliest technological breakthroughs, that somehow, this improving mind-set became pervasive. +

+

+ You had societies explicitly — like the Hartlib Circle or the Lunar Society, or the Select Society, and the club, and so on — all these societies explicitly devoted to figuring out ways to advance the state of affairs that prevailed. And these societies were comprised of many of the leading people and thinkers and so on of the day. +

+
+ +
+
+
+
+

+ Advertisement +

+
Continue reading the main story +
+ +
+
+
+
+
+
+

+ And it seems maybe a bit satisfyingly squishy to attribute it to something so hard to pin down. But as you run through all the possible other explanations, it’s differences in IP law. It’s difference in the Malthusian conditions. It’s difference in the prevalence of coal, you know, et cetera, et cetera. Through various cross-sectional analyses, you can exclude most of these in looking at all of Ireland, Scotland, and England. +

+

+ It really does seem to me that differences in the mind-set and in the culture are where you have to net out. And that’s not to say maybe that it’s fully sufficient. There might be other preconditions that are important. And then, maybe as a last thing to say, it is striking to me that many of these kind of original 18th-century economic writers and thinkers — and again, the kind of people we look to as the founders of much of the discipline — that they themselves were kind of centrally preoccupied with this. +

+

+ And yeah, they were in favor of free trade and specialization and human labor and lots of these concepts that we’re now very familiar with, but they really thought that general mind-set played a big role, too. +

+

+ EZRA KLEIN: So let’s talk about Joel Mokyr ideas for a minute. So Mokyr is an economic historian. People should read his book, “The Culture of Growth,” which is really fascinating. He argues, as you’re saying, that in this period, this mind-set that we can increase the store of usable knowledge, and then use it to alter nature, to better the human condition, takes hold. +

+

+ That’s a new mind-set. It’s different than religious ideas of the past. It’s different than cultural ideas of the present. And that, plus a bunch of other things, particularly the republic of letters, the way people are writing letters back and forth, kind of combine into a culture that is able to grow. +

+

+ But one of the things that I really take from his work, that sits in my head, is he believes it’s all very contingent. He really believes it might have not happened. But the other is that I think it opens up this question that as a tech person, I’m curious to hear your thoughts on, which is, he really believes — Mokyr really believes — that there is a communications infrastructure that arises at that time, that has a kind of culture of generosity and argument and honesty in it, and is built on writing letters slowly to one another, and then copying those letters over to other people. +

+

+ And that culture is really good for intellectual advancement. I think one of the promises of the internet and the age we live in is, it’s all faster. We can write to people immediately. Things we write can go viral and be seen by 5 million people all of a sudden. +

+
+ +
+
+
+
+

+ Advertisement +

+
Continue reading the main story +
+ +
+
+
+
+
+
+

+ And that was going to speed up economic growth really, really rapidly. And I would say, you don’t see that. So I’m curious how you think about communication cultures here and what you think for all the advantages of ours we might not have. +

+

+ PATRICK COLLISON: I mean, I think it’s hard to say in aggregate. I feel it’s pretty likely that the effects are very heterogeneous across different populations. And you’ve made the case that you think Twitter is bad for journalism and for journalists. +

+

+ And I guess you live this yourself with your now mostly inactive Twitter account, I guess, apart from announcements. And I think in the case of the internet, that it’s almost certainly a tremendously large gain that billions of people now have access to educational materials. And some of the otherwise hard-to-communicate tacit knowledge — that things like YouTube videos now made legible and available. +

+

+ And I think it’s true that there are various gravity equations that we see across different disciplines. I mean, in economies themselves, in trade, where you rapidly decline in propensities to trade as countries get further from each other — but you have versions of this in academic disciplines as well, where geographic distance correlates inversely with likelihood of the exchange of ideas and so on. +

+

+ And I think it’s clearly the case that the sort of reaction surface area has increased substantially by the internet there and represents a kind of efficiency gain for people looking to exchange in ideas. Many of the companies that Stripe works with are remote companies, and they might employ people across myriad countries, and that’s a kind of communication and efficiency gain that would certainly not otherwise be achievable. +

+

+ I think it’s worth recognizing that the aggregate amount of G.D.P. that we are creating or gaining every year is so much larger now than — I mean, the percentage might be the same. But the total amount of stuff happening, or the increasing amount of stuff happening, is so much larger now than it was 100 or 200 or 300 years ago. +

+

+ And so for all of those reasons, I think we should give superior communication technologies and faster communication technologies a significant amount of credit, even though the ways in which those are manifests might be hard to measure and somewhat prosaic. +

+
+ +
+
+
+
+

+ Advertisement +

+
Continue reading the main story +
+ +
+
+
+
+
+
+

+ Take my mom, for example. My mom works with a hospital in Minnesota. Our youngest brother has a physical disability. And in the course of that, she trained herself in treatment for cerebral palsy, this condition, and she wrote a book about it, and she did a master’s in this. And now, she’s trying to improve treatment for this condition throughout Ireland, in the U.S. and other countries as well. +

+

+ She’s a retired Irish mother who spends some of her year living in the U.S. near her sons, spends the rest of her year living in Ireland, working at a hospital in Minnesota, who just got a proposal to have her book translated into German a couple of days ago. And that’s a relatively prosaic story, but literally, millions of these stories exist in kind of aggregate form around the world. +

+

+ To circle back to the initial thrust of your question, though, I think it’s at least possible that the internet is bad for civic discourse. I’m not saying it is, but it’s certainly in the realm of plausibility — and that perhaps both things are true, where there’s some kind of iceberg where there are these enormous welfare gains that are not that legible, not that visible, lie beneath the surface, and then certain of the most visible manifestations, like what we see on cable news or what we see written in the papers — perhaps that is worse, and perhaps, slightly more structural judiciousness would be desirable there. +

+

+ EZRA KLEIN: I want to try to flip that and suggest that — because I’m going to push some counter ideas on why we maybe don’t see as much progress as we wish we did. But one is that I think possibly, very large welfare losses lie beneath the surface. And beneath the surface of stories like the one you just told about your mother, I think we all have stories of ways or people for whom the internet has unlocked a possibility. +

+

+ I mean, my whole career is built on the internet. I was an early blogger. I got rejected from my student newspaper. And if there was no blogging, like, god knows what would have happened to me. [LAUGHS] I mean, nothing too terrible, probably, but I wouldn’t have the career I have today. +

+

+ And at the same time, I think that the group of people who, by luck or by temperament, proved very, very good at using the internet, to some degree, distracts from the many, many, many people for whom the internet is fundamentally a distraction machine, or for whom the internet is creating, because of what we built on it. You know, shorter attention spans — how many people would have had an idea, sitting in a room by themselves, or taking a walk, that they never have now, because they never have to have a moment where they’re thinking alone? +

+

+ And so one thing that I think we’re all loathe to do is we’ll talk a lot about how it’s weird that we have so much more knowledge, but productivity isn’t increasing faster. It’s weird that we have so much more rapid communication between researchers, but science isn’t advancing faster. And then, the idea that maybe there are things happening to us that makes us less able to use that increasing stock of knowledge well, or makes us less able to collaborate in a useful way, I think, gets dismissed rather quickly. +

+
+ +
+
+
+
+

+ Advertisement +

+
Continue reading the main story +
+ +
+
+
+
+
+
+

+ But I don’t think it’s totally implausible. Now, I don’t want to say, like, the greatest technology we ever had was letter-writing. Obviously, the greatest technology we ever had was blogging in the early aughts when I became a blogger. And whatever happened in your 20s is, like, as good as it was ever going to get. +

+

+ But I do wonder about these questions. And I think something Mokyr is right to put a lot of attention on is communicative cultures. Communication is how we collaborate. And if communication is in any way getting worse, it’s going to have pretty big macro effects. +

+

+ The other thing is if you believe these cultures matter, weirdly, as big as we’re getting, the internet allows a certain disciplines culture to stretch boundaries and borders in time in a way that it would have been harder. I suspect that labs were more different 50 years ago than they are today. +

+

+ The countries and the disciplines of researchers and the cultures of researchers in countries or cities are more different from each other 50 years ago than today, which is great if we have the best of all cultures today, but it’s not that great if you actually think variation is really important. +

+

+ PATRICK COLLISON: Let’s wrap up there. So first, I agree, as a basic matter, that there are welfare losses occurring across society that we should be worried about, and probably everybody listening to this is familiar with the Stephen Pinker case for optimism, and rather than focusing in the headlines, you zoom out, look at these long-term time series. And once one does that, things seem a lot more encouraging, whether you look at it by income or life expectancy or infant mortality or choose your metric. +

+

+ Something that’s been striking to me of late is if you change the x-axis on those time series, and look at many of those phenomena and trends over a much shorter window, the valence changes substantially, and life expectancy in the U.S. is now, in fact, declining. According to C.D.C. data, 54 percent of teenage girls now report persistent feelings of sadness and hopelessness. And you could say, well, teenagers were never stereotyped as the most cheerful lot, but we do have some degree of longitudinal data here, and that number is up from being in the 20s as recently as 2009. +

+

+ ½ the population now is either prediabetic or diabetic — again, according to the C.D.C. Basically, point is, when we look at more recent windows, I think there are plenty of aggregate, emergent, complicated outcomes and phenomena that should give us concern. On the degree to which we should attribute the diagnosis to the internet or to our kind of communication media more broadly, it’s less clear to me in that — not saying it’s not true, but presumably, the life expectancy one is not — or at least if it is, the mechanism has to be very complicated. +

+
+ +
+
+
+
+

+ Advertisement +

+
Continue reading the main story +
+ +
+
+
+
+
+
+

+ There are a bunch of other health-related ones. So take, for example, say, the incidence of diabetes or pre-diabetes. And you could say, OK, fine, all those things might be true, but they’re totally different. I guess the question I wonder about is, well, we know that lots of basic biological outcomes are correlated with mental states and so on. +

+

+ And so to what degree is there some more nuanced and complicated relationship there? But I think it’s a fair question, and I wonder a lot about it myself. +

+

+ EZRA KLEIN: Let me ask you about how you think, over the long period here, about the relationship between technology and equity or egalitarianism. And something specific is in my mind. I flicked earlier at the way the Industrial Revolution, for an extended period of time, seems to have reduced a lot of people’s living standards. And it wasn’t till later you had changes in redistribution in labor unions and labor protections that the amount of material prosperity that was generating created more broad-based prosperity, particularly at a very high level. +

+

+ I don’t know that you can sustain that kind of thing today. We have much more a small-d democratic culture. If things aren’t working for people, it’s much easier for them to organize and be heard. +

+

+ I think there’s a much more direct and complicated relationship now between whether or not people feel benefited by technology, and whether or not they are going to accept the conditions and the risks of rapid technological advance. But I’m curious, from your vantage point, how you see that both kind of historically and currently. +

+

+ PATRICK COLLISON: I agree with that. I worry a lot about the basic stability of a society that does not successfully generate and make sufficiently broadly accessible the benefits of economic growth. The world simply has too little prosperity. And if it is not the case that people in the U.S. or people in any country — if they either feel like things aren’t progressing, or if they feel like maybe somewhere distant from them, things are progressing but they personally will never be able to benefit from it, I think we put ourselves in a very dangerous and likely unstable equilibrium. +

+

+ And if you go back to — well, you don’t have to go back very far in history to see, obviously, plenty of instances where this kind of instability brought the whole house of cards down. And so as a consequence of that, I worry a lot about, how do we simply make sure that — or one of the small things we each individually can do to try to make sure that society is generating enough economic gain and enough broadly experienced welfare gain that the whole compact can be maintained? +

+
+ +
+
+
+
+

+ Advertisement +

+
Continue reading the main story +
+ +
+
+
+
+
+
+

+ And again, I don’t think there’s a ready neat kind of singular answer to that. Maybe Stripe as part of our small little contribution in one little fissure. But I think the central question you’re getting at is super important. +

+

+ EZRA KLEIN: And one of the questions I wonder about there — we’ve talked about the way progress has been very geographically lumpy, let’s call it, right? There’s a lot that happens in very small places, and it ends up affecting the whole world. Obviously, then, the gains of progress sometimes have that quality, too. +

+

+ And I do think of one of the politically destabilizing effects of the past, let’s call it, 30 or 40 years of digital progress, is being the concentrations of wealth. We just used to have a lot more spread. Even putting the questions of rising inequality aside, just where rich people were was different. +

+

+ And so where they were giving a lot of money to the local hospital was more spread out, say, across the country or in other countries across the land. But here, even as the internet is supposed to democratize distance, and in many ways, has — I mean, telework is not a fake phenomenon. It has really concentrated the wealth of that to, literally, where we’re sitting, but to New York. There’s a lot of money now in Austin. +

+

+ And then, on top of that, you often have barriers of entry, in terms of how many homes can be bought. So it’s not even like people can move to the place where all the economic opportunity is happening. And I do think that creates some of the skepticism you see of technology. +

+

+ I don’t think a lot of people’s — I think people are really excited about a lot of the goods they’ve gotten from it. But in this kind of macro political sense, as you’re saying, in a period of a lot of change, a lot of folks with real backing in the data don’t feel life has gotten better at the macro level. +

+

+ Life expectancy, happiness, political stability — it’s not like you can look around and say, well, I got this computer in my pocket, and everything else is going great, too. It’s like, I got this computer in my pocket, and what it keeps telling me is that everything is going to hell. Now, maybe it’s telling me that a little bit too much, but there is validity to the narrative. +

+
+ +
+
+
+
+

+ Advertisement +

+
Continue reading the main story +
+ +
+
+
+
+
+
+

+ PATRICK COLLISON: Yeah. So again, vehement in agreement on the sort of central importance of making sure that improvements in the standard of living are actually broadly realized across the society. That, too, I think, could serve as a manifesto for some of these Progress Studies ideas. +

+

+ On the internet in particular, or on technology and the technology sector and so forth, I think it’s complicated and difficult to try to sort of fully collapse or linearize it or something, where on the one hand, you have some of these concentration dynamics you identify. At the same time, of course, it is also a tremendous and incredible dispersal agent in making some of those possibilities and opportunities be more broadly available. +

+

+ I think it’s dangerous to take an excessively U.S.-centric perspective here. If you interact with or look at survey data, or otherwise try to assess what’s the sentiment of people in Poland, what’s the sentiment of people in India, or what’s the sentiment of people in Indonesia, they view the internet extremely positively. And I think correctly so, where their opportunities for advancement would be substantially curtailed in the absence of much of what the internet makes possible. +

+

+ I think in places like the U.S., or actually, even at home in Ireland, some of this story is complicated by lots of other things, but including — and I think, substantially — dynamics around housing policy, where there’s an extensive literature showing that, for example, a very substantial moderating effect on the increase in wealth inequality that would otherwise have happened, or income inequality, was geographic reallocation and people responding rationally to, OK, things in New York are going great, or things in California are going great. +

+

+ And if you look at the rate of increase of the Californian population, say, through the 1960s, that was a tremendously potent mechanism for us redistributing some of the economic gains that were being realized at the time. And of course, now, we have this crazy position, where California is losing population at the same time where the market caps of these companies and the profits of these companies are increasing very rapidly. +

+

+ But as one assesses that dynamic and tries to ask the question of, well, why aren’t these gains being better or more broadly distributed, it’s certainly not clear to me that the answer even lies in the realm of technology qua technology. And I think it’s certainly more broadly, again, some of these considerations like geographic allocation. +

+

+ EZRA KLEIN: Let me ask one more question on the geographic dimension, and then I’ll move on to it. And it’s on my mind, in part because when I try to think about progress, when I try to think about what inventions and innovations are coming really quickly, I actually see a bunch here. And I’ll use A.I. as an example. +

+
+ +
+
+
+
+

+ Advertisement +

+
Continue reading the main story +
+ +
+
+
+
+
+
+

+ I’ve met people who are trying to automate a bunch of legal contracts. It makes a ton of sense. People pay a lot all over the country — to some degree, all over the world — to get fairly basic legal contracts drawn up — wills and real estate documents and merger agreements and all kinds of — from the small to the large. +

+

+ If you imagine that getting really effectively automated, though — +

+

+ you think about Saint Louis, Missouri, where some of the people who are important pillars of the community work in law firms there, and what they do is contracts. They do estate planning and all the things that people have to do in contracts. And if it actually does get concentrated to really, really great contracting firms in the Bay Area or in New York, on the one hand, the democratizing potential will really be realized. Those contracts will get cheaper. +

+

+ There’s also a theory in crypto of smart contracts. And on the other hand, you really will have a lot of that — the gains of that, economically, going to smaller areas and aggregated across a bunch of different domains. So graphic design, in all kinds of areas of the country — midlevel graphic designers get paid to make logos for local businesses. +

+

+ It’s pretty clear they’re going to be able to do that really, really easily on things like DALL-E pretty fast. So you can imagine a lot of that area getting wiped out. And you kind of run through a couple of these. And before you get to really unbelievable and sci-fi-like dimensions of artificial intelligence, you just have a thing that is going to democratize a lot of capabilities in a way that’s going to put the money for those capabilities both a little bit back into the pockets of the people who need them, and then a lot into the people who run the best A.I. rigs and is going to have a really weird geographically destabilizing effect. +

+

+ And that paradox of the internet both democratizing geography, and then concentrating wealth and capital in very small areas is, to me, a central challenge. Because if you get that wrong, if it goes too much in the concentration area, I think we’re going to lose a lot of the political stability we need here. But you’re more on top of these technological advances than I am. Do you think the trends there are going to play out differently than I’m worried they will? +

+

+ PATRICK COLLISON: First, yeah, it’s not — I don’t think it’s foreordained whether or not these are going to be centralized technologies. I don’t know. And on the one hand, there’s, I think, an obvious feature we can contemplate, where there are only three A.I. models, and they are rooted in the hegemons, the citadels of Silicon Valley technology, and we all are digital serfs who are subsistence-farming on their gains. +

+

+ I think there’s also a very plausible story where these technologies prove substantially less defensible than we might have expected, and where, instead, they have this enormously decentralizing effect. Because otherwise, economies of scale that only large firms could benefit from can now be realized and pursued, even by massively smaller firms. And if we look at the recent history of A.I. — I don’t think any clear story there, but it does feel to me that it has been more biased towards the second story than the first. +

+
+ +
+
+
+
+

+ Advertisement +

+
Continue reading the main story +
+ +
+
+
+
+
+
+

+ There are now multiple companies with large language models. There are a number of very successful open-source A.I. efforts. Actually, there was a really cool example from Replit, which is a service — it’s a programming I.D. in the browser, used by kids learning to code, but also increasingly used by people who are pursuing serious programming. +

+

+ And they recently released a GitHub copilot-like technology, where it will kind of autocomplete your code in the editor, and where you can do some pretty cool things. Like, you can highlight a block of code and ask it to be explained, and it’ll turn code into natural language, into English, and say, hey, here’s what this code is doing. +

+

+ Anyway, they wrote a blog post about how they built this, and they describe how it was built by one guy over the course of a couple of weeks. And maybe that’s only the case in the early days of this AI technology. I mean, in early computer games, the first games were built by a single heroic person, and now, it’s these gigantic studios and enormous CapEx budgets. +

+

+ And so I think the fact that this is the case today doesn’t mean that it will remain the case through time. But my takeaway is that at least not foreordained that AI or any of these other technologies will be centralizing forces. And then, the other thing to observe is that when we talk about these being centralizing, I think there’s a question as to, do we look at it in relative or absolute terms? +

+

+ And my contention would be that, both from a moral standpoint, but maybe more importantly from kind of a political-economy standpoint, what will matter is whether, on an absolute basis, people feel like they are realizing opportunities, their lives are improving, that things are getting better, that their kids will be in a better situation and so forth. And exactly how much value is realized by the companies themselves doesn’t actually matter that much, compared to that former question. +

+

+ And whether A.W.S. or whether any of these organizations has super high or super low profit margins, I don’t know is nearly as important as what is the actual effect on these communities and individuals across the society. And so in as much as one means — by centralizing, one means a large share of the profits, I think it is probably a more useful framing to look at it instead in terms of absolutes, and in particular, the absolute surplus generated by the users. +

+

+ [MUSIC PLAYING] +

+

+ EZRA KLEIN: What have you come to believe about the relationship between progress and war? +

+
+ +
+
+
+
+

+ Advertisement +

+
Continue reading the main story +
+ +
+
+
+
+
+
+

+ PATRICK COLLISON: I am somewhat skeptical that war is as conducive to breakthroughs as we might intuitively conclude, or as is sometimes claimed. You’re probably familiar with Alexander Field’s work on the ’30s here. And his basic claim is, the productivity gains we often attribute to the Second World War in the U.S. — like, those foundations actually were laid in the ’30s, and then the first half of the ’40s were a period of decreasing productivity as we massively, inefficiently reallocated our economic resources for the purposes of winning the war, which was probably a good thing to do, but inefficient in narrow economic terms. +

+

+ And he has a new book coming out, I think, next month, that sort of extends this argument into the ’50s. +

+

+ And as one takes stock of the scientific breakthroughs — and so Stripe Press recently republished Vannevar Bush’s memoir, where he takes stock of this. +

+

+ And obviously, you have, say, the Manhattan Project, and that’s a big deal, certainly. But it doesn’t feel to me that had the Manhattan Project not occurred, that peaceful development of nuclear technology would have been massively stymied. Maybe it would have taken another 10 years, but it was already happening to some meaningful extent. +

+

+ And then, as you take stock of all the other breakthroughs that took place in the U.S. during the Second World War, there were some meaningful stuff like blood plasma and blood transfusions. There was some significant breakthroughs there. Some of the first antimalarial medications, radar, the proximity fuse, which I’m not sure is all that useful outside of military applications. +

+

+ But by the time you get down to invention 6 on the list, I don’t know that as you compare that list to, again, some counterfactual of what would otherwise have ensued, that it looks radically better as you take stock of the Cold War and the enormous fraction of our economic resources and human capital that were devoted towards us, that the gains necessarily look that impressive. And so again, it’s super hard to judge. You don’t have proper controls and so on. But I’m skeptical. +

+

+ EZRA KLEIN: Let me take the other side. So there’s a question of, during war, how much did we invent during World War II. And that’s a question of how much the threat of war or the competition with an adversary ends up charging up innovation and convinces us to put resources, both in terms of people and in terms of money, and maybe in terms of institutions, into projects we wouldn’t otherwise have done. +

+
+ +
+
+
+
+

+ Advertisement +

+
Continue reading the main story +
+ +
+
+
+
+
+
+

+ I think there’s an argument, at least, that we went to the moon because of the Soviet Union. It would not have done that for some time. Probably would have eventually done it, but also, who knows? +

+

+ As Derek Thompson, who I’m working on a lot of these ideas with, likes to point out, the Apollo Project was unpopular. It was not something that commanded wide popular support. +

+

+ Even now, if you look at the CHIPS Act that passed, it passed, with all that spending on semiconductor research and other kinds of next-generation technologies, under the framework of, let’s compete more effectively with China. If you look at all the things Darpa has done or been part of, the fact that “defense” is the first word in the Darpa acronym, I think, is meaningful. +

+

+ There’s something about what threat persuades societies to do, and persuades them to do technologically or what risks it allows otherwise-more-cautious governments to take, or what failures they could justify that allows them to have big successes. Something there doesn’t seem to small to me. +

+

+ And maybe it’s my political side, where I so often see scientific funding justified in Congress in terms of countries we’re competing with or are adversaries with. And I see what the defense industry can do that other institutions cannot, because they don’t get a lot of political blowback. But I don’t know. +

+

+ I worry a little bit about how much we seem to need the threat of another to accelerate things. And in a small way, maybe, we see what the pandemic — where we were willing to move much, much quicker on things like mRNA technology than I think we would have outside of it. +

+

+ PATRICK COLLISON: Yeah. So I think it’s certainly true that the crisis can cause the discontinuous shifts that have large effects, which in your example, say, are probably super beneficial. I don’t know. If you take Darpa as an example, it started as Arpa, as a more open-ended research institution and set of programs, and then with the Vietnam War, had the D pretended to it. +

+
+ +
+
+
+
+

+ Advertisement +

+
Continue reading the main story +
+ +
+
+
+
+
+
+

+ And we decided, in the face of threat, to make it more applied, to take more seriously its translational and kind of, quote unquote, “competition-oriented mandate.” And I think that was bad for Darpa. And the internet, which arose under Arpa — it’s hard to think of innovations of similar magnitudes that then occurred in then-Darpa’s subsequent, say, two decades. +

+

+ If you take, say, U.S. science in general, the war — the Second World War — to some extent, the first, but much more so the second — precipitated an enormous centralization of U.S. science in its aftermath. Because we really marshaled together all of the — or a significant fraction of the scientific capacity of the U.S. in service of the war effort. +

+

+ For, example the 50 percent overhead, the fraction of government grants that goes to universities — that was chosen in the early days of the coordination of the war effort, and has now become a kind of a pillar of academic and research funding in the U.S. And in the aftermath of the war, we sort have this question of OK, we’ve kind of pulled everything together. Now, what do we do? +

+

+ And that became, in various ways, the N.I.H. and the N.S.F. and so on. I mean, the N.I.H. predated it, but the growth of the N.I.H. really occurred after the war. And the federal government, shortly thereafter, for the first time, became the majority funder of US science. And all that centralization — and I mean, you pointed out the benefits of variety and of experimentation and of heterogeneity, and having some degree of institutional and structural diversity and so on, I totally agree with all of that. +

+

+ And I think all of that was very meaningfully curtailed by, again, the aftershocks of some of the threats that we faced during the war. And if you think about the things that we’re maybe happiest about having happened — the founding of the major new U.S. research universities in the latter parts of the 19th century or the revolution in health care and kind of medical practice that first happened at Johns Hopkins, and then kind of codified in the Flexner Report, or the great industrial research labs of Bell and Park and so on — or excuse me — Xerox — they didn’t obviously come from a place of fear or a threat. +

+

+ They came from a place of hope and optimism and opportunity. And maybe there are some inventions that you’re more likely to get to from some of these external pressures. But yeah, if you gave me a dial, and I can kind of turn up or down the threat or fear index of society, it’s not super obvious to me that one would want to turn it up if what one cared about was the aggregate rate of progress. +

+

+ EZRA KLEIN: That’s a good bridge, I think, to the question of institutions. And I take one of the main concerns of yours, of progress studies, as being around institutional slowdown. You have this idea that we don’t meta-maintain institutions very well. +

+
+ +
+
+
+
+

+ Advertisement +

+
Continue reading the main story +
+ +
+
+
+
+
+
+

+ They start in one place, and then over time, they crust over, and we don’t really know what to do with that. Give me a little bit of your thinking there. +

+

+ PATRICK COLLISON: I think institutions, the cultures they instill and act as kind of coordination points and training sites for — those of enormous consequence — I think much of the success of the U.S. and of various other Western countries has, in substantial part, been attributable to successful institutions. Most people would accept, I think, that there is, to some extent, consistent trends that tend to happen with institutions through time. +

+

+ Just maybe most basically, the problem that gives rise to an institution in the first place is probably a pretty real and significant problem. No one would have taken the time to found the institution if it wasn’t. And there can be some degree of drift there, where we don’t necessarily decommission the institution once the problem has subsided or abated. +

+

+ And then, you tend to attract a certain kind of person in the early days of an institution — people who are slightly less status and reputation and procedure-oriented, because a new institution almost never has that. And then, through time, the sort of collective or the mission-oriented incentives of the institution can kind of drift somewhat from the individual incentives that particular people are subject to. +

+

+ I think all this stuff exists. And the thing that I observe, or that I just find myself thinking about is, we’ve had eras of institution formation in the U.S. It has not been kind of a constant rate through time. And the New Deal maybe, and say, the 30 years afterwards, and the Great Society — we bookend it with those start and endpoints. +

+

+ That was a period of tremendously active institution construction and formation in the U.S., Darpa being — or Arpa originally being a good example, and indeed, NASA. And I guess I find myself wondering, one, if we didn’t have any of these institutions — and I’m not saying we should get rid of them. But if we didn’t have them, what institutions would we found today, first, and how high in the list would NASA be, for example? +

+

+ And then, secondly, in as much as we accept that some of these institutional dynamics exist, like the fact that sclerosis as an emergent property arises, what do we do about that? And lots of people have told us it’s pretty — doesn’t need a lot of teasing apart to see it as one compares NASA and SpaceX and the respective budgets, and the respective achievements, and so forth, I think it’s hard to not at least wonder about their respective efficiencies. +

+
+ +
+
+
+
+

+ Advertisement +

+
Continue reading the main story +
+ +
+
+
+
+
+
+

+ And say, if society could only have SpaceX or NASA, which one would we choose, and what should we conclude from that, and to what extent do those phenomena generalize elsewhere? I don’t have answers to these questions. But I find that in the political discourse — not that anybody is celebrating that, but in the discourse, it’s very easy to get, I think, very wrapped up in questions of optimal funding levels, and should this number be 10 percent or 50 percent or higher or whatever, whereas to me, a lot of our satisfaction with the outcomes seems to hinge on deeper questions about the nature of the institution. +

+

+ I don’t know that the problem or benefit, or anything good or bad about NASA is attributable to the budget, per se. It seems more, kind of, resonant in some of these deeper cultural questions. And then, in the recent pandemic, or in the — I don’t know. I was going to say, ongoing pandemic. But I guess as of two days ago, with the President’s verdict, it is now over. +

+

+ EZRA KLEIN: It’s over. Congratulations, everybody. +

+

+ PATRICK COLLISON: Exactly. But anyway, I think that was maybe a vivid demonstration of many of these dynamics, where I don’t know this any of the story about the institutional response to the pandemic should be primarily one of funding. I think it’s much more about the dispositions and the attitudes and the cultural biases of entities like the N.I.H. and the F.D.A. and the C.D.C. +

+

+ EZRA KLEIN: I find the NASA SpaceX example an interesting and provocative one. Because on the one hand, I think what you’re saying is completely true. And where a lot of the NASA programs and projects have gone in recent decades, is just — it’s sad. It’s just a sad story. +

+

+ And on the other hand, the idea that you — the thought experiment of choosing between NASA and SpaceX — the thing that it immediately asks is, well, you can’t. Because without NASA, there is no SpaceX. And one way the private sector handles a lot of these questions — I mean, I’m always struck by how much of the way biotech research works is that big pharmaceutical companies acquire small biotech firms that have made a breakthrough or have come up with a very promising candidate. This is kind of an accepted thing that the big companies — they do a fair amount of research, but a major, major innovation transmission there is small groups do more, quicker, and they’re just going to buy them. And the NASA SpaceX example has a little bit of that dynamic to it, although with a different mechanism of financing. And I don’t know. I wonder if there aren’t deeper lessons there. +

+

+ PATRICK COLLISON: Yeah, I don’t mean here in the NASA example — like, I don’t think reducing it to a simple binary of this-or-that is correct. It’s more, what should we make of the differences in these two organizations? And given those observations or beliefs, what do we then think an efficient outcome might look like? +

+

+ And do we think that where we are today — this prevailing status quo — is optimal? Or are there other things we can do better? And maybe an important thing to say within all of this is, to the extent that these are all kind of inevitably determined outcomes, maybe it doesn’t really matter if we think things would be better or worse. +

+
+ +
+
+
+
+

+ Advertisement +

+
Continue reading the main story +
+ +
+
+
+
+
+
+

+ We’re going to end up in the same place, regardless. But I think for all of these, it’s super contingent. And various aspects of both funding decisions and, kind of, the precepts and methodologies of the N.I.H., how we design I.P. law, how we regulate and require and run clinical trials — there are tons of individual contingent decisions that we kind of have collectively made that give rise to the biotech and to the pharma ecosystem. +

+

+ And certainly, in the case of space, you know, like, it doesn’t have to be this way other. Universes, no pun intended, are possible. I think perhaps the thing that people underappreciated with science in the U.S. is, it has been very different in the not-too-distant past. +

+

+ Peer review is a relatively recent invention. Modern journals are a relatively recent invention. As I mentioned, the federal government being the primary funder of basic research is a relatively recent invention. +

+

+ And molecular biology was, in significant part, a thesis by Warren Weaver at the Rockefeller Foundation. There’s a thing here, and we should aggressively pursue it. But importantly, it was not — it required an institution, an organization, that was not part of the standard apparatus, for want of a better term. +

+

+ And the Broad Institute, over the last 25 years, has been enormously successful in the field of genomics and functional genomics and CRISPR, et cetera. And the Broad Institute is itself a kind of structural innovation, breaking somewhat from the more traditional prevailing university model. And so I think the fact that so many of our successes are associated with some degree of structural and institutional change should be somewhat thought-provoking for us. +

+

+ EZRA KLEIN: You’ve been trying to work in the space of institution-building here, too. I want to talk about Fast Grants and about Arc a little bit. So let’s begin with Fast Grants. What is it, and what has it taught you? +

+

+ PATRICK COLLISON: Well, it’s mostly “what was it.” In the early days of the pandemic — well, I should preface all of this by saying — well, I’ll reaffirm my preface that I don’t know, to every question. But more importantly here, I will say, my now-wife is herself a scientist. We’ve known each other since we were teenagers. We spend a lot of time talking about science in various forms. +

+
+ +
+
+
+
+

+ Advertisement +

+
Continue reading the main story +
+ +
+
+
+
+
+
+

+ EZRA KLEIN: You met — am I allowed to say this? You met at a science competition. +

+

+ PATRICK COLLISON: That is true. We met at a science competition, 100 teenagers, and — +

+

+ EZRA KLEIN: And she beat you. +

+

+ PATRICK COLLISON: And yes. I was the runner-up, and she was the winner. I had created a programming language and a new dialect of lisp, and she had created a new treatment for urinary tract infections. And so I really don’t envy the judges for having to figure out what framework one should use to make all these comparisons and lots of other people. +

+

+ EZRA KLEIN: You sound a little bitter, man. +

+

+ PATRICK COLLISON: [CHUCKLES] I was gonna say, but no, we can all agree this the correct outcomes ensued. Anyway, so we were living together in March of 2020, holed up. And a number of her friends and colleagues were unsurprisingly with, I guess, a large fraction of all biology scientists, were trying to urgently repurpose their work to figure out, well, could they do something that would be somehow benefit to accelerating the end of the pandemic? +

+

+ And by early April, so a couple of weeks into lockdown, when it was becoming apparent and striking to us, which was it is difficult for these people to get funding for their work. And that might sound a bit, kind of, surprising, because you think, well, don’t they have some degree of money already? And couldn’t they just go and just spend that? +

+

+ But there are, obviously, significant rules around and restrictions around that which one can do with one’s grant money. This is money provided by the government for a purpose. And so it’s not like you can go and readily spend it on something totally unrelated. +

+

+ And the money is administered by the university, and so you have to go through their proper procurement processes. Point is, lots of restrictions on scientists’ pecuniary ability to suddenly repurpose the research agendas. And we kind of thought, well — we assume maybe in the early weeks, that presumably various bodies — I don’t know who — some kind of amorphous other, some combination of C.D.C., F.D.A., N.I.H., philanthropies — whatever. +

+

+ Somebody will come along and just give these scientists the obvious money that society clearly should, so they can go, and they can pursue these programs. Didn’t seem to be happening. I mean, to be fair, I don’t want to give us too much credit. +

+
+ +
+
+
+
+

+ Advertisement +

+
Continue reading the main story +
+ +
+
+
+
+
+
+

+ Various people were doing things right off the bat in various different places, but we just personally knew of lots of specific examples of really good scientists who were unable to make progress of their work to the extent that they would like. So we tried to set up what we thought would be a pretty small initiative, and called Fast Grants. +

+

+ The basic idea would be, you send us some kind of proposal. And initially, within 48 hours, you would get a funding decision and either receive money or not. We started out with a pretty small amount of money. +

+

+ The initial donors — we were among them, but there were a number — contributed, best I recall, about $10 million. Launched the website early April 2020. Quickly inundated with, I think, four and a half thousand applications, which, given our promised 48-hour turnaround, was somewhat challenging. +

+

+ I should say this was myself. This was Silvana, my wife, and this was Tyler Cohen. So we had an immediate question as to, how do we actually run a philanthropic endeavor? And how do we stand it up in very short order? And he, through Mercatus and through Emergent Ventures, had some experience of very efficient and somewhat-scaled grant-giving. +

+

+ And so the three of us worked together to put it together over the course of a week or so. We proceeded over the course of, roughly speaking, the next year, slightly more, to make about 200 grants, eventually dispersing almost — or slightly over, actually — $50 million in total, to universities around the world, though primarily in the U.S. +

+

+ And you ask, kind of, what did we learn? A big surprise was how slowly other parts of the establishment mobilized. And various of the projects we funded or the labs we funded and so on — they’ve gone on to now do — none of them were directly implicated in the vaccine research project that ended up yielding so much fruit. So again, I don’t want to give Fast Grants too much credit. Eventually, the thing that really mattered, we had nothing to do with. +

+

+ But versus the projects, things like Saliva Direct, which was in the summer an early discovery that saliva tests work basically as well as the nasopharyngeal swabs we were all being subject to, or various discoveries around possible therapeutics, some of which are — still continue to go through clinical trials, and may still turn out to matter to a significant extent. And that 500 people are still dying in the U.S. per day from Covid, and — despite the existence of the vaccines and so on. +

+
+ +
+
+
+
+

+ Advertisement +

+
Continue reading the main story +
+ +
+
+
+
+
+
+

+ So anyway, various discoveries ensued that I think will prove to be important. And the second thing we learned, which is not really related to Covid or the pandemic, but has certainly been significant for us, is — it just got us thinking more deeply and broadly about the questions of, how do scientists choose what to do? And what are the constraints they’re subject to as a practical and applied matter? +

+

+ And this gets back to all this discussion about both culture and institutions. And towards the end of Fast grants, we ran a survey of the grant recipients. And these are essentially all people who don’t normally — certainly don’t normally work on Covid. Covid didn’t exist. +

+

+ But they don’t even normally work on viruses, for the most part. These are basically kind of broadly drawn as a cross section across biology. And we just asked them, as a general matter in your regular research, if you could spend your grant money however you want, how much would you change your research agenda? +

+

+ So not an increase in the funding level, which tends to be what we discuss in as much as we’re discussing science policy across society. But much more specifically and narrowly, if you had complete autonomy in how you spend whatever grant money you’re getting, how much of your research agenda would change? And our intuition was that maybe a third of people would like to be doing something meaningfully different to what they actually are. +

+

+ But of these scientists, and these are really good scientists, four out of five told us that they would change their research agendas, quote, “a lot.” We gave them three options. Not much, or not at all, a little, and then a lot. Four out of five chose the maximum option on our survey. So I just find this incredibly thought-provoking. +

+

+ Basically, we seem to be in a situation where most of our top scientists aren’t doing what they think would be best for them to do. And we could say, no, our various committees and governing bodies and decision-making apparatus and so on, they know better. And I’m not saying it would be completely unreasonable for one to maintain that. But that would seem to be a very central question about the construction of our scientific apparatus. And I think that should give us some pause. +

+

+ EZRA KLEIN: There are a couple things there. One is that it is a consistent observation I have learning about new areas that there is a way we’re taught the thing works, or people think the thing works, and there’s this huge middle layer. Right? So in politics, which I know very well, and legislation, you have the “Schoolhouse Rock” version of how a bill becomes a law. And then it’s, like, a filibuster is how a bill becomes a law or does not become a law. +

+
+ +
+
+
+
+

+ Advertisement +

+
Continue reading the main story +
+ +
+
+
+
+
+
+

+ We were talking about drug innovation earlier. I think the folk way people think it works is we make a discovery about a drug, and then, like, we make a drug out of it after some tests. But you talk to people who work on pharmaceuticals and just clinical trials. +

+

+ And in science — I think if you had asked me as a high schooler, had some science classes, I’d have told you something about the scientific method. And then you talk to a scientist, and it’s grants. Like, grants are how science works. Grants are the middle layer between — you are a scientist, and you can do some science. And grants are how the N.S.F. and the N.I.H. work. They’re how a lot of the universities work. There’s fund-raising. +

+

+ I mean, there are different ways that it happens. To make the question of “Are we doing science well?” a little bit more precise, I think one version of that question is, “Are we doing grants well?” And I think that question is more tractable. People don’t feel as defensive about it. +

+

+ But I’ve talked to a lot of scientists in the course of my work. I’ve covered health care for my entire career. And I don’t know any who think we’re doing grants well. I don’t know any who will not complain to you for hours. +

+

+ And they may be wrong. And I do want to note — because they also just have somewhat different incentives. I mean, I was noting earlier, and I think it’s very real. The government, particularly when it gives out grants, needs to worry about the reputational cost of the grant. If the grant goes wrong, if not enough of the grants pay out into useful research. If Rand Paul can stand up in Senate and make what you did sounds silly, these things really end up mattering. And so you get a process that is optimizing for a lot of different things. +

+

+ But the question of whether or not we do grants well ends up being really, really, really important in every country that does major capital science that I know of, and is just not the main question for a bunch of different reasons we ask. +

+

+ PATRICK COLLISON: Yeah. Another question we asked in our survey was how much time they spend on the grants. I think to some extent, this is perhaps — at least, of those who’ve spent some amount of time interacting with scientists, kind of more broadly known than perhaps the finding with respect to how they do — or the degree to which they can choose what they work on. But we found that — or they reported to us that they spend on the order of 40 percent of their time on grant administration. +

+
+ +
+
+
+
+

+ Advertisement +

+
Continue reading the main story +
+ +
+
+
+
+
+
+

+ And even if one were to maintain that the decision-making apparatus around what scientists do is somehow efficient, I think it is a very tenuous position to also try to argue that 40 percent of the best scientist’s time is optimally allocated towards grant applications, authorship and administration. And we’re not talking about an inconsequential 40 percent here. +

+

+ I mean, this is 40 percent of the time of this super-elite 10,000, 100,000, whatever it is, some relatively finite number of people. And we’ve chosen to take and to redeploy almost half of their time in service of technocratic, bureaucratic undertaking. And getting back again to this point about people perhaps falsely assuming that things have been more inter-temporally consistent than they have, that percentage has increased very substantially over the last couple of decades as the overall edifice of science has grown, and as the kind of acceptance rates and the various thresholds for various grants has become more exacting. +

+

+ EZRA KLEIN: How we allocate people’s time is really important. But also, just how we allocate talent is really important. And it brings me to something you said that I wanted to ask you about. This was in response to a question about whether big tech companies are hogging all the talent in society. +

+

+ And you said, quote, “I don’t think that the ambitious upstarts who go into high speed rail in America, anyway, are going to have a great time or have much success in convincing their friends to follow them. And I suspect that for various reasons, too many domains look somewhat like high speed rail.” And so you go on to say that there’s a view that the internet is a frontier of last resort, and that you don’t think that’s totally wrong. +

+

+ So tell me about that. Tell me about the idea of the internet as a frontier of last resort. But behind that, this idea that other frontiers where talented people might want to go and make their mark on society have closed. +

+

+ PATRICK COLLISON: You’re familiar with and you’ve probably written about the Stephen Teles idea of kludgeocracy. And I kind of like the term “kludgeocracy,” because rather than making some of the inhibitions that people might encounter in pursuing something like high speed rail, rather than casting those as being deliberate, the valence is more that it’s this kind of emergent, inadvertent and kind of complicated phenomena that nobody perhaps particularly wants or chose. +

+

+ And I think the case of California’s high speed rail is quite striking, where — you’ve written about this and kind of similar projects and the New York subway expansion and so on. And congestion pricing and so on. But it’s striking where it’s not actually obviously a question of first order political will. +

+
+ +
+
+
+
+

+ Advertisement +

+
Continue reading the main story +
+ +
+
+
+
+
+
+

+ Like, we’re willing to fund the high speed rail in California. We’re clearly willing to invest in building the subway expansion in New York. But somehow, somewhere between that first order decision and desire and our actual ability to kind of instantiate it, something really goes wrong. +

+

+ And you contrast that with stories of — in the case of, say, California, Henry Kaiser and these various other early part of the 20th century operators in the physical realm. And for a variety of reasons, but mostly prosaic state and county-level complications and things that would extend the time horizon of one’s project, it has simply become meaningfully less-appealing for those people to undertake these initiatives. I mean, Foster City, not too far from where we are now, that’s named after the eponymous Mr. Foster. +

+

+ He was a developer. He decided, well, with reclaimed wetlands, I’m going to build a city. California is growing quickly. The Bay Area is a — kind of propitious and will be a long-term successful area. And I think this place simply needs more housing. +

+

+ And he, with that kind of founder energy, was able to give birth and rise to the city that now bears his name. I haven’t met anybody pitching me on a similar city on the shores of the Bay in the last couple of years. But I would imagine that were one to adopt that ambition today and to propose that maybe the San Jose Marsh wetlands should themselves be an expansion of San Jose, I don’t think one would get very far. +

+

+ And in fact, even for much more sort of limited things, like additional runways or runway expansions at S.F.O., even they have now been stymied for decades at this point. That ability to translate that into something enunciated has dissipated and deteriorated. And then I think the kind of individual version is, and if I want to be that heroic solar farm entrepreneur or railway magnate, that my practical ability to do so has been meaningfully curtailed. +

+

+ EZRA KLEIN: Yeah. I mean, that’s what I’m getting at here a little bit, which is talent really matters for a society. Where the most talented people go really matters for society. And a lot of those people want to go somewhere where they can have a really big effect. +

+

+ I think there’s been a huge rush to digital land because you can build on digital land. You can build quickly. I mean, it’s interesting to some of the dynamics we’re talking about, the temporal dynamics we’re talking about, that you see this dynamic even within the tech world. +

+
+ +
+
+
+
+

+ Advertisement +

+
Continue reading the main story +
+ +
+
+
+
+
+
+

+ There was a while where it was really exciting to go join Facebook, go join Google, go join one of the big companies. Because you could do so much. And that’s still, to some degree, true. But they got really big. +

+

+ And so crypto got — whatever you think of crypto, one thing that is exciting about it to people is the idea that it’s open land. That you can go in there and have a really big effect on it. Build something new just with a couple of friends that might change the whole direction of the field. +

+

+ And on some level, it’s always going to be harder for, say, putting high speed rail through the middle of California. Right? I mean, just building things in the world is just going to be tougher. +

+

+ But on the other hand, if you make building things in the world too hard, if you make grants too difficult — if you — I know a lot of doctors who their advice to young people is don’t become a doctor. And it always breaks my heart a little bit. We need really great people to be doctors. +

+

+ And their point is not, don’t go heal sick people. Their point is, being a doctor is too hard now. The amount of time you spend dealing with insurance agencies and malpractice insurance and boards, and this and that, it’s just too much administration. +

+

+ When industries become very complicated to operate in, you want to select for people who are good at operating complicated industries, which may be different than the people who are good at moving really fast and changing things dramatically. But two, you kind of subtly bias where different kinds of people in your society go. I think in China, if you want to change a lot, you still probably go into infrastructure construction, among other things. Right? +

+

+ The idea that you might be a genius rail mind, in China, that’s great. There’s probably a lot of rail you can make. That’s not true here. And the point is not to make too much of the rail example, but to make a lot of the idea that talent flows towards where it can have an effect and people can live the kinds of heroic lives they want to lead. And if we have subtly pushed a lot of people into maybe not the right — not the socially optimal directions, that over time will have a pretty big effect on a society. +

+
+ +
+
+
+
+

+ Advertisement +

+
Continue reading the main story +
+ +
+
+
+
+
+
+

+ PATRICK COLLISON: I think a constant is that some number of ambitious young people will want to do something, as you say, heroic. And yeah, I think maybe two things have changed. For one, for whatever reason, our predisposition to putting those people in positions of authority has diminished. And then secondly, even if placed, their ability to actually execute, again for various reasons, has been attenuated. +

+

+ I’ve been reading about the university founders and presidents and those associated with some of the great US research institutions. And one thing that is striking is how many of them were so young when placed in those positions of authority. You know, Daniel Coit Gilman at Johns Hopkins, or William Rainey Harper at the University of Chicago. I think he was 32 when he was appointed president of the University of Chicago. Even in the recent past. +

+

+ So my dad was in the first year of the University of Limerick in Ireland. Or at the time, it was called N.I.H.E. It kind of acquired university status later in its life. And the Irish guy who founded it and was really the dynamo behind it, I think he was 29 when he was put in charge of that project. +

+

+ And I think it was in 1970 or ’71 that he was charged with this mission. But as recently as 1970 in Ireland, we were willing to put a 29-year-old — I mean, that’s a person meaningfully younger than me in charge of the project of overseeing the creation of a major new research institution. And I don’t know that I have compelling or confident observations to offer in terms of the etiology underlying these changes. But I think the changes themselves are important, or at least we should assume they’re important if we come from a place of humility, where this is what has worked in the past. Enabling these ambitious young people who are willing to contemplate spending multiple decades in pursuit of some ambitious and idiosyncratic vision. +

+

+ And maybe we’re more enlightened now. Maybe we figured out how to get all the same innovation and all the same breakthroughs without unleashing that force. But I guess my starting point, at least, would be, well, we should — before getting super confident in that or before really being deliberate about it, I think we should give some kind of credit and credence to the prescription and the methodology that’s worked heretofore. +

+

+ EZRA KLEIN: And before books, let me end on this. We’ve talked a lot about scientific slowdown, about technological slowdown. But let’s say in the next 15-year time frame, what are the three technological or scientific possibilities you’re most excited by? If in 20 — I guess it’d be 2037, we’re having a conversation about how dumb this conversation was because it was right on the cusp of so much incredible stuff happening, what do you think is likely to be on that list? +

+

+ PATRICK COLLISON: I don’t know that I’ve super non-consensus answers. I think that there are fundamental a priori reasons to believe that the rate of progress in biology could increase substantially over the years, and to your question, kind of decades to come. So if in 2037 we are enormously impressed and struck by the discontinuity there, that would not shock me. +

+
+ +
+
+
+
+

+ Advertisement +

+
Continue reading the main story +
+ +
+
+
+
+
+
+

+ Clearly, over the past couple of years, there’s been acceleration in progress in A.I. And kind of far for me to try to point estimate for kind of where that is in 2037. But I would be surprised if that is not somewhere on that list. +

+

+ And then I think there’s something about education in the broadest sense that feels to me like a very significant, and hopefully very positive change happening in the world right now. Maybe best embodied by YouTube. But also by Twitter and by blogs and Substacks and even Zoom and kind of the growing ease of being in some kind of cultural proximity to people one aspires to emulating, or following in the footsteps of, or otherwise kind of being more like. +

+

+ And to the extent that one believes my story about the significance of sociology, and culture, and mentorship, and the kind of delicate transmission of tacit knowledge, it has until very recently only been possible for that to happen to a meaningful extent through physical co-location. And the fact that we’ve now thrown open those doors to such an extent feels to me like a really compelling and plausibly transformative change. And if it were the case in 2037 that we have multiplied by 20 the number of people who can — who have the initial mental models and understanding to become successful entrepreneurs, or successful scientists, or successful writers, or successful in whatever one might choose one’s domain to be, again, I think that would not be shocking. And I think it’s a pretty hopeful fact about the world. +

+

+ EZRA KLEIN: And then always our final question. What are the three books you’d recommend to the audience? +

+

+ PATRICK COLLISON: Well, I’m right now reading “Revolution and Empire,” which is a book about Edmund Burke. And it is just fabulous. I very highly recommend it. +

+

+ Edmund Burke, Ireland’s foremost political philosopher. And I’m embarrassed to say that I have known less about him than I feel like I ought to have. So I recommend that very highly. +

+

+ And the autobiography by Warren Weaver, who I mentioned, at Rockefeller. I can’t remember if it’s called “Scene of Change” or “Scene of the Action.” But it’s Warren Weaver’s autobiography. +

+
+ +
+
+
+
+

+ Advertisement +

+
Continue reading the main story +
+ +
+
+
+
+
+
+

+ That’s not a great book in the sense that you don’t read it — you don’t find it to be a vivid, compelling page-turner. And your mind is not blown on every page. But I find myself thinking back to it quite a lot and having various parts of it sort of ricochet to my mind. +

+

+ And then it all depends on what people are interested in and all the rest. And you should read the things you like. But I have on my desk at home right now “A Widening Sphere,” which is a history of M.I.T. And I was re-reading it recently. And my — +

+

+ EZRA KLEIN: Who doesn’t re-read the histories of M.I.T.? +

+

+ PATRICK COLLISON: [LAUGHS] Well, William Barton Rogers, the founder, was the son of an Irishman, and started M.I.T. substantially with his brother. And I find it very inspiring, I guess back to what we were saying earlier, how motivated he was and they were by a kind of broad-based desire for societal betterment. Like, M.I.T. didn’t inadvertently end up being a significant contribution to American prosperity and ingenuity and welfare. He was really immersed in that milieu. +

+

+ He paid a lot of attention to some of the cultural dynamics we were describing in England, and the Darwins. And the early writing on M.I.T., if you go and just read the first two pages of the founding manifesto, it wasn’t utopian in some kind of implausibly lofty sense. But it was somebody who knew they weren’t founding a run of the mill nth technical college. +

+

+ And I feel like it’s easy to get cynical always. It’s easy to assume that the things that really worked out worked out through happenstance, as opposed to optimism and ambition. But yeah, I find the history of MIT to be a kind of inspiring reminder that sometimes these implausible, lofty, ambitious, long-term initiatives can work out much better than one would hope. +

+

+ [MUSIC PLAYING] +

+

+ EZRA KLEIN: Patrick Collison, thank you very much. +

+

+ PATRICK COLLISON: Thanks for having me. +

+

+ [MUSIC PLAYING] +

+

+ EZRA KLEIN: “The Ezra Klein Show” is produced by Annie Galvin and Rogé Karma. Fact-checking by Michelle Harris, Mary Marge Locker and Kate Sinclair. Original music by Isaac Jones. +

+
+ +
+
+
+
+

+ Advertisement +

+
Continue reading the main story +
+ +
+
+
+
+
+
+

+ Mixing by Sonia Herrero, Isaac Jones and Carole Sabouraud. Audience strategy by Shannon Busta. Special thanks to Kristin Lin and Kristina Samulewski. +

+

+ [MUSIC PLAYING] +

+
+
+
+ Real conversations. Ideas that matter. So many book recommendations.
+
+ Listen to “The Ezra Klein Show”: Apple Podcasts, Spotify, Pocket Casts, Google Podcasts, Stitcher, How to Listen +
+
+
+
+ +
+
+
+
+
+
+

+ Advertisement +

+
Continue reading the main story +
+ +
+
+
+
+
+
+
+
+
+
+
+ Special offer: Subscribe for +
+ $2 $0.50 a week for the first year. +
+
+
+
+
+
+
+
+
+
+

+ VIEW OFFER +

+
+
+
+
+
+
+
+
+ + + VIEW OFFER + + + + + +
+
+ + +
+
+
+
+
+
+
+ + +
+
+
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/packages/readabilityjs/test/test-pages/nytimes-podcasts/url.txt b/packages/readabilityjs/test/test-pages/nytimes-podcasts/url.txt new file mode 100644 index 000000000..fff4a6db0 --- /dev/null +++ b/packages/readabilityjs/test/test-pages/nytimes-podcasts/url.txt @@ -0,0 +1 @@ +https://www.nytimes.com/2022/09/27/podcasts/transcript-ezra-klein-interviews-patrick-collison.html \ No newline at end of file diff --git a/packages/readabilityjs/test/test-pages/nytimes/expected.html b/packages/readabilityjs/test/test-pages/nytimes/expected.html index 8b990443d..251653e4c 100644 --- a/packages/readabilityjs/test/test-pages/nytimes/expected.html +++ b/packages/readabilityjs/test/test-pages/nytimes/expected.html @@ -1,14 +1,7 @@ -
+
-
-

Americas|As Virus and Economic Woes Ravage Brazil, Bolsonaro Improvises and Confounds -

-
-

https://www.nytimes.com/2021/03/31/world/americas/brazil-coronavirus-bolsonaro.html

-
-

Critics see the recent behavior of Brazil’s president — polarizing in the best of times — as an unnerving sign of a flailing leader. His strategy, if there is one, is difficult to discern.

@@ -16,9 +9,9 @@
- - - President Jair Bolsonaro of Brazil during a news conference on Wednesday. + + + President Jair Bolsonaro of Brazil during a news conference on Wednesday.
@@ -46,9 +39,9 @@

Image

- - - People over 71 and their relatives standing in a line for vaccinations on Wednesday in Rio de Janiero. + + + People over 71 and their relatives standing in a line for vaccinations on Wednesday in Rio de Janiero.
@@ -75,9 +68,9 @@

Image

- - - Health care workers intubating a Covid-19 patient this month at a hospital in Porto Alegre, Brazil. + + + Health care workers intubating a Covid-19 patient this month at a hospital in Porto Alegre, Brazil.
@@ -102,9 +95,9 @@

Image

- - - Supporters of Mr. Bolsonaro on Wednesday in Rio de Janiero. + + + Supporters of Mr. Bolsonaro on Wednesday in Rio de Janiero.
@@ -135,18 +128,8 @@

Ernesto Londoño reported from Rio de Janeiro, and Letícia Casado from Brasília.

-
-
-

OFFER EXTENDED: Subscribe for $0.50 a week. -

-
-
-

Thanks for reading The Times. -

-
-
- \ No newline at end of file + \ No newline at end of file diff --git a/packages/text-to-speech/package.json b/packages/text-to-speech/package.json index 8d9bf3306..e326eb8f9 100644 --- a/packages/text-to-speech/package.json +++ b/packages/text-to-speech/package.json @@ -3,7 +3,7 @@ "version": "1.0.0", "description": "", "main": "build/src/index.js", - "types": "build/src/index.d.ts", + "types": "build/src/htmlToSsml.d.ts", "files": [ "build/src" ], diff --git a/packages/text-to-speech/tsconfig.json b/packages/text-to-speech/tsconfig.json index f450acf38..42c16d244 100644 --- a/packages/text-to-speech/tsconfig.json +++ b/packages/text-to-speech/tsconfig.json @@ -3,7 +3,9 @@ "compilerOptions": { "outDir": "build", "rootDir": ".", - "lib": ["dom"] + "lib": ["dom"], + // Generate d.ts files + "declaration": true }, - "include": ["src", "test"] + "include": ["src"], } diff --git a/packages/web/components/templates/LoginForm.tsx b/packages/web/components/templates/LoginForm.tsx index a98d7c4a2..566276e80 100644 --- a/packages/web/components/templates/LoginForm.tsx +++ b/packages/web/components/templates/LoginForm.tsx @@ -38,7 +38,7 @@ export function LoginForm(props: LoginFormProps): JSX.Element { }} > - A read-it-later app for serious readers. + Read-it-later for serious readers. - Get Started + Sign Up @@ -33,7 +33,7 @@ export function GetStartedButton(): JSX.Element { const containerStyles = { px: '2vw', - pt: 100, + pt: 32, pb: 100, width: '100%', background: 'linear-gradient(0deg, rgba(255, 255, 255, 0.2), rgba(255, 255, 255, 0.2)), linear-gradient(0deg, rgba(253, 250, 236, 0.7), rgba(253, 250, 236, 0.7))', @@ -125,16 +125,23 @@ type LandingSectionsContainerProps = { export function LandingSectionsContainer({ hideFirst = false, hideSecond = false, - hideThird = true, - hideFourth = true, + hideThird = false, + hideFourth = false, }: LandingSectionsContainerProps): JSX.Element { const iconColor = 'rgb(255, 210, 52)' return ( {!hideFirst && ( + Omnivore strips away the ads, trackers, and clutter and + formats pages for easy reading without distractions. The + text-focused view also makes articles smaller and quicker + to load. +

+ } image={ +

+ Read actively, not passively. Highlight key sections and add + notes as you read. You can access your highlights and notes any + time — they stay with your articles forever. +

+

+ Fun fact: research shows that highlighting while you read + improves retention and makes you a more effective reader. +

+ + } image={ + Send subscriptions directly to your Omnivore library, and + read them on your own time, away from the constant distractions + and interruptions of your email inbox. +

+ } image={ landing-3 } - icon={} + icon={} /> )} {!hideFourth && ( +

With the Omnivore app for iOS and Android and extensions for all + major web browsers, you can add to your reading list any time. +

+

Saved articles remain in your Omnivore library forever — even if the + site where you found them goes away. Access them any time in our reader + view or in their original format. +

+ + } image={ landing-4 } - icon={} + icon={} containerStyles={reversedSectionStyles} /> )} - Get started with Omnivore today + Sign up for free diff --git a/packages/web/pages/about.tsx b/packages/web/pages/about.tsx index b55ac637e..c6c505f96 100644 --- a/packages/web/pages/about.tsx +++ b/packages/web/pages/about.tsx @@ -4,10 +4,17 @@ import { LandingHeader } from '../components/templates/landing/LandingHeader' import { LandingFooter } from '../components/templates/landing/LandingFooter' const mobileContainerStyles = { - maxWidth: 430, alignSelf: 'center', marginTop: 80, - padding: '10px', + maxWidth: 960, + + px: '2vw', + '@md': { + px: '6vw', + }, + '@xl': { + px: '120px', + } } const headingStyles = { @@ -16,12 +23,12 @@ const headingStyles = { fontSize: 45, lineHeight: '53px', padding: '10px', + paddingBottom: '0px', } const subHeadingStyles = { color: 'rgb(125, 125, 125)', - mb: 32, padding: '10px', } @@ -31,12 +38,16 @@ export default function LandingPage(): JSX.Element { - A read-it-later app for serious readers. + Omnivore is the read-it-later app for serious readers. - Omnivore is a privacy focused, open-source read-it-later app. - Use it to save interesting articles and read distraction free. - Add notes and highlights. - Organise your reading queue the way you want and sync it across all your devices. + Distraction free. Privacy focused. Open source. + + + + Save interesting articles, newsletter subscriptions, and documents and + read them later — focused and distraction free. Add notes and highlights. + Organize your reading list the way you want and sync it across all your + devices. diff --git a/packages/web/public/static/landing/landing-3.png b/packages/web/public/static/landing/landing-3.png index 62c71adbf..bd49932cf 100644 Binary files a/packages/web/public/static/landing/landing-3.png and b/packages/web/public/static/landing/landing-3.png differ diff --git a/packages/web/public/static/landing/landing-3@2x.png b/packages/web/public/static/landing/landing-3@2x.png index 07cf79bea..a1784cf0b 100644 Binary files a/packages/web/public/static/landing/landing-3@2x.png and b/packages/web/public/static/landing/landing-3@2x.png differ diff --git a/packages/web/public/static/landing/landing-4.png b/packages/web/public/static/landing/landing-4.png index 3032d15b4..bcb786cff 100644 Binary files a/packages/web/public/static/landing/landing-4.png and b/packages/web/public/static/landing/landing-4.png differ diff --git a/packages/web/public/static/landing/landing-4@2x.png b/packages/web/public/static/landing/landing-4@2x.png index f8d86f031..5cbadd221 100644 Binary files a/packages/web/public/static/landing/landing-4@2x.png and b/packages/web/public/static/landing/landing-4@2x.png differ diff --git a/yarn.lock b/yarn.lock index 97a48bba4..b76eae9d8 100644 --- a/yarn.lock +++ b/yarn.lock @@ -10104,6 +10104,13 @@ brace-expansion@^1.1.7: balanced-match "^1.0.0" concat-map "0.0.1" +brace-expansion@^2.0.1: + version "2.0.1" + resolved "https://registry.yarnpkg.com/brace-expansion/-/brace-expansion-2.0.1.tgz#1edc459e0f0c548486ecf9fc99f2221364b9a0ae" + integrity sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA== + dependencies: + balanced-match "^1.0.0" + braces@^2.3.1, braces@^2.3.2: version "2.3.2" resolved "https://registry.yarnpkg.com/braces/-/braces-2.3.2.tgz#5979fd3f14cd531565e5fa2df1abfff1dfaee729" @@ -10579,6 +10586,19 @@ chai@^4.3.4: pathval "^1.1.1" type-detect "^4.0.5" +chai@^4.3.6: + version "4.3.6" + resolved "https://registry.yarnpkg.com/chai/-/chai-4.3.6.tgz#ffe4ba2d9fa9d6680cc0b370adae709ec9011e9c" + integrity sha512-bbcp3YfHCUzMOvKqsztczerVgBKSsEijCySNlHHbX3VG1nskvqjz5Rfso1gGwD6w6oOV3eI60pKuMOV5MV7p3Q== + dependencies: + assertion-error "^1.1.0" + check-error "^1.0.2" + deep-eql "^3.0.1" + get-func-name "^2.0.0" + loupe "^2.3.1" + pathval "^1.1.1" + type-detect "^4.0.5" + chalk@^1.0.0, chalk@^1.1.3: version "1.1.3" resolved "https://registry.yarnpkg.com/chalk/-/chalk-1.1.3.tgz#a8115c55e4a702fe4d150abd3872822a7e09fc98" @@ -10748,6 +10768,21 @@ chokidar@3.5.2: optionalDependencies: fsevents "~2.3.2" +chokidar@3.5.3, chokidar@^3.4.1, chokidar@^3.4.2, chokidar@^3.5.1, chokidar@^3.5.2, chokidar@^3.5.3: + version "3.5.3" + resolved "https://registry.yarnpkg.com/chokidar/-/chokidar-3.5.3.tgz#1cf37c8707b932bd1af1ae22c0432e2acd1903bd" + integrity sha512-Dr3sfKRP6oTcjf2JmUmFJfeVMvXBdegxB0iVQ5eb2V10uFJUCAS8OByZdVAyVb8xXNz3GjjTgj9kLWsZTqE6kw== + dependencies: + anymatch "~3.1.2" + braces "~3.0.2" + glob-parent "~5.1.2" + is-binary-path "~2.1.0" + is-glob "~4.0.1" + normalize-path "~3.0.0" + readdirp "~3.6.0" + optionalDependencies: + fsevents "~2.3.2" + chokidar@^2.1.8: version "2.1.8" resolved "https://registry.yarnpkg.com/chokidar/-/chokidar-2.1.8.tgz#804b3a7b6a99358c3c5c61e71d8728f041cff917" @@ -10767,21 +10802,6 @@ chokidar@^2.1.8: optionalDependencies: fsevents "^1.2.7" -chokidar@^3.4.1, chokidar@^3.4.2, chokidar@^3.5.1, chokidar@^3.5.2, chokidar@^3.5.3: - version "3.5.3" - resolved "https://registry.yarnpkg.com/chokidar/-/chokidar-3.5.3.tgz#1cf37c8707b932bd1af1ae22c0432e2acd1903bd" - integrity sha512-Dr3sfKRP6oTcjf2JmUmFJfeVMvXBdegxB0iVQ5eb2V10uFJUCAS8OByZdVAyVb8xXNz3GjjTgj9kLWsZTqE6kw== - dependencies: - anymatch "~3.1.2" - braces "~3.0.2" - glob-parent "~5.1.2" - is-binary-path "~2.1.0" - is-glob "~4.0.1" - normalize-path "~3.0.0" - readdirp "~3.6.0" - optionalDependencies: - fsevents "~2.3.2" - chownr@^1.1.1, chownr@^1.1.4: version "1.1.4" resolved "https://registry.yarnpkg.com/chownr/-/chownr-1.1.4.tgz#6fc9d7b42d32a583596337666e7d08084da2cc6b" @@ -14489,7 +14509,7 @@ glob@7.1.7: once "^1.3.0" path-is-absolute "^1.0.0" -glob@^7.1.1, glob@^7.1.2, glob@^7.1.3, glob@^7.1.4, glob@^7.1.6, glob@^7.2.0: +glob@7.2.0, glob@^7.1.1, glob@^7.1.2, glob@^7.1.3, glob@^7.1.4, glob@^7.1.6, glob@^7.2.0: version "7.2.0" resolved "https://registry.yarnpkg.com/glob/-/glob-7.2.0.tgz#d15535af7732e02e948f4c41628bd910293f6023" integrity sha512-lmLf6gtyrPq8tTjSmrO94wBeQbFR3HbLHbuyD69wuyQkImp2hWqMGB47OX65FBkPffO641IP9jWa1z4ivqG26Q== @@ -17652,6 +17672,17 @@ linkedom@^0.14.12: htmlparser2 "^8.0.1" uhyphen "^0.1.0" +linkedom@^0.14.16: + version "0.14.16" + resolved "https://registry.yarnpkg.com/linkedom/-/linkedom-0.14.16.tgz#124eb006fad1dfe7ed8f96ec8ae74ab0fb0fd88e" + integrity sha512-a4QWl4W93P15/x+4d9k8K+C81nOzQeGOs3D37uG0TFqKZYGLEyZwXweSFrypK8yvUx5U2cuZKkdDIOjaouv3ag== + dependencies: + css-select "^5.1.0" + cssom "^0.5.0" + html-escaper "^3.0.3" + htmlparser2 "^8.0.1" + uhyphen "^0.1.0" + linkedom@^0.14.9: version "0.14.9" resolved "https://registry.yarnpkg.com/linkedom/-/linkedom-0.14.9.tgz#34c6f15eddc809406f42d8ee48cd30b0222eccb0" @@ -18054,6 +18085,13 @@ loose-envify@^1.0.0, loose-envify@^1.1.0, loose-envify@^1.4.0: dependencies: js-tokens "^3.0.0 || ^4.0.0" +loupe@^2.3.1: + version "2.3.4" + resolved "https://registry.yarnpkg.com/loupe/-/loupe-2.3.4.tgz#7e0b9bffc76f148f9be769cb1321d3dcf3cb25f3" + integrity sha512-OvKfgCC2Ndby6aSTREl5aCCPTNIzlDfQZvZxNUrBrihDhL3xcrYegTblhmEiCrg2kKQz4XsFIaemE5BF4ybSaQ== + dependencies: + get-func-name "^2.0.0" + lower-case-first@^1.0.0: version "1.0.2" resolved "https://registry.yarnpkg.com/lower-case-first/-/lower-case-first-1.0.2.tgz#e5da7c26f29a7073be02d52bac9980e5922adfa1" @@ -18151,6 +18189,11 @@ luxon@^2.3.1: resolved "https://registry.yarnpkg.com/luxon/-/luxon-2.3.1.tgz#f276b1b53fd9a740a60e666a541a7f6dbed4155a" integrity sha512-I8vnjOmhXsMSlNMZlMkSOvgrxKJl0uOsEzdGgGNZuZPaS9KlefpE9KV95QFftlJSC+1UyCC9/I69R02cz/zcCA== +luxon@^3.0.4: + version "3.0.4" + resolved "https://registry.yarnpkg.com/luxon/-/luxon-3.0.4.tgz#d179e4e9f05e092241e7044f64aaa54796b03929" + integrity sha512-aV48rGUwP/Vydn8HT+5cdr26YYQiUZ42NM6ToMoaGKwYfWbfLeRkEu1wXWMHBZT6+KyLfcbbtVcoQFCbbPjKlw== + lz-string@^1.4.4: version "1.4.4" resolved "https://registry.yarnpkg.com/lz-string/-/lz-string-1.4.4.tgz#c0d8eaf36059f705796e1e344811cf4c498d3a26" @@ -18605,6 +18648,13 @@ minimatch@3.0.4: dependencies: brace-expansion "^1.1.7" +minimatch@5.0.1: + version "5.0.1" + resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-5.0.1.tgz#fb9022f7528125187c92bd9e9b6366be1cf3415b" + integrity sha512-nLDxIFRyhDblz3qMuq+SoRZED4+miJ/G+tdDrjkkkRnjAsBexeGpgjLEQ0blJy7rHhR2b93rhQY4SvyWu9v03g== + dependencies: + brace-expansion "^2.0.1" + minimatch@^3.0.2, minimatch@^3.0.4: version "3.1.2" resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-3.1.2.tgz#19cd194bfd3e428f049a70817c038d89ab4be35b" @@ -18787,6 +18837,34 @@ mocha-unfunk-reporter@^0.4.0: miniwrite "~0.1.3" unfunk-diff "~0.0.1" +mocha@^10.0.0: + version "10.0.0" + resolved "https://registry.yarnpkg.com/mocha/-/mocha-10.0.0.tgz#205447d8993ec755335c4b13deba3d3a13c4def9" + integrity sha512-0Wl+elVUD43Y0BqPZBzZt8Tnkw9CMUdNYnUsTfOM1vuhJVZL+kiesFYsqwBkEEuEixaiPe5ZQdqDgX2jddhmoA== + dependencies: + "@ungap/promise-all-settled" "1.1.2" + ansi-colors "4.1.1" + browser-stdout "1.3.1" + chokidar "3.5.3" + debug "4.3.4" + diff "5.0.0" + escape-string-regexp "4.0.0" + find-up "5.0.0" + glob "7.2.0" + he "1.2.0" + js-yaml "4.1.0" + log-symbols "4.1.0" + minimatch "5.0.1" + ms "2.1.3" + nanoid "3.3.3" + serialize-javascript "6.0.0" + strip-json-comments "3.1.1" + supports-color "8.1.1" + workerpool "6.2.1" + yargs "16.2.0" + yargs-parser "20.2.4" + yargs-unparser "2.0.0" + mocha@^8.2.0: version "8.4.0" resolved "https://registry.yarnpkg.com/mocha/-/mocha-8.4.0.tgz#677be88bf15980a3cae03a73e10a0fc3997f0cff" @@ -18949,7 +19027,7 @@ nan@^2.12.1: resolved "https://registry.yarnpkg.com/nan/-/nan-2.15.0.tgz#3f34a473ff18e15c1b5626b62903b5ad6e665fee" integrity sha512-8ZtvEnA2c5aYCZYd1cvgdnU6cqwixRoYg70xPLWUws5ORTa/lnw+u4amixRS/Ac5U5mQVgp9pnlSUnbNWFaWZQ== -nanoid@*, nanoid@^3.1.23, nanoid@^3.1.25, nanoid@^3.1.29, nanoid@^3.1.30, nanoid@^3.3.1: +nanoid@*, nanoid@3.3.3, nanoid@^3.1.23, nanoid@^3.1.25, nanoid@^3.1.29, nanoid@^3.1.30, nanoid@^3.3.1: version "3.3.3" resolved "https://registry.yarnpkg.com/nanoid/-/nanoid-3.3.3.tgz#fd8e8b7aa761fe807dba2d1b98fb7241bb724a25" integrity sha512-p1sjXuopFs0xg+fPASzQ28agW1oHD7xDsd9Xkf3T15H3c/cifrFHVwrh74PdoklAPi+i7MdRsE47vm2r6JoB+w== @@ -19120,6 +19198,16 @@ nock@^13.2.4: lodash.set "^4.3.2" propagate "^2.0.0" +nock@^13.2.9: + version "13.2.9" + resolved "https://registry.yarnpkg.com/nock/-/nock-13.2.9.tgz#4faf6c28175d36044da4cfa68e33e5a15086ad4c" + integrity sha512-1+XfJNYF1cjGB+TKMWi29eZ0b82QOvQs2YoLNzbpWGqFMtRQHTa57osqdGj4FrFPgkO4D4AZinzUJR9VvW3QUA== + dependencies: + debug "^4.1.0" + json-stringify-safe "^5.0.1" + lodash "^4.17.21" + propagate "^2.0.0" + node-addon-api@^1.2.0: version "1.7.2" resolved "https://registry.yarnpkg.com/node-addon-api/-/node-addon-api-1.7.2.tgz#3df30b95720b53c24e59948b49532b662444f54d" @@ -24493,6 +24581,11 @@ underscore@^1.13.4, underscore@^1.9.1: resolved "https://registry.yarnpkg.com/underscore/-/underscore-1.13.4.tgz#7886b46bbdf07f768e0052f1828e1dcab40c0dee" integrity sha512-BQFnUDuAQ4Yf/cYY5LNrK9NCJFKriaRbD9uR1fTeXnBeoa97W0i41qkZfGO9pSo8I5KzjAcSY2XYtdf0oKd7KQ== +underscore@^1.13.6: + version "1.13.6" + resolved "https://registry.yarnpkg.com/underscore/-/underscore-1.13.6.tgz#04786a1f589dc6c09f761fc5f45b89e935136441" + integrity sha512-+A5Sja4HP1M08MaXya7p5LvjuM7K6q/2EaC0+iovj/wOcMsTzMvDFbasi/oSapiwOlt252IqsKqPjCl7huKS0A== + undici@^4.9.3: version "4.14.1" resolved "https://registry.yarnpkg.com/undici/-/undici-4.14.1.tgz#7633b143a8a10d6d63335e00511d071e8d52a1d9" @@ -24912,6 +25005,11 @@ uuid@^8.0.0, uuid@^8.3.0, uuid@^8.3.1, uuid@^8.3.2: resolved "https://registry.yarnpkg.com/uuid/-/uuid-8.3.2.tgz#80d5b5ced271bb9af6c445f21a1a04c606cefbe2" integrity sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg== +uuid@^9.0.0: + version "9.0.0" + resolved "https://registry.yarnpkg.com/uuid/-/uuid-9.0.0.tgz#592f550650024a38ceb0c562f2f6aa435761efb5" + integrity sha512-MXcSTerfPa4uqyzStbRoTgt5XIe3x5+42+q1sDuy3R5MDk66URdLMOZe5aPX/SQd+kuYAh0FdP/pO28IkQyTeg== + v8-compile-cache-lib@^3.0.1: version "3.0.1" resolved "https://registry.yarnpkg.com/v8-compile-cache-lib/-/v8-compile-cache-lib-3.0.1.tgz#6336e8d71965cb3d35a1bbb7868445a7c05264bf" @@ -25582,6 +25680,11 @@ workerpool@6.1.5: resolved "https://registry.yarnpkg.com/workerpool/-/workerpool-6.1.5.tgz#0f7cf076b6215fd7e1da903ff6f22ddd1886b581" integrity sha512-XdKkCK0Zqc6w3iTxLckiuJ81tiD/o5rBE/m+nXpRCB+/Sq4DqkfXZ/x0jW02DG1tGsfUGXbTJyZDP+eu67haSw== +workerpool@6.2.1: + version "6.2.1" + resolved "https://registry.yarnpkg.com/workerpool/-/workerpool-6.2.1.tgz#46fc150c17d826b86a008e5a4508656777e9c343" + integrity sha512-ILEIE97kDZvF9Wb9f6h5aXK4swSlKGUcOEGiIYb2OOu/IrDU9iwj0fD//SsA6E5ibwJxpEvhullJY4Sl4GcpAw== + wrap-ansi@^3.0.1: version "3.0.1" resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-3.0.1.tgz#288a04d87eda5c286e060dfe8f135ce8d007f8ba"