prehistoric-systems/omnivore
1
0
Fork 0
mirror of https://github.com/omnivore-app/omnivore.git synced 2026-03-11 08:54:26 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge branch 'main' of github.com:omnivore-app/omnivore into feat/1078

Browse source
This commit is contained in:
Rupin Khandelwal 2022-10-01 14:55:39 -05:00
commit 6ae0948ca9
155 changed files with 23259 additions and 2033 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

13
android/Omnivore/app/src/main/graphql/CreateHighlight.graphql Normal file
View file

@ -0,0 +1,13 @@
mutation CreateHighlight($input: CreateHighlightInput!) {
createHighlight(input: $input) {
... on CreateHighlightSuccess {
highlight {
...HighlightFields
}
}
... on CreateHighlightError {
errorCodes
}
}
}

12
android/Omnivore/app/src/main/graphql/DeleteHighlight.graphql Normal file
View file

@ -0,0 +1,12 @@
mutation DeleteHighlight($highlightId: ID!) {
deleteHighlight(highlightId: $highlightId) {
... on DeleteHighlightSuccess {
highlight {
id
}
}
... on DeleteHighlightError {
errorCodes
}
}
}

14
android/Omnivore/app/src/main/graphql/ReadingProgressMutation.graphql Normal file
View file

@ -0,0 +1,14 @@
mutation SaveArticleReadingProgress($input: SaveArticleReadingProgressInput!) {
saveArticleReadingProgress(input: $input) {
... on SaveArticleReadingProgressSuccess {
updatedArticle {
id
readingProgressPercent
readingProgressAnchorIndex
}
}
... on SaveArticleReadingProgressError {
errorCodes
}
}
}

13
android/Omnivore/app/src/main/graphql/UpdateHighlight.graphql Normal file
View file

@ -0,0 +1,13 @@
mutation UpdateHighlight($input: UpdateHighlightInput!) {
updateHighlight(input: $input) {
... on UpdateHighlightSuccess {
highlight {
id
}
}
... on UpdateHighlightError {
errorCodes
}
}
}

36
android/Omnivore/app/src/main/java/app/omnivore/omnivore/networking/HighlightMutations.kt Normal file
View file

@ -0,0 +1,36 @@
package app.omnivore.omnivore.networking
import android.util.Log
import app.omnivore.omnivore.graphql.generated.CreateHighlightMutation
import app.omnivore.omnivore.graphql.generated.type.CreateHighlightInput
import com.apollographql.apollo3.api.Optional
import com.google.gson.Gson
data class CreateHighlightParams(
val shortId: String?,
val highlightID: String?,
val quote: String?,
val patch: String?,
val articleId: String?,
val `annotation`: String?
) {
fun asCreateHighlightInput() = CreateHighlightInput(
annotation = Optional.presentIfNotNull(`annotation`),
articleId = articleId ?: "",
id = highlightID ?: "",
patch = patch ?: "",
quote = quote ?: "",
shortId = shortId ?: ""
)
}
suspend fun Networker.createHighlight(jsonString: String): Boolean {
val input = Gson().fromJson(jsonString, CreateHighlightParams::class.java).asCreateHighlightInput()
Log.d("Loggo", "created highlight input: $input")
val result = authenticatedApolloClient().mutation(CreateHighlightMutation(input)).execute()
val highlight = result.data?.createHighlight?.onCreateHighlightSuccess?.highlight
return highlight != null
}

36
android/Omnivore/app/src/main/java/app/omnivore/omnivore/networking/ReadingProgressMutations.kt Normal file
View file

@ -0,0 +1,36 @@
package app.omnivore.omnivore.networking
import app.omnivore.omnivore.graphql.generated.SaveArticleReadingProgressMutation
import app.omnivore.omnivore.graphql.generated.type.SaveArticleReadingProgressInput
import android.util.Log
import com.google.gson.Gson
data class ReadingProgressParams(
val id: String?,
val readingProgressPercent: Double?,
val readingProgressAnchorIndex: Int?
) {
fun asSaveReadingProgressInput() = SaveArticleReadingProgressInput(
id = id ?: "",
readingProgressPercent = readingProgressPercent ?: 0.0,
readingProgressAnchorIndex = readingProgressAnchorIndex ?: 0
)
}
suspend fun Networker.updateReadingProgress(jsonString: String): Boolean {
val input = Gson().fromJson(jsonString, ReadingProgressParams::class.java).asSaveReadingProgressInput()
Log.d("Loggo", "created reading progress input: $input")
val result = authenticatedApolloClient()
.mutation(SaveArticleReadingProgressMutation(input))
.execute()
val articleID = result.data?.saveArticleReadingProgress?.onSaveArticleReadingProgressSuccess?.updatedArticle?.id
Log.d("Loggo", "updated article with id: $articleID")
return articleID != null
}

4
android/Omnivore/app/src/main/java/app/omnivore/omnivore/networking/SearchQuery.kt
View file

@ -22,7 +22,7 @@ suspend fun Networker.search(
)
).execute()
val cursor = result.data?.search?.onSearchSuccess?.pageInfo?.endCursor
val newCursor = result.data?.search?.onSearchSuccess?.pageInfo?.endCursor
val itemList = result.data?.search?.onSearchSuccess?.edges ?: listOf()
val items = itemList.map {
@ -49,5 +49,5 @@ suspend fun Networker.search(
)
}
return SearchQueryResponse(cursor, items)
return SearchQueryResponse(newCursor, items)
}

68
android/Omnivore/app/src/main/java/app/omnivore/omnivore/ui/reader/AnnotationEditView.kt Normal file
View file

@ -0,0 +1,68 @@
package app.omnivore.omnivore.ui.reader
import androidx.compose.foundation.background
import androidx.compose.foundation.layout.*
import androidx.compose.foundation.shape.RoundedCornerShape
import androidx.compose.material3.*
import androidx.compose.runtime.Composable
import androidx.compose.runtime.mutableStateOf
import androidx.compose.runtime.remember
import androidx.compose.ui.Alignment
import androidx.compose.ui.Modifier
import androidx.compose.ui.draw.clip
import androidx.compose.ui.unit.dp
// TODO: better layout and styling for this view
@OptIn(ExperimentalMaterial3Api::class)
@Composable
fun AnnotationEditView(
initialAnnotation: String,
onSave: (String) -> Unit,
onCancel: () -> Unit,
) {
val annotation = remember { mutableStateOf(initialAnnotation) }
Column(
modifier = Modifier
.clip(RoundedCornerShape(4.dp))
.background(MaterialTheme.colorScheme.background)
.padding(8.dp),
) {
Column(
modifier = Modifier.padding(16.dp),
) {
Text(text = "Note")
Spacer(modifier = Modifier.height(8.dp))
TextField(
value = annotation.value,
onValueChange = { annotation.value = it }
)
}
Spacer(modifier = Modifier.height(8.dp))
Row(
modifier = Modifier.align(Alignment.End)
) {
Button(
onClick = {
onCancel()
}
) {
Text("Cancel")
}
Spacer(modifier = Modifier.width(8.dp))
Button(
onClick = {
onSave(annotation.value)
}
) {
Text("Save")
}
}
}
}

137
android/Omnivore/app/src/main/java/app/omnivore/omnivore/ui/reader/WebReader.kt
View file

@ -8,12 +8,17 @@ import android.view.*
import android.webkit.JavascriptInterface
import android.webkit.WebView
import android.webkit.WebViewClient
import androidx.compose.foundation.layout.Box
import androidx.compose.material3.Text
import androidx.compose.runtime.Composable
import androidx.compose.runtime.getValue
import androidx.compose.runtime.livedata.observeAsState
import androidx.compose.runtime.mutableStateOf
import androidx.compose.runtime.remember
import androidx.compose.ui.viewinterop.AndroidView
import app.omnivore.omnivore.R
import app.omnivore.omnivore.networking.ReadingProgressParams
import com.google.gson.Gson
import org.json.JSONObject
@ -36,6 +41,11 @@ fun WebReaderLoadingContainer(slug: String, webReaderViewModel: WebReaderViewMod
@SuppressLint("SetJavaScriptEnabled")
@Composable
fun WebReader(params: WebReaderParams, webReaderViewModel: WebReaderViewModel) {
// TODO: maybe handle cases where js can be queued up?
val javascriptToExecute = remember { mutableStateOf<String?>(null) }
val annotation: String? by webReaderViewModel.annotationLiveData.observeAsState(null)
WebView.setWebContentsDebuggingEnabled(true)
val webReaderContent = WebReaderContent(
@ -51,39 +61,81 @@ fun WebReader(params: WebReaderParams, webReaderViewModel: WebReaderViewModel) {
val styledContent = webReaderContent.styledContent()
AndroidView(factory = {
OmnivoreWebView(it).apply {
layoutParams = ViewGroup.LayoutParams(
ViewGroup.LayoutParams.MATCH_PARENT,
ViewGroup.LayoutParams.MATCH_PARENT
Box {
AndroidView(factory = {
OmnivoreWebView(it).apply {
layoutParams = ViewGroup.LayoutParams(
ViewGroup.LayoutParams.MATCH_PARENT,
ViewGroup.LayoutParams.MATCH_PARENT
)
settings.javaScriptEnabled = true
settings.allowContentAccess = true
settings.allowFileAccess = true
settings.domStorageEnabled = true
webViewClient = object : WebViewClient() {
}
val javascriptInterface = AndroidWebKitMessenger { actionID, json ->
when (actionID) {
"existingHighlightTap" -> {
isExistingHighlightSelected = true
actionTapCoordinates = Gson().fromJson(json, ActionTapCoordinates::class.java)
Log.d("Loggo", "receive existing highlight tap action: $actionTapCoordinates")
startActionMode(null, ActionMode.TYPE_PRIMARY)
}
else -> {
webReaderViewModel.handleIncomingWebMessage(actionID, json)
}
}
}
addJavascriptInterface(javascriptInterface, "AndroidWebKitMessenger")
loadDataWithBaseURL(
"file:///android_asset/",
styledContent,
"text/html; charset=utf-8",
"utf-8",
null
)
}
}, update = {
if (javascriptToExecute.value != null) {
it.evaluateJavascript(javascriptToExecute.value!!, null)
}
})
if (annotation != null) {
AnnotationEditView(
initialAnnotation = annotation!!,
onSave = {
val script = "var event = new Event('saveAnnotation');event.annotation = '$it';document.dispatchEvent(event);"
javascriptToExecute.value = script
webReaderViewModel.cancelAnnotationEdit()
},
onCancel = {
webReaderViewModel.cancelAnnotationEdit()
}
)
settings.javaScriptEnabled = true
settings.allowContentAccess = true
settings.allowFileAccess = true
settings.domStorageEnabled = true
webViewClient = object : WebViewClient() {
}
val javascriptInterface = AndroidWebKitMessenger { actionID, json ->
webReaderViewModel.handleIncomingWebMessage(actionID, json)
}
addJavascriptInterface(javascriptInterface, "AndroidWebKitMessenger")
loadDataWithBaseURL("file:///android_asset/", styledContent, "text/html; charset=utf-8", "utf-8", null);
}
}, update = {
it.loadDataWithBaseURL("file:///android_asset/", styledContent, "text/html; charset=utf-8", "utf-8", null);
})
}
}
class OmnivoreWebView(context: Context) : WebView(context) {
var isExistingHighlightSelected = false
var actionTapCoordinates: ActionTapCoordinates? = null
private val actionModeCallback = object : ActionMode.Callback2() {
// Called when the action mode is created; startActionMode() was called
override fun onCreateActionMode(mode: ActionMode, menu: Menu): Boolean {
mode.menuInflater.inflate(R.menu.text_selection_menu, menu)
if (isExistingHighlightSelected) {
mode.menuInflater.inflate(R.menu.highlight_selection_menu, menu)
isExistingHighlightSelected = false
} else {
mode.menuInflater.inflate(R.menu.text_selection_menu, menu)
}
return true
}
@ -97,7 +149,8 @@ class OmnivoreWebView(context: Context) : WebView(context) {
override fun onActionItemClicked(mode: ActionMode, item: MenuItem): Boolean {
return when (item.itemId) {
R.id.annotate -> {
Log.d("Loggo", "Annotate action selected")
val script = "var event = new Event('annotate');document.dispatchEvent(event);"
evaluateJavascript(script, null)
mode.finish()
true
}
@ -108,6 +161,13 @@ class OmnivoreWebView(context: Context) : WebView(context) {
mode.finish()
true
}
R.id.delete -> {
val script = "var event = new Event('remove');document.dispatchEvent(event);"
evaluateJavascript(script, null)
clearFocus()
mode.finish()
true
}
else -> {
Log.d("Loggo", "${item.itemId} selected")
false
@ -117,35 +177,44 @@ class OmnivoreWebView(context: Context) : WebView(context) {
// Called when the user exits the action mode
override fun onDestroyActionMode(mode: ActionMode) {
// actionMode = null
Log.d("Loggo", "destroying menu: $mode")
isExistingHighlightSelected = false
actionTapCoordinates = null
}
override fun onGetContentRect(mode: ActionMode?, view: View?, outRect: Rect?) {
Log.d("Loggo", "outRect: $outRect, View: $view")
outRect?.set(left, top, right, bottom)
}
}
private var currentActionModeCallback: ActionMode.Callback? = actionModeCallback
override fun startActionMode(callback: ActionMode.Callback?): ActionMode {
return super.startActionMode(currentActionModeCallback)
return super.startActionMode(actionModeCallback)
}
override fun startActionModeForChild(
originalView: View?,
callback: ActionMode.Callback?
): ActionMode {
return super.startActionModeForChild(originalView, currentActionModeCallback)
return super.startActionModeForChild(originalView, actionModeCallback)
}
override fun startActionMode(callback: ActionMode.Callback?, type: Int): ActionMode {
return super.startActionMode(currentActionModeCallback, type)
Log.d("Loggo", "startActionMode:type called")
return super.startActionMode(actionModeCallback, type)
}
}
class AndroidWebKitMessenger(val messageHandler: (String, JSONObject) -> Unit) {
class AndroidWebKitMessenger(val messageHandler: (String, String) -> Unit) {
@JavascriptInterface
fun handleIdentifiableMessage(actionID: String, jsonString: String) {
messageHandler(actionID, JSONObject(jsonString))
messageHandler(actionID, jsonString)
}
}
data class ActionTapCoordinates(
val rectX: Double,
val rectY: Double,
val rectWidth: Double,
val rectHeight: Double,
)

41
android/Omnivore/app/src/main/java/app/omnivore/omnivore/ui/reader/WebReaderViewModel.kt
View file

@ -6,8 +6,7 @@ import androidx.lifecycle.ViewModel
import androidx.lifecycle.viewModelScope
import app.omnivore.omnivore.DatastoreRepository
import app.omnivore.omnivore.models.LinkedItem
import app.omnivore.omnivore.networking.Networker
import app.omnivore.omnivore.networking.linkedItem
import app.omnivore.omnivore.networking.*
import com.google.gson.Gson
import dagger.hilt.android.lifecycle.HiltViewModel
import kotlinx.coroutines.launch
@ -19,12 +18,17 @@ data class WebReaderParams(
val articleContent: ArticleContent
)
data class AnnotationWebViewMessage(
val annotation: String?
)
@HiltViewModel
class WebReaderViewModel @Inject constructor(
private val datastoreRepo: DatastoreRepository,
private val networker: Networker
): ViewModel() {
val webReaderParamsLiveData = MutableLiveData<WebReaderParams?>(null)
val annotationLiveData = MutableLiveData<String?>(null)
fun loadItem(slug: String) {
viewModelScope.launch {
@ -45,37 +49,50 @@ class WebReaderViewModel @Inject constructor(
}
}
fun handleIncomingWebMessage(actionID: String, json: JSONObject) {
fun handleIncomingWebMessage(actionID: String, jsonString: String) {
when (actionID) {
"createHighlight" -> {
Log.d("Loggo", "receive create highlight action: $json")
viewModelScope.launch {
val isHighlightSynced = networker.createHighlight(jsonString)
Log.d("Network", "isHighlightSynced = $isHighlightSynced")
}
}
"deleteHighlight" -> {
// { highlightId }
Log.d("Loggo", "receive delete highlight action: $json")
Log.d("Loggo", "receive delete highlight action: $jsonString")
}
"updateHighlight" -> {
Log.d("Loggo", "receive update highlight action: $json")
Log.d("Loggo", "receive update highlight action: $jsonString")
}
"articleReadingProgress" -> {
Log.d("Loggo", "received article reading progress action: $json")
viewModelScope.launch {
val isReadingProgressSynced = networker.updateReadingProgress(jsonString)
Log.d("Network", "isReadingProgressSynced = $isReadingProgressSynced")
}
}
"annotate" -> {
Log.d("Loggo", "received annotate action: $json")
}
"existingHighlightTap" -> {
Log.d("Loggo", "receive existing highlight tap action: $json")
viewModelScope.launch {
val annotation = Gson()
.fromJson(jsonString, AnnotationWebViewMessage::class.java)
.annotation ?: ""
annotationLiveData.value = annotation
}
}
"shareHighlight" -> {
// unimplemented
}
else -> {
Log.d("Loggo", "receive unrecognized action of $actionID with json: $json")
Log.d("Loggo", "receive unrecognized action of $actionID with json: $jsonString")
}
}
}
fun reset() {
webReaderParamsLiveData.value = null
annotationLiveData.value = null
}
fun cancelAnnotationEdit() {
annotationLiveData.value = null
}
}

15
android/Omnivore/app/src/main/res/menu/highlight_selection_menu.xml Normal file
View file

@ -0,0 +1,15 @@
<?xml version="1.0" encoding="utf-8"?>
<menu xmlns:app="http://schemas.android.com/apk/res-auto"
xmlns:android="http://schemas.android.com/apk/res/android">
<item
android:id="@+id/delete"
android:title="@string/delete_highlight_menu_title"
app:showAsAction="always">
</item>
<item
android:id="@+id/annotate"
android:title="@string/annotate_menu_action"
app:showAsAction="always">
</item>
</menu>

1
android/Omnivore/app/src/main/res/values/strings.xml
View file

@ -5,4 +5,5 @@
<string name="welcome_subtitle">Save articles and read them later in our distraction-free reader.</string>
<string name="highlight_menu_action">Highlight</string>
<string name="annotate_menu_action">Annotate</string>
<string name="delete_highlight_menu_title">Delete</string>
</resources>

36
apple/Omnivore.xcodeproj/project.pbxproj
View file

@ -1233,7 +1233,7 @@
CODE_SIGN_IDENTITY = "Apple Development";
CODE_SIGN_STYLE = Automatic;
COMBINE_HIDPI_IMAGES = YES;
CURRENT_PROJECT_VERSION = 70;
CURRENT_PROJECT_VERSION = 75;
DEVELOPMENT_TEAM = QJF2XZ86HB;
ENABLE_HARDENED_RUNTIME = YES;
INFOPLIST_FILE = InfoPlists/ShareExtensionMac.plist;
@ -1243,7 +1243,7 @@
"@executable_path/../../../../Frameworks",
);
MACOSX_DEPLOYMENT_TARGET = 12.0;
MARKETING_VERSION = 1.14.0;
MARKETING_VERSION = 1.15.0;
MTL_ENABLE_DEBUG_INFO = INCLUDE_SOURCE;
MTL_FAST_MATH = YES;
PRODUCT_BUNDLE_IDENTIFIER = "app.omnivore.app.ShareExtension-Mac";
@ -1265,7 +1265,7 @@
CODE_SIGN_IDENTITY = "Apple Development";
CODE_SIGN_STYLE = Automatic;
COMBINE_HIDPI_IMAGES = YES;
CURRENT_PROJECT_VERSION = 70;
CURRENT_PROJECT_VERSION = 75;
DEVELOPMENT_TEAM = QJF2XZ86HB;
ENABLE_HARDENED_RUNTIME = YES;
INFOPLIST_FILE = InfoPlists/ShareExtensionMac.plist;
@ -1275,7 +1275,7 @@
"@executable_path/../../../../Frameworks",
);
MACOSX_DEPLOYMENT_TARGET = 12.0;
MARKETING_VERSION = 1.14.0;
MARKETING_VERSION = 1.15.0;
MTL_FAST_MATH = YES;
PRODUCT_BUNDLE_IDENTIFIER = "app.omnivore.app.ShareExtension-Mac";
PRODUCT_NAME = "$(TARGET_NAME)";
@ -1347,7 +1347,7 @@
CODE_SIGN_IDENTITY = "Apple Development";
CODE_SIGN_STYLE = Automatic;
COMBINE_HIDPI_IMAGES = YES;
CURRENT_PROJECT_VERSION = 70;
CURRENT_PROJECT_VERSION = 75;
DEVELOPMENT_ASSET_PATHS = "";
DEVELOPMENT_TEAM = QJF2XZ86HB;
ENABLE_HARDENED_RUNTIME = YES;
@ -1358,7 +1358,7 @@
"@executable_path/../Frameworks",
);
MACOSX_DEPLOYMENT_TARGET = 12.0;
MARKETING_VERSION = 1.14.0;
MARKETING_VERSION = 1.15.0;
MTL_ENABLE_DEBUG_INFO = INCLUDE_SOURCE;
MTL_FAST_MATH = YES;
PRODUCT_BUNDLE_IDENTIFIER = app.omnivore.app;
@ -1381,7 +1381,7 @@
CODE_SIGN_IDENTITY = "Apple Development";
CODE_SIGN_STYLE = Automatic;
COMBINE_HIDPI_IMAGES = YES;
CURRENT_PROJECT_VERSION = 70;
CURRENT_PROJECT_VERSION = 75;
DEVELOPMENT_ASSET_PATHS = "";
DEVELOPMENT_TEAM = QJF2XZ86HB;
ENABLE_HARDENED_RUNTIME = YES;
@ -1392,7 +1392,7 @@
"@executable_path/../Frameworks",
);
MACOSX_DEPLOYMENT_TARGET = 12.0;
MARKETING_VERSION = 1.14.0;
MARKETING_VERSION = 1.15.0;
MTL_FAST_MATH = YES;
PRODUCT_BUNDLE_IDENTIFIER = app.omnivore.app;
PRODUCT_NAME = "$(TARGET_NAME)";
@ -1447,7 +1447,7 @@
"$(inherited)",
"@executable_path/Frameworks",
);
MARKETING_VERSION = 1.14.0;
MARKETING_VERSION = 1.15.0;
PRODUCT_BUNDLE_IDENTIFIER = app.omnivore.app;
PRODUCT_NAME = Omnivore;
PROVISIONING_PROFILE_SPECIFIER = "";
@ -1479,7 +1479,7 @@
"@executable_path/Frameworks",
"@executable_path/../../Frameworks",
);
MARKETING_VERSION = 1.14.0;
MARKETING_VERSION = 1.15.0;
MTL_ENABLE_DEBUG_INFO = INCLUDE_SOURCE;
MTL_FAST_MATH = YES;
OTHER_LDFLAGS = (
@ -1518,7 +1518,7 @@
"@executable_path/Frameworks",
"@executable_path/../../Frameworks",
);
MARKETING_VERSION = 1.14.0;
MARKETING_VERSION = 1.15.0;
MTL_FAST_MATH = YES;
OTHER_LDFLAGS = (
"-framework",
@ -1544,7 +1544,7 @@
CODE_SIGN_ENTITLEMENTS = "Entitlements/SafariExtension-Mac.entitlements";
CODE_SIGN_IDENTITY = "Apple Development";
CODE_SIGN_STYLE = Automatic;
CURRENT_PROJECT_VERSION = 70;
CURRENT_PROJECT_VERSION = 75;
DEVELOPMENT_TEAM = QJF2XZ86HB;
ENABLE_HARDENED_RUNTIME = YES;
GENERATE_INFOPLIST_FILE = YES;
@ -1557,7 +1557,7 @@
"@executable_path/../../../../Frameworks",
);
MACOSX_DEPLOYMENT_TARGET = 12.0;
MARKETING_VERSION = 1.14.0;
MARKETING_VERSION = 1.15.0;
MTL_ENABLE_DEBUG_INFO = INCLUDE_SOURCE;
MTL_FAST_MATH = YES;
OTHER_LDFLAGS = (
@ -1583,7 +1583,7 @@
CODE_SIGN_ENTITLEMENTS = "Entitlements/SafariExtension-Mac.entitlements";
CODE_SIGN_IDENTITY = "Apple Development";
CODE_SIGN_STYLE = Automatic;
CURRENT_PROJECT_VERSION = 70;
CURRENT_PROJECT_VERSION = 75;
DEVELOPMENT_TEAM = QJF2XZ86HB;
ENABLE_HARDENED_RUNTIME = YES;
GENERATE_INFOPLIST_FILE = YES;
@ -1596,7 +1596,7 @@
"@executable_path/../../../../Frameworks",
);
MACOSX_DEPLOYMENT_TARGET = 12.0;
MARKETING_VERSION = 1.14.0;
MARKETING_VERSION = 1.15.0;
MTL_FAST_MATH = YES;
OTHER_LDFLAGS = (
"-framework",
@ -1683,7 +1683,7 @@
"@executable_path/Frameworks",
"@executable_path/../../Frameworks",
);
MARKETING_VERSION = 1.14.0;
MARKETING_VERSION = 1.15.0;
PRODUCT_BUNDLE_IDENTIFIER = "app.omnivore.app.share-extension";
PRODUCT_NAME = ShareExtension;
PROVISIONING_PROFILE_SPECIFIER = "";
@ -1738,7 +1738,7 @@
"$(inherited)",
"@executable_path/Frameworks",
);
MARKETING_VERSION = 1.14.0;
MARKETING_VERSION = 1.15.0;
PRODUCT_BUNDLE_IDENTIFIER = app.omnivore.app;
PRODUCT_NAME = Omnivore;
PROVISIONING_PROFILE_SPECIFIER = "";
@ -1767,7 +1767,7 @@
"@executable_path/Frameworks",
"@executable_path/../../Frameworks",
);
MARKETING_VERSION = 1.14.0;
MARKETING_VERSION = 1.15.0;
PRODUCT_BUNDLE_IDENTIFIER = "app.omnivore.app.share-extension";
PRODUCT_NAME = ShareExtension;
PROVISIONING_PROFILE_SPECIFIER = "";

1
apple/OmnivoreKit/Sources/App/PDFSupport/PDFViewer.swift
View file

@ -39,6 +39,7 @@ import Utils
.useParentNavigationBar(true)
.updateConfiguration { builder in
builder.textSelectionShouldSnapToWord = true
builder.shouldAskForAnnotationUsername = false
}
.updateControllerConfiguration { controller in
// Store config state so we only run this update closure once

61
apple/OmnivoreKit/Sources/App/Views/AudioPlayer/MiniPlayer.swift
View file

@ -192,11 +192,11 @@ public struct MiniPlayer: View {
if !expanded {
Text(itemAudioProperties.title)
.font(expanded ? .appTitle : .appCallout)
.font(.appCallout)
.lineSpacing(1.25)
.foregroundColor(.appGrayTextContrast)
.fixedSize(horizontal: false, vertical: false)
.frame(maxWidth: .infinity, alignment: expanded ? .center : .leading)
.frame(maxWidth: .infinity, alignment: .leading)
.matchedGeometryEffect(id: "ArticleTitle", in: animation)
playPauseButtonItem
@ -210,55 +210,26 @@ public struct MiniPlayer: View {
Spacer()
if expanded {
Text(itemAudioProperties.title)
.lineLimit(1)
.font(expanded ? .appTitle : .appCallout)
.lineSpacing(1.25)
Marquee(text: itemAudioProperties.title, font: UIFont(name: "Inter-Regular", size: 22)!)
.foregroundColor(.appGrayTextContrast)
.frame(maxWidth: .infinity, alignment: expanded ? .center : .leading)
.matchedGeometryEffect(id: "ArticleTitle", in: animation)
.onTapGesture {
viewArticle()
}
HStack {
Spacer()
if let byline = itemAudioProperties.byline {
Text(byline)
.lineLimit(1)
.font(.appCallout)
.lineSpacing(1.25)
.foregroundColor(.appGrayText)
.frame(alignment: .trailing)
}
Spacer()
if let byline = itemAudioProperties.byline {
Marquee(text: byline, font: UIFont(name: "Inter-Regular", size: 16)!)
.foregroundColor(.appGrayText)
}
Slider(value: $audioController.timeElapsed,
in: 0 ... self.audioController.duration,
onEditingChanged: { scrubStarted in
if scrubStarted {
self.audioController.scrubState = .scrubStarted
} else {
self.audioController.scrubState = .scrubEnded(self.audioController.timeElapsed)
}
})
.accentColor(.appCtaYellow)
.introspectSlider { slider in
// Make the thumb a little smaller than the default and give it the CTA color
// for some reason this doesn't work on my iPad though.
let tintColor = UIColor(Color.appCtaYellow)
let image = UIImage(systemName: "circle.fill",
withConfiguration: UIImage.SymbolConfiguration(scale: .small))?
.withTintColor(tintColor)
.withRenderingMode(.alwaysOriginal)
slider.setThumbImage(image, for: .selected)
slider.setThumbImage(image, for: .normal)
slider.minimumTrackTintColor = tintColor
}
ScrubberView(value: $audioController.timeElapsed,
minValue: 0, maxValue: self.audioController.duration,
onEditingChanged: { scrubStarted in
if scrubStarted {
self.audioController.scrubState = .scrubStarted
} else {
self.audioController.scrubState = .scrubEnded(self.audioController.timeElapsed)
}
})
HStack {
Text(audioController.timeElapsedString ?? "0:00")
@ -333,7 +304,7 @@ public struct MiniPlayer: View {
withAnimation(.easeIn(duration: 0.08)) { expanded = true }
}.sheet(isPresented: $showVoiceSheet) {
NavigationView {
TextToSpeechVoiceSelectionView(forLanguage: audioController.currentVoiceLanguage)
TextToSpeechVoiceSelectionView(forLanguage: audioController.currentVoiceLanguage, showLanguageChanger: true)
.navigationBarTitle("Voice")
.navigationBarTitleDisplayMode(.inline)
.navigationBarItems(leading: Button(action: { self.showVoiceSheet = false }) {

73
apple/OmnivoreKit/Sources/App/Views/AudioPlayer/ScrubberView.swift Normal file
View file

@ -0,0 +1,73 @@
//
// ScrubberView.swift
//
//
// Created by Jackson Harper on 9/27/22.
//
import Foundation
import SwiftUI
struct ScrubberView: UIViewRepresentable {
typealias UIViewType = UISlider
@Binding var value: Double
var minValue: Double
var maxValue: Double
var onEditingChanged: (Bool) -> Void
init(value: Binding<Double>, minValue: Double, maxValue: Double, onEditingChanged: @escaping (Bool) -> Void) {
self._value = value
self.minValue = minValue
self.maxValue = maxValue
self.onEditingChanged = onEditingChanged
}
func makeUIView(context: Context) -> UISlider {
let slider = UISlider(frame: .zero)
slider.maximumValue = Float(minValue)
slider.maximumValue = Float(maxValue)
let tintColor = UIColor(Color.appCtaYellow)
let image = UIImage(systemName: "circle.fill",
withConfiguration: UIImage.SymbolConfiguration(scale: .small))?
.withTintColor(tintColor)
.withRenderingMode(.alwaysOriginal)
slider.setThumbImage(image, for: .selected)
slider.setThumbImage(image, for: .normal)
slider.minimumTrackTintColor = tintColor
slider.addTarget(context.coordinator,
action: #selector(Coordinator.valueChanged(_:)),
for: .valueChanged)
return slider
}
func updateUIView(_ uiView: UISlider, context _: Context) {
uiView.value = Float(value)
}
func makeCoordinator() -> Coordinator {
let coordinator = Coordinator(value: $value, onEditingChanged: onEditingChanged)
return coordinator
}
class Coordinator: NSObject {
var value: Binding<Double>
var onEditingChanged: (Bool) -> Void
init(value: Binding<Double>, onEditingChanged: @escaping (Bool) -> Void) {
self.value = value
self.onEditingChanged = onEditingChanged
super.init()
}
@objc func valueChanged(_ sender: UISlider) {
value.wrappedValue = Double(sender.value)
onEditingChanged(sender.isTracking)
}
}
}

204
apple/OmnivoreKit/Sources/App/Views/Labels/MarqueTextView.swift Normal file
View file

@ -0,0 +1,204 @@
import SwiftUI
// Mostly from: https://kavsoft.dev/swiftui_3.0_marquee_text_animation with some customizations
struct Marquee: View {
var text: String
var font: UIFont
// Storing Text Size
@State var storedSize: CGSize = .zero
@State var offset: CGFloat = 0
@State var animatedText: String = ""
var animationSpeed: Double = 0.03
var delayTime: Double = 3.0
var body: some View {
// Since it scrolls horizontal using ScrollView
GeometryReader { proxy in
let size = proxy.size
let condition = textSize(text: text).width < (size.width - 50)
ScrollView(condition ? .init() : .horizontal, showsIndicators: false) {
HStack(alignment: .center) {
Spacer(minLength: 0)
Text(condition ? text : animatedText)
.font(Font(font))
.offset(x: condition ? 0 : offset)
.padding(.horizontal, 15)
Spacer(minLength: 0)
}
}
.frame(maxWidth: .infinity, maxHeight: .infinity, alignment: .center)
}
.frame(height: storedSize.height)
.overlay(content: {
HStack {
let color: Color = .systemBackground
LinearGradient(colors: [color, color.opacity(0.7), color.opacity(0.5), color.opacity(0.3)], startPoint: .leading, endPoint: .trailing)
.frame(width: 8)
Spacer()
LinearGradient(colors: [color, color.opacity(0.7), color.opacity(0.5), color.opacity(0.3)].reversed(), startPoint: .leading, endPoint: .trailing)
.frame(width: 8)
}
})
.disabled(true)
.onAppear {
startAnimation(text: text)
}
.onReceive(Timer.publish(every: (animationSpeed * storedSize.width) + delayTime,
on: .main,
in: .default).autoconnect()
) { _ in
offset = 0
withAnimation(.linear(duration: animationSpeed * storedSize.width).delay(delayTime)) {
offset = -storedSize.width
}
}
.onChange(of: text) { newValue in
animatedText = ""
offset = 0
startAnimation(text: newValue)
}
}
func startAnimation(text: String) {
// Double the text with some spacing so that we can create a continuous loop
animatedText.append(text)
(1 ... 15).forEach { _ in
animatedText.append(" ")
}
storedSize = textSize(text: animatedText)
animatedText.append(text)
let timing: Double = (animationSpeed * storedSize.width)
withAnimation(.linear(duration: timing).delay(delayTime)) {
offset = -storedSize.width
}
}
func textSize(text: String) -> CGSize {
let attributes = [NSAttributedString.Key.font: font]
let size = (text as NSString).size(withAttributes: attributes)
return size
}
}
// Old version:
//
// struct MarqueTextView: View {
// let font: Font
//
// @State var text: String
// @State private var intrinsicSize: CGSize = .zero
// @State private var truncatedSize: CGSize = .zero
//
// @State private var shouldAnimate: Bool = false
// @State private var animationOffset: Double = 0.0
//
// var body: some View {
// GeometryReader { geo in
// ScrollView(.horizontal, showsIndicators: false) {
// HStack(alignment: .center) {
// Spacer(minLength: 0)
// Text(text)
// .font(font)
// .lineLimit(1)
// .lineSpacing(1.25)
// .offset(x: animationOffset)
// .readSize { size in
// truncatedSize = size
// intrinsicSize = geo.size
//
// shouldAnimate = textSize().width > intrinsicSize.width
// }
// Spacer(minLength: 0)
// }
// .frame(width: max(geo.size.width, textSize().width + 10))
// }
// .frame(maxWidth: .infinity, alignment: .center)
// .disabled(true)
// .onChange(of: shouldAnimate) { _ in
//
// let baseText = text
// text.append(" ")
// let initialSize = textSize()
//
// print("starting animation, truncatedSize: ", truncatedSize, "geo width: ", geo.size)
// if shouldAnimate {
// DispatchQueue.main.asyncAfter(deadline: .now() + .milliseconds(500)) {
// withAnimation(.linear(duration: 0.05 * truncatedSize.width)) {
// animationOffset = -truncatedSize.width
// }
// }
// }
// }
// .onReceive(Timer.publish(every: 0.05 * truncatedSize.width + 0.5, on: .main, in: .default).autoconnect()) { _ in
// if shouldAnimate {
// animationOffset = 0
// withAnimation(.linear(duration: 0.05 * truncatedSize.width)) {
// animationOffset = -truncatedSize.width
// }
// }
// }
// }
// }
//
// func textSize() -> CGSize {
// let attributes = [NSAttributedString.Key.font: UIFont(name: "Inter-Regular", size: 16)!]
// return (text as NSString).size(withAttributes: attributes)
// }
// }
//
//// text()
//// .lineLimit(lineLimit)
//// .offset(x: animationOffset)
//// .readSize { size in
//// truncatedSize = size
//// shouldAnimate = truncatedSize != intrinsicSize
//// print("trunvatedSize: ", truncatedSize, "intrinsicSize: ", intrinsicSize)
//// }
//// .background(
//// text()
//// .fixedSize(horizontal: false, vertical: true)
//// .hidden()
//// .readSize { size in
//// intrinsicSize = size
//// shouldAnimate = truncatedSize != intrinsicSize
//// }
//// )
//// .onChange(of: shouldAnimate, perform: { _ in
//// print("starting animation")
//// DispatchQueue.main.asyncAfter(deadline: .now() + .milliseconds(500)) {
//// withAnimation(.linear(duration: 0.2 * intrinsicSize.width)) {
//// animationOffset = intrinsicSize.width
//// }
//// }
//// })
//// }
//// }
//
// extension View {
// func readSize(onChange: @escaping (CGSize) -> Void) -> some View {
// background(
// GeometryReader { geometryProxy in
// Color.clear
// .preference(key: SizePreferenceKey.self, value: geometryProxy.size)
// }
// )
// .onPreferenceChange(SizePreferenceKey.self, perform: onChange)
// }
// }
//
// struct SizePreferenceKey: PreferenceKey {
// static var defaultValue: CGSize = .zero
// static func reduce(value _: inout CGSize, nextValue _: () -> CGSize) {}
// }

2
apple/OmnivoreKit/Sources/App/Views/Profile/TextToSpeechView.swift
View file

@ -30,7 +30,7 @@ struct TextToSpeechView: View {
private var innerBody: some View {
Section("Voices") {
ForEach(Voices.Languages, id: \.key) { language in
NavigationLink(destination: TextToSpeechVoiceSelectionView(forLanguage: language)) {
NavigationLink(destination: TextToSpeechVoiceSelectionView(forLanguage: language, showLanguageChanger: false)) {
Text(language.name)
}
}

12
apple/OmnivoreKit/Sources/App/Views/Profile/TextToSpeechVoiceSelectionView.swift
View file

@ -6,18 +6,22 @@ import Views
struct TextToSpeechVoiceSelectionView: View {
@EnvironmentObject var audioController: AudioController
let language: VoiceLanguage
let showLanguageChanger: Bool
init(forLanguage: VoiceLanguage) {
init(forLanguage: VoiceLanguage, showLanguageChanger: Bool) {
self.language = forLanguage
self.showLanguageChanger = showLanguageChanger
}
var body: some View {
Group {
#if os(iOS)
Form {
Section("Language") {
NavigationLink(destination: TextToSpeechLanguageView().navigationTitle("Language")) {
Text(audioController.currentVoiceLanguage.name)
if showLanguageChanger {
Section("Language") {
NavigationLink(destination: TextToSpeechLanguageView().navigationTitle("Language")) {
Text(audioController.currentVoiceLanguage.name)
}
}
}
innerBody

3
apple/OmnivoreKit/Sources/App/Views/WelcomeView.swift
View file

@ -33,7 +33,7 @@ struct WelcomeView: View {
var headlineText: some View {
Group {
Text("Never miss a great read.")
Text("Read-it-later for serious readers.")
}
.font(.appLargeTitle)
}
@ -153,6 +153,7 @@ struct WelcomeView: View {
action: { showEmailLoginModal = true },
label: {
Text("Continue with Email")
.font(.appHeadline)
.foregroundColor(.appGrayTextContrast)
.underline()
}

2
apple/OmnivoreKit/Sources/Services/AudioSession/AudioController.swift
View file

@ -390,7 +390,7 @@ public class AudioController: NSObject, ObservableObject, AVAudioPlayerDelegate
}
}
@AppStorage(UserDefaultKey.textToSpeechPreloadEnabled.rawValue) public var preloadEnabled = true
@AppStorage(UserDefaultKey.textToSpeechPreloadEnabled.rawValue) public var preloadEnabled = false
public var currentVoiceLanguage: VoiceLanguage {
Voices.Languages.first(where: { $0.key == currentLanguage }) ?? Voices.English

24
apple/OmnivoreKit/Sources/Services/DataService/Mutations/DeleteHighlight.swift
View file

@ -5,20 +5,30 @@ import SwiftGraphQL
public extension DataService {
func deleteHighlight(highlightID: String) {
if let highlight = Highlight.lookup(byID: highlightID, inContext: backgroundContext) {
if let highlight = Highlight.lookup(byID: highlightID, inContext: viewContext) {
deleteHighlight(objectID: highlight.objectID)
}
}
private func deleteHighlight(objectID: NSManagedObjectID) {
// Update CoreData
backgroundContext.perform { [weak self] in
guard let self = self else { return }
guard let highlight = self.backgroundContext.object(with: objectID) as? Highlight else { return }
highlight.remove(inContext: self.backgroundContext)
viewContext.performAndWait {
guard let highlight = viewContext.object(with: objectID) as? Highlight else { return }
highlight.serverSyncStatus = Int64(ServerSyncStatus.needsDeletion.rawValue)
// Send update to server
self.syncHighlightDeletion(highlightID: highlight.unwrappedID, objectID: objectID)
do {
try viewContext.save()
logger.debug("Highlight succesfully marked for deletion")
} catch {
viewContext.rollback()
logger.debug("Failed to mark Highlight for deletion: \(error.localizedDescription)")
}
}
// Send update to server
backgroundContext.perform { [weak self] in
guard let highlight = self?.backgroundContext.object(with: objectID) as? Highlight else { return }
self?.syncHighlightDeletion(highlightID: highlight.unwrappedID, objectID: objectID)
}
}

19
apple/OmnivoreKit/Sources/Services/DataService/Mutations/RemoveLabelPublisher.swift
View file

@ -5,14 +5,21 @@ import SwiftGraphQL
extension DataService {
public func removeLabel(labelID: String, name: String) {
// Update CoreData
backgroundContext.perform { [weak self] in
guard let self = self else { return }
guard let label = LinkedItemLabel.lookup(byID: labelID, inContext: self.backgroundContext) else { return }
label.remove(inContext: self.backgroundContext)
viewContext.performAndWait {
guard let label = LinkedItemLabel.lookup(byID: labelID, inContext: self.viewContext) else { return }
label.serverSyncStatus = Int64(ServerSyncStatus.needsDeletion.rawValue)
// Send update to server
self.syncLabelDeletion(labelID: labelID, labelName: name)
do {
try viewContext.save()
logger.debug("Label succesfully marked for deletion")
} catch {
viewContext.rollback()
logger.debug("Failed to mark Label for deletion: \(error.localizedDescription)")
}
}
// Send update to server
syncLabelDeletion(labelID: labelID, labelName: name)
}
func syncLabelDeletion(labelID: String, labelName _: String) {

26
apple/OmnivoreKit/Sources/Services/DataService/Mutations/RemoveLink.swift
View file

@ -3,16 +3,26 @@ import Foundation
import Models
import SwiftGraphQL
extension DataService {
public func removeLink(objectID: NSManagedObjectID) {
public extension DataService {
func removeLink(objectID: NSManagedObjectID) {
// Update CoreData
backgroundContext.perform { [weak self] in
guard let self = self else { return }
guard let linkedItem = self.backgroundContext.object(with: objectID) as? LinkedItem else { return }
linkedItem.remove(inContext: self.backgroundContext)
viewContext.performAndWait {
guard let linkedItem = viewContext.object(with: objectID) as? LinkedItem else { return }
linkedItem.serverSyncStatus = Int64(ServerSyncStatus.needsDeletion.rawValue)
// Send update to server
self.syncLinkDeletion(itemID: linkedItem.unwrappedID, objectID: objectID)
do {
try viewContext.save()
logger.debug("LinkedItem succesfully marked for deletion")
} catch {
viewContext.rollback()
logger.debug("Failed to mark LinkedItem for deletion: \(error.localizedDescription)")
}
}
// Send update to server
backgroundContext.perform { [weak self] in
guard let linkedItem = self?.backgroundContext.object(with: objectID) as? LinkedItem else { return }
self?.syncLinkDeletion(itemID: linkedItem.unwrappedID, objectID: objectID)
}
}

2
apple/OmnivoreKit/Sources/Views/Fonts.swift
View file

@ -6,7 +6,7 @@ import SwiftUI
public extension Font {
/// 34pt, Inter-Regular
static var appLargeTitle: Font {
.customFont(InterFont.regular.rawValue, size: 34, relativeTo: .largeTitle)
.customFont(InterFont.bold.rawValue, size: 34, relativeTo: .largeTitle)
}
/// 28pt, Inter-Regular

2
packages/api/src/elastic/types.ts
View file

@ -247,6 +247,8 @@ export interface SearchItem {
labels?: Label[]
highlights?: Highlight[]
wordsCount?: number
siteName?: string
siteIcon?: string
}
const keys = ['_id', 'url', 'slug', 'userId', 'uploadFileId', 'state'] as const

3
packages/api/src/entity/subscription.ts
View file

@ -43,6 +43,9 @@ export class Subscription {
@Column('text', { nullable: true })
unsubscribeHttpUrl?: string
@Column('text', { nullable: true })
icon?: string
@CreateDateColumn()
createdAt!: Date

4
packages/api/src/generated/graphql.ts
View file

@ -1644,6 +1644,7 @@ export type SearchItem = {
readingProgressPercent: Scalars['Float'];
savedAt: Scalars['Date'];
shortId?: Maybe<Scalars['String']>;
siteIcon?: Maybe<Scalars['String']>;
siteName?: Maybe<Scalars['String']>;
slug: Scalars['String'];
state?: Maybe<ArticleSavingRequestStatus>;
@ -1987,6 +1988,7 @@ export type Subscription = {
__typename?: 'Subscription';
createdAt: Scalars['Date'];
description?: Maybe<Scalars['String']>;
icon?: Maybe<Scalars['String']>;
id: Scalars['ID'];
name: Scalars['String'];
newsletterEmail: Scalars['String'];
@ -4159,6 +4161,7 @@ export type SearchItemResolvers<ContextType = ResolverContext, ParentType extend
readingProgressPercent?: Resolver<ResolversTypes['Float'], ParentType, ContextType>;
savedAt?: Resolver<ResolversTypes['Date'], ParentType, ContextType>;
shortId?: Resolver<Maybe<ResolversTypes['String']>, ParentType, ContextType>;
siteIcon?: Resolver<Maybe<ResolversTypes['String']>, ParentType, ContextType>;
siteName?: Resolver<Maybe<ResolversTypes['String']>, ParentType, ContextType>;
slug?: Resolver<ResolversTypes['String'], ParentType, ContextType>;
state?: Resolver<Maybe<ResolversTypes['ArticleSavingRequestStatus']>, ParentType, ContextType>;
@ -4368,6 +4371,7 @@ export type SubscribeSuccessResolvers<ContextType = ResolverContext, ParentType
export type SubscriptionResolvers<ContextType = ResolverContext, ParentType extends ResolversParentTypes['Subscription'] = ResolversParentTypes['Subscription']> = {
createdAt?: SubscriptionResolver<ResolversTypes['Date'], "createdAt", ParentType, ContextType>;
description?: SubscriptionResolver<Maybe<ResolversTypes['String']>, "description", ParentType, ContextType>;
icon?: SubscriptionResolver<Maybe<ResolversTypes['String']>, "icon", ParentType, ContextType>;
id?: SubscriptionResolver<ResolversTypes['ID'], "id", ParentType, ContextType>;
name?: SubscriptionResolver<ResolversTypes['String'], "name", ParentType, ContextType>;
newsletterEmail?: SubscriptionResolver<ResolversTypes['String'], "newsletterEmail", ParentType, ContextType>;

2
packages/api/src/generated/schema.graphql
View file

@ -1170,6 +1170,7 @@ type SearchItem {
readingProgressPercent: Float!
savedAt: Date!
shortId: String
siteIcon: String
siteName: String
slug: String!
state: ArticleSavingRequestStatus
@ -1485,6 +1486,7 @@ type SubscribeSuccess {
type Subscription {
createdAt: Date!
description: String
icon: String
id: ID!
name: String!
newsletterEmail: String!

1
packages/api/src/resolvers/article/index.ts
View file

@ -899,6 +899,7 @@ export const searchResolver = authorized<
publishedAt: validatedDate(r.publishedAt),
ownedByViewer: r.userId === claims.uid,
pageType: r.pageType || PageType.Highlights,
siteIcon: r.siteIcon && createImageProxyUrl(r.siteIcon, 32, 32),
} as SearchItem,
cursor: endCursor,
}

6
packages/api/src/resolvers/subscriptions/index.ts
View file

@ -23,6 +23,7 @@ import { User } from '../../entity/user'
import { Subscription } from '../../entity/subscription'
import { getSubscribeHandler, unsubscribe } from '../../services/subscriptions'
import { ILike } from 'typeorm'
import { createImageProxyUrl } from '../../utils/imageproxy'
export const subscriptionsResolver = authorized<
SubscriptionsSuccess,
@ -57,7 +58,10 @@ export const subscriptionsResolver = authorized<
})
return {
subscriptions,
subscriptions: subscriptions.map((s) => ({
...s,
icon: s.icon && createImageProxyUrl(s.icon, 32, 32),
})),
}
} catch (error) {
log.error(error)

22
packages/api/src/routers/svc/emails.ts
View file

@ -8,14 +8,11 @@ import { analytics } from '../../utils/analytics'
import { getNewsletterEmail } from '../../services/newsletters'
import { env } from '../../env'
import {
findNewsletterUrl,
generateUniqueUrl,
getTitleFromEmailSubject,
isProbablyArticle,
isProbablyNewsletter,
parseEmailAddress,
} from '../../utils/parser'
import { saveNewsletterEmail } from '../../services/save_newsletter_email'
import { saveEmail } from '../../services/save_email'
import { buildLogger } from '../../utils/logger'
@ -80,25 +77,6 @@ export function emailsServiceRouter() {
const ctx = { pubsub: createPubSubClient(), uid: user.id }
const parsedFrom = parseEmailAddress(data.from)
if (await isProbablyNewsletter(data.html)) {
logger.info('handling as newsletter', data)
await saveNewsletterEmail(
{
email: data.to,
title: data.subject,
content: data.html,
author: parsedFrom.name,
url: (await findNewsletterUrl(data.html)) || generateUniqueUrl(),
unsubMailTo: data.unsubMailTo,
unsubHttpUrl: data.unsubHttpUrl,
newsletterEmail,
},
ctx
)
res.status(200).send('Newsletter')
return
}
if (
await isProbablyArticle(
data.forwardedFrom || parsedFrom.address,

2
packages/api/src/schema.ts
View file

@ -1495,6 +1495,7 @@ const schema = gql`
readAt: Date
savedAt: Date!
highlights: [Highlight!]
siteIcon: String
}
type SearchItemEdge {
@ -1530,6 +1531,7 @@ const schema = gql`
status: SubscriptionStatus!
unsubscribeMailTo: String
unsubscribeHttpUrl: String
icon: String
createdAt: Date!
updatedAt: Date!
}

1
packages/api/src/services/save_email.ts
View file

@ -70,6 +70,7 @@ export const saveEmail = async (
readingProgressPercent: 0,
subscription: input.author,
state: ArticleSavingRequestStatus.Succeeded,
siteIcon: parseResult.parsedContent?.siteIcon,
}
const page = await getPageByParam({

31
packages/api/src/services/save_newsletter_email.ts
View file

@ -12,6 +12,8 @@ import { Page } from '../elastic/types'
import { addLabelToPage } from './labels'
import { saveSubscription } from './subscriptions'
import { NewsletterEmail } from '../entity/newsletter_email'
import { fetchFavicon } from '../utils/parser'
import { updatePage } from '../elastic/pages'
interface NewsletterMessage {
email: string
@ -33,7 +35,6 @@ export const saveNewsletterEmail = async (
// get user from newsletter email
const newsletterEmail =
data.newsletterEmail || (await getNewsletterEmail(data.email))
if (!newsletterEmail) {
console.log('newsletter email not found', data.email)
return false
@ -54,7 +55,6 @@ export const saveNewsletterEmail = async (
pubsub: createPubSubClient(),
uid: newsletterEmail.user.id,
}
const input: SaveEmailInput = {
url: data.url,
originalContent: data.content,
@ -63,22 +63,31 @@ export const saveNewsletterEmail = async (
unsubMailTo: data.unsubMailTo,
unsubHttpUrl: data.unsubHttpUrl,
}
const page = await saveEmail(saveCtx, input)
if (!page) {
console.log('newsletter not created:', input)
return false
}
if (!page.siteIcon) {
// fetch favicon if not already set
const favicon = await fetchFavicon(page.url)
if (favicon) {
page.siteIcon = favicon
await updatePage(page.id, { siteIcon: favicon }, saveCtx)
}
}
// creates or updates subscription
const subscription = await saveSubscription(
newsletterEmail.user.id,
data.author,
newsletterEmail.address,
data.unsubMailTo,
data.unsubHttpUrl
)
console.log('subscription', subscription)
const subscription = await saveSubscription({
userId: newsletterEmail.user.id,
name: data.author,
newsletterEmail: newsletterEmail.address,
unsubscribeMailTo: data.unsubMailTo,
unsubscribeHttpUrl: data.unsubHttpUrl,
icon: page.siteIcon,
})
console.log('subscription saved', subscription)
// adds newsletters label to page
const result = await addLabelToPage(saveCtx, page.id, {

26
packages/api/src/services/subscriptions.ts
View file

@ -6,6 +6,15 @@ import axios from 'axios'
import { NewsletterEmail } from '../entity/newsletter_email'
import { createNewsletterEmail } from './newsletters'
interface SaveSubscriptionInput {
userId: string
name: string
newsletterEmail: string
unsubscribeMailTo?: string
unsubscribeHttpUrl?: string
icon?: string
}
const sendUnsubscribeEmail = async (
unsubscribeMailTo: string,
newsletterEmail: string
@ -30,13 +39,14 @@ const sendUnsubscribeHttpRequest = async (url: string): Promise<void> => {
}
}
export const saveSubscription = async (
userId: string,
name: string,
newsletterEmail: string,
unsubscribeMailTo?: string,
unsubscribeHttpUrl?: string
): Promise<Subscription> => {
export const saveSubscription = async ({
userId,
name,
newsletterEmail,
unsubscribeMailTo,
unsubscribeHttpUrl,
icon,
}: SaveSubscriptionInput): Promise<Subscription> => {
const subscription = await getRepository(Subscription).findOneBy({
name,
user: { id: userId },
@ -46,6 +56,7 @@ export const saveSubscription = async (
// if subscription already exists, updates updatedAt
subscription.status = SubscriptionStatus.Active
subscription.newsletterEmail = newsletterEmail
icon && (subscription.icon = icon)
unsubscribeMailTo && (subscription.unsubscribeMailTo = unsubscribeMailTo)
unsubscribeHttpUrl && (subscription.unsubscribeHttpUrl = unsubscribeHttpUrl)
return getRepository(Subscription).save(subscription)
@ -59,6 +70,7 @@ export const saveSubscription = async (
status: SubscriptionStatus.Active,
unsubscribeHttpUrl,
unsubscribeMailTo,
icon,
})
}

31
packages/api/src/textToSpeech.d.ts vendored
View file

@ -1,31 +0,0 @@
declare module '@omnivore/text-to-speech-handler' {
export function htmlToSpeechFile(htmlInput: HtmlInput): SpeechFile
export interface HtmlInput {
title?: string
content: string
options: SSMLOptions
}
export interface SSMLOptions {
primaryVoice?: string
secondaryVoice?: string
rate?: string
language?: string
}
interface Utterance {
idx: string
wordOffset: number
wordCount: number
voice?: string
text: string
}
export interface SpeechFile {
wordCount: number
language: string
defaultVoice: string
utterances: Utterance[]
}
}

159
packages/api/src/utils/parser.ts
View file

@ -450,150 +450,6 @@ export const parseUrlMetadata = async (
}
}
// Attempt to determine if an HTML blob is a newsletter
// based on it's contents.
// TODO: when we consolidate the handlers we could include this
// as a utility method on each one.
export const isProbablyNewsletter = async (html: string): Promise<boolean> => {
const dom = parseHTML(html).document
const domCopy = parseHTML(dom.documentElement.outerHTML).document
const article = await new Readability(domCopy, {
debug: false,
keepTables: true,
}).parse()
if (!article || !article.content) {
return false
}
// substack newsletter emails have tables with a *post-meta class
if (dom.querySelector('table[class$="post-meta"]')) {
return true
}
// If the article has a header link, and substack icons its probably a newsletter
const href = findNewsletterHeaderHref(dom)
const heartIcon = dom.querySelector(
'table tbody td span a img[src*="HeartIcon"]'
)
const recommendIcon = dom.querySelector(
'table tbody td span a img[src*="RecommendIconRounded"]'
)
if (href && (heartIcon || recommendIcon)) {
return true
}
// Check if this is a beehiiv.net newsletter
if (dom.querySelectorAll('img[src*="beehiiv.net"]').length > 0) {
const beehiivUrl = beehiivNewsletterHref(dom)
if (beehiivUrl) {
return true
}
}
// Check if this is a newsletter from revue
if (
dom.querySelectorAll('img[src*="getrevue.co"], img[src*="revue.email"]')
.length > 0
) {
const getrevueUrl = revueNewsletterHref(dom)
if (getrevueUrl) {
return true
}
}
// Check if this is a convertkit.com newsletter
return (
dom.querySelectorAll(
'img[src*="convertkit.com"], img[src*="convertkit-mail.com"]'
).length > 0
)
}
const beehiivNewsletterHref = (dom: Document): string | undefined => {
const readOnline = dom.querySelectorAll('table tr td div a[class*="link"]')
let res: string | undefined = undefined
readOnline.forEach((e) => {
if (e.textContent === 'Read Online') {
res = e.getAttribute('href') || undefined
}
})
return res
}
const convertkitNewsletterHref = (dom: Document): string | undefined => {
const readOnline = dom.querySelectorAll('table tr td a')
let res: string | undefined = undefined
readOnline.forEach((e) => {
if (e.textContent === 'View this email in your browser') {
res = e.getAttribute('href') || undefined
}
})
return res
}
const revueNewsletterHref = (dom: Document): string | undefined => {
const viewOnline = dom.querySelectorAll('table tr td a[target="_blank"]')
let res: string | undefined = undefined
viewOnline.forEach((e) => {
if (e.textContent === 'View online') {
res = e.getAttribute('href') || undefined
}
})
return res
}
const findNewsletterHeaderHref = (dom: Document): string | undefined => {
// Substack header links
const postLink = dom.querySelector('h1 a ')
if (postLink) {
return postLink.getAttribute('href') || undefined
}
// Check if this is a beehiiv.net newsletter
const beehiiv = beehiivNewsletterHref(dom)
if (beehiiv) {
return beehiiv
}
// Check if this is a revue newsletter
const revue = revueNewsletterHref(dom)
if (revue) {
return revue
}
// Check if this is a convertkit.com newsletter
const convertkitUrl = convertkitNewsletterHref(dom)
if (convertkitUrl) {
return convertkitUrl
}
return undefined
}
// Given an HTML blob tries to find a URL to use for
// a canonical URL.
export const findNewsletterUrl = async (
html: string
): Promise<string | undefined> => {
const dom = parseHTML(html).document
// Check if this is a substack newsletter
const href = findNewsletterHeaderHref(dom)
if (href) {
// Try to make a HEAD request so we get the redirected URL, since these
// will usually be behind tracking url redirects
return axios({
method: 'HEAD',
url: href,
})
.then((res) => res.request.res.responseUrl as string | undefined)
.catch((e) => href)
}
return undefined
}
export const isProbablyArticle = async (
email: string,
subject: string
@ -621,3 +477,18 @@ export const parseEmailAddress = (from: string): addressparser.EmailAddress => {
}
return { name: '', address: from }
}
export const fetchFavicon = async (
url: string
): Promise<string | undefined> => {
try {
// get the correct url if it's a redirect
const response = await axios.head(url, { timeout: 5000 })
const realUrl = response.request.res.responseUrl
const domain = new URL(realUrl).hostname
return `https://api.faviconkit.com/${domain}/32`
} catch (e) {
console.log('Error fetching favicon', e)
return undefined
}
}

14
packages/api/test/resolvers/article.test.ts
View file

@ -10,7 +10,11 @@ import { expect } from 'chai'
import 'mocha'
import { User } from '../../src/entity/user'
import chaiString from 'chai-string'
import { UpdateReason, UploadFileStatus } from '../../src/generated/graphql'
import {
SyncUpdatedItemEdge,
UpdateReason,
UploadFileStatus,
} from '../../src/generated/graphql'
import {
ArticleSavingRequestStatus,
Highlight,
@ -1009,7 +1013,7 @@ describe('Article API', () => {
}
// set the since to be the timestamp before deletion
since = pages[4].createdAt.toISOString()
since = pages[4].updatedAt!.toISOString()
// Delete some pages
for (let i = 0; i < 3; i++) {
@ -1033,7 +1037,11 @@ describe('Article API', () => {
authToken
).expect(200)
expect(res.body.data.updatesSince.edges.length).to.eql(3)
expect(
res.body.data.updatesSince.edges.filter(
(e: SyncUpdatedItemEdge) => e.updateReason === UpdateReason.Deleted
).length
).to.eql(3)
expect(res.body.data.updatesSince.edges[0].itemID).to.eq(
deletedPages[0].id
)

32
packages/api/test/routers/emails.test.ts
View file

@ -52,35 +52,8 @@ describe('Emails Router', () => {
sinon.restore()
})
context('when email is a newsletter', () => {
before(() => {
sinon.replace(parser, 'isProbablyNewsletter', sinon.fake.resolves(true))
})
it('saves the email as a newsletter', async () => {
const data = {
message: {
data: Buffer.from(
JSON.stringify({ from, to, subject, html })
).toString('base64'),
publishTime: new Date().toISOString(),
},
}
const res = await request
.post(`/svc/pubsub/emails/forward?token=${token}`)
.send(data)
.expect(200)
expect(res.text).to.eql('Newsletter')
})
})
context('when email is an article', () => {
before(() => {
sinon.replace(
parser,
'isProbablyNewsletter',
sinon.fake.resolves(false)
)
sinon.replace(parser, 'isProbablyArticle', sinon.fake.resolves(true))
})
@ -103,11 +76,6 @@ describe('Emails Router', () => {
context('when email is a regular email', () => {
before(() => {
sinon.replace(
parser,
'isProbablyNewsletter',
sinon.fake.resolves(false)
)
sinon.replace(parser, 'isProbablyArticle', sinon.fake.resolves(false))
})

75
packages/api/test/utils/parser.test.ts
View file

@ -4,11 +4,8 @@ import { expect } from 'chai'
import 'chai/register-should'
import fs from 'fs'
import {
findNewsletterUrl,
generateUniqueUrl,
getTitleFromEmailSubject,
isProbablyArticle,
isProbablyNewsletter,
parseEmailAddress,
parsePageMetadata,
parsePreparedContent,
@ -24,69 +21,6 @@ const load = (path: string): string => {
return fs.readFileSync(path, 'utf8')
}
describe('isProbablyNewsletter', () => {
it('returns true for substack newsletter', async () => {
const html = load('./test/utils/data/substack-forwarded-newsletter.html')
await expect(isProbablyNewsletter(html)).to.eventually.be.true
})
it('returns true for private forwarded substack newsletter', async () => {
const html = load(
'./test/utils/data/substack-private-forwarded-newsletter.html'
)
await expect(isProbablyNewsletter(html)).to.eventually.be.true
})
it('returns false for substack welcome email', async () => {
const html = load('./test/utils/data/substack-forwarded-welcome-email.html')
await expect(isProbablyNewsletter(html)).to.eventually.be.false
})
it('returns true for beehiiv.com newsletter', async () => {
const html = load('./test/utils/data/beehiiv-newsletter.html')
await expect(isProbablyNewsletter(html)).to.eventually.be.true
})
})
describe('findNewsletterUrl', async () => {
it('gets the URL from the header if it is a substack newsletter', async () => {
nock('https://email.mg2.substack.com')
.head(
'/c/eJxNkk2TojAQhn-N3KTyQfg4cGDGchdnYcsZx9K5UCE0EMVAkTiKv36iHnarupNUd7rfVJ4W3EDTj1M89No496Uw0wCxgovuwBgYnbOGsZBVjDHzKPWYU8VehUMWOlIX9Qhw4rKLzXgGZziXnRTcyF7dK0iIGMVOG_OS1aTmKPRDilgVhTQUPCQIcE0x-MFTmJ8rCUpA3KtuenR2urg1ZtAzmszI0tq_Z7m66y-ilQo0uAqMTQ7WRX8auJKg56blZg7WB-iHDuYEBzO6NP0R1IwuYFphQbbTjnTH9NBfs80nym4Zyj8uUvyKbtUyGr5eUz9fNDQ7JCxfJDo9dW1lY9lmj_JNivPbGmf2Pt_lN9tDit9b-WeTetni85Z9pDpVOd7L1E_Vy7egayNO23ZP34eSeLJeux1b0rer_xaZ7ykS78nuSjMY-nL98rparNZNcv07JCjN06_EkTFBxBqOUMACErnELUNMSxTUjLDQZwzcqa4bRjCfeejUEFefS224OLr2S5wxPtij7lVrs80d2CNseRV2P52VNFMBipcdVE-U5jkRD7hFAwpGOylVwU2Mfc9qBh7DoR89yVnWXhgQFHnIsbpVb6tU_B-hH_2yzWY'
)
.reply(302, undefined, {
Location:
'https://newsletter.slowchinese.net/p/companies-that-eat-people-217',
})
.get('/p/companies-that-eat-people-217')
.reply(200, '')
const html = load('./test/utils/data/substack-forwarded-newsletter.html')
const url = await findNewsletterUrl(html)
// Not sure if the redirects from substack expire, this test could eventually fail
expect(url).to.startWith(
'https://newsletter.slowchinese.net/p/companies-that-eat-people-217'
)
})
it('gets the URL from the header if it is a beehiiv newsletter', async () => {
nock('https://u23463625.ct.sendgrid.net')
.head(
'/ss/c/AX1lEgEQaxtvFxLaVo0GBo_geajNrlI1TGeIcmMViR3pL3fEDZnbbkoeKcaY62QZk0KPFudUiUXc_uMLerV4nA/3k5/3TFZmreTR0qKSCgowABnVg/h30/zzLik7UXd1H_n4oyd5W8Xu639AYQQB2UXz-CsssSnno'
)
.reply(302, undefined, {
Location: 'https://www.milkroad.com/p/talked-guy-spent-30m-beeple',
})
.get('/p/talked-guy-spent-30m-beeple')
.reply(200, '')
const html = load('./test/utils/data/beehiiv-newsletter.html')
const url = await findNewsletterUrl(html)
expect(url).to.startWith(
'https://www.milkroad.com/p/talked-guy-spent-30m-beeple'
)
})
it('returns undefined if it is not a newsletter', async () => {
const html = load('./test/utils/data/substack-forwarded-welcome-email.html')
const url = await findNewsletterUrl(html)
expect(url).to.be.undefined
})
})
describe('parseMetadata', async () => {
it('gets author, title, image, description', async () => {
const html = load('./test/utils/data/substack-post.html')
@ -164,15 +98,6 @@ describe('isProbablyArticle', () => {
})
})
describe('generateUniqueUrl', () => {
it('generates a unique URL', () => {
const url1 = generateUniqueUrl()
const url2 = generateUniqueUrl()
expect(url1).to.not.eql(url2)
})
})
describe('getTitleFromEmailSubject', () => {
it('returns the title from the email subject', () => {
const title = 'test subject'

36
packages/content-fetch/apple-news-handler.js
View file

@ -1,36 +0,0 @@
/* eslint-disable no-undef */
/* eslint-disable no-empty */
/* eslint-disable @typescript-eslint/explicit-function-return-type */
/* eslint-disable @typescript-eslint/no-var-requires */
/* eslint-disable @typescript-eslint/no-require-imports */
require('dotenv').config();
const Url = require('url');
const axios = require('axios');
const { promisify } = require('util');
const { DateTime } = require('luxon');
const os = require('os');
const { Cipher } = require('crypto');
const { parseHTML } = require('linkedom');
exports.appleNewsHandler = {
shouldPrehandle: (url, env) => {
const u = new URL(url);
if (u.hostname === 'apple.news') {
return true;
}
return false
},
prehandle: async (url, env) => {
const MOBILE_USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.61 Safari/537.36'
const response = await axios.get(url, { headers: { 'User-Agent': MOBILE_USER_AGENT } } );
const data = response.data;
const dom = parseHTML(data).document;
// make sure its a valid URL by wrapping in new URL
const u = new URL(dom.querySelector('span.click-here').parentNode.href);
return { url: u.href };
}
}

39
packages/content-fetch/bloomberg-handler.js
View file

@ -1,39 +0,0 @@
/* eslint-disable no-undef */
/* eslint-disable no-empty */
/* eslint-disable @typescript-eslint/explicit-function-return-type */
/* eslint-disable @typescript-eslint/no-var-requires */
/* eslint-disable @typescript-eslint/no-require-imports */
require('dotenv').config();
const axios = require('axios');
const os = require('os');
const { parseHTML } = require('linkedom');
exports.bloombergHandler = {
shouldPrehandle: (url, env) => {
const BLOOMBERG_URL_MATCH =
/https?:\/\/(www\.)?bloomberg.[a-zA-Z0-9()]{1,6}\b([-a-zA-Z0-9()@:%_\+.~#?&//=]*)/
return BLOOMBERG_URL_MATCH.test(url.toString())
},
prehandle: async (url, env) => {
console.log('prehandling bloomberg url', url)
try {
const response = await axios.get('https://app.scrapingbee.com/api/v1', {
params: {
'api_key': process.env.SCRAPINGBEE_API_KEY,
'url': url,
'return_page_source': true,
'block_ads': true,
'block_resources': false,
}
})
const dom = parseHTML(response.data).document;
return { title: dom.title, content: dom.querySelector('body').innerHTML, url: url }
} catch (error) {
console.error('error prehandling bloomberg url', error)
throw error
}
}
}

35
packages/content-fetch/derstandard-handler.js
View file

@ -1,35 +0,0 @@
/* eslint-disable no-undef */
/* eslint-disable no-empty */
/* eslint-disable @typescript-eslint/explicit-function-return-type */
/* eslint-disable @typescript-eslint/no-var-requires */
/* eslint-disable @typescript-eslint/no-require-imports */
require('dotenv').config();
const axios = require('axios');
const { parseHTML } = require('linkedom');
exports.derstandardHandler = {
shouldPrehandle: (url, env) => {
const u = new URL(url);
return u.hostname === 'www.derstandard.at';
},
prehandle: async (url, env) => {
const response = await axios.get(url, {
// set cookie to give consent to get the article
headers: {
'cookie': `DSGVO_ZUSAGE_V1=true; consentUUID=2bacb9c1-1e80-4be0-9f7b-ee987cf4e7b0_6`
},
});
const content = response.data;
var title = undefined;
const dom = parseHTML(content).document;
const titleElement = dom.querySelector('.article-title')
if (!titleElement) {
title = titleElement.textContent
titleElement.remove()
}
return { content: dom.body.outerHTML, title: title };
}
}

93
packages/content-fetch/fetch-content.js
View file

@ -9,16 +9,10 @@ const puppeteer = require('puppeteer-core');
const axios = require('axios');
const jwt = require('jsonwebtoken');
const { promisify } = require('util');
const { parseHTML } = require('linkedom');
const { preHandleContent } = require('@omnivore/content-handler');
const signToken = promisify(jwt.sign);
const { appleNewsHandler } = require('./apple-news-handler');
const { twitterHandler } = require('./twitter-handler');
const { youtubeHandler } = require('./youtube-handler');
const { tDotCoHandler } = require('./t-dot-co-handler');
const { pdfHandler } = require('./pdf-handler');
const { mediumHandler } = require('./medium-handler');
const { derstandardHandler } = require('./derstandard-handler');
const { imageHandler } = require('./image-handler');
const { scrapingBeeHandler } = require('./scrapingBee-handler')
const MOBILE_USER_AGENT = 'Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/93.0.4577.62 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)'
const DESKTOP_USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 11_6_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4372.0 Safari/537.36'
@ -29,8 +23,6 @@ const NON_SCRIPT_HOSTS= ['medium.com', 'fastcompany.com'];
const ALLOWED_CONTENT_TYPES = ['text/html', 'application/octet-stream', 'text/plain', 'application/pdf'];
const { parseHTML } = require('linkedom');
// Add stealth plugin to hide puppeteer usage
// const StealthPlugin = require('puppeteer-extra-plugin-stealth');
// puppeteer.use(StealthPlugin());
@ -207,19 +199,6 @@ const saveUploadedPdf = async (userId, url, uploadFileId, articleSavingRequestId
);
};
const handlers = {
'pdf': pdfHandler,
'apple-news': appleNewsHandler,
'twitter': twitterHandler,
'youtube': youtubeHandler,
't-dot-co': tDotCoHandler,
'medium': mediumHandler,
'derstandard': derstandardHandler,
'image': imageHandler,
'scrapingBee': scrapingBeeHandler,
};
async function fetchContent(req, res) {
functionStartTime = Date.now();
@ -246,61 +225,19 @@ async function fetchContent(req, res) {
return res.sendStatus(400);
}
// if (!userId || !articleSavingRequestId) {
// Object.assign(logRecord, { invalidParams: true, body: req.body, query: req.query });
// console.log(`Invalid parameters`, logRecord);
// return res.sendStatus(400);
// }
// Before we run the regular handlers we check to see if we need tp
// pre-resolve the URL. TODO: This should probably happen recursively,
// so URLs can be pre-resolved, handled, pre-resolved, handled, etc.
for (const [key, handler] of Object.entries(handlers)) {
if (handler.shouldResolve && handler.shouldResolve(url)) {
try {
url = await handler.resolve(url);
validateUrlString(url);
} catch (err) {
console.log('error resolving url with handler', key, err);
}
break;
}
}
// Before we fetch the page we check the handlers, to see if they want
// to perform a prefetch action that can modify our requests.
// enumerate the handlers and see if any of them want to handle the request
const handler = Object.keys(handlers).find(key => {
try {
return handlers[key].shouldPrehandle(url)
} catch (e) {
console.log('error with handler: ', key, e);
}
return false;
});
var title = undefined;
var content = undefined;
var contentType = undefined;
if (handler) {
try {
// The only handler we have now can modify the URL, but in the
// future maybe we let it modify content. In that case
// we might exit the request early.
console.log('pre-handling url with handler: ', handler);
const result = await handlers[handler].prehandle(url);
if (result && result.url) {
url = result.url
validateUrlString(url);
}
if (result && result.title) { title = result.title }
if (result && result.content) { content = result.content }
if (result && result.contentType) { contentType = result.contentType }
} catch (e) {
console.log('error with handler: ', handler, e);
// pre handle url with custom handlers
let title, content, contentType;
try {
const result = await preHandleContent(url);
if (result && result.url) {
url = result.url
validateUrlString(url);
}
if (result && result.title) { title = result.title }
if (result && result.content) { content = result.content }
if (result && result.contentType) { contentType = result.contentType }
} catch (e) {
console.log('error with handler: ', e);
}
let context, page, finalUrl;

34
packages/content-fetch/image-handler.js
View file

@ -1,34 +0,0 @@
/* eslint-disable no-undef */
/* eslint-disable no-empty */
/* eslint-disable @typescript-eslint/explicit-function-return-type */
/* eslint-disable @typescript-eslint/no-var-requires */
/* eslint-disable @typescript-eslint/no-require-imports */
require('dotenv').config();
exports.imageHandler = {
shouldPrehandle: (url, env) => {
const IMAGE_URL_PATTERN =
/(https?:\/\/.*\.(?:jpg|jpeg|png|webp))/i
return IMAGE_URL_PATTERN.test(url.toString())
},
prehandle: async (url, env) => {
const title = url.toString().split('/').pop();
const content = `
<html>
<head>
<title>${title}</title>
<meta property="og:image" content="${url}" />
<meta property="og:title" content="${title}" />
</head>
<body>
<div>
<img src="${url}" alt="${title}">
</div>
</body>
</html>`
return { title, content };
}
}

29
packages/content-fetch/medium-handler.js
View file

@ -1,29 +0,0 @@
/* eslint-disable no-undef */
/* eslint-disable no-empty */
/* eslint-disable @typescript-eslint/explicit-function-return-type */
/* eslint-disable @typescript-eslint/no-var-requires */
/* eslint-disable @typescript-eslint/no-require-imports */
require('dotenv').config();
const axios = require('axios');
const os = require('os');
exports.mediumHandler = {
shouldPrehandle: (url, env) => {
const u = new URL(url);
return u.hostname.endsWith('medium.com')
},
prehandle: async (url, env) => {
console.log('prehandling medium url', url)
try {
const res = new URL(url);
res.searchParams.delete('source');
return { url: res.toString() }
} catch (error) {
console.error('error prehandling medium url', error)
throw error
}
}
}

3
packages/content-fetch/package.json
View file

@ -11,7 +11,8 @@
"linkedom": "^0.14.9",
"luxon": "^2.3.1",
"puppeteer-core": "^16.1.0",
"underscore": "^1.13.4"
"underscore": "^1.13.4",
"@omnivore/content-handler": "1.0.0"
},
"scripts": {
"start": "node app.js",

21
packages/content-fetch/pdf-handler.js
View file

@ -1,21 +0,0 @@
/* eslint-disable no-undef */
/* eslint-disable no-empty */
/* eslint-disable @typescript-eslint/explicit-function-return-type */
/* eslint-disable @typescript-eslint/no-var-requires */
/* eslint-disable @typescript-eslint/no-require-imports */
require('dotenv').config();
const Url = require('url');
exports.pdfHandler = {
shouldPrehandle: (url, env) => {
const u = Url.parse(url)
const path = u.path.replace(u.search, '')
return path.endsWith('.pdf')
},
prehandle: async (url, env) => {
return { contentType: 'application/pdf' };
}
}

44
packages/content-fetch/scrapingBee-handler.js
View file

@ -1,44 +0,0 @@
/* eslint-disable no-undef */
/* eslint-disable no-empty */
/* eslint-disable @typescript-eslint/explicit-function-return-type */
/* eslint-disable @typescript-eslint/no-var-requires */
/* eslint-disable @typescript-eslint/no-require-imports */
require('dotenv').config();
const axios = require('axios');
const { parseHTML } = require('linkedom');
const os = require('os');
exports.scrapingBeeHandler = {
shouldPrehandle: (url, env) => {
const u = new URL(url);
const hostnames = [
'nytimes.com',
'news.google.com',
]
return hostnames.some((h) => u.hostname.endsWith(h))
},
prehandle: async (url, env) => {
console.log('prehandling url with scrapingbee', url)
try {
const response = await axios.get('https://app.scrapingbee.com/api/v1', {
params: {
'api_key': process.env.SCRAPINGBEE_API_KEY,
'url': url,
'return_page_source': true,
'block_ads': true,
'block_resources': false,
}
})
const dom = parseHTML(response.data).document;
return { title: dom.title, content: response.data, url: url }
} catch (error) {
console.error('error prehandling url w/scrapingbee', error)
throw error
}
}
}

31
packages/content-fetch/t-dot-co-handler.js
View file

@ -1,31 +0,0 @@
/* eslint-disable no-undef */
/* eslint-disable no-empty */
/* eslint-disable @typescript-eslint/explicit-function-return-type */
/* eslint-disable @typescript-eslint/no-var-requires */
/* eslint-disable @typescript-eslint/no-require-imports */
require('dotenv').config();
const axios = require('axios');
const Url = require('url');
exports.tDotCoHandler = {
shouldResolve: function (url, env) {
const T_DOT_CO_URL_MATCH = /^https:\/\/(?:www\.)?t\.co\/.*$/;
return T_DOT_CO_URL_MATCH.test(url);
},
resolve: async function(url, env) {
return await axios.get(url, { maxRedirects: 0, validateStatus: null })
.then(res => {
return Url.parse(res.headers.location).href;
}).catch((err) => {
console.log('err with t.co url', err);
return undefined;
});
},
shouldPrehandle: (url, env) => {
return false
},
}

9
packages/content-fetch/test/apple-news-handler.test.js
View file

@ -1,9 +0,0 @@
const { expect } = require('chai')
const { appleNewsHandler } = require('../apple-news-handler')
describe('open a simple web page', () => {
it('should return a response', async () => {
const response = await appleNewsHandler.prehandle('https://apple.news/AxjzaZaPvSn23b67LhXI5EQ')
console.log('response', response)
})
})

3
packages/content-fetch/test/babel-register.js Normal file
View file

@ -0,0 +1,3 @@
const register = require('@babel/register').default
register({ extensions: ['.ts', '.tsx', '.js', '.jsx'] })

13
packages/content-fetch/test/stub.test.ts Normal file
View file

@ -0,0 +1,13 @@
import 'mocha'
import * as chai from 'chai'
import { expect } from 'chai'
import 'chai/register-should'
import chaiString from 'chai-string'
chai.use(chaiString)
describe('Stub test', () => {
it('should pass', () => {
expect(true).to.be.true
})
})

12
packages/content-fetch/test/youtube-handler.test.js
View file

@ -1,12 +0,0 @@
const { expect } = require('chai')
const { getYoutubeVideoId } = require('../youtube-handler')
describe('getYoutubeVideoId', () => {
it('should parse video id out of a URL', async () => {
expect('BnSUk0je6oo').to.eq(getYoutubeVideoId('https://www.youtube.com/watch?v=BnSUk0je6oo&t=269s'));
expect('vFD2gu007dc').to.eq(getYoutubeVideoId('https://www.youtube.com/watch?v=vFD2gu007dc&list=RDvFD2gu007dc&start_radio=1'));
expect('vFD2gu007dc').to.eq(getYoutubeVideoId('https://youtu.be/vFD2gu007dc'));
expect('BMFVCnbRaV4').to.eq(getYoutubeVideoId('https://youtube.com/watch?v=BMFVCnbRaV4&feature=share'));
expect('cg9b4RC87LI').to.eq(getYoutubeVideoId('https://youtu.be/cg9b4RC87LI?t=116'));
})
})

172
packages/content-fetch/twitter-handler.js
View file

@ -1,172 +0,0 @@
/* eslint-disable no-undef */
/* eslint-disable no-empty */
/* eslint-disable @typescript-eslint/explicit-function-return-type */
/* eslint-disable @typescript-eslint/no-var-requires */
/* eslint-disable @typescript-eslint/no-require-imports */
require('dotenv').config();
const axios = require('axios');
const { DateTime } = require('luxon');
const _ = require('underscore');
const TWITTER_BEARER_TOKEN = process.env.TWITTER_BEARER_TOKEN;
const TWITTER_URL_MATCH = /twitter\.com\/(?:#!\/)?(\w+)\/status(?:es)?\/(\d+)(?:\/.*)?/
const embeddedTweet = async (url) => {
const BASE_ENDPOINT = 'https://publish.twitter.com/oembed'
const apiUrl = new URL(BASE_ENDPOINT)
apiUrl.searchParams.append('url', url);
apiUrl.searchParams.append('omit_script', true);
apiUrl.searchParams.append('dnt', true);
return await axios.get(apiUrl.toString(), {
headers: {
Authorization: `Bearer ${TWITTER_BEARER_TOKEN}`,
redirect: "follow",
},
});
};
const getTweetFields = () => {
const TWEET_FIELDS =
"&tweet.fields=attachments,author_id,conversation_id,created_at," +
"entities,geo,in_reply_to_user_id,lang,possibly_sensitive,public_metrics,referenced_tweets," +
"source,withheld";
const EXPANSIONS = "&expansions=author_id,attachments.media_keys";
const USER_FIELDS =
"&user.fields=created_at,description,entities,location,pinned_tweet_id,profile_image_url,protected,public_metrics,url,verified,withheld";
const MEDIA_FIELDS =
"&media.fields=duration_ms,height,preview_image_url,url,media_key,public_metrics,width";
return `${TWEET_FIELDS}${EXPANSIONS}${USER_FIELDS}${MEDIA_FIELDS}`;
}
const getTweetById = async (id) => {
const BASE_ENDPOINT = "https://api.twitter.com/2/tweets/";
const apiUrl = new URL(BASE_ENDPOINT + id + '?' + getTweetFields())
return await axios.get(apiUrl.toString(), {
headers: {
Authorization: `Bearer ${TWITTER_BEARER_TOKEN}`,
redirect: "follow",
},
});
};
const getUserByUsername = async (username) => {
const BASE_ENDPOINT = "https://api.twitter.com/2/users/by/username/";
const apiUrl = new URL(BASE_ENDPOINT + username)
apiUrl.searchParams.append('user.fields', 'profile_image_url');
return await axios.get(apiUrl.toString(), {
headers: {
Authorization: `Bearer ${TWITTER_BEARER_TOKEN}`,
redirect: "follow",
},
});
};
const titleForTweet = (tweet) => {
return `${tweet.data.author_name} on Twitter`
};
const titleForAuthor = (author) => {
return `${author.name} on Twitter`
};
const usernameFromStatusUrl = (url) => {
const match = url.toString().match(TWITTER_URL_MATCH)
return match[1]
};
const tweetIdFromStatusUrl = (url) => {
const match = url.toString().match(TWITTER_URL_MATCH)
return match[2]
};
const formatTimestamp = (timestamp) => {
return DateTime.fromJSDate(new Date(timestamp)).toLocaleString(DateTime.DATETIME_FULL);
};
exports.twitterHandler = {
shouldPrehandle: (url, env) => {
return TWITTER_BEARER_TOKEN && TWITTER_URL_MATCH.test(url.toString())
},
// version of the handler that uses the oembed API
// This isn't great as it doesn't work well with our
// readability API. But could potentially give a more consistent
// look to the tweets
// prehandle: async (url, env) => {
// const oeTweet = await embeddedTweet(url)
// const dom = new JSDOM(oeTweet.data.html);
// const bq = dom.window.document.querySelector('blockquote')
// console.log('blockquote:', bq);
// const title = titleForTweet(oeTweet)
// return { title, content: '<div>' + bq.innerHTML + '</div>', url: oeTweet.data.url };
// }
prehandle: async (url, env) => {
console.log('prehandling twitter url', url)
const tweetId = tweetIdFromStatusUrl(url)
const tweetData = (await getTweetById(tweetId)).data;
const authorId = tweetData.data.author_id;
const author = tweetData.includes.users.filter(u => u.id = authorId)[0];
// escape html entities in title
const title = _.escape(titleForAuthor(author))
const authorImage = author.profile_image_url.replace('_normal', '_400x400')
let text = tweetData.data.text;
if (tweetData.data.entities && tweetData.data.entities.urls) {
for (let urlObj of tweetData.data.entities.urls) {
text = text.replace(
urlObj.url,
`<a href="${urlObj.expanded_url}">${urlObj.display_url}</a>`
);
}
}
const front = `
<div>
<p>${text}</p>
`
var includesHtml = '';
if (tweetData.includes.media) {
includesHtml = tweetData.includes.media.map(m => {
const linkUrl = m.type == 'photo' ? m.url : url;
const previewUrl = m.type == 'photo' ? m.url : m.preview_image_url;
const mediaOpen = `<a class="media-link" href=${linkUrl}>
<picture>
<img class="tweet-img" src=${previewUrl} />
</picture>
</a>`
return mediaOpen
}).join('\n');
}
const back = `
— <a href="https://twitter.com/${author.username}">${author.username}</a> ${author.name} <a href="${url}">${formatTimestamp(tweetData.data.created_at)}</a>
</div>
`
const content = `
<head>
<meta property="og:image" content="${authorImage}" />
<meta property="og:image:secure_url" content="${authorImage}" />
<meta property="og:title" content="${title}" />
<meta property="og:description" content="${_.escape(tweetData.data.text)}" />
</head>
<body>
${front}
${includesHtml}
${back}
</body>`
return { content, url, title };
}
}

1
packages/content-handler/.eslintignore Normal file
View file

@ -0,0 +1 @@
node_modules/

6
packages/content-handler/.eslintrc Normal file
View file

@ -0,0 +1,6 @@
{
"extends": "../../.eslintrc",
"parserOptions": {
"project": "tsconfig.json"
}
}

2
packages/content-handler/.gitignore vendored Normal file
View file

@ -0,0 +1,2 @@
node_modules
/lib

7
packages/content-handler/.npmignore Normal file
View file

@ -0,0 +1,7 @@
/test/
src
tsconfig.json
.eslintrc
.eslintignore
.gitignore
mocha-config.json

5
packages/content-handler/mocha-config.json Normal file
View file

@ -0,0 +1,5 @@
{
"extension": ["ts"],
"spec": "test/**/*.test.ts",
"require": "test/babel-register.js"
}

34
packages/content-handler/package.json Normal file
View file

@ -0,0 +1,34 @@
{
"name": "@omnivore/content-handler",
"version": "1.0.0",
"description": "A standalone version of content handler to parse and format each type of content",
"main": "build/src/index.js",
"types": "build/src/index.d.ts",
"files": [
"build/src"
],
"license": "Apache-2.0",
"scripts": {
"test": "yarn mocha -r ts-node/register --config mocha-config.json",
"lint": "eslint src --ext ts,js,tsx,jsx",
"compile": "tsc",
"build": "tsc"
},
"devDependencies": {
"chai": "^4.3.6",
"chai-as-promised": "^7.1.1",
"chai-string": "^1.5.0",
"eslint-plugin-prettier": "^4.0.0",
"mocha": "^10.0.0",
"nock": "^13.2.9"
},
"dependencies": {
"addressparser": "^1.0.1",
"axios": "^0.27.2",
"linkedom": "^0.14.16",
"luxon": "^3.0.4",
"rfc2047": "^4.0.1",
"underscore": "^1.13.6",
"uuid": "^9.0.0"
}
}

175
packages/content-handler/src/content-handler.ts Normal file
View file

@ -0,0 +1,175 @@
import addressparser from 'addressparser'
import rfc2047 from 'rfc2047'
import { v4 as uuid } from 'uuid'
import { parseHTML } from 'linkedom'
import axios from 'axios'
interface Unsubscribe {
mailTo?: string
httpUrl?: string
}
export interface NewsletterInput {
postHeader: string
from: string
unSubHeader: string
email: string
html: string
title: string
}
export interface NewsletterResult {
email: string
content: string
url: string
title: string
author: string
unsubMailTo?: string
unsubHttpUrl?: string
}
export interface PreHandleResult {
url?: string
title?: string
content?: string
contentType?: string
dom?: Document
}
export const FAKE_URL_PREFIX = 'https://omnivore.app/no_url?q='
export const generateUniqueUrl = () => FAKE_URL_PREFIX + uuid()
export abstract class ContentHandler {
protected senderRegex: RegExp
protected urlRegex: RegExp
name: string
protected constructor() {
this.senderRegex = new RegExp(/NEWSLETTER_SENDER_REGEX/)
this.urlRegex = new RegExp(/NEWSLETTER_URL_REGEX/)
this.name = 'Handler name'
}
shouldResolve(url: string): boolean {
return false
}
async resolve(url: string): Promise<string | undefined> {
return Promise.resolve(url)
}
shouldPreHandle(url: string, dom?: Document): boolean {
return false
}
async preHandle(url: string, dom?: Document): Promise<PreHandleResult> {
return Promise.resolve({ url, dom })
}
async isNewsletter(input: {
postHeader: string
from: string
unSubHeader: string
html?: string
}): Promise<boolean> {
const re = new RegExp(this.senderRegex)
return Promise.resolve(
re.test(input.from) && (!!input.postHeader || !!input.unSubHeader)
)
}
findNewsletterHeaderHref(dom: Document): string | undefined {
return undefined
}
// Given an HTML blob tries to find a URL to use for
// a canonical URL.
async findNewsletterUrl(html: string): Promise<string | undefined> {
const dom = parseHTML(html).document
// Check if this is a substack newsletter
const href = this.findNewsletterHeaderHref(dom)
if (href) {
// Try to make a HEAD request, so we get the redirected URL, since these
// will usually be behind tracking url redirects
try {
const response = await axios.head(href, { timeout: 5000 })
return Promise.resolve(
// eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
response.request.res.responseUrl as string | undefined
)
} catch (e) {
console.log('error making HEAD request', e)
return Promise.resolve(href)
}
}
return Promise.resolve(undefined)
}
async parseNewsletterUrl(
_postHeader: string,
html: string
): Promise<string | undefined> {
// get newsletter url from html
const matches = html.match(this.urlRegex)
if (matches) {
return Promise.resolve(matches[1])
}
return Promise.resolve(undefined)
}
parseAuthor(from: string): string {
// get author name from email
// e.g. 'Jackson Harper from Omnivore App <jacksonh@substack.com>'
// or 'Mike Allen <mike@axios.com>'
const parsed = addressparser(from)
if (parsed.length > 0) {
return parsed[0].name
}
return from
}
parseUnsubscribe(unSubHeader: string): Unsubscribe {
// parse list-unsubscribe header
// e.g. List-Unsubscribe: <https://omnivore.com/unsub>, <mailto:unsub@omnivore.com>
const decoded = rfc2047.decode(unSubHeader)
return {
mailTo: decoded.match(/<(https?:\/\/[^>]*)>/)?.[1],
httpUrl: decoded.match(/<mailto:([^>]*)>/)?.[1],
}
}
async handleNewsletter({
email,
html,
postHeader,
title,
from,
unSubHeader,
}: NewsletterInput): Promise<NewsletterResult> {
console.log('handleNewsletter', email, postHeader, title, from)
if (!email || !html || !title || !from) {
console.log('invalid newsletter email')
throw new Error('invalid newsletter email')
}
// fallback to default url if newsletter url does not exist
// assign a random uuid to the default url to avoid duplicate url
const url =
(await this.parseNewsletterUrl(postHeader, html)) || generateUniqueUrl()
const author = this.parseAuthor(from)
const unsubscribe = this.parseUnsubscribe(unSubHeader)
return {
email,
content: html,
url,
title,
author,
unsubMailTo: unsubscribe.mailTo || '',
unsubHttpUrl: unsubscribe.httpUrl || '',
}
}
}

116
packages/content-handler/src/index.ts Normal file
View file

@ -0,0 +1,116 @@
import { AppleNewsHandler } from './websites/apple-news-handler'
import { BloombergHandler } from './websites/bloomberg-handler'
import { DerstandardHandler } from './websites/derstandard-handler'
import { ImageHandler } from './websites/image-handler'
import { MediumHandler } from './websites/medium-handler'
import { PdfHandler } from './websites/pdf-handler'
import { ScrapingBeeHandler } from './websites/scrapingBee-handler'
import { TDotCoHandler } from './websites/t-dot-co-handler'
import { TwitterHandler } from './websites/twitter-handler'
import { YoutubeHandler } from './websites/youtube-handler'
import { WikipediaHandler } from './websites/wikipedia-handler'
import {
ContentHandler,
NewsletterInput,
NewsletterResult,
PreHandleResult,
} from './content-handler'
import { SubstackHandler } from './newsletters/substack-handler'
import { AxiosHandler } from './newsletters/axios-handler'
import { GolangHandler } from './newsletters/golang-handler'
import { MorningBrewHandler } from './newsletters/morning-brew-handler'
import { BloombergNewsletterHandler } from './newsletters/bloomberg-newsletter-handler'
import { BeehiivHandler } from './newsletters/beehiiv-handler'
import { ConvertkitHandler } from './newsletters/convertkit-handler'
import { RevueHandler } from './newsletters/revue-handler'
const validateUrlString = (url: string) => {
const u = new URL(url)
// Make sure the URL is http or https
if (u.protocol !== 'http:' && u.protocol !== 'https:') {
throw new Error('Invalid URL protocol check failed')
}
// Make sure the domain is not localhost
if (u.hostname === 'localhost' || u.hostname === '0.0.0.0') {
throw new Error('Invalid URL is localhost')
}
// Make sure the domain is not a private IP
if (/^(10|172\.16|192\.168)\..*/.test(u.hostname)) {
throw new Error('Invalid URL is private ip')
}
}
const contentHandlers: ContentHandler[] = [
new AppleNewsHandler(),
new BloombergHandler(),
new DerstandardHandler(),
new ImageHandler(),
new MediumHandler(),
new PdfHandler(),
new ScrapingBeeHandler(),
new TDotCoHandler(),
new TwitterHandler(),
new YoutubeHandler(),
new WikipediaHandler(),
]
const newsletterHandlers: ContentHandler[] = [
new AxiosHandler(),
new BloombergNewsletterHandler(),
new GolangHandler(),
new SubstackHandler(),
new MorningBrewHandler(),
new SubstackHandler(),
new BeehiivHandler(),
new ConvertkitHandler(),
new RevueHandler(),
]
export const preHandleContent = async (
url: string,
dom?: Document
): Promise<PreHandleResult | undefined> => {
// Before we run the regular handlers we check to see if we need tp
// pre-resolve the URL. TODO: This should probably happen recursively,
// so URLs can be pre-resolved, handled, pre-resolved, handled, etc.
for (const handler of contentHandlers) {
if (handler.shouldResolve(url)) {
try {
const resolvedUrl = await handler.resolve(url)
if (resolvedUrl && validateUrlString(resolvedUrl)) {
url = resolvedUrl
}
} catch (err) {
console.log('error resolving url with handler', handler.name, err)
}
break
}
}
// Before we fetch the page we check the handlers, to see if they want
// to perform a prefetch action that can modify our requests.
// enumerate the handlers and see if any of them want to handle the request
for (const handler of contentHandlers) {
if (handler.shouldPreHandle(url, dom)) {
console.log('preHandleContent', handler.name, url)
return handler.preHandle(url, dom)
}
}
return undefined
}
export const handleNewsletter = async (
input: NewsletterInput
): Promise<NewsletterResult | undefined> => {
for (const handler of newsletterHandlers) {
if (await handler.isNewsletter(input)) {
return handler.handleNewsletter(input)
}
}
return undefined
}
module.exports = {
preHandleContent,
handleNewsletter,
}

46
packages/content-handler/src/newsletters/axios-handler.ts Normal file
View file

@ -0,0 +1,46 @@
import { ContentHandler, PreHandleResult } from '../content-handler'
export class AxiosHandler extends ContentHandler {
constructor() {
super()
this.senderRegex = /<.+@axios.com>/
this.urlRegex = /View in browser at <a.*>(.*)<\/a>/
this.name = 'axios'
}
shouldPreHandle(url: string, dom?: Document): boolean {
const host = this.name + '.com'
// check if url ends with axios.com
return new URL(url).hostname.endsWith(host)
}
async preHandle(url: string, dom: Document): Promise<PreHandleResult> {
const body = dom.querySelector('table')
let isFooter = false
// this removes ads and replaces table with a div
body?.querySelectorAll('table').forEach((el) => {
// remove the footer and the ads
if (!el.textContent || el.textContent.length < 20 || isFooter) {
el.remove()
} else {
// removes the first few rows of the table (the header)
// remove the last two rows of the table (they are ads)
el.querySelectorAll('tr').forEach((tr, i) => {
if (i <= 7 || i >= el.querySelectorAll('tr').length - 2) {
console.log('removing', tr)
tr.remove()
}
})
// replace the table with a div
const div = dom.createElement('div')
div.innerHTML = el.innerHTML
el.parentNode?.replaceChild(div, el)
// set the isFooter flag to true because the next table is the footer
isFooter = true
}
})
return Promise.resolve({ dom })
}
}

43
packages/content-handler/src/newsletters/beehiiv-handler.ts Normal file
View file

@ -0,0 +1,43 @@
import { ContentHandler } from '../content-handler'
import { parseHTML } from 'linkedom'
export class BeehiivHandler extends ContentHandler {
constructor() {
super()
this.name = 'beehiiv'
}
findNewsletterHeaderHref(dom: Document): string | undefined {
const readOnline = dom.querySelectorAll('table tr td div a[class*="link"]')
let res: string | undefined = undefined
readOnline.forEach((e) => {
if (e.textContent === 'Read Online') {
res = e.getAttribute('href') || undefined
}
})
return res
}
async isNewsletter(input: {
postHeader: string
from: string
unSubHeader: string
html: string
}): Promise<boolean> {
const dom = parseHTML(input.html).document
if (dom.querySelectorAll('img[src*="beehiiv.net"]').length > 0) {
const beehiivUrl = this.findNewsletterHeaderHref(dom)
if (beehiivUrl) {
return Promise.resolve(true)
}
}
return false
}
async parseNewsletterUrl(
postHeader: string,
html: string
): Promise<string | undefined> {
return this.findNewsletterUrl(html)
}
}

37
packages/content-handler/src/newsletters/bloomberg-newsletter-handler.ts Normal file
View file

@ -0,0 +1,37 @@
import { ContentHandler, PreHandleResult } from '../content-handler'
export class BloombergNewsletterHandler extends ContentHandler {
constructor() {
super()
this.senderRegex = /<.+@mail.bloomberg.*.com>/
this.urlRegex = /<a class="view-in-browser__url" href=["']([^"']*)["']/
this.name = 'bloomberg'
}
shouldPreHandle(url: string, dom: Document): boolean {
const host = this.name + '.com'
// check if url ends with bloomberg.com
return (
new URL(url).hostname.endsWith(host) ||
dom.querySelector('.logo-image')?.getAttribute('alt')?.toLowerCase() ===
this.name
)
}
async preHandle(url: string, dom: Document): Promise<PreHandleResult> {
const body = dom.querySelector('.wrapper')
// this removes header
body?.querySelector('.sailthru-variables')?.remove()
body?.querySelector('.preview-text')?.remove()
body?.querySelector('.logo-wrapper')?.remove()
body?.querySelector('.by-the-number-wrapper')?.remove()
// this removes footer
body?.querySelector('.quote-box-wrapper')?.remove()
body?.querySelector('.header-wrapper')?.remove()
body?.querySelector('.component-wrapper')?.remove()
body?.querySelector('.footer')?.remove()
return Promise.resolve({ dom })
}
}

41
packages/content-handler/src/newsletters/convertkit-handler.ts Normal file
View file

@ -0,0 +1,41 @@
import { ContentHandler } from '../content-handler'
import { parseHTML } from 'linkedom'
export class ConvertkitHandler extends ContentHandler {
constructor() {
super()
this.name = 'convertkit'
}
findNewsletterHeaderHref(dom: Document): string | undefined {
const readOnline = dom.querySelectorAll('table tr td a')
let res: string | undefined = undefined
readOnline.forEach((e) => {
if (e.textContent === 'View this email in your browser') {
res = e.getAttribute('href') || undefined
}
})
return res
}
async isNewsletter(input: {
postHeader: string
from: string
unSubHeader: string
html: string
}): Promise<boolean> {
const dom = parseHTML(input.html).document
return Promise.resolve(
dom.querySelectorAll(
'img[src*="convertkit.com"], img[src*="convertkit-mail.com"]'
).length > 0
)
}
async parseNewsletterUrl(
postHeader: string,
html: string
): Promise<string | undefined> {
return this.findNewsletterUrl(html)
}
}

27
packages/content-handler/src/newsletters/golang-handler.ts Normal file
View file

@ -0,0 +1,27 @@
import { ContentHandler, PreHandleResult } from '../content-handler'
export class GolangHandler extends ContentHandler {
constructor() {
super()
this.senderRegex = /<.+@golangweekly.com>/
this.urlRegex = /<a href=["']([^"']*)["'].*>Read on the Web<\/a>/
this.name = 'golangweekly'
}
shouldPreHandle(url: string, dom?: Document): boolean {
const host = this.name + '.com'
// check if url ends with golangweekly.com
return new URL(url).hostname.endsWith(host)
}
async preHandle(url: string, dom: Document): Promise<PreHandleResult> {
const body = dom.querySelector('body')
// this removes the "Subscribe" button
body?.querySelector('.el-splitbar')?.remove()
// this removes the title
body?.querySelector('.el-masthead')?.remove()
return Promise.resolve({ dom })
}
}

35
packages/content-handler/src/newsletters/morning-brew-handler.ts Normal file
View file

@ -0,0 +1,35 @@
import { ContentHandler, PreHandleResult } from '../content-handler'
export class MorningBrewHandler extends ContentHandler {
constructor() {
super()
this.senderRegex = /Morning Brew <crew@morningbrew.com>/
this.urlRegex = /<a.* href=["']([^"']*)["'].*>View Online<\/a>/
this.name = 'morningbrew'
}
shouldPreHandle(url: string, dom?: Document): boolean {
const host = this.name + '.com'
// check if url ends with morningbrew.com
return new URL(url).hostname.endsWith(host)
}
async preHandle(url: string, dom: Document): Promise<PreHandleResult> {
// retain the width of the cells in the table of market info
dom.querySelectorAll('.markets-arrow-cell').forEach((td) => {
const table = td.closest('table')
if (table) {
const bubbleTable = table.querySelector('.markets-bubble')
if (bubbleTable) {
// replace the nested table with the text
const e = bubbleTable.querySelector('.markets-table-text')
e && bubbleTable.parentNode?.replaceChild(e, bubbleTable)
}
// set custom class for the table
table.className = 'morning-brew-markets'
}
})
return Promise.resolve({ dom })
}
}

46
packages/content-handler/src/newsletters/revue-handler.ts Normal file
View file

@ -0,0 +1,46 @@
import { ContentHandler } from '../content-handler'
import { parseHTML } from 'linkedom'
export class RevueHandler extends ContentHandler {
constructor() {
super()
this.name = 'revue'
}
findNewsletterHeaderHref(dom: Document): string | undefined {
const viewOnline = dom.querySelectorAll('table tr td a[target="_blank"]')
let res: string | undefined = undefined
viewOnline.forEach((e) => {
if (e.textContent === 'View online') {
res = e.getAttribute('href') || undefined
}
})
return res
}
async isNewsletter(input: {
postHeader: string
from: string
unSubHeader: string
html: string
}): Promise<boolean> {
const dom = parseHTML(input.html).document
if (
dom.querySelectorAll('img[src*="getrevue.co"], img[src*="revue.email"]')
.length > 0
) {
const getrevueUrl = this.findNewsletterHeaderHref(dom)
if (getrevueUrl) {
return Promise.resolve(true)
}
}
return false
}
async parseNewsletterUrl(
postHeader: string,
html: string
): Promise<string | undefined> {
return this.findNewsletterUrl(html)
}
}

90
packages/content-handler/src/newsletters/substack-handler.ts Normal file
View file

@ -0,0 +1,90 @@
import addressparser from 'addressparser'
import { ContentHandler, PreHandleResult } from '../content-handler'
import { parseHTML } from 'linkedom'
export class SubstackHandler extends ContentHandler {
constructor() {
super()
this.name = 'substack'
}
shouldPreHandle(url: string, dom: Document): boolean {
const host = this.name + '.com'
// check if url ends with substack.com
// or has a profile image hosted at substack.com
return (
new URL(url).hostname.endsWith(host) ||
!!dom
.querySelector('.email-body img')
?.getAttribute('src')
?.includes(host)
)
}
async preHandle(url: string, dom: Document): Promise<PreHandleResult> {
const body = dom.querySelector('.email-body-container')
// this removes header and profile avatar
body?.querySelector('.header')?.remove()
body?.querySelector('.preamble')?.remove()
body?.querySelector('.meta-author-wrap')?.remove()
// this removes meta button
body?.querySelector('.post-meta')?.remove()
// this removes footer
body?.querySelector('.post-cta')?.remove()
body?.querySelector('.container-border')?.remove()
body?.querySelector('.footer')?.remove()
return Promise.resolve(dom)
}
findNewsletterHeaderHref(dom: Document): string | undefined {
// Substack header links
const postLink = dom.querySelector('h1 a ')
if (postLink) {
return postLink.getAttribute('href') || undefined
}
return undefined
}
async isNewsletter({
postHeader,
html,
}: {
postHeader: string
from: string
unSubHeader: string
html: string
}): Promise<boolean> {
if (postHeader) {
return Promise.resolve(true)
}
const dom = parseHTML(html).document
// substack newsletter emails have tables with a *post-meta class
if (dom.querySelector('table[class$="post-meta"]')) {
return true
}
// If the article has a header link, and substack icons its probably a newsletter
const href = this.findNewsletterHeaderHref(dom)
const heartIcon = dom.querySelector(
'table tbody td span a img[src*="HeartIcon"]'
)
const recommendIcon = dom.querySelector(
'table tbody td span a img[src*="RecommendIconRounded"]'
)
return Promise.resolve(!!(href && (heartIcon || recommendIcon)))
}
async parseNewsletterUrl(
postHeader: string,
html: string
): Promise<string | undefined> {
// raw SubStack newsletter url is like <https://hongbo130.substack.com/p/tldr>
// we need to get the real url from the raw url
if (postHeader && addressparser(postHeader).length > 0) {
return Promise.resolve(addressparser(postHeader)[0].name)
}
return this.findNewsletterUrl(html)
}
}

31
packages/content-handler/src/websites/apple-news-handler.ts Normal file
View file

@ -0,0 +1,31 @@
import axios from 'axios'
import { parseHTML } from 'linkedom'
import { ContentHandler, PreHandleResult } from '../content-handler'
export class AppleNewsHandler extends ContentHandler {
constructor() {
super()
this.name = 'Apple News'
}
shouldPreHandle(url: string, dom?: Document): boolean {
const u = new URL(url)
return u.hostname === 'apple.news'
}
async preHandle(url: string, document?: Document): Promise<PreHandleResult> {
const MOBILE_USER_AGENT =
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.61 Safari/537.36'
const response = await axios.get(url, {
headers: { 'User-Agent': MOBILE_USER_AGENT },
})
const data = response.data as string
const dom = parseHTML(data).document
// make sure it's a valid URL by wrapping in new URL
const href = dom
.querySelector('span.click-here')
?.parentElement?.getAttribute('href')
const u = href ? new URL(href) : undefined
return { url: u?.href }
}
}

41
packages/content-handler/src/websites/bloomberg-handler.ts Normal file
View file

@ -0,0 +1,41 @@
import axios from 'axios'
import { parseHTML } from 'linkedom'
import { ContentHandler, PreHandleResult } from '../content-handler'
export class BloombergHandler extends ContentHandler {
constructor() {
super()
this.name = 'Bloomberg'
}
shouldPreHandle(url: string, dom?: Document): boolean {
const BLOOMBERG_URL_MATCH =
/https?:\/\/(www\.)?bloomberg.[a-zA-Z0-9()]{1,6}\b([-a-zA-Z0-9()@:%_+.~#?&/=]*)/
return BLOOMBERG_URL_MATCH.test(url.toString())
}
async preHandle(url: string, document?: Document): Promise<PreHandleResult> {
console.log('prehandling bloomberg url', url)
try {
const response = await axios.get('https://app.scrapingbee.com/api/v1', {
params: {
api_key: process.env.SCRAPINGBEE_API_KEY,
url: url,
return_page_source: true,
block_ads: true,
block_resources: false,
},
})
const dom = parseHTML(response.data).document
return {
title: dom.title,
content: dom.querySelector('body')?.innerHTML,
url: url,
}
} catch (error) {
console.error('error prehandling bloomberg url', error)
throw error
}
}
}

34
packages/content-handler/src/websites/derstandard-handler.ts Normal file
View file

@ -0,0 +1,34 @@
import { ContentHandler, PreHandleResult } from '../content-handler'
import axios from 'axios'
import { parseHTML } from 'linkedom'
export class DerstandardHandler extends ContentHandler {
constructor() {
super()
this.name = 'Derstandard'
}
shouldPreHandle(url: string, dom?: Document): boolean {
const u = new URL(url)
return u.hostname === 'www.derstandard.at'
}
async preHandle(url: string, document?: Document): Promise<PreHandleResult> {
const response = await axios.get(url, {
// set cookie to give consent to get the article
headers: {
cookie: `DSGVO_ZUSAGE_V1=true; consentUUID=2bacb9c1-1e80-4be0-9f7b-ee987cf4e7b0_6`,
},
})
const content = response.data as string
const dom = parseHTML(content).document
const titleElement = dom.querySelector('.article-title')
titleElement && titleElement.remove()
return {
content: dom.body.outerHTML,
title: titleElement?.textContent || undefined,
}
}
}

32
packages/content-handler/src/websites/image-handler.ts Normal file
View file

@ -0,0 +1,32 @@
import { ContentHandler, PreHandleResult } from '../content-handler'
export class ImageHandler extends ContentHandler {
constructor() {
super()
this.name = 'Image'
}
shouldPreHandle(url: string, dom?: Document): boolean {
const IMAGE_URL_PATTERN = /(https?:\/\/.*\.(?:jpg|jpeg|png|webp))/i
return IMAGE_URL_PATTERN.test(url.toString())
}
async preHandle(url: string, document?: Document): Promise<PreHandleResult> {
const title = url.toString().split('/').pop() || 'Image'
const content = `
<html>
<head>
<title>${title}</title>
<meta property="og:image" content="${url}" />
<meta property="og:title" content="${title}" />
</head>
<body>
<div>
<img src="${url}" alt="${title}">
</div>
</body>
</html>`
return Promise.resolve({ title, content })
}
}

26
packages/content-handler/src/websites/medium-handler.ts Normal file
View file

@ -0,0 +1,26 @@
import { ContentHandler, PreHandleResult } from '../content-handler'
export class MediumHandler extends ContentHandler {
constructor() {
super()
this.name = 'Medium'
}
shouldPreHandle(url: string, dom?: Document): boolean {
const u = new URL(url)
return u.hostname.endsWith('medium.com')
}
async preHandle(url: string, document?: Document): Promise<PreHandleResult> {
console.log('prehandling medium url', url)
try {
const res = new URL(url)
res.searchParams.delete('source')
return Promise.resolve({ url: res.toString() })
} catch (error) {
console.error('error prehandling medium url', error)
throw error
}
}
}

18
packages/content-handler/src/websites/pdf-handler.ts Normal file
View file

@ -0,0 +1,18 @@
import { ContentHandler, PreHandleResult } from '../content-handler'
export class PdfHandler extends ContentHandler {
constructor() {
super()
this.name = 'PDF'
}
shouldPreHandle(url: string, dom?: Document): boolean {
const u = new URL(url)
const path = u.pathname.replace(u.search, '')
return path.endsWith('.pdf')
}
async preHandle(_url: string, document?: Document): Promise<PreHandleResult> {
return Promise.resolve({ contentType: 'application/pdf' })
}
}

38
packages/content-handler/src/websites/scrapingBee-handler.ts Normal file
View file

@ -0,0 +1,38 @@
import { ContentHandler, PreHandleResult } from '../content-handler'
import axios from 'axios'
import { parseHTML } from 'linkedom'
export class ScrapingBeeHandler extends ContentHandler {
constructor() {
super()
this.name = 'ScrapingBee'
}
shouldPreHandle(url: string, dom?: Document): boolean {
const u = new URL(url)
const hostnames = ['nytimes.com', 'news.google.com']
return hostnames.some((h) => u.hostname.endsWith(h))
}
async preHandle(url: string, document?: Document): Promise<PreHandleResult> {
console.log('prehandling url with scrapingbee', url)
try {
const response = await axios.get('https://app.scrapingbee.com/api/v1', {
params: {
api_key: process.env.SCRAPINGBEE_API_KEY,
url: url,
return_page_source: true,
block_ads: true,
block_resources: false,
},
})
const dom = parseHTML(response.data).document
return { title: dom.title, content: response.data as string, url: url }
} catch (error) {
console.error('error prehandling url w/scrapingbee', error)
throw error
}
}
}

26
packages/content-handler/src/websites/t-dot-co-handler.ts Normal file
View file

@ -0,0 +1,26 @@
import { ContentHandler } from '../content-handler'
import axios from 'axios'
export class TDotCoHandler extends ContentHandler {
constructor() {
super()
this.name = 't.co'
}
shouldResolve(url: string): boolean {
const T_DOT_CO_URL_MATCH = /^https:\/\/(?:www\.)?t\.co\/.*$/
return T_DOT_CO_URL_MATCH.test(url)
}
async resolve(url: string) {
return axios
.get(url, { maxRedirects: 0, validateStatus: null })
.then((res) => {
return new URL(res.headers.location).href
})
.catch((err) => {
console.log('err with t.co url', err)
return undefined
})
}
}

167
packages/content-handler/src/websites/twitter-handler.ts Normal file
View file

@ -0,0 +1,167 @@
import { ContentHandler, PreHandleResult } from '../content-handler'
import axios from 'axios'
import { DateTime } from 'luxon'
import _ from 'underscore'
const TWITTER_BEARER_TOKEN = process.env.TWITTER_BEARER_TOKEN
const TWITTER_URL_MATCH =
/twitter\.com\/(?:#!\/)?(\w+)\/status(?:es)?\/(\d+)(?:\/.*)?/
const getTweetFields = () => {
const TWEET_FIELDS =
'&tweet.fields=attachments,author_id,conversation_id,created_at,' +
'entities,geo,in_reply_to_user_id,lang,possibly_sensitive,public_metrics,referenced_tweets,' +
'source,withheld'
const EXPANSIONS = '&expansions=author_id,attachments.media_keys'
const USER_FIELDS =
'&user.fields=created_at,description,entities,location,pinned_tweet_id,profile_image_url,protected,public_metrics,url,verified,withheld'
const MEDIA_FIELDS =
'&media.fields=duration_ms,height,preview_image_url,url,media_key,public_metrics,width'
return `${TWEET_FIELDS}${EXPANSIONS}${USER_FIELDS}${MEDIA_FIELDS}`
}
const getTweetById = async (id: string) => {
const BASE_ENDPOINT = 'https://api.twitter.com/2/tweets/'
const apiUrl = new URL(BASE_ENDPOINT + id + '?' + getTweetFields())
if (!TWITTER_BEARER_TOKEN) {
throw new Error('No Twitter bearer token found')
}
return axios.get(apiUrl.toString(), {
headers: {
Authorization: `Bearer ${TWITTER_BEARER_TOKEN}`,
redirect: 'follow',
},
})
}
const titleForAuthor = (author: { name: string }) => {
return `${author.name} on Twitter`
}
const tweetIdFromStatusUrl = (url: string): string | undefined => {
const match = url.toString().match(TWITTER_URL_MATCH)
return match?.[2]
}
const formatTimestamp = (timestamp: string) => {
return DateTime.fromJSDate(new Date(timestamp)).toLocaleString(
DateTime.DATETIME_FULL
)
}
export class TwitterHandler extends ContentHandler {
constructor() {
super()
this.name = 'Twitter'
}
shouldPreHandle(url: string, dom?: Document): boolean {
return !!TWITTER_BEARER_TOKEN && TWITTER_URL_MATCH.test(url.toString())
}
async preHandle(url: string, document?: Document): Promise<PreHandleResult> {
console.log('prehandling twitter url', url)
const tweetId = tweetIdFromStatusUrl(url)
if (!tweetId) {
throw new Error('could not find tweet id in url')
}
const tweetData = (await getTweetById(tweetId)).data as {
data: {
author_id: string
text: string
entities: {
urls: [
{
url: string
expanded_url: string
display_url: string
}
]
}
created_at: string
}
includes: {
users: [
{
id: string
name: string
profile_image_url: string
username: string
}
]
media: [
{
preview_image_url: string
type: string
url: string
}
]
}
}
const authorId = tweetData.data.author_id
const author = tweetData.includes.users.filter((u) => (u.id = authorId))[0]
// escape html entities in title
const title = _.escape(titleForAuthor(author))
const authorImage = author.profile_image_url.replace('_normal', '_400x400')
let text = tweetData.data.text
if (tweetData.data.entities && tweetData.data.entities.urls) {
for (const urlObj of tweetData.data.entities.urls) {
text = text.replace(
urlObj.url,
`<a href="${urlObj.expanded_url}">${urlObj.display_url}</a>`
)
}
}
const front = `
<div>
<p>${text}</p>
`
let includesHtml = ''
if (tweetData.includes.media) {
includesHtml = tweetData.includes.media
.map((m) => {
const linkUrl = m.type == 'photo' ? m.url : url
const previewUrl = m.type == 'photo' ? m.url : m.preview_image_url
const mediaOpen = `<a class="media-link" href=${linkUrl}>
<picture>
<img class="tweet-img" src=${previewUrl} />
</picture>
</a>`
return mediaOpen
})
.join('\n')
}
const back = `
— <a href="https://twitter.com/${author.username}">${
author.username
}</a> ${author.name} <a href="${url}">${formatTimestamp(
tweetData.data.created_at
)}</a>
</div>
`
const content = `
<head>
<meta property="og:image" content="${authorImage}" />
<meta property="og:image:secure_url" content="${authorImage}" />
<meta property="og:title" content="${title}" />
<meta property="og:description" content="${_.escape(
tweetData.data.text
)}" />
</head>
<body>
${front}
${includesHtml}
${back}
</body>`
return { content, url, title }
}
}

20
packages/content-handler/src/websites/wikipedia-handler.ts Normal file
View file

@ -0,0 +1,20 @@
import { ContentHandler, PreHandleResult } from '../content-handler'
export class WikipediaHandler extends ContentHandler {
constructor() {
super()
this.name = 'wikipedia'
}
shouldPreHandle(url: string, dom?: Document): boolean {
return new URL(url).hostname.endsWith('wikipedia.org')
}
async preHandle(url: string, dom: Document): Promise<PreHandleResult> {
// This removes the [edit] anchors from wikipedia pages
dom.querySelectorAll('.mw-editsection').forEach((e) => e.remove())
// this removes the sidebar
dom.querySelector('.infobox')?.remove()
return Promise.resolve({ dom })
}
}

58
packages/content-fetch/youtube-handler.js → packages/content-handler/src/websites/youtube-handler.ts
View file

@ -1,18 +1,13 @@
/* eslint-disable no-undef */
/* eslint-disable no-empty */
/* eslint-disable @typescript-eslint/explicit-function-return-type */
/* eslint-disable @typescript-eslint/no-var-requires */
/* eslint-disable @typescript-eslint/no-require-imports */
require('dotenv').config();
const axios = require('axios');
const _ = require('underscore');
import { ContentHandler, PreHandleResult } from '../content-handler'
import axios from 'axios'
import _ from 'underscore'
const YOUTUBE_URL_MATCH =
/^((?:https?:)?\/\/)?((?:www|m)\.)?((?:youtube\.com|youtu.be))(\/(?:[\w-]+\?v=|embed\/|v\/)?)([\w-]+)(\S+)?$/
function getYoutubeVideoId(url) {
const u = new URL(url);
const videoId = u.searchParams.get('v');
export const getYoutubeVideoId = (url: string) => {
const u = new URL(url)
const videoId = u.searchParams.get('v')
if (!videoId) {
const match = url.toString().match(YOUTUBE_URL_MATCH)
if (match === null || match.length < 6 || !match[5]) {
@ -22,28 +17,41 @@ function getYoutubeVideoId(url) {
}
return videoId
}
exports.getYoutubeVideoId = getYoutubeVideoId
exports.youtubeHandler = {
shouldPrehandle: (url, env) => {
export class YoutubeHandler extends ContentHandler {
constructor() {
super()
this.name = 'Youtube'
}
shouldPreHandle(url: string, dom?: Document): boolean {
return YOUTUBE_URL_MATCH.test(url.toString())
},
}
prehandle: async (url, env) => {
async preHandle(url: string, document?: Document): Promise<PreHandleResult> {
const videoId = getYoutubeVideoId(url)
if (!videoId) {
return {}
}
const oembedUrl = `https://www.youtube.com/oembed?format=json&url=` + encodeURIComponent(`https://www.youtube.com/watch?v=${videoId}`)
const oembed = (await axios.get(oembedUrl.toString())).data;
const oembedUrl =
`https://www.youtube.com/oembed?format=json&url=` +
encodeURIComponent(`https://www.youtube.com/watch?v=${videoId}`)
const oembed = (await axios.get(oembedUrl.toString())).data as {
title: string
width: number
height: number
thumbnail_url: string
author_name: string
author_url: string
}
// escape html entities in title
const title = _.escape(oembed.title);
const ratio = oembed.width / oembed.height;
const thumbnail = oembed.thumbnail_url;
const height = 350;
const width = height * ratio;
const authorName = _.escape(oembed.author_name);
const title = _.escape(oembed.title)
const ratio = oembed.width / oembed.height
const thumbnail = oembed.thumbnail_url
const height = 350
const width = height * ratio
const authorName = _.escape(oembed.author_name)
const content = `
<html>
@ -63,6 +71,6 @@ exports.youtubeHandler = {
console.log('got video id', videoId)
return { content, title: 'Youtube Content' };
return { content, title: 'Youtube Content' }
}
}

10
packages/content-handler/test/apple-news-handler.test.ts Normal file
View file

@ -0,0 +1,10 @@
import { AppleNewsHandler } from '../src/websites/apple-news-handler'
describe('open a simple web page', () => {
it('should return a response', async () => {
const response = await new AppleNewsHandler().preHandle(
'https://apple.news/AxjzaZaPvSn23b67LhXI5EQ'
)
console.log('response', response)
})
})

3
packages/content-handler/test/babel-register.js Normal file
View file

@ -0,0 +1,3 @@
const register = require('@babel/register').default
register({ extensions: ['.ts', '.tsx', '.js', '.jsx'] })

0
packages/api/test/utils/data/beehiiv-newsletter.html → packages/content-handler/test/data/beehiiv-newsletter.html
View file

0
packages/api/test/utils/data/substack-forwarded-newsletter.html → packages/content-handler/test/data/substack-forwarded-newsletter.html
View file

0
packages/api/test/utils/data/substack-forwarded-welcome-email.html → packages/content-handler/test/data/substack-forwarded-welcome-email.html
View file

0
packages/api/test/utils/data/substack-private-forwarded-newsletter.html → packages/content-handler/test/data/substack-private-forwarded-newsletter.html
View file

191
packages/content-handler/test/newsletter.test.ts Normal file
View file

@ -0,0 +1,191 @@
import 'mocha'
import * as chai from 'chai'
import { expect } from 'chai'
import chaiAsPromised from 'chai-as-promised'
import chaiString from 'chai-string'
import { SubstackHandler } from '../src/newsletters/substack-handler'
import { AxiosHandler } from '../src/newsletters/axios-handler'
import { BloombergNewsletterHandler } from '../src/newsletters/bloomberg-newsletter-handler'
import { GolangHandler } from '../src/newsletters/golang-handler'
import { MorningBrewHandler } from '../src/newsletters/morning-brew-handler'
import nock from 'nock'
import { generateUniqueUrl } from '../src/content-handler'
import fs from 'fs'
import { BeehiivHandler } from '../src/newsletters/beehiiv-handler'
chai.use(chaiAsPromised)
chai.use(chaiString)
const load = (path: string): string => {
return fs.readFileSync(path, 'utf8')
}
describe('Newsletter email test', () => {
describe('#getNewsletterUrl()', () => {
it('returns url when email is from SubStack', async () => {
const rawUrl = '<https://hongbo130.substack.com/p/tldr>'
await expect(
new SubstackHandler().parseNewsletterUrl(rawUrl, '')
).to.eventually.equal('https://hongbo130.substack.com/p/tldr')
})
it('returns url when email is from Axios', async () => {
const url = 'https://axios.com/blog/the-best-way-to-build-a-web-app'
const html = `View in browser at <a>${url}</a>`
await expect(
new AxiosHandler().parseNewsletterUrl('', html)
).to.eventually.equal(url)
})
it('returns url when email is from Bloomberg', async () => {
const url = 'https://www.bloomberg.com/news/google-is-now-a-partner'
const html = `
<a class="view-in-browser__url" href="${url}">
View in browser
</a>
`
await expect(
new BloombergNewsletterHandler().parseNewsletterUrl('', html)
).to.eventually.equal(url)
})
it('returns url when email is from Golang Weekly', async () => {
const url = 'https://www.golangweekly.com/first'
const html = `
<a href="${url}" style="text-decoration: none">Read on the Web</a>
`
await expect(
new GolangHandler().parseNewsletterUrl('', html)
).to.eventually.equal(url)
})
it('returns url when email is from Morning Brew', async () => {
const url = 'https://www.morningbrew.com/daily/issues/first'
const html = `
<a style="color: #000000; text-decoration: none;" target="_blank" rel="noopener" href="${url}">View Online</a>
`
await expect(
new MorningBrewHandler().parseNewsletterUrl('', html)
).to.eventually.equal(url)
})
})
describe('get author from email address', () => {
it('returns author when email is from Substack', () => {
const from = 'Jackson Harper from Omnivore App <jacksonh@substack.com>'
expect(new AxiosHandler().parseAuthor(from)).to.equal(
'Jackson Harper from Omnivore App'
)
})
it('returns author when email is from Axios', () => {
const from = 'Mike Allen <mike@axios.com>'
expect(new AxiosHandler().parseAuthor(from)).to.equal('Mike Allen')
})
})
describe('isProbablyNewsletter', () => {
it('returns true for substack newsletter', async () => {
const html = load('./test/data/substack-forwarded-newsletter.html')
await expect(
new SubstackHandler().isNewsletter({
html,
postHeader: '',
from: '',
unSubHeader: '',
})
).to.eventually.be.true
})
it('returns true for private forwarded substack newsletter', async () => {
const html = load(
'./test/data/substack-private-forwarded-newsletter.html'
)
await expect(
new SubstackHandler().isNewsletter({
html,
postHeader: '',
from: '',
unSubHeader: '',
})
).to.eventually.be.true
})
it('returns false for substack welcome email', async () => {
const html = load('./test/data/substack-forwarded-welcome-email.html')
await expect(
new SubstackHandler().isNewsletter({
html,
postHeader: '',
from: '',
unSubHeader: '',
})
).to.eventually.be.false
})
it('returns true for beehiiv.com newsletter', async () => {
const html = load('./test/data/beehiiv-newsletter.html')
await expect(
new BeehiivHandler().isNewsletter({
html,
postHeader: '',
from: '',
unSubHeader: '',
})
).to.eventually.be.true
})
})
describe('findNewsletterUrl', async () => {
it('gets the URL from the header if it is a substack newsletter', async () => {
nock('https://email.mg2.substack.com')
.head(
'/c/eJxNkk2TojAQhn-N3KTyQfg4cGDGchdnYcsZx9K5UCE0EMVAkTiKv36iHnarupNUd7rfVJ4W3EDTj1M89No496Uw0wCxgovuwBgYnbOGsZBVjDHzKPWYU8VehUMWOlIX9Qhw4rKLzXgGZziXnRTcyF7dK0iIGMVOG_OS1aTmKPRDilgVhTQUPCQIcE0x-MFTmJ8rCUpA3KtuenR2urg1ZtAzmszI0tq_Z7m66y-ilQo0uAqMTQ7WRX8auJKg56blZg7WB-iHDuYEBzO6NP0R1IwuYFphQbbTjnTH9NBfs80nym4Zyj8uUvyKbtUyGr5eUz9fNDQ7JCxfJDo9dW1lY9lmj_JNivPbGmf2Pt_lN9tDit9b-WeTetni85Z9pDpVOd7L1E_Vy7egayNO23ZP34eSeLJeux1b0rer_xaZ7ykS78nuSjMY-nL98rparNZNcv07JCjN06_EkTFBxBqOUMACErnELUNMSxTUjLDQZwzcqa4bRjCfeejUEFefS224OLr2S5wxPtij7lVrs80d2CNseRV2P52VNFMBipcdVE-U5jkRD7hFAwpGOylVwU2Mfc9qBh7DoR89yVnWXhgQFHnIsbpVb6tU_B-hH_2yzWY'
)
.reply(302, undefined, {
Location:
'https://newsletter.slowchinese.net/p/companies-that-eat-people-217',
})
.get('/p/companies-that-eat-people-217')
.reply(200, '')
const html = load('./test/data/substack-forwarded-newsletter.html')
const url = await new SubstackHandler().findNewsletterUrl(html)
// Not sure if the redirects from substack expire, this test could eventually fail
expect(url).to.startWith(
'https://newsletter.slowchinese.net/p/companies-that-eat-people-217'
)
}).timeout(10000)
it('gets the URL from the header if it is a beehiiv newsletter', async () => {
nock('https://u23463625.ct.sendgrid.net')
.head(
'/ss/c/AX1lEgEQaxtvFxLaVo0GBo_geajNrlI1TGeIcmMViR3pL3fEDZnbbkoeKcaY62QZk0KPFudUiUXc_uMLerV4nA/3k5/3TFZmreTR0qKSCgowABnVg/h30/zzLik7UXd1H_n4oyd5W8Xu639AYQQB2UXz-CsssSnno'
)
.reply(302, undefined, {
Location: 'https://www.milkroad.com/p/talked-guy-spent-30m-beeple',
})
.get('/p/talked-guy-spent-30m-beeple')
.reply(200, '')
const html = load('./test/data/beehiiv-newsletter.html')
const url = await new BeehiivHandler().findNewsletterUrl(html)
expect(url).to.startWith(
'https://www.milkroad.com/p/talked-guy-spent-30m-beeple'
)
})
it('returns undefined if it is not a newsletter', async () => {
const html = load('./test/data/substack-forwarded-welcome-email.html')
const url = await new SubstackHandler().findNewsletterUrl(html)
expect(url).to.be.undefined
})
})
describe('generateUniqueUrl', () => {
it('generates a unique URL', () => {
const url1 = generateUniqueUrl()
const url2 = generateUniqueUrl()
expect(url1).to.not.eql(url2)
})
})
})

25
packages/content-handler/test/youtube-handler.test.ts Normal file
View file

@ -0,0 +1,25 @@
import { expect } from 'chai'
import 'mocha'
import { getYoutubeVideoId } from '../src/websites/youtube-handler'
describe('getYoutubeVideoId', () => {
it('should parse video id out of a URL', async () => {
expect('BnSUk0je6oo').to.eq(
getYoutubeVideoId('https://www.youtube.com/watch?v=BnSUk0je6oo&t=269s')
)
expect('vFD2gu007dc').to.eq(
getYoutubeVideoId(
'https://www.youtube.com/watch?v=vFD2gu007dc&list=RDvFD2gu007dc&start_radio=1'
)
)
expect('vFD2gu007dc').to.eq(
getYoutubeVideoId('https://youtu.be/vFD2gu007dc')
)
expect('BMFVCnbRaV4').to.eq(
getYoutubeVideoId('https://youtube.com/watch?v=BMFVCnbRaV4&feature=share')
)
expect('cg9b4RC87LI').to.eq(
getYoutubeVideoId('https://youtu.be/cg9b4RC87LI?t=116')
)
})
})

10
packages/content-handler/tsconfig.json Normal file
View file

@ -0,0 +1,10 @@
{
"extends": "@tsconfig/node14/tsconfig.json",
"compilerOptions": {
"rootDir": ".",
"declaration": true,
"outDir": "build",
"lib": ["dom"]
},
"include": ["src"]
}

9
packages/db/migrations/0096.do.subscriptions_icon.sql Executable file
View file

@ -0,0 +1,9 @@
-- Type: DO
-- Name: subscriptions_icon
-- Description: Add icon field to subscriptions table
BEGIN;
ALTER TABLE omnivore.subscriptions ADD COLUMN icon text;
COMMIT;

9
packages/db/migrations/0096.undo.subscriptions_icon.sql Executable file
View file

@ -0,0 +1,9 @@
-- Type: UNDO
-- Name: subscriptions_icon
-- Description: Add icon field to subscriptions table
BEGIN;
ALTER TABLE omnivore.subscriptions DROP COLUMN IF EXISTS icon;
COMMIT;

1
packages/inbound-email-handler/package.json
View file

@ -31,6 +31,7 @@
"@google-cloud/pubsub": "^2.18.4",
"@sendgrid/client": "^7.6.0",
"@sentry/serverless": "^6.16.1",
"@omnivore/content-handler": "1.0.0",
"addressparser": "^1.0.1",
"axios": "^0.27.2",
"jsonwebtoken": "^8.5.1",

10
packages/inbound-email-handler/src/axios-handler.ts
View file

@ -1,10 +0,0 @@
import { NewsletterHandler } from './newsletter'
export class AxiosHandler extends NewsletterHandler {
constructor() {
super()
this.senderRegex = /<.+@axios.com>/
this.urlRegex = /View in browser at <a.*>(.*)<\/a>/
this.defaultUrl = 'https://axios.com'
}
}

10
packages/inbound-email-handler/src/bloomberg-handler.ts
View file

@ -1,10 +0,0 @@
import { NewsletterHandler } from './newsletter'
export class BloombergHandler extends NewsletterHandler {
constructor() {
super()
this.senderRegex = /<.+@mail.bloomberg.*.com>/
this.urlRegex = /<a class="view-in-browser__url" href=["']([^"']*)["']/
this.defaultUrl = 'https://www.bloomberg.com'
}
}

10
packages/inbound-email-handler/src/golang-handler.ts
View file

@ -1,10 +0,0 @@
import { NewsletterHandler } from './newsletter'
export class GolangHandler extends NewsletterHandler {
constructor() {
super()
this.senderRegex = /<.+@golangweekly.com>/
this.urlRegex = /<a href=["']([^"']*)["'].*>Read on the Web<\/a>/
this.defaultUrl = 'https://golangweekly.com'
}
}

61
packages/inbound-email-handler/src/index.ts
View file

@ -9,35 +9,27 @@ import * as multipart from 'parse-multipart-data'
import {
handleConfirmation,
isConfirmationEmail,
NewsletterHandler,
parseUnsubscribe,
} from './newsletter'
import { PubSub } from '@google-cloud/pubsub'
import { handlePdfAttachment } from './pdf'
import { SubstackHandler } from './substack-handler'
import { AxiosHandler } from './axios-handler'
import { BloombergHandler } from './bloomberg-handler'
import { GolangHandler } from './golang-handler'
import { MorningBrewHandler } from './morning-brew-handler'
import { handleNewsletter } from '@omnivore/content-handler'
const NEWSLETTER_EMAIL_RECEIVED_TOPIC = 'newsletterEmailReceived'
const NON_NEWSLETTER_EMAIL_TOPIC = 'nonNewsletterEmailReceived'
const pubsub = new PubSub()
const NEWSLETTER_HANDLERS = [
new SubstackHandler(),
new AxiosHandler(),
new BloombergHandler(),
new GolangHandler(),
new MorningBrewHandler(),
]
export const getNewsletterHandler = (
postHeader: string,
from: string,
unSubHeader: string
): NewsletterHandler | undefined => {
return NEWSLETTER_HANDLERS.find((h) => {
return h.isNewsletter(postHeader, from, unSubHeader)
})
export const publishMessage = async (
topic: string,
message: any
): Promise<string | undefined> => {
return pubsub
.topic(topic)
.publishMessage({ json: message })
.catch((err) => {
console.log('error publishing message:', err)
return undefined
})
}
export const inboundEmailHandler = Sentry.GCPFunction.wrapHttpFunction(
@ -86,23 +78,20 @@ export const inboundEmailHandler = Sentry.GCPFunction.wrapHttpFunction(
try {
// check if it is a confirmation email or forwarding newsletter
const newsletterHandler = getNewsletterHandler(
postHeader,
const newsletterMessage = await handleNewsletter({
from,
unSubHeader
)
if (newsletterHandler) {
console.log('handleNewsletter', from, to)
await newsletterHandler.handleNewsletter(
to,
html,
postHeader,
subject,
from,
unSubHeader
html,
postHeader,
unSubHeader,
email: to,
title: subject,
})
if (newsletterMessage) {
await publishMessage(
NEWSLETTER_EMAIL_RECEIVED_TOPIC,
newsletterMessage
)
return res.send('ok')
return res.status(200).send('newsletter received')
}
console.log('non-newsletter email from', from, 'to', to)

10
packages/inbound-email-handler/src/morning-brew-handler.ts
View file

@ -1,10 +0,0 @@
import { NewsletterHandler } from './newsletter'
export class MorningBrewHandler extends NewsletterHandler {
constructor() {
super()
this.senderRegex = /Morning Brew <crew@morningbrew.com>/
this.urlRegex = /<a.* href=["']([^"']*)["'].*>View Online<\/a>/
this.defaultUrl = 'https://www.morningbrew.com'
}
}

Some files were not shown because too many files have changed in this diff Show more