prehistoric-systems/omnivore
1
0
Fork 0
mirror of https://github.com/omnivore-app/omnivore.git synced 2026-03-11 08:54:26 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge commit '6675688361d457fe12adda8f33f17fb7a4528210' into OMN-506

Browse source
This commit is contained in:
gitstart-omnivore 2022-05-09 17:10:57 +00:00
commit caeef23bc5
32 changed files with 1886 additions and 138 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

77
apple/OmnivoreKit/Sources/App/Views/Home/HomeFeedViewIOS.swift
View file

@ -5,10 +5,12 @@ import UserNotifications
import Utils
import Views
private let enableGrid = UIDevice.isIPad || FeatureFlag.enableGridCardsOnPhone
#if os(iOS)
struct HomeFeedContainerView: View {
@EnvironmentObject var dataService: DataService
@AppStorage(UserDefaultKey.homeFeedlayoutPreference.rawValue) var prefersListLayout = UIDevice.isIPhone
@AppStorage(UserDefaultKey.homeFeedlayoutPreference.rawValue) var prefersListLayout = false
@ObservedObject var viewModel: HomeFeedViewModel
func loadItems(isRefresh: Bool) {
@ -49,6 +51,44 @@ import Views
.sheet(item: $viewModel.itemUnderLabelEdit) { item in
ApplyLabelsView(mode: .item(item), onSave: nil)
}
.toolbar {
ToolbarItem(placement: .barTrailing) {
Button("", action: {})
.disabled(true)
.overlay {
if viewModel.isLoading, !prefersListLayout, enableGrid {
ProgressView()
}
}
}
ToolbarItem(placement: UIDevice.isIPhone ? .barLeading : .barTrailing) {
if enableGrid {
Button(
action: { prefersListLayout.toggle() },
label: {
Label("Toggle Feed Layout", systemImage: prefersListLayout ? "square.grid.2x2" : "list.bullet")
}
)
} else {
EmptyView()
}
}
ToolbarItem(placement: .barTrailing) {
if UIDevice.isIPhone {
NavigationLink(
destination: { ProfileView() },
label: {
Image.profile
.resizable()
.frame(width: 26, height: 26)
.padding(.vertical)
}
)
} else {
EmptyView()
}
}
}
}
.navigationTitle("Home")
.navigationBarTitleDisplayMode(.inline)
@ -115,31 +155,10 @@ import Views
}
}
}
if prefersListLayout {
if prefersListLayout || !enableGrid {
HomeFeedListView(prefersListLayout: $prefersListLayout, viewModel: viewModel)
} else {
HomeFeedGridView(viewModel: viewModel)
.toolbar {
ToolbarItem {
Button("", action: {})
.disabled(true)
.overlay {
if viewModel.isLoading {
ProgressView()
}
}
}
ToolbarItem {
if UIDevice.isIPad {
Button(
action: { prefersListLayout.toggle() },
label: {
Label("Toggle Feed Layout", systemImage: prefersListLayout ? "square.grid.2x2" : "list.bullet")
}
)
}
}
}
}
}
}
@ -256,18 +275,6 @@ import Views
}
}
.listStyle(PlainListStyle())
.toolbar {
ToolbarItem {
if UIDevice.isIPad {
Button(
action: { prefersListLayout.toggle() },
label: {
Label("Toggle Feed Layout", systemImage: prefersListLayout ? "square.grid.2x2" : "list.bullet")
}
)
}
}
}
}
}

14
apple/OmnivoreKit/Sources/App/Views/Home/HomeView.swift
View file

@ -8,20 +8,8 @@ struct HomeView: View {
if UIDevice.isIPhone {
NavigationView {
HomeFeedContainerView(viewModel: viewModel)
.toolbar {
ToolbarItem {
NavigationLink(
destination: { ProfileView() },
label: {
Image.profile
.resizable()
.frame(width: 26, height: 26)
.padding()
}
)
}
}
}
.navigationViewStyle(StackNavigationViewStyle())
.accentColor(.appGrayTextContrast)
} else {
HomeFeedContainerView(viewModel: viewModel)

4
apple/OmnivoreKit/Sources/App/Views/Labels/LabelsViewModel.swift
View file

@ -22,7 +22,7 @@ import Views
dataService.viewContext.performAndWait {
self.labels = labelIDs.compactMap { dataService.viewContext.object(with: $0) as? LinkedItemLabel }
}
let selLabels = initiallySelectedLabels ?? item?.labels.asArray(of: LinkedItemLabel.self) ?? []
let selLabels = initiallySelectedLabels ?? item?.sortedLabels ?? []
for label in labels {
if selLabels.contains(label) {
selectedLabels.append(label)
@ -43,6 +43,7 @@ import Views
color: color.hex ?? "",
description: description
) else {
isLoading = false
return
}
@ -51,6 +52,7 @@ import Views
unselectedLabels.insert(label, at: 0)
}
isLoading = false
showCreateEmailModal = false
}

28
apple/OmnivoreKit/Sources/App/Views/WebReader/WebReaderContainer.swift
View file

@ -14,7 +14,7 @@ import WebKit
@State var safariWebLink: SafariWebLink?
@State private var navBarVisibilityRatio = 1.0
@State private var showDeleteConfirmation = false
@State private var showOverlay = true
@State private var progressViewOpacity = 0.0
@State var increaseFontActionID: UUID?
@State var decreaseFontActionID: UUID?
@State var annotationSaveTransactionID: UUID?
@ -156,21 +156,6 @@ import WebKit
annotationSaveTransactionID: $annotationSaveTransactionID,
annotation: $annotation
)
.overlay(
Group {
if showOverlay {
Color.systemBackground
.transition(.opacity)
.onAppear {
DispatchQueue.main.asyncAfter(deadline: .now() + .milliseconds(100)) {
withAnimation(.linear(duration: 0.2)) {
showOverlay = false
}
}
}
}
}
)
.sheet(item: $safariWebLink) {
SafariView(url: $0.url)
}
@ -186,9 +171,16 @@ import WebKit
}
)
}
} else if let errorMessage = viewModel.errorMessage {
Text(errorMessage).padding()
} else {
Text(viewModel.contentFetchFailed ? "Unable to fetch content." : "Processing...")
.padding()
ProgressView()
.opacity(progressViewOpacity)
.onAppear {
DispatchQueue.main.asyncAfter(deadline: .now() + .milliseconds(1000)) {
progressViewOpacity = 1
}
}
.task {
await viewModel.loadContent(dataService: dataService, itemID: item.unwrappedID)
}

15
apple/OmnivoreKit/Sources/App/Views/WebReader/WebReaderViewModel.swift
View file

@ -10,15 +10,24 @@ struct SafariWebLink: Identifiable {
@MainActor final class WebReaderViewModel: ObservableObject {
@Published var articleContent: ArticleContent?
@Published var contentFetchFailed = false
@Published var errorMessage: String?
func loadContent(dataService: DataService, itemID: String) async {
contentFetchFailed = false
errorMessage = nil
do {
articleContent = try await dataService.fetchArticleContent(itemID: itemID)
} catch {
contentFetchFailed = true
if let fetchError = error as? ContentFetchError {
switch fetchError {
case .network:
errorMessage = "We were unable to retrieve your content. Please ccheck network connectivity and try again."
default:
errorMessage = "We were unable to parse your content."
}
} else {
errorMessage = "We were unable to retrieve your content."
}
}
}

6
apple/OmnivoreKit/Sources/Models/DataModels/FeedItem.swift
View file

@ -58,6 +58,12 @@ public extension LinkedItem {
return URL(string: pageURLString ?? "")
}
var sortedLabels: [LinkedItemLabel] {
labels.asArray(of: LinkedItemLabel.self).sorted {
($0.name ?? "").lowercased() < ($1.name ?? "").lowercased()
}
}
var labelsJSONString: String {
let labels = self.labels.asArray(of: LinkedItemLabel.self).map { label in
[

2
apple/OmnivoreKit/Sources/Models/ErrorModels/SaveArticleError.swift
View file

@ -1,5 +1,7 @@
import Foundation
public typealias ContentFetchError = SaveArticleError
public enum SaveArticleError: Error {
case unauthorized
case network

34
apple/OmnivoreKit/Sources/Services/DataService/Queries/ArticleContentQuery.swift
View file

@ -20,7 +20,7 @@ extension DataService {
func prefetchPage(pendingLink: PendingLink, username: String) async {
let content = try? await articleContent(username: username, itemID: pendingLink.itemID, useCache: false)
if content?.contentStatus == .processing, pendingLink.retryCount < 6 {
if content?.contentStatus == .processing, pendingLink.retryCount < 7 {
let retryDelayInNanoSeconds = UInt64(pendingLink.retryCount * 2 * 1_000_000_000)
do {
@ -45,26 +45,24 @@ extension DataService {
username: String? = nil,
requestCount: Int = 1
) async throws -> ArticleContent {
guard let username = username ?? currentViewer?.username else {
throw BasicError.message(messageText: "username could not be fetched from core data")
guard requestCount < 7 else {
throw ContentFetchError.badData
}
guard let fetchedContent = try? await articleContent(username: username, itemID: itemID, useCache: true) else {
throw BasicError.message(messageText: "networking error")
guard let username = username ?? currentViewer?.username else {
throw ContentFetchError.unauthorized
}
let fetchedContent = try await articleContent(username: username, itemID: itemID, useCache: true)
switch fetchedContent.contentStatus {
case .failed:
throw BasicError.message(messageText: "content processing failed")
throw ContentFetchError.badData
case .processing:
do {
let retryDelayInNanoSeconds = UInt64(requestCount * 2 * 1_000_000_000)
try await Task.sleep(nanoseconds: retryDelayInNanoSeconds)
logger.debug("fetching content for \(itemID). request count: \(requestCount)")
return try await fetchArticleContent(itemID: itemID, username: username, requestCount: requestCount + 1)
} catch {
throw BasicError.message(messageText: "content fetch failed")
}
let retryDelayInNanoSeconds = UInt64(requestCount * 2 * 1_000_000_000)
try await Task.sleep(nanoseconds: retryDelayInNanoSeconds)
logger.debug("fetching content for \(itemID). request count: \(requestCount)")
return try await fetchArticleContent(itemID: itemID, username: username, requestCount: requestCount + 1)
case .succeeded, .unknown:
return fetchedContent
}
@ -120,7 +118,7 @@ extension DataService {
return try await withCheckedThrowingContinuation { continuation in
send(query, to: path, headers: headers) { [weak self] queryResult in
guard let payload = try? queryResult.get() else {
continuation.resume(throwing: BasicError.message(messageText: "network error"))
continuation.resume(throwing: ContentFetchError.network)
return
}
@ -129,6 +127,10 @@ extension DataService {
// Default to suceeded since older links will return a nil status
// (but the content is almost always there)
let status = result.contentStatus ?? .succeeded
if status == .failed {
continuation.resume(throwing: ContentFetchError.badData)
return
}
if status == .succeeded {
self?.persistArticleContent(
@ -146,7 +148,7 @@ extension DataService {
continuation.resume(returning: articleContent)
case .error:
continuation.resume(throwing: BasicError.message(messageText: "LinkedItem fetch error"))
continuation.resume(throwing: ContentFetchError.badData)
}
}
}

1
apple/OmnivoreKit/Sources/Utils/FeatureFlags.swift
View file

@ -14,4 +14,5 @@ public enum FeatureFlag {
public static let enablePushNotifications = false
public static let enableShareButton = false
public static let enableSnooze = false
public static let enableGridCardsOnPhone = false
}

3
apple/OmnivoreKit/Sources/Views/FeedItem/GridCard.swift
View file

@ -151,12 +151,11 @@ public struct GridCard: View {
// Category Labels
ScrollView(.horizontal, showsIndicators: false) {
HStack {
ForEach(item.labels.asArray(of: LinkedItemLabel.self), id: \.self) {
ForEach(item.sortedLabels, id: \.self) {
TextChip(feedItemLabel: $0)
}
Spacer()
}
.frame(height: 30)
.padding(.horizontal)
.padding(.bottom, 8)
}

34
apple/OmnivoreKit/Sources/Views/FeedItem/HomeFeedCardView.swift
View file

@ -12,12 +12,11 @@ public struct FeedCard: View {
public var body: some View {
VStack {
HStack(alignment: .top, spacing: 6) {
VStack(alignment: .leading, spacing: 6) {
VStack(alignment: .leading, spacing: 4) {
Text(item.unwrappedTitle)
.font(.appSubheadline)
.font(.appCallout)
.foregroundColor(.appGrayTextContrast)
.lineLimit(2)
.frame(maxWidth: .infinity, alignment: .leading)
.fixedSize(horizontal: false, vertical: true)
if let author = item.author {
Text("By \(author)")
@ -34,7 +33,13 @@ public struct FeedCard: View {
.lineLimit(1)
}
}
.frame(maxWidth: .infinity)
.frame(
minWidth: 0,
maxWidth: .infinity,
minHeight: 0,
maxHeight: .infinity,
alignment: .topLeading
)
.multilineTextAlignment(.leading)
.padding(0)
@ -52,7 +57,7 @@ public struct FeedCard: View {
.frame(width: 80, height: 80)
.cornerRadius(6)
} else {
EmptyView()
EmptyView().frame(width: 80, height: 80, alignment: .top)
}
}
}
@ -62,14 +67,25 @@ public struct FeedCard: View {
// Category Labels
ScrollView(.horizontal, showsIndicators: false) {
HStack {
ForEach(item.labels.asArray(of: LinkedItemLabel.self), id: \.self) {
ForEach(item.sortedLabels, id: \.self) {
TextChip(feedItemLabel: $0)
}
Spacer()
}
}
.padding(.bottom, 5)
.padding(.top, 8)
.padding(.bottom, 2)
}
.padding(.top, 5)
.padding(.top, 16)
.padding(.bottom, 8)
.frame(
minWidth: nil,
idealWidth: nil,
maxWidth: nil,
minHeight: 70,
idealHeight: nil,
maxHeight: nil,
alignment: .topLeading
)
}
}

8
apple/OmnivoreKit/Sources/Views/TextChip.swift
View file

@ -17,7 +17,6 @@ public struct TextChip: View {
let text: String
let color: Color
let cornerRadius = 20.0
public var body: some View {
Text(text)
@ -26,8 +25,7 @@ public struct TextChip: View {
.font(.appFootnote)
.foregroundColor(color.isDark ? .white : .black)
.lineLimit(1)
.background(color)
.cornerRadius(cornerRadius)
.background(Capsule().fill(color))
}
}
@ -86,7 +84,6 @@ public struct TextChipButton: View {
let color: Color
let onTap: () -> Void
let actionType: ActionType
let cornerRadius = 20.0
let foregroundColor: Color
public var body: some View {
@ -102,8 +99,7 @@ public struct TextChipButton: View {
.font(.appFootnote)
.foregroundColor(foregroundColor)
.lineLimit(1)
.background(color)
.cornerRadius(cornerRadius)
.background(Capsule().fill(color))
Color.clear.contentShape(Rectangle()).frame(height: 15)
}

1
packages/api/src/readability.d.ts vendored
View file

@ -163,6 +163,7 @@ declare module '@omnivore/readability' {
previewImage?: string
/** Article published date */
publishedDate?: Date
dom?: Element
}
}

6
packages/api/src/resolvers/article/index.ts
View file

@ -54,7 +54,6 @@ import {
} from '../../utils/helpers'
import {
ParsedContentPuppeteer,
parseOriginalContent,
parsePreparedContent,
} from '../../utils/parser'
import { isSiteBlockedForParse } from '../../utils/blocked'
@ -230,14 +229,13 @@ export const createArticleResolver = authorized<
const parseResults = await traceAs<Promise<ParsedContentPuppeteer>>(
{ spanName: 'article.parse' },
async (): Promise<ParsedContentPuppeteer> => {
return await parsePreparedContent(url, preparedDocument)
return parsePreparedContent(url, preparedDocument)
}
)
parsedContent = parseResults.parsedContent
canonicalUrl = parseResults.canonicalUrl
domContent = parseResults.domContent
pageType = parseOriginalContent(url, domContent)
pageType = parseResults.pageType
} else if (!preparedDocument?.document) {
// We have a URL but no document, so we try to send this to puppeteer
// and return a dummy response.

9
packages/api/src/services/save_email.ts
View file

@ -1,9 +1,5 @@
import { generateSlug, stringToHash, validatedDate } from '../utils/helpers'
import {
parseOriginalContent,
parsePreparedContent,
parseUrlMetadata,
} from '../utils/parser'
import { parsePreparedContent, parseUrlMetadata } from '../utils/parser'
import normalizeUrl from 'normalize-url'
import { PubsubClient } from '../datalayer/pubsub'
import { ArticleSavingRequestStatus, Page } from '../elastic/types'
@ -44,7 +40,6 @@ export const saveEmail = async (
const content = parseResult.parsedContent?.content || input.originalContent
const slug = generateSlug(title)
const pageType = parseOriginalContent(url, input.originalContent)
const metadata = await parseUrlMetadata(url)
const articleToSave: Page = {
@ -60,7 +55,7 @@ export const saveEmail = async (
stripHash: true,
stripWWW: false,
}),
pageType: pageType,
pageType: parseResult.pageType,
hash: stringToHash(content),
image: metadata?.previewImage || parseResult.parsedContent?.previewImage,
publishedAt: validatedDate(parseResult.parsedContent?.publishedDate),

6
packages/api/src/services/save_page.ts
View file

@ -3,7 +3,7 @@ import { homePageURL } from '../env'
import { Maybe, SavePageInput, SaveResult } from '../generated/graphql'
import { DataModels } from '../resolvers/types'
import { generateSlug, stringToHash, validatedDate } from '../utils/helpers'
import { parseOriginalContent, parsePreparedContent } from '../utils/parser'
import { parsePreparedContent } from '../utils/parser'
import normalizeUrl from 'normalize-url'
import { createPageSaveRequest } from './create_page_save_request'
@ -72,8 +72,6 @@ export const savePage = async (
},
})
const pageType = parseOriginalContent(input.url, input.originalContent)
const articleToSave: Page = {
id: input.clientRequestId,
slug,
@ -87,7 +85,7 @@ export const savePage = async (
stripHash: true,
stripWWW: false,
}),
pageType: pageType,
pageType: parseResult.pageType,
hash: stringToHash(parseResult.parsedContent?.content || input.url),
image: parseResult.parsedContent?.previewImage,
publishedAt: validatedDate(parseResult.parsedContent?.publishedDate),

15
packages/api/src/utils/parser.ts
View file

@ -80,6 +80,7 @@ export type ParsedContentPuppeteer = {
domContent: string
parsedContent: Readability.ParseResult | null
canonicalUrl?: string | null
pageType: PageType
}
/* eslint-disable @typescript-eslint/no-explicit-any */
@ -101,9 +102,8 @@ type ArticleParseLogRecord = LogRecord & {
const DEBUG_MODE = process.env.DEBUG === 'true' || false
export const parseOriginalContent = (url: string, html: string): PageType => {
const parseOriginalContent = (window: DOMWindow): PageType => {
try {
const { window } = new JSDOM(html, { url })
const e = window.document.querySelector("head meta[property='og:type']")
const content = e?.getAttribute('content')
if (!content) {
@ -121,7 +121,7 @@ export const parseOriginalContent = (url: string, html: string): PageType => {
return PageType.Website
}
} catch (error) {
logger.error('Error extracting og:type from content for url', url, error)
logger.error('Error extracting og:type from content', error)
}
return PageType.Unknown
@ -232,6 +232,7 @@ export const parsePreparedContent = async (
canonicalUrl: url,
parsedContent: null,
domContent: preparedDocument.document,
pageType: PageType.Unknown,
}
}
@ -253,9 +254,8 @@ export const parsePreparedContent = async (
// Format code blocks
// TODO: we probably want to move this type of thing
// to the handlers, and have some concept of postHandle
if (article?.content) {
const cWindow = new JSDOM(article?.content).window
cWindow.document.querySelectorAll('code').forEach((e) => {
if (article?.dom) {
article.dom.querySelectorAll('code').forEach((e) => {
console.log(e.textContent)
if (e.textContent) {
const att = hljs.highlightAuto(e.textContent)
@ -270,7 +270,7 @@ export const parsePreparedContent = async (
e.replaceWith(code)
}
})
article.content = cWindow.document.body.outerHTML
article.content = article.dom.outerHTML
}
const newWindow = new JSDOM('').window
@ -310,6 +310,7 @@ export const parsePreparedContent = async (
domContent: preparedDocument.document,
parsedContent: article,
canonicalUrl,
pageType: parseOriginalContent(window),
}
}

4
packages/db/migrate.ts
View file

@ -89,6 +89,7 @@ const logAppliedMigrations = (
export const INDEX_ALIAS = 'pages_alias'
export const esClient = new Client({
node: process.env.ELASTIC_URL || 'http://localhost:9200',
requestTimeout: 60000 * 30, // 30 minutes
auth: {
username: process.env.ELASTIC_USERNAME || '',
password: process.env.ELASTIC_PASSWORD || '',
@ -133,6 +134,9 @@ log('Starting adding default state to pages in elasticsearch...')
esClient
.update_by_query({
index: INDEX_ALIAS,
requests_per_second: 250,
scroll_size: 500,
timeout: '30m',
body: {
script: {
source: 'ctx._source.state = params.state',

34
packages/puppeteer-parse/image-handler.js Normal file
View file

@ -0,0 +1,34 @@
/* eslint-disable no-undef */
/* eslint-disable no-empty */
/* eslint-disable @typescript-eslint/explicit-function-return-type */
/* eslint-disable @typescript-eslint/no-var-requires */
/* eslint-disable @typescript-eslint/no-require-imports */
require('dotenv').config();
exports.imageHandler = {
shouldPrehandle: (url, env) => {
const IMAGE_URL_PATTERN =
/(https?:\/\/.*\.(?:jpg|jpeg|png|webp))/i
return IMAGE_URL_PATTERN.test(url.toString())
},
prehandle: async (url, env) => {
const title = url.toString().split('/').pop();
const content = `
<html>
<head>
<title>${title}</title>
<meta property="og:image" content="${url}" />
<meta property="og:title" content="${title}" />
</head>
<body>
<div>
<img src="${url}" alt="${title}">
</div>
</body>
</html>`
return { title, content };
}
}

2
packages/puppeteer-parse/index.js
View file

@ -24,6 +24,7 @@ const { tDotCoHandler } = require('./t-dot-co-handler');
const { pdfHandler } = require('./pdf-handler');
const { mediumHandler } = require('./medium-handler');
const { derstandardHandler } = require('./derstandard-handler');
const { imageHandler } = require('./image-handler');
const storage = new Storage();
const ALLOWED_ORIGINS = process.env.ALLOWED_ORIGINS ? process.env.ALLOWED_ORIGINS.split(',') : [];
@ -228,6 +229,7 @@ const handlers = {
't-dot-co': tDotCoHandler,
'medium': mediumHandler,
'derstandard': derstandardHandler,
'image': imageHandler,
};
/**

7
packages/readabilityjs/Readability.js
View file

@ -167,8 +167,8 @@ Readability.prototype = {
// NOTE: These two regular expressions are duplicated in
// Readability-readerable.js. Please keep both copies in sync.
articleNegativeLookBehindCandidates: /breadcrumbs|breadcrumb|utils|trilist/i,
articleNegativeLookAheadCandidates: /outstream(.?)_|sub(.?)_|m_/i,
unlikelyCandidates: /-ad-|ai2html|banner|breadcrumbs|breadcrumb|combx|comment|community|cover-wrap|disqus|extra|footer|gdpr|header|legends|menu|related|remark|replies|rss|shoutbox|sidebar|skyscraper|social|sponsor|supplemental|ad-break|agegate|pagination|pager(?!ow)|popup|yom-remote|copyright|keywords|outline|infinite-list|beta|recirculation|site-index|hide-for-print|post-end-share-cta|post-end-cta-full|post-footer|main-navigation|programtic-ads|outstream_article|hfeed|comment-holder|back-to-top|show-up-next/i,
articleNegativeLookAheadCandidates: /outstream(.?)_|sub(.?)_|m_|omeda-promo-|in-article-advert/i,
unlikelyCandidates: /-ad-|ai2html|banner|breadcrumbs|breadcrumb|combx|comment|community|cover-wrap|disqus|extra|footer|gdpr|header|legends|menu|related|remark|replies|rss|shoutbox|sidebar|skyscraper|social|sponsor|supplemental|ad-break|agegate|pagination|pager(?!ow)|popup|yom-remote|copyright|keywords|outline|infinite-list|beta|recirculation|site-index|hide-for-print|post-end-share-cta|post-end-cta-full|post-footer|main-navigation|programtic-ads|outstream_article|hfeed|comment-holder|back-to-top|show-up-next|onward-journey|topic-tracker/i,
// okMaybeItsACandidate: /and|article(?!-breadcrumb)|body|column|content|main|shadow|post-header/i,
get okMaybeItsACandidate() {
return new RegExp(`and|(?<!${this.articleNegativeLookAheadCandidates.source})article(?!-(${this.articleNegativeLookBehindCandidates.source}))|body|column|content|^(?!main-navigation)main|shadow|post-header|hfeed site|blog-posts hfeed`, 'i')
@ -1579,7 +1579,7 @@ Readability.prototype = {
});
// But first check if we actually have something
if (!this._attempts[0].textLength) {
if (!this._attempts[0].textLength && !this._attempts[0].articleContent) {
return null;
}
@ -2899,6 +2899,7 @@ Readability.prototype = {
siteIcon: metadata.siteIcon,
previewImage: metadata.previewImage,
publishedDate: metadata.publishedDate || publishedAt || this._articlePublishedDate,
dom: articleContent,
};
}
};

11
packages/readabilityjs/test/test-pages/fiercepharma/expected-metadata.json Normal file
View file

@ -0,0 +1,11 @@
{
"title": "Novavax, eyeing the COVID 'vaccine hesitant' and kids, unveils new education campaigns as Nuvaxovid nears US finish line",
"byline": " 03:23pm",
"dir": null,
"excerpt": "Pfizer, Moderna and Johnson & Johnson were quickest off the mark in getting COVID vaccines into American arms, but Novavax is hoping to add another pandemic vaccine to the U.S. | Pfizer, Moderna and Johnson & Johnson were quickest off the mark in getting COVID vaccines into American arms, but Novavax is hoping to add another pandemic vaccine to the U.S. mix soon—and it's pushing new campaigns to get the word out.",
"siteName": "Fierce Pharma",
"siteIcon": "https://qtxasset.com/quartz/qcloud5/fiercefavicon.ico",
"previewImage": "https://qtxasset.com/quartz/qcloud5/media/image/Novavaximage.png?VersionId=f9UDAFMgagg4Y_igFXWmnc3TJKOoTl_k",
"publishedDate": "2022-03-10T13:30:00.000Z",
"readerable": true
}

68
packages/readabilityjs/test/test-pages/fiercepharma/expected.html Normal file
View file

@ -0,0 +1,68 @@
<div id="readability-page-1" class="page">
<div data-v-0ba5eb0f="">
<div id="article-media-row" data-ga-read-media-recorded="1" data-v-0ba5eb0f="">
<figure>
<figure data-v-2243ef14="">
<img src="https://qtxasset.com/cdn-cgi/image/w=850,h=478,f=auto,fit=crop,g=0.5x0.5/https://qtxasset.com/quartz/qcloud5/media/image/Novavaximage.png?VersionId=f9UDAFMgagg4Y_igFXWmnc3TJKOoTl_k" alt="Novavax" width="850" height="478" data-v-2243ef14="">
</figure>
<figcaption class="caption"> Novavax is betting its reliance on traditional vaccine technology can bring several missed population groups back on board for their COVID shots. (Novavax ) </figcaption>
</figure>
<!---->
<!---->
</div>
<div id="article-body-row" data-v-0ba5eb0f="" data-ga-read-body-start-recorded="1">
<p> Pfizer, Moderna and Johnson &amp; Johnson were quickest off the mark in getting COVID vaccines into American arms, but Novavax is hoping to add another pandemic vaccine to the U.S. mix soon—and it's pushing new campaigns to get the word out. </p>
<p> The biopharma, which has approvals and authorizations in Europe and around the world, is now on the cusp of a potential green light in the U.S. And with a market comes the need for marketing. </p>
<p> But because it still has no U.S. approval—and it cannot under law advertise to consumers&nbsp;in Europe—Novavax&nbsp;is launching two new global, unbranded vaccine education programs: "We Do Vaccines" and "Know Our Vax." They're designed to offer up vaccine information and&nbsp;"explain Novavax’ commitment to vaccine development and innovation,” the company told Fierce Pharma Marketing. </p>
<p> The main message of the campaign is that “people have options when it comes to their vaccine,” Silvia Taylor, senior vice president of&nbsp;global corporate affairs at Novavax, said in an interview. “We want people to understand that we have this vaccine, and that this vaccine is different.” </p>
<div data-embed-button="node" data-entity-embed-display="view_mode:node.related_content" data-entity-type="node" data-entity-uuid="4ba732ba-8a0c-43f5-bed9-e0aa30cff883" data-langcode="en">
<p>
<h3> Related </h3>
</p>
</div>
<p> Novavax&nbsp;knows it has some tough competition—Pfizer and Moderna's vaccines dominate the U.S. market—but the small biotech is eyeing certain market niches: The "vaccine hesitant" who might be leery of the brand-new mRNA tech in Pfizer and Moderna's shots, and children. And it does have a strong, vocal following online that's eagerly awaiting a U.S. decision. </p>
<p> Nuvaxovid&nbsp;taps&nbsp;older tech that's been used in influenza shots and others for decades. The&nbsp;vaccine contains a version of the SARS-CoV-2 spike protein made in the lab as well as an adjuvant, which is a booster ingredient designed to&nbsp;strengthen&nbsp;immune responses to the vaccine. </p>
<h3> A new option </h3>
<p> Pfizer and Moderna's shots obviously weren't the only two vaccines in the U.S.: Johnson &amp; Johnson’s single-dose vaccine alternative also&nbsp;uses older vaccine technology. But it fell out of favor amid weakening efficacy and&nbsp;major manufacturing issues. Then, late last year, a Centers for Disease Control and Prevention panel recommended it <a href="https://www.fiercepharma.com/pharma/citing-risk-potentially-deadly-blood-clots-cdc-vaccine-advisors-recommend-mrna-pandemic">should be sidelined</a>&nbsp;because of serious safety concerns. </p>
<p> AstraZeneca's&nbsp;COVID vaccine—which itself uses&nbsp;more traditional vaccine technology—hasn’t been approved in the U.S.&nbsp; </p>
<p> Enter Novavax, now&nbsp;looking to position its vaccine as an mRNA alternative. </p>
<p> People hesitant about vaccinations may not want an&nbsp;mRNA vaccine because it's new technology, without years of proven safety behind it. But they might use an older, “tried and tested” tech, as Novavax puts it. &nbsp; </p>
<p> “There’s a recognition; a familiarity with this type of [protein-based] vaccine technology that many are comfortable with, and would have had with HPV, shingles and flu shots,”&nbsp;Taylor said. </p>
<p> “There is a segment of the population that are the so-called vaccine hesitant; they are the people we know are waiting for our vaccine. Never before have I seen a company and a product so closely followed, and that’s a big opportunity," Taylor said. </p>
<p> "There are people who want to know they can have a new option; they want to know who is making that option," she added. "So, these two education campaigns are set up to really help people understand that.” </p>
<div data-embed-button="node" data-entity-embed-display="view_mode:node.related_content" data-entity-type="node" data-entity-uuid="b9513585-1462-4bb6-bd66-5513a14a5a4b" data-langcode="en">
<p>
<h3> Related </h3>
</p>
</div>
<p> She added that people are telling the company directly that access to&nbsp;Nuvaxovid “will convince them to get their vaccine. So that’s the first target audience for us.” </p>
<p> That group not only includes people getting their first shots but also those who may need boosters&nbsp;but&nbsp;put them off because of concerns about&nbsp;mRNA. </p>
<p> And the choice goes both ways: Not only does Novavax want consumers to have a choice, they want to arm doctors with a different shot for their vaccine arsenal. </p>
<p> Novavax is also targeting the pediatric&nbsp;population. There are questions about how well mRNA vaccines&nbsp;work in younger children. There are also safety concerns, notably&nbsp;the rates of myocarditis in young and adolescent boys, who appear to be more at risk from this condition, which can cause dangerous inflammation of the heart. </p>
<p> Taylor believes Nuvaxovid can be a safe and efficacious second choice for children and adolescents outside of mRNA. “When you are talking to caregivers, there are certain considerations that are going to be front and center for them: So, tolerability and efficacy and the big question, how will it be tolerated by my child? That becomes very important, and that’s also the market we are starting to make inroads in," Taylor said. </p>
<p> The education program route is one well-traveled by pharmas.&nbsp;In this case, it allows Novavax to talk up vaccines—and itself—without running afoul of rules against branded advertising.&nbsp;And awareness campaigns&nbsp;help prime the pump ahead of what could be branded&nbsp;DTC campaigns if and when&nbsp;the shot wins&nbsp;full FDA approval. </p>
<div data-embed-button="node" data-entity-embed-display="view_mode:node.related_content" data-entity-type="node" data-entity-uuid="0f6ab279-b69b-4c81-9d31-bbd99378281e" data-langcode="en">
<p>
<h3> Related </h3>
</p>
</div>
<p> The "We Do Vaccines"&nbsp;program&nbsp;offers up educational information about&nbsp;common vaccine types&nbsp;and how they work, how vaccines are made and tested, and how Novavax believes its approach to technology makes its vaccines different. </p>
<p> It has an <a href="https://www.wedovaccines.com/about-vaccines#how">accompanying website</a> that's a straightforward look at the different types of vaccine technologies&nbsp;and how they can help stop the spread of certain infectious diseases, from COVID to influenza. This particular campaign is aimed at consumers, Taylor said. </p>
<p> Novavax’s name is on the website, though not prominently, and it doesn’t directly talk about the COVID shot. But the site does link to a <a href="https://www.wedovaccines.com/about-novavax">second site</a> that dives much more deeply into the protein technology Novavax uses for the COVID&nbsp;vaccine, approved with the brand name&nbsp;Nuvaxovid in Europe. (The name hasn’t been confirmed in the U.S.&nbsp;yet.) </p>
<p> The "Know Our Vax"&nbsp;program, meanwhile, targets doctors and other healthcare professionals with&nbsp;educational information about Novavax, its global approach and technology. This&nbsp;campaign's <a href="https://www.knowourvax.com/#vaccine-program">website</a>&nbsp;talks a little about Novavax itself and its history—and more about&nbsp;its vaccine tech and its pipeline,&nbsp;which includes work on other respiratory diseases.&nbsp; </p>
<p> Both sites invite visitors&nbsp;to sign up for “vaccine updates” from the company. Novavax said it used an agency to create the campaigns, though it did not name which one. </p>
<p> Pfizer and Moderna have both been relatively quiet on the marketing front. Neither want to talk to journalists about their marketing or education plans (at least this one). Marketing wasn't allowed while they were sold under emergency authorization,&nbsp;but now that they have&nbsp;full FDA approval, they can. Still,&nbsp;Pfizer has over the past four months been releasing a series of new DTC ads&nbsp;similar in tone to what Novavax is doing. </p>
<p> In its first series of ads, which first aired late last year, <a href="https://www.fiercepharma.com/marketing/pfizer-biontech-select-comirnaty-as-brand-name-for-covid-19-vaccine">Pfizer</a> doesn’t mention the words “COVID-19” or “vaccine,” but rather takes the viewer to “the pursuit of normal” and features the deliciously mundane aspects of everyday life that vaccines have allowed to return. </p>
<div data-embed-button="node" data-entity-embed-display="view_mode:node.related_content" data-entity-type="node" data-entity-uuid="e3809caf-199c-40c8-a3ea-10c1c77db700" data-langcode="en">
<p>
<h3> Related </h3>
</p>
</div>
<p> While nearing the finish line in the U.S., Novavax still has some way to go to actually cross it. First, the biopharma has been around 34 years now, but until last year, never saw a drug authorized or approved. As it nears a possible green light in its native U.S., that’s a lot of pressure for management to deliver. </p>
<p> And it’s struggled to get here: Manufacturing issues have hampered delivery of its vaccine, with the company reportedly <a href="https://www.fiercepharma.com/manufacturing/novavax-struggling-to-make-covid-19-vax-to-meet-global-demand-report">struggling to meet quality standards</a>. It has since said it has cleared up any remaining issues with the FDA in a recent <a href="https://www.wsj.com/articles/novavaxs-covid-19-vaccine-moves-closer-to-fda-authorization-decision-11646562601" target="_blank">interview</a> with The Wall Street Journal. </p>
<p> Novavax is also preparing for a&nbsp;full BLA filing in the second half of the year; should it win that approval, it can really hit the gas on its DTC plans. Taylor said Novavax isn’t thinking too far ahead in terms of marketing after an approval, saying they “are solely focused on delivering these new campaigns and our vaccine around the world.” </p>
</div>
<!---->
<!---->
</div>
</div>

1575
packages/readabilityjs/test/test-pages/fiercepharma/source.html Normal file
View file

File diff suppressed because one or more lines are too long

1
packages/readabilityjs/test/test-pages/fiercepharma/url.txt Normal file
View file

@ -0,0 +1 @@
https://www.fiercepharma.com/marketing/novavax-eyeing-covid-vaccine-hesitant-and-kids-unveils-new-education-campaigns-nuvaxovid

11
packages/readabilityjs/test/test-pages/ft.com/expected-metadata.json Normal file
View file

@ -0,0 +1,11 @@
{
"title": "Tiger Global slumps more than 40% in first four months of 2022",
"byline": "Harriet Agnew, Robin Wigglesworth, Laurence Fletcher",
"dir": null,
"excerpt": "Tumble in tech stocks deals fresh blow to Chase Coleman’s flagship hedge fund",
"siteName": "Financial Times",
"siteIcon": "https://www.ft.com/__origami/service/image/v2/images/raw/ftlogo-v1%3Abrand-ft-logo-square-coloured?source=update-logos&format=svg",
"previewImage": "https://d1e00ek4ebabms.cloudfront.net/production/fcdb88dd-1d9b-4b1a-b6ea-590293ae813f.jpg",
"publishedDate": "2022-05-03T17:53:22.094Z",
"readerable": true
}

27
packages/readabilityjs/test/test-pages/ft.com/expected.html Normal file
View file

@ -0,0 +1,27 @@
<div id="readability-page-1" class="page">
<div id="o-topper">
<p>Tumble in tech stocks deals fresh blow to Chase Coleman’s flagship hedge fund</p>
</div>
<article id="site-content" role="main" data-content-id="31ff7d44-f8a4-4cde-989e-50aa08301155" data-syndicatable="yes" data-access-level="subscribed" data-article-type="no-full-width-graphics">
<div>
<figure><img src="https://www.ft.com/__origami/service/image/v2/images/raw/https%3A%2F%2Fd1e00ek4ebabms.cloudfront.net%2Fproduction%2Ffcdb88dd-1d9b-4b1a-b6ea-590293ae813f.jpg?fit=scale-down&amp;source=next&amp;width=700" data-id="https://api.ft.com/content/fcdb88dd-1d9b-4b1a-b6ea-590293ae813f" data-image-type="image" data-original-image-width="2400" data-original-image-height="1350" alt="Tiger Global logo on a smartphone" srcset="https://www.ft.com/__origami/service/image/v2/images/raw/https%3A%2F%2Fd1e00ek4ebabms.cloudfront.net%2Fproduction%2Ffcdb88dd-1d9b-4b1a-b6ea-590293ae813f.jpg?fit=scale-down&amp;source=next&amp;width=700 700w, https://www.ft.com/__origami/service/image/v2/images/raw/https%3A%2F%2Fd1e00ek4ebabms.cloudfront.net%2Fproduction%2Ffcdb88dd-1d9b-4b1a-b6ea-590293ae813f.jpg?fit=scale-down&amp;source=next&amp;width=500 500w, https://www.ft.com/__origami/service/image/v2/images/raw/https%3A%2F%2Fd1e00ek4ebabms.cloudfront.net%2Fproduction%2Ffcdb88dd-1d9b-4b1a-b6ea-590293ae813f.jpg?fit=scale-down&amp;source=next&amp;width=300 300w" sizes="(min-width: 76.25em) 700px, (min-width: 61.25em) 620px, (min-width: 46.25em) 700px, calc(100vw - 20px)">
<figcaption>Tiger Global’s hedge fund lost 15.2% in April, according to a letter to investors, taking it down 43.7% in the first four months of 2022 © Timon Schneider/Alamy</figcaption>
</figure>
</div>
<div data-attribute="article-content-body" data-trackable="article-body" data-component="article-body">
<p>Tiger Global’s flagship hedge fund was dealt a fresh blow in April and is down more than 40 per cent this year, in the latest sign of how star investors who rode the big rally in tech stocks have been wrongfooted by a sharp pullback.</p>
<p>The losses marked a dramatic fall from grace for Tiger Global’s founder Chase Coleman, who has emerged as one of the world’s most prominent growth investors after founding the firm in 2001.</p>
<p>Tiger Global’s hedge fund lost 15.2 per cent in April, according to a person familiar with the matter, taking it down 43.7 per cent in the first four months of 2022. This year’s losses and a 7 per cent reversal in 2021 mean that the Tiger Global hedge fund’s gain of 48 per cent in 2020 has been completely erased. </p>
<p>The group’s long-only fund lost 24.9 per cent in April and is down 51.7 per cent in 2022, the person said. Across the two funds, the firm managed about $35bn in public equities at the end of 2021. Tiger Global declined to comment on the performance numbers, which were first reported by Bloomberg.</p>
<p>Last month was a <a href="https://www.ft.com/content/2437bc7a-3758-4660-900c-34f83654bd72" data-trackable="link">miserable one</a> for many hedge funds, with both global bond and stock markets losing money as investors fretted about high inflation pushing central banks into an aggressive interest rate hiking cycle.</p>
<p>The so-called <a href="https://www.ft.com/content/e1d1c558-9a87-4843-9cd8-29ab203b7911" data-trackable="link">Tiger Cub hedge funds</a>, spawned from Julian Robertson’s Tiger Management and big investors in tech stocks, have been <a href="https://www.ft.com/content/4af5c796-b8a5-4046-baf4-c9c7411929c2" data-trackable="link">hit particularly hard</a> in recent months as a boom in high-growth technology stocks that was accelerated by the pandemic has <a href="https://www.ft.com/content/6211eadb-c101-46d1-98b2-95347bd9d413" data-trackable="link">turned into a bear market</a>. This has put the brakes on one of the most lucrative trades in recent years.</p>
<p>The Nasdaq Composite lost 13.3 per cent in April, its <a href="https://www.ft.com/content/fa46a15f-9144-4e7c-a86f-7c75c87611aa" data-trackable="link">worst monthly performance</a> since 2008. Despite a bounce in recent days, the tech-heavy benchmark has fallen almost 22 per cent since its November peak.</p>
<p>In a brief letter to investors, the Tiger Global investment team said: “April added to a very disappointing start to 2022 for our public funds. Markets have not been co-operative given the macroeconomic backdrop, but we do not believe in excuses and so will not offer any.”</p>
<p>The letter added that the team was managing the portfolio in the ways it described in its first-quarter letter. “[We] know we will look back on this as one point in time on a long journey,” it said.</p>
<p>By the start of last year, Coleman was ranked the 14th best-performing hedge fund manager ever after a bumper year in 2020 in which he made $10.4bn of gains for investors, according to research by LCH Investments. But a bruising few months meant <a href="https://www.ft.com/content/c155a678-8f38-4f95-b4ed-1fe6473804de" data-trackable="link">he lost $1.5bn</a> for investors last year, pushing him down the rankings even before this year’s fall.</p>
<p>This year’s losses come as expectations of a sharp rise in interest rates have pushed investors out of stocks with high rates of growth but little in the way of earnings. Higher interest rates make such companies’ future cash flows look relatively less attractive.</p>
<p>Other high-profile casualties among growth investors include Baillie Gifford’s <a href="https://www.ft.com/content/e962ac7d-2bf7-415e-bb3f-ffed3238463f" data-trackable="link">Scottish Mortgage Investment Trust</a> and Cathie Wood’s flagship <a href="https://www.ft.com/content/a93f4de2-35d2-44e1-a6a1-0000cba0dd4d" data-trackable="link">Ark Innovation ETF</a>, both of which have nursed big losses in the past 12 months.</p>
<p>Tiger Global is also a prolific investor in private markets and holds stakes in more billion-dollar private start-ups than any other firm, according to CB Insights. It has recently gained notoriety for something else: a fast-paced style of investing that has unsettled the clubby ranks of Silicon Valley venture capitalists.</p>
</div>
</article>
</div>

1
packages/readabilityjs/test/test-pages/ft.com/source.html Normal file
View file

File diff suppressed because one or more lines are too long

1
packages/readabilityjs/test/test-pages/ft.com/url.txt Normal file
View file

@ -0,0 +1 @@
https://on.ft.com/3LSG1iw

11
packages/web/components/patterns/LibraryCards/HighlightItemCard.tsx
View file

@ -19,12 +19,13 @@ export function HighlightItemCard(props: HighlightItemCardProps): JSX.Element {
css={{
p: '$2',
height: '100%',
maxWidth: '498px',
borderRadius: '6px',
width: '100%',
borderRadius: '0px',
cursor: 'pointer',
wordBreak: 'break-word',
overflow: 'clip',
border: '1px solid $grayBorder',
borderBottom: 'none',
boxShadow: '0px 3px 11px rgba(32, 31, 29, 0.04)',
bg: '$grayBg',
'&:focus': {
@ -49,6 +50,7 @@ export function HighlightItemCard(props: HighlightItemCardProps): JSX.Element {
css={{
background: '$highlightBackground',
color: '$highlightText',
fontSize: '14px',
}}
>
{props.item.quote}
@ -75,13 +77,10 @@ export function HighlightItemCard(props: HighlightItemCardProps): JSX.Element {
)}
<StyledText
css={{
marginLeft: '$2',
fontWeight: '700',
}}
>
{props.item.title
.substring(0, 50)
.concat(props.item.title.length > 50 ? '...' : '')}
{props.item.title}
</StyledText>
</HStack>
</VStack>

3
packages/web/components/templates/homeFeed/HomeFeedContainer.tsx
View file

@ -37,7 +37,6 @@ import { showErrorToast, showSuccessToast } from '../../../lib/toastHelpers'
import { ConfirmationModal } from '../../patterns/ConfirmationModal'
import { SetLabelsModal } from '../article/SetLabelsModal'
import { Label } from '../../../lib/networking/fragments/labelFragment'
import { isVipUser } from '../../../lib/featureFlag'
import { EmptyLibrary } from './EmptyLibrary'
import TopBarProgress from 'react-topbar-progress-indicator'
import { State, PageType } from '../../../lib/networking/fragments/articleFragment'
@ -601,7 +600,7 @@ function HomeFeedGrid(props: HomeFeedContentProps): JSX.Element {
searchTerm={props.searchTerm}
applySearchQuery={props.applySearchQuery}
/>
{viewerData?.me && isVipUser(viewerData?.me) && (
{viewerData?.me && (
<Box
css={{
display: 'flex',

5
packages/web/lib/networking/queries/useGetLibraryItemsQuery.tsx
View file

@ -1,12 +1,12 @@
import { gql } from 'graphql-request'
import useSWRInfinite from 'swr/infinite'
import { gqlFetcher } from '../networkHelpers'
import type { ArticleFragmentData, PageType, State } from '../fragments/articleFragment'
import type { PageType, State } from '../fragments/articleFragment'
import { ContentReader } from '../fragments/articleFragment'
import { setLinkArchivedMutation } from '../mutations/setLinkArchivedMutation'
import { deleteLinkMutation } from '../mutations/deleteLinkMutation'
import { articleReadingProgressMutation } from '../mutations/articleReadingProgressMutation'
import { Label, labelFragment } from './../fragments/labelFragment'
import { Label } from './../fragments/labelFragment'
import { showErrorToast, showSuccessToast } from '../../toastHelpers'
export type LibraryItemsQueryInput = {
@ -124,6 +124,7 @@ export function useGetLibraryItemsQuery({
shortId
quote
annotation
state
}
}
pageInfo {