mirror of
https://github.com/omnivore-app/omnivore.git
synced 2026-03-11 08:54:26 +00:00
Merge pull request #2311 from omnivore-app/feature/metadata
Feature: Set thumbnail and pre-cache all the images in the content
This commit is contained in:
commit
c0872939a7
28 changed files with 566 additions and 22 deletions
|
|
@ -2874,6 +2874,7 @@ export type UpdatePageInput = {
|
|||
byline?: InputMaybe<Scalars['String']>;
|
||||
description?: InputMaybe<Scalars['String']>;
|
||||
pageId: Scalars['ID'];
|
||||
previewImage?: InputMaybe<Scalars['String']>;
|
||||
publishedAt?: InputMaybe<Scalars['Date']>;
|
||||
savedAt?: InputMaybe<Scalars['Date']>;
|
||||
title?: InputMaybe<Scalars['String']>;
|
||||
|
|
|
|||
|
|
@ -2219,6 +2219,7 @@ input UpdatePageInput {
|
|||
byline: String
|
||||
description: String
|
||||
pageId: ID!
|
||||
previewImage: String
|
||||
publishedAt: Date
|
||||
savedAt: Date
|
||||
title: String
|
||||
|
|
|
|||
|
|
@ -247,7 +247,6 @@ export const importFromIntegrationResolver = authorized<
|
|||
)) as string
|
||||
// create a task to import all the pages
|
||||
const taskName = await enqueueImportFromIntegration(
|
||||
uid,
|
||||
integration.id,
|
||||
authToken
|
||||
)
|
||||
|
|
|
|||
|
|
@ -1,13 +1,13 @@
|
|||
import { getPageById, updatePage } from '../../elastic/pages'
|
||||
import { Page } from '../../elastic/types'
|
||||
import {
|
||||
MutationUpdatePageArgs,
|
||||
UpdatePageError,
|
||||
UpdatePageErrorCode,
|
||||
UpdatePageSuccess,
|
||||
} from '../../generated/graphql'
|
||||
import { authorized, userDataToUser } from '../../utils/helpers'
|
||||
import { getPageById, updatePage } from '../../elastic/pages'
|
||||
import { Page } from '../../entity/page'
|
||||
import { Merge } from '../../util'
|
||||
import { authorized, userDataToUser } from '../../utils/helpers'
|
||||
|
||||
export type UpdatePageSuccessPartial = Merge<
|
||||
UpdatePageSuccess,
|
||||
|
|
@ -42,6 +42,7 @@ export const updatePageResolver = authorized<
|
|||
author: input.byline ?? undefined,
|
||||
savedAt: input.savedAt ? new Date(input.savedAt) : undefined,
|
||||
publishedAt: input.publishedAt ? new Date(input.publishedAt) : undefined,
|
||||
image: input.previewImage ?? undefined,
|
||||
}
|
||||
|
||||
const updateResult = await updatePage(input.pageId, pageData, { ...ctx, uid })
|
||||
|
|
|
|||
|
|
@ -580,6 +580,7 @@ const schema = gql`
|
|||
byline: String
|
||||
savedAt: Date
|
||||
publishedAt: Date
|
||||
previewImage: String @sanitize
|
||||
}
|
||||
|
||||
type UpdatePageSuccess {
|
||||
|
|
|
|||
|
|
@ -1,3 +1,8 @@
|
|||
import normalizeUrl from 'normalize-url'
|
||||
import { PubsubClient } from '../datalayer/pubsub'
|
||||
import { createPage, getPageByParam, updatePage } from '../elastic/pages'
|
||||
import { ArticleSavingRequestStatus, Page } from '../elastic/types'
|
||||
import { enqueueThumbnailTask } from '../utils/createTask'
|
||||
import {
|
||||
generateSlug,
|
||||
stringToHash,
|
||||
|
|
@ -9,10 +14,6 @@ import {
|
|||
parsePreparedContent,
|
||||
parseUrlMetadata,
|
||||
} from '../utils/parser'
|
||||
import normalizeUrl from 'normalize-url'
|
||||
import { PubsubClient } from '../datalayer/pubsub'
|
||||
import { ArticleSavingRequestStatus, Page } from '../elastic/types'
|
||||
import { createPage, getPageByParam, updatePage } from '../elastic/pages'
|
||||
|
||||
export type SaveContext = {
|
||||
pubsub: PubsubClient
|
||||
|
|
@ -105,6 +106,18 @@ export const saveEmail = async (
|
|||
return undefined
|
||||
}
|
||||
|
||||
// create a task to update thumbnail and pre-cache all images
|
||||
try {
|
||||
const taskId = await enqueueThumbnailTask(
|
||||
ctx.uid,
|
||||
slug,
|
||||
articleToSave.content
|
||||
)
|
||||
console.debug('Created thumbnail task', taskId)
|
||||
} catch (e) {
|
||||
console.log('Failed to create thumbnail task', e)
|
||||
}
|
||||
|
||||
articleToSave.id = pageId
|
||||
|
||||
return articleToSave
|
||||
|
|
|
|||
|
|
@ -14,6 +14,7 @@ import {
|
|||
SaveResult,
|
||||
} from '../generated/graphql'
|
||||
import { DataModels } from '../resolvers/types'
|
||||
import { enqueueThumbnailTask } from '../utils/createTask'
|
||||
import {
|
||||
generateSlug,
|
||||
stringToHash,
|
||||
|
|
@ -164,6 +165,18 @@ export const savePage = async (
|
|||
pageId = newPageId
|
||||
}
|
||||
|
||||
// create a task to update thumbnail and pre-cache all images
|
||||
try {
|
||||
const taskId = await enqueueThumbnailTask(
|
||||
saver.userId,
|
||||
slug,
|
||||
articleToSave.content
|
||||
)
|
||||
console.debug('Created thumbnail task', taskId)
|
||||
} catch (e) {
|
||||
console.log('Failed to create thumbnail task', e)
|
||||
}
|
||||
|
||||
if (parseResult.highlightData) {
|
||||
const highlight = {
|
||||
updatedAt: new Date(),
|
||||
|
|
|
|||
|
|
@ -66,6 +66,7 @@ interface BackendEnv {
|
|||
integrationTaskHandlerUrl: string
|
||||
textToSpeechTaskHandlerUrl: string
|
||||
recommendationTaskHandlerUrl: string
|
||||
thumbnailTaskHandlerUrl: string
|
||||
}
|
||||
fileUpload: {
|
||||
gcsUploadBucket: string
|
||||
|
|
@ -159,6 +160,7 @@ const nullableEnvVars = [
|
|||
'GCP_LOCATION',
|
||||
'RECOMMENDATION_TASK_HANDLER_URL',
|
||||
'POCKET_CONSUMER_KEY',
|
||||
'THUMBNAIL_TASK_HANDLER_URL',
|
||||
] // Allow some vars to be null/empty
|
||||
|
||||
/* If not in GAE and Prod/QA/Demo env (f.e. on localhost/dev env), allow following env vars to be null */
|
||||
|
|
@ -245,6 +247,7 @@ export function getEnv(): BackendEnv {
|
|||
integrationTaskHandlerUrl: parse('INTEGRATION_TASK_HANDLER_URL'),
|
||||
textToSpeechTaskHandlerUrl: parse('TEXT_TO_SPEECH_TASK_HANDLER_URL'),
|
||||
recommendationTaskHandlerUrl: parse('RECOMMENDATION_TASK_HANDLER_URL'),
|
||||
thumbnailTaskHandlerUrl: parse('THUMBNAIL_TASK_HANDLER_URL'),
|
||||
}
|
||||
const imageProxy = {
|
||||
url: parse('IMAGE_PROXY_URL'),
|
||||
|
|
|
|||
|
|
@ -12,6 +12,7 @@ import {
|
|||
CreateLabelInput,
|
||||
} from '../generated/graphql'
|
||||
import { signFeatureToken } from '../services/features'
|
||||
import { generateVerificationToken } from './auth'
|
||||
import { CreateTaskError } from './errors'
|
||||
import { buildLogger } from './logger'
|
||||
import View = google.cloud.tasks.v2.Task.View
|
||||
|
|
@ -22,7 +23,7 @@ const logger = buildLogger('app.dispatch')
|
|||
const client = new CloudTasksClient()
|
||||
|
||||
const createHttpTaskWithToken = async ({
|
||||
project,
|
||||
project = process.env.GOOGLE_CLOUD_PROJECT,
|
||||
queue = env.queue.name,
|
||||
location = env.queue.location,
|
||||
taskHandlerUrl = env.queue.contentFetchUrl,
|
||||
|
|
@ -32,7 +33,7 @@ const createHttpTaskWithToken = async ({
|
|||
scheduleTime,
|
||||
requestHeaders,
|
||||
}: {
|
||||
project: string
|
||||
project?: string
|
||||
queue?: string
|
||||
location?: string
|
||||
taskHandlerUrl?: string
|
||||
|
|
@ -42,12 +43,18 @@ const createHttpTaskWithToken = async ({
|
|||
scheduleTime?: number
|
||||
requestHeaders?: Record<string, string>
|
||||
}): Promise<
|
||||
[
|
||||
protos.google.cloud.tasks.v2.ITask,
|
||||
protos.google.cloud.tasks.v2.ICreateTaskRequest | undefined,
|
||||
unknown | undefined
|
||||
]
|
||||
| [
|
||||
protos.google.cloud.tasks.v2.ITask,
|
||||
protos.google.cloud.tasks.v2.ICreateTaskRequest | undefined,
|
||||
unknown | undefined
|
||||
]
|
||||
| null
|
||||
> => {
|
||||
// If there is no Google Cloud Project Id exposed, it means that we are in local environment
|
||||
if (env.dev.isLocal || !project) {
|
||||
return null
|
||||
}
|
||||
|
||||
// Construct the fully qualified queue name.
|
||||
priority === 'low' && (queue = `${queue}-low`)
|
||||
|
||||
|
|
@ -458,7 +465,6 @@ export const enqueueRecommendation = async (
|
|||
}
|
||||
|
||||
export const enqueueImportFromIntegration = async (
|
||||
userId: string,
|
||||
integrationId: string,
|
||||
authToken: string
|
||||
): Promise<string> => {
|
||||
|
|
@ -493,4 +499,51 @@ export const enqueueImportFromIntegration = async (
|
|||
return createdTasks[0].name
|
||||
}
|
||||
|
||||
export const enqueueThumbnailTask = async (
|
||||
userId: string,
|
||||
slug: string,
|
||||
content: string
|
||||
): Promise<string> => {
|
||||
const { GOOGLE_CLOUD_PROJECT } = process.env
|
||||
const payload = {
|
||||
userId,
|
||||
slug,
|
||||
content,
|
||||
}
|
||||
|
||||
const requestHeaders = {
|
||||
Authorization: generateVerificationToken(userId),
|
||||
}
|
||||
|
||||
// If there is no Google Cloud Project Id exposed, it means that we are in local environment
|
||||
if (env.dev.isLocal || !GOOGLE_CLOUD_PROJECT) {
|
||||
// Calling the handler function directly.
|
||||
setTimeout(() => {
|
||||
axios
|
||||
.post(env.queue.thumbnailTaskHandlerUrl, payload, {
|
||||
headers: requestHeaders,
|
||||
})
|
||||
.catch((error) => {
|
||||
console.error(error)
|
||||
})
|
||||
}, 0)
|
||||
return ''
|
||||
}
|
||||
|
||||
const createdTasks = await createHttpTaskWithToken({
|
||||
payload,
|
||||
taskHandlerUrl: env.queue.thumbnailTaskHandlerUrl,
|
||||
requestHeaders,
|
||||
})
|
||||
|
||||
if (!createdTasks || !createdTasks[0].name) {
|
||||
logger.error(`Unable to get the name of the task`, {
|
||||
payload,
|
||||
createdTasks,
|
||||
})
|
||||
throw new CreateTaskError(`Unable to get the name of the task`)
|
||||
}
|
||||
return createdTasks[0].name
|
||||
}
|
||||
|
||||
export default createHttpTaskWithToken
|
||||
|
|
|
|||
|
|
@ -83,7 +83,7 @@ export function authorized<
|
|||
return (parent, args, ctx, info) => {
|
||||
const { claims } = ctx
|
||||
if (claims?.uid) {
|
||||
return resolver(parent, args, { ...ctx, claims }, info)
|
||||
return resolver(parent, args, { ...ctx, claims, uid: claims.uid }, info)
|
||||
}
|
||||
return { errorCodes: ['UNAUTHORIZED'] } as TError
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,9 +1,9 @@
|
|||
import { createTestUser, deleteTestUser } from '../db'
|
||||
import { createTestElasticPage, graphqlRequest, request } from '../util'
|
||||
import { expect } from 'chai'
|
||||
import 'mocha'
|
||||
import { User } from '../../src/entity/user'
|
||||
import { Page } from '../../src/elastic/types'
|
||||
import { User } from '../../src/entity/user'
|
||||
import { createTestUser, deleteTestUser } from '../db'
|
||||
import { createTestElasticPage, graphqlRequest, request } from '../util'
|
||||
|
||||
describe('Update API', () => {
|
||||
let user: User
|
||||
|
|
@ -28,8 +28,9 @@ describe('Update API', () => {
|
|||
|
||||
describe('update page', () => {
|
||||
let query: string
|
||||
let title = 'New Title'
|
||||
let description = 'New Description'
|
||||
const title = 'New Title'
|
||||
const description = 'New Description'
|
||||
const previewImage = 'https://omnivore.app/image.png'
|
||||
|
||||
beforeEach(() => {
|
||||
query = `
|
||||
|
|
@ -39,12 +40,14 @@ describe('Update API', () => {
|
|||
pageId: "${page.id}"
|
||||
title: "${title}"
|
||||
description: "${description}"
|
||||
previewImage: "${previewImage}"
|
||||
}
|
||||
) {
|
||||
... on UpdatePageSuccess {
|
||||
updatedPage {
|
||||
title
|
||||
description
|
||||
image
|
||||
}
|
||||
}
|
||||
... on UpdatePageError {
|
||||
|
|
@ -61,6 +64,7 @@ describe('Update API', () => {
|
|||
const updatedPage = res?.body.data.updatePage.updatedPage
|
||||
expect(updatedPage?.title).to.eql(title)
|
||||
expect(updatedPage?.description).to.eql(description)
|
||||
expect(updatedPage?.image).to.eql(previewImage)
|
||||
})
|
||||
})
|
||||
})
|
||||
|
|
|
|||
5
packages/thumbnail-handler/.dockerignore
Normal file
5
packages/thumbnail-handler/.dockerignore
Normal file
|
|
@ -0,0 +1,5 @@
|
|||
node_modules
|
||||
build
|
||||
.env*
|
||||
Dockerfile
|
||||
.dockerignore
|
||||
2
packages/thumbnail-handler/.eslintignore
Normal file
2
packages/thumbnail-handler/.eslintignore
Normal file
|
|
@ -0,0 +1,2 @@
|
|||
node_modules/
|
||||
build/
|
||||
6
packages/thumbnail-handler/.eslintrc
Normal file
6
packages/thumbnail-handler/.eslintrc
Normal file
|
|
@ -0,0 +1,6 @@
|
|||
{
|
||||
"extends": "../../.eslintrc",
|
||||
"parserOptions": {
|
||||
"project": "tsconfig.json"
|
||||
}
|
||||
}
|
||||
16
packages/thumbnail-handler/.gcloudignore
Normal file
16
packages/thumbnail-handler/.gcloudignore
Normal file
|
|
@ -0,0 +1,16 @@
|
|||
# This file specifies files that are *not* uploaded to Google Cloud Platform
|
||||
# using gcloud. It follows the same syntax as .gitignore, with the addition of
|
||||
# "#!include" directives (which insert the entries of the given .gitignore-style
|
||||
# file at that point).
|
||||
#
|
||||
# For more information, run:
|
||||
# $ gcloud topic gcloudignore
|
||||
#
|
||||
.gcloudignore
|
||||
# If you would like to upload your .git directory, .gitignore file or files
|
||||
# from your .gitignore file, remove the corresponding line
|
||||
# below:
|
||||
.git
|
||||
.gitignore
|
||||
|
||||
node_modules
|
||||
26
packages/thumbnail-handler/Dockerfile
Normal file
26
packages/thumbnail-handler/Dockerfile
Normal file
|
|
@ -0,0 +1,26 @@
|
|||
FROM node:14.18-alpine
|
||||
|
||||
# Run everything after as non-privileged user.
|
||||
WORKDIR /app
|
||||
|
||||
COPY package.json .
|
||||
COPY yarn.lock .
|
||||
COPY tsconfig.json .
|
||||
COPY .eslintrc .
|
||||
|
||||
COPY /packages/thumbnail-handler/package.json ./packages/thumbnail-handler/package.json
|
||||
|
||||
RUN yarn install --pure-lockfile
|
||||
|
||||
ADD /packages/thumbnail-handler ./packages/thumbnail-handler
|
||||
RUN yarn workspace @omnivore/thumbnail-handler build
|
||||
|
||||
# After building, fetch the production dependencies
|
||||
RUN rm -rf /app/packages/thumbnail-handler/node_modules
|
||||
RUN rm -rf /app/node_modules
|
||||
RUN yarn install --pure-lockfile --production
|
||||
|
||||
EXPOSE 8080
|
||||
|
||||
CMD ["yarn", "workspace", "@omnivore/thumbnail-handler", "start"]
|
||||
|
||||
5
packages/thumbnail-handler/mocha-config.json
Normal file
5
packages/thumbnail-handler/mocha-config.json
Normal file
|
|
@ -0,0 +1,5 @@
|
|||
{
|
||||
"extension": ["ts"],
|
||||
"spec": "test/**/*.test.ts",
|
||||
"require": "test/babel-register.js"
|
||||
}
|
||||
32
packages/thumbnail-handler/package.json
Normal file
32
packages/thumbnail-handler/package.json
Normal file
|
|
@ -0,0 +1,32 @@
|
|||
{
|
||||
"name": "@omnivore/thumbnail-handler",
|
||||
"version": "1.0.0",
|
||||
"main": "build/src/index.js",
|
||||
"files": [
|
||||
"build/src"
|
||||
],
|
||||
"license": "Apache-2.0",
|
||||
"scripts": {
|
||||
"test": "yarn mocha -r ts-node/register --config mocha-config.json",
|
||||
"lint": "eslint src --ext ts,js,tsx,jsx",
|
||||
"compile": "tsc",
|
||||
"build": "tsc",
|
||||
"start": "functions-framework --target=thumbnailHandler",
|
||||
"dev": "concurrently \"tsc -w\" \"nodemon --watch ./build/ --exec npm run start\""
|
||||
},
|
||||
"devDependencies": {
|
||||
"chai": "^4.3.6",
|
||||
"eslint-plugin-prettier": "^4.0.0",
|
||||
"mocha": "^10.0.0",
|
||||
"nock": "^13.3.1"
|
||||
},
|
||||
"dependencies": {
|
||||
"@google-cloud/functions-framework": "3.1.2",
|
||||
"@sentry/serverless": "^6.16.1",
|
||||
"axios": "^1.4.0",
|
||||
"dotenv": "^16.0.1",
|
||||
"image-size": "^1.0.2",
|
||||
"jsonwebtoken": "^8.5.1",
|
||||
"linkedom": "^0.14.26"
|
||||
}
|
||||
}
|
||||
274
packages/thumbnail-handler/src/index.ts
Normal file
274
packages/thumbnail-handler/src/index.ts
Normal file
|
|
@ -0,0 +1,274 @@
|
|||
import * as Sentry from '@sentry/serverless'
|
||||
import axios from 'axios'
|
||||
import * as dotenv from 'dotenv' // see https://github.com/motdotla/dotenv#how-do-i-use-dotenv-with-import
|
||||
import sizeOf from 'image-size'
|
||||
import * as jwt from 'jsonwebtoken'
|
||||
import { parseHTML } from 'linkedom'
|
||||
import { promisify } from 'util'
|
||||
|
||||
interface ArticleResponse {
|
||||
data: {
|
||||
article: {
|
||||
article: Page
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
interface Page {
|
||||
id: string
|
||||
content: string
|
||||
image?: string
|
||||
}
|
||||
|
||||
interface UpdatePageResponse {
|
||||
data: {
|
||||
updatePage: {
|
||||
updatedPage: Page
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
interface ThumbnailRequest {
|
||||
slug: string
|
||||
content: string
|
||||
}
|
||||
|
||||
dotenv.config()
|
||||
Sentry.GCPFunction.init({
|
||||
dsn: process.env.SENTRY_DSN,
|
||||
tracesSampleRate: 0,
|
||||
})
|
||||
|
||||
const signToken = promisify(jwt.sign)
|
||||
|
||||
const articleQuery = async (userId: string, slug: string): Promise<Page> => {
|
||||
const JWT_SECRET = process.env.JWT_SECRET
|
||||
const REST_BACKEND_ENDPOINT = process.env.REST_BACKEND_ENDPOINT
|
||||
|
||||
if (!JWT_SECRET || !REST_BACKEND_ENDPOINT) {
|
||||
throw 'Environment not configured correctly'
|
||||
}
|
||||
|
||||
const data = JSON.stringify({
|
||||
query: `query article ($username: String!, $slug: String!){
|
||||
article(username: $username, slug: $slug){
|
||||
... on ArticleSuccess {
|
||||
article {
|
||||
id
|
||||
content
|
||||
image
|
||||
}
|
||||
}
|
||||
... on ArticleError {
|
||||
errorCodes
|
||||
}
|
||||
}
|
||||
}`,
|
||||
variables: {
|
||||
username: 'me',
|
||||
slug,
|
||||
},
|
||||
})
|
||||
const auth = (await signToken({ uid: userId }, JWT_SECRET)) as string
|
||||
|
||||
const response = await axios.post<ArticleResponse>(
|
||||
`${REST_BACKEND_ENDPOINT}/graphql`,
|
||||
data,
|
||||
{
|
||||
headers: {
|
||||
Cookie: `auth=${auth};`,
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
return response.data.data.article.article
|
||||
}
|
||||
|
||||
const updatePageMutation = async (
|
||||
userId: string,
|
||||
pageId: string,
|
||||
image: string
|
||||
) => {
|
||||
const JWT_SECRET = process.env.JWT_SECRET
|
||||
const REST_BACKEND_ENDPOINT = process.env.REST_BACKEND_ENDPOINT
|
||||
|
||||
if (!JWT_SECRET || !REST_BACKEND_ENDPOINT) {
|
||||
throw 'Environment not configured correctly'
|
||||
}
|
||||
|
||||
const data = JSON.stringify({
|
||||
query: `mutation UpdatePage ($input: UpdatePageInput!) {
|
||||
updatePage(input: $input) {
|
||||
... on UpdatePageSuccess {
|
||||
updatedPage {
|
||||
id
|
||||
}
|
||||
}
|
||||
... on UpdatePageError {
|
||||
errorCodes
|
||||
}
|
||||
}
|
||||
}`,
|
||||
variables: {
|
||||
input: {
|
||||
pageId,
|
||||
previewImage: image,
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
const auth = (await signToken({ uid: userId }, JWT_SECRET)) as string
|
||||
const response = await axios.post<UpdatePageResponse>(
|
||||
`${REST_BACKEND_ENDPOINT}/graphql`,
|
||||
data,
|
||||
{
|
||||
headers: {
|
||||
Cookie: `auth=${auth};`,
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
return !!response.data.data.updatePage
|
||||
}
|
||||
|
||||
const isThumbnailRequest = (body: any): body is ThumbnailRequest => {
|
||||
return 'slug' in body && 'content' in body
|
||||
}
|
||||
|
||||
const getImageSize = async (url: string): Promise<[number, number] | null> => {
|
||||
try {
|
||||
// get image file by url
|
||||
const response = await axios.get(url, {
|
||||
responseType: 'arraybuffer',
|
||||
})
|
||||
|
||||
// eslint-disable-next-line @typescript-eslint/no-unsafe-argument
|
||||
const buffer = Buffer.from(response.data, 'binary')
|
||||
|
||||
// get image size
|
||||
const { width, height } = sizeOf(buffer)
|
||||
|
||||
if (!width || !height) {
|
||||
return null
|
||||
}
|
||||
|
||||
return [width, height]
|
||||
} catch (e) {
|
||||
console.log(e)
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
// credit: https://github.com/reddit-archive/reddit/blob/753b17407e9a9dca09558526805922de24133d53/r2/r2/lib/media.py#L706
|
||||
export const findThumbnail = async (
|
||||
content: string
|
||||
): Promise<string | null> => {
|
||||
const dom = parseHTML(content).document
|
||||
|
||||
// find the largest and squarest image as the thumbnail
|
||||
// and pre-cache all images
|
||||
const images = dom.querySelectorAll('img[src]')
|
||||
if (!images || images.length === 0) {
|
||||
console.debug('no images')
|
||||
return null
|
||||
}
|
||||
|
||||
let thumbnail = null
|
||||
let largestArea = 0
|
||||
for await (const image of Array.from(images)) {
|
||||
const src = image.getAttribute('src')
|
||||
if (!src) {
|
||||
continue
|
||||
}
|
||||
|
||||
const size = await getImageSize(src)
|
||||
if (!size) {
|
||||
continue
|
||||
}
|
||||
|
||||
let area = size[0] * size[1]
|
||||
|
||||
// ignore small images
|
||||
if (area < 5000) {
|
||||
console.debug('ignore small', src)
|
||||
continue
|
||||
}
|
||||
|
||||
// penalize excessively long/wide images
|
||||
const ratio = Math.max(...size) / Math.min(...size)
|
||||
if (ratio > 1.5) {
|
||||
console.debug('penalizing long/wide', src)
|
||||
area /= ratio * 2
|
||||
}
|
||||
|
||||
// penalize images with "sprite" in their name
|
||||
if (src.toLowerCase().includes('sprite')) {
|
||||
console.debug('penalizing sprite', src)
|
||||
area /= 10
|
||||
}
|
||||
|
||||
if (area > largestArea) {
|
||||
largestArea = area
|
||||
thumbnail = src
|
||||
}
|
||||
}
|
||||
|
||||
return thumbnail
|
||||
}
|
||||
|
||||
/**
|
||||
* request structure
|
||||
* {
|
||||
* userId: string
|
||||
* slug: string
|
||||
* }
|
||||
*/
|
||||
|
||||
export const thumbnailHandler = Sentry.GCPFunction.wrapHttpFunction(
|
||||
async (req, res) => {
|
||||
const token = req.headers?.authorization
|
||||
if (!token) {
|
||||
console.debug('no token')
|
||||
return res.status(401).send('UNAUTHORIZED')
|
||||
}
|
||||
const { uid } = jwt.decode(token) as { uid: string }
|
||||
if (!uid) {
|
||||
console.debug('no uid')
|
||||
return res.status(401).send('UNAUTHORIZED')
|
||||
}
|
||||
|
||||
if (!isThumbnailRequest(req.body)) {
|
||||
console.debug('bad request')
|
||||
return res.status(400).send('BAD_REQUEST')
|
||||
}
|
||||
|
||||
const { slug, content } = req.body
|
||||
|
||||
try {
|
||||
// find thumbnail from all images & pre-cache
|
||||
const thumbnail = await findThumbnail(content)
|
||||
if (!thumbnail) {
|
||||
console.debug('no thumbnail')
|
||||
return res.status(200).send('NOT_FOUND')
|
||||
}
|
||||
|
||||
const page = await articleQuery(uid, slug)
|
||||
console.debug('find page', page.id)
|
||||
// update page with thumbnail if not already set
|
||||
if (page.image) {
|
||||
console.debug('thumbnail already set')
|
||||
return res.status(200).send('OK')
|
||||
}
|
||||
|
||||
const updated = await updatePageMutation(uid, page.id, thumbnail)
|
||||
console.debug('thumbnail updated', updated)
|
||||
|
||||
res.send('ok')
|
||||
} catch (e) {
|
||||
console.error(e)
|
||||
res.status(500).send('INTERNAL_SERVER_ERROR')
|
||||
}
|
||||
}
|
||||
)
|
||||
3
packages/thumbnail-handler/test/babel-register.js
Normal file
3
packages/thumbnail-handler/test/babel-register.js
Normal file
|
|
@ -0,0 +1,3 @@
|
|||
const register = require('@babel/register').default
|
||||
|
||||
register({ extensions: ['.ts', '.tsx', '.js', '.jsx'] })
|
||||
9
packages/thumbnail-handler/test/fixtures/findThumbnail.html
vendored
Normal file
9
packages/thumbnail-handler/test/fixtures/findThumbnail.html
vendored
Normal file
|
|
@ -0,0 +1,9 @@
|
|||
<DIV class="page" id="readability-page-1">
|
||||
<img src="https://omnivore.app/small.png" alt="small image" />
|
||||
|
||||
<img src="https://omnivore.app/large_and_square.png" alt="large and square image" />
|
||||
|
||||
<img src="https://omnivore.app/wide.png" alt="wide image" />
|
||||
|
||||
<img src="https://omnivore.app/sprite.png" alt="sprite image" />
|
||||
</DIV>
|
||||
BIN
packages/thumbnail-handler/test/fixtures/large_and_square.png
vendored
Normal file
BIN
packages/thumbnail-handler/test/fixtures/large_and_square.png
vendored
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 117 KiB |
BIN
packages/thumbnail-handler/test/fixtures/small.png
vendored
Normal file
BIN
packages/thumbnail-handler/test/fixtures/small.png
vendored
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 872 B |
BIN
packages/thumbnail-handler/test/fixtures/sprite.png
vendored
Normal file
BIN
packages/thumbnail-handler/test/fixtures/sprite.png
vendored
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 117 KiB |
BIN
packages/thumbnail-handler/test/fixtures/wide.png
vendored
Normal file
BIN
packages/thumbnail-handler/test/fixtures/wide.png
vendored
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 174 KiB |
30
packages/thumbnail-handler/test/index.test.ts
Normal file
30
packages/thumbnail-handler/test/index.test.ts
Normal file
|
|
@ -0,0 +1,30 @@
|
|||
import { expect } from 'chai'
|
||||
import fs from 'fs'
|
||||
import 'mocha'
|
||||
import nock from 'nock'
|
||||
import path from 'path'
|
||||
import { findThumbnail } from '../src'
|
||||
|
||||
describe('findThumbnail', () => {
|
||||
it('finds the largest and squarest image', async () => {
|
||||
const images = ['large_and_square', 'small', 'sprite', 'wide']
|
||||
// mock getting image by url
|
||||
images.forEach((image) => {
|
||||
nock('https://omnivore.app')
|
||||
.get(`/${image}.png`)
|
||||
.replyWithFile(200, path.join(__dirname, 'fixtures', `${image}.png`))
|
||||
})
|
||||
// get html content from file
|
||||
const content = fs.readFileSync(
|
||||
path.join(__dirname, 'fixtures', 'findThumbnail.html'),
|
||||
'utf8'
|
||||
)
|
||||
// find thumbnail
|
||||
const thumbnail = await findThumbnail(content)
|
||||
|
||||
expect(thumbnail).to.eql('https://omnivore.app/large_and_square.png')
|
||||
|
||||
// clean up
|
||||
nock.cleanAll()
|
||||
})
|
||||
})
|
||||
8
packages/thumbnail-handler/tsconfig.json
Normal file
8
packages/thumbnail-handler/tsconfig.json
Normal file
|
|
@ -0,0 +1,8 @@
|
|||
{
|
||||
"extends": "./../../tsconfig.json",
|
||||
"compilerOptions": {
|
||||
"outDir": "build",
|
||||
"rootDir": "."
|
||||
},
|
||||
"include": ["src"]
|
||||
}
|
||||
39
yarn.lock
39
yarn.lock
|
|
@ -10426,6 +10426,15 @@ axios@^1.2.0, axios@^1.2.2:
|
|||
form-data "^4.0.0"
|
||||
proxy-from-env "^1.1.0"
|
||||
|
||||
axios@^1.4.0:
|
||||
version "1.4.0"
|
||||
resolved "https://registry.yarnpkg.com/axios/-/axios-1.4.0.tgz#38a7bf1224cd308de271146038b551d725f0be1f"
|
||||
integrity sha512-S4XCWMEmzvo64T9GfvQDOXgYRDJ/wsSZc7Jvdgx5u1sd0JwsuPLqb3SYmusag+edF6ziyMensPVqLTSc1PiSEA==
|
||||
dependencies:
|
||||
follow-redirects "^1.15.0"
|
||||
form-data "^4.0.0"
|
||||
proxy-from-env "^1.1.0"
|
||||
|
||||
axobject-query@^2.2.0:
|
||||
version "2.2.0"
|
||||
resolved "https://registry.yarnpkg.com/axobject-query/-/axobject-query-2.2.0.tgz#943d47e10c0b704aa42275e20edf3722648989be"
|
||||
|
|
@ -16662,6 +16671,13 @@ ignore@^5.2.0:
|
|||
resolved "https://registry.yarnpkg.com/ignore/-/ignore-5.2.0.tgz#6d3bac8fa7fe0d45d9f9be7bac2fc279577e345a"
|
||||
integrity sha512-CmxgYGiEPCLhfLnpPp1MoRmifwEIOgjcHXxOBjv7mY96c+eWScsOP9c112ZyLdWHi0FxHjI+4uVhKYp/gcdRmQ==
|
||||
|
||||
image-size@^1.0.2:
|
||||
version "1.0.2"
|
||||
resolved "https://registry.yarnpkg.com/image-size/-/image-size-1.0.2.tgz#d778b6d0ab75b2737c1556dd631652eb963bc486"
|
||||
integrity sha512-xfOoWjceHntRb3qFCrh5ZFORYH8XCdYpASltMhZ/Q0KZiOwjdE/Yl2QCiWdwD+lygV5bMCvauzgu5PxBX/Yerg==
|
||||
dependencies:
|
||||
queue "6.0.2"
|
||||
|
||||
immediate@~3.0.5:
|
||||
version "3.0.6"
|
||||
resolved "https://registry.yarnpkg.com/immediate/-/immediate-3.0.6.tgz#9db1dbd0faf8de6fbe0f5dd5e56bb606280de69b"
|
||||
|
|
@ -18823,6 +18839,17 @@ linkedom@^0.14.21:
|
|||
htmlparser2 "^8.0.1"
|
||||
uhyphen "^0.1.0"
|
||||
|
||||
linkedom@^0.14.26:
|
||||
version "0.14.26"
|
||||
resolved "https://registry.yarnpkg.com/linkedom/-/linkedom-0.14.26.tgz#fd8ddaef1a052e1191fb2e881605a1a001409f3b"
|
||||
integrity sha512-mK6TrydfFA7phrnp+1j57ycBwFI5bGSW6YXlw9acHoqF+mP/y+FooEYYyniOt5Ot57FSKB3iwmnuQ1UUyNLm5A==
|
||||
dependencies:
|
||||
css-select "^5.1.0"
|
||||
cssom "^0.5.0"
|
||||
html-escaper "^3.0.3"
|
||||
htmlparser2 "^8.0.1"
|
||||
uhyphen "^0.2.0"
|
||||
|
||||
linkedom@^0.14.9:
|
||||
version "0.14.9"
|
||||
resolved "https://registry.yarnpkg.com/linkedom/-/linkedom-0.14.9.tgz#34c6f15eddc809406f42d8ee48cd30b0222eccb0"
|
||||
|
|
@ -23161,6 +23188,13 @@ querystring@^0.2.0:
|
|||
resolved "https://registry.yarnpkg.com/querystring/-/querystring-0.2.1.tgz#40d77615bb09d16902a85c3e38aa8b5ed761c2dd"
|
||||
integrity sha512-wkvS7mL/JMugcup3/rMitHmd9ecIGd2lhFhK9N3UUQ450h66d1r3Y9nvXzQAW1Lq+wyx61k/1pfKS5KuKiyEbg==
|
||||
|
||||
queue@6.0.2:
|
||||
version "6.0.2"
|
||||
resolved "https://registry.yarnpkg.com/queue/-/queue-6.0.2.tgz#b91525283e2315c7553d2efa18d83e76432fed65"
|
||||
integrity sha512-iHZWu+q3IdFZFX36ro/lKBkSvfkztY5Y7HMiPlOUjhupPcG2JMfst2KKEpu5XndviX/3UhFbRngUPNKtgvtZiA==
|
||||
dependencies:
|
||||
inherits "~2.0.3"
|
||||
|
||||
quick-lru@^4.0.1:
|
||||
version "4.0.1"
|
||||
resolved "https://registry.yarnpkg.com/quick-lru/-/quick-lru-4.0.1.tgz#5b8878f113a58217848c6482026c73e1ba57727f"
|
||||
|
|
@ -26919,6 +26953,11 @@ uhyphen@^0.1.0:
|
|||
resolved "https://registry.yarnpkg.com/uhyphen/-/uhyphen-0.1.0.tgz#3cc22afa790daa802b9f6789f3583108d5b4a08c"
|
||||
integrity sha512-o0QVGuFg24FK765Qdd5kk0zU/U4dEsCtN/GSiwNI9i8xsSVtjIAOdTaVhLwZ1nrbWxFVMxNDDl+9fednsOMsBw==
|
||||
|
||||
uhyphen@^0.2.0:
|
||||
version "0.2.0"
|
||||
resolved "https://registry.yarnpkg.com/uhyphen/-/uhyphen-0.2.0.tgz#8fdf0623314486e020a3c00ee5cc7a12fe722b81"
|
||||
integrity sha512-qz3o9CHXmJJPGBdqzab7qAYuW8kQGKNEuoHFYrBwV6hWIMcpAmxDLXojcHfFr9US1Pe6zUswEIJIbLI610fuqA==
|
||||
|
||||
uid-number@0.0.6:
|
||||
version "0.0.6"
|
||||
resolved "https://registry.yarnpkg.com/uid-number/-/uid-number-0.0.6.tgz#0ea10e8035e8eb5b8e4449f06da1c730663baa81"
|
||||
|
|
|
|||
Loading…
Reference in a new issue