diff --git a/packages/api/src/generated/graphql.ts b/packages/api/src/generated/graphql.ts index 7a259576f..e722f0d57 100644 --- a/packages/api/src/generated/graphql.ts +++ b/packages/api/src/generated/graphql.ts @@ -2874,6 +2874,7 @@ export type UpdatePageInput = { byline?: InputMaybe; description?: InputMaybe; pageId: Scalars['ID']; + previewImage?: InputMaybe; publishedAt?: InputMaybe; savedAt?: InputMaybe; title?: InputMaybe; diff --git a/packages/api/src/generated/schema.graphql b/packages/api/src/generated/schema.graphql index ab629b0c8..e8d6cb93c 100644 --- a/packages/api/src/generated/schema.graphql +++ b/packages/api/src/generated/schema.graphql @@ -2219,6 +2219,7 @@ input UpdatePageInput { byline: String description: String pageId: ID! + previewImage: String publishedAt: Date savedAt: Date title: String diff --git a/packages/api/src/resolvers/integrations/index.ts b/packages/api/src/resolvers/integrations/index.ts index d302aa97c..35f697d39 100644 --- a/packages/api/src/resolvers/integrations/index.ts +++ b/packages/api/src/resolvers/integrations/index.ts @@ -247,7 +247,6 @@ export const importFromIntegrationResolver = authorized< )) as string // create a task to import all the pages const taskName = await enqueueImportFromIntegration( - uid, integration.id, authToken ) diff --git a/packages/api/src/resolvers/update/index.ts b/packages/api/src/resolvers/update/index.ts index 0570e43b8..409b31771 100644 --- a/packages/api/src/resolvers/update/index.ts +++ b/packages/api/src/resolvers/update/index.ts @@ -1,13 +1,13 @@ +import { getPageById, updatePage } from '../../elastic/pages' +import { Page } from '../../elastic/types' import { MutationUpdatePageArgs, UpdatePageError, UpdatePageErrorCode, UpdatePageSuccess, } from '../../generated/graphql' -import { authorized, userDataToUser } from '../../utils/helpers' -import { getPageById, updatePage } from '../../elastic/pages' -import { Page } from '../../entity/page' import { Merge } from '../../util' +import { authorized, userDataToUser } from '../../utils/helpers' export type UpdatePageSuccessPartial = Merge< UpdatePageSuccess, @@ -42,6 +42,7 @@ export const updatePageResolver = authorized< author: input.byline ?? undefined, savedAt: input.savedAt ? new Date(input.savedAt) : undefined, publishedAt: input.publishedAt ? new Date(input.publishedAt) : undefined, + image: input.previewImage ?? undefined, } const updateResult = await updatePage(input.pageId, pageData, { ...ctx, uid }) diff --git a/packages/api/src/schema.ts b/packages/api/src/schema.ts index 34943c903..5b9801aa9 100755 --- a/packages/api/src/schema.ts +++ b/packages/api/src/schema.ts @@ -580,6 +580,7 @@ const schema = gql` byline: String savedAt: Date publishedAt: Date + previewImage: String @sanitize } type UpdatePageSuccess { diff --git a/packages/api/src/services/save_email.ts b/packages/api/src/services/save_email.ts index 1fdf423bd..7e49494b0 100644 --- a/packages/api/src/services/save_email.ts +++ b/packages/api/src/services/save_email.ts @@ -1,3 +1,8 @@ +import normalizeUrl from 'normalize-url' +import { PubsubClient } from '../datalayer/pubsub' +import { createPage, getPageByParam, updatePage } from '../elastic/pages' +import { ArticleSavingRequestStatus, Page } from '../elastic/types' +import { enqueueThumbnailTask } from '../utils/createTask' import { generateSlug, stringToHash, @@ -9,10 +14,6 @@ import { parsePreparedContent, parseUrlMetadata, } from '../utils/parser' -import normalizeUrl from 'normalize-url' -import { PubsubClient } from '../datalayer/pubsub' -import { ArticleSavingRequestStatus, Page } from '../elastic/types' -import { createPage, getPageByParam, updatePage } from '../elastic/pages' export type SaveContext = { pubsub: PubsubClient @@ -105,6 +106,18 @@ export const saveEmail = async ( return undefined } + // create a task to update thumbnail and pre-cache all images + try { + const taskId = await enqueueThumbnailTask( + ctx.uid, + slug, + articleToSave.content + ) + console.debug('Created thumbnail task', taskId) + } catch (e) { + console.log('Failed to create thumbnail task', e) + } + articleToSave.id = pageId return articleToSave diff --git a/packages/api/src/services/save_page.ts b/packages/api/src/services/save_page.ts index 4a43f6e3f..c58ded776 100644 --- a/packages/api/src/services/save_page.ts +++ b/packages/api/src/services/save_page.ts @@ -14,6 +14,7 @@ import { SaveResult, } from '../generated/graphql' import { DataModels } from '../resolvers/types' +import { enqueueThumbnailTask } from '../utils/createTask' import { generateSlug, stringToHash, @@ -164,6 +165,18 @@ export const savePage = async ( pageId = newPageId } + // create a task to update thumbnail and pre-cache all images + try { + const taskId = await enqueueThumbnailTask( + saver.userId, + slug, + articleToSave.content + ) + console.debug('Created thumbnail task', taskId) + } catch (e) { + console.log('Failed to create thumbnail task', e) + } + if (parseResult.highlightData) { const highlight = { updatedAt: new Date(), diff --git a/packages/api/src/util.ts b/packages/api/src/util.ts index f8c6f9d4c..2f2c9a5ef 100755 --- a/packages/api/src/util.ts +++ b/packages/api/src/util.ts @@ -66,6 +66,7 @@ interface BackendEnv { integrationTaskHandlerUrl: string textToSpeechTaskHandlerUrl: string recommendationTaskHandlerUrl: string + thumbnailTaskHandlerUrl: string } fileUpload: { gcsUploadBucket: string @@ -159,6 +160,7 @@ const nullableEnvVars = [ 'GCP_LOCATION', 'RECOMMENDATION_TASK_HANDLER_URL', 'POCKET_CONSUMER_KEY', + 'THUMBNAIL_TASK_HANDLER_URL', ] // Allow some vars to be null/empty /* If not in GAE and Prod/QA/Demo env (f.e. on localhost/dev env), allow following env vars to be null */ @@ -245,6 +247,7 @@ export function getEnv(): BackendEnv { integrationTaskHandlerUrl: parse('INTEGRATION_TASK_HANDLER_URL'), textToSpeechTaskHandlerUrl: parse('TEXT_TO_SPEECH_TASK_HANDLER_URL'), recommendationTaskHandlerUrl: parse('RECOMMENDATION_TASK_HANDLER_URL'), + thumbnailTaskHandlerUrl: parse('THUMBNAIL_TASK_HANDLER_URL'), } const imageProxy = { url: parse('IMAGE_PROXY_URL'), diff --git a/packages/api/src/utils/createTask.ts b/packages/api/src/utils/createTask.ts index 5d48e2a54..17dae7831 100644 --- a/packages/api/src/utils/createTask.ts +++ b/packages/api/src/utils/createTask.ts @@ -12,6 +12,7 @@ import { CreateLabelInput, } from '../generated/graphql' import { signFeatureToken } from '../services/features' +import { generateVerificationToken } from './auth' import { CreateTaskError } from './errors' import { buildLogger } from './logger' import View = google.cloud.tasks.v2.Task.View @@ -22,7 +23,7 @@ const logger = buildLogger('app.dispatch') const client = new CloudTasksClient() const createHttpTaskWithToken = async ({ - project, + project = process.env.GOOGLE_CLOUD_PROJECT, queue = env.queue.name, location = env.queue.location, taskHandlerUrl = env.queue.contentFetchUrl, @@ -32,7 +33,7 @@ const createHttpTaskWithToken = async ({ scheduleTime, requestHeaders, }: { - project: string + project?: string queue?: string location?: string taskHandlerUrl?: string @@ -42,12 +43,18 @@ const createHttpTaskWithToken = async ({ scheduleTime?: number requestHeaders?: Record }): Promise< - [ - protos.google.cloud.tasks.v2.ITask, - protos.google.cloud.tasks.v2.ICreateTaskRequest | undefined, - unknown | undefined - ] + | [ + protos.google.cloud.tasks.v2.ITask, + protos.google.cloud.tasks.v2.ICreateTaskRequest | undefined, + unknown | undefined + ] + | null > => { + // If there is no Google Cloud Project Id exposed, it means that we are in local environment + if (env.dev.isLocal || !project) { + return null + } + // Construct the fully qualified queue name. priority === 'low' && (queue = `${queue}-low`) @@ -458,7 +465,6 @@ export const enqueueRecommendation = async ( } export const enqueueImportFromIntegration = async ( - userId: string, integrationId: string, authToken: string ): Promise => { @@ -493,4 +499,51 @@ export const enqueueImportFromIntegration = async ( return createdTasks[0].name } +export const enqueueThumbnailTask = async ( + userId: string, + slug: string, + content: string +): Promise => { + const { GOOGLE_CLOUD_PROJECT } = process.env + const payload = { + userId, + slug, + content, + } + + const requestHeaders = { + Authorization: generateVerificationToken(userId), + } + + // If there is no Google Cloud Project Id exposed, it means that we are in local environment + if (env.dev.isLocal || !GOOGLE_CLOUD_PROJECT) { + // Calling the handler function directly. + setTimeout(() => { + axios + .post(env.queue.thumbnailTaskHandlerUrl, payload, { + headers: requestHeaders, + }) + .catch((error) => { + console.error(error) + }) + }, 0) + return '' + } + + const createdTasks = await createHttpTaskWithToken({ + payload, + taskHandlerUrl: env.queue.thumbnailTaskHandlerUrl, + requestHeaders, + }) + + if (!createdTasks || !createdTasks[0].name) { + logger.error(`Unable to get the name of the task`, { + payload, + createdTasks, + }) + throw new CreateTaskError(`Unable to get the name of the task`) + } + return createdTasks[0].name +} + export default createHttpTaskWithToken diff --git a/packages/api/src/utils/helpers.ts b/packages/api/src/utils/helpers.ts index f65994d21..315a46804 100644 --- a/packages/api/src/utils/helpers.ts +++ b/packages/api/src/utils/helpers.ts @@ -83,7 +83,7 @@ export function authorized< return (parent, args, ctx, info) => { const { claims } = ctx if (claims?.uid) { - return resolver(parent, args, { ...ctx, claims }, info) + return resolver(parent, args, { ...ctx, claims, uid: claims.uid }, info) } return { errorCodes: ['UNAUTHORIZED'] } as TError } diff --git a/packages/api/test/resolvers/update.test.ts b/packages/api/test/resolvers/update.test.ts index 7ac09d563..290325b6b 100644 --- a/packages/api/test/resolvers/update.test.ts +++ b/packages/api/test/resolvers/update.test.ts @@ -1,9 +1,9 @@ -import { createTestUser, deleteTestUser } from '../db' -import { createTestElasticPage, graphqlRequest, request } from '../util' import { expect } from 'chai' import 'mocha' -import { User } from '../../src/entity/user' import { Page } from '../../src/elastic/types' +import { User } from '../../src/entity/user' +import { createTestUser, deleteTestUser } from '../db' +import { createTestElasticPage, graphqlRequest, request } from '../util' describe('Update API', () => { let user: User @@ -28,8 +28,9 @@ describe('Update API', () => { describe('update page', () => { let query: string - let title = 'New Title' - let description = 'New Description' + const title = 'New Title' + const description = 'New Description' + const previewImage = 'https://omnivore.app/image.png' beforeEach(() => { query = ` @@ -39,12 +40,14 @@ describe('Update API', () => { pageId: "${page.id}" title: "${title}" description: "${description}" + previewImage: "${previewImage}" } ) { ... on UpdatePageSuccess { updatedPage { title description + image } } ... on UpdatePageError { @@ -61,6 +64,7 @@ describe('Update API', () => { const updatedPage = res?.body.data.updatePage.updatedPage expect(updatedPage?.title).to.eql(title) expect(updatedPage?.description).to.eql(description) + expect(updatedPage?.image).to.eql(previewImage) }) }) }) diff --git a/packages/thumbnail-handler/.dockerignore b/packages/thumbnail-handler/.dockerignore new file mode 100644 index 000000000..d8aea4ee6 --- /dev/null +++ b/packages/thumbnail-handler/.dockerignore @@ -0,0 +1,5 @@ +node_modules +build +.env* +Dockerfile +.dockerignore diff --git a/packages/thumbnail-handler/.eslintignore b/packages/thumbnail-handler/.eslintignore new file mode 100644 index 000000000..b38db2f29 --- /dev/null +++ b/packages/thumbnail-handler/.eslintignore @@ -0,0 +1,2 @@ +node_modules/ +build/ diff --git a/packages/thumbnail-handler/.eslintrc b/packages/thumbnail-handler/.eslintrc new file mode 100644 index 000000000..e006282a6 --- /dev/null +++ b/packages/thumbnail-handler/.eslintrc @@ -0,0 +1,6 @@ +{ + "extends": "../../.eslintrc", + "parserOptions": { + "project": "tsconfig.json" + } +} \ No newline at end of file diff --git a/packages/thumbnail-handler/.gcloudignore b/packages/thumbnail-handler/.gcloudignore new file mode 100644 index 000000000..ccc4eb240 --- /dev/null +++ b/packages/thumbnail-handler/.gcloudignore @@ -0,0 +1,16 @@ +# This file specifies files that are *not* uploaded to Google Cloud Platform +# using gcloud. It follows the same syntax as .gitignore, with the addition of +# "#!include" directives (which insert the entries of the given .gitignore-style +# file at that point). +# +# For more information, run: +# $ gcloud topic gcloudignore +# +.gcloudignore +# If you would like to upload your .git directory, .gitignore file or files +# from your .gitignore file, remove the corresponding line +# below: +.git +.gitignore + +node_modules diff --git a/packages/thumbnail-handler/Dockerfile b/packages/thumbnail-handler/Dockerfile new file mode 100644 index 000000000..1ba49d688 --- /dev/null +++ b/packages/thumbnail-handler/Dockerfile @@ -0,0 +1,26 @@ +FROM node:14.18-alpine + +# Run everything after as non-privileged user. +WORKDIR /app + +COPY package.json . +COPY yarn.lock . +COPY tsconfig.json . +COPY .eslintrc . + +COPY /packages/thumbnail-handler/package.json ./packages/thumbnail-handler/package.json + +RUN yarn install --pure-lockfile + +ADD /packages/thumbnail-handler ./packages/thumbnail-handler +RUN yarn workspace @omnivore/thumbnail-handler build + +# After building, fetch the production dependencies +RUN rm -rf /app/packages/thumbnail-handler/node_modules +RUN rm -rf /app/node_modules +RUN yarn install --pure-lockfile --production + +EXPOSE 8080 + +CMD ["yarn", "workspace", "@omnivore/thumbnail-handler", "start"] + diff --git a/packages/thumbnail-handler/mocha-config.json b/packages/thumbnail-handler/mocha-config.json new file mode 100644 index 000000000..44d1d24c1 --- /dev/null +++ b/packages/thumbnail-handler/mocha-config.json @@ -0,0 +1,5 @@ +{ + "extension": ["ts"], + "spec": "test/**/*.test.ts", + "require": "test/babel-register.js" + } \ No newline at end of file diff --git a/packages/thumbnail-handler/package.json b/packages/thumbnail-handler/package.json new file mode 100644 index 000000000..1825752bf --- /dev/null +++ b/packages/thumbnail-handler/package.json @@ -0,0 +1,32 @@ +{ + "name": "@omnivore/thumbnail-handler", + "version": "1.0.0", + "main": "build/src/index.js", + "files": [ + "build/src" + ], + "license": "Apache-2.0", + "scripts": { + "test": "yarn mocha -r ts-node/register --config mocha-config.json", + "lint": "eslint src --ext ts,js,tsx,jsx", + "compile": "tsc", + "build": "tsc", + "start": "functions-framework --target=thumbnailHandler", + "dev": "concurrently \"tsc -w\" \"nodemon --watch ./build/ --exec npm run start\"" + }, + "devDependencies": { + "chai": "^4.3.6", + "eslint-plugin-prettier": "^4.0.0", + "mocha": "^10.0.0", + "nock": "^13.3.1" + }, + "dependencies": { + "@google-cloud/functions-framework": "3.1.2", + "@sentry/serverless": "^6.16.1", + "axios": "^1.4.0", + "dotenv": "^16.0.1", + "image-size": "^1.0.2", + "jsonwebtoken": "^8.5.1", + "linkedom": "^0.14.26" + } +} diff --git a/packages/thumbnail-handler/src/index.ts b/packages/thumbnail-handler/src/index.ts new file mode 100644 index 000000000..41e6ab640 --- /dev/null +++ b/packages/thumbnail-handler/src/index.ts @@ -0,0 +1,274 @@ +import * as Sentry from '@sentry/serverless' +import axios from 'axios' +import * as dotenv from 'dotenv' // see https://github.com/motdotla/dotenv#how-do-i-use-dotenv-with-import +import sizeOf from 'image-size' +import * as jwt from 'jsonwebtoken' +import { parseHTML } from 'linkedom' +import { promisify } from 'util' + +interface ArticleResponse { + data: { + article: { + article: Page + } + } +} + +interface Page { + id: string + content: string + image?: string +} + +interface UpdatePageResponse { + data: { + updatePage: { + updatedPage: Page + } + } +} + +interface ThumbnailRequest { + slug: string + content: string +} + +dotenv.config() +Sentry.GCPFunction.init({ + dsn: process.env.SENTRY_DSN, + tracesSampleRate: 0, +}) + +const signToken = promisify(jwt.sign) + +const articleQuery = async (userId: string, slug: string): Promise => { + const JWT_SECRET = process.env.JWT_SECRET + const REST_BACKEND_ENDPOINT = process.env.REST_BACKEND_ENDPOINT + + if (!JWT_SECRET || !REST_BACKEND_ENDPOINT) { + throw 'Environment not configured correctly' + } + + const data = JSON.stringify({ + query: `query article ($username: String!, $slug: String!){ + article(username: $username, slug: $slug){ + ... on ArticleSuccess { + article { + id + content + image + } + } + ... on ArticleError { + errorCodes + } + } + }`, + variables: { + username: 'me', + slug, + }, + }) + const auth = (await signToken({ uid: userId }, JWT_SECRET)) as string + + const response = await axios.post( + `${REST_BACKEND_ENDPOINT}/graphql`, + data, + { + headers: { + Cookie: `auth=${auth};`, + 'Content-Type': 'application/json', + }, + } + ) + + return response.data.data.article.article +} + +const updatePageMutation = async ( + userId: string, + pageId: string, + image: string +) => { + const JWT_SECRET = process.env.JWT_SECRET + const REST_BACKEND_ENDPOINT = process.env.REST_BACKEND_ENDPOINT + + if (!JWT_SECRET || !REST_BACKEND_ENDPOINT) { + throw 'Environment not configured correctly' + } + + const data = JSON.stringify({ + query: `mutation UpdatePage ($input: UpdatePageInput!) { + updatePage(input: $input) { + ... on UpdatePageSuccess { + updatedPage { + id + } + } + ... on UpdatePageError { + errorCodes + } + } + }`, + variables: { + input: { + pageId, + previewImage: image, + }, + }, + }) + + const auth = (await signToken({ uid: userId }, JWT_SECRET)) as string + const response = await axios.post( + `${REST_BACKEND_ENDPOINT}/graphql`, + data, + { + headers: { + Cookie: `auth=${auth};`, + 'Content-Type': 'application/json', + }, + } + ) + + return !!response.data.data.updatePage +} + +const isThumbnailRequest = (body: any): body is ThumbnailRequest => { + return 'slug' in body && 'content' in body +} + +const getImageSize = async (url: string): Promise<[number, number] | null> => { + try { + // get image file by url + const response = await axios.get(url, { + responseType: 'arraybuffer', + }) + + // eslint-disable-next-line @typescript-eslint/no-unsafe-argument + const buffer = Buffer.from(response.data, 'binary') + + // get image size + const { width, height } = sizeOf(buffer) + + if (!width || !height) { + return null + } + + return [width, height] + } catch (e) { + console.log(e) + return null + } +} + +// credit: https://github.com/reddit-archive/reddit/blob/753b17407e9a9dca09558526805922de24133d53/r2/r2/lib/media.py#L706 +export const findThumbnail = async ( + content: string +): Promise => { + const dom = parseHTML(content).document + + // find the largest and squarest image as the thumbnail + // and pre-cache all images + const images = dom.querySelectorAll('img[src]') + if (!images || images.length === 0) { + console.debug('no images') + return null + } + + let thumbnail = null + let largestArea = 0 + for await (const image of Array.from(images)) { + const src = image.getAttribute('src') + if (!src) { + continue + } + + const size = await getImageSize(src) + if (!size) { + continue + } + + let area = size[0] * size[1] + + // ignore small images + if (area < 5000) { + console.debug('ignore small', src) + continue + } + + // penalize excessively long/wide images + const ratio = Math.max(...size) / Math.min(...size) + if (ratio > 1.5) { + console.debug('penalizing long/wide', src) + area /= ratio * 2 + } + + // penalize images with "sprite" in their name + if (src.toLowerCase().includes('sprite')) { + console.debug('penalizing sprite', src) + area /= 10 + } + + if (area > largestArea) { + largestArea = area + thumbnail = src + } + } + + return thumbnail +} + +/** + * request structure + * { + * userId: string + * slug: string + * } + */ + +export const thumbnailHandler = Sentry.GCPFunction.wrapHttpFunction( + async (req, res) => { + const token = req.headers?.authorization + if (!token) { + console.debug('no token') + return res.status(401).send('UNAUTHORIZED') + } + const { uid } = jwt.decode(token) as { uid: string } + if (!uid) { + console.debug('no uid') + return res.status(401).send('UNAUTHORIZED') + } + + if (!isThumbnailRequest(req.body)) { + console.debug('bad request') + return res.status(400).send('BAD_REQUEST') + } + + const { slug, content } = req.body + + try { + // find thumbnail from all images & pre-cache + const thumbnail = await findThumbnail(content) + if (!thumbnail) { + console.debug('no thumbnail') + return res.status(200).send('NOT_FOUND') + } + + const page = await articleQuery(uid, slug) + console.debug('find page', page.id) + // update page with thumbnail if not already set + if (page.image) { + console.debug('thumbnail already set') + return res.status(200).send('OK') + } + + const updated = await updatePageMutation(uid, page.id, thumbnail) + console.debug('thumbnail updated', updated) + + res.send('ok') + } catch (e) { + console.error(e) + res.status(500).send('INTERNAL_SERVER_ERROR') + } + } +) diff --git a/packages/thumbnail-handler/test/babel-register.js b/packages/thumbnail-handler/test/babel-register.js new file mode 100644 index 000000000..a6f65f60a --- /dev/null +++ b/packages/thumbnail-handler/test/babel-register.js @@ -0,0 +1,3 @@ +const register = require('@babel/register').default + +register({ extensions: ['.ts', '.tsx', '.js', '.jsx'] }) diff --git a/packages/thumbnail-handler/test/fixtures/findThumbnail.html b/packages/thumbnail-handler/test/fixtures/findThumbnail.html new file mode 100644 index 000000000..66192deac --- /dev/null +++ b/packages/thumbnail-handler/test/fixtures/findThumbnail.html @@ -0,0 +1,9 @@ +
+ small image + + large and square image + + wide image + + sprite image +
diff --git a/packages/thumbnail-handler/test/fixtures/large_and_square.png b/packages/thumbnail-handler/test/fixtures/large_and_square.png new file mode 100644 index 000000000..4c4992af7 Binary files /dev/null and b/packages/thumbnail-handler/test/fixtures/large_and_square.png differ diff --git a/packages/thumbnail-handler/test/fixtures/small.png b/packages/thumbnail-handler/test/fixtures/small.png new file mode 100644 index 000000000..930657193 Binary files /dev/null and b/packages/thumbnail-handler/test/fixtures/small.png differ diff --git a/packages/thumbnail-handler/test/fixtures/sprite.png b/packages/thumbnail-handler/test/fixtures/sprite.png new file mode 100644 index 000000000..4c4992af7 Binary files /dev/null and b/packages/thumbnail-handler/test/fixtures/sprite.png differ diff --git a/packages/thumbnail-handler/test/fixtures/wide.png b/packages/thumbnail-handler/test/fixtures/wide.png new file mode 100644 index 000000000..232acac91 Binary files /dev/null and b/packages/thumbnail-handler/test/fixtures/wide.png differ diff --git a/packages/thumbnail-handler/test/index.test.ts b/packages/thumbnail-handler/test/index.test.ts new file mode 100644 index 000000000..575cfc87d --- /dev/null +++ b/packages/thumbnail-handler/test/index.test.ts @@ -0,0 +1,30 @@ +import { expect } from 'chai' +import fs from 'fs' +import 'mocha' +import nock from 'nock' +import path from 'path' +import { findThumbnail } from '../src' + +describe('findThumbnail', () => { + it('finds the largest and squarest image', async () => { + const images = ['large_and_square', 'small', 'sprite', 'wide'] + // mock getting image by url + images.forEach((image) => { + nock('https://omnivore.app') + .get(`/${image}.png`) + .replyWithFile(200, path.join(__dirname, 'fixtures', `${image}.png`)) + }) + // get html content from file + const content = fs.readFileSync( + path.join(__dirname, 'fixtures', 'findThumbnail.html'), + 'utf8' + ) + // find thumbnail + const thumbnail = await findThumbnail(content) + + expect(thumbnail).to.eql('https://omnivore.app/large_and_square.png') + + // clean up + nock.cleanAll() + }) +}) diff --git a/packages/thumbnail-handler/tsconfig.json b/packages/thumbnail-handler/tsconfig.json new file mode 100644 index 000000000..7ebe093f6 --- /dev/null +++ b/packages/thumbnail-handler/tsconfig.json @@ -0,0 +1,8 @@ +{ + "extends": "./../../tsconfig.json", + "compilerOptions": { + "outDir": "build", + "rootDir": "." + }, + "include": ["src"] +} diff --git a/yarn.lock b/yarn.lock index 339090cfe..0a239f537 100644 --- a/yarn.lock +++ b/yarn.lock @@ -10426,6 +10426,15 @@ axios@^1.2.0, axios@^1.2.2: form-data "^4.0.0" proxy-from-env "^1.1.0" +axios@^1.4.0: + version "1.4.0" + resolved "https://registry.yarnpkg.com/axios/-/axios-1.4.0.tgz#38a7bf1224cd308de271146038b551d725f0be1f" + integrity sha512-S4XCWMEmzvo64T9GfvQDOXgYRDJ/wsSZc7Jvdgx5u1sd0JwsuPLqb3SYmusag+edF6ziyMensPVqLTSc1PiSEA== + dependencies: + follow-redirects "^1.15.0" + form-data "^4.0.0" + proxy-from-env "^1.1.0" + axobject-query@^2.2.0: version "2.2.0" resolved "https://registry.yarnpkg.com/axobject-query/-/axobject-query-2.2.0.tgz#943d47e10c0b704aa42275e20edf3722648989be" @@ -16662,6 +16671,13 @@ ignore@^5.2.0: resolved "https://registry.yarnpkg.com/ignore/-/ignore-5.2.0.tgz#6d3bac8fa7fe0d45d9f9be7bac2fc279577e345a" integrity sha512-CmxgYGiEPCLhfLnpPp1MoRmifwEIOgjcHXxOBjv7mY96c+eWScsOP9c112ZyLdWHi0FxHjI+4uVhKYp/gcdRmQ== +image-size@^1.0.2: + version "1.0.2" + resolved "https://registry.yarnpkg.com/image-size/-/image-size-1.0.2.tgz#d778b6d0ab75b2737c1556dd631652eb963bc486" + integrity sha512-xfOoWjceHntRb3qFCrh5ZFORYH8XCdYpASltMhZ/Q0KZiOwjdE/Yl2QCiWdwD+lygV5bMCvauzgu5PxBX/Yerg== + dependencies: + queue "6.0.2" + immediate@~3.0.5: version "3.0.6" resolved "https://registry.yarnpkg.com/immediate/-/immediate-3.0.6.tgz#9db1dbd0faf8de6fbe0f5dd5e56bb606280de69b" @@ -18823,6 +18839,17 @@ linkedom@^0.14.21: htmlparser2 "^8.0.1" uhyphen "^0.1.0" +linkedom@^0.14.26: + version "0.14.26" + resolved "https://registry.yarnpkg.com/linkedom/-/linkedom-0.14.26.tgz#fd8ddaef1a052e1191fb2e881605a1a001409f3b" + integrity sha512-mK6TrydfFA7phrnp+1j57ycBwFI5bGSW6YXlw9acHoqF+mP/y+FooEYYyniOt5Ot57FSKB3iwmnuQ1UUyNLm5A== + dependencies: + css-select "^5.1.0" + cssom "^0.5.0" + html-escaper "^3.0.3" + htmlparser2 "^8.0.1" + uhyphen "^0.2.0" + linkedom@^0.14.9: version "0.14.9" resolved "https://registry.yarnpkg.com/linkedom/-/linkedom-0.14.9.tgz#34c6f15eddc809406f42d8ee48cd30b0222eccb0" @@ -23161,6 +23188,13 @@ querystring@^0.2.0: resolved "https://registry.yarnpkg.com/querystring/-/querystring-0.2.1.tgz#40d77615bb09d16902a85c3e38aa8b5ed761c2dd" integrity sha512-wkvS7mL/JMugcup3/rMitHmd9ecIGd2lhFhK9N3UUQ450h66d1r3Y9nvXzQAW1Lq+wyx61k/1pfKS5KuKiyEbg== +queue@6.0.2: + version "6.0.2" + resolved "https://registry.yarnpkg.com/queue/-/queue-6.0.2.tgz#b91525283e2315c7553d2efa18d83e76432fed65" + integrity sha512-iHZWu+q3IdFZFX36ro/lKBkSvfkztY5Y7HMiPlOUjhupPcG2JMfst2KKEpu5XndviX/3UhFbRngUPNKtgvtZiA== + dependencies: + inherits "~2.0.3" + quick-lru@^4.0.1: version "4.0.1" resolved "https://registry.yarnpkg.com/quick-lru/-/quick-lru-4.0.1.tgz#5b8878f113a58217848c6482026c73e1ba57727f" @@ -26919,6 +26953,11 @@ uhyphen@^0.1.0: resolved "https://registry.yarnpkg.com/uhyphen/-/uhyphen-0.1.0.tgz#3cc22afa790daa802b9f6789f3583108d5b4a08c" integrity sha512-o0QVGuFg24FK765Qdd5kk0zU/U4dEsCtN/GSiwNI9i8xsSVtjIAOdTaVhLwZ1nrbWxFVMxNDDl+9fednsOMsBw== +uhyphen@^0.2.0: + version "0.2.0" + resolved "https://registry.yarnpkg.com/uhyphen/-/uhyphen-0.2.0.tgz#8fdf0623314486e020a3c00ee5cc7a12fe722b81" + integrity sha512-qz3o9CHXmJJPGBdqzab7qAYuW8kQGKNEuoHFYrBwV6hWIMcpAmxDLXojcHfFr9US1Pe6zUswEIJIbLI610fuqA== + uid-number@0.0.6: version "0.0.6" resolved "https://registry.yarnpkg.com/uid-number/-/uid-number-0.0.6.tgz#0ea10e8035e8eb5b8e4449f06da1c730663baa81"