mirror of
https://github.com/omnivore-app/omnivore.git
synced 2026-03-11 08:54:26 +00:00
create a following task for each rss item fetched
This commit is contained in:
Hongbo Wu
parent
4bef0a006a
commit
a71181982f
13 changed files with 40 additions and 132 deletions
|
|
@ -1,5 +0,0 @@
|
|||
node_modules
|
||||
build
|
||||
.env*
|
||||
Dockerfile
|
||||
.dockerignore
|
||||
|
|
@ -1,2 +0,0 @@
|
|||
node_modules/
|
||||
build/
|
||||
|
|
@ -1,6 +0,0 @@
|
|||
{
|
||||
"extends": "../../.eslintrc",
|
||||
"parserOptions": {
|
||||
"project": "tsconfig.json"
|
||||
}
|
||||
}
|
||||
|
|
@ -1,16 +0,0 @@
|
|||
# This file specifies files that are *not* uploaded to Google Cloud Platform
|
||||
# using gcloud. It follows the same syntax as .gitignore, with the addition of
|
||||
# "#!include" directives (which insert the entries of the given .gitignore-style
|
||||
# file at that point).
|
||||
#
|
||||
# For more information, run:
|
||||
# $ gcloud topic gcloudignore
|
||||
#
|
||||
.gcloudignore
|
||||
# If you would like to upload your .git directory, .gitignore file or files
|
||||
# from your .gitignore file, remove the corresponding line
|
||||
# below:
|
||||
.git
|
||||
.gitignore
|
||||
|
||||
node_modules
|
||||
|
|
@ -1,26 +0,0 @@
|
|||
FROM node:18.16-alpine
|
||||
|
||||
# Run everything after as non-privileged user.
|
||||
WORKDIR /app
|
||||
|
||||
COPY package.json .
|
||||
COPY yarn.lock .
|
||||
COPY tsconfig.json .
|
||||
COPY .eslintrc .
|
||||
|
||||
COPY /packages/following-handler/package.json ./packages/following-handler/package.json
|
||||
|
||||
RUN yarn install --pure-lockfile
|
||||
|
||||
COPY /packages/following-handler ./packages/following-handler
|
||||
RUN yarn workspace @omnivore/rss-handler build
|
||||
|
||||
# After building, fetch the production dependencies
|
||||
RUN rm -rf /app/packages/following-handler/node_modules
|
||||
RUN rm -rf /app/node_modules
|
||||
RUN yarn install --pure-lockfile --production
|
||||
|
||||
EXPOSE 8080
|
||||
|
||||
CMD ["yarn", "workspace", "@omnivore/following-handler", "start"]
|
||||
|
||||
|
|
@ -1,5 +0,0 @@
|
|||
{
|
||||
"extension": ["ts"],
|
||||
"spec": "test/**/*.test.ts",
|
||||
"require": "test/babel-register.js"
|
||||
}
|
||||
|
|
@ -1,31 +0,0 @@
|
|||
{
|
||||
"name": "@omnivore/following-handler",
|
||||
"version": "1.0.0",
|
||||
"main": "build/src/index.js",
|
||||
"files": [
|
||||
"build/src"
|
||||
],
|
||||
"license": "Apache-2.0",
|
||||
"scripts": {
|
||||
"test": "yarn mocha -r ts-node/register --config mocha-config.json",
|
||||
"test:typecheck": "tsc --noEmit",
|
||||
"lint": "eslint src --ext ts,js,tsx,jsx",
|
||||
"compile": "tsc",
|
||||
"build": "tsc",
|
||||
"start": "functions-framework --target=followingHandler",
|
||||
"dev": "concurrently \"tsc -w\" \"nodemon --watch ./build/ --exec npm run start\""
|
||||
},
|
||||
"devDependencies": {
|
||||
"chai": "^4.3.6",
|
||||
"eslint-plugin-prettier": "^4.0.0",
|
||||
"mocha": "^10.0.0"
|
||||
},
|
||||
"dependencies": {
|
||||
"@google-cloud/functions-framework": "3.1.2",
|
||||
"@sentry/serverless": "^7.77.0",
|
||||
"dotenv": "^16.0.1"
|
||||
},
|
||||
"volta": {
|
||||
"extends": "../../package.json"
|
||||
}
|
||||
}
|
||||
|
|
@ -1,19 +0,0 @@
|
|||
import * as Sentry from '@sentry/serverless'
|
||||
import * as dotenv from 'dotenv' // see https://github.com/motdotla/dotenv#how-do-i-use-dotenv-with-import
|
||||
|
||||
dotenv.config()
|
||||
Sentry.GCPFunction.init({
|
||||
dsn: process.env.SENTRY_DSN,
|
||||
tracesSampleRate: 0,
|
||||
})
|
||||
|
||||
export const followingHandler = Sentry.GCPFunction.wrapHttpFunction(
|
||||
(req, res) => {
|
||||
if (req.query.token !== process.env.PUBSUB_VERIFICATION_TOKEN) {
|
||||
console.log('query does not include valid token')
|
||||
return res.sendStatus(403)
|
||||
}
|
||||
|
||||
res.send('ok')
|
||||
}
|
||||
)
|
||||
|
|
@ -1,3 +0,0 @@
|
|||
const register = require('@babel/register').default
|
||||
|
||||
register({ extensions: ['.ts', '.tsx', '.js', '.jsx'] })
|
||||
|
|
@ -1,8 +0,0 @@
|
|||
import { expect } from 'chai'
|
||||
import 'mocha'
|
||||
|
||||
describe('stub test', () => {
|
||||
it('should pass', () => {
|
||||
expect(true).to.be.true
|
||||
})
|
||||
})
|
||||
|
|
@ -1,8 +0,0 @@
|
|||
{
|
||||
"extends": "./../../tsconfig.json",
|
||||
"compilerOptions": {
|
||||
"outDir": "build",
|
||||
"rootDir": "."
|
||||
},
|
||||
"include": ["src"]
|
||||
}
|
||||
|
|
@ -5,7 +5,11 @@ import * as dotenv from 'dotenv' // see https://github.com/motdotla/dotenv#how-d
|
|||
import * as jwt from 'jsonwebtoken'
|
||||
import Parser, { Item } from 'rss-parser'
|
||||
import { promisify } from 'util'
|
||||
import { CONTENT_FETCH_URL, createCloudTask } from './task'
|
||||
import {
|
||||
CONTENT_FETCH_URL,
|
||||
createCloudTask,
|
||||
FOLLOWING_HANDLER_URL,
|
||||
} from './task'
|
||||
|
||||
interface RssFeedRequest {
|
||||
subscriptionIds: string[]
|
||||
|
|
@ -151,6 +155,38 @@ const createSavingItemTask = async (
|
|||
}
|
||||
}
|
||||
|
||||
const createFollowingTask = async (
|
||||
userId: string,
|
||||
feedUrl: string,
|
||||
item: Item
|
||||
) => {
|
||||
const input = {
|
||||
userId,
|
||||
url: item.link,
|
||||
title: item.title,
|
||||
author: item.creator,
|
||||
description: item.summary,
|
||||
sharedSource: 'rss-feeder',
|
||||
previewContent: item.content,
|
||||
sharedBy: feedUrl,
|
||||
savedAt: item.isoDate,
|
||||
publishedAt: item.isoDate,
|
||||
sharedAt: item.isoDate,
|
||||
}
|
||||
|
||||
try {
|
||||
console.log('Creating task', input.url)
|
||||
// save page
|
||||
const task = await createCloudTask(FOLLOWING_HANDLER_URL, input)
|
||||
console.log('Created task', task)
|
||||
|
||||
return !!task
|
||||
} catch (error) {
|
||||
console.error('Error while creating task', error)
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
dotenv.config()
|
||||
Sentry.GCPFunction.init({
|
||||
dsn: process.env.SENTRY_DSN,
|
||||
|
|
@ -330,7 +366,7 @@ const processSubscription = async (
|
|||
continue
|
||||
}
|
||||
|
||||
const created = await createSavingItemTask(userId, feedUrl, item)
|
||||
const created = await createFollowingTask(userId, feedUrl, item)
|
||||
if (!created) {
|
||||
console.error('Failed to create task for feed item', item.link)
|
||||
continue
|
||||
|
|
@ -353,7 +389,7 @@ const processSubscription = async (
|
|||
}
|
||||
|
||||
// the feed has never been fetched, save at least the last valid item
|
||||
const created = await createSavingItemTask(userId, feedUrl, lastValidItem)
|
||||
const created = await createFollowingTask(userId, feedUrl, lastValidItem)
|
||||
if (!created) {
|
||||
console.error('Failed to create task for feed item', lastValidItem.link)
|
||||
throw new Error('Failed to create task for feed item')
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ import { CloudTasksClient, protos } from '@google-cloud/tasks'
|
|||
const cloudTask = new CloudTasksClient()
|
||||
|
||||
export const CONTENT_FETCH_URL = process.env.CONTENT_FETCH_GCF_URL
|
||||
export const FOLLOWING_HANDLER_URL = process.env.FOLLOWING_HANDLER_URL
|
||||
|
||||
export const createCloudTask = async (
|
||||
taskHandlerUrl: string | undefined,
|
||||
|
|
|
|||
Loading…
Reference in a new issue