mirror of
https://github.com/bewcloud/bewcloud.git
synced 2026-03-11 08:54:49 +00:00
c26cae625e
This implements a huge change, where Fresh is removed as a framework and serving files, allowing more control over importing, bundling, and serving files and components. The biggest challenge was to continue making sure that there weren't too many places to look into for import versions, and `PasswordlessPasskeyLogin.tsx` became a prototype in migrating a component to fully SSR, no need for frontend parsing (via Babel) or bundling (via a custom-script, downloading frontend dependencies from esm.sh). Still, there are too many components to migrate like that, and it's all working, so I likely won't even attempt it unless there's some bug, new feature, or security vulnerability to address that warrants a rewrite of those. This also updates all dependencies (except `@libs/xml` because that still causes some breaking in DAV endpoints), including Deno! All other advantages can be seen in the related issues, and the breaking change this (v4.0.0) introduces is related simply to `config.email.tlsMode` (which had a deprecation warning throughout v3), and because, while I tested many things exhaustively, it's not impossible something broke that I didn't see. Closes #141 Closes #132
68 lines
2.3 KiB
TypeScript
68 lines
2.3 KiB
TypeScript
import Database, { sql } from '/lib/interfaces/database.ts';
|
|
import { NewsFeed } from '/lib/types.ts';
|
|
import { concurrentPromises } from '/public/ts/utils/misc.ts';
|
|
import { FeedModel } from '/lib/models/news.ts';
|
|
|
|
const db = new Database();
|
|
|
|
export async function fetchNewArticles(forceFetch = false) {
|
|
const fourHoursAgo = forceFetch ? new Date() : new Date(new Date().setUTCHours(new Date().getUTCHours() - 4));
|
|
|
|
try {
|
|
const feedsToCrawl = await db.query<NewsFeed>(
|
|
sql`SELECT * FROM "bewcloud_news_feeds" WHERE "last_crawled_at" IS NULL OR "last_crawled_at" <= $1 ORDER BY "last_crawled_at" ASC`,
|
|
[
|
|
fourHoursAgo.toISOString(),
|
|
],
|
|
);
|
|
|
|
console.info('Will crawl', feedsToCrawl.length, 'news feeds');
|
|
|
|
await concurrentPromises(feedsToCrawl.map((newsFeed) => () => FeedModel.crawl(newsFeed)), 3);
|
|
|
|
console.info('Crawled', feedsToCrawl.length, 'news feeds');
|
|
} catch (error) {
|
|
console.error(error);
|
|
}
|
|
}
|
|
|
|
export async function cleanupOldArticles() {
|
|
const oneMonthAgo = new Date(new Date().setUTCMonth(new Date().getUTCMonth() - 1));
|
|
|
|
try {
|
|
console.info('Will cleanup old articles');
|
|
|
|
const feedIdsToSkip = new Set<string>();
|
|
|
|
const feeds = await db.query<Pick<NewsFeed, 'id' | 'feed_url'>>(
|
|
sql`SELECT "id", "feed_url" FROM "bewcloud_news_feeds" ORDER BY "last_crawled_at" ASC`,
|
|
);
|
|
|
|
for (const feed of feeds) {
|
|
const recentArticlesCount = (await db.query<{ count: number }>(
|
|
sql`SELECT COUNT("id") AS "count" FROM "bewcloud_news_feed_articles" WHERE "feed_id" = $1 AND "article_date" >= $2`,
|
|
[feed.id, oneMonthAgo.toISOString().substring(0, 10)],
|
|
))[0].count;
|
|
|
|
// Don't delete old articles if the feed doesn't have many recent articles (low frequency feeds)
|
|
if (recentArticlesCount <= 5) {
|
|
feedIdsToSkip.add(feed.id);
|
|
}
|
|
}
|
|
|
|
const result = await db.query<{ count: number }>(
|
|
sql`WITH "deleted" AS (
|
|
DELETE FROM "bewcloud_news_feed_articles" WHERE "is_read" = TRUE AND "article_date" <= $1 AND NOT ("feed_id" != ANY($2)) RETURNING *
|
|
)
|
|
SELECT COUNT(*) FROM "deleted"`,
|
|
[
|
|
oneMonthAgo.toISOString().substring(0, 10),
|
|
[...feedIdsToSkip],
|
|
],
|
|
);
|
|
|
|
console.info('Deleted', result[0].count, 'old articles');
|
|
} catch (error) {
|
|
console.error(error);
|
|
}
|
|
}
|