diff --git a/packages/api/src/routers/svc/user.ts b/packages/api/src/routers/svc/user.ts index ca6e9dbea..0c7fc5991 100644 --- a/packages/api/src/routers/svc/user.ts +++ b/packages/api/src/routers/svc/user.ts @@ -5,7 +5,7 @@ import express from 'express' import { LessThan } from 'typeorm' import { StatusType } from '../../entity/user' import { readPushSubscription } from '../../pubsub' -import { deleteUsers } from '../../services/user' +import { batchDeleteUsers } from '../../services/user' import { corsConfig } from '../../utils/corsConfig' import { logger } from '../../utils/logger' @@ -52,7 +52,7 @@ export function userServiceRouter() { const subTime = cleanupMessage.subDays * 1000 * 60 * 60 * 24 // convert days to milliseconds try { - const result = await deleteUsers({ + const result = await batchDeleteUsers({ status: StatusType.Deleted, updatedAt: LessThan(new Date(Date.now() - subTime)), // subDays ago }) diff --git a/packages/api/src/services/user.ts b/packages/api/src/services/user.ts index f3d01e489..848c12b0c 100644 --- a/packages/api/src/services/user.ts +++ b/packages/api/src/services/user.ts @@ -1,6 +1,6 @@ import { DeepPartial, FindOptionsWhere, In } from 'typeorm' import { StatusType, User } from '../entity/user' -import { authTrx } from '../repository' +import { authTrx, getRepository, queryBuilderToRawSql } from '../repository' import { userRepository } from '../repository/user' import { SetClaimsRole } from '../utils/dictionary' @@ -47,3 +47,45 @@ export const createUsers = async (users: DeepPartial[]) => { SetClaimsRole.ADMIN ) } + +export const batchDeleteUsers = async (criteria: FindOptionsWhere) => { + const userQb = getRepository(User).createQueryBuilder().where(criteria) + const userCountSql = queryBuilderToRawSql(userQb.select('COUNT(1)')) + const userSubQuery = queryBuilderToRawSql(userQb.select('id INTO user_ids')) + + const batchSize = 1000 + const start = new Date().toISOString() + const sql = ` + -- Set batch size + DO $$ + DECLARE + batch_size INT := ${batchSize}; + user_ids UUID[]; + BEGIN + -- Loop through batches of users + FOR i IN 0..CEIL((${userCountSql})) * 1.0 / batch_size) - 1 LOOP + -- GET batch of user ids + ${userSubQuery} LIMIT ${batchSize} OFFSET i * batch_size; + + -- Loop through batches of items + FOR j IN 0..CEIL((SELECT COUNT(1) FROM omnivore.library_item WHERE user_id = ANY(user_ids))) * 1.0 / batch_size) - 1 LOOP + -- Delete batch of items + DELETE FROM omnivore.library_item + WHERE user_id = ANY(user_ids) + AND updated_at < '${start}' + LIMIT ${batchSize}; + END LOOP; + + -- Delete the batch of users + DELETE FROM omnivore.user WHERE id = ANY(user_ids); + END LOOP; + END $$ + ` + + return authTrx( + async (t) => t.query(sql), + undefined, + undefined, + SetClaimsRole.ADMIN + ) +}