From 431640605f50a0e250be618157a8b923a3a6fa27 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Mon, 5 Aug 2024 15:34:56 +0800 Subject: [PATCH 1/4] optimize the batch update script to use index by adding order by user_id --- packages/db/remove_original_content.py | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/db/remove_original_content.py b/packages/db/remove_original_content.py index 29285e134..60516804d 100755 --- a/packages/db/remove_original_content.py +++ b/packages/db/remove_original_content.py @@ -23,6 +23,7 @@ def batch_update_library_items(conn): SELECT ctid FROM omnivore.library_item WHERE original_content IS NOT NULL + ORDER BY user_id LIMIT {batch_size} ) """) From 498790d22157c6909b5ca258ff04c793a671c4b4 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Mon, 5 Aug 2024 15:38:36 +0800 Subject: [PATCH 2/4] commit changes first --- packages/db/remove_original_content.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/db/remove_original_content.py b/packages/db/remove_original_content.py index 60516804d..fa050c3b0 100755 --- a/packages/db/remove_original_content.py +++ b/packages/db/remove_original_content.py @@ -27,8 +27,9 @@ def batch_update_library_items(conn): LIMIT {batch_size} ) """) - rows_updated = cursor.rowcount conn.commit() + + rows_updated = cursor.rowcount if rows_updated == 0: break From 7295b90397fe66b8061b1b68ec514e2d1d776aa4 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Mon, 5 Aug 2024 15:39:55 +0800 Subject: [PATCH 3/4] fix typo --- packages/db/remove_original_content.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/db/remove_original_content.py b/packages/db/remove_original_content.py index fa050c3b0..8869c8780 100755 --- a/packages/db/remove_original_content.py +++ b/packages/db/remove_original_content.py @@ -19,8 +19,8 @@ def batch_update_library_items(conn): cursor.execute(f""" UPDATE omnivore.library_item SET original_content = NULL - WHERE ctid IN ( - SELECT ctid + WHERE id IN ( + SELECT id FROM omnivore.library_item WHERE original_content IS NOT NULL ORDER BY user_id From a6637fcbed3d2d12ce045d67618821154a0bdbef Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Mon, 5 Aug 2024 15:49:29 +0800 Subject: [PATCH 4/4] update batch_size --- packages/db/remove_original_content.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/db/remove_original_content.py b/packages/db/remove_original_content.py index 8869c8780..29574c94e 100755 --- a/packages/db/remove_original_content.py +++ b/packages/db/remove_original_content.py @@ -9,10 +9,10 @@ PG_USER = os.getenv('PG_USER', 'app_user') PG_PASSWORD = os.getenv('PG_PASSWORD', 'app_pass') PG_DB = os.getenv('PG_DB', 'omnivore') PG_TIMEOUT = os.getenv('PG_TIMEOUT', 10) +BATCH_SIZE = os.getenv('BATCH_SIZE', 100) def batch_update_library_items(conn): - batch_size = 100 # update original_content to NULL in batches with conn.cursor() as cursor: while True: @@ -24,7 +24,7 @@ def batch_update_library_items(conn): FROM omnivore.library_item WHERE original_content IS NOT NULL ORDER BY user_id - LIMIT {batch_size} + LIMIT {BATCH_SIZE} ) """) conn.commit()