From 330e50fba5467b589c01d5f8fd26932157454df4 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Thu, 21 Sep 2023 09:25:11 +0800 Subject: [PATCH] increase scroll to 2m and cool down for 5 seconds --- packages/db/elastic_migrations/migrate_from_elastic.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/db/elastic_migrations/migrate_from_elastic.py b/packages/db/elastic_migrations/migrate_from_elastic.py index 0f596d6a2..75cba1145 100755 --- a/packages/db/elastic_migrations/migrate_from_elastic.py +++ b/packages/db/elastic_migrations/migrate_from_elastic.py @@ -309,7 +309,7 @@ async def main(): # Scan API for larger library docs = async_scan(es_client, index=ES_INDEX, query=query, preserve_order=True, size=ES_SCAN_SIZE, - request_timeout=60, scroll='1m') + request_timeout=60, scroll='2m') # convert _id to uuid async for doc in docs: @@ -438,6 +438,8 @@ async def main(): if len(recommendations) > 0: await insert_recommendations(db_conn, recommendations) recommendations = [] + # cool down for 5 seconds + await asyncio.sleep(5) # copy remaining records to postgres if len(library_items) > 0: await insert_library_items(db_conn, library_items)