prehistoric-systems/omnivore
1
0
Fork 0
mirror of https://github.com/omnivore-app/omnivore.git synced 2026-03-11 08:54:26 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

increase scroll to 2m and cool down for 5 seconds

Browse source
This commit is contained in:
Hongbo Wu 2023-09-21 09:25:11 +08:00
parent 7d4948a43b
commit 330e50fba5
1 changed files with 3 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
packages/db/elastic_migrations/migrate_from_elastic.py
View file

@ -309,7 +309,7 @@ async def main():
# Scan API for larger library
docs = async_scan(es_client, index=ES_INDEX, query=query,
preserve_order=True, size=ES_SCAN_SIZE,
request_timeout=60, scroll='1m')
request_timeout=60, scroll='2m')
# convert _id to uuid
async for doc in docs:
@ -438,6 +438,8 @@ async def main():
if len(recommendations) > 0:
await insert_recommendations(db_conn, recommendations)
recommendations = []
# cool down for 5 seconds
await asyncio.sleep(5)
# copy remaining records to postgres
if len(library_items) > 0:
await insert_library_items(db_conn, library_items)