prehistoric-systems/omnivore
1
0
Fork 0
mirror of https://github.com/omnivore-app/omnivore.git synced 2026-03-11 08:54:26 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

add assertion of migrated data in python script

Browse source
This commit is contained in:
Hongbo Wu 2022-03-23 12:09:25 +08:00
parent 42e2beb53d
commit 1b0399f286
1 changed files with 27 additions and 5 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

32
packages/api/elastic_migrate.py
View file

@ -64,6 +64,26 @@ UPDATE omnivore.highlight
'''
def assertData(conn, client):
# get all users from postgres
try:
cursor = conn.cursor(cursor_factory=RealDictCursor)
cursor.execute('''SELECT id FROM omnivore.user''')
result = cursor.fetchall()
for row in result:
userId = row['id']
cursor.execute(
f'''SELECT COUNT(id) FROM omnivore.links WHERE user_id = "{userId}"''')
countInPostgres = cursor.fetchone()
countInElastic = client.count(
index='pages', body={'query': {'term': {'userId': userId}}})['count']
assert countInPostgres == countInElastic
cursor.close()
except Exception as err:
print('Assert data ERROR:', err)
exit(1)
def create_index(client):
print('Creating index')
try:
@ -183,15 +203,17 @@ conn = psycopg2.connect(
password={PG_PASSWORD}')
print('Postgres connection:', conn.info)
create_index(client)
# create_index(client)
# ingest data from postgres to es and json file (for debugging)
ingest_data_to_elastic(conn, QUERY, DATA_FILE)
# ingest_data_to_elastic(conn, QUERY, DATA_FILE)
# update existing tables
update_postgres_data(conn, UPDATE_ARTICLE_SAVING_REQUEST_SQL,
'article_saving_request')
update_postgres_data(conn, UPDATE_HIGHLIGHT_SQL, 'highlight')
# update_postgres_data(conn, UPDATE_ARTICLE_SAVING_REQUEST_SQL,
# 'article_saving_request')
# update_postgres_data(conn, UPDATE_HIGHLIGHT_SQL, 'highlight')
assertData(conn, client)
client.close()
conn.close()