From 8a485af20dbdd14a1de890849c368e00db460646 Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Wed, 23 Mar 2022 14:30:33 +0800
Subject: [PATCH] validate publishedAt before inserting to elastic

---
 packages/api/elastic_migrate.py | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)

diff --git a/packages/api/elastic_migrate.py b/packages/api/elastic_migrate.py
index 64c06bced..ea081200f 100644
--- a/packages/api/elastic_migrate.py
+++ b/packages/api/elastic_migrate.py
@@ -1,4 +1,5 @@
 #!/usr/bin/python
+from datetime import datetime
 import os
 import json
 import psycopg2
@@ -178,6 +179,7 @@ def import_data_to_es(client, docs) -> int:
 
     doc_list = []
     for doc in docs:
+        doc['publishedAt'] = validated_date(doc['publishedAt'])
         # convert the string to a dict object
         dict_doc = {
             '_index': 'pages',
@@ -191,6 +193,22 @@ def import_data_to_es(client, docs) -> int:
     return count
 
 
+def validated_date(date):
+    try:
+        if date is None:
+            return None
+
+        datetime_object = datetime.strptime(date, '%Y-%m-%dT%H:%M:%S.%fZ')
+        # Make sure the date year is not greater than 9999
+        if datetime_object.year > 9999:
+            return None
+
+        return date
+    except Exception as err:
+        print('error validating date', err)
+        return None
+
+
 print('Starting migration')
 
 # test elastic client