DBOARD3-1040: fallback to redis cache when IMS fails for flexils

inventory_provider/tasks/worker.py

@@ -32,6 +32,7 @@ from inventory_provider import environment
 from inventory_provider import snmp
 from inventory_provider import juniper
 from redis import RedisError
+from requests import HTTPError
 
 FINALIZER_POLLING_FREQUENCY_S = 2.5
 FINALIZER_TIMEOUT_S = 300
@@ -1062,7 +1063,7 @@ def cache_extracted_ims_data(extracted_data, use_current=False):
 @log_task_entry_and_exit
 def ims_task(self, use_current=False):
     try:
-        extracted_data = extract_ims_data()
+        extracted_data = extract_ims_data(log_warning=self.log_warning)
         cache_extracted_ims_data(extracted_data)
         transformed_data = transform_ims_data(extracted_data)
 
@@ -1074,17 +1075,20 @@ def ims_task(self, use_current=False):
         update_latch_status(InventoryTask.config, pending=True, failure=True)
 
 
-def extract_ims_data():
+def extract_ims_data(log_warning=lambda s: None):
     c = InventoryTask.config["ims"]
     return _extract_ims_data(
         ims_api_url=c['api'],
         ims_username=c['username'],
         ims_password=c['password'],
-        verify_ssl=c.get('verify-ssl', False)
+        verify_ssl=c.get('verify-ssl', False),
+        log_warning=log_warning
     )
 
 
-def _extract_ims_data(ims_api_url, ims_username, ims_password, verify_ssl):
+def _extract_ims_data(
+    ims_api_url, ims_username, ims_password, verify_ssl, log_warning
+):
     """
     convenient entry point for testing ...
 
@@ -1212,7 +1216,22 @@ def _extract_ims_data(ims_api_url, ims_username, ims_password, verify_ssl):
     @log_task_entry_and_exit
     def _populate_flexils_data():
         nonlocal flexils_data
-        flexils_data = ims_data.get_flexils_by_circuitid(ds=_ds())
+
+        def _transform_key(key):
+            if key == "null":
+                return None
+            try:
+                return int(key)
+            except ValueError:
+                return key
+
+        try:
+            flexils_data = ims_data.get_flexils_by_circuitid(ds=_ds())
+        except HTTPError:
+            log_warning("Failure reading FlexILS data from IMS. Using cache instead")
+            redis = get_current_redis(InventoryTask.config)
+            raw_data = json.loads(redis.get("ims:cache:flexils_data"))
+            flexils_data = {_transform_key(k): v for k, v in raw_data.items()}
 
     @log_task_entry_and_exit
     def _populate_hierarchy():