use current data, in case of snmp/netconf errors

inventory_provider/tasks/worker.py

@@ -12,7 +12,8 @@ import jsonschema
 
 from inventory_provider.tasks.app import app
 from inventory_provider.tasks.common \
-    import get_next_redis, latch_db, get_latch, set_latch, update_latch_status
+    import get_next_redis, get_current_redis, \
+    latch_db, get_latch, set_latch, update_latch_status
 from inventory_provider.tasks import data
 from inventory_provider import config
 from inventory_provider import environment
@@ -108,22 +109,57 @@ class InventoryTask(Task):
 @app.task(base=InventoryTask, bind=True, name='snmp_refresh_interfaces')
 @log_task_entry_and_exit
 def snmp_refresh_interfaces(self, hostname, community):
-    # TODO: [DBOARD3-242] copy from current redis in case of error
-    value = list(snmp.get_router_snmp_indexes(hostname, community))
+    warning = False
+    try:
+        value = list(snmp.get_router_snmp_indexes(hostname, community))
+    except ConnectionError:
+        logger.exception(f'error loading snmp data from {hostname}')
+        warning = True
+        r = get_current_redis(InventoryTask.config)
+        value = r.get(f'snmp-interfaces:{hostname}')
+        if not value:
+            raise InventoryTaskError(
+                f'snmp error with {hostname}'
+                f' and no cached netconf data found')
+        # unnecessary json encode/decode here ... could be optimized
+        value = json.loads(value.decode('utf-8'))
+
     r = get_next_redis(InventoryTask.config)
-    r.set('snmp-interfaces:' + hostname, json.dumps(value))
-    return self.success(message=f'snmp info loaded from {hostname}')
+    r.set(f'snmp-interfaces:{hostname}', json.dumps(value))
+
+    if warning:
+        return self.warning(
+            message=f'i/o error with {hostname}, using cached snmp info')
+    else:
+        return self.success(message=f'snmp info loaded from {hostname}')
 
 
 @app.task(base=InventoryTask, bind=True, name='netconf_refresh_config')
 @log_task_entry_and_exit
 def netconf_refresh_config(self, hostname):
-    # TODO: [DBOARD3-242] copy from current redis in case of error
-    netconf_doc = juniper.load_config(hostname, InventoryTask.config["ssh"])
-    netconf_str = etree.tostring(netconf_doc, encoding='unicode')
+
+    warning = False
+    try:
+        netconf_doc = juniper.load_config(hostname, InventoryTask.config["ssh"])
+        netconf_str = etree.tostring(netconf_doc, encoding='unicode')
+    except ConnectionError:
+        logger.exception(f'error loading netconf data from {hostname}')
+        warning = True
+        r = get_current_redis(InventoryTask.config)
+        netconf_str = r.get(f'netconf:{hostname}')
+        if not netconf_str:
+            raise InventoryTaskError(
+                f'netconf error with {hostname}'
+                f' and no cached netconf data found')
+
     r = get_next_redis(InventoryTask.config)
-    r.set('netconf:' + hostname, netconf_str)
-    return self.success(message=f'netconf info loaded from {hostname}')
+    r.set(f'netconf:{hostname}', netconf_str)
+
+    if warning:
+        return self.warning(
+            message=f'i/o error with {hostname}, using cached netconf info')
+    else:
+        return self.success(message=f'netconf info loaded from {hostname}')
 
 
 @app.task(base=InventoryTask, bind=True, name='update_interfaces_to_services')