catch RedisError's and reset latch status

inventory_provider/tasks/worker.py

@@ -3,6 +3,8 @@ import logging
 import os
 import time
 
+from redis.exceptions import RedisError
+
 from celery import Task, states
 from celery.result import AsyncResult
 
@@ -548,26 +550,30 @@ def launch_refresh_cache_all(config):
     :return:
     """
 
-    _erase_next_db(config)
-    monitor.clear_joblog(get_current_redis(config))
-
-    update_latch_status(config, pending=True)
-
-    # TODO: [DBOARD3-242] catch exceptions & reset latch status
-
-    # first batch of subtasks: refresh cached opsdb data
-    subtasks = [
-        update_neteng_managed_device_list.apply_async(),
-        update_interfaces_to_services.apply_async(),
-        update_geant_lambdas.apply_async(),
-        update_circuit_hierarchy.apply_async()
-    ]
-    [x.get() for x in subtasks]
-
-    # now launch the task whose only purpose is to
-    # act as a convenient parent for all of the remaining tasks
-    t = internal_refresh_phase_2.apply_async()
-    return t.id
+    try:
+        _erase_next_db(config)
+        monitor.clear_joblog(get_current_redis(config))
+
+        update_latch_status(config, pending=True)
+
+        # first batch of subtasks: refresh cached opsdb data
+        subtasks = [
+            update_neteng_managed_device_list.apply_async(),
+            update_interfaces_to_services.apply_async(),
+            update_geant_lambdas.apply_async(),
+            update_circuit_hierarchy.apply_async()
+        ]
+        [x.get() for x in subtasks]
+
+        # now launch the task whose only purpose is to
+        # act as a convenient parent for all of the remaining tasks
+        t = internal_refresh_phase_2.apply_async()
+        return t.id
+
+    except RedisError:
+        update_latch_status(config, pending=False, failure=True)
+        logger.exception('error launching refresh subtasks')
+        raise
 
 
 def _wait_for_tasks(task_ids, update_callback=lambda s: None):