diff --git a/inventory_provider/tasks/worker.py b/inventory_provider/tasks/worker.py index e76e3f6b882c3892ed8389caf4d5a0a55f69a683..d4779f6fe85135ff0fa2aa1188317f9593f81579 100644 --- a/inventory_provider/tasks/worker.py +++ b/inventory_provider/tasks/worker.py @@ -3,6 +3,8 @@ import logging import os import time +from redis.exceptions import RedisError + from celery import Task, states from celery.result import AsyncResult @@ -548,26 +550,30 @@ def launch_refresh_cache_all(config): :return: """ - _erase_next_db(config) - monitor.clear_joblog(get_current_redis(config)) - - update_latch_status(config, pending=True) - - # TODO: [DBOARD3-242] catch exceptions & reset latch status - - # first batch of subtasks: refresh cached opsdb data - subtasks = [ - update_neteng_managed_device_list.apply_async(), - update_interfaces_to_services.apply_async(), - update_geant_lambdas.apply_async(), - update_circuit_hierarchy.apply_async() - ] - [x.get() for x in subtasks] - - # now launch the task whose only purpose is to - # act as a convenient parent for all of the remaining tasks - t = internal_refresh_phase_2.apply_async() - return t.id + try: + _erase_next_db(config) + monitor.clear_joblog(get_current_redis(config)) + + update_latch_status(config, pending=True) + + # first batch of subtasks: refresh cached opsdb data + subtasks = [ + update_neteng_managed_device_list.apply_async(), + update_interfaces_to_services.apply_async(), + update_geant_lambdas.apply_async(), + update_circuit_hierarchy.apply_async() + ] + [x.get() for x in subtasks] + + # now launch the task whose only purpose is to + # act as a convenient parent for all of the remaining tasks + t = internal_refresh_phase_2.apply_async() + return t.id + + except RedisError: + update_latch_status(config, pending=False, failure=True) + logger.exception('error launching refresh subtasks') + raise def _wait_for_tasks(task_ids, update_callback=lambda s: None):