From ce6dcfce61ef5c2b885846ed9f22a87c28736ebd Mon Sep 17 00:00:00 2001
From: Erik Reid <erik.reid@geant.org>
Date: Thu, 4 Jun 2020 11:54:30 +0200
Subject: [PATCH] catch RedisError's and reset latch status
---
inventory_provider/tasks/worker.py | 46 +++++++++++++++++-------------
1 file changed, 26 insertions(+), 20 deletions(-)
diff --git a/inventory_provider/tasks/worker.py b/inventory_provider/tasks/worker.py
index e76e3f6b..d4779f6f 100644
--- a/inventory_provider/tasks/worker.py
+++ b/inventory_provider/tasks/worker.py
@@ -3,6 +3,8 @@ import logging
import os
import time
+from redis.exceptions import RedisError
+
from celery import Task, states
from celery.result import AsyncResult
@@ -548,26 +550,30 @@ def launch_refresh_cache_all(config):
:return:
"""
- _erase_next_db(config)
- monitor.clear_joblog(get_current_redis(config))
-
- update_latch_status(config, pending=True)
-
- # TODO: [DBOARD3-242] catch exceptions & reset latch status
-
- # first batch of subtasks: refresh cached opsdb data
- subtasks = [
- update_neteng_managed_device_list.apply_async(),
- update_interfaces_to_services.apply_async(),
- update_geant_lambdas.apply_async(),
- update_circuit_hierarchy.apply_async()
- ]
- [x.get() for x in subtasks]
-
- # now launch the task whose only purpose is to
- # act as a convenient parent for all of the remaining tasks
- t = internal_refresh_phase_2.apply_async()
- return t.id
+ try:
+ _erase_next_db(config)
+ monitor.clear_joblog(get_current_redis(config))
+
+ update_latch_status(config, pending=True)
+
+ # first batch of subtasks: refresh cached opsdb data
+ subtasks = [
+ update_neteng_managed_device_list.apply_async(),
+ update_interfaces_to_services.apply_async(),
+ update_geant_lambdas.apply_async(),
+ update_circuit_hierarchy.apply_async()
+ ]
+ [x.get() for x in subtasks]
+
+ # now launch the task whose only purpose is to
+ # act as a convenient parent for all of the remaining tasks
+ t = internal_refresh_phase_2.apply_async()
+ return t.id
+
+ except RedisError:
+ update_latch_status(config, pending=False, failure=True)
+ logger.exception('error launching refresh subtasks')
+ raise
def _wait_for_tasks(task_ids, update_callback=lambda s: None):
--
GitLab