diff --git a/inventory_provider/tasks/common.py b/inventory_provider/tasks/common.py index 533295b34fdd0ea33eda914a9aacc06fb414b127..427d51d8a18e4128f3dd3b174274e65fa17f5f8f 100644 --- a/inventory_provider/tasks/common.py +++ b/inventory_provider/tasks/common.py @@ -1,5 +1,6 @@ import json import logging +import time import jsonschema import redis @@ -87,17 +88,24 @@ def update_latch_status(config, pending=False, failure=False): logger.debug('updating latch status: pending={}, failure={}'.format( pending, failure)) + now = time.time() for db in config['redis-databases']: r = _get_redis(config, dbid=db) latch = get_latch(r) if not latch: continue + if not pending and not failure: + if not latch['pending'] and not latch['failure']: + logger.error( + 'updating latch for db {db} with pending=failure=True, ' + f'but latch is already {latch}') + latch['timestamp'] = now latch['pending'] = pending latch['failure'] = failure r.set('db:latch', json.dumps(latch)) -def set_latch(config, new_current, new_next): +def set_latch(config, new_current, new_next, timestamp): logger.debug('setting latch: new current={}, new next={}'.format( new_current, new_next)) @@ -108,7 +116,8 @@ def set_latch(config, new_current, new_next): 'next': new_next, 'this': db, 'pending': False, - 'failure': False + 'failure': False, + 'timestamp': timestamp } r = _get_redis(config, dbid=db) @@ -130,7 +139,11 @@ def latch_db(config): next_idx = db_ids.index(latch['next']) next_idx = (next_idx + 1) % len(db_ids) - set_latch(config, new_current=latch['next'], new_next=db_ids[next_idx]) + set_latch( + config, + new_current=latch['next'], + new_next=db_ids[next_idx], + timestamp=time.time()) def _get_redis(config, dbid=None): diff --git a/inventory_provider/tasks/worker.py b/inventory_provider/tasks/worker.py index d0613236b21c8b987a92b94a878751ea14f7a9c4..9102731490f31548ccb06818d5e109196635dce2 100644 --- a/inventory_provider/tasks/worker.py +++ b/inventory_provider/tasks/worker.py @@ -541,7 +541,8 @@ def _erase_next_db(config): set_latch( config, new_current=saved_latch['current'], - new_next=saved_latch['next']) + new_next=saved_latch['next'], + timestamp=saved_latch.get('timestamp', 0)) @app.task(base=InventoryTask, bind=True, name='internal_refresh_phase_2')