diff --git a/brian_polling_manager/main.py b/brian_polling_manager/main.py index 90ba9f94c8841906efaa2f50bab5a7f4c518781e..1228b238a0b512b172b8cc631e8e646ca81e4797 100644 --- a/brian_polling_manager/main.py +++ b/brian_polling_manager/main.py @@ -100,6 +100,9 @@ def refresh(config, force=False): } jsonschema.validate(result, REFRESH_RESULT_SCHEMA) # sanity + # remove events for deleted checks, otherwise they stick around forever + sensu.clean_events(config['sensu']) + statsd_config = config.get('statsd', None) if statsd_config: for key, counts in result.items(): diff --git a/brian_polling_manager/sensu.py b/brian_polling_manager/sensu.py index 1ac0bf705ca11978dabdc480f7959e1591eec132..9252b6a3528838572f664c1a5242845cd4d11630 100644 --- a/brian_polling_manager/sensu.py +++ b/brian_polling_manager/sensu.py @@ -5,6 +5,7 @@ import copy import logging import random import requests +from datetime import datetime logger = logging.getLogger(__name__) @@ -100,6 +101,38 @@ def checks_match(a, b) -> bool: return True +def clean_events(params, namespace='default'): + logger.info('cleaning events for deleted checks') + url = random.choice(params['api-base']) + r = requests.get(f'{url}/api/core/v2/namespaces/{namespace}/events', + headers={'Authorization': f'Key {params["api-key"]}'}) + r.raise_for_status() + + events = r.json() + + all_checks = load_all_checks(params, namespace) + check_names = {c['metadata']['name'] for c in all_checks} + + with requests.Session() as session: + session.headers.update({'Authorization': f'Key {params["api-key"]}'}) + for event in events: + if event['check']['metadata']['name'] not in check_names: + last_execution = event['check']['executed'] + days_since_last_execution = (datetime.utcnow() - datetime.fromtimestamp(last_execution)).days + + if days_since_last_execution <= 1: + continue + + logger.info(f'deleting event for deleted check: ' + f'{event["check"]["metadata"]["name"]}' + f' (last execution: {days_since_last_execution} days ago)') + r = session.delete( + f'{url}/api/core/v2/namespaces/{namespace}/events/' + f'{event["entity"]["metadata"]["name"]}/' + f'{event["check"]["metadata"]["name"]}') + r.raise_for_status() + + class AbstractCheck(object): """ not explicitly using abc.ABC ... more readable than stacks of decorators @@ -230,7 +263,6 @@ def refresh(sensu_params, required_checks, current_checks): :param current_checks: dict of {name:check_dict} from sensu :return: dict with change counts """ - # cf. main.REFRESH_RESULT_SCHEMA result = { 'checks': len(current_checks), diff --git a/tox.ini b/tox.ini index ebb3d1ca90d07d59f47a42090cb6e06d477041c5..20d441f6d20caa681ffee4f78241e59791920b80 100644 --- a/tox.ini +++ b/tox.ini @@ -2,6 +2,7 @@ envlist = py36 [flake8] +max-line-length = 120 exclude = venv,.tox [testenv] @@ -12,7 +13,7 @@ deps = commands = coverage erase - coverage run --source brian_polling_manager -m py.test {posargs} + coverage run --source brian_polling_manager -m pytest {posargs} coverage xml coverage html coverage report