cleanup events for deleted checks

brian_polling_manager/main.py

@@ -100,6 +100,9 @@ def refresh(config, force=False):
     }
     jsonschema.validate(result, REFRESH_RESULT_SCHEMA)  # sanity
 
+    # remove events for deleted checks, otherwise they stick around forever
+    sensu.clean_events(config['sensu'])
+
     statsd_config = config.get('statsd', None)
     if statsd_config:
         for key, counts in result.items():

brian_polling_manager/sensu.py

@@ -5,6 +5,7 @@ import copy
 import logging
 import random
 import requests
+from datetime import datetime
 
 logger = logging.getLogger(__name__)
 
@@ -100,6 +101,38 @@ def checks_match(a, b) -> bool:
     return True
 
 
+def clean_events(params, namespace='default'):
+    logger.info('cleaning events for deleted checks')
+    url = random.choice(params['api-base'])
+    r = requests.get(f'{url}/api/core/v2/namespaces/{namespace}/events',
+                     headers={'Authorization': f'Key {params["api-key"]}'})
+    r.raise_for_status()
+
+    events = r.json()
+
+    all_checks = load_all_checks(params, namespace)
+    check_names = {c['metadata']['name'] for c in all_checks}
+
+    with requests.Session() as session:
+        session.headers.update({'Authorization': f'Key {params["api-key"]}'})
+        for event in events:
+            if event['check']['metadata']['name'] not in check_names:
+                last_execution = event['check']['executed']
+                days_since_last_execution = (datetime.utcnow() - datetime.fromtimestamp(last_execution)).days
+
+                if days_since_last_execution <= 1:
+                    continue
+
+                logger.info(f'deleting event for deleted check: '
+                            f'{event["check"]["metadata"]["name"]}'
+                            f' (last execution: {days_since_last_execution} days ago)')
+                r = session.delete(
+                    f'{url}/api/core/v2/namespaces/{namespace}/events/'
+                    f'{event["entity"]["metadata"]["name"]}/'
+                    f'{event["check"]["metadata"]["name"]}')
+                r.raise_for_status()
+
+
 class AbstractCheck(object):
     """
     not explicitly using abc.ABC ... more readable than stacks of decorators
@@ -230,7 +263,6 @@ def refresh(sensu_params, required_checks, current_checks):
     :param current_checks: dict of {name:check_dict} from sensu
     :return: dict with change counts
     """
-
     # cf. main.REFRESH_RESULT_SCHEMA
     result = {
         'checks': len(current_checks),

tox.ini

@@ -2,6 +2,7 @@
 envlist = py36
 
 [flake8]
+max-line-length = 120
 exclude = venv,.tox
 
 [testenv]
@@ -12,7 +13,7 @@ deps =
 
 commands =
     coverage erase
-    coverage run --source brian_polling_manager -m py.test {posargs}
+    coverage run --source brian_polling_manager -m pytest {posargs}
     coverage xml
     coverage html
     coverage report