From ab0fb71f1da77be483fb50dfc65dc68df99f9cb9 Mon Sep 17 00:00:00 2001
From: Erik Reid <erik.reid@geant.org>
Date: Thu, 27 May 2021 16:07:18 +0200
Subject: [PATCH] get inventory interfaces, save state
---
brian_polling_manager/cli.py | 125 ++++++++++++++++++++++-------
brian_polling_manager/inventory.py | 122 ++++++++++++++++++++++++++++
2 files changed, 217 insertions(+), 30 deletions(-)
create mode 100644 brian_polling_manager/inventory.py
diff --git a/brian_polling_manager/cli.py b/brian_polling_manager/cli.py
index 0d2e1a7..0865ea7 100644
--- a/brian_polling_manager/cli.py
+++ b/brian_polling_manager/cli.py
@@ -1,11 +1,12 @@
import json
import logging
+import os
import re
import click
import jsonschema
-from brian_polling_manager import sensu
+from brian_polling_manager import sensu, inventory
logger = logging.getLogger(__name__)
@@ -21,7 +22,8 @@ _DEFAULT_CONFIG = {
'https://test-poller-sensu-agent03.geant.org:8080'
],
'token': '696a815c-607e-4090-81de-58988c83033e'
- }
+ },
+ 'statedir': '/tmp/'
}
CONFIG_SCHEMA = {
@@ -46,13 +48,76 @@ CONFIG_SCHEMA = {
'items': {'type': 'string'},
'minItems': 1
},
- 'sensu': {'$ref': '#/definitions/sensu'}
+ 'sensu': {'$ref': '#/definitions/sensu'},
+ 'statedir': {'type': 'string'}
},
- 'required': ['inventory', 'sensu'],
+ 'required': ['inventory', 'sensu', 'statedir'],
'additionalProperties': False
}
+class State(object):
+
+ INTERFACES = 'interfaces.json'
+ STATE = 'state.json'
+
+ STATE_SCHEMA = {
+ '$schema': 'http://json-schema.org/draft-07/schema#',
+ 'type': 'object',
+ 'properties': {
+ 'last': {'type': 'number'}
+ },
+ 'required': ['last'],
+ 'additionalProperties': False
+ }
+
+ def __init__(self, state_dir: str):
+ assert os.path.isdir(state_dir)
+ self.filenames = {
+ 'state': os.path.join(state_dir, State.STATE),
+ 'cache': os.path.join(state_dir, State.INTERFACES)
+ }
+
+ @staticmethod
+ def _load_json(filename, schema):
+ try:
+ with open(filename) as f:
+ state = json.loads(f.read())
+ jsonschema.validate(state, schema)
+ return state
+ except (json.JSONDecodeError, jsonschema.ValidationError, OSError):
+ logger.exception(
+ f'unable to open state file {filename}')
+ return None
+
+ @property
+ def last(self) -> int:
+ state = State._load_json(self.filenames['state'], State.STATE_SCHEMA)
+ return state['last'] if state else -1
+
+ @last.setter
+ def last(self, new_last: float):
+ state = {'last': new_last}
+ with open(self.filenames['state'], 'w') as f:
+ f.write(json.dumps(state))
+
+ @property
+ def interfaces(self) -> list:
+ return State._load_json(self.filenames['cache'], inventory.INVENTORY_INTERFACES_SCHEMA)
+
+ @interfaces.setter
+ def interfaces(self, new_interfaces):
+ try:
+ jsonschema.validate(new_interfaces, inventory.INVENTORY_INTERFACES_SCHEMA)
+ except jsonschema.ValidationError:
+ logger.exception('invalid interface state data')
+ return
+
+ with open(self.filenames['cache'], 'w') as f:
+ f.write(json.dumps(new_interfaces))
+
+
+
def _validate_config(ctx, param, value):
"""
loads, validates and returns configuration parameters
@@ -79,6 +144,14 @@ def _validate_config(ctx, param, value):
return config
+def load_ifc_checks(sensu_params):
+ def _is_ifc_check(check):
+ name = check['metadata']['name']
+ return re.match(r'^check-([^-]+\.geant\.net)-(.+)$', name)
+ ifc_checks = filter(_is_ifc_check, sensu.load_all_checks(sensu_params))
+ return {c['metadata']['name']:c for c in ifc_checks}
+
+
@click.command()
@click.option(
'--config',
@@ -86,35 +159,27 @@ def _validate_config(ctx, param, value):
type=click.STRING,
callback=_validate_config,
help='configuration filename')
-def main(config):
+@click.option(
+ '--force/--no-force',
+ default=False,
+ help="update even if inventory hasn't been updated")
+def main(config, force):
+ state = State(config['statedir'])
+ last = inventory.last_update_timestamp(config['inventory'])
+ if force or last != state.last:
+ interfaces = inventory.load_interfaces(config['inventory'])
+ state.last = last
+ state.interfaces = interfaces
+
+ assert interfaces
- def _categorize_ifc_check(check):
- name = check['metadata']['name']
- m = re.match(r'^check-([^-]+\.geant\.net)-(.+)$', name)
- if not m:
- return None
- return {
- 'hostname': m.group(1),
- 'interface': m.group(2),
- 'check': check
- }
- ifc_checks = {}
- for check in filter(None, map(_categorize_ifc_check, sensu.load_all_checks(config['sensu']))):
- host_checks = ifc_checks.setdefault(check['hostname'], {})
- if check['interface'] in host_checks:
- other_check = host_checks[check['interface']]
- other_name = other_check['metadata']['name']
- this_name = check['metadata']['name']
- logger.warning(f'found both {this_name} and {other_name}')
- host_checks[check['interface']] = check['check']
+ ifc_checks = load_ifc_checks(config['sensu'])
+ print(ifc_checks.keys())
- for host, interfaces in ifc_checks.items():
- print(host)
- for ifc, check in interfaces.items():
- name = check['metadata']['name']
- print(f'\t{ifc}: {name}')
+ print('here')
if __name__ == '__main__':
- main()
\ No newline at end of file
+ logging.basicConfig(level=logging.DEBUG)
+ main()
diff --git a/brian_polling_manager/inventory.py b/brian_polling_manager/inventory.py
new file mode 100644
index 0000000..b521104
--- /dev/null
+++ b/brian_polling_manager/inventory.py
@@ -0,0 +1,122 @@
+import logging
+import random
+
+import jsonschema
+import requests
+
+logger = logging.getLogger(__name__)
+
+# minimal inventory response schema
+INVENTORY_VERSION_SCHEMA = {
+ '$schema': 'http://json-schema.org/draft-07/schema#',
+
+ 'definitions': {
+ 'latch': {
+ 'type': 'object',
+ 'properties': {
+ 'timestamp': {'type': 'number'}
+ },
+ 'required': ['timestamp'],
+ 'additionalProperties': True
+ }
+ },
+
+ 'type': 'object',
+ 'properties': {
+ 'latch': {'$ref': '#/definitions/latch'}
+ },
+ 'required': ['latch'],
+ 'additionalProperties': True
+}
+
+INVENTORY_INTERFACES_SCHEMA = {
+ '$schema': 'http://json-schema.org/draft-07/schema#',
+
+ 'definitions': {
+ 'service': {
+ 'type': 'object',
+ 'properties': {
+ 'id': {'type': 'integer'},
+ 'name': {'type': 'string'},
+ 'type': {'type': 'string'},
+ 'status': {'type': 'string'},
+ },
+ 'required': ['id', 'name', 'type', 'status'],
+ 'additionalProperties': False
+ },
+ 'interface': {
+ 'type': 'object',
+ 'properties': {
+ 'router': {'type': 'string'},
+ 'name': {'type': 'string'},
+ 'description': {'type': 'string'},
+ 'snmp-index': {
+ 'type': 'integer',
+ 'minimum': 1
+ },
+ 'bundle': {
+ 'type': 'array',
+ 'items': {'type': 'string'}
+ },
+ 'bundle-parents': {
+ 'type': 'array',
+ 'items': {'type': 'string'}
+ },
+ 'circuits': {
+ 'type': 'array',
+ 'items': {'$ref': '#/definitions/service'}
+ }
+ },
+ 'required': [
+ 'router', 'name', 'description',
+ 'snmp-index', 'bundle', 'bundle-parents',
+ 'circuits'],
+ 'additionalProperties': False
+ },
+ },
+
+ 'type': 'array',
+ 'items': {'$ref': '#/definitions/interface'}
+}
+
+
+def _pick_one(things):
+ if not isinstance(things, (list, tuple, set)):
+ things = [things]
+ return random.choice(things)
+
+
+def load_interfaces(base_urls):
+ """
+ Load /poller/interfaces from inventory provider
+ and return a slightly reformatted dict.
+
+ :param base_urls: inventory provider base api url, or a list of them
+ :return: a dict like [<router>][<interface>] = inventory leaf data
+ """
+ url = _pick_one(base_urls)
+ logger.debug(f'using inventory base api url: {url}')
+
+ rsp = requests.get(
+ f'{url}/poller/interfaces',
+ headers={'Accept': 'application/json'})
+ rsp.raise_for_status()
+
+ result = rsp.json()
+ jsonschema.validate(result, INVENTORY_INTERFACES_SCHEMA)
+ return result
+
+
+def last_update_timestamp(base_urls) -> float:
+ try:
+ r = requests.get(
+ f'{_pick_one(base_urls)}/version',
+ headers={'Accept': 'application/json'})
+ r.raise_for_status()
+
+ result = r.json()
+ jsonschema.validate(result, INVENTORY_VERSION_SCHEMA)
+ return result['latch']['timestamp']
+ except (IOError, jsonschema.ValidationError, ValueError):
+ logger.exception('connection error')
+ return None
--
GitLab