diff --git a/.gitignore b/.gitignore
index 2258b7a4102ede94bd49bf3d1a65c7fc7f25142e..e087291d2752e55c0e2e301c1b7f2d675a4f5949 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,3 +1,7 @@
.idea
*.egg-info
__pycache__
+.tox
+coverage.xml
+.coverage
+htmlcov
diff --git a/Changelog.md b/Changelog.md
new file mode 100644
index 0000000000000000000000000000000000000000..573ab94989c7e4d33e8710f6f9ef2ffb4404289e
--- /dev/null
+++ b/Changelog.md
@@ -0,0 +1,6 @@
+# Changelog
+
+All notable changes to this project will be documented in this file.
+
+## [0.1] - 2021-06-01
+- initial release
diff --git a/MANIFEST.in b/MANIFEST.in
new file mode 100644
index 0000000000000000000000000000000000000000..5adb2b83cf466768d3165102fcdc58db524e6e4e
--- /dev/null
+++ b/MANIFEST.in
@@ -0,0 +1 @@
+include brian_polling_manager/logging_default_config.json
diff --git a/README.md b/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..cd6712732928ae185525c2de6176326663b7441e
--- /dev/null
+++ b/README.md
@@ -0,0 +1,9 @@
+# brian-polling-manager
+
+This module contains tooling for configuring
+SNMP polling for BRIAN.
+
+The documents should be viewable in the
+workspace of the most recent
+[Jenkins job](https://jenkins.geant.org/job/brian-polling-manager%20(develop)/ws/docs/build/html/index.html).
+
diff --git a/brian_polling_manager/__init__.py b/brian_polling_manager/__init__.py
index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..d3dbc19f3741c5d05983ac8f9878a570c22d5247 100644
--- a/brian_polling_manager/__init__.py
+++ b/brian_polling_manager/__init__.py
@@ -0,0 +1,37 @@
+"""
+automatically invoked app factory
+"""
+import logging
+import os
+
+from flask import Flask
+
+from brian_polling_manager import configuration
+
+CONFIG_KEY = 'CONFIG_PARAMS'
+SESSION_SECRET = 'super-secret'
+
+
+def create_app():
+ """
+ overrides default settings with those found
+ in the file read from env var CONFIG_FILENAME
+
+ :return: a new flask app instance
+ """
+
+ app = Flask(__name__)
+ app.secret_key = SESSION_SECRET
+
+ if 'CONFIG_FILENAME' in os.environ:
+ with open(os.environ['CONFIG_FILENAME']) as f:
+ app.config[CONFIG_KEY] = configuration.load_config(f)
+ else:
+ # this inits with the defaults
+ app.config[CONFIG_KEY] = configuration.load_config()
+
+ from brian_polling_manager import api
+ app.register_blueprint(api.routes, url_prefix='/api')
+
+ logging.info('Flask app initialized')
+ return app
diff --git a/brian_polling_manager/api.py b/brian_polling_manager/api.py
new file mode 100644
index 0000000000000000000000000000000000000000..424813f44665e73976f4681cf2b570d6d083c212
--- /dev/null
+++ b/brian_polling_manager/api.py
@@ -0,0 +1,163 @@
+"""
+These endpoints are used to update BRIAN snmp polling checks to Sensu
+
+.. contents:: :local:
+
+
+/api/version
+---------------------
+
+.. autofunction:: brian_polling_manager.api.version
+
+
+/api/update
+-----------------------------
+
+.. autofunction:: brian_polling_manager.api.update
+
+"""
+import functools
+import logging
+import pkg_resources
+
+from flask import Blueprint, current_app, request, Response, jsonify
+
+from brian_polling_manager import CONFIG_KEY
+from brian_polling_manager import main
+
+routes = Blueprint("api-routes", __name__)
+logger = logging.getLogger(__name__)
+
+
+API_VERSION = '0.1'
+
+VERSION_SCHEMA = {
+ '$schema': 'http://json-schema.org/draft-07/schema#',
+
+ 'definitions': {
+ 'latch': {
+ 'type': 'object',
+ 'properties': {
+ 'current': {'type': 'integer'},
+ 'next': {'type': 'integer'},
+ 'this': {'type': 'integer'},
+ 'failure': {'type': 'boolean'},
+ 'pending': {'type': 'boolean'},
+ 'timestamp': {'type': 'number'}
+ },
+ 'required': ['current', 'next', 'this', 'pending', 'failure'],
+ 'additionalProperties': False
+ }
+ },
+
+ 'type': 'object',
+ 'properties': {
+ 'api': {
+ 'type': 'string',
+ 'pattern': r'\d+\.\d+'
+ },
+ 'module': {
+ 'type': 'string',
+ 'pattern': r'\d+\.\d+'
+ },
+ 'latch': {'$ref': '#/definitions/latch'}
+ },
+ 'required': ['api', 'module'],
+ 'additionalProperties': False
+}
+
+
+class APIUpstreamError(Exception):
+ def __init__(self, message, status_code=504):
+ super().__init__()
+ self.message = message
+ self.status_code = status_code
+
+
+@routes.errorhandler(APIUpstreamError)
+def handle_request_error(error):
+ logger.error(f'api error: {error.message}')
+ return Response(
+ response=error.message,
+ status=error.status_code,
+ mimetype='text/html')
+
+
+def require_accepts_json(f):
+ """
+ used as a route handler decorator to return an error
+ unless the request allows responses with type "application/json"
+ :param f: the function to be decorated
+ :return: the decorated function
+ """
+ @functools.wraps(f)
+ def decorated_function(*args, **kwargs):
+ # TODO: use best_match to disallow */* ...?
+ if not request.accept_mimetypes.accept_json:
+ return Response(
+ response="response will be json",
+ status=406,
+ mimetype="text/html")
+ return f(*args, **kwargs)
+ return decorated_function
+
+
+@routes.after_request
+def after_request(response):
+ """
+ Generic function to do additional logging of requests & responses.
+
+ :param response:
+ :return:
+ """
+ if response.status_code != 200:
+
+ try:
+ data = response.data.decode('utf-8')
+ except Exception:
+ # never expected to happen, but we don't want any failures here
+ logging.exception('INTERNAL DECODING ERROR')
+ data = 'decoding error (see logs)'
+
+ logger.warning(
+ f'[{response.status_code}] {request.method} {request.path} {data}')
+ else:
+ logger.info(
+ f'[{response.status_code}] {request.method} {request.path}')
+ return response
+
+
+@routes.route('/version', methods=['GET', 'POST'])
+@require_accepts_json
+def version():
+ """
+ Returns a json object with information about the module version.
+
+ The response will be formatted according to the following schema:
+
+ .. asjson:: brian_polling_manager.api.VERSION_SCHEMA
+
+ :return:
+ """
+ version_params = {
+ 'api': API_VERSION,
+ 'module':
+ pkg_resources.get_distribution('brian_polling_manager').version
+ }
+ return jsonify(version_params)
+
+
+@routes.route('/update', methods=['GET', 'POST'])
+@require_accepts_json
+def update():
+ """
+ Update checks based on current inventory provider state.
+
+ The response will be formatted according to the following schema:
+
+ .. asjson:: brian_polling_manager.main.REFRESH_RESULT_SCHEMA
+
+ :return:
+ """
+ response = main.refresh(config=current_app.config[CONFIG_KEY])
+ return jsonify(response)
diff --git a/brian_polling_manager/app.py b/brian_polling_manager/app.py
new file mode 100644
index 0000000000000000000000000000000000000000..c998a1b8bf4c650f23cfe981be3cf5c3f479e4be
--- /dev/null
+++ b/brian_polling_manager/app.py
@@ -0,0 +1,9 @@
+"""
+default app creation
+"""
+import brian_polling_manager
+
+app = brian_polling_manager.create_app()
+
+if __name__ == "__main__":
+ app.run(host="::", port="7878")
diff --git a/brian_polling_manager/cli.py b/brian_polling_manager/configuration.py
similarity index 62%
rename from brian_polling_manager/cli.py
rename to brian_polling_manager/configuration.py
index e9015a357db3a3362763571069fef1850697b281..bbb136dd4601b54c3863813d80a39eb4b2463f6d 100644
--- a/brian_polling_manager/cli.py
+++ b/brian_polling_manager/configuration.py
@@ -1,14 +1,19 @@
import json
import logging
+import logging.config
import os
+from typing import Union
-import click
import jsonschema
-from brian_polling_manager import inventory, interfaces
+from brian_polling_manager import inventory
logger = logging.getLogger(__name__)
+_DEFAULT_LOGGING_FILENAME = os.path.join(
+ os.path.dirname(__file__),
+ 'logging_default_config.json')
+
_DEFAULT_CONFIG = {
'inventory': [
'http://test-inventory-provider01.geant.org:8080',
@@ -20,7 +25,7 @@ _DEFAULT_CONFIG = {
'https://test-poller-sensu-agent02.geant.org:8080',
'https://test-poller-sensu-agent03.geant.org:8080'
],
- 'token': '696a815c-607e-4090-81de-58988c83033e',
+ 'api-key': '696a815c-607e-4090-81de-58988c83033e',
'interface-check': {
'script': '/var/lib/sensu/bin/counter2influx.sh',
'measurement': 'counters',
@@ -29,10 +34,18 @@ _DEFAULT_CONFIG = {
'output_metric_handlers': ['influx-db-handler'],
'namespace': 'default',
'round_robin': True,
- 'command': '{script} {measurement} {community} {hostname} {interface} {ifIndex}',
+ 'command': ('{script} {measurement} '
+ '{community} {hostname} '
+ '{interface} {ifIndex}'),
}
},
- 'statedir': '/tmp/'
+ 'statedir': '/tmp/',
+ 'logging': _DEFAULT_LOGGING_FILENAME,
+ 'statsd': {
+ 'hostname': 'localhost',
+ 'port': 8125,
+ 'prefix': 'brian_polling'
+ }
}
CONFIG_SCHEMA = {
@@ -69,10 +82,20 @@ CONFIG_SCHEMA = {
'items': {'type': 'string'},
'minItems': 1
},
- 'token': {'type': 'string'},
+ 'api-key': {'type': 'string'},
'interface-check': {'$ref': '#/definitions/influx-check'}
},
- 'required': ['api-base', 'token', 'interface-check'],
+ 'required': ['api-base', 'api-key', 'interface-check'],
+ 'additionalProperties': False
+ },
+ 'statsd': {
+ 'type': 'object',
+ 'properties': {
+ 'hostname': {'type': 'string'},
+ 'port': {'type': 'integer'},
+ 'prefix': {'type': 'string'}
+ },
+ 'required': ['hostname', 'port', 'prefix'],
'additionalProperties': False
}
},
@@ -84,7 +107,9 @@ CONFIG_SCHEMA = {
'minItems': 1
},
'sensu': {'$ref': '#/definitions/sensu'},
- 'statedir': {'type': 'string'}
+ 'statedir': {'type': 'string'},
+ 'logging': {'type': 'string'},
+ 'statsd': {'$ref': '#/definitions/statsd'}
},
'required': ['inventory', 'sensu', 'statedir'],
'additionalProperties': False
@@ -131,19 +156,26 @@ class State(object):
return state['last'] if state else -1
@last.setter
- def last(self, new_last: float):
- state = {'last': new_last}
- with open(self.filenames['state'], 'w') as f:
- f.write(json.dumps(state))
+ def last(self, new_last: Union[float, None]):
+ if not new_last or new_last < 0:
+ os.unlink(self.filenames['state'])
+ else:
+ state = {'last': new_last}
+ with open(self.filenames['state'], 'w') as f:
+ f.write(json.dumps(state))
@property
def interfaces(self) -> list:
- return State._load_json(self.filenames['cache'], inventory.INVENTORY_INTERFACES_SCHEMA)
+ return State._load_json(
+ self.filenames['cache'],
+ inventory.INVENTORY_INTERFACES_SCHEMA)
@interfaces.setter
def interfaces(self, new_interfaces):
try:
- jsonschema.validate(new_interfaces, inventory.INVENTORY_INTERFACES_SCHEMA)
+ jsonschema.validate(
+ new_interfaces,
+ inventory.INVENTORY_INTERFACES_SCHEMA)
except jsonschema.ValidationError:
logger.exception('invalid interface state data')
return
@@ -152,56 +184,41 @@ class State(object):
f.write(json.dumps(new_interfaces))
-def _validate_config(ctx, param, value):
+def _setup_logging(filename=None):
+ """
+ set up logging using the configured filename
+
+ :raises: json.JSONDecodeError, OSError, AttributeError,
+ ValueError, TypeError, ImportError
+ """
+ if not filename:
+ filename = _DEFAULT_LOGGING_FILENAME
+
+ with open(filename) as f:
+ # TODO: this mac workaround should be removed ...
+ d = json.loads(f.read())
+ import platform
+ if platform.system() == 'Darwin':
+ d['handlers']['syslog_handler']['address'] = '/var/run/syslog'
+ logging.config.dictConfig(d)
+ # logging.config.dictConfig(json.loads(f.read()))
+
+
+def load_config(file=None):
"""
loads, validates and returns configuration parameters
- :param ctx:
- :param param:
- :param value: filename (string)
+ :param value: filename (file-like object, opened for reading)
:return: a dict containing configuration parameters
+ :raises: json.JSONDecodeError, jsonschema.ValidationError,
+ OSError, AttributeError, ValueError, TypeError, ImportError
"""
- if value is None:
+ if not file:
config = _DEFAULT_CONFIG
else:
- try:
- with open(value) as f:
- config = json.loads(f.read())
- except (json.JSONDecodeError, jsonschema.ValidationError) as e:
- raise click.BadParameter(str(e))
-
- try:
+ config = json.loads(file.read())
jsonschema.validate(config, CONFIG_SCHEMA)
- except jsonschema.ValidationError as e:
- raise click.BadParameter(str(e))
-
- return config
-
+ _setup_logging(config.get('logging', None))
-
-@click.command()
-@click.option(
- '--config',
- default=None,
- type=click.STRING,
- callback=_validate_config,
- help='configuration filename')
-@click.option(
- '--force/--no-force',
- default=False,
- help="update even if inventory hasn't been updated")
-def main(config, force):
-
- state = State(config['statedir'])
- last = inventory.last_update_timestamp(config['inventory'])
- if force or last != state.last:
- state.last = last
- state.interfaces = inventory.load_interfaces(config['inventory'])
-
- interfaces.refresh(config['sensu'], state)
-
-
-if __name__ == '__main__':
- logging.basicConfig(level=logging.DEBUG)
- main()
+ return config
diff --git a/brian_polling_manager/interfaces.py b/brian_polling_manager/interfaces.py
index f50b975cafd3a0fa87b8926505682d96c73bd1e7..2c0adef6f595f21ee42d64d32c1a89394cf40727 100644
--- a/brian_polling_manager/interfaces.py
+++ b/brian_polling_manager/interfaces.py
@@ -5,17 +5,20 @@ from brian_polling_manager import sensu
logger = logging.getLogger(__name__)
+
def load_ifc_checks(sensu_params):
def _is_ifc_check(check):
name = check['metadata']['name']
- return re.match(r'^check-([^-]+\.geant\.net)-(.+)$', name)
+ # check-* is the old-style name (add to the returned
+ # data so it can be deleted)
+ return re.match(r'^(check|ifc)-([^-]+\.geant\.net)-(.+)$', name)
ifc_checks = filter(_is_ifc_check, sensu.load_all_checks(sensu_params))
- return {c['metadata']['name']:c for c in ifc_checks}
+ return {c['metadata']['name']: c for c in ifc_checks}
def _check_name(interface):
ifc_name = interface['name'].replace('/', '-')
- return f'check-{interface["router"]}-{ifc_name}'
+ return f'ifc-{interface["router"]}-{ifc_name}'
def _make_check(check_params, interface):
@@ -35,7 +38,8 @@ def _make_check(check_params, interface):
'proxy_entity_name': interface['router'],
'round_robin': check_params['round_robin'],
'output_metric_format': 'influxdb_line',
- 'output_metric_handlers': sorted(check_params['output_metric_handlers']),
+ 'output_metric_handlers': sorted(
+ check_params['output_metric_handlers']),
'metadata': {
'name': _check_name(interface),
'namespace': check_params['namespace']
@@ -44,8 +48,8 @@ def _make_check(check_params, interface):
def _checks_match(a, b) -> bool:
- # if a['command'] != b['command']:
- # return False
+ if a['command'] != b['command']:
+ return False
if a['interval'] != b['interval']:
return False
if a['proxy_entity_name'] != b['proxy_entity_name']:
@@ -56,7 +60,8 @@ def _checks_match(a, b) -> bool:
return False
if sorted(a['subscriptions']) != sorted(b['subscriptions']):
return False
- if sorted(a['output_metric_handlers']) != sorted(b['output_metric_handlers']):
+ if sorted(a['output_metric_handlers']) \
+ != sorted(b['output_metric_handlers']):
return False
if a['metadata']['name'] != b['metadata']['name']:
return False
@@ -69,17 +74,34 @@ def refresh(sensu_params, state):
ifc_checks = load_ifc_checks(sensu_params)
+ created = 0
+ updated = 0
+ interfaces = 0
for interface in state.interfaces:
- expected_check = _make_check(sensu_params['interface-check'], interface)
+ interfaces += 1
+
+ expected_check = _make_check(
+ sensu_params['interface-check'], interface)
expected_name = _check_name(interface)
if expected_name not in ifc_checks:
+ created += 1
sensu.create_check(sensu_params, expected_check)
elif not _checks_match(ifc_checks[expected_name], expected_check):
+ updated += 1
sensu.update_check(sensu_params, expected_check)
wanted_checks = {_check_name(ifc) for ifc in state.interfaces}
extra_checks = set(ifc_checks.keys()) - wanted_checks
for name in extra_checks:
sensu.delete_check(sensu_params, name)
+
+ # cf. main.REFRESH_RESULT_SCHEMA
+ return {
+ 'checks': len(ifc_checks),
+ 'input': interfaces,
+ 'created': created,
+ 'updated': updated,
+ 'deleted': len(extra_checks)
+ }
diff --git a/brian_polling_manager/logging_default_config.json b/brian_polling_manager/logging_default_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..bfab907634c09de48cbd2f0bd29c28bf292753dd
--- /dev/null
+++ b/brian_polling_manager/logging_default_config.json
@@ -0,0 +1,59 @@
+{
+ "version": 1,
+ "disable_existing_loggers": false,
+ "formatters": {
+ "simple": {
+ "format": "%(asctime)s - %(name)s (%(lineno)d) - %(levelname)s - %(message)s"
+ }
+ },
+
+ "handlers": {
+ "console": {
+ "class": "logging.StreamHandler",
+ "level": "DEBUG",
+ "formatter": "simple",
+ "stream": "ext://sys.stdout"
+ },
+
+ "syslog_handler": {
+ "class": "logging.handlers.SysLogHandler",
+ "level": "DEBUG",
+ "address": "/dev/log",
+ "facility": "user",
+ "formatter": "simple"
+ },
+
+ "info_file_handler": {
+ "class": "logging.handlers.RotatingFileHandler",
+ "level": "INFO",
+ "formatter": "simple",
+ "filename": "info.log",
+ "maxBytes": 10485760,
+ "backupCount": 20,
+ "encoding": "utf8"
+ },
+
+ "error_file_handler": {
+ "class": "logging.handlers.RotatingFileHandler",
+ "level": "ERROR",
+ "formatter": "simple",
+ "filename": "errors.log",
+ "maxBytes": 10485760,
+ "backupCount": 20,
+ "encoding": "utf8"
+ }
+ },
+
+ "loggers": {
+ "brian_polling_manager": {
+ "level": "DEBUG",
+ "handlers": ["console", "syslog_handler"],
+ "propagate": false
+ }
+ },
+
+ "root": {
+ "level": "DEBUG",
+ "handlers": ["console", "syslog_handler"]
+ }
+}
diff --git a/brian_polling_manager/main.py b/brian_polling_manager/main.py
new file mode 100644
index 0000000000000000000000000000000000000000..5e92a164b9706344ec9d558fd37ed2fd2fdc4473
--- /dev/null
+++ b/brian_polling_manager/main.py
@@ -0,0 +1,135 @@
+"""
+This script queries Inventory Provider for changes
+and configures Sensu with the snmp polling checks
+required by BRIAN.
+
+.. code-block:: console
+
+ % brian-polling-manager --help
+ Usage: brian-polling-manager [OPTIONS]
+
+ Update BRIAN snmp checks based on Inventory Provider data.
+
+ Options:
+ --config TEXT configuration filename
+ --force / --no-force update even if inventory hasn't been updated
+ --help Show this message and exit.
+
+The required configuration file must be
+formatted according to the following schema:
+
+.. asjson::
+ brian_polling_manager.configuration.CONFIG_SCHEMA
+
+"""
+import json
+import logging
+
+import click
+import jsonschema
+from statsd import StatsClient
+
+from brian_polling_manager import inventory, interfaces, configuration
+
+logger = logging.getLogger(__name__)
+
+REFRESH_RESULT_SCHEMA = {
+ '$schema': 'http://json-schema.org/draft-07/schema#',
+ 'definitions': {
+ 'refresh-result': {
+ 'type': 'object',
+ 'properties': {
+ 'checks': {'type': 'integer'},
+ 'input': {'type': 'integer'},
+ 'created': {'type': 'integer'},
+ 'updated': {'type': 'integer'},
+ 'deleted': {'type': 'integer'}
+ },
+ 'required': ['checks', 'input', 'created', 'updated', 'deleted'],
+ 'additionalProperties': False
+ }
+ },
+ 'type': 'object',
+ 'properties': {
+ 'interfaces': {'$ref': '#/definitions/refresh-result'}
+ },
+ 'required': ['interfaces'],
+ 'additionalProperties': False
+}
+
+
+def refresh(config, force=False):
+ """
+ reload inventory data & update sensu checks
+
+ The output will be a dict formatted according to the following schema:
+
+ .. asjson::
+ brian_polling_manager.main.REFRESH_RESULT_SCHEMA
+
+ :param config: a dict returned by configuration.load_config
+ :param force: if True, reload inventory data even if timestamp is same
+ :return: a dict, formatted as above
+ """
+ state = configuration.State(config['statedir'])
+ last = inventory.last_update_timestamp(config['inventory'])
+ if force or not last or last != state.last:
+ state.last = last
+ state.interfaces = inventory.load_interfaces(config['inventory'])
+
+ result = {
+ 'interfaces': interfaces.refresh(config['sensu'], state)
+ }
+
+ statsd_config = config.get('statsd', None)
+ if statsd_config:
+ statsd = StatsClient(
+ host=statsd_config['hostname'],
+ port=statsd_config['port'],
+ prefix=f'{statsd_config["prefix"]}_interfaces')
+ statsd.gauge('checks', result['interfaces']['checks'])
+ statsd.gauge('input', result['interfaces']['input'])
+ statsd.gauge('created', result['interfaces']['created'])
+ statsd.gauge('updated', result['interfaces']['updated'])
+ statsd.gauge('deleted', result['interfaces']['deleted'])
+
+ jsonschema.validate(result, REFRESH_RESULT_SCHEMA) # sanity
+ return result
+
+
+def _validate_config(_ctx, _param, file):
+ """
+ loads, validates and returns configuration parameters
+
+ :param _ctx: unused
+ :param _param: unused
+ :param value: file (file-like object open for reading)
+ :return: a dict containing configuration parameters
+ """
+ try:
+ return configuration.load_config(file)
+ except (json.JSONDecodeError, jsonschema.ValidationError, OSError,
+ AttributeError, ValueError, TypeError, ImportError) as e:
+ raise click.BadParameter(str(e))
+
+
+@click.command()
+@click.option(
+ '--config',
+ default=None,
+ type=click.File('r'),
+ callback=_validate_config,
+ help='configuration filename')
+@click.option(
+ '--force/--no-force',
+ default=False,
+ help="refresh inventory data even if it hasn't been updated")
+def cli(config, force):
+ """
+ Update BRIAN snmp checks based on Inventory Provider data.
+ """
+ logger.info(json.dumps(refresh(config, force)))
+
+
+if __name__ == '__main__':
+ cli()
diff --git a/brian_polling_manager/sensu.py b/brian_polling_manager/sensu.py
index 246e68996b96f6f8d19f3fa53e9bdc5d9448853d..0c60d091320f7ddfbc8a94aa5423b34c7a1dc9f2 100644
--- a/brian_polling_manager/sensu.py
+++ b/brian_polling_manager/sensu.py
@@ -9,13 +9,11 @@ logger = logging.getLogger(__name__)
def load_all_checks(params, namespace='default'):
-
url = random.choice(params['api-base'])
- # url = params['api-base'][0]
r = requests.get(
f'{url}/api/core/v2/namespaces/{namespace}/checks',
headers={
- 'Authorization': f'Key {params["token"]}',
+ 'Authorization': f'Key {params["api-key"]}',
'Accepts': 'application/json',
})
r.raise_for_status()
@@ -25,30 +23,30 @@ def load_all_checks(params, namespace='default'):
def create_check(params, check, namespace='default'):
- logger.error(f'creating missing check: {check["metadata"]["name"]}')
- # url = random.choice(params['api-base'])
- # r = requests.post(
- # f'{url}/api/core/v2/namespaces/{namespace}/checks',
- # headers={
- # 'Authorization': f'Key {params["token"]}',
- # 'Content-Type': 'application/json',
- # },
- # json=check)
- # r.raise_for_status()
+ logger.info(f'creating missing check: {check["metadata"]["name"]}')
+ url = random.choice(params['api-base'])
+ r = requests.post(
+ f'{url}/api/core/v2/namespaces/{namespace}/checks',
+ headers={
+ 'Authorization': f'Key {params["api-key"]}',
+ 'Content-Type': 'application/json',
+ },
+ json=check)
+ r.raise_for_status()
def update_check(params, check, namespace='default'):
name = check["metadata"]["name"]
- logger.error(f'updating existing check: {name}')
- # url = random.choice(params['api-base'])
- # r = requests.post(
- # f'{url}/api/core/v2/namespaces/{namespace}/checks/{name}',
- # headers={
- # 'Authorization': f'Key {params["token"]}',
- # 'Content-Type': 'application/json',
- # },
- # json=check)
- # r.raise_for_status()
+ logger.info(f'updating existing check: {name}')
+ url = random.choice(params['api-base'])
+ r = requests.put(
+ f'{url}/api/core/v2/namespaces/{namespace}/checks/{name}',
+ headers={
+ 'Authorization': f'Key {params["api-key"]}',
+ 'Content-Type': 'application/json',
+ },
+ json=check)
+ r.raise_for_status()
def delete_check(params, check, namespace='default'):
@@ -56,9 +54,9 @@ def delete_check(params, check, namespace='default'):
name = check
else:
name = check["metadata"]["name"]
- logger.error(f'deleting unwanted check: {name}')
- # url = random.choice(params['api-base'])
- # r = requests.delete(
- # f'{url}/api/core/v2/namespaces/{namespace}/checks',
- # headers={'Authorization': f'Key {params["token"]}'})
- # r.raise_for_status()
+ logger.info(f'deleting unwanted check: {name}')
+ url = random.choice(params['api-base'])
+ r = requests.delete(
+ f'{url}/api/core/v2/namespaces/{namespace}/checks/{name}',
+ headers={'Authorization': f'Key {params["api-key"]}'})
+ r.raise_for_status()
diff --git a/docs/Makefile b/docs/Makefile
new file mode 100644
index 0000000000000000000000000000000000000000..d0c3cbf1020d5c292abdedf27627c6abe25e2293
--- /dev/null
+++ b/docs/Makefile
@@ -0,0 +1,20 @@
+# Minimal makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line, and also
+# from the environment for the first two.
+SPHINXOPTS ?=
+SPHINXBUILD ?= sphinx-build
+SOURCEDIR = source
+BUILDDIR = build
+
+# Put it first so that "make" without argument is like "make help".
+help:
+ @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+
+.PHONY: help Makefile
+
+# Catch-all target: route all unknown targets to Sphinx using the new
+# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS).
+%: Makefile
+ @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
diff --git a/docs/source/api.rst b/docs/source/api.rst
new file mode 100644
index 0000000000000000000000000000000000000000..efea063542a57dd21192254370ab61c5dc45f421
--- /dev/null
+++ b/docs/source/api.rst
@@ -0,0 +1,6 @@
+
+HTTP API
+===================================================
+
+.. automodule:: brian_polling_manager.api
+
diff --git a/docs/source/conf.py b/docs/source/conf.py
new file mode 100644
index 0000000000000000000000000000000000000000..2db47d2af9be401adc20354305d350e8eb9ef255
--- /dev/null
+++ b/docs/source/conf.py
@@ -0,0 +1,100 @@
+# Configuration file for the Sphinx documentation builder.
+#
+# This file only contains a selection of the most common options. For a full
+# list see the documentation:
+# https://www.sphinx-doc.org/en/master/usage/configuration.html
+
+# -- Path setup --------------------------------------------------------------
+
+# If extensions (or modules to document with autodoc) are in another directory,
+# add these directories to sys.path here. If the directory is relative to the
+# documentation root, use os.path.abspath to make it absolute, like shown here.
+#
+# import os
+# import sys
+# sys.path.insert(0, os.path.abspath('.'))
+
+from importlib import import_module
+from docutils.parsers.rst import Directive
+from docutils import nodes
+from sphinx import addnodes
+import json
+import os
+import sys
+
+sys.path.insert(0, os.path.abspath(
+ os.path.join(
+ os.path.dirname(__file__),
+ '..', '..', 'brian_polling_manager')))
+
+
+class RenderAsJSON(Directive):
+ # cf. https://stackoverflow.com/a/59883833
+
+ required_arguments = 1
+
+ def run(self):
+ module_path, member_name = self.arguments[0].rsplit('.', 1)
+
+ member_data = getattr(import_module(module_path), member_name)
+ code = json.dumps(member_data, indent=2)
+
+ literal = nodes.literal_block(code, code)
+ literal['language'] = 'json'
+
+ return [
+ addnodes.desc_name(text=member_name),
+ addnodes.desc_content('', literal)
+ ]
+
+
+def setup(app):
+ app.add_directive('asjson', RenderAsJSON)
+
+
+# -- Project information -----------------------------------------------------
+
+project = 'BRIAN Polling Manager'
+copyright = '2021, swd@geant.org'
+author = 'swd@geant.org'
+
+# The full version, including alpha/beta/rc tags
+release = '0.0.1'
+
+
+# -- General configuration ---------------------------------------------------
+
+# Add any Sphinx extension module names here, as strings. They can be
+# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
+# ones.
+extensions = [
+ 'sphinx_rtd_theme',
+ 'sphinx.ext.autodoc',
+ 'sphinx.ext.coverage'
+]
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ['_templates']
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+# This pattern also affects html_static_path and html_extra_path.
+exclude_patterns = []
+
+
+# -- Options for HTML output -------------------------------------------------
+
+# The theme to use for HTML and HTML Help pages. See the documentation for
+# a list of builtin themes.
+#
+html_theme = 'sphinx_rtd_theme'
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+html_static_path = ['_static']
+
+# Both the class’ and the __init__ method’s docstring
+# are concatenated and inserted.
+autoclass_content = "both"
+autodoc_typehints = "none"
diff --git a/docs/source/index.rst b/docs/source/index.rst
new file mode 100644
index 0000000000000000000000000000000000000000..006d51f03004ee54b7dd4a21996274b94e633c8a
--- /dev/null
+++ b/docs/source/index.rst
@@ -0,0 +1,20 @@
+.. BRIAN Polling Manager documentation master file, created by
+ sphinx-quickstart on Mon May 31 09:43:14 2021.
+ You can adapt this file completely to your liking, but it should at least
+ contain the root `toctree` directive.
+
+
+BRIAN Polling Manager
+======================
+
+Documentation for BRIAN Polling Manager.
+This application queries Inventory Provider and configures
+Sensu checks for polling the data required by BRIAN.
+
+
+.. toctree::
+ :maxdepth: 2
+ :caption: Contents:
+
+ main
+ api
diff --git a/docs/source/main.rst b/docs/source/main.rst
new file mode 100644
index 0000000000000000000000000000000000000000..4ef198ec2450507e2c450a9ce8c6c13bc548b8c3
--- /dev/null
+++ b/docs/source/main.rst
@@ -0,0 +1,6 @@
+
+brian-polling-manager
+===================================================
+
+.. automodule:: brian_polling_manager.main
+
diff --git a/requirements.txt b/requirements.txt
index a66ebff5328f50b5d15353f4b9c6febd21ae94e6..a5a0af502b4e289530624c546268288480910240 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,3 +1,10 @@
click
jsonschema
requests
+statsd
+flask
+
+pytest
+responses
+sphinx
+sphinx-rtd-theme
diff --git a/setup.py b/setup.py
index 6e52a005fe6a3372b5476e5ccb1e464e57a13053..59ddd2c76c53cfbd00d4366fd7c9ac62f482ac9d 100644
--- a/setup.py
+++ b/setup.py
@@ -2,7 +2,7 @@ from setuptools import setup, find_packages
setup(
name='brian-polling-manager',
- version='0.1',
+ version="0.1",
author='GEANT',
author_email='swd@geant.org',
description='service for managing BRIAN polling checks',
@@ -11,12 +11,14 @@ setup(
install_requires=[
'click',
'requests',
- 'jsonschema'
+ 'jsonschema',
+ 'statsd',
+ 'flask'
],
entry_points={
'console_scripts': [
- 'cli=brian_polling_manager.cli:main'
+ 'brian-polling-manager=brian_polling_manager.main:cli'
]
},
+ include_package_data=True,
)
-
diff --git a/test/conftest.py b/test/conftest.py
new file mode 100644
index 0000000000000000000000000000000000000000..2553e8fcafecbeb13bf967ab4ae92e28ebe5f704
--- /dev/null
+++ b/test/conftest.py
@@ -0,0 +1,177 @@
+import json
+import os
+import random
+import re
+import tempfile
+
+import jsonschema
+import pytest
+import responses
+
+
+def _load_test_data(filename):
+ full_path = os.path.join(
+ os.path.dirname(__file__),
+ 'data', filename)
+ with open(full_path) as f:
+ return f.read()
+
+
+@pytest.fixture
+def config():
+ with tempfile.TemporaryDirectory() as state_dir_name:
+ yield {
+ 'inventory': [
+ 'http://bogus-inventory01.xxx.yyy:12345',
+ 'http://bogus-inventory02.xxx.yyy:12345',
+ 'http://bogus-inventory03.xxx.yyy:12345'
+ ],
+ 'sensu': {
+ 'api-base': [
+ 'https://bogus-sensu01.xxx.yyy:12345',
+ 'https://bogus-sensu02.xxx.yyy:12345',
+ 'https://bogus-sensu03.xxx.yyy:12345'
+ ],
+ 'api-key': 'abc-sensu-key-blah-blah',
+ 'interface-check': {
+ 'script': '/var/lib/sensu/bin/counter2influx.sh',
+ 'measurement': 'counters',
+ 'interval': 300,
+ 'subscriptions': ['interfacecounters'],
+ 'output_metric_handlers': ['influx-db-handler'],
+ 'namespace': 'default',
+ 'round_robin': True,
+ 'command': ('{script} {measurement} '
+ '{community} {hostname} '
+ '{interface} {ifIndex}'),
+ }
+ },
+ 'statedir': state_dir_name,
+ 'statsd': {
+ 'hostname': 'localhost',
+ 'port': 11119,
+ 'prefix': 'zzzzz'
+ }
+ }
+
+
+@pytest.fixture
+def config_filename(config):
+ with tempfile.NamedTemporaryFile(mode='w') as f:
+ f.write(json.dumps(config))
+ f.flush()
+ yield f.name
+
+
+@pytest.fixture
+def mocked_sensu():
+
+ saved_sensu_checks = {
+ c['metadata']['name']: c
+ for c in json.loads(_load_test_data('checks.json'))}
+
+ _check_schema = {
+ '$schema': 'http://json-schema.org/draft-07/schema#',
+
+ 'type': 'object',
+ 'properties': {
+ 'command': {'type': 'string'},
+ 'interval': {'type': 'integer'},
+ 'subscriptions': {'type': 'array', 'items': {'type': 'string'}},
+ 'proxy_entity_name': {'type': 'string'},
+ 'round_robin': {'type': 'boolean'},
+ 'output_metric_format': {'enum': ['influxdb_line']},
+ 'output_metric_handlers': {
+ 'type': 'array',
+ 'items': {'type': 'string'}
+ },
+ 'metadata': {
+ 'type': 'object',
+ 'properties': {
+ 'name': {'type': 'string'},
+ 'namespace': {'type': 'string'}
+ },
+ 'required': ['name', 'namespace'],
+ 'additionalProperties': True
+ },
+ },
+ 'required': [
+ 'command', 'interval', 'subscriptions',
+ 'output_metric_format', 'output_metric_handlers',
+ 'round_robin', 'metadata'],
+ 'additionalProperties': True
+ }
+
+ # mocked api for returning all checks
+ responses.add(
+ method=responses.GET,
+ url=re.compile(r'.*sensu.+/api/core/v2/namespaces/[^\/]+/checks$'),
+ json=list(saved_sensu_checks.values())
+ )
+
+ def new_check_callback(request):
+ check = json.loads(request.body)
+ jsonschema.validate(check, _check_schema)
+ path_elems = request.path_url.split('/')
+ assert len(path_elems) == 7 # sanity
+ assert path_elems[5] == check['metadata']['namespace']
+ assert check['metadata']['name'] not in saved_sensu_checks
+ saved_sensu_checks[check['metadata']['name']] = check
+ return (201, {}, '')
+
+ # mocked api for creating a check
+ responses.add_callback(
+ method=responses.POST,
+ url=re.compile(r'.*sensu.+/api/core/v2/namespaces/[^\/]+/checks$'),
+ callback=new_check_callback)
+
+ def update_check_callback(request):
+ check = json.loads(request.body)
+ jsonschema.validate(check, _check_schema)
+ path_elems = request.path_url.split('/')
+ assert len(path_elems) == 8 # sanity
+ assert path_elems[5] == check['metadata']['namespace']
+ assert path_elems[-1] == check['metadata']['name']
+ assert check['metadata']['name'] in saved_sensu_checks, \
+ 'we only intend to call this method for updating existing checks'
+ saved_sensu_checks[check['metadata']['name']] = check
+ return 201, {}, ''
+
+ # mocked api for updating a check
+ responses.add_callback(
+ method=responses.PUT,
+ url=re.compile(
+ r'.*sensu.+/api/core/v2/namespaces/[^\/]+/checks/[^\/]+$'),
+ callback=update_check_callback)
+
+ def delete_check_callback(request):
+ path_elems = request.path_url.split('/')
+ assert len(path_elems) == 8 # sanity
+ del saved_sensu_checks[path_elems[-1]]
+ return 204, {}, ''
+
+ # mocked api for deleting a check
+ responses.add_callback(
+ method=responses.DELETE,
+ url=re.compile(
+ r'.*sensu.+/api/core/v2/namespaces/[^\/]+/checks/[^\/]+$'),
+ callback=delete_check_callback)
+
+ yield saved_sensu_checks
+
+
+@pytest.fixture
+def mocked_inventory():
+
+ # mocked api for returning all checks
+ responses.add(
+ method=responses.GET,
+ url=re.compile(r'.*inventory.+/poller/interfaces.*'),
+ body=_load_test_data('interfaces.json'))
+
+ bogus_version = {'latch': {'timestamp': 10000 * random.random()}}
+ # mocked api for returning all checks
+ responses.add(
+ method=responses.GET,
+ url=re.compile(r'.*inventory.+/version.*'),
+ body=json.dumps(bogus_version))
diff --git a/test/data/checks.json b/test/data/checks.json
new file mode 100644
index 0000000000000000000000000000000000000000..6ff1cee3be38169120c93b88181119e626d559e2
--- /dev/null
+++ b/test/data/checks.json
@@ -0,0 +1,93 @@
+[
+ {
+ "command": "/var/lib/sensu/bin/counter2influx.sh counters 0pBiFbD mx1.ams.nl.geant.net ae1 1211",
+ "handlers": [],
+ "high_flap_threshold": 0,
+ "interval": 300,
+ "low_flap_threshold": 0,
+ "publish": false,
+ "runtime_assets": null,
+ "subscriptions": [
+ "interfacecounters"
+ ],
+ "proxy_entity_name": "mx1.ams.nl.geant.net",
+ "check_hooks": null,
+ "stdin": false,
+ "subdue": null,
+ "ttl": 0,
+ "timeout": 0,
+ "round_robin": true,
+ "output_metric_format": "influxdb_line",
+ "output_metric_handlers": [
+ "influx-db-handler"
+ ],
+ "env_vars": null,
+ "metadata": {
+ "name": "ifc-mx1.ams.nl.geant.net-ae1",
+ "namespace": "default",
+ "created_by": "admin"
+ },
+ "secrets": null
+ },
+ {
+ "command": "/var/lib/sensu/bin/counter2influx.sh counters 0pBiFbD mx1.fra.de.geant.net ae10 1006",
+ "handlers": [],
+ "high_flap_threshold": 0,
+ "interval": 300,
+ "low_flap_threshold": 0,
+ "publish": false,
+ "runtime_assets": null,
+ "subscriptions": [
+ "interfacecounters"
+ ],
+ "proxy_entity_name": "mx1.fra.de.geant.net",
+ "check_hooks": null,
+ "stdin": false,
+ "subdue": null,
+ "ttl": 0,
+ "timeout": 0,
+ "round_robin": true,
+ "output_metric_format": "influxdb_line",
+ "output_metric_handlers": [
+ "influx-db-handler"
+ ],
+ "env_vars": null,
+ "metadata": {
+ "name": "ifc-mx1.fra.de.geant.net-ae10",
+ "namespace": "default",
+ "created_by": "admin"
+ },
+ "secrets": null
+ },
+ {
+ "command": "/var/lib/sensu/bin/counter2influx.sh counters 0pBiFbD mx1.ams.nl.geant.net ae1 1211",
+ "handlers": [],
+ "high_flap_threshold": 0,
+ "interval": 300,
+ "low_flap_threshold": 0,
+ "publish": false,
+ "runtime_assets": null,
+ "subscriptions": [
+ "interfacecounters"
+ ],
+ "proxy_entity_name": "mx1.ams.nl.geant.net",
+ "check_hooks": null,
+ "stdin": false,
+ "subdue": null,
+ "ttl": 0,
+ "timeout": 0,
+ "round_robin": true,
+ "output_metric_format": "influxdb_line",
+ "output_metric_handlers": [
+ "influx-db-handler"
+ ],
+ "env_vars": null,
+ "metadata": {
+ "name": "check-mx1.ams.nl.geant.net-ae1",
+ "namespace": "default",
+ "created_by": "admin"
+ },
+ "secrets": null
+ }
+]
+
diff --git a/test/data/interfaces.json b/test/data/interfaces.json
new file mode 100644
index 0000000000000000000000000000000000000000..da1e1231cf5cf96bc4c465a8a432fd9ce5bbdabb
--- /dev/null
+++ b/test/data/interfaces.json
@@ -0,0 +1,50 @@
+[
+ {
+ "router": "mx1.ams.nl.geant.net",
+ "name": "ae1",
+ "bundle": [],
+ "bundle-parents": [],
+ "description": "blah blah",
+ "circuits": [
+ {
+ "id": 12112,
+ "name": "something",
+ "type": "SERVICE",
+ "status": "operational"
+ }
+ ],
+ "snmp-index": 1211
+ },
+ {
+ "router": "mx1.fra.de.geant.net",
+ "name": "ae10",
+ "bundle": [],
+ "bundle-parents": [],
+ "description": "blah blah",
+ "circuits": [
+ {
+ "id": 50028,
+ "name": "something",
+ "type": "SERVICE",
+ "status": "operational"
+ }
+ ],
+ "snmp-index": 1006
+ },
+ {
+ "router": "mx1.fra.de.geant.net",
+ "name": "ae99",
+ "bundle": [],
+ "bundle-parents": [],
+ "description": "blah blah",
+ "circuits": [
+ {
+ "id": 50028,
+ "name": "something",
+ "type": "SERVICE",
+ "status": "operational"
+ }
+ ],
+ "snmp-index": 9999
+ }
+]
diff --git a/test/test_api.py b/test/test_api.py
new file mode 100644
index 0000000000000000000000000000000000000000..0a647feed8c14f664ae78a5074f68bdd1835a8d1
--- /dev/null
+++ b/test/test_api.py
@@ -0,0 +1,38 @@
+import json
+import os
+
+import jsonschema
+import pytest
+import responses
+
+import brian_polling_manager
+from brian_polling_manager.api import VERSION_SCHEMA
+from brian_polling_manager.main import REFRESH_RESULT_SCHEMA
+
+
+@pytest.fixture
+def client(config_filename, mocked_sensu, mocked_inventory):
+ os.environ['CONFIG_FILENAME'] = config_filename
+ with brian_polling_manager.create_app().test_client() as c:
+ yield c
+
+
+def test_version(client):
+ rv = client.get(
+ '/api/version',
+ headers={'Accept': ['application/json']})
+ assert rv.status_code == 200
+ assert rv.is_json
+ response_data = json.loads(rv.data.decode('utf-8'))
+ jsonschema.validate(response_data, VERSION_SCHEMA)
+
+
+@responses.activate
+def test_update(client):
+ rv = client.get(
+ '/api/update',
+ headers={'Accept': ['application/json']})
+ assert rv.status_code == 200
+ assert rv.is_json
+ response_data = json.loads(rv.data.decode('utf-8'))
+ jsonschema.validate(response_data, REFRESH_RESULT_SCHEMA)
diff --git a/test/test_e2e.py b/test/test_e2e.py
new file mode 100644
index 0000000000000000000000000000000000000000..7486e959f447d984528a2f9e44f9e34b7c79b669
--- /dev/null
+++ b/test/test_e2e.py
@@ -0,0 +1,15 @@
+from click.testing import CliRunner
+import responses
+
+from brian_polling_manager import main
+
+
+@responses.activate
+def test_run_flashtest(config_filename, mocked_sensu, mocked_inventory):
+
+ runner = CliRunner()
+ result = runner.invoke(
+ main.cli,
+ ['--config', config_filename, '--force']
+ )
+ assert result.exit_code == 0
diff --git a/test/test_sensu_checks.py b/test/test_sensu_checks.py
new file mode 100644
index 0000000000000000000000000000000000000000..7a5a8178cfd72ff4d3d6a1899b1d1012e45f5df4
--- /dev/null
+++ b/test/test_sensu_checks.py
@@ -0,0 +1,40 @@
+import copy
+import random
+
+import responses
+
+from brian_polling_manager import sensu, inventory, interfaces
+
+
+@responses.activate
+def test_load_checks(config, mocked_sensu):
+ checks = list(sensu.load_all_checks(config['sensu']))
+ assert len(checks) > 0 # test data has checks in it
+
+
+@responses.activate
+def test_check_lifecycle(config, mocked_sensu, mocked_inventory):
+
+ test_interface = random.choice(
+ inventory.load_interfaces(config['inventory']))
+ test_interface['name'] = 'xyz'
+
+ new_check = interfaces._make_check(
+ config['sensu']['interface-check'],
+ test_interface)
+
+ # create the new check
+ check_name = new_check['metadata']['name']
+ assert check_name not in mocked_sensu
+ sensu.create_check(config['sensu'], new_check)
+ assert check_name in mocked_sensu
+
+ # modify interval & update, then verify the correct call was made
+ updated_check = copy.copy(new_check)
+ updated_check['interval'] += 1
+ sensu.update_check(config['sensu'], updated_check)
+ assert mocked_sensu[check_name]['interval'] == updated_check['interval']
+
+ # delete the check and confirm the correct call was made
+ sensu.delete_check(config['sensu'], check_name)
+ assert check_name not in mocked_sensu
diff --git a/tox.ini b/tox.ini
new file mode 100644
index 0000000000000000000000000000000000000000..88439ca5e63e6c272d36794aad7324d44ed927e4
--- /dev/null
+++ b/tox.ini
@@ -0,0 +1,19 @@
+[tox]
+envlist = py36
+
+[testenv]
+deps =
+ coverage
+ flake8
+ -r requirements.txt
+
+commands =
+ coverage erase
+ coverage run --source brian_polling_manager -m py.test {posargs}
+ coverage xml
+ coverage html
+ coverage report
+ coverage report --fail-under 80
+ flake8
+ sphinx-build -M html docs/source docs/build
+