From 702e96bdb1803bbef34e1b44dd794b3a05b8b655 Mon Sep 17 00:00:00 2001 From: Erik Reid <erik.reid@geant.org> Date: Tue, 1 Jun 2021 08:31:39 +0200 Subject: [PATCH] refactored: moved config loading to own module --- brian_polling_manager/__init__.py | 37 ++++ brian_polling_manager/cli.py | 202 ++-------------------- brian_polling_manager/configuration.py | 224 +++++++++++++++++++++++++ brian_polling_manager/environment.py | 24 --- 4 files changed, 271 insertions(+), 216 deletions(-) create mode 100644 brian_polling_manager/configuration.py delete mode 100644 brian_polling_manager/environment.py diff --git a/brian_polling_manager/__init__.py b/brian_polling_manager/__init__.py index e69de29..50c8ab5 100644 --- a/brian_polling_manager/__init__.py +++ b/brian_polling_manager/__init__.py @@ -0,0 +1,37 @@ +# """ +# automatically invoked app factory +# """ +# import logging +# import os +# +# from flask import Flask +# +# import dashboard_v3_webapp.config as config +# +# CONFIG_KEY = 'CONFIG_PARAMS' +# SESSION_SECRET = 'super-secret' +# +# +# def create_app(): +# """ +# overrides default settings with those found +# in the file read from env var CONFIG_FILENAME +# +# :return: a new flask app instance +# """ +# +# app_config = config.DEFAULT_PARAMS +# if 'CONFIG_FILENAME' in os.environ: +# with open(os.environ['CONFIG_FILENAME']) as f: +# app_config.update(config.load(f)) +# +# app = Flask(__name__) +# app.secret_key = SESSION_SECRET +# app.config[CONFIG_KEY] = app_config +# +# from dashboard_v3_webapp import api +# app.register_blueprint(api.routes, url_prefix='/api') +# +# logging.info('Flask app initialized') +# # environment.setup_logging() +# return app diff --git a/brian_polling_manager/cli.py b/brian_polling_manager/cli.py index f78df64..5be7ba2 100644 --- a/brian_polling_manager/cli.py +++ b/brian_polling_manager/cli.py @@ -19,218 +19,36 @@ The required configuration file must be formatted according to the following schema: .. asjson:: - brian_polling_manager.cli.CONFIG_SCHEMA + brian_polling_manager.configuration.CONFIG_SCHEMA """ import json import logging -import os -from typing import Union import click import jsonschema from statsd import StatsClient -from brian_polling_manager import inventory, interfaces, environment +from brian_polling_manager import inventory, interfaces, configuration logger = logging.getLogger(__name__) -_DEFAULT_CONFIG = { - 'inventory': [ - 'http://test-inventory-provider01.geant.org:8080', - 'http://test-inventory-provider02.geant.org:8080' - ], - 'sensu': { - 'api-base': [ - 'https://test-poller-sensu-agent01.geant.org:8080', - 'https://test-poller-sensu-agent02.geant.org:8080', - 'https://test-poller-sensu-agent03.geant.org:8080' - ], - 'api-key': '696a815c-607e-4090-81de-58988c83033e', - 'interface-check': { - 'script': '/var/lib/sensu/bin/counter2influx.sh', - 'measurement': 'counters', - 'interval': 300, - 'subscriptions': ['interfacecounters'], - 'output_metric_handlers': ['influx-db-handler'], - 'namespace': 'default', - 'round_robin': True, - 'command': ('{script} {measurement} ' - '{community} {hostname} ' - '{interface} {ifIndex}'), - } - }, - 'statedir': '/tmp/', - 'logging': environment.DEFAULT_LOGGING_FILENAME, - 'statsd': { - 'hostname': 'localhost', - 'port': 8125, - 'prefix': 'brian_polling' - } -} -CONFIG_SCHEMA = { - '$schema': 'http://json-schema.org/draft-07/schema#', - 'definitions': { - 'influx-check': { - 'type': 'object', - 'properties': { - 'script': {'type': 'string'}, - 'measurement': {'type': 'string'}, - 'interval': {'type': 'integer'}, - 'subscriptions': { - 'type': 'array', - 'items': {'type': 'string'} - }, - 'output_metric_handlers': { - 'type': 'array', - 'items': {'type': 'string'} - }, - 'namespace': {'type': 'string'}, - 'round_robin': {'type': 'boolean'}, - 'command': {'type': 'string'}, - }, - 'required': ['script', 'measurement', 'interval', - 'subscriptions', 'output_metric_handlers', - 'namespace', 'round_robin', 'command'], - 'additionalProperties': False - }, - 'sensu': { - 'type': 'object', - 'properties': { - 'api-base': { - 'type': 'array', - 'items': {'type': 'string'}, - 'minItems': 1 - }, - 'api-key': {'type': 'string'}, - 'interface-check': {'$ref': '#/definitions/influx-check'} - }, - 'required': ['api-base', 'api-key', 'interface-check'], - 'additionalProperties': False - }, - 'statsd': { - 'type': 'object', - 'properties': { - 'hostname': {'type': 'string'}, - 'port': {'type': 'integer'}, - 'prefix': {'type': 'string'} - }, - 'required': ['hostname', 'port', 'prefix'], - 'additionalProperties': False - } - }, - 'type': 'object', - 'properties': { - 'inventory': { - 'type': 'array', - 'items': {'type': 'string'}, - 'minItems': 1 - }, - 'sensu': {'$ref': '#/definitions/sensu'}, - 'statedir': {'type': 'string'}, - 'logging': {'type': 'string'}, - 'statsd': {'$ref': '#/definitions/statsd'} - }, - 'required': ['inventory', 'sensu', 'statedir'], - 'additionalProperties': False -} - - -class State(object): - - INTERFACES = 'interfaces.json' - STATE = 'state.json' - - STATE_SCHEMA = { - '$schema': 'http://json-schema.org/draft-07/schema#', - 'type': 'object', - 'properties': { - 'last': {'type': 'number'} - }, - 'required': ['last'], - 'additionalProperties': False - } - - def __init__(self, state_dir: str): - assert os.path.isdir(state_dir) - self.filenames = { - 'state': os.path.join(state_dir, State.STATE), - 'cache': os.path.join(state_dir, State.INTERFACES) - } - - @staticmethod - def _load_json(filename, schema): - try: - with open(filename) as f: - state = json.loads(f.read()) - jsonschema.validate(state, schema) - return state - except (json.JSONDecodeError, jsonschema.ValidationError, OSError): - logger.exception( - f'unable to open state file {filename}') - return None - - @property - def last(self) -> int: - state = State._load_json(self.filenames['state'], State.STATE_SCHEMA) - return state['last'] if state else -1 - - @last.setter - def last(self, new_last: Union[float, None]): - if not new_last or new_last < 0: - os.unlink(self.filenames['state']) - else: - state = {'last': new_last} - with open(self.filenames['state'], 'w') as f: - f.write(json.dumps(state)) - - @property - def interfaces(self) -> list: - return State._load_json( - self.filenames['cache'], - inventory.INVENTORY_INTERFACES_SCHEMA) - - @interfaces.setter - def interfaces(self, new_interfaces): - try: - jsonschema.validate( - new_interfaces, - inventory.INVENTORY_INTERFACES_SCHEMA) - except jsonschema.ValidationError: - logger.exception('invalid interface state data') - return - - with open(self.filenames['cache'], 'w') as f: - f.write(json.dumps(new_interfaces)) - - -def _validate_config(ctx, param, file): +def _validate_config(_ctx, _param, file): """ loads, validates and returns configuration parameters - :param ctx: - :param param: - :param value: filename (string) + :param _ctx: unused + :param _param: unused + :param value: file (file-like object open for reading) :return: a dict containing configuration parameters """ - if file is None: - config = _DEFAULT_CONFIG - else: - try: - config = json.loads(file.read()) - except (json.JSONDecodeError, jsonschema.ValidationError) as e: - raise click.BadParameter(str(e)) - try: - jsonschema.validate(config, CONFIG_SCHEMA) - except jsonschema.ValidationError as e: + return configuration.load_config(file) + except (json.JSONDecodeError, jsonschema.ValidationError, OSError, + AttributeError, ValueError, TypeError, ImportError) as e: raise click.BadParameter(str(e)) - environment.setup_logging(config.get('logging', None)) - - return config - @click.command() @click.option( @@ -248,7 +66,7 @@ def main(config, force): Update BRIAN snmp checks based on Inventory Provider data. """ - state = State(config['statedir']) + state = configuration.State(config['statedir']) last = inventory.last_update_timestamp(config['inventory']) if force or not last or last != state.last: state.last = last diff --git a/brian_polling_manager/configuration.py b/brian_polling_manager/configuration.py new file mode 100644 index 0000000..c09ab5d --- /dev/null +++ b/brian_polling_manager/configuration.py @@ -0,0 +1,224 @@ +import json +import logging +import logging.config +import os +from typing import Union + +import jsonschema + +from brian_polling_manager import inventory + +logger = logging.getLogger(__name__) + +_DEFAULT_LOGGING_FILENAME = os.path.join( + os.path.dirname(__file__), + 'logging_default_config.json') + +_DEFAULT_CONFIG = { + 'inventory': [ + 'http://test-inventory-provider01.geant.org:8080', + 'http://test-inventory-provider02.geant.org:8080' + ], + 'sensu': { + 'api-base': [ + 'https://test-poller-sensu-agent01.geant.org:8080', + 'https://test-poller-sensu-agent02.geant.org:8080', + 'https://test-poller-sensu-agent03.geant.org:8080' + ], + 'api-key': '696a815c-607e-4090-81de-58988c83033e', + 'interface-check': { + 'script': '/var/lib/sensu/bin/counter2influx.sh', + 'measurement': 'counters', + 'interval': 300, + 'subscriptions': ['interfacecounters'], + 'output_metric_handlers': ['influx-db-handler'], + 'namespace': 'default', + 'round_robin': True, + 'command': ('{script} {measurement} ' + '{community} {hostname} ' + '{interface} {ifIndex}'), + } + }, + 'statedir': '/tmp/', + 'logging': _DEFAULT_LOGGING_FILENAME, + 'statsd': { + 'hostname': 'localhost', + 'port': 8125, + 'prefix': 'brian_polling' + } +} + +CONFIG_SCHEMA = { + '$schema': 'http://json-schema.org/draft-07/schema#', + 'definitions': { + 'influx-check': { + 'type': 'object', + 'properties': { + 'script': {'type': 'string'}, + 'measurement': {'type': 'string'}, + 'interval': {'type': 'integer'}, + 'subscriptions': { + 'type': 'array', + 'items': {'type': 'string'} + }, + 'output_metric_handlers': { + 'type': 'array', + 'items': {'type': 'string'} + }, + 'namespace': {'type': 'string'}, + 'round_robin': {'type': 'boolean'}, + 'command': {'type': 'string'}, + }, + 'required': ['script', 'measurement', 'interval', + 'subscriptions', 'output_metric_handlers', + 'namespace', 'round_robin', 'command'], + 'additionalProperties': False + }, + 'sensu': { + 'type': 'object', + 'properties': { + 'api-base': { + 'type': 'array', + 'items': {'type': 'string'}, + 'minItems': 1 + }, + 'api-key': {'type': 'string'}, + 'interface-check': {'$ref': '#/definitions/influx-check'} + }, + 'required': ['api-base', 'api-key', 'interface-check'], + 'additionalProperties': False + }, + 'statsd': { + 'type': 'object', + 'properties': { + 'hostname': {'type': 'string'}, + 'port': {'type': 'integer'}, + 'prefix': {'type': 'string'} + }, + 'required': ['hostname', 'port', 'prefix'], + 'additionalProperties': False + } + }, + 'type': 'object', + 'properties': { + 'inventory': { + 'type': 'array', + 'items': {'type': 'string'}, + 'minItems': 1 + }, + 'sensu': {'$ref': '#/definitions/sensu'}, + 'statedir': {'type': 'string'}, + 'logging': {'type': 'string'}, + 'statsd': {'$ref': '#/definitions/statsd'} + }, + 'required': ['inventory', 'sensu', 'statedir'], + 'additionalProperties': False +} + + +class State(object): + + INTERFACES = 'interfaces.json' + STATE = 'state.json' + + STATE_SCHEMA = { + '$schema': 'http://json-schema.org/draft-07/schema#', + 'type': 'object', + 'properties': { + 'last': {'type': 'number'} + }, + 'required': ['last'], + 'additionalProperties': False + } + + def __init__(self, state_dir: str): + assert os.path.isdir(state_dir) + self.filenames = { + 'state': os.path.join(state_dir, State.STATE), + 'cache': os.path.join(state_dir, State.INTERFACES) + } + + @staticmethod + def _load_json(filename, schema): + try: + with open(filename) as f: + state = json.loads(f.read()) + jsonschema.validate(state, schema) + return state + except (json.JSONDecodeError, jsonschema.ValidationError, OSError): + logger.exception( + f'unable to open state file {filename}') + return None + + @property + def last(self) -> int: + state = State._load_json(self.filenames['state'], State.STATE_SCHEMA) + return state['last'] if state else -1 + + @last.setter + def last(self, new_last: Union[float, None]): + if not new_last or new_last < 0: + os.unlink(self.filenames['state']) + else: + state = {'last': new_last} + with open(self.filenames['state'], 'w') as f: + f.write(json.dumps(state)) + + @property + def interfaces(self) -> list: + return State._load_json( + self.filenames['cache'], + inventory.INVENTORY_INTERFACES_SCHEMA) + + @interfaces.setter + def interfaces(self, new_interfaces): + try: + jsonschema.validate( + new_interfaces, + inventory.INVENTORY_INTERFACES_SCHEMA) + except jsonschema.ValidationError: + logger.exception('invalid interface state data') + return + + with open(self.filenames['cache'], 'w') as f: + f.write(json.dumps(new_interfaces)) + + +def _setup_logging(filename=None): + """ + set up logging using the configured filename + + :raises: json.JSONDecodeError, OSError, AttributeError, + ValueError, TypeError, ImportError + """ + if not filename: + filename = _DEFAULT_LOGGING_FILENAME + + with open(filename) as f: + # TODO: this mac workaround should be removed ... + d = json.loads(f.read()) + import platform + if platform.system() == 'Darwin': + d['handlers']['syslog_handler']['address'] = '/var/run/syslog' + logging.config.dictConfig(d) + # logging.config.dictConfig(json.loads(f.read())) + + +def load_config(file): + """ + loads, validates and returns configuration parameters + + :param value: filename (file-like object, opened for reading) + :return: a dict containing configuration parameters + :raises: json.JSONDecodeError, jsonschema.ValidationError, + OSError, AttributeError, ValueError, TypeError, ImportError + """ + if file is None: + config = _DEFAULT_CONFIG + else: + config = json.loads(file.read()) + jsonschema.validate(config, CONFIG_SCHEMA) + + _setup_logging(config.get('logging', None)) + + return config diff --git a/brian_polling_manager/environment.py b/brian_polling_manager/environment.py deleted file mode 100644 index 9662849..0000000 --- a/brian_polling_manager/environment.py +++ /dev/null @@ -1,24 +0,0 @@ -import json -import logging.config -import os - -DEFAULT_LOGGING_FILENAME = os.path.join( - os.path.dirname(__file__), - 'logging_default_config.json') - - -def setup_logging(filename=None): - """ - set up logging using the configured filename - """ - if not filename: - filename = DEFAULT_LOGGING_FILENAME - - with open(filename) as f: - # TODO: this mac workaround should be removed ... - d = json.loads(f.read()) - import platform - if platform.system() == 'Darwin': - d['handlers']['syslog_handler']['address'] = '/var/run/syslog' - logging.config.dictConfig(d) - # logging.config.dictConfig(json.loads(f.read())) -- GitLab