Skip to content
Snippets Groups Projects
Commit 702e96bd authored by Erik Reid's avatar Erik Reid
Browse files

refactored: moved config loading to own module

parent ab0c098b
No related branches found
No related tags found
No related merge requests found
Show whitespace changes
Inline Side-by-side
brian_polling_manager/__init__.py
+ 37
− 0
View file @ 702e96bd
# """
# automatically invoked app factory
# """
# import logging
# import os
#
# from flask import Flask
#
# import dashboard_v3_webapp.config as config
#
# CONFIG_KEY = 'CONFIG_PARAMS'
# SESSION_SECRET = 'super-secret'
#
#
# def create_app():
# """
# overrides default settings with those found
# in the file read from env var CONFIG_FILENAME
#
# :return: a new flask app instance
# """
#
# app_config = config.DEFAULT_PARAMS
# if 'CONFIG_FILENAME' in os.environ:
# with open(os.environ['CONFIG_FILENAME']) as f:
# app_config.update(config.load(f))
#
# app = Flask(__name__)
# app.secret_key = SESSION_SECRET
# app.config[CONFIG_KEY] = app_config
#
# from dashboard_v3_webapp import api
# app.register_blueprint(api.routes, url_prefix='/api')
#
# logging.info('Flask app initialized')
# # environment.setup_logging()
# return app
brian_polling_manager/cli.py
+ 10
− 192
View file @ 702e96bd
......@@ -19,218 +19,36 @@ The required configuration file must be
formatted according to the following schema:
.. asjson::
brian_polling_manager.cli.CONFIG_SCHEMA
brian_polling_manager.configuration.CONFIG_SCHEMA
"""
import json
import logging
import os
from typing import Union
import click
import jsonschema
from statsd import StatsClient
from brian_polling_manager import inventory, interfaces, environment
from brian_polling_manager import inventory, interfaces, configuration
logger = logging.getLogger(__name__)
_DEFAULT_CONFIG = {
'inventory': [
'http://test-inventory-provider01.geant.org:8080',
'http://test-inventory-provider02.geant.org:8080'
],
'sensu': {
'api-base': [
'https://test-poller-sensu-agent01.geant.org:8080',
'https://test-poller-sensu-agent02.geant.org:8080',
'https://test-poller-sensu-agent03.geant.org:8080'
],
'api-key': '696a815c-607e-4090-81de-58988c83033e',
'interface-check': {
'script': '/var/lib/sensu/bin/counter2influx.sh',
'measurement': 'counters',
'interval': 300,
'subscriptions': ['interfacecounters'],
'output_metric_handlers': ['influx-db-handler'],
'namespace': 'default',
'round_robin': True,
'command': ('{script} {measurement} '
'{community} {hostname} '
'{interface} {ifIndex}'),
}
},
'statedir': '/tmp/',
'logging': environment.DEFAULT_LOGGING_FILENAME,
'statsd': {
'hostname': 'localhost',
'port': 8125,
'prefix': 'brian_polling'
}
}
CONFIG_SCHEMA = {
'$schema': 'http://json-schema.org/draft-07/schema#',
'definitions': {
'influx-check': {
'type': 'object',
'properties': {
'script': {'type': 'string'},
'measurement': {'type': 'string'},
'interval': {'type': 'integer'},
'subscriptions': {
'type': 'array',
'items': {'type': 'string'}
},
'output_metric_handlers': {
'type': 'array',
'items': {'type': 'string'}
},
'namespace': {'type': 'string'},
'round_robin': {'type': 'boolean'},
'command': {'type': 'string'},
},
'required': ['script', 'measurement', 'interval',
'subscriptions', 'output_metric_handlers',
'namespace', 'round_robin', 'command'],
'additionalProperties': False
},
'sensu': {
'type': 'object',
'properties': {
'api-base': {
'type': 'array',
'items': {'type': 'string'},
'minItems': 1
},
'api-key': {'type': 'string'},
'interface-check': {'$ref': '#/definitions/influx-check'}
},
'required': ['api-base', 'api-key', 'interface-check'],
'additionalProperties': False
},
'statsd': {
'type': 'object',
'properties': {
'hostname': {'type': 'string'},
'port': {'type': 'integer'},
'prefix': {'type': 'string'}
},
'required': ['hostname', 'port', 'prefix'],
'additionalProperties': False
}
},
'type': 'object',
'properties': {
'inventory': {
'type': 'array',
'items': {'type': 'string'},
'minItems': 1
},
'sensu': {'$ref': '#/definitions/sensu'},
'statedir': {'type': 'string'},
'logging': {'type': 'string'},
'statsd': {'$ref': '#/definitions/statsd'}
},
'required': ['inventory', 'sensu', 'statedir'],
'additionalProperties': False
}
class State(object):
INTERFACES = 'interfaces.json'
STATE = 'state.json'
STATE_SCHEMA = {
'$schema': 'http://json-schema.org/draft-07/schema#',
'type': 'object',
'properties': {
'last': {'type': 'number'}
},
'required': ['last'],
'additionalProperties': False
}
def __init__(self, state_dir: str):
assert os.path.isdir(state_dir)
self.filenames = {
'state': os.path.join(state_dir, State.STATE),
'cache': os.path.join(state_dir, State.INTERFACES)
}
@staticmethod
def _load_json(filename, schema):
try:
with open(filename) as f:
state = json.loads(f.read())
jsonschema.validate(state, schema)
return state
except (json.JSONDecodeError, jsonschema.ValidationError, OSError):
logger.exception(
f'unable to open state file {filename}')
return None
@property
def last(self) -> int:
state = State._load_json(self.filenames['state'], State.STATE_SCHEMA)
return state['last'] if state else -1
@last.setter
def last(self, new_last: Union[float, None]):
if not new_last or new_last < 0:
os.unlink(self.filenames['state'])
else:
state = {'last': new_last}
with open(self.filenames['state'], 'w') as f:
f.write(json.dumps(state))
@property
def interfaces(self) -> list:
return State._load_json(
self.filenames['cache'],
inventory.INVENTORY_INTERFACES_SCHEMA)
@interfaces.setter
def interfaces(self, new_interfaces):
try:
jsonschema.validate(
new_interfaces,
inventory.INVENTORY_INTERFACES_SCHEMA)
except jsonschema.ValidationError:
logger.exception('invalid interface state data')
return
with open(self.filenames['cache'], 'w') as f:
f.write(json.dumps(new_interfaces))
def _validate_config(ctx, param, file):
def _validate_config(_ctx, _param, file):
"""
loads, validates and returns configuration parameters
:param ctx:
:param param:
:param value: filename (string)
:param _ctx: unused
:param _param: unused
:param value: file (file-like object open for reading)
:return: a dict containing configuration parameters
"""
if file is None:
config = _DEFAULT_CONFIG
else:
try:
config = json.loads(file.read())
except (json.JSONDecodeError, jsonschema.ValidationError) as e:
return configuration.load_config(file)
except (json.JSONDecodeError, jsonschema.ValidationError, OSError,
AttributeError, ValueError, TypeError, ImportError) as e:
raise click.BadParameter(str(e))
try:
jsonschema.validate(config, CONFIG_SCHEMA)
except jsonschema.ValidationError as e:
raise click.BadParameter(str(e))
environment.setup_logging(config.get('logging', None))
return config
@click.command()
@click.option(
......@@ -248,7 +66,7 @@ def main(config, force):
Update BRIAN snmp checks based on Inventory Provider data.
"""
state = State(config['statedir'])
state = configuration.State(config['statedir'])
last = inventory.last_update_timestamp(config['inventory'])
if force or not last or last != state.last:
state.last = last
......
brian_polling_manager/configuration.py 0 → 100644
+ 224
− 0
View file @ 702e96bd
import json
import logging
import logging.config
import os
from typing import Union
import jsonschema
from brian_polling_manager import inventory
logger = logging.getLogger(__name__)
_DEFAULT_LOGGING_FILENAME = os.path.join(
os.path.dirname(__file__),
'logging_default_config.json')
_DEFAULT_CONFIG = {
'inventory': [
'http://test-inventory-provider01.geant.org:8080',
'http://test-inventory-provider02.geant.org:8080'
],
'sensu': {
'api-base': [
'https://test-poller-sensu-agent01.geant.org:8080',
'https://test-poller-sensu-agent02.geant.org:8080',
'https://test-poller-sensu-agent03.geant.org:8080'
],
'api-key': '696a815c-607e-4090-81de-58988c83033e',
'interface-check': {
'script': '/var/lib/sensu/bin/counter2influx.sh',
'measurement': 'counters',
'interval': 300,
'subscriptions': ['interfacecounters'],
'output_metric_handlers': ['influx-db-handler'],
'namespace': 'default',
'round_robin': True,
'command': ('{script} {measurement} '
'{community} {hostname} '
'{interface} {ifIndex}'),
}
},
'statedir': '/tmp/',
'logging': _DEFAULT_LOGGING_FILENAME,
'statsd': {
'hostname': 'localhost',
'port': 8125,
'prefix': 'brian_polling'
}
}
CONFIG_SCHEMA = {
'$schema': 'http://json-schema.org/draft-07/schema#',
'definitions': {
'influx-check': {
'type': 'object',
'properties': {
'script': {'type': 'string'},
'measurement': {'type': 'string'},
'interval': {'type': 'integer'},
'subscriptions': {
'type': 'array',
'items': {'type': 'string'}
},
'output_metric_handlers': {
'type': 'array',
'items': {'type': 'string'}
},
'namespace': {'type': 'string'},
'round_robin': {'type': 'boolean'},
'command': {'type': 'string'},
},
'required': ['script', 'measurement', 'interval',
'subscriptions', 'output_metric_handlers',
'namespace', 'round_robin', 'command'],
'additionalProperties': False
},
'sensu': {
'type': 'object',
'properties': {
'api-base': {
'type': 'array',
'items': {'type': 'string'},
'minItems': 1
},
'api-key': {'type': 'string'},
'interface-check': {'$ref': '#/definitions/influx-check'}
},
'required': ['api-base', 'api-key', 'interface-check'],
'additionalProperties': False
},
'statsd': {
'type': 'object',
'properties': {
'hostname': {'type': 'string'},
'port': {'type': 'integer'},
'prefix': {'type': 'string'}
},
'required': ['hostname', 'port', 'prefix'],
'additionalProperties': False
}
},
'type': 'object',
'properties': {
'inventory': {
'type': 'array',
'items': {'type': 'string'},
'minItems': 1
},
'sensu': {'$ref': '#/definitions/sensu'},
'statedir': {'type': 'string'},
'logging': {'type': 'string'},
'statsd': {'$ref': '#/definitions/statsd'}
},
'required': ['inventory', 'sensu', 'statedir'],
'additionalProperties': False
}
class State(object):
INTERFACES = 'interfaces.json'
STATE = 'state.json'
STATE_SCHEMA = {
'$schema': 'http://json-schema.org/draft-07/schema#',
'type': 'object',
'properties': {
'last': {'type': 'number'}
},
'required': ['last'],
'additionalProperties': False
}
def __init__(self, state_dir: str):
assert os.path.isdir(state_dir)
self.filenames = {
'state': os.path.join(state_dir, State.STATE),
'cache': os.path.join(state_dir, State.INTERFACES)
}
@staticmethod
def _load_json(filename, schema):
try:
with open(filename) as f:
state = json.loads(f.read())
jsonschema.validate(state, schema)
return state
except (json.JSONDecodeError, jsonschema.ValidationError, OSError):
logger.exception(
f'unable to open state file {filename}')
return None
@property
def last(self) -> int:
state = State._load_json(self.filenames['state'], State.STATE_SCHEMA)
return state['last'] if state else -1
@last.setter
def last(self, new_last: Union[float, None]):
if not new_last or new_last < 0:
os.unlink(self.filenames['state'])
else:
state = {'last': new_last}
with open(self.filenames['state'], 'w') as f:
f.write(json.dumps(state))
@property
def interfaces(self) -> list:
return State._load_json(
self.filenames['cache'],
inventory.INVENTORY_INTERFACES_SCHEMA)
@interfaces.setter
def interfaces(self, new_interfaces):
try:
jsonschema.validate(
new_interfaces,
inventory.INVENTORY_INTERFACES_SCHEMA)
except jsonschema.ValidationError:
logger.exception('invalid interface state data')
return
with open(self.filenames['cache'], 'w') as f:
f.write(json.dumps(new_interfaces))
def _setup_logging(filename=None):
"""
set up logging using the configured filename
:raises: json.JSONDecodeError, OSError, AttributeError,
ValueError, TypeError, ImportError
"""
if not filename:
filename = _DEFAULT_LOGGING_FILENAME
with open(filename) as f:
# TODO: this mac workaround should be removed ...
d = json.loads(f.read())
import platform
if platform.system() == 'Darwin':
d['handlers']['syslog_handler']['address'] = '/var/run/syslog'
logging.config.dictConfig(d)
# logging.config.dictConfig(json.loads(f.read()))
def load_config(file):
"""
loads, validates and returns configuration parameters
:param value: filename (file-like object, opened for reading)
:return: a dict containing configuration parameters
:raises: json.JSONDecodeError, jsonschema.ValidationError,
OSError, AttributeError, ValueError, TypeError, ImportError
"""
if file is None:
config = _DEFAULT_CONFIG
else:
config = json.loads(file.read())
jsonschema.validate(config, CONFIG_SCHEMA)
_setup_logging(config.get('logging', None))
return config
brian_polling_manager/environment.py deleted 100644 → 0
+ 0
− 24
View file @ ab0c098b
import json
import logging.config
import os
DEFAULT_LOGGING_FILENAME = os.path.join(
os.path.dirname(__file__),
'logging_default_config.json')
def setup_logging(filename=None):
"""
set up logging using the configured filename
"""
if not filename:
filename = DEFAULT_LOGGING_FILENAME
with open(filename) as f:
# TODO: this mac workaround should be removed ...
d = json.loads(f.read())
import platform
if platform.system() == 'Darwin':
d['handlers']['syslog_handler']['address'] = '/var/run/syslog'
logging.config.dictConfig(d)
# logging.config.dictConfig(json.loads(f.read()))
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please to comment