From 702e96bdb1803bbef34e1b44dd794b3a05b8b655 Mon Sep 17 00:00:00 2001
From: Erik Reid <erik.reid@geant.org>
Date: Tue, 1 Jun 2021 08:31:39 +0200
Subject: [PATCH] refactored: moved config loading to own module

---
 brian_polling_manager/__init__.py      |  37 ++++
 brian_polling_manager/cli.py           | 202 ++--------------------
 brian_polling_manager/configuration.py | 224 +++++++++++++++++++++++++
 brian_polling_manager/environment.py   |  24 ---
 4 files changed, 271 insertions(+), 216 deletions(-)
 create mode 100644 brian_polling_manager/configuration.py
 delete mode 100644 brian_polling_manager/environment.py

diff --git a/brian_polling_manager/__init__.py b/brian_polling_manager/__init__.py
index e69de29..50c8ab5 100644
--- a/brian_polling_manager/__init__.py
+++ b/brian_polling_manager/__init__.py
@@ -0,0 +1,37 @@
+# """
+# automatically invoked app factory
+# """
+# import logging
+# import os
+#
+# from flask import Flask
+#
+# import dashboard_v3_webapp.config as config
+#
+# CONFIG_KEY = 'CONFIG_PARAMS'
+# SESSION_SECRET = 'super-secret'
+#
+#
+# def create_app():
+#     """
+#     overrides default settings with those found
+#     in the file read from env var CONFIG_FILENAME
+#
+#     :return: a new flask app instance
+#     """
+#
+#     app_config = config.DEFAULT_PARAMS
+#     if 'CONFIG_FILENAME' in os.environ:
+#         with open(os.environ['CONFIG_FILENAME']) as f:
+#             app_config.update(config.load(f))
+#
+#     app = Flask(__name__)
+#     app.secret_key = SESSION_SECRET
+#     app.config[CONFIG_KEY] = app_config
+#
+#     from dashboard_v3_webapp import api
+#     app.register_blueprint(api.routes, url_prefix='/api')
+#
+#     logging.info('Flask app initialized')
+#     # environment.setup_logging()
+#     return app
diff --git a/brian_polling_manager/cli.py b/brian_polling_manager/cli.py
index f78df64..5be7ba2 100644
--- a/brian_polling_manager/cli.py
+++ b/brian_polling_manager/cli.py
@@ -19,218 +19,36 @@ The required configuration file must be
 formatted according to the following schema:
 
 .. asjson::
-    brian_polling_manager.cli.CONFIG_SCHEMA
+    brian_polling_manager.configuration.CONFIG_SCHEMA
 
 """
 import json
 import logging
-import os
-from typing import Union
 
 import click
 import jsonschema
 from statsd import StatsClient
 
-from brian_polling_manager import inventory, interfaces, environment
+from brian_polling_manager import inventory, interfaces, configuration
 
 logger = logging.getLogger(__name__)
 
-_DEFAULT_CONFIG = {
-    'inventory': [
-        'http://test-inventory-provider01.geant.org:8080',
-        'http://test-inventory-provider02.geant.org:8080'
-    ],
-    'sensu': {
-        'api-base': [
-            'https://test-poller-sensu-agent01.geant.org:8080',
-            'https://test-poller-sensu-agent02.geant.org:8080',
-            'https://test-poller-sensu-agent03.geant.org:8080'
-        ],
-        'api-key': '696a815c-607e-4090-81de-58988c83033e',
-        'interface-check': {
-            'script': '/var/lib/sensu/bin/counter2influx.sh',
-            'measurement': 'counters',
-            'interval': 300,
-            'subscriptions': ['interfacecounters'],
-            'output_metric_handlers': ['influx-db-handler'],
-            'namespace': 'default',
-            'round_robin': True,
-            'command': ('{script} {measurement} '
-                        '{community} {hostname} '
-                        '{interface} {ifIndex}'),
-        }
-    },
-    'statedir': '/tmp/',
-    'logging': environment.DEFAULT_LOGGING_FILENAME,
-    'statsd': {
-        'hostname': 'localhost',
-        'port': 8125,
-        'prefix': 'brian_polling'
-    }
-}
 
-CONFIG_SCHEMA = {
-    '$schema': 'http://json-schema.org/draft-07/schema#',
-    'definitions': {
-        'influx-check': {
-            'type': 'object',
-            'properties': {
-                'script': {'type': 'string'},
-                'measurement': {'type': 'string'},
-                'interval': {'type': 'integer'},
-                'subscriptions': {
-                    'type': 'array',
-                    'items': {'type': 'string'}
-                },
-                'output_metric_handlers': {
-                    'type': 'array',
-                    'items': {'type': 'string'}
-                },
-                'namespace': {'type': 'string'},
-                'round_robin': {'type': 'boolean'},
-                'command': {'type': 'string'},
-            },
-            'required': ['script', 'measurement', 'interval',
-                         'subscriptions', 'output_metric_handlers',
-                         'namespace', 'round_robin', 'command'],
-            'additionalProperties': False
-        },
-        'sensu': {
-            'type': 'object',
-            'properties': {
-                'api-base': {
-                    'type': 'array',
-                    'items': {'type': 'string'},
-                    'minItems': 1
-                },
-                'api-key': {'type': 'string'},
-                'interface-check': {'$ref': '#/definitions/influx-check'}
-            },
-            'required': ['api-base', 'api-key', 'interface-check'],
-            'additionalProperties': False
-        },
-        'statsd': {
-            'type': 'object',
-            'properties': {
-                'hostname': {'type': 'string'},
-                'port': {'type': 'integer'},
-                'prefix': {'type': 'string'}
-            },
-            'required': ['hostname', 'port', 'prefix'],
-            'additionalProperties': False
-        }
-    },
-    'type': 'object',
-    'properties': {
-        'inventory': {
-            'type': 'array',
-            'items': {'type': 'string'},
-            'minItems': 1
-        },
-        'sensu': {'$ref': '#/definitions/sensu'},
-        'statedir': {'type': 'string'},
-        'logging': {'type': 'string'},
-        'statsd': {'$ref': '#/definitions/statsd'}
-    },
-    'required': ['inventory', 'sensu', 'statedir'],
-    'additionalProperties': False
-}
-
-
-class State(object):
-
-    INTERFACES = 'interfaces.json'
-    STATE = 'state.json'
-
-    STATE_SCHEMA = {
-        '$schema': 'http://json-schema.org/draft-07/schema#',
-        'type': 'object',
-        'properties': {
-            'last': {'type': 'number'}
-        },
-        'required': ['last'],
-        'additionalProperties': False
-    }
-
-    def __init__(self, state_dir: str):
-        assert os.path.isdir(state_dir)
-        self.filenames = {
-            'state': os.path.join(state_dir, State.STATE),
-            'cache': os.path.join(state_dir, State.INTERFACES)
-        }
-
-    @staticmethod
-    def _load_json(filename, schema):
-        try:
-            with open(filename) as f:
-                state = json.loads(f.read())
-                jsonschema.validate(state, schema)
-                return state
-        except (json.JSONDecodeError, jsonschema.ValidationError, OSError):
-            logger.exception(
-                f'unable to open state file {filename}')
-            return None
-
-    @property
-    def last(self) -> int:
-        state = State._load_json(self.filenames['state'], State.STATE_SCHEMA)
-        return state['last'] if state else -1
-
-    @last.setter
-    def last(self, new_last: Union[float, None]):
-        if not new_last or new_last < 0:
-            os.unlink(self.filenames['state'])
-        else:
-            state = {'last': new_last}
-            with open(self.filenames['state'], 'w') as f:
-                f.write(json.dumps(state))
-
-    @property
-    def interfaces(self) -> list:
-        return State._load_json(
-            self.filenames['cache'],
-            inventory.INVENTORY_INTERFACES_SCHEMA)
-
-    @interfaces.setter
-    def interfaces(self, new_interfaces):
-        try:
-            jsonschema.validate(
-                new_interfaces,
-                inventory.INVENTORY_INTERFACES_SCHEMA)
-        except jsonschema.ValidationError:
-            logger.exception('invalid interface state data')
-            return
-
-        with open(self.filenames['cache'], 'w') as f:
-            f.write(json.dumps(new_interfaces))
-
-
-def _validate_config(ctx, param, file):
+def _validate_config(_ctx, _param, file):
     """
     loads, validates and returns configuration parameters
 
-    :param ctx:
-    :param param:
-    :param value: filename (string)
+    :param _ctx: unused
+    :param _param: unused
+    :param value: file (file-like object open for reading)
     :return: a dict containing configuration parameters
     """
-    if file is None:
-        config = _DEFAULT_CONFIG
-    else:
-        try:
-            config = json.loads(file.read())
-        except (json.JSONDecodeError, jsonschema.ValidationError) as e:
-            raise click.BadParameter(str(e))
-
     try:
-        jsonschema.validate(config, CONFIG_SCHEMA)
-    except jsonschema.ValidationError as e:
+        return configuration.load_config(file)
+    except (json.JSONDecodeError, jsonschema.ValidationError, OSError,
+            AttributeError, ValueError, TypeError, ImportError) as e:
         raise click.BadParameter(str(e))
 
-    environment.setup_logging(config.get('logging', None))
-
-    return config
-
 
 @click.command()
 @click.option(
@@ -248,7 +66,7 @@ def main(config, force):
     Update BRIAN snmp checks based on Inventory Provider data.
     """
 
-    state = State(config['statedir'])
+    state = configuration.State(config['statedir'])
     last = inventory.last_update_timestamp(config['inventory'])
     if force or not last or last != state.last:
         state.last = last
diff --git a/brian_polling_manager/configuration.py b/brian_polling_manager/configuration.py
new file mode 100644
index 0000000..c09ab5d
--- /dev/null
+++ b/brian_polling_manager/configuration.py
@@ -0,0 +1,224 @@
+import json
+import logging
+import logging.config
+import os
+from typing import Union
+
+import jsonschema
+
+from brian_polling_manager import inventory
+
+logger = logging.getLogger(__name__)
+
+_DEFAULT_LOGGING_FILENAME = os.path.join(
+    os.path.dirname(__file__),
+    'logging_default_config.json')
+
+_DEFAULT_CONFIG = {
+    'inventory': [
+        'http://test-inventory-provider01.geant.org:8080',
+        'http://test-inventory-provider02.geant.org:8080'
+    ],
+    'sensu': {
+        'api-base': [
+            'https://test-poller-sensu-agent01.geant.org:8080',
+            'https://test-poller-sensu-agent02.geant.org:8080',
+            'https://test-poller-sensu-agent03.geant.org:8080'
+        ],
+        'api-key': '696a815c-607e-4090-81de-58988c83033e',
+        'interface-check': {
+            'script': '/var/lib/sensu/bin/counter2influx.sh',
+            'measurement': 'counters',
+            'interval': 300,
+            'subscriptions': ['interfacecounters'],
+            'output_metric_handlers': ['influx-db-handler'],
+            'namespace': 'default',
+            'round_robin': True,
+            'command': ('{script} {measurement} '
+                        '{community} {hostname} '
+                        '{interface} {ifIndex}'),
+        }
+    },
+    'statedir': '/tmp/',
+    'logging': _DEFAULT_LOGGING_FILENAME,
+    'statsd': {
+        'hostname': 'localhost',
+        'port': 8125,
+        'prefix': 'brian_polling'
+    }
+}
+
+CONFIG_SCHEMA = {
+    '$schema': 'http://json-schema.org/draft-07/schema#',
+    'definitions': {
+        'influx-check': {
+            'type': 'object',
+            'properties': {
+                'script': {'type': 'string'},
+                'measurement': {'type': 'string'},
+                'interval': {'type': 'integer'},
+                'subscriptions': {
+                    'type': 'array',
+                    'items': {'type': 'string'}
+                },
+                'output_metric_handlers': {
+                    'type': 'array',
+                    'items': {'type': 'string'}
+                },
+                'namespace': {'type': 'string'},
+                'round_robin': {'type': 'boolean'},
+                'command': {'type': 'string'},
+            },
+            'required': ['script', 'measurement', 'interval',
+                         'subscriptions', 'output_metric_handlers',
+                         'namespace', 'round_robin', 'command'],
+            'additionalProperties': False
+        },
+        'sensu': {
+            'type': 'object',
+            'properties': {
+                'api-base': {
+                    'type': 'array',
+                    'items': {'type': 'string'},
+                    'minItems': 1
+                },
+                'api-key': {'type': 'string'},
+                'interface-check': {'$ref': '#/definitions/influx-check'}
+            },
+            'required': ['api-base', 'api-key', 'interface-check'],
+            'additionalProperties': False
+        },
+        'statsd': {
+            'type': 'object',
+            'properties': {
+                'hostname': {'type': 'string'},
+                'port': {'type': 'integer'},
+                'prefix': {'type': 'string'}
+            },
+            'required': ['hostname', 'port', 'prefix'],
+            'additionalProperties': False
+        }
+    },
+    'type': 'object',
+    'properties': {
+        'inventory': {
+            'type': 'array',
+            'items': {'type': 'string'},
+            'minItems': 1
+        },
+        'sensu': {'$ref': '#/definitions/sensu'},
+        'statedir': {'type': 'string'},
+        'logging': {'type': 'string'},
+        'statsd': {'$ref': '#/definitions/statsd'}
+    },
+    'required': ['inventory', 'sensu', 'statedir'],
+    'additionalProperties': False
+}
+
+
+class State(object):
+
+    INTERFACES = 'interfaces.json'
+    STATE = 'state.json'
+
+    STATE_SCHEMA = {
+        '$schema': 'http://json-schema.org/draft-07/schema#',
+        'type': 'object',
+        'properties': {
+            'last': {'type': 'number'}
+        },
+        'required': ['last'],
+        'additionalProperties': False
+    }
+
+    def __init__(self, state_dir: str):
+        assert os.path.isdir(state_dir)
+        self.filenames = {
+            'state': os.path.join(state_dir, State.STATE),
+            'cache': os.path.join(state_dir, State.INTERFACES)
+        }
+
+    @staticmethod
+    def _load_json(filename, schema):
+        try:
+            with open(filename) as f:
+                state = json.loads(f.read())
+                jsonschema.validate(state, schema)
+                return state
+        except (json.JSONDecodeError, jsonschema.ValidationError, OSError):
+            logger.exception(
+                f'unable to open state file {filename}')
+            return None
+
+    @property
+    def last(self) -> int:
+        state = State._load_json(self.filenames['state'], State.STATE_SCHEMA)
+        return state['last'] if state else -1
+
+    @last.setter
+    def last(self, new_last: Union[float, None]):
+        if not new_last or new_last < 0:
+            os.unlink(self.filenames['state'])
+        else:
+            state = {'last': new_last}
+            with open(self.filenames['state'], 'w') as f:
+                f.write(json.dumps(state))
+
+    @property
+    def interfaces(self) -> list:
+        return State._load_json(
+            self.filenames['cache'],
+            inventory.INVENTORY_INTERFACES_SCHEMA)
+
+    @interfaces.setter
+    def interfaces(self, new_interfaces):
+        try:
+            jsonschema.validate(
+                new_interfaces,
+                inventory.INVENTORY_INTERFACES_SCHEMA)
+        except jsonschema.ValidationError:
+            logger.exception('invalid interface state data')
+            return
+
+        with open(self.filenames['cache'], 'w') as f:
+            f.write(json.dumps(new_interfaces))
+
+
+def _setup_logging(filename=None):
+    """
+    set up logging using the configured filename
+
+    :raises: json.JSONDecodeError, OSError, AttributeError,
+        ValueError, TypeError, ImportError
+    """
+    if not filename:
+        filename = _DEFAULT_LOGGING_FILENAME
+
+    with open(filename) as f:
+        # TODO: this mac workaround should be removed ...
+        d = json.loads(f.read())
+        import platform
+        if platform.system() == 'Darwin':
+            d['handlers']['syslog_handler']['address'] = '/var/run/syslog'
+        logging.config.dictConfig(d)
+        # logging.config.dictConfig(json.loads(f.read()))
+
+
+def load_config(file):
+    """
+    loads, validates and returns configuration parameters
+
+    :param value: filename (file-like object, opened for reading)
+    :return: a dict containing configuration parameters
+    :raises: json.JSONDecodeError, jsonschema.ValidationError,
+        OSError, AttributeError, ValueError, TypeError, ImportError
+    """
+    if file is None:
+        config = _DEFAULT_CONFIG
+    else:
+        config = json.loads(file.read())
+        jsonschema.validate(config, CONFIG_SCHEMA)
+
+    _setup_logging(config.get('logging', None))
+
+    return config
diff --git a/brian_polling_manager/environment.py b/brian_polling_manager/environment.py
deleted file mode 100644
index 9662849..0000000
--- a/brian_polling_manager/environment.py
+++ /dev/null
@@ -1,24 +0,0 @@
-import json
-import logging.config
-import os
-
-DEFAULT_LOGGING_FILENAME = os.path.join(
-    os.path.dirname(__file__),
-    'logging_default_config.json')
-
-
-def setup_logging(filename=None):
-    """
-    set up logging using the configured filename
-    """
-    if not filename:
-        filename = DEFAULT_LOGGING_FILENAME
-
-    with open(filename) as f:
-        # TODO: this mac workaround should be removed ...
-        d = json.loads(f.read())
-        import platform
-        if platform.system() == 'Darwin':
-            d['handlers']['syslog_handler']['address'] = '/var/run/syslog'
-        logging.config.dictConfig(d)
-        # logging.config.dictConfig(json.loads(f.read()))
-- 
GitLab