Finished feature flask-api.

brian_polling_manager/__init__.py

+"""
+automatically invoked app factory
+"""
+import logging
+import os
+
+from flask import Flask
+
+from brian_polling_manager import configuration
+
+CONFIG_KEY = 'CONFIG_PARAMS'
+SESSION_SECRET = 'super-secret'
+
+
+def create_app():
+    """
+    overrides default settings with those found
+    in the file read from env var CONFIG_FILENAME
+
+    :return: a new flask app instance
+    """
+
+    app = Flask(__name__)
+    app.secret_key = SESSION_SECRET
+
+    if 'CONFIG_FILENAME' in os.environ:
+        with open(os.environ['CONFIG_FILENAME']) as f:
+            app.config[CONFIG_KEY] = configuration.load_config(f)
+    else:
+        # this inits with the defaults
+        app.config[CONFIG_KEY] = configuration.load_config()
+
+    from brian_polling_manager import api
+    app.register_blueprint(api.routes, url_prefix='/api')
+
+    logging.info('Flask app initialized')
+    return app

brian_polling_manager/api.py

+"""
+These endpoints are used to update BRIAN snmp polling checks to Sensu
+
+.. contents:: :local:
+
+
+/api/version
+---------------------
+
+.. autofunction:: brian_polling_manager.api.version
+
+
+/api/update
+-----------------------------
+
+.. autofunction:: brian_polling_manager.api.update
+
+"""
+import functools
+import logging
+import pkg_resources
+
+from flask import Blueprint, current_app, request, Response, jsonify
+
+from brian_polling_manager import CONFIG_KEY
+from brian_polling_manager import main
+
+routes = Blueprint("api-routes", __name__)
+logger = logging.getLogger(__name__)
+
+
+API_VERSION = '0.1'
+
+VERSION_SCHEMA = {
+    '$schema': 'http://json-schema.org/draft-07/schema#',
+
+    'definitions': {
+        'latch': {
+            'type': 'object',
+            'properties': {
+                'current': {'type': 'integer'},
+                'next': {'type': 'integer'},
+                'this': {'type': 'integer'},
+                'failure': {'type': 'boolean'},
+                'pending': {'type': 'boolean'},
+                'timestamp': {'type': 'number'}
+            },
+            'required': ['current', 'next', 'this', 'pending', 'failure'],
+            'additionalProperties': False
+        }
+    },
+
+    'type': 'object',
+    'properties': {
+        'api': {
+            'type': 'string',
+            'pattern': r'\d+\.\d+'
+        },
+        'module': {
+            'type': 'string',
+            'pattern': r'\d+\.\d+'
+        },
+        'latch': {'$ref': '#/definitions/latch'}
+    },
+    'required': ['api', 'module'],
+    'additionalProperties': False
+}
+
+
+class APIUpstreamError(Exception):
+    def __init__(self, message, status_code=504):
+        super().__init__()
+        self.message = message
+        self.status_code = status_code
+
+
+@routes.errorhandler(APIUpstreamError)
+def handle_request_error(error):
+    logger.error(f'api error: {error.message}')
+    return Response(
+        response=error.message,
+        status=error.status_code,
+        mimetype='text/html')
+
+
+def require_accepts_json(f):
+    """
+    used as a route handler decorator to return an error
+    unless the request allows responses with type "application/json"
+    :param f: the function to be decorated
+    :return: the decorated function
+    """
+    @functools.wraps(f)
+    def decorated_function(*args, **kwargs):
+        # TODO: use best_match to disallow */* ...?
+        if not request.accept_mimetypes.accept_json:
+            return Response(
+                response="response will be json",
+                status=406,
+                mimetype="text/html")
+        return f(*args, **kwargs)
+    return decorated_function
+
+
+@routes.after_request
+def after_request(response):
+    """
+    Generic function to do additional logging of requests & responses.
+
+    :param response:
+    :return:
+    """
+    if response.status_code != 200:
+
+        try:
+            data = response.data.decode('utf-8')
+        except Exception:
+            # never expected to happen, but we don't want any failures here
+            logging.exception('INTERNAL DECODING ERROR')
+            data = 'decoding error (see logs)'
+
+        logger.warning(
+            f'[{response.status_code}] {request.method} {request.path} {data}')
+    else:
+        logger.info(
+            f'[{response.status_code}] {request.method} {request.path}')
+    return response
+
+
+@routes.route('/version', methods=['GET', 'POST'])
+@require_accepts_json
+def version():
+    """
+    Returns a json object with information about the module version.
+
+    The response will be formatted according to the following schema:
+
+    .. asjson:: brian_polling_manager.api.VERSION_SCHEMA
+
+    :return:
+    """
+    version_params = {
+        'api': API_VERSION,
+        'module':
+            pkg_resources.get_distribution('brian_polling_manager').version
+    }
+    return jsonify(version_params)
+
+
+@routes.route('/update', methods=['GET', 'POST'])
+@require_accepts_json
+def update():
+    """
+    Update checks based on current inventory provider state.
+
+    The response will be formatted according to the following schema:
+
+    .. asjson:: brian_polling_manager.main.REFRESH_RESULT_SCHEMA
+
+    :return:
+    """
+    response = main.refresh(config=current_app.config[CONFIG_KEY])
+    return jsonify(response)

brian_polling_manager/app.py

+"""
+default app creation
+"""
+import brian_polling_manager
+from brian_polling_manager import configuration
+
+app = brian_polling_manager.create_app()
+
+if __name__ == "__main__":
+    app.run(host="::", port="7878")

brian_polling_manager/cli.py → brian_polling_manager/configuration.py

-"""
-This script queries Inventory Provider for changes
-and configures Sensu with the snmp polling checks
-required by BRIAN.
-
-.. code-block:: console
-
-    % brian-polling-manager --help
-    Usage: brian-polling-manager [OPTIONS]
-
-      Update BRIAN snmp checks based on Inventory Provider data.
-
-    Options:
-      --config TEXT         configuration filename
-      --force / --no-force  update even if inventory hasn't been updated
-      --help                Show this message and exit.
-
-The required configuration file must be
-formatted according to the following schema:
-
-.. asjson::
-    brian_polling_manager.cli.CONFIG_SCHEMA
-
-"""
 import json
 import logging
+import logging.config
 import os
 from typing import Union
 
-import click
 import jsonschema
-from statsd import StatsClient
 
-from brian_polling_manager import inventory, interfaces, environment
+from brian_polling_manager import inventory
 
 logger = logging.getLogger(__name__)
 
+_DEFAULT_LOGGING_FILENAME = os.path.join(
+    os.path.dirname(__file__),
+    'logging_default_config.json')
+
 _DEFAULT_CONFIG = {
     'inventory': [
         'http://test-inventory-provider01.geant.org:8080',
@@ -61,7 +40,7 @@ _DEFAULT_CONFIG = {
         }
     },
     'statedir': '/tmp/',
-    'logging': environment.DEFAULT_LOGGING_FILENAME,
+    'logging': _DEFAULT_LOGGING_FILENAME,
     'statsd': {
         'hostname': 'localhost',
         'port': 8125,
@@ -205,66 +184,41 @@ class State(object):
             f.write(json.dumps(new_interfaces))
 
 
-def _validate_config(ctx, param, file):
+def _setup_logging(filename=None):
     """
-    loads, validates and returns configuration parameters
+    set up logging using the configured filename
 
-    :param ctx:
-    :param param:
-    :param value: filename (string)
-    :return: a dict containing configuration parameters
+    :raises: json.JSONDecodeError, OSError, AttributeError,
+        ValueError, TypeError, ImportError
     """
-    if file is None:
-        config = _DEFAULT_CONFIG
-    else:
-        try:
-            config = json.loads(file.read())
-        except (json.JSONDecodeError, jsonschema.ValidationError) as e:
-            raise click.BadParameter(str(e))
+    if not filename:
+        filename = _DEFAULT_LOGGING_FILENAME
 
-    try:
-        jsonschema.validate(config, CONFIG_SCHEMA)
-    except jsonschema.ValidationError as e:
-        raise click.BadParameter(str(e))
-
-    environment.setup_logging(config.get('logging', None))
-
-    return config
+    with open(filename) as f:
+        # TODO: this mac workaround should be removed ...
+        d = json.loads(f.read())
+        import platform
+        if platform.system() == 'Darwin':
+            d['handlers']['syslog_handler']['address'] = '/var/run/syslog'
+        logging.config.dictConfig(d)
+        # logging.config.dictConfig(json.loads(f.read()))
 
 
-@click.command()
-@click.option(
-    '--config',
-    default=None,
-    type=click.File('r'),
-    callback=_validate_config,
-    help='configuration filename')
-@click.option(
-    '--force/--no-force',
-    default=False,
-    help="refresh inventory data even if it hasn't been updated")
-def main(config, force):
+def load_config(file=None):
     """
-    Update BRIAN snmp checks based on Inventory Provider data.
-    """
-
-    state = State(config['statedir'])
-    last = inventory.last_update_timestamp(config['inventory'])
-    if force or not last or last != state.last:
-        state.last = last
-        state.interfaces = inventory.load_interfaces(config['inventory'])
+    loads, validates and returns configuration parameters
 
-    statsd_config = config.get('statsd', None)
-    if statsd_config:
-        statsd = StatsClient(
-            host=statsd_config['hostname'],
-            port=statsd_config['port'],
-            prefix=statsd_config['prefix'])
+    :param value: filename (file-like object, opened for reading)
+    :return: a dict containing configuration parameters
+    :raises: json.JSONDecodeError, jsonschema.ValidationError,
+        OSError, AttributeError, ValueError, TypeError, ImportError
+    """
+    if not file:
+        config = _DEFAULT_CONFIG
     else:
-        statsd = None
-
-    interfaces.refresh(config['sensu'], state, statsd=statsd)
+        config = json.loads(file.read())
+        jsonschema.validate(config, CONFIG_SCHEMA)
 
+    _setup_logging(config.get('logging', None))
 
-if __name__ == '__main__':
-    main()
+    return config

brian_polling_manager/environment.py

-import json
-import logging.config
-import os
-
-DEFAULT_LOGGING_FILENAME = os.path.join(
-    os.path.dirname(__file__),
-    'logging_default_config.json')
-
-
-def setup_logging(filename=None):
-    """
-    set up logging using the configured filename
-    """
-    if not filename:
-        filename = DEFAULT_LOGGING_FILENAME
-
-    with open(filename) as f:
-        # TODO: this mac workaround should be removed ...
-        d = json.loads(f.read())
-        import platform
-        if platform.system() == 'Darwin':
-            d['handlers']['syslog_handler']['address'] = '/var/run/syslog'
-        logging.config.dictConfig(d)
-        # logging.config.dictConfig(json.loads(f.read()))

brian_polling_manager/interfaces.py

@@ -70,7 +70,7 @@ def _checks_match(a, b) -> bool:
     return True
 
 
-def refresh(sensu_params, state, statsd=None):
+def refresh(sensu_params, state):
 
     ifc_checks = load_ifc_checks(sensu_params)
 
@@ -97,9 +97,11 @@ def refresh(sensu_params, state, statsd=None):
     for name in extra_checks:
         sensu.delete_check(sensu_params, name)
 
-    if statsd:
-        statsd.gauge('checks', len(ifc_checks))
-        statsd.gauge('interfaces', interfaces)
-        statsd.gauge('checks_created', created)
-        statsd.gauge('checks_updated', updated)
-        statsd.gauge('checks_deleted', len(extra_checks))
+    # cf. main.REFRESH_RESULT_SCHEMA
+    return {
+        'checks': len(ifc_checks),
+        'input': interfaces,
+        'created': created,
+        'updated': updated,
+        'deleted': len(extra_checks)
+    }

brian_polling_manager/main.py

+"""
+This script queries Inventory Provider for changes
+and configures Sensu with the snmp polling checks
+required by BRIAN.
+
+.. code-block:: console
+
+    % brian-polling-manager --help
+    Usage: brian-polling-manager [OPTIONS]
+
+      Update BRIAN snmp checks based on Inventory Provider data.
+
+    Options:
+      --config TEXT         configuration filename
+      --force / --no-force  update even if inventory hasn't been updated
+      --help                Show this message and exit.
+
+The required configuration file must be
+formatted according to the following schema:
+
+.. asjson::
+    brian_polling_manager.configuration.CONFIG_SCHEMA
+
+"""
+import json
+import logging
+
+import click
+import jsonschema
+from statsd import StatsClient
+
+from brian_polling_manager import inventory, interfaces, configuration
+
+logger = logging.getLogger(__name__)
+
+REFRESH_RESULT_SCHEMA = {
+    '$schema': 'http://json-schema.org/draft-07/schema#',
+    'definitions': {
+        'refresh-result': {
+            'type': 'object',
+            'properties': {
+                'checks': {'type': 'integer'},
+                'input': {'type': 'integer'},
+                'created': {'type': 'integer'},
+                'updated': {'type': 'integer'},
+                'deleted': {'type': 'integer'}
+            },
+            'required': ['checks', 'input', 'created', 'updated', 'deleted'],
+            'additionalProperties': False
+        }
+    },
+    'type': 'object',
+    'properties': {
+        'interfaces': {'$ref': '#/definitions/refresh-result'}
+    },
+    'required': ['interfaces'],
+    'additionalProperties': False
+}
+
+
+def refresh(config, force=False):
+    """
+    reload inventory data & update sensu checks
+
+    The output will be a dict formatted according to the following schema:
+
+    .. asjson::
+        brian_polling_manager.main.REFRESH_RESULT_SCHEMA
+
+    :param config: a dict returned by configuration.load_config
+    :param force: if True, reload inventory data even if timestamp is same
+    :return: a dict, formatted as above
+    """
+    state = configuration.State(config['statedir'])
+    last = inventory.last_update_timestamp(config['inventory'])
+    if force or not last or last != state.last:
+        state.last = last
+        state.interfaces = inventory.load_interfaces(config['inventory'])
+
+    result = {
+        'interfaces': interfaces.refresh(config['sensu'], state)
+    }
+
+    statsd_config = config.get('statsd', None)
+    if statsd_config:
+        statsd = StatsClient(
+            host=statsd_config['hostname'],
+            port=statsd_config['port'],
+            prefix=f'{statsd_config["prefix"]}_interfaces')
+        statsd.gauge('checks', result['interfaces']['checks'])
+        statsd.gauge('input', result['interfaces']['input'])
+        statsd.gauge('created', result['interfaces']['created'])
+        statsd.gauge('updated', result['interfaces']['updated'])
+        statsd.gauge('deleted', result['interfaces']['deleted'])
+
+    jsonschema.validate(result, REFRESH_RESULT_SCHEMA)  # sanity
+    return result
+
+
+def _validate_config(_ctx, _param, file):
+    """
+    loads, validates and returns configuration parameters
+
+    :param _ctx: unused
+    :param _param: unused
+    :param value: file (file-like object open for reading)
+    :return: a dict containing configuration parameters
+    """
+    try:
+        return configuration.load_config(file)
+    except (json.JSONDecodeError, jsonschema.ValidationError, OSError,
+            AttributeError, ValueError, TypeError, ImportError) as e:
+        raise click.BadParameter(str(e))
+
+
+@click.command()
+@click.option(
+    '--config',
+    default=None,
+    type=click.File('r'),
+    callback=_validate_config,
+    help='configuration filename')
+@click.option(
+    '--force/--no-force',
+    default=False,
+    help="refresh inventory data even if it hasn't been updated")
+def cli(config, force):
+    """
+    Update BRIAN snmp checks based on Inventory Provider data.
+    """
+    logger.info(json.dumps(refresh(config, force)))
+
+
+if __name__ == '__main__':
+    cli()

docs/source/api.rst

+
+HTTP API
+===================================================
+
+.. automodule:: brian_polling_manager.api
+

docs/source/index.rst

@@ -16,4 +16,5 @@ Sensu checks for polling the data required by BRIAN.
    :maxdepth: 2
    :caption: Contents:
 
-   cli
+   main
+   api

docs/source/cli.rst → docs/source/main.rst

@@ -2,5 +2,5 @@
 brian-polling-manager
 ===================================================
 
-.. automodule:: brian_polling_manager.cli
+.. automodule:: brian_polling_manager.main
 

requirements.txt

@@ -2,6 +2,7 @@ click
 jsonschema
 requests
 statsd
+flask
 
 pytest
 responses

setup.py

@@ -12,11 +12,12 @@ setup(
         'click',
         'requests',
         'jsonschema',
-        'statsd'
+        'statsd',
+        'flask'
     ],
     entry_points={
         'console_scripts': [
-            'brian-polling-manager=brian_polling_manager.cli:main'
+            'brian-polling-manager=brian_polling_manager.main:cli'
         ]
     },
     include_package_data=True,

test/conftest.py

@@ -55,6 +55,14 @@ def config():
         }
 
 
+@pytest.fixture
+def config_filename(config):
+    with tempfile.NamedTemporaryFile(mode='w') as f:
+        f.write(json.dumps(config))
+        f.flush()
+        yield f.name
+
+
 @pytest.fixture
 def mocked_sensu():
 

test/test_api.py

+import json
+import os
+
+import jsonschema
+import pytest
+import responses
+
+import brian_polling_manager
+from brian_polling_manager.api import VERSION_SCHEMA
+from brian_polling_manager.main import REFRESH_RESULT_SCHEMA
+
+
+@pytest.fixture
+def client(config_filename, mocked_sensu, mocked_inventory):
+    os.environ['CONFIG_FILENAME'] = config_filename
+    with brian_polling_manager.create_app().test_client() as c:
+        yield c
+
+
+def test_version(client):
+    rv = client.get(
+        '/api/version',
+        headers={'Accept': ['application/json']})
+    assert rv.status_code == 200
+    assert rv.is_json
+    response_data = json.loads(rv.data.decode('utf-8'))
+    jsonschema.validate(response_data, VERSION_SCHEMA)
+
+
+@responses.activate
+def test_update(client):
+    rv = client.get(
+        '/api/update',
+        headers={'Accept': ['application/json']})
+    assert rv.status_code == 200
+    assert rv.is_json
+    response_data = json.loads(rv.data.decode('utf-8'))
+    jsonschema.validate(response_data, REFRESH_RESULT_SCHEMA)

test/test_e2e.py

-import json
-import tempfile
-
 from click.testing import CliRunner
 import responses
 
-from brian_polling_manager import cli
+from brian_polling_manager import main
 
 
 @responses.activate
-def test_run_flashtest(config, mocked_sensu, mocked_inventory):
-
-    with tempfile.NamedTemporaryFile(mode='w') as f:
-        f.write(json.dumps(config))
-        f.flush()
-
-        runner = CliRunner()
-        result = runner.invoke(
-            cli.main,
-            ['--config', f.name, '--force']
-        )
-        assert result.exit_code == 0
+def test_run_flashtest(config_filename, mocked_sensu, mocked_inventory):
+
+    runner = CliRunner()
+    result = runner.invoke(
+        main.cli,
+        ['--config', config_filename, '--force']
+    )
+    assert result.exit_code == 0

tox.ini

@@ -13,7 +13,7 @@ commands =
     coverage xml
     coverage html
     coverage report
-    coverage report --fail-under 75
+    coverage report --fail-under 80
     flake8
     sphinx-build -M html docs/source docs/build