created entry point

brian_polling_manager/interface_stats/cli.py

+import click
+from functools import partial
+import json
+
+import jsonschema
+
+from brian_polling_manager.interface_stats import config, brian, juniper
+from brian_polling_manager import inventory, influx
+
+
+import json
+import logging.config
+import os
+
+
+LOGGING_DEFAULT_CONFIG = {
+    'version': 1,
+    'disable_existing_loggers': False,
+    'formatters': {
+        'simple': {
+            'format': '%(asctime)s - %(name)s '
+                      '(%(lineno)d) - %(levelname)s - %(message)s'
+        }
+    },
+
+    'handlers': {
+        'console': {
+            'class': 'logging.StreamHandler',
+            'level': 'DEBUG',
+            'formatter': 'simple',
+            'stream': 'ext://sys.stdout'
+        },
+    },
+
+    'loggers': {
+        'brian_polling_manager': {
+            'level': 'DEBUG',
+            'handlers': ['console'],
+            'propagate': False
+        }
+    },
+
+    'root': {
+        'level': 'INFO',
+        'handlers': ['console']
+    }
+}
+
+
+def setup_logging():
+    """
+    set up logging using the configured filename
+
+    if LOGGING_CONFIG is defined in the environment, use this for
+    the filename, otherwise use LOGGING_DEFAULT_CONFIG
+    """
+    logging_config = LOGGING_DEFAULT_CONFIG
+    if 'LOGGING_CONFIG' in os.environ:
+        filename = os.environ['LOGGING_CONFIG']
+        with open(filename) as f:
+            logging_config = json.loads(f.read())
+
+    # # TODO: this mac workaround should be removed ...
+    # import platform
+    # if platform.system() == 'Darwin':
+    #     logging_config['handlers']['syslog_handler']['address'] \
+    #         = '/var/run/syslog'
+
+    logging.config.dictConfig(logging_config)
+
+
+def _validate_config(_unused_ctx, _unused_param, file):
+    try:
+        return config.load(file)
+    except json.JSONDecodeError:
+        raise click.BadParameter('config file is not valid json')
+    except jsonschema.ValidationError as e:
+        raise click.BadParameter(e)
+
+
+def _managed_routers(inventory_base_urls):
+    return {_ifc['router'] for _ifc in inventory.load_interfaces(inventory_base_urls)}
+
+
+def fqdn_netconf_doc_map(app_config_params):
+    _result = {}
+    for fqdn in _managed_routers(inventory_base_urls=app_config_params['inventory']):
+        _result[fqdn] = juniper.get_interface_info_ncrpc(fqdn)
+
+    return _result
+
+
+def _brian_points(router_fqdn, netconf_doc, measurement_name):
+    interfaces = juniper.physical_interface_counters(netconf_doc)
+    counters = brian.counters(router_fqdn=router_fqdn, interface_counters=interfaces)
+    yield from map(partial(brian.ctr2point, measurement_name), counters)
+
+    interfaces = juniper.logical_interface_counters(netconf_doc)
+    counters = brian.counters(router_fqdn=router_fqdn, interface_counters=interfaces)
+    yield from map(partial(brian.ctr2point, measurement_name), counters)
+
+
+def _main(app_config_params: dict):
+    """
+    callable entry point, without click
+    ... tmp, for testing
+
+    :param app_config_params:
+    :return:
+    """
+
+    setup_logging()
+
+    nc_doc_map = fqdn_netconf_doc_map(app_config_params)
+    influx_params = app_config_params['influx']['brian-counters']
+    for _fqdn, _ncdoc in nc_doc_map.items():
+        points = _brian_points(
+            router_fqdn=_fqdn,
+            netconf_doc=_ncdoc,
+            measurement_name=influx_params['measurement'])
+
+        with influx.influx_client(influx_params) as client:
+            client.write_points(points)
+
+
+@click.command()
+@click.option(
+    '--config', 'app_config_params',
+    required=True,
+    type=click.File('r'),
+    help='Config filename',
+    callback=_validate_config)
+def main(app_config_params: dict):
+    _main(app_config_params)
+
+
+if __name__ == '__main__':
+    main()

brian_polling_manager/interface_stats/config.py

+import json
+import logging
+import logging.config
+import os
+from typing import Union
+
+import jsonschema
+
+from brian_polling_manager import inventory
+
+logger = logging.getLogger(__name__)
+
+DEFAULT_INFLUX_SSL = True
+DEFAULT_INFLUX_PORT = 8086
+
+CONFIG_SCHEMA = {
+    '$schema': 'https://json-schema.org/draft/2020-12/schema',
+
+    'definitions': {
+        'influx-db-measurement': {
+            'type': 'object',
+            'properties': {
+                'ssl': {'type': 'boolean'},
+                'hostname': {'type': 'string'},
+                'port': {'type': 'integer'},
+                'username': {'type': 'string'},
+                'password': {'type': 'string'},
+                'database': {'type': 'string'},
+                'measurement': {'type': 'string'}
+            },
+            'required': [
+                # ssl, port are optional
+                'hostname',
+                'username',
+                'password',
+                'database',
+                'measurement'
+            ],
+            'additionalProperties': False
+        }
+    },
+
+    'type': 'object',
+    'properties': {
+        'ssh-config': {'type': 'string'},
+        'inventory': {
+            'type': 'array',
+            'items': {'type': 'string', 'format': 'uri'},
+            'minItems': 1
+        },
+        'influx': {
+            'type': 'object',
+            'properties': {
+                'brian-counters': {'$ref': '#/definitions/influx-db-measurement'},
+                'error-counters': {'$ref': '#/definitions/influx-db-measurement'},
+            },
+            'required': ['brian-counters', 'error-counters'],
+            'additionalProperties': False
+        },
+    },
+    'required': ['ssh-config', 'inventory', 'influx'],
+    'additionalProperties': False
+}
+
+
+def load(config_file):
+    """
+    loads, validates and returns configuration parameters
+
+    :param config_file: filename (file-like object, opened for reading)
+    :return: a dict containing configuration parameters
+    :raises: json.JSONDecodeError, jsonschema.ValidationError
+    """
+
+    config = json.loads(config_file.read())
+    jsonschema.validate(config, CONFIG_SCHEMA)
+
+    for db in config['influx']:
+        db.setdefault('ssl', DEFAULT_INFLUX_SSL)
+        db.setdefault('port', DEFAULT_INFLUX_PORT)
+
+    return config

test/test_interface_stats.py

@@ -2,6 +2,7 @@ from functools import partial
 import itertools
 import os
 import re
+import tempfile
 from unittest.mock import patch
 
 import jsonschema
@@ -11,7 +12,7 @@ import responses
 
 from brian_polling_manager.interface_stats import \
     PHYSICAL_INTERFACE_COUNTER_SCHEMA, LOGICAL_INTERFACE_COUNTER_SCHEMA
-from brian_polling_manager.interface_stats import brian, juniper
+from brian_polling_manager.interface_stats import brian, juniper, cli
 from brian_polling_manager import inventory, influx
 
 # logging.basicConfig(level=logging.INFO)
@@ -55,16 +56,20 @@ def test_validate_logical_counters_schema(router_fqdn, ifc_netconf_rpc):
         jsonschema.validate(ifc, LOGICAL_INTERFACE_COUNTER_SCHEMA)
 
 
-def poller_interfaces():
+def mock_poller_interfaces():
     # note: responses.activate doesn't seem to work if this is a fixture
     with open(os.path.join(DATA_DIRNAME, 'poller-interfaces.json')) as f:
         responses.add(
             method=responses.GET,
-            url='https://bogus-hostname/poller/interfaces',
+            url=re.compile(r'.*/poller/interfaces$'),
             body=f.read(),
             status=200,
             content_type="application/json")
 
+
+def poller_interfaces():
+    mock_poller_interfaces()
+
     polled = {}
     for ifc in inventory.load_interfaces(['https://bogus-hostname']):
         if ifc['dashboards']:
@@ -155,3 +160,56 @@ def test_logical_brian_points(router_fqdn, ifc_netconf_rpc):
     for _p in map(partial(brian.ctr2point, 'bogus-measurement'), counters):
         jsonschema.validate(_p, influx.INFLUX_POINT)
 
+
+@pytest.fixture
+def app_config_params():
+    with tempfile.NamedTemporaryFile() as f:
+        yield {
+            'ssh-config': f.name,
+            'inventory': [
+                'https://bogus-hostname-1',
+                'https://bogus-hostname-2',
+                'https://bogus-hostname-3',
+            ],
+            'influx': {
+                'brian-counters': {
+                    'hostname': 'bogus-hostname',
+                    'port': 8086,
+                    'database': 'bogus-database',
+                    'measurement': 'bogus-measurement',
+                    'username': 'bogus-username',
+                    'password': 'bogus-password',
+                    'ssl': True
+                },
+                'error-counters': {
+                    'hostname': 'bogus-hostname',
+                    'port': 8086,
+                    'database': 'bogus-database',
+                    'measurement': 'bogus-measurement',
+                    'username': 'bogus-username',
+                    'password': 'bogus-password',
+                    'ssl': True
+                },
+            }
+        }
+
+
+@responses.activate
+def test_validate_all_points(app_config_params, ifc_netconf_rpc):
+
+    def _mocked_managed_routers(inventory_base_urls):
+        all_routers = {_ifc['router'] for _ifc in inventory.load_interfaces(['https://abc'])}
+        return [_ifc for _ifc in all_routers if not _ifc.startswith('srx')]
+
+    with patch('brian_polling_manager.interface_stats.cli._managed_routers') as rpc:
+        rpc.side_effect = _mocked_managed_routers
+
+        mock_poller_interfaces()
+        nc_doc_map = cli.fqdn_netconf_doc_map(app_config_params)
+        influx_params = app_config_params['influx']['brian-counters']
+        for _fqdn, _ncdoc in nc_doc_map.items():
+            for _p in cli._brian_points(
+                    router_fqdn=_fqdn,
+                    netconf_doc=_ncdoc,
+                    measurement_name=influx_params['measurement']):
+                jsonschema.validate(_p, influx.INFLUX_POINT)