Finished release 0.34.

changelog

@@ -49,4 +49,6 @@
 0.31: Added top-level-services to the related services
       Fix canonicalization error of v6 addresses
 0.32: Ensured all Related Services are returned for juniper links
-0.33: Added Related Services for Infinera info
+0.33: Added Related Services for Infinera info
\ No newline at end of file
+0.34: POL1-135: initial support for service category api
+      DBOARD3-203: omite 'inactive' interfaces
\ No newline at end of file

inventory_provider/juniper.py

@@ -240,6 +240,8 @@ def list_interfaces(netconf_config):
 
     def _units(base_name, node):
         for u in node.xpath('./unit'):
+            if u.get('inactive', None) == 'inactive':
+                continue
             unit_info = _ifc_info(u)
             unit_info['name'] = "%s.%s" % (base_name, unit_info['name'])
             yield unit_info

inventory_provider/routes/poller.py

@@ -82,3 +82,23 @@ def poller_interface_oids(hostname):
         result.append(ifc_data)
 
     return jsonify(result)
+
+
+@routes.route('/services/<category>', methods=['GET', 'POST'])
+@common.require_accepts_json
+def service_category_interfaces(category):
+
+    result = []
+
+    r = common.get_current_redis()
+    for k in r.scan_iter(f'interface-services:{category.lower()}:*'):
+        ifc = r.get(k.decode('utf-8'))
+        result.append(json.loads(ifc.decode('utf-8')))
+
+    if not result:
+        return Response(
+            response=f'no info available for service category {category}',
+            status=404,
+            mimetype="text/html")
+
+    return jsonify(result)

inventory_provider/tasks/worker.py

@@ -560,6 +560,7 @@ def refresh_finalizer(self, pending_task_ids_json):
 
         _wait_for_tasks(task_ids, update_callback=_update)
         _build_subnet_db(update_callback=_update)
+        _build_interface_services(update_callback=_update)
 
     except (jsonschema.ValidationError,
             json.JSONDecodeError,
@@ -573,6 +574,54 @@ def refresh_finalizer(self, pending_task_ids_json):
     logger.debug('<<< refresh_finalizer')
 
 
+def _build_interface_services(update_callback=lambda s: None):
+    logger.debug('>>> _build_interface_services')
+
+    r = get_next_redis(InventoryTask.config)
+
+    def _interfaces():
+        for k in r.scan_iter('netconf-interfaces:*'):
+            k = k.decode('utf-8')
+            (_, router_name, ifc_name) = k.split(':')
+
+            info = r.get(k).decode('utf-8')
+            info = json.loads(info)
+
+            assert ifc_name == info['name']
+            yield {
+                'router': router_name,
+                'interface': info['name'],
+                'description': info['description']
+            }
+
+    def _classify(ifc):
+        if ifc['description'].startswith('SRV_MDVPN'):
+            return 'mdvpn'
+        if 'LHCONE' in ifc['description']:
+            return 'lhcone'
+        return None
+
+    r = get_next_redis(InventoryTask.config)
+    rp = r.pipeline()
+
+    update_callback('loading all known interfaces')
+    interfaces = list(_interfaces())
+    update_callback(f'loaded {len(interfaces)} interfaces, '
+                    'saving by service category')
+
+    for ifc in interfaces:
+        service_type = _classify(ifc)
+        if not service_type:
+            continue
+        rp.set(
+            f'interface-services:{service_type}'
+            f':{ifc["router"]}:{ifc["interface"]}',
+            json.dumps(ifc))
+
+    rp.execute()
+    logger.debug('<<< _build_interface_services')
+
+
 def _build_subnet_db(update_callback=lambda s: None):
 
     r = get_next_redis(InventoryTask.config)

setup.py

@@ -2,7 +2,7 @@ from setuptools import setup, find_packages
 
 setup(
     name='inventory-provider',
-    version="0.33",
+    version="0.34",
     author='GEANT',
     author_email='swd@geant.org',
     description='Dashboard inventory provider',

test/test_general_poller_routes.py

+import json
+
+import jsonschema
+import pytest
+
+from inventory_provider.tasks import worker
+
+DEFAULT_REQUEST_HEADERS = {
+    "Content-type": "application/json",
+    "Accept": ["application/json"]
+}
+
+
+INTERFACE_LIST_SCHEMA = {
+    '$schema': 'http://json-schema.org/draft-07/schema#',
+
+    'definitions': {
+        'ifc-info': {
+            'type': 'object',
+            'properties': {
+                'description': {'type': 'string'},
+                'router': {'type': 'string'},
+                'interface': {'type': 'string'}
+            },
+            'required': ['router', 'interface', 'description'],
+            'additionalProperties': False
+        },
+    },
+
+    'type': 'array',
+    'items': {'$ref': '#/definitions/ifc-info'}
+}
+
+
+@pytest.mark.parametrize('category', ['mdvpn', 'lhcone', 'MDVpn', 'LHCONE'])
+def test_service_category(client, mocked_worker_module, category):
+    worker._build_interface_services()
+    rv = client.get(
+        f'/poller/services/{category}',
+        headers=DEFAULT_REQUEST_HEADERS)
+    assert rv.status_code == 200
+    assert rv.is_json
+    response_data = json.loads(rv.data.decode('utf-8'))
+    jsonschema.validate(response_data, INTERFACE_LIST_SCHEMA)
+    assert response_data, 'expected a non-empty list'
+
+
+@pytest.mark.parametrize('category', ['mdvpn ', ' mdvpn', 'mdvpn1', 'aaa'])
+def test_service_category_not_found(client, mocked_worker_module, category):
+    worker._build_interface_services()
+    rv = client.get(
+        f'/poller/services/{category}',
+        headers=DEFAULT_REQUEST_HEADERS)
+    assert rv.status_code == 404

test/test_worker_utils.py

+"""
+tests of a few worker utilities
+"""
+import json
+import re
+
+import jsonschema
+
+from inventory_provider.tasks import worker
+from inventory_provider.tasks import common
+
+
+def backend_db():
+    return common._get_redis({
+        'redis': {
+            'hostname': None,
+            'port': None
+        },
+        'redis-databases': [0, 7]
+    }).db
+
+
+def test_build_interface_services(mocked_worker_module):
+    """
+    checks that valid interface service objects are created
+    :param mocked_worker_module: fixture
+    :return:
+    """
+    ifc_schema = {
+        '$schema': 'http://json-schema.org/draft-07/schema#',
+
+        'type': 'object',
+        'properties': {
+            'description': {'type': 'string'},
+            'router': {'type': 'string'},
+            'interface': {'type': 'string'}
+        },
+        'required': ['router', 'interface', 'description'],
+        'additionalProperties': False
+    }
+
+    db = backend_db()  # also forces initialization
+    worker._build_interface_services()
+
+    seen_types = set()
+    for k, v in db.items():
+        if not k.startswith('interface-services:'):
+            continue
+
+        (_, type, router, ifc_name) = k.split(':')
+
+        ifc_info = json.loads(v)
+        jsonschema.validate(json.loads(v), ifc_schema)
+
+        assert ifc_info['router'] == router
+        assert ifc_info['interface'] == ifc_name
+
+        seen_types.add(type)
+
+    assert type in ('mdvpn', 'lhcone')
+
+    expected_seen_types = set(['mdvpn', 'lhcone'])
+    assert seen_types == expected_seen_types
+
+
+def test_build_subnet_db(mocked_worker_module):
+    """
+    checks that valid reverse subnet objects are created
+    :param mocked_worker_module: fixture
+    :return:
+    """
+
+    address_schema = {
+        '$schema': 'http://json-schema.org/draft-07/schema#',
+
+        'type': 'object',
+        'properties': {
+            'name': {'type': 'string'},
+            'interface address': {'type': 'string'},
+            'interface name': {'type': 'string'},
+            'router': {'type': 'string'}
+        },
+        'required': ['name', 'interface address', 'interface name', 'router'],
+        'additionalProperties': False
+    }
+
+    db = backend_db()  # also forces initialization
+    worker._build_subnet_db()
+
+    found_record = False
+    for key, value in db.items():
+
+        if not key.startswith('reverse_interface_addresses:'):
+            continue
+
+        found_record = True
+
+        m = re.match('^reverse_interface_addresses:(.+)', key)
+        assert m
+        address = m.group(1)
+
+        value = json.loads(value)
+        jsonschema.validate(value, address_schema)
+        assert value['name'] == address
+
+    assert found_record