Finished release 0.65.

Changelog.md

@@ -2,7 +2,15 @@
 
 All notable changes to this project will be documented in this file.
 
-## [0.62] - 2021-04-01
+## [0.65] - 2021-05-28
+- DBOARD3-438: Changed status of non-monitored services
+- DBOARD3-439: initial LnetD endpoint
+
+
+## [0.64] - 2021-05-19
+- DBOARD3-303: Replace dependency on OpsDB with IMS
+
+## [0.63] - 2021-04-01
 - POL1-370: load & return lab interfaces for poller
 
 ## [0.62] - 2021-03-24

docs/source/protocol/index.rst

@@ -33,4 +33,5 @@ API modules
    lg
    data
    jobs
-   msr
\ No newline at end of file
+   msr
+   lnetd
\ No newline at end of file

docs/source/protocol/lnetd.rst

+.. LnetD endpoint docs
+
+
+LnetD support
+=========================
+
+This endpoint is intended for use with LnetD
+
+.. contents:: :local:
+
+/LnetD/interfaces</hostname>
+---------------------------------
+
+.. autofunction:: inventory_provider.routes.lnetd.interfaces

inventory_provider/__init__.py

@@ -78,6 +78,9 @@ def create_app():
     from inventory_provider.routes import msr
     app.register_blueprint(msr.routes, url_prefix='/msr')
 
+    from inventory_provider.routes import lnetd
+    app.register_blueprint(lnetd.routes, url_prefix='/LnetD')
+
     if app.config.get('ENABLE_TESTING_ROUTES', False):
         from inventory_provider.routes import testing
         app.register_blueprint(testing.routes, url_prefix='/testing')

inventory_provider/routes/classifier.py

@@ -143,15 +143,13 @@ def get_interface_services_and_loc(ims_source_equipment, ims_interface, redis):
         for s in json.loads(raw_services.decode('utf-8')):
             related_services.update(
                 {r['id']: r for r in s['related-services']})
-            if s['monitored'] and s['circuit_type'] == 'service':
+            if s['circuit_type'] == 'service':
                 contacts.update(set(s.pop('contacts', set())))
                 _format_service(s)
                 result['services'].append(s)
         result['related-services'] = list(related_services.values())
         result['contacts'] = sorted(list(contacts))
 
-        # non-monitored related services are not added by the worker so don't
-        # need filtering out here
         if not result['services']:
             result.pop('services', None)
         if result['related-services']:

inventory_provider/routes/common.py

@@ -252,3 +252,17 @@ def load_json_docs(config_params, key_pattern, num_threads=10):
 def load_xml_docs(config_params, key_pattern, num_threads=10):
     yield from _load_redis_docs(
         config_params, key_pattern, num_threads, doc_type=_DECODE_TYPE_XML)
+
+
+def load_snmp_indexes(hostname=None):
+    result = dict()
+    key_pattern = f'snmp-interfaces:{hostname}*' \
+        if hostname else 'snmp-interfaces:*'
+
+    for doc in load_json_docs(
+            config_params=current_app.config['INVENTORY_PROVIDER_CONFIG'],
+            key_pattern=key_pattern):
+        router = doc['key'][len('snmp-interfaces:'):]
+        result[router] = {e['name']: e for e in doc['value']}
+
+    return result

inventory_provider/routes/lnetd.py

+import json
+import logging
+import re
+
+from flask import Blueprint, Response, current_app, request
+
+from inventory_provider import juniper
+from inventory_provider.routes import common
+from inventory_provider.routes.common import _ignore_cache_or_retrieve
+
+logger = logging.getLogger(__name__)
+routes = Blueprint('lnetd-support-routes', __name__)
+
+INTERFACE_LIST_SCHEMA = {
+    '$schema': 'http://json-schema.org/draft-07/schema#',
+
+    'definitions': {
+
+        'ipv4-interface-address': {
+            'type': 'string',
+            'pattern': r'^(\d+\.){3}\d+/\d+$'
+        },
+        'ipv6-interface-address': {
+            'type': 'string',
+            'pattern': r'^[a-f\d:]+/\d+$'
+        },
+
+        'interface': {
+            'type': 'object',
+            'properties': {
+                'hostname': {'type': 'string'},
+                'interface': {'type': 'string'},
+                'ifIndex': {'type': 'integer', 'minimum': 1},
+                'ipv4': {
+                    'type': 'array',
+                    'items': {'$ref': '#/definitions/ipv4-interface-address'}
+                },
+                'ipv6': {
+                    'type': 'array',
+                    'items': {'$ref': '#/definitions/ipv6-interface-address'}
+                },
+            },
+            'required': ['hostname', 'interface', 'ifIndex', 'ipv4', 'ipv6'],
+            'additionalProperties': False
+        },
+    },
+
+    'type': 'array',
+    'items': {'$ref': '#/definitions/interface'}
+}
+
+
+def _add_snmp_indexes(interfaces, hostname=None):
+    """
+    generator that adds snmp ifIndex to each interface, if available
+    (only interfaces with an snmp index are yielded)
+
+    :param interfaces: result of _load_interfaces
+    :param hostname: hostname or None for all
+    :return: generator that yields interfaces with 'ifIndex' added
+    """
+    snmp_indexes = common.load_snmp_indexes(hostname)
+    for ifc in interfaces:
+
+        hostname = ifc['hostname']
+        if hostname not in snmp_indexes:
+            continue
+
+        interface = ifc['interface']
+        if interface not in snmp_indexes[hostname]:
+            continue
+
+        ifc['ifIndex'] = snmp_indexes[hostname][interface]['index']
+        yield ifc
+
+
+def _load_router_interfaces(hostname):
+    """
+    loads basic interface data for production & lab routers
+
+    :param hostname:
+    :return:
+    """
+    def _load_docs(key_pattern):
+
+        m = re.match(r'^(.*netconf:).+', key_pattern)
+        assert m  # sanity
+        key_prefix_len = len(m.group(1))
+        assert key_prefix_len >= len('netconf:')  # sanity
+
+        for doc in common.load_xml_docs(
+                config_params=current_app.config['INVENTORY_PROVIDER_CONFIG'],
+                key_pattern=key_pattern,
+                num_threads=10):
+
+            router = doc['key'][key_prefix_len:]
+
+            for ifc in juniper.list_interfaces(doc['value']):
+                if not ifc['description']:
+                    continue
+                # seems we're only interested in interfaces with addresses?
+                if not ifc['ipv4'] and not ifc['ipv6']:
+                    continue
+
+                yield {
+                    'hostname': router,
+                    'interface': ifc['name'],
+                    'ipv4': ifc['ipv4'],
+                    'ipv6': ifc['ipv6']
+                }
+
+    base_key_pattern = f'netconf:{hostname}*' if hostname else 'netconf:*'
+    yield from _load_docs(base_key_pattern)
+    yield from _load_docs(f'lab:{base_key_pattern}')
+
+
+def _load_interfaces(hostname=None):
+    """
+    prepares the result of a call to /interfaces
+
+    :param hostname: hostname or None for all
+    :return: generator yielding interface elements
+    """
+    basic_interfaces = _load_router_interfaces(hostname)
+    return _add_snmp_indexes(basic_interfaces, hostname)
+
+
+@routes.route("/interfaces", methods=['GET', 'POST'])
+@routes.route('/interfaces/<hostname>', methods=['GET', 'POST'])
+@common.require_accepts_json
+def interfaces(hostname=None):
+    """
+    Handler for `/LnetD/interfaces` and
+    `/LnetD/interfaces/<hostname>`
+    which returns information for either all interfaces
+    or those on the requested hostname.
+
+    .. asjson::
+       inventory_provider.routes.lnetd.INTERFACE_LIST_SCHEMA
+
+    :param hostname: optional, if present should be a router hostname
+    :return:
+    """
+
+    cache_key = f'classifier-cache:lnetd-interfaces:{hostname}' \
+        if hostname else 'classifier-cache:lnetd-interfaces:all'
+
+    r = common.get_current_redis()
+
+    result = _ignore_cache_or_retrieve(request, cache_key, r)
+
+    if not result:
+        result = list(_load_interfaces(hostname))
+        if not result:
+            return Response(
+                response='no interfaces found',
+                status=404,
+                mimetype='text/html')
+
+        result = json.dumps(result)
+        # cache this data for the next call
+        r.set(cache_key, result.encode('utf-8'))
+
+    return Response(result, mimetype="application/json")

inventory_provider/routes/poller.py

@@ -123,20 +123,6 @@ def after_request(resp):
     return common.after_request(resp)
 
 
-def _load_snmp_indexes(hostname=None):
-    result = dict()
-    key_pattern = f'snmp-interfaces:{hostname}*' \
-        if hostname else 'snmp-interfaces:*'
-
-    for doc in common.load_json_docs(
-            config_params=current_app.config['INVENTORY_PROVIDER_CONFIG'],
-            key_pattern=key_pattern):
-        router = doc['key'][len('snmp-interfaces:'):]
-        result[router] = {e['name']: e for e in doc['value']}
-
-    return result
-
-
 def _load_interface_bundles(hostname=None):
     result = dict()
 
@@ -295,7 +281,7 @@ def _add_snmp_indexes(interfaces, hostname=None):
     :param hostname: hostname or None for all
     :return: generator with 'snmp-index' optionally added to each element
     """
-    snmp_indexes = _load_snmp_indexes(hostname)
+    snmp_indexes = common.load_snmp_indexes(hostname)
     for ifc in interfaces:
         router_snmp = snmp_indexes.get(ifc['router'], None)
         if router_snmp and ifc['name'] in router_snmp:

inventory_provider/tasks/worker.py

@@ -571,8 +571,6 @@ def update_circuit_hierarchy_and_port_id_services(self, use_current=False):
     tls_names = list(ims_data.get_service_types(ds1))
     customer_contacts = \
         {k: v for k, v in ims_data.get_customer_service_emails(ds1)}
-    circuit_ids_not_to_monitor = \
-        list(ims_data.get_non_monitored_circuit_ids(ds1))
     circuit_ids_to_monitor = \
         list(ims_data.get_monitored_circuit_ids(ds1))
 
@@ -618,7 +616,6 @@ def update_circuit_hierarchy_and_port_id_services(self, use_current=False):
         nonlocal hierarchy
         hierarchy = {}
         for d in ims_data.get_circuit_hierarchy(ds1):
-            d['monitored'] = d['id'] in circuit_ids_to_monitor
             d['contacts'] = customer_contacts.get(d['customerid'], [])
             hierarchy[d['id']] = d
         logger.debug("hierarchy complete")
@@ -667,25 +664,20 @@ def update_circuit_hierarchy_and_port_id_services(self, use_current=False):
         c = hierarchy.get(circuit_id, None)
         if c:
 
-            def _is_rs(candidate):
-                if candidate['id'] in circuit_ids_not_to_monitor:
-                    return False
-                if candidate['product'] in tls_names:
-                    return True
-                # if candidate['speed'] == 'BGP':
-                #     return True
-                return False
-
-            if _is_rs(c):
+            if c['circuit-type'] == 'service':
                 rs[c['id']] = {
                     'id': c['id'],
                     'name': c['name'],
-                    'status': c['status'],
-                    'circuit_type': 'service',
+                    'circuit_type': c['circuit-type'],
                     'service_type': c['product'],
                     'project': c['project'],
                     'contacts': sorted(list(c['contacts']))
                 }
+                if c['id'] in circuit_ids_to_monitor:
+                    rs[c['id']]['status'] = c['status']
+                else:
+                    rs[c['id']]['status'] = 'non-monitored'
+
             if c['sub-circuits']:
                 for sub in c['sub-circuits']:
                     temp_parents = \
@@ -694,10 +686,10 @@ def update_circuit_hierarchy_and_port_id_services(self, use_current=False):
         return list(rs.values())
 
     def _format_service(s):
-        if s['id'] in circuit_ids_not_to_monitor:
-            s['monitored'] = False
-        else:
-            s['monitored'] = True
+
+        if s['circuit_type'] == 'service' \
+                and s['id'] not in circuit_ids_to_monitor:
+            s['status'] = 'non-monitored'
         pd_a = port_id_details[s['port_a_id']][0]
         location_a = locations.get(pd_a['equipment_name'], None)
         if location_a:
@@ -758,10 +750,6 @@ def update_circuit_hierarchy_and_port_id_services(self, use_current=False):
                         'name': hierarchy[x]['name'],
                         'status': hierarchy[x]['status']
                     }
-                    if c['id'] in circuit_ids_not_to_monitor:
-                        c['monitored'] = False
-                    else:
-                        c['monitored'] = True
                     circ['fibre-routes'].append(c)
 
                 circ['related-services'] = \

setup.py

@@ -2,7 +2,7 @@ from setuptools import setup, find_packages
 
 setup(
     name='inventory-provider',
-    version="0.64",
+    version="0.65",
     author='GEANT',
     author_email='swd@geant.org',
     description='Dashboard inventory provider',

test/per_router/test_lnetd_routes.py

+import json
+import jsonschema
+import pytest
+from inventory_provider.routes.lnetd import INTERFACE_LIST_SCHEMA
+
+DEFAULT_REQUEST_HEADERS = {
+    'Accept': ['application/json']
+}
+
+
+def test_router_interfaces(router, client):
+    if router.startswith('qfx'):
+        pytest.skip('no interfaces expected for {router}, skipping')
+
+    rv = client.post(
+        f'/LnetD/interfaces/{router}',
+        headers=DEFAULT_REQUEST_HEADERS)
+
+    assert rv.status_code == 200
+    response = json.loads(rv.data.decode('utf-8'))
+    jsonschema.validate(response, INTERFACE_LIST_SCHEMA)
+    assert response  # at least shouldn't be empty
+    assert all(ifc['hostname'] == router for ifc in response)

test/test_general_lnetd_routes.py

+import json
+import jsonschema
+from inventory_provider.routes import lnetd
+
+DEFAULT_REQUEST_HEADERS = {
+    'Accept': ['application/json']
+}
+
+
+def test_get_all_interfaces(client):
+    rv = client.get(
+        '/LnetD/interfaces',
+        headers=DEFAULT_REQUEST_HEADERS)
+    assert rv.status_code == 200
+    assert rv.is_json
+    response_data = json.loads(rv.data.decode('utf-8'))
+    jsonschema.validate(response_data, lnetd.INTERFACE_LIST_SCHEMA)
+    response_routers = {ifc['hostname'] for ifc in response_data}
+    assert len(response_routers) > 1, \
+        'there should data from be lots of routers'