Finished feature DBOARD3-439-new-lnetd-endpoint.

docs/source/protocol/index.rst

@@ -33,4 +33,5 @@ API modules
    lg
    data
    jobs
-   msr
+   msr
\ No newline at end of file
+   lnetd
\ No newline at end of file

docs/source/protocol/lnetd.rst

+.. LnetD endpoint docs
+
+
+LnetD support
+=========================
+
+This endpoint is intended for use with LnetD
+
+.. contents:: :local:
+
+/LnetD/interfaces</hostname>
+---------------------------------
+
+.. autofunction:: inventory_provider.routes.lnetd.interfaces

inventory_provider/__init__.py

@@ -78,6 +78,9 @@ def create_app():
     from inventory_provider.routes import msr
     app.register_blueprint(msr.routes, url_prefix='/msr')
 
+    from inventory_provider.routes import lnetd
+    app.register_blueprint(lnetd.routes, url_prefix='/LnetD')
+
     if app.config.get('ENABLE_TESTING_ROUTES', False):
         from inventory_provider.routes import testing
         app.register_blueprint(testing.routes, url_prefix='/testing')

inventory_provider/routes/common.py

@@ -252,3 +252,17 @@ def load_json_docs(config_params, key_pattern, num_threads=10):
 def load_xml_docs(config_params, key_pattern, num_threads=10):
     yield from _load_redis_docs(
         config_params, key_pattern, num_threads, doc_type=_DECODE_TYPE_XML)
+
+
+def load_snmp_indexes(hostname=None):
+    result = dict()
+    key_pattern = f'snmp-interfaces:{hostname}*' \
+        if hostname else 'snmp-interfaces:*'
+
+    for doc in load_json_docs(
+            config_params=current_app.config['INVENTORY_PROVIDER_CONFIG'],
+            key_pattern=key_pattern):
+        router = doc['key'][len('snmp-interfaces:'):]
+        result[router] = {e['name']: e for e in doc['value']}
+
+    return result

inventory_provider/routes/lnetd.py

+import json
+import logging
+import re
+
+from flask import Blueprint, Response, current_app, request
+
+from inventory_provider import juniper
+from inventory_provider.routes import common
+from inventory_provider.routes.common import _ignore_cache_or_retrieve
+
+logger = logging.getLogger(__name__)
+routes = Blueprint('lnetd-support-routes', __name__)
+
+INTERFACE_LIST_SCHEMA = {
+    '$schema': 'http://json-schema.org/draft-07/schema#',
+
+    'definitions': {
+
+        'ipv4-interface-address': {
+            'type': 'string',
+            'pattern': r'^(\d+\.){3}\d+/\d+$'
+        },
+        'ipv6-interface-address': {
+            'type': 'string',
+            'pattern': r'^[a-f\d:]+/\d+$'
+        },
+
+        'interface': {
+            'type': 'object',
+            'properties': {
+                'hostname': {'type': 'string'},
+                'interface': {'type': 'string'},
+                'ifIndex': {'type': 'integer', 'minimum': 1},
+                'ipv4': {
+                    'type': 'array',
+                    'items': {'$ref': '#/definitions/ipv4-interface-address'}
+                },
+                'ipv6': {
+                    'type': 'array',
+                    'items': {'$ref': '#/definitions/ipv6-interface-address'}
+                },
+            },
+            'required': ['hostname', 'interface', 'ifIndex', 'ipv4', 'ipv6'],
+            'additionalProperties': False
+        },
+    },
+
+    'type': 'array',
+    'items': {'$ref': '#/definitions/interface'}
+}
+
+
+def _add_snmp_indexes(interfaces, hostname=None):
+    """
+    generator that adds snmp ifIndex to each interface, if available
+    (only interfaces with an snmp index are yielded)
+
+    :param interfaces: result of _load_interfaces
+    :param hostname: hostname or None for all
+    :return: generator that yields interfaces with 'ifIndex' added
+    """
+    snmp_indexes = common.load_snmp_indexes(hostname)
+    for ifc in interfaces:
+
+        hostname = ifc['hostname']
+        if hostname not in snmp_indexes:
+            continue
+
+        interface = ifc['interface']
+        if interface not in snmp_indexes[hostname]:
+            continue
+
+        ifc['ifIndex'] = snmp_indexes[hostname][interface]['index']
+        yield ifc
+
+
+def _load_router_interfaces(hostname):
+    """
+    loads basic interface data for production & lab routers
+
+    :param hostname:
+    :return:
+    """
+    def _load_docs(key_pattern):
+
+        m = re.match(r'^(.*netconf:).+', key_pattern)
+        assert m  # sanity
+        key_prefix_len = len(m.group(1))
+        assert key_prefix_len >= len('netconf:')  # sanity
+
+        for doc in common.load_xml_docs(
+                config_params=current_app.config['INVENTORY_PROVIDER_CONFIG'],
+                key_pattern=key_pattern,
+                num_threads=10):
+
+            router = doc['key'][key_prefix_len:]
+
+            for ifc in juniper.list_interfaces(doc['value']):
+                if not ifc['description']:
+                    continue
+                # seems we're only interested in interfaces with addresses?
+                if not ifc['ipv4'] and not ifc['ipv6']:
+                    continue
+
+                yield {
+                    'hostname': router,
+                    'interface': ifc['name'],
+                    'ipv4': ifc['ipv4'],
+                    'ipv6': ifc['ipv6']
+                }
+
+    base_key_pattern = f'netconf:{hostname}*' if hostname else 'netconf:*'
+    yield from _load_docs(base_key_pattern)
+    yield from _load_docs(f'lab:{base_key_pattern}')
+
+
+def _load_interfaces(hostname=None):
+    """
+    prepares the result of a call to /interfaces
+
+    :param hostname: hostname or None for all
+    :return: generator yielding interface elements
+    """
+    basic_interfaces = _load_router_interfaces(hostname)
+    return _add_snmp_indexes(basic_interfaces, hostname)
+
+
+@routes.route("/interfaces", methods=['GET', 'POST'])
+@routes.route('/interfaces/<hostname>', methods=['GET', 'POST'])
+@common.require_accepts_json
+def interfaces(hostname=None):
+    """
+    Handler for `/LnetD/interfaces` and
+    `/LnetD/interfaces/<hostname>`
+    which returns information for either all interfaces
+    or those on the requested hostname.
+
+    .. asjson::
+       inventory_provider.routes.lnetd.INTERFACE_LIST_SCHEMA
+
+    :param hostname: optional, if present should be a router hostname
+    :return:
+    """
+
+    cache_key = f'classifier-cache:lnetd-interfaces:{hostname}' \
+        if hostname else 'classifier-cache:lnetd-interfaces:all'
+
+    r = common.get_current_redis()
+
+    result = _ignore_cache_or_retrieve(request, cache_key, r)
+
+    if not result:
+        result = list(_load_interfaces(hostname))
+        if not result:
+            return Response(
+                response='no interfaces found',
+                status=404,
+                mimetype='text/html')
+
+        result = json.dumps(result)
+        # cache this data for the next call
+        r.set(cache_key, result.encode('utf-8'))
+
+    return Response(result, mimetype="application/json")

inventory_provider/routes/poller.py

@@ -123,20 +123,6 @@ def after_request(resp):
     return common.after_request(resp)
 
 
-def _load_snmp_indexes(hostname=None):
-    result = dict()
-    key_pattern = f'snmp-interfaces:{hostname}*' \
-        if hostname else 'snmp-interfaces:*'
-
-    for doc in common.load_json_docs(
-            config_params=current_app.config['INVENTORY_PROVIDER_CONFIG'],
-            key_pattern=key_pattern):
-        router = doc['key'][len('snmp-interfaces:'):]
-        result[router] = {e['name']: e for e in doc['value']}
-
-    return result
-
-
 def _load_interface_bundles(hostname=None):
     result = dict()
 
@@ -295,7 +281,7 @@ def _add_snmp_indexes(interfaces, hostname=None):
     :param hostname: hostname or None for all
     :return: generator with 'snmp-index' optionally added to each element
     """
-    snmp_indexes = _load_snmp_indexes(hostname)
+    snmp_indexes = common.load_snmp_indexes(hostname)
     for ifc in interfaces:
         router_snmp = snmp_indexes.get(ifc['router'], None)
         if router_snmp and ifc['name'] in router_snmp:

test/per_router/test_lnetd_routes.py

+import json
+import jsonschema
+import pytest
+from inventory_provider.routes.lnetd import INTERFACE_LIST_SCHEMA
+
+DEFAULT_REQUEST_HEADERS = {
+    'Accept': ['application/json']
+}
+
+
+def test_router_interfaces(router, client):
+    if router.startswith('qfx'):
+        pytest.skip('no interfaces expected for {router}, skipping')
+
+    rv = client.post(
+        f'/LnetD/interfaces/{router}',
+        headers=DEFAULT_REQUEST_HEADERS)
+
+    assert rv.status_code == 200
+    response = json.loads(rv.data.decode('utf-8'))
+    jsonschema.validate(response, INTERFACE_LIST_SCHEMA)
+    assert response  # at least shouldn't be empty
+    assert all(ifc['hostname'] == router for ifc in response)

test/test_general_lnetd_routes.py

+import json
+import jsonschema
+from inventory_provider.routes import lnetd
+
+DEFAULT_REQUEST_HEADERS = {
+    'Accept': ['application/json']
+}
+
+
+def test_get_all_interfaces(client):
+    rv = client.get(
+        '/LnetD/interfaces',
+        headers=DEFAULT_REQUEST_HEADERS)
+    assert rv.status_code == 200
+    assert rv.is_json
+    response_data = json.loads(rv.data.decode('utf-8'))
+    jsonschema.validate(response_data, lnetd.INTERFACE_LIST_SCHEMA)
+    response_routers = {ifc['hostname'] for ifc in response_data}
+    assert len(response_routers) > 1, \
+        'there should data from be lots of routers'