diff --git a/Changelog.md b/Changelog.md
index 944ddac918962f57821d2e20eaf6356a4e866cd4..178d74d11ea6ad368c68fc798669e7cb1c98ae8b 100644
--- a/Changelog.md
+++ b/Changelog.md
@@ -2,7 +2,15 @@
All notable changes to this project will be documented in this file.
-## [0.62] - 2021-04-01
+## [0.65] - 2021-05-28
+- DBOARD3-438: Changed status of non-monitored services
+- DBOARD3-439: initial LnetD endpoint
+
+
+## [0.64] - 2021-05-19
+- DBOARD3-303: Replace dependency on OpsDB with IMS
+
+## [0.63] - 2021-04-01
- POL1-370: load & return lab interfaces for poller
## [0.62] - 2021-03-24
diff --git a/docs/source/protocol/index.rst b/docs/source/protocol/index.rst
index e0ef1976524f092d255755ee35ba77f4ca5edbaf..0ec3dbb58ae1dd95b59fc4093a1ae47b4e794309 100644
--- a/docs/source/protocol/index.rst
+++ b/docs/source/protocol/index.rst
@@ -33,4 +33,5 @@ API modules
lg
data
jobs
- msr
\ No newline at end of file
+ msr
+ lnetd
\ No newline at end of file
diff --git a/docs/source/protocol/lnetd.rst b/docs/source/protocol/lnetd.rst
new file mode 100644
index 0000000000000000000000000000000000000000..6502451c057217084217ed77821b64e3945b8ea9
--- /dev/null
+++ b/docs/source/protocol/lnetd.rst
@@ -0,0 +1,14 @@
+.. LnetD endpoint docs
+
+
+LnetD support
+=========================
+
+This endpoint is intended for use with LnetD
+
+.. contents:: :local:
+
+/LnetD/interfaces</hostname>
+---------------------------------
+
+.. autofunction:: inventory_provider.routes.lnetd.interfaces
diff --git a/inventory_provider/__init__.py b/inventory_provider/__init__.py
index 5767a1da9c37b94dbc8446877ce9d4772e6c4d3b..aa92cb410dffe41600160d1d94bd1b98e3f9a75d 100644
--- a/inventory_provider/__init__.py
+++ b/inventory_provider/__init__.py
@@ -78,6 +78,9 @@ def create_app():
from inventory_provider.routes import msr
app.register_blueprint(msr.routes, url_prefix='/msr')
+ from inventory_provider.routes import lnetd
+ app.register_blueprint(lnetd.routes, url_prefix='/LnetD')
+
if app.config.get('ENABLE_TESTING_ROUTES', False):
from inventory_provider.routes import testing
app.register_blueprint(testing.routes, url_prefix='/testing')
diff --git a/inventory_provider/routes/classifier.py b/inventory_provider/routes/classifier.py
index 9d72b5b89a79e1ec2e54e0fb247294a09a5c7ad5..f470dd3393c58b94fccec10a0597b388b5b16757 100644
--- a/inventory_provider/routes/classifier.py
+++ b/inventory_provider/routes/classifier.py
@@ -143,15 +143,13 @@ def get_interface_services_and_loc(ims_source_equipment, ims_interface, redis):
for s in json.loads(raw_services.decode('utf-8')):
related_services.update(
{r['id']: r for r in s['related-services']})
- if s['monitored'] and s['circuit_type'] == 'service':
+ if s['circuit_type'] == 'service':
contacts.update(set(s.pop('contacts', set())))
_format_service(s)
result['services'].append(s)
result['related-services'] = list(related_services.values())
result['contacts'] = sorted(list(contacts))
- # non-monitored related services are not added by the worker so don't
- # need filtering out here
if not result['services']:
result.pop('services', None)
if result['related-services']:
diff --git a/inventory_provider/routes/common.py b/inventory_provider/routes/common.py
index af20bf9a4b614897a362e6e26b90ad2d24909795..56ed526edd5f5fa87a6b0c37947f73bf36fb8f73 100644
--- a/inventory_provider/routes/common.py
+++ b/inventory_provider/routes/common.py
@@ -252,3 +252,17 @@ def load_json_docs(config_params, key_pattern, num_threads=10):
def load_xml_docs(config_params, key_pattern, num_threads=10):
yield from _load_redis_docs(
config_params, key_pattern, num_threads, doc_type=_DECODE_TYPE_XML)
+
+
+def load_snmp_indexes(hostname=None):
+ result = dict()
+ key_pattern = f'snmp-interfaces:{hostname}*' \
+ if hostname else 'snmp-interfaces:*'
+
+ for doc in load_json_docs(
+ config_params=current_app.config['INVENTORY_PROVIDER_CONFIG'],
+ key_pattern=key_pattern):
+ router = doc['key'][len('snmp-interfaces:'):]
+ result[router] = {e['name']: e for e in doc['value']}
+
+ return result
diff --git a/inventory_provider/routes/lnetd.py b/inventory_provider/routes/lnetd.py
new file mode 100644
index 0000000000000000000000000000000000000000..0025ef218b40e08d322e72c62a947c798435bbe1
--- /dev/null
+++ b/inventory_provider/routes/lnetd.py
@@ -0,0 +1,164 @@
+import json
+import logging
+import re
+
+from flask import Blueprint, Response, current_app, request
+
+from inventory_provider import juniper
+from inventory_provider.routes import common
+from inventory_provider.routes.common import _ignore_cache_or_retrieve
+
+logger = logging.getLogger(__name__)
+routes = Blueprint('lnetd-support-routes', __name__)
+
+INTERFACE_LIST_SCHEMA = {
+ '$schema': 'http://json-schema.org/draft-07/schema#',
+
+ 'definitions': {
+
+ 'ipv4-interface-address': {
+ 'type': 'string',
+ 'pattern': r'^(\d+\.){3}\d+/\d+$'
+ },
+ 'ipv6-interface-address': {
+ 'type': 'string',
+ 'pattern': r'^[a-f\d:]+/\d+$'
+ },
+
+ 'interface': {
+ 'type': 'object',
+ 'properties': {
+ 'hostname': {'type': 'string'},
+ 'interface': {'type': 'string'},
+ 'ifIndex': {'type': 'integer', 'minimum': 1},
+ 'ipv4': {
+ 'type': 'array',
+ 'items': {'$ref': '#/definitions/ipv4-interface-address'}
+ },
+ 'ipv6': {
+ 'type': 'array',
+ 'items': {'$ref': '#/definitions/ipv6-interface-address'}
+ },
+ },
+ 'required': ['hostname', 'interface', 'ifIndex', 'ipv4', 'ipv6'],
+ 'additionalProperties': False
+ },
+ },
+
+ 'type': 'array',
+ 'items': {'$ref': '#/definitions/interface'}
+}
+
+
+def _add_snmp_indexes(interfaces, hostname=None):
+ """
+ generator that adds snmp ifIndex to each interface, if available
+ (only interfaces with an snmp index are yielded)
+
+ :param interfaces: result of _load_interfaces
+ :param hostname: hostname or None for all
+ :return: generator that yields interfaces with 'ifIndex' added
+ """
+ snmp_indexes = common.load_snmp_indexes(hostname)
+ for ifc in interfaces:
+
+ hostname = ifc['hostname']
+ if hostname not in snmp_indexes:
+ continue
+
+ interface = ifc['interface']
+ if interface not in snmp_indexes[hostname]:
+ continue
+
+ ifc['ifIndex'] = snmp_indexes[hostname][interface]['index']
+ yield ifc
+
+
+def _load_router_interfaces(hostname):
+ """
+ loads basic interface data for production & lab routers
+
+ :param hostname:
+ :return:
+ """
+ def _load_docs(key_pattern):
+
+ m = re.match(r'^(.*netconf:).+', key_pattern)
+ assert m # sanity
+ key_prefix_len = len(m.group(1))
+ assert key_prefix_len >= len('netconf:') # sanity
+
+ for doc in common.load_xml_docs(
+ config_params=current_app.config['INVENTORY_PROVIDER_CONFIG'],
+ key_pattern=key_pattern,
+ num_threads=10):
+
+ router = doc['key'][key_prefix_len:]
+
+ for ifc in juniper.list_interfaces(doc['value']):
+ if not ifc['description']:
+ continue
+ # seems we're only interested in interfaces with addresses?
+ if not ifc['ipv4'] and not ifc['ipv6']:
+ continue
+
+ yield {
+ 'hostname': router,
+ 'interface': ifc['name'],
+ 'ipv4': ifc['ipv4'],
+ 'ipv6': ifc['ipv6']
+ }
+
+ base_key_pattern = f'netconf:{hostname}*' if hostname else 'netconf:*'
+ yield from _load_docs(base_key_pattern)
+ yield from _load_docs(f'lab:{base_key_pattern}')
+
+
+def _load_interfaces(hostname=None):
+ """
+ prepares the result of a call to /interfaces
+
+ :param hostname: hostname or None for all
+ :return: generator yielding interface elements
+ """
+ basic_interfaces = _load_router_interfaces(hostname)
+ return _add_snmp_indexes(basic_interfaces, hostname)
+
+
+@routes.route("/interfaces", methods=['GET', 'POST'])
+@routes.route('/interfaces/<hostname>', methods=['GET', 'POST'])
+@common.require_accepts_json
+def interfaces(hostname=None):
+ """
+ Handler for `/LnetD/interfaces` and
+ `/LnetD/interfaces/<hostname>`
+ which returns information for either all interfaces
+ or those on the requested hostname.
+
+ .. asjson::
+ inventory_provider.routes.lnetd.INTERFACE_LIST_SCHEMA
+
+ :param hostname: optional, if present should be a router hostname
+ :return:
+ """
+
+ cache_key = f'classifier-cache:lnetd-interfaces:{hostname}' \
+ if hostname else 'classifier-cache:lnetd-interfaces:all'
+
+ r = common.get_current_redis()
+
+ result = _ignore_cache_or_retrieve(request, cache_key, r)
+
+ if not result:
+ result = list(_load_interfaces(hostname))
+ if not result:
+ return Response(
+ response='no interfaces found',
+ status=404,
+ mimetype='text/html')
+
+ result = json.dumps(result)
+ # cache this data for the next call
+ r.set(cache_key, result.encode('utf-8'))
+
+ return Response(result, mimetype="application/json")
diff --git a/inventory_provider/routes/poller.py b/inventory_provider/routes/poller.py
index 73681162903e04ee84751946535c6a05e1c1526d..900fd11c08c439bab616242296fe4360b2c6f660 100644
--- a/inventory_provider/routes/poller.py
+++ b/inventory_provider/routes/poller.py
@@ -123,20 +123,6 @@ def after_request(resp):
return common.after_request(resp)
-def _load_snmp_indexes(hostname=None):
- result = dict()
- key_pattern = f'snmp-interfaces:{hostname}*' \
- if hostname else 'snmp-interfaces:*'
-
- for doc in common.load_json_docs(
- config_params=current_app.config['INVENTORY_PROVIDER_CONFIG'],
- key_pattern=key_pattern):
- router = doc['key'][len('snmp-interfaces:'):]
- result[router] = {e['name']: e for e in doc['value']}
-
- return result
-
-
def _load_interface_bundles(hostname=None):
result = dict()
@@ -295,7 +281,7 @@ def _add_snmp_indexes(interfaces, hostname=None):
:param hostname: hostname or None for all
:return: generator with 'snmp-index' optionally added to each element
"""
- snmp_indexes = _load_snmp_indexes(hostname)
+ snmp_indexes = common.load_snmp_indexes(hostname)
for ifc in interfaces:
router_snmp = snmp_indexes.get(ifc['router'], None)
if router_snmp and ifc['name'] in router_snmp:
diff --git a/inventory_provider/tasks/worker.py b/inventory_provider/tasks/worker.py
index 82644b92c8cdcd2ab1c9ebf7fe9c3b2eaa461a9e..a36a5faf2214f572c9f0eca9170fea3782f51e84 100644
--- a/inventory_provider/tasks/worker.py
+++ b/inventory_provider/tasks/worker.py
@@ -571,8 +571,6 @@ def update_circuit_hierarchy_and_port_id_services(self, use_current=False):
tls_names = list(ims_data.get_service_types(ds1))
customer_contacts = \
{k: v for k, v in ims_data.get_customer_service_emails(ds1)}
- circuit_ids_not_to_monitor = \
- list(ims_data.get_non_monitored_circuit_ids(ds1))
circuit_ids_to_monitor = \
list(ims_data.get_monitored_circuit_ids(ds1))
@@ -618,7 +616,6 @@ def update_circuit_hierarchy_and_port_id_services(self, use_current=False):
nonlocal hierarchy
hierarchy = {}
for d in ims_data.get_circuit_hierarchy(ds1):
- d['monitored'] = d['id'] in circuit_ids_to_monitor
d['contacts'] = customer_contacts.get(d['customerid'], [])
hierarchy[d['id']] = d
logger.debug("hierarchy complete")
@@ -667,25 +664,20 @@ def update_circuit_hierarchy_and_port_id_services(self, use_current=False):
c = hierarchy.get(circuit_id, None)
if c:
- def _is_rs(candidate):
- if candidate['id'] in circuit_ids_not_to_monitor:
- return False
- if candidate['product'] in tls_names:
- return True
- # if candidate['speed'] == 'BGP':
- # return True
- return False
-
- if _is_rs(c):
+ if c['circuit-type'] == 'service':
rs[c['id']] = {
'id': c['id'],
'name': c['name'],
- 'status': c['status'],
- 'circuit_type': 'service',
+ 'circuit_type': c['circuit-type'],
'service_type': c['product'],
'project': c['project'],
'contacts': sorted(list(c['contacts']))
}
+ if c['id'] in circuit_ids_to_monitor:
+ rs[c['id']]['status'] = c['status']
+ else:
+ rs[c['id']]['status'] = 'non-monitored'
+
if c['sub-circuits']:
for sub in c['sub-circuits']:
temp_parents = \
@@ -694,10 +686,10 @@ def update_circuit_hierarchy_and_port_id_services(self, use_current=False):
return list(rs.values())
def _format_service(s):
- if s['id'] in circuit_ids_not_to_monitor:
- s['monitored'] = False
- else:
- s['monitored'] = True
+
+ if s['circuit_type'] == 'service' \
+ and s['id'] not in circuit_ids_to_monitor:
+ s['status'] = 'non-monitored'
pd_a = port_id_details[s['port_a_id']][0]
location_a = locations.get(pd_a['equipment_name'], None)
if location_a:
@@ -758,10 +750,6 @@ def update_circuit_hierarchy_and_port_id_services(self, use_current=False):
'name': hierarchy[x]['name'],
'status': hierarchy[x]['status']
}
- if c['id'] in circuit_ids_not_to_monitor:
- c['monitored'] = False
- else:
- c['monitored'] = True
circ['fibre-routes'].append(c)
circ['related-services'] = \
diff --git a/setup.py b/setup.py
index 5ce7b85e79ccc418c5fb07e9e92e810a62b811fc..758d2bcd73c16d0e2032cdc1989c74adbfc3d1de 100644
--- a/setup.py
+++ b/setup.py
@@ -2,7 +2,7 @@ from setuptools import setup, find_packages
setup(
name='inventory-provider',
- version="0.64",
+ version="0.65",
author='GEANT',
author_email='swd@geant.org',
description='Dashboard inventory provider',
diff --git a/test/per_router/test_lnetd_routes.py b/test/per_router/test_lnetd_routes.py
new file mode 100644
index 0000000000000000000000000000000000000000..d4017fa36fc09107af505aa37bc81ed563e3df22
--- /dev/null
+++ b/test/per_router/test_lnetd_routes.py
@@ -0,0 +1,23 @@
+import json
+import jsonschema
+import pytest
+from inventory_provider.routes.lnetd import INTERFACE_LIST_SCHEMA
+
+DEFAULT_REQUEST_HEADERS = {
+ 'Accept': ['application/json']
+}
+
+
+def test_router_interfaces(router, client):
+ if router.startswith('qfx'):
+ pytest.skip('no interfaces expected for {router}, skipping')
+
+ rv = client.post(
+ f'/LnetD/interfaces/{router}',
+ headers=DEFAULT_REQUEST_HEADERS)
+
+ assert rv.status_code == 200
+ response = json.loads(rv.data.decode('utf-8'))
+ jsonschema.validate(response, INTERFACE_LIST_SCHEMA)
+ assert response # at least shouldn't be empty
+ assert all(ifc['hostname'] == router for ifc in response)
diff --git a/test/test_general_lnetd_routes.py b/test/test_general_lnetd_routes.py
new file mode 100644
index 0000000000000000000000000000000000000000..4cec6a5643046d37374b3ee6f1471ebe78e1233c
--- /dev/null
+++ b/test/test_general_lnetd_routes.py
@@ -0,0 +1,20 @@
+import json
+import jsonschema
+from inventory_provider.routes import lnetd
+
+DEFAULT_REQUEST_HEADERS = {
+ 'Accept': ['application/json']
+}
+
+
+def test_get_all_interfaces(client):
+ rv = client.get(
+ '/LnetD/interfaces',
+ headers=DEFAULT_REQUEST_HEADERS)
+ assert rv.status_code == 200
+ assert rv.is_json
+ response_data = json.loads(rv.data.decode('utf-8'))
+ jsonschema.validate(response_data, lnetd.INTERFACE_LIST_SCHEMA)
+ response_routers = {ifc['hostname'] for ifc in response_data}
+ assert len(response_routers) > 1, \
+ 'there should data from be lots of routers'