diff --git a/inventory_provider/routes/poller.py b/inventory_provider/routes/poller.py
index 34d3a81e26332f08561a42a9b37f4e2836bc9365..a0748eb2cc05b24d7fb7b6def65bce99cc1d0b49 100644
--- a/inventory_provider/routes/poller.py
+++ b/inventory_provider/routes/poller.py
@@ -54,6 +54,12 @@ These endpoints are intended for use by BRIAN.
.. autofunction:: inventory_provider.routes.poller.get_service_types
+/poller/error-report-interfaces</hostname>
+------------------------------------------
+
+.. autofunction:: inventory_provider.routes.poller.error_report_interfaces
+
+
support method: _get_dashboards
---------------------------------
@@ -209,6 +215,25 @@ INTERFACE_LIST_SCHEMA = {
'items': {'$ref': '#/definitions/interface'}
}
+ERROR_REPORT_INTERFACE_LIST_SCHEMA = {
+ '$schema': 'https://json-schema.org/draft-07/schema#',
+ 'definitions': {
+ 'interface': {
+ 'type': 'object',
+ 'properties': {
+ 'router': {'type': 'string'},
+ 'name': {'type': 'string'},
+ 'description': {'type': 'string'},
+ 'vendor': {'type': 'string', 'enum': ['juniper', 'nokia']}
+ },
+ 'required': ['router', 'name', 'description', 'vendor'],
+ 'additionalProperties': False
+ },
+ },
+ 'type': 'array',
+ 'items': {'$ref': '#/definitions/interface'}
+}
+
INTERFACE_SPEED_LIST_SCHEMA = {
'$schema': 'https://json-schema.org/draft-07/schema#',
@@ -883,6 +908,89 @@ def interfaces(hostname=None):
return Response(result, mimetype="application/json")
+def load_error_report_interfaces(
+ config, hostname=None, use_next_redis=False
+):
+ interfaces = _load_interfaces(config, hostname, use_next_redis=use_next_redis)
+
+ def filter_interface(interface: dict):
+ return all(
+ (
+ "phy" in interface["description"].lower(),
+ "spare" not in interface["description"].lower(),
+ "non-operational" not in interface["description"].lower(),
+ "reserved" not in interface["description"].lower(),
+ "test" not in interface["description"].lower(),
+ "dsc." not in interface["name"].lower(),
+ "fxp" not in interface["name"].lower(),
+ )
+ )
+
+ def transform_interface(interface: dict):
+ return {
+ "router": interface["router"],
+ "name": interface["name"],
+ "description": interface["description"],
+ # TODO: This is a complete hack until we have a proper way to determine
+ # router vendor
+ "vendor": "nokia" if interface["router"].startswith("rt0") else "juniper"
+ }
+
+ return sorted(
+ map(transform_interface, filter(filter_interface, interfaces)),
+ key=lambda i: (i["router"], i["name"]),
+ )
+
+
+@routes.route("/error-report-interfaces", methods=['GET'])
+@routes.route('/error-report-interfaces/<hostname>', methods=['GET'])
+@common.require_accepts_json
+def error_report_interfaces(hostname=None):
+ """
+ Handler for `/poller/error-report-interfaces` and
+ `/poller/error-report-interfaces/<hostname>`
+ which returns information for either all interfaces
+ or those on the requested hostname.
+
+ The optional `no-lab` parameter omits lab routers
+ if it's truthiness evaluates to True.
+
+ The response is a list of information for all
+ interfaces that should be included in the neteng error report
+ and includes vendor information (either juniper or nokia)
+
+ .. asjson::
+ inventory_provider.routes.poller.ERROR_REPORT_INTERFACE_LIST_SCHEMA
+
+ :param hostname: optional, if present should be a router hostname
+ :return:
+ """
+
+ suffix = hostname or "all"
+ cache_key = f'classifier-cache:error-report-interfaces:{suffix}'
+
+ r = common.get_current_redis()
+
+ result = _ignore_cache_or_retrieve(request, cache_key, r)
+
+ if not result:
+ interfaces = load_error_report_interfaces(
+ current_app.config['INVENTORY_PROVIDER_CONFIG'], hostname
+ )
+ result = json.dumps(interfaces).encode('utf-8')
+ # cache this data for the next call
+ r.set(cache_key, result)
+
+ if not result or result == b'[]':
+ return Response(
+ response='no interfaces found',
+ status=404,
+ mimetype='text/plain'
+ )
+
+ return Response(result, mimetype="application/json")
+
+
def interface_speed(ifc):
"""
Return the maximum bits per second expected for the given interface.
diff --git a/inventory_provider/tasks/worker.py b/inventory_provider/tasks/worker.py
index eafdb92cea8029a970f4205466e7d65dba5d0417..a360124dd6e344324ce5316caf77e21f278598ae 100644
--- a/inventory_provider/tasks/worker.py
+++ b/inventory_provider/tasks/worker.py
@@ -21,7 +21,7 @@ from ncclient.transport import TransportError
from inventory_provider.db import ims_data
from inventory_provider.db.ims import IMS
-from inventory_provider.routes.poller import load_interfaces_to_poll
+from inventory_provider.routes.poller import load_error_report_interfaces, load_interfaces_to_poll
from inventory_provider.tasks.app import app
from inventory_provider.tasks.common \
import get_next_redis, get_current_redis, \
@@ -1827,6 +1827,7 @@ def final_task(self):
_build_snmp_peering_db(update_callback=self.log_info)
_build_juniper_peering_db(update_callback=self.log_info)
populate_poller_interfaces_cache(warning_callback=self.log_warning)
+ populate_error_report_interfaces_cache(warning_callback=self.log_warning)
collate_netconf_interfaces_all_cache(warning_callback=self.log_warning)
latch_db(InventoryTask.config)
@@ -1879,6 +1880,50 @@ def populate_poller_interfaces_cache(warning_callback=lambda s: None):
r.set(all_cache_key, json.dumps(all_populated_interfaces))
+@log_task_entry_and_exit
+def populate_error_report_interfaces_cache(warning_callback=lambda s: None):
+ cache_ns = 'classifier-cache:error-report-interfaces:'
+ all_cache_key = cache_ns + 'all'
+ all_populated_interfaces = None
+
+ r = get_next_redis(InventoryTask.config)
+
+ try:
+ all_populated_interfaces = load_error_report_interfaces(
+ InventoryTask.config, use_next_redis=True
+ )
+
+ except Exception as e:
+ warning_callback(f"Failed to retrieve all required data {e}")
+ logger.exception(
+ "Failed to retrieve all required data, logging exception")
+
+ if not all_populated_interfaces:
+ previous_r = get_current_redis(InventoryTask.config)
+
+ try:
+ warning_callback(f"populating {all_cache_key} from previously cached data")
+ previous = json.loads(previous_r.get(all_cache_key))
+ all_populated_interfaces = sorted(
+ previous, key=lambda i: (i["router"], i["name"])
+ )
+ except Exception as e:
+ warning_callback(
+ f"Failed to load {all_cache_key} from previously cached data: {e}"
+ )
+ return
+
+ router_interfaces = {}
+ for ifc in all_populated_interfaces:
+ interfaces = router_interfaces.setdefault(ifc['router'], [])
+ interfaces.append(ifc)
+
+ for router, ifcs in router_interfaces.items():
+ r.set(cache_ns + router, json.dumps(ifcs))
+
+ r.set(all_cache_key, json.dumps(all_populated_interfaces))
+
+
@log_task_entry_and_exit
def collate_netconf_interfaces_all_cache(warning_callback=lambda s: None):
"""
diff --git a/test/conftest.py b/test/conftest.py
index 245f1f3b97d78dba94b814833eed2a6fc13c4b6d..c77fc91d9877f8d274bbbf1987d2a64a5430dbfc 100644
--- a/test/conftest.py
+++ b/test/conftest.py
@@ -258,7 +258,9 @@ def mocked_redis(mocker):
def client(flask_config_filename, data_config_filename, mocked_redis):
os.environ['FLASK_SETTINGS_FILENAME'] = flask_config_filename
os.environ['INVENTORY_PROVIDER_CONFIG_FILENAME'] = data_config_filename
- with inventory_provider.create_app(setup_logging=False).test_client() as c:
+ app = inventory_provider.create_app(setup_logging=False)
+ app.testing = True # Show exceptions instead of generid 500 status
+ with app.test_client() as c:
yield c
diff --git a/test/test_general_poller_routes.py b/test/test_general_poller_routes.py
index a42700acd659488fea874a41680058e951f13d69..611bd3df301f56bdd57159b143cb183071b74282 100644
--- a/test/test_general_poller_routes.py
+++ b/test/test_general_poller_routes.py
@@ -466,3 +466,25 @@ def test_gws_config_html(client):
# just a sanity check, no validation
# ... for now, this isn't an important interface
assert response_data.endswith('</html>')
+
+
+def test_get_all_error_report_interfaces(client):
+ rv = client.get('/poller/error-report-interfaces', headers=DEFAULT_REQUEST_HEADERS)
+ assert rv.status_code == 200
+ assert rv.is_json
+ response_data = json.loads(rv.data)
+ jsonschema.validate(response_data, poller.ERROR_REPORT_INTERFACE_LIST_SCHEMA)
+ response_routers = {ifc['router'] for ifc in response_data}
+ assert len(response_routers) > 1, 'there should data from be lots of routers'
+
+
+def test_get_single_router_error_report_interfaces(client):
+ rv = client.get(
+ '/poller/error-report-interfaces/mx1.ams.nl.geant.net',
+ headers=DEFAULT_REQUEST_HEADERS,
+ )
+ assert rv.status_code == 200
+ assert rv.is_json
+ response_data = json.loads(rv.data)
+ jsonschema.validate(response_data, poller.ERROR_REPORT_INTERFACE_LIST_SCHEMA)
+ assert {ifc['router'] for ifc in response_data} == {"mx1.ams.nl.geant.net"}
diff --git a/test/test_worker.py b/test/test_worker.py
index 0120d47f9de55b0806afac65d659290ea909538d..066393e45c61e4c0025c830d0a8557148c468bab 100644
--- a/test/test_worker.py
+++ b/test/test_worker.py
@@ -5,7 +5,7 @@ import jsonschema
from lxml import etree
from inventory_provider.tasks import common
-from inventory_provider.tasks.worker import transform_ims_data, \
+from inventory_provider.tasks.worker import populate_error_report_interfaces_cache, transform_ims_data, \
extract_ims_data, persist_ims_data, \
retrieve_and_persist_neteng_managed_device_list, \
populate_poller_interfaces_cache, refresh_nokia_interface_list
@@ -941,3 +941,104 @@ def test_refresh_nokia_interface_list(mocked_redis, data_config):
interface = json.loads(r.get(k).decode('utf-8'))
interfaces[k] = interface
jsonschema.validate(interface, interfaces_schema)
+
+
+def test_populate_error_report_interfaces_cache(mocker, data_config, mocked_redis):
+ r = common._get_redis(data_config)
+
+ mocker.patch('inventory_provider.tasks.worker.get_next_redis', return_value=r)
+ all_interfaces = [
+ {
+ "router": "router_a.geant.net",
+ "name": "interface_a",
+ "bundle": ["ae_a"],
+ "bundle-parents": [],
+ "description": "PHY DESCRIPTION A",
+ "circuits": []
+ },
+ {
+ "router": "router_a.geant.net",
+ "name": "ae_a",
+ "bundle": [],
+ "bundle-parents": [],
+ "description": "PHY DESCRIPTION B",
+ "circuits": []
+ },
+ {
+ "router": "router_a.geant.net",
+ "name": "ae_a.123",
+ "bundle": [],
+ "bundle-parents": [],
+ "description": "PHY DESCRIPTION C",
+ "circuits": []
+ },
+ {
+ "router": "rt0.geant.net",
+ "name": "lag-1.0",
+ "bundle": ["ae_c"],
+ "bundle-parents": [],
+ "description": "PHY DESCRIPTION D",
+ "circuits": []
+ },
+ {
+ "router": "router_a.geant.net",
+ "name": "ae_a.456",
+ "bundle": [],
+ "bundle-parents": [],
+ "description": "Spare Not included",
+ "circuits": []
+ },
+ {
+ "router": "router_a.geant.net",
+ "name": "dsc.1",
+ "bundle": [],
+ "bundle-parents": [],
+ "description": "PHY discard",
+ "circuits": []
+ },
+ ]
+
+ mocker.patch('inventory_provider.routes.poller._load_interfaces',
+ return_value=all_interfaces)
+ mocker.patch(
+ 'inventory_provider.tasks.worker.InventoryTask.config'
+ )
+ exp_router_a_interfaces = [
+ {
+ "router": "router_a.geant.net",
+ "name": "ae_a",
+ "description": "PHY DESCRIPTION B",
+ "vendor": "juniper"
+ },
+ {
+ "router": "router_a.geant.net",
+ "name": "ae_a.123",
+ "description": "PHY DESCRIPTION C",
+ "vendor": "juniper"
+ },
+ {
+ "router": "router_a.geant.net",
+ "name": "interface_a",
+ "description": "PHY DESCRIPTION A",
+ "vendor": "juniper"
+ },
+ ]
+ exp_nokia_router_interfaces = [
+ {
+ "router": "rt0.geant.net",
+ "name": "lag-1.0",
+ "description": "PHY DESCRIPTION D",
+ "vendor": "nokia"
+ }
+ ]
+
+ populate_error_report_interfaces_cache()
+
+ all = r.get("classifier-cache:error-report-interfaces:all").decode("utf-8")
+ assert json.loads(all) == exp_router_a_interfaces + exp_nokia_router_interfaces
+
+ router_a = r.get("classifier-cache:error-report-interfaces:router_a.geant.net")
+ assert json.loads(router_a) == exp_router_a_interfaces
+
+ nokia_router = r.get("classifier-cache:error-report-interfaces:rt0.geant.net")
+ assert json.loads(nokia_router) == exp_nokia_router_interfaces