diff --git a/Changelog.md b/Changelog.md
index 85f13623c2e8e4d948132baad83021a952152c22..ef5ee4915b713335fe0a96b5976c267cee77bd25 100644
--- a/Changelog.md
+++ b/Changelog.md
@@ -2,6 +2,12 @@
All notable changes to this project will be documented in this file.
+## [0.117] - 2024-04-16
+- adding monitored id check for third party
+
+## [0.116] - 2024-04-15
+- adding noc and planned noc details for third party circuit
+
## [0.115] - 2024-04-12
- adding related service for circuit hierarchy
diff --git a/inventory_provider/routes/poller.py b/inventory_provider/routes/poller.py
index 34d3a81e26332f08561a42a9b37f4e2836bc9365..4fc27222f6630cc471eafa8c72b45a01db765ecd 100644
--- a/inventory_provider/routes/poller.py
+++ b/inventory_provider/routes/poller.py
@@ -54,6 +54,12 @@ These endpoints are intended for use by BRIAN.
.. autofunction:: inventory_provider.routes.poller.get_service_types
+/poller/error-report-interfaces</hostname>
+------------------------------------------
+
+.. autofunction:: inventory_provider.routes.poller.error_report_interfaces
+
+
support method: _get_dashboards
---------------------------------
@@ -209,6 +215,25 @@ INTERFACE_LIST_SCHEMA = {
'items': {'$ref': '#/definitions/interface'}
}
+ERROR_REPORT_INTERFACE_LIST_SCHEMA = {
+ '$schema': 'https://json-schema.org/draft-07/schema#',
+ 'definitions': {
+ 'interface': {
+ 'type': 'object',
+ 'properties': {
+ 'router': {'type': 'string'},
+ 'name': {'type': 'string'},
+ 'description': {'type': 'string'},
+ 'vendor': {'type': 'string', 'enum': ['juniper', 'nokia']}
+ },
+ 'required': ['router', 'name', 'description', 'vendor'],
+ 'additionalProperties': False
+ },
+ },
+ 'type': 'array',
+ 'items': {'$ref': '#/definitions/interface'}
+}
+
INTERFACE_SPEED_LIST_SCHEMA = {
'$schema': 'https://json-schema.org/draft-07/schema#',
@@ -479,10 +504,10 @@ def _get_dashboards(interface):
if re.match(r'(SRV_GLOBAL|LAG|PHY) INFRASTRUCTURE BACKBONE', description):
yield BRIAN_DASHBOARDS.INFRASTRUCTURE_BACKBONE
if router == 'mx1.lon.uk.geant.net' \
- and re.match(r'^ae12(\.\d+|$)$', ifc_name):
+ and re.match(r'^ae12\.\d+$', ifc_name):
yield BRIAN_DASHBOARDS.CAE1
if router == 'rt1.mar.fr.geant.net' \
- and re.match(r'^ae12(\.\d+|$)$', ifc_name):
+ and re.match(r'^ae12\.\d+$', ifc_name):
yield BRIAN_DASHBOARDS.IC1
if re.match(r'PHY UPSTREAM\s', description):
yield BRIAN_DASHBOARDS.GWS_PHY_UPSTREAM
@@ -883,6 +908,89 @@ def interfaces(hostname=None):
return Response(result, mimetype="application/json")
+def load_error_report_interfaces(
+ config, hostname=None, use_next_redis=False
+):
+ interfaces = _load_interfaces(config, hostname, use_next_redis=use_next_redis)
+
+ def filter_interface(interface: dict):
+ return all(
+ (
+ "phy" in interface["description"].lower(),
+ "spare" not in interface["description"].lower(),
+ "non-operational" not in interface["description"].lower(),
+ "reserved" not in interface["description"].lower(),
+ "test" not in interface["description"].lower(),
+ "dsc." not in interface["name"].lower(),
+ "fxp" not in interface["name"].lower(),
+ )
+ )
+
+ def transform_interface(interface: dict):
+ return {
+ "router": interface["router"],
+ "name": interface["name"],
+ "description": interface["description"],
+ # TODO: This is a complete hack until we have a proper way to determine
+ # router vendor
+ "vendor": "nokia" if interface["router"].startswith("rt0") else "juniper"
+ }
+
+ return sorted(
+ map(transform_interface, filter(filter_interface, interfaces)),
+ key=lambda i: (i["router"], i["name"]),
+ )
+
+
+@routes.route("/error-report-interfaces", methods=['GET'])
+@routes.route('/error-report-interfaces/<hostname>', methods=['GET'])
+@common.require_accepts_json
+def error_report_interfaces(hostname=None):
+ """
+ Handler for `/poller/error-report-interfaces` and
+ `/poller/error-report-interfaces/<hostname>`
+ which returns information for either all interfaces
+ or those on the requested hostname.
+
+ The optional `no-lab` parameter omits lab routers
+ if it's truthiness evaluates to True.
+
+ The response is a list of information for all
+ interfaces that should be included in the neteng error report
+ and includes vendor information (either juniper or nokia)
+
+ .. asjson::
+ inventory_provider.routes.poller.ERROR_REPORT_INTERFACE_LIST_SCHEMA
+
+ :param hostname: optional, if present should be a router hostname
+ :return:
+ """
+
+ suffix = hostname or "all"
+ cache_key = f'classifier-cache:error-report-interfaces:{suffix}'
+
+ r = common.get_current_redis()
+
+ result = _ignore_cache_or_retrieve(request, cache_key, r)
+
+ if not result:
+ interfaces = load_error_report_interfaces(
+ current_app.config['INVENTORY_PROVIDER_CONFIG'], hostname
+ )
+ result = json.dumps(interfaces).encode('utf-8')
+ # cache this data for the next call
+ r.set(cache_key, result)
+
+ if not result or result == b'[]':
+ return Response(
+ response='no interfaces found',
+ status=404,
+ mimetype='text/plain'
+ )
+
+ return Response(result, mimetype="application/json")
+
+
def interface_speed(ifc):
"""
Return the maximum bits per second expected for the given interface.
diff --git a/inventory_provider/tasks/worker.py b/inventory_provider/tasks/worker.py
index eafdb92cea8029a970f4205466e7d65dba5d0417..627985b15c32f4ee9c333e301943abf3ceb5bddf 100644
--- a/inventory_provider/tasks/worker.py
+++ b/inventory_provider/tasks/worker.py
@@ -21,7 +21,7 @@ from ncclient.transport import TransportError
from inventory_provider.db import ims_data
from inventory_provider.db.ims import IMS
-from inventory_provider.routes.poller import load_interfaces_to_poll
+from inventory_provider.routes.poller import load_error_report_interfaces, load_interfaces_to_poll
from inventory_provider.tasks.app import app
from inventory_provider.tasks.common \
import get_next_redis, get_current_redis, \
@@ -1493,6 +1493,7 @@ def transform_ims_data(data):
'locations': data['locations'],
'site_locations': data['site_locations'],
'lg_routers': data['lg_routers'],
+ 'circuit_ids_to_monitor': data['circuit_ids_to_monitor'],
}
@@ -1506,6 +1507,7 @@ def persist_ims_data(data, use_current=False):
node_pair_services = data['node_pair_services']
sid_services = data['sid_services']
pop_nodes = data['pop_nodes']
+ circuit_ids_to_monitor = data['circuit_ids_to_monitor']
def _get_sites():
# de-dupe the sites (by abbreviation)
@@ -1578,7 +1580,7 @@ def persist_ims_data(data, use_current=False):
populate_poller_cache(interface_services, r)
populate_mic_cache(interface_services, r)
- populate_mic_with_third_party_data(interface_services, hierarchy, r)
+ populate_mic_with_third_party_data(interface_services, hierarchy, circuit_ids_to_monitor, r)
for service_type, services in services_by_type.items():
for v in services.values():
@@ -1613,7 +1615,7 @@ def persist_ims_data(data, use_current=False):
rp.execute()
-def populate_mic_with_third_party_data(interface_services, hierarchy, r):
+def populate_mic_with_third_party_data(interface_services, hierarchy, circuit_ids_to_monitor, r):
cache_key = "mic:impact:third-party-data"
third_party_data = defaultdict(lambda: defaultdict(dict))
third_party_interface_data = defaultdict(lambda: defaultdict(dict))
@@ -1629,7 +1631,8 @@ def populate_mic_with_third_party_data(interface_services, hierarchy, r):
return s
def get_formatted_third_party_rs(_circuit_data):
- if _circuit_data and _circuit_data['status'] == 'operational' and _circuit_data['circuit-type'] == 'service':
+ if _circuit_data and _circuit_data['status'] == 'operational' and \
+ _circuit_data['id'] in circuit_ids_to_monitor and _circuit_data['circuit-type'] == 'service':
return {
'id': _circuit_data['id'],
'name': _circuit_data['name'] + ' (' + _circuit_data['sid'] + ')',
@@ -1660,7 +1663,7 @@ def populate_mic_with_third_party_data(interface_services, hierarchy, r):
related_services_info['related_services'].append({
'id': formatted_rs['id'],
'name': name,
- 'service_type': formatted_rs['service_type']
+ 'service_type': formatted_rs['service_type'],
})
for contact in formatted_rs['contacts']:
related_services_info['contacts'].add(contact)
@@ -1827,6 +1830,7 @@ def final_task(self):
_build_snmp_peering_db(update_callback=self.log_info)
_build_juniper_peering_db(update_callback=self.log_info)
populate_poller_interfaces_cache(warning_callback=self.log_warning)
+ populate_error_report_interfaces_cache(warning_callback=self.log_warning)
collate_netconf_interfaces_all_cache(warning_callback=self.log_warning)
latch_db(InventoryTask.config)
@@ -1879,6 +1883,50 @@ def populate_poller_interfaces_cache(warning_callback=lambda s: None):
r.set(all_cache_key, json.dumps(all_populated_interfaces))
+@log_task_entry_and_exit
+def populate_error_report_interfaces_cache(warning_callback=lambda s: None):
+ cache_ns = 'classifier-cache:error-report-interfaces:'
+ all_cache_key = cache_ns + 'all'
+ all_populated_interfaces = None
+
+ r = get_next_redis(InventoryTask.config)
+
+ try:
+ all_populated_interfaces = load_error_report_interfaces(
+ InventoryTask.config, use_next_redis=True
+ )
+
+ except Exception as e:
+ warning_callback(f"Failed to retrieve all required data {e}")
+ logger.exception(
+ "Failed to retrieve all required data, logging exception")
+
+ if not all_populated_interfaces:
+ previous_r = get_current_redis(InventoryTask.config)
+
+ try:
+ warning_callback(f"populating {all_cache_key} from previously cached data")
+ previous = json.loads(previous_r.get(all_cache_key))
+ all_populated_interfaces = sorted(
+ previous, key=lambda i: (i["router"], i["name"])
+ )
+ except Exception as e:
+ warning_callback(
+ f"Failed to load {all_cache_key} from previously cached data: {e}"
+ )
+ return
+
+ router_interfaces = {}
+ for ifc in all_populated_interfaces:
+ interfaces = router_interfaces.setdefault(ifc['router'], [])
+ interfaces.append(ifc)
+
+ for router, ifcs in router_interfaces.items():
+ r.set(cache_ns + router, json.dumps(ifcs))
+
+ r.set(all_cache_key, json.dumps(all_populated_interfaces))
+
+
@log_task_entry_and_exit
def collate_netconf_interfaces_all_cache(warning_callback=lambda s: None):
"""
diff --git a/setup.py b/setup.py
index a668b097593320381568bb87d77cf0f2af973a9c..e70ce07badf0b4bc5b6995b7e0029aaea86bbb47 100644
--- a/setup.py
+++ b/setup.py
@@ -2,7 +2,7 @@ from setuptools import setup, find_packages
setup(
name='inventory-provider',
- version="0.116",
+ version="0.117",
author='GEANT',
author_email='swd@geant.org',
description='Dashboard inventory provider',
diff --git a/test/conftest.py b/test/conftest.py
index 245f1f3b97d78dba94b814833eed2a6fc13c4b6d..c77fc91d9877f8d274bbbf1987d2a64a5430dbfc 100644
--- a/test/conftest.py
+++ b/test/conftest.py
@@ -258,7 +258,9 @@ def mocked_redis(mocker):
def client(flask_config_filename, data_config_filename, mocked_redis):
os.environ['FLASK_SETTINGS_FILENAME'] = flask_config_filename
os.environ['INVENTORY_PROVIDER_CONFIG_FILENAME'] = data_config_filename
- with inventory_provider.create_app(setup_logging=False).test_client() as c:
+ app = inventory_provider.create_app(setup_logging=False)
+ app.testing = True # Show exceptions instead of generid 500 status
+ with app.test_client() as c:
yield c
diff --git a/test/test_general_poller_routes.py b/test/test_general_poller_routes.py
index a42700acd659488fea874a41680058e951f13d69..86064e7aba058162a1d756a987cad4bb6a8ba36e 100644
--- a/test/test_general_poller_routes.py
+++ b/test/test_general_poller_routes.py
@@ -318,24 +318,19 @@ def test_interface_dashboard_mapping(description, expected_dashboards):
assert set(d.name for d in dashboards) == set(expected_dashboards)
-def test__CAE_1_dashboards():
- interface = {
- 'router': 'mx1.lon.uk.geant.net',
- 'name': 'ae12.123',
- 'description': ''
- }
+@pytest.mark.parametrize(
+ "router, interface, exp_dashboards",
+ [
+ ("mx1.lon.uk.geant.net", "ae12.123", ["CAE1"]),
+ ("mx1.lon.uk.geant.net", "ae12", []), # POL1-704
+ ("rt1.mar.fr.geant.net", "ae12.123", ["IC1"]), # POL1_703
+ ("rt1.mar.fr.geant.net", "ae12", []), # POL1-704
+ ],
+)
+def test_ae12_aggregate_dashboards(router, interface, exp_dashboards):
+ interface = {"router": router, "name": interface, "description": ""}
dashboards = poller._get_dashboards(interface)
- assert set(d.name for d in dashboards) == {"CAE1"}
-
-
-def test_POL1_703_IC1_dashboards():
- interface = {
- 'router': 'rt1.mar.fr.geant.net',
- 'name': 'ae12.123',
- 'description': ''
- }
- dashboards = poller._get_dashboards(interface)
- assert set(d.name for d in dashboards) == {"IC1"}
+ assert [d.name for d in dashboards] == exp_dashboards
@pytest.mark.parametrize('interface,customers,dashboard_info', [
@@ -466,3 +461,25 @@ def test_gws_config_html(client):
# just a sanity check, no validation
# ... for now, this isn't an important interface
assert response_data.endswith('</html>')
+
+
+def test_get_all_error_report_interfaces(client):
+ rv = client.get('/poller/error-report-interfaces', headers=DEFAULT_REQUEST_HEADERS)
+ assert rv.status_code == 200
+ assert rv.is_json
+ response_data = json.loads(rv.data)
+ jsonschema.validate(response_data, poller.ERROR_REPORT_INTERFACE_LIST_SCHEMA)
+ response_routers = {ifc['router'] for ifc in response_data}
+ assert len(response_routers) > 1, 'there should data from be lots of routers'
+
+
+def test_get_single_router_error_report_interfaces(client):
+ rv = client.get(
+ '/poller/error-report-interfaces/mx1.ams.nl.geant.net',
+ headers=DEFAULT_REQUEST_HEADERS,
+ )
+ assert rv.status_code == 200
+ assert rv.is_json
+ response_data = json.loads(rv.data)
+ jsonschema.validate(response_data, poller.ERROR_REPORT_INTERFACE_LIST_SCHEMA)
+ assert {ifc['router'] for ifc in response_data} == {"mx1.ams.nl.geant.net"}
diff --git a/test/test_worker.py b/test/test_worker.py
index 0120d47f9de55b0806afac65d659290ea909538d..50a4f2e03035770d62bd2ed3042075b3db64f79c 100644
--- a/test/test_worker.py
+++ b/test/test_worker.py
@@ -5,7 +5,7 @@ import jsonschema
from lxml import etree
from inventory_provider.tasks import common
-from inventory_provider.tasks.worker import transform_ims_data, \
+from inventory_provider.tasks.worker import populate_error_report_interfaces_cache, transform_ims_data, \
extract_ims_data, persist_ims_data, \
retrieve_and_persist_neteng_managed_device_list, \
populate_poller_interfaces_cache, refresh_nokia_interface_list
@@ -595,7 +595,8 @@ def test_persist_ims_data(mocker, data_config, mocked_redis):
},
"sid_services": {"SID-001": [{"k1": "data"}, {"k1": "data"}]},
"services_by_type": {},
- "geant_nodes": []
+ "geant_nodes": [],
+ "circuit_ids_to_monitor": ["123", "456", "id1", "sub_circuit_1"]
}
for k in r.keys("ims:*"):
r.delete(k)
@@ -941,3 +942,104 @@ def test_refresh_nokia_interface_list(mocked_redis, data_config):
interface = json.loads(r.get(k).decode('utf-8'))
interfaces[k] = interface
jsonschema.validate(interface, interfaces_schema)
+
+
+def test_populate_error_report_interfaces_cache(mocker, data_config, mocked_redis):
+ r = common._get_redis(data_config)
+
+ mocker.patch('inventory_provider.tasks.worker.get_next_redis', return_value=r)
+ all_interfaces = [
+ {
+ "router": "router_a.geant.net",
+ "name": "interface_a",
+ "bundle": ["ae_a"],
+ "bundle-parents": [],
+ "description": "PHY DESCRIPTION A",
+ "circuits": []
+ },
+ {
+ "router": "router_a.geant.net",
+ "name": "ae_a",
+ "bundle": [],
+ "bundle-parents": [],
+ "description": "PHY DESCRIPTION B",
+ "circuits": []
+ },
+ {
+ "router": "router_a.geant.net",
+ "name": "ae_a.123",
+ "bundle": [],
+ "bundle-parents": [],
+ "description": "PHY DESCRIPTION C",
+ "circuits": []
+ },
+ {
+ "router": "rt0.geant.net",
+ "name": "lag-1.0",
+ "bundle": ["ae_c"],
+ "bundle-parents": [],
+ "description": "PHY DESCRIPTION D",
+ "circuits": []
+ },
+ {
+ "router": "router_a.geant.net",
+ "name": "ae_a.456",
+ "bundle": [],
+ "bundle-parents": [],
+ "description": "Spare Not included",
+ "circuits": []
+ },
+ {
+ "router": "router_a.geant.net",
+ "name": "dsc.1",
+ "bundle": [],
+ "bundle-parents": [],
+ "description": "PHY discard",
+ "circuits": []
+ },
+ ]
+
+ mocker.patch('inventory_provider.routes.poller._load_interfaces',
+ return_value=all_interfaces)
+ mocker.patch(
+ 'inventory_provider.tasks.worker.InventoryTask.config'
+ )
+ exp_router_a_interfaces = [
+ {
+ "router": "router_a.geant.net",
+ "name": "ae_a",
+ "description": "PHY DESCRIPTION B",
+ "vendor": "juniper"
+ },
+ {
+ "router": "router_a.geant.net",
+ "name": "ae_a.123",
+ "description": "PHY DESCRIPTION C",
+ "vendor": "juniper"
+ },
+ {
+ "router": "router_a.geant.net",
+ "name": "interface_a",
+ "description": "PHY DESCRIPTION A",
+ "vendor": "juniper"
+ },
+ ]
+ exp_nokia_router_interfaces = [
+ {
+ "router": "rt0.geant.net",
+ "name": "lag-1.0",
+ "description": "PHY DESCRIPTION D",
+ "vendor": "nokia"
+ }
+ ]
+
+ populate_error_report_interfaces_cache()
+
+ all = r.get("classifier-cache:error-report-interfaces:all").decode("utf-8")
+ assert json.loads(all) == exp_router_a_interfaces + exp_nokia_router_interfaces
+
+ router_a = r.get("classifier-cache:error-report-interfaces:router_a.geant.net")
+ assert json.loads(router_a) == exp_router_a_interfaces
+
+ nokia_router = r.get("classifier-cache:error-report-interfaces:rt0.geant.net")
+ assert json.loads(nokia_router) == exp_nokia_router_interfaces