From d2c01b118e52021b20828c8c20f4d3dec5f90341 Mon Sep 17 00:00:00 2001
From: Robert Latta <robert.latta@geant.org>
Date: Wed, 23 Mar 2022 09:24:56 +0000
Subject: [PATCH] Added dashboards_info property RE. POL1-526
---
inventory_provider/routes/poller.py | 104 +++++++++++++++-------------
test/test_general_poller_routes.py | 2 +-
test/test_worker.py | 40 ++++++-----
3 files changed, 81 insertions(+), 65 deletions(-)
diff --git a/inventory_provider/routes/poller.py b/inventory_provider/routes/poller.py
index 525564e8..3d5c7cf3 100644
--- a/inventory_provider/routes/poller.py
+++ b/inventory_provider/routes/poller.py
@@ -61,6 +61,7 @@ support method: _get_dashboards
"""
+from collections import defaultdict
from enum import Enum, auto
import itertools
import json
@@ -140,6 +141,15 @@ INTERFACE_LIST_SCHEMA = {
'required': ['id', 'name', 'type', 'status'],
'additionalProperties': False
},
+ 'db_info': {
+ 'type': 'object',
+ 'properties': {
+ 'name': {'type': 'string'},
+ 'interface_type': {'enum': _INTERFACE_TYPES}
+ },
+ 'required': ['name', 'interface_type'],
+ 'additionalProperties': False
+ },
'interface': {
'type': 'object',
'properties': {
@@ -167,13 +177,11 @@ INTERFACE_LIST_SCHEMA = {
'items': {'enum': _DASHBOARD_IDS}
},
'dashboard_info': {
- 'type': 'object',
- 'properties': {
- 'name': {'type': 'string'},
- 'interface_type': {'enum': _INTERFACE_TYPES}
- },
- 'required': ['name', 'interface_type'],
- 'additionalProperties': False
+ '$ref': '#/definitions/db_info',
+ },
+ 'dashboards_info': {
+ 'type': 'array',
+ 'items': {'$ref': '#/definitions/db_info'}
}
},
'required': [
@@ -467,7 +475,7 @@ def _get_dashboards(interface):
yield BRIAN_DASHBOARDS.NREN
-def _get_dashboard_data(ifc):
+def _get_dashboard_data(ifc, customers):
def _get_interface_type(description):
if re.match(r'^PHY', description):
@@ -508,6 +516,7 @@ def _get_dashboard_data(ifc):
if BRIAN_DASHBOARDS.INFRASTRUCTURE_BACKBONE.name in dashboards:
name = _get_backbone_name(description)
+ customers.add(name)
elif BRIAN_DASHBOARDS.GWS_PHY_UPSTREAM.name in dashboards:
name = _get_customer_name(description)
host = ifc['router']
@@ -521,7 +530,11 @@ def _get_dashboard_data(ifc):
'dashboard_info': {
'name': name,
'interface_type': interface_type.name
- }
+ },
+ 'dashboards_info': [{
+ 'name': customer_name,
+ 'interface_type': interface_type.name
+ } for customer_name in customers]
}
@@ -554,46 +567,40 @@ def _load_interface_bundles(config, hostname=None, use_next_redis=False):
return result
-def _load_services(config, hostname=None, use_next_redis=False):
+def _get_services_and_customers(config, hostname=None, use_next_redis=False):
if hostname:
hostname = get_ims_equipment_name(hostname)
- result = dict()
+ result = defaultdict(dict)
+ included_service_ids = set()
key_pattern = f'ims:interface_services:{hostname}:*' \
if hostname else 'ims:interface_services:*'
- def _filter_and_format_services(_services):
- included_service_ids = set()
- for s in _services:
- if s['id'] in included_service_ids:
- continue
- if s['circuit_type'] == 'service':
- included_service_ids.add(s['id'])
- yield {
- 'id': s['id'],
- 'name': s['name'],
- 'type': s['service_type'],
- 'status': s['status']
- }
-
for doc in common.load_json_docs(
config_params=config,
key_pattern=key_pattern,
num_threads=20,
use_next_redis=use_next_redis):
+ cs = {
+ 'services': [],
+ 'customers': set()
+ }
+ for s in doc['value']:
+ if s['id'] in included_service_ids:
+ continue
+ included_service_ids.add(s['id'])
- m = re.match(r'^ims:interface_services:([^:]+):(.+)', doc['key'])
- if not m:
- logger.warning(f'can\'t parse redis service key {doc["key"]}')
- # there are some weird records (dtn*, dp1*)
- continue
-
- router = m.group(1)
- interface = m.group(2)
- result.setdefault(router, dict())
- result[router][interface] = \
- list(_filter_and_format_services(doc['value']))
-
+ cs['customers'].add(s['customer'])
+ for c in s.get('additional_customers', []):
+ cs['customers'].add(c['name'])
+ if s['circuit_type'] == 'service':
+ cs['services'].append({
+ 'id': s['id'],
+ 'name': s['name'],
+ 'type': s['service_type'],
+ 'status': s['status'],
+ })
+ result[s['equipment']][s['port']] = cs
return result
@@ -641,9 +648,6 @@ def _load_interfaces(
:return:
"""
def _load_docs(key_pattern):
- # print('')
- # logger.debug(f'docs Key: {key_pattern}')
- # print('')
for doc in _load_netconf_docs(config, key_pattern, use_next_redis):
@@ -691,7 +695,8 @@ def load_interfaces_to_poll(
basic_interfaces = \
list(_load_interfaces(config, hostname, no_lab, use_next_redis))
bundles = _load_interface_bundles(config, hostname, use_next_redis)
- services = _load_services(config, hostname, use_next_redis)
+ services_and_customers = \
+ _get_services_and_customers(config, hostname, use_next_redis)
snmp_indexes = common.load_snmp_indexes(config, hostname, use_next_redis)
def _get_populated_interfaces(all_interfaces):
@@ -709,16 +714,21 @@ def load_interfaces_to_poll(
base_ifc = ifc['name'].split('.')[0]
ifc['bundle-parents'] = router_bundle.get(base_ifc, [])
- router_services = services.get(
- get_ims_equipment_name(ifc['router'], r), None)
- if router_services:
- ifc['circuits'] = router_services.get(
- get_ims_interface(ifc['name']), []
+ router_services_and_customers = services_and_customers.get(
+ get_ims_equipment_name(ifc['router'], r), {})
+ ifc_services_and_customers = \
+ router_services_and_customers.get(
+ get_ims_interface(ifc['name']), {}
)
+ if 'services' in ifc_services_and_customers \
+ and ifc_services_and_customers['services']:
+ ifc['circuits'] = ifc_services_and_customers['services']
+
dashboards = _get_dashboards(ifc)
ifc['dashboards'] = sorted([d.name for d in dashboards])
- yield _get_dashboard_data(ifc)
+ yield _get_dashboard_data(
+ ifc, ifc_services_and_customers.get('customers', set()))
else:
continue
return _get_populated_interfaces(basic_interfaces)
diff --git a/test/test_general_poller_routes.py b/test/test_general_poller_routes.py
index 4a40cb4f..e863657f 100644
--- a/test/test_general_poller_routes.py
+++ b/test/test_general_poller_routes.py
@@ -395,7 +395,7 @@ def test_interface_dashboard_mapping(description, expected_dashboards):
])
def test_description_dashboard_parsing(interface, dashboard_info):
- updated = poller._get_dashboard_data(interface)
+ updated = poller._get_dashboard_data(interface, set())
info = updated['dashboard_info']
assert info == dashboard_info
diff --git a/test/test_worker.py b/test/test_worker.py
index 2ff0ce69..f98bc1b2 100644
--- a/test/test_worker.py
+++ b/test/test_worker.py
@@ -488,8 +488,7 @@ def test_populate_poller_interfaces_cache(
mocker, data_config, mocked_redis):
r = common._get_redis(data_config)
- mocker.patch('inventory_provider.tasks.common.get_next_redis',
- return_value=r)
+ mocker.patch('inventory_provider.tasks.common.get_next_redis')
mocker.patch('inventory_provider.tasks.worker.get_next_redis',
return_value=r)
all_interfaces = [
@@ -574,20 +573,26 @@ def test_populate_poller_interfaces_cache(
}
},
}
- services = {
+ services_and_customers = {
"ROUTER_A": {
- "AE_A.123": [{
- "id": 321,
- "name": "SERVICE A",
- "type": "SERVICE TYPE",
- "status": "operational"
- }],
- "AE_A.456": [{
- "id": 654,
- "name": "SERVICE B",
- "type": "SERVICE TYPE",
- "status": "operational"
- }]
+ "AE_A.123": {
+ 'services': [{
+ "id": 321,
+ "name": "SERVICE A",
+ "type": "SERVICE TYPE",
+ "status": "operational"
+ }],
+ 'customers': {'geant'}
+ },
+ "AE_A.456": {
+ 'services': [{
+ "id": 654,
+ "name": "SERVICE B",
+ "type": "SERVICE TYPE",
+ "status": "operational"
+ }],
+ 'customers': {'geant'}
+ }
}
}
@@ -652,8 +657,9 @@ def test_populate_poller_interfaces_cache(
return_value=bundles)
mocker.patch('inventory_provider.routes.common.load_snmp_indexes',
return_value=snmp_indexes)
- mocker.patch('inventory_provider.routes.poller._load_services',
- return_value=services)
+ mocker.patch(
+ 'inventory_provider.routes.poller._get_services_and_customers',
+ return_value=services_and_customers)
mocker.patch(
'inventory_provider.tasks.worker.InventoryTask.config'
)
--
GitLab