Added dashboards_info property RE. POL1-526

inventory_provider/routes/poller.py

@@ -61,6 +61,7 @@ support method: _get_dashboards
 
 
 """
+from collections import defaultdict
 from enum import Enum, auto
 import itertools
 import json
@@ -140,6 +141,15 @@ INTERFACE_LIST_SCHEMA = {
             'required': ['id', 'name', 'type', 'status'],
             'additionalProperties': False
         },
+        'db_info': {
+            'type': 'object',
+            'properties': {
+                'name': {'type': 'string'},
+                'interface_type': {'enum': _INTERFACE_TYPES}
+            },
+            'required': ['name', 'interface_type'],
+            'additionalProperties': False
+        },
         'interface': {
             'type': 'object',
             'properties': {
@@ -167,13 +177,11 @@ INTERFACE_LIST_SCHEMA = {
                     'items': {'enum': _DASHBOARD_IDS}
                 },
                 'dashboard_info': {
-                    'type': 'object',
-                    'properties': {
-                        'name': {'type': 'string'},
-                        'interface_type': {'enum': _INTERFACE_TYPES}
-                    },
-                    'required': ['name', 'interface_type'],
-                    'additionalProperties': False
+                    '$ref': '#/definitions/db_info',
+                },
+                'dashboards_info': {
+                    'type': 'array',
+                    'items': {'$ref': '#/definitions/db_info'}
                 }
             },
             'required': [
@@ -467,7 +475,7 @@ def _get_dashboards(interface):
         yield BRIAN_DASHBOARDS.NREN
 
 
-def _get_dashboard_data(ifc):
+def _get_dashboard_data(ifc, customers):
 
     def _get_interface_type(description):
         if re.match(r'^PHY', description):
@@ -508,6 +516,7 @@ def _get_dashboard_data(ifc):
 
     if BRIAN_DASHBOARDS.INFRASTRUCTURE_BACKBONE.name in dashboards:
         name = _get_backbone_name(description)
+        customers.add(name)
     elif BRIAN_DASHBOARDS.GWS_PHY_UPSTREAM.name in dashboards:
         name = _get_customer_name(description)
         host = ifc['router']
@@ -521,7 +530,11 @@ def _get_dashboard_data(ifc):
         'dashboard_info': {
             'name': name,
             'interface_type': interface_type.name
-        }
+        },
+        'dashboards_info': [{
+            'name': customer_name,
+            'interface_type': interface_type.name
+        } for customer_name in customers]
     }
 
 
@@ -554,46 +567,40 @@ def _load_interface_bundles(config, hostname=None, use_next_redis=False):
     return result
 
 
-def _load_services(config, hostname=None, use_next_redis=False):
+def _get_services_and_customers(config, hostname=None, use_next_redis=False):
     if hostname:
         hostname = get_ims_equipment_name(hostname)
 
-    result = dict()
+    result = defaultdict(dict)
+    included_service_ids = set()
     key_pattern = f'ims:interface_services:{hostname}:*' \
         if hostname else 'ims:interface_services:*'
 
-    def _filter_and_format_services(_services):
-        included_service_ids = set()
-        for s in _services:
-            if s['id'] in included_service_ids:
-                continue
-            if s['circuit_type'] == 'service':
-                included_service_ids.add(s['id'])
-                yield {
-                    'id': s['id'],
-                    'name': s['name'],
-                    'type': s['service_type'],
-                    'status': s['status']
-                }
-
     for doc in common.load_json_docs(
             config_params=config,
             key_pattern=key_pattern,
             num_threads=20,
             use_next_redis=use_next_redis):
+        cs = {
+            'services': [],
+            'customers': set()
+        }
+        for s in doc['value']:
+            if s['id'] in included_service_ids:
+                continue
+            included_service_ids.add(s['id'])
 
-        m = re.match(r'^ims:interface_services:([^:]+):(.+)', doc['key'])
-        if not m:
-            logger.warning(f'can\'t parse redis service key {doc["key"]}')
-            # there are some weird records (dtn*, dp1*)
-            continue
-
-        router = m.group(1)
-        interface = m.group(2)
-        result.setdefault(router, dict())
-        result[router][interface] = \
-            list(_filter_and_format_services(doc['value']))
-
+            cs['customers'].add(s['customer'])
+            for c in s.get('additional_customers', []):
+                cs['customers'].add(c['name'])
+            if s['circuit_type'] == 'service':
+                cs['services'].append({
+                    'id': s['id'],
+                    'name': s['name'],
+                    'type': s['service_type'],
+                    'status': s['status'],
+                })
+            result[s['equipment']][s['port']] = cs
     return result
 
 
@@ -641,9 +648,6 @@ def _load_interfaces(
     :return:
     """
     def _load_docs(key_pattern):
-        # print('')
-        # logger.debug(f'docs Key: {key_pattern}')
-        # print('')
 
         for doc in _load_netconf_docs(config, key_pattern, use_next_redis):
 
@@ -691,7 +695,8 @@ def load_interfaces_to_poll(
     basic_interfaces = \
         list(_load_interfaces(config, hostname, no_lab, use_next_redis))
     bundles = _load_interface_bundles(config, hostname, use_next_redis)
-    services = _load_services(config, hostname, use_next_redis)
+    services_and_customers = \
+        _get_services_and_customers(config, hostname, use_next_redis)
     snmp_indexes = common.load_snmp_indexes(config, hostname, use_next_redis)
 
     def _get_populated_interfaces(all_interfaces):
@@ -709,16 +714,21 @@ def load_interfaces_to_poll(
                     base_ifc = ifc['name'].split('.')[0]
                     ifc['bundle-parents'] = router_bundle.get(base_ifc, [])
 
-                router_services = services.get(
-                    get_ims_equipment_name(ifc['router'], r), None)
-                if router_services:
-                    ifc['circuits'] = router_services.get(
-                        get_ims_interface(ifc['name']), []
+                router_services_and_customers = services_and_customers.get(
+                    get_ims_equipment_name(ifc['router'], r), {})
+                ifc_services_and_customers = \
+                    router_services_and_customers.get(
+                        get_ims_interface(ifc['name']), {}
                     )
 
+                if 'services' in ifc_services_and_customers \
+                        and ifc_services_and_customers['services']:
+                    ifc['circuits'] = ifc_services_and_customers['services']
+
                 dashboards = _get_dashboards(ifc)
                 ifc['dashboards'] = sorted([d.name for d in dashboards])
-                yield _get_dashboard_data(ifc)
+                yield _get_dashboard_data(
+                    ifc, ifc_services_and_customers.get('customers', set()))
             else:
                 continue
     return _get_populated_interfaces(basic_interfaces)

test/test_general_poller_routes.py

@@ -395,7 +395,7 @@ def test_interface_dashboard_mapping(description, expected_dashboards):
 ])
 def test_description_dashboard_parsing(interface, dashboard_info):
 
-    updated = poller._get_dashboard_data(interface)
+    updated = poller._get_dashboard_data(interface, set())
     info = updated['dashboard_info']
     assert info == dashboard_info
 

test/test_worker.py

@@ -488,8 +488,7 @@ def test_populate_poller_interfaces_cache(
         mocker, data_config, mocked_redis):
     r = common._get_redis(data_config)
 
-    mocker.patch('inventory_provider.tasks.common.get_next_redis',
-                 return_value=r)
+    mocker.patch('inventory_provider.tasks.common.get_next_redis')
     mocker.patch('inventory_provider.tasks.worker.get_next_redis',
                  return_value=r)
     all_interfaces = [
@@ -574,20 +573,26 @@ def test_populate_poller_interfaces_cache(
             }
         },
     }
-    services = {
+    services_and_customers = {
         "ROUTER_A": {
-            "AE_A.123": [{
-                "id": 321,
-                "name": "SERVICE A",
-                "type": "SERVICE TYPE",
-                "status": "operational"
-            }],
-            "AE_A.456": [{
-                "id": 654,
-                "name": "SERVICE B",
-                "type": "SERVICE TYPE",
-                "status": "operational"
-            }]
+            "AE_A.123": {
+                'services': [{
+                    "id": 321,
+                    "name": "SERVICE A",
+                    "type": "SERVICE TYPE",
+                    "status": "operational"
+                }],
+                'customers': {'geant'}
+            },
+            "AE_A.456": {
+                'services': [{
+                    "id": 654,
+                    "name": "SERVICE B",
+                    "type": "SERVICE TYPE",
+                    "status": "operational"
+                }],
+                'customers': {'geant'}
+            }
         }
     }
 
@@ -652,8 +657,9 @@ def test_populate_poller_interfaces_cache(
                  return_value=bundles)
     mocker.patch('inventory_provider.routes.common.load_snmp_indexes',
                  return_value=snmp_indexes)
-    mocker.patch('inventory_provider.routes.poller._load_services',
-                 return_value=services)
+    mocker.patch(
+        'inventory_provider.routes.poller._get_services_and_customers',
+        return_value=services_and_customers)
     mocker.patch(
         'inventory_provider.tasks.worker.InventoryTask.config'
     )