Changelog.md

@@ -2,6 +2,22 @@
 
 All notable changes to this project will be documented in this file.
 
+## [0.115] - 2024-04-12
+- adding related service for circuit hierarchy
+
+## [0.114] - 2024-03-27
+- adding new api for mic third party
+- additional search for hostname in IMS cache
+- rewrote _load_interfaces to not parse the entire netconf docs
+- POL1-0703: add support for IC1 dashboard 
+- DBOARD3-894 : Extract and persist interface-host information
+- DBOARD3-888 : Add netconf retrieval
+- added link-info and nokia-link-info routes
+- added verify_option to allow querying of lab IMS
+- initial nokia module, with netconf retrieval
+- added retrieval of nokia netconf
+- added retrieval of router vendor
+
 ## [0.113] - 2024-02-06
 - adding third party id as part of DBOARD3-676
 - pinning lxml

inventory_provider/nokia.py

-import ipaddress
 import logging
 import re
 
@@ -32,8 +31,8 @@ example:
         3 or 'unit' - g
 '''
 SPEED_UNITS = {
-    'g': 'Gbs',
-    'G': 'Gbs',
+    'g': 'Gbps',
+    'G': 'Gbps',
 }
 
 
@@ -47,6 +46,41 @@ def load_config(hostname, ssh_params, hostkey_verify=False):
         return m.get_config(source='running').data
 
 
+def get_ports(netconf_config):
+    def _port_info(e):
+        pi = {
+            'port-id': e.find('n:port-id', namespaces=NS).text,
+            'admin-state': e.find('n:admin-state', namespaces=NS).text,
+        }
+        description = e.find('n:description', namespaces=NS)
+
+        pi['description'] = description.text if description is not None else ''
+        breakout = e.find('./n:connector/n:breakout', namespaces=NS)
+        if breakout is not None:
+            breakout = breakout.text
+            breakout_match = BREAKOUT_PATTERN.match(breakout)
+            pi['breakout'] = breakout
+            pi['speed'] = int(breakout_match.group('speed'))
+            pi['speed-unit'] = SPEED_UNITS.get(
+                breakout_match.group('unit'), 'Unknown')
+        return pi
+
+    # making the assumption that the breakout ports are listed directly before their
+    # child ports
+    ports = netconf_config.findall('./n:configure/n:port', namespaces=NS)
+    current_parent_port = None
+    for port in ports:
+        port_info = _port_info(port)
+        if 'breakout' in port_info:
+            current_parent_port = port_info
+        elif current_parent_port is not None and port_info['port-id'].startswith(
+                current_parent_port['port-id']):
+            port_info['speed'] = current_parent_port['speed']
+            port_info['speed-unit'] = current_parent_port['speed-unit']
+
+        yield port_info
+
+
 def get_lags(netconf_config):
     def _lag_info(e):
         _name = e.find('./n:lag-name', namespaces=NS).text
@@ -73,20 +107,39 @@ def get_lags(netconf_config):
         yield _lag_info(lag)
 
 
-def interface_addresses(netconf_config):
-    def _interface_info(e, _name):
+def interface_info(netconf_config):
+
+    def _get_ip_address(e):
         for details_parent in e:
+            # example element
+            # <primary> # details_parent - not always primary
+            #     <address>62.40.119.9</address>
+            #     <prefix-length>32</prefix-length>
+            # </primary>
             address = details_parent[0].text
             prefix_length = details_parent[1].text
             ip_string = f'{address}/{prefix_length}'
-            yield {
-                'name': ipaddress.ip_interface(ip_string).ip.exploded,
-                'interface address': ip_string,
-                'interface name': _name,
-            }
+            yield ip_string
 
-    interfaces = netconf_config.findall('./n:configure/n:router/n:interface', namespaces=NS)
+    interfaces = netconf_config.findall(
+        'n:configure/n:router/n:interface', namespaces=NS)
     for interface in interfaces:
-        interface_name = interface.find('./n:interface-name', namespaces=NS).text
-        for element in interface.xpath('./n:ipv4|./n:ipv6', namespaces=NS):
-            yield from _interface_info(element, interface_name)
+        details = {
+            "interface-name": interface.find('n:interface-name', namespaces=NS).text,
+            "ipv4": [],
+            "ipv6": [],
+        }
+        description = interface.find('n:description', namespaces=NS)
+        details["description"] = description.text if description is not None else ""
+
+        admin_state = interface.find('n:admin-state', namespaces=NS)
+        if admin_state is not None:
+            details["admin-state"] = admin_state.text
+
+        for element in interface.xpath('n:ipv4', namespaces=NS):
+            details["ipv4"].extend(_get_ip_address(element))
+
+        for element in interface.xpath('n:ipv6', namespaces=NS):
+            details["ipv6"].extend(_get_ip_address(element))
+
+        yield details

inventory_provider/routes/mic.py

@@ -23,7 +23,6 @@ from inventory_provider.routes.common import _ignore_cache_or_retrieve
 logger = logging.getLogger(__name__)
 routes = Blueprint('mic-support-routes', __name__)
 
-
 ALL_DATA_SCHEMA = {
     "schema": "http://json-schema.org/draft-07/schema#",
     "type": "object",
@@ -52,7 +51,7 @@ ALL_DATA_SCHEMA = {
                                         "type": "array",
                                         "items": {"type": "string"}
                                     },
-                                    "third_party_id":  {"type": "string"}
+                                    "third_party_id": {"type": "string"}
                                 },
                                 "required": ["id", "sid", "status", "name",
                                              "service_type", "contacts",
@@ -68,6 +67,79 @@ ALL_DATA_SCHEMA = {
     "additionalProperties": False
 }
 
+THIRD_PARTY_RESPONSE_SCHEMA = {
+    "schema": "http://json-schema.org/draft-07/schema#",
+    "type": "object",
+    "properties": {
+        "circuit_hierarchy": {
+            "type": "array",
+            "items": {
+                "type": "object",
+                "properties": {
+                    "id": {"type": "integer"},
+                    "status": {"type": "string"},
+                    "name": {"type": "string"},
+                    "service_type": {"type": "string"},
+                    "contacts": {
+                        "type": "array",
+                        "items": {"type": "string"}
+                    },
+                    "planned_work_contacts": {
+                        "type": "array",
+                        "items": {"type": "string"}
+                    },
+                    "third_party_id": {"type": "string"}
+                },
+                "required": ["id", "status", "name", "service_type", "contacts", "planned_work_contacts",
+                             "third_party_id"],
+                "additionalProperties": False
+            }
+        },
+        "interface_services": {
+            "type": "object",
+            "patternProperties": {
+                "^.*$": {
+                    "type": "object",
+                    "patternProperties": {
+                        "^.*$": {
+                            "type": "object",
+                            "patternProperties": {
+                                "^.*$": {
+                                    "type": "array",
+                                    "items": {
+                                        "type": "object",
+                                        "properties": {
+                                            "id": {"type": "integer"},
+                                            "sid": {"type": "string"},
+                                            "status": {"type": "string"},
+                                            "name": {"type": "string"},
+                                            "service_type": {"type": "string"},
+                                            "contacts": {
+                                                "type": "array",
+                                                "items": {"type": "string"}
+                                            },
+                                            "planned_work_contacts": {
+                                                "type": "array",
+                                                "items": {"type": "string"}
+                                            },
+                                            "third_party_id": {"type": "string"}
+                                        },
+                                        "required": ["id", "sid", "status", "name", "service_type", "contacts",
+                                                     "planned_work_contacts", "third_party_id"],
+                                        "additionalProperties": False
+                                    }
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+        }
+    },
+    "required": ["circuit_hierarchy", "interface_services"],
+    "additionalProperties": False
+}
+
 
 @routes.route("/all-data")
 def get_everything():
@@ -81,3 +153,17 @@ def get_everything():
             status=404,
             mimetype="text/html")
     return Response(result, mimetype='application/json')
+
+
+@routes.route("/third-party-data")
+def get_third_party_data():
+    cache_key = "mic:impact:third-party-data"
+    logger.debug(cache_key)
+    r = common.get_current_redis()
+    result = _ignore_cache_or_retrieve(request, cache_key, r)
+    if not result:
+        return Response(
+            response='no data found',
+            status=404,
+            mimetype="text/html")
+    return Response(result, mimetype='application/json')

inventory_provider/tasks/worker.py

 import concurrent.futures
 import functools
+import ipaddress
+import itertools
 import json
 import logging
 import os
@@ -139,8 +141,7 @@ def refresh_juniper_interface_list(hostname, netconf, interface_info, lab=False)
     # scan with bigger batches, to mitigate network latency effects
     for k in r.scan_iter(f'{interfaces_keybase}:*', count=1000):
         rp.delete(k)
-    for k in r.scan_iter(
-            f'{bundles_keybase}:*', count=1000):
+    for k in r.scan_iter(f'{bundles_keybase}:*', count=1000):
         rp.delete(k)
     rp.execute()
 
@@ -653,28 +654,92 @@ def retrieve_and_persist_netconf_config_nokia(
 
 
 def refresh_nokia_interface_list(hostname, netconf_config, redis, lab=False):
-
     bundles_keybase = f'netconf-interface-bundles:{hostname}'
     interfaces_all_key = f'netconf-interfaces-hosts:{hostname}'
+    interfaces_key_base = f'netconf-interfaces:{hostname}'
     if lab:
         bundles_keybase = f'lab:{bundles_keybase}'
         interfaces_all_key = f'lab:{interfaces_all_key}'
+        interfaces_key_base = f'lab:{interfaces_key_base}'
 
     logger.debug(f'removing cached netconf-interfaces for {hostname}')
     rp = redis.pipeline()
     rp.delete(interfaces_all_key)
     for k in redis.scan_iter(f'{bundles_keybase}:*', count=1000):
         rp.delete(k)
+    for k in redis.scan_iter(f'{interfaces_key_base}:*', count=1000):
+        rp.delete(k)
     rp.execute()
+
+    ports_by_port_id = {p['port-id']: p for p in nokia.get_ports(netconf_config)}
+    lags_by_name = {lag['name']: lag for lag in nokia.get_lags(netconf_config)}
+    interfaces_by_name = \
+        {ifc['interface-name']: ifc for ifc in nokia.interface_info(netconf_config)}
+
+    def _save_interfaces_details(_details, _rp):
+        _rp.set(
+            f'{interfaces_key_base}:{_details["name"]}',
+            json.dumps(_details))
+
     rp = redis.pipeline()
-    for lag in nokia.get_lags(netconf_config):
+    for port in ports_by_port_id.values():
+        details = {
+            'name': port['port-id'],
+            'description': port['description'],
+            'bundle': [],
+            'ipv4': [],
+            'ipv6': [],
+        }
+        if 'speed' in port and 'speed-unit' in port:
+            details['speed'] = f'{port["speed"]}{port["speed-unit"]}'
+        else:
+            details['speed'] = ''
+        _save_interfaces_details(details, rp)
+
+    for interface in interfaces_by_name.values():
+        details = {
+            'name': interface['interface-name'],
+            'description': interface['description'],
+            'bundle':
+                lags_by_name.get(interface['interface-name'], {}).get('ports', []),
+            'speed': '',
+            'ipv4': interface['ipv4'],
+            'ipv6': interface['ipv6'],
+        }
+        _save_interfaces_details(details, rp)
+
+    def _get_lag_speed(_lag):
+        _ports = [ports_by_port_id[p] for p in _lag['ports'] if p in ports_by_port_id]
+        assert len({p['speed-unit'] for p in _ports}) == 1
+        return f'{sum(p["speed"] for p in _ports)}{_ports[0]["speed-unit"]}'
+
+    for lag in lags_by_name.values():
+        details = {
+            'name': lag['name'],
+            'description': lag['description'],
+            'bundle': lag['ports'],
+            'speed': _get_lag_speed(lag),
+            'ipv4': [],
+            'ipv6': [],
+        }
+        _save_interfaces_details(details, rp)
+
+    for lag in lags_by_name.values():
         rp.set(
             f'{bundles_keybase}:{lag["name"]}',
             json.dumps(lag['ports']))
 
+    interfaces = []
+    for interface in interfaces_by_name.values():
+        for addr in itertools.chain(interface['ipv4'], interface['ipv6']):
+            interfaces.append({
+                'name': ipaddress.ip_interface(addr).ip.exploded,
+                'interface address': addr,
+                'interface name': interface['interface-name'],
+            })
     rp.set(
         interfaces_all_key,
-        json.dumps(list(nokia.interface_addresses(netconf_config))))
+        json.dumps(interfaces))
     rp.execute()
 
 
@@ -1159,6 +1224,10 @@ def transform_ims_data(data):
         c, ttc = _get_circuit_contacts(d)
         d['contacts'] = sorted(list(c))
         d['planned_work_contacts'] = sorted(list(ttc))
+        d['sid'] = circuit_ids_and_sids.get(d['id'], '')
+
+        if d['id'] in circuit_ids_and_third_party_ids:
+            d['third_party_id'] = circuit_ids_and_third_party_ids[d['id']]
 
         # add flexils data to port_id_details and port_id_services
         all_ils_details = flexils_data.get(d['id'])
@@ -1255,9 +1324,6 @@ def transform_ims_data(data):
                 if c['id'] in circuit_ids_and_sids:
                     rs[c['id']]['sid'] = circuit_ids_and_sids[c['id']]
 
-                if c['id'] in circuit_ids_and_third_party_ids:
-                    rs[c['id']]['third_party_id'] = circuit_ids_and_third_party_ids[c['id']]
-
             if c['sub-circuits']:
                 for sub in c['sub-circuits']:
                     temp_parents = \
@@ -1512,6 +1578,7 @@ def persist_ims_data(data, use_current=False):
 
     populate_poller_cache(interface_services, r)
     populate_mic_cache(interface_services, r)
+    populate_mic_with_third_party_data(interface_services, hierarchy, r)
 
     for service_type, services in services_by_type.items():
         for v in services.values():
@@ -1546,6 +1613,135 @@ def persist_ims_data(data, use_current=False):
     rp.execute()
 
 
+def populate_mic_with_third_party_data(interface_services, hierarchy, r):
+    cache_key = "mic:impact:third-party-data"
+    third_party_data = defaultdict(lambda: defaultdict(dict))
+    third_party_interface_data = defaultdict(lambda: defaultdict(dict))
+
+    def get_related_services_ids(base_id):
+        s = set()
+        base = hierarchy[base_id]
+        if base['circuit-type'] == 'service':
+            s.add(base['id'])
+        if base['sub-circuits']:
+            for sub_circuit in base['sub-circuits']:
+                s |= get_related_services_ids(sub_circuit)
+        return s
+
+    def get_formatted_third_party_rs(_circuit_data):
+        if _circuit_data and _circuit_data['status'] == 'operational' and _circuit_data['circuit-type'] == 'service':
+            return {
+                'id': _circuit_data['id'],
+                'name': _circuit_data['name'] + ' (' + _circuit_data['sid'] + ')',
+                'service_type': _circuit_data['product'],
+            }
+        else:
+            return None
+
+    def get_related_services_for_third_party_type_circuit(_circuit_data):
+        related_service_ids = list(get_related_services_ids(_circuit_data['id']))
+        related_services = []
+        seen_names = set()  # Set to keep track of already added names
+        for rs_id in related_service_ids:
+            rs_info = hierarchy.get(rs_id)
+            if rs_info is not None:
+                formatted_rs = get_formatted_third_party_rs(rs_info)
+                if formatted_rs is None:
+                    # print("Formatted RS is None for ID:", rs_id)
+                    continue
+                name = formatted_rs['name']
+                if name not in seen_names:  # Check if the name has not been seen before
+                    related_services.append({
+                        'id': formatted_rs['id'],
+                        'name': name,
+                        'service_type': formatted_rs['service_type']
+                    })
+                    seen_names.add(name)  # Add the name to the set of seen names
+        return related_services
+
+    def _get_formatted_third_party_data(_circuit_data):
+        if _circuit_data['status'] == 'operational' and _circuit_data.get('third_party_id'):
+            return {
+                'id': _circuit_data['id'],
+                'status': _circuit_data['status'],
+                'name': _circuit_data['name'],
+                'sid': _circuit_data.get('sid', ''),
+                'service': _circuit_data['circuit-type'],
+                'service_type': 'circuit_hierarchy',
+                'contacts': _circuit_data['contacts'],
+                'planned_work_contacts': _circuit_data['planned_work_contacts'],
+                'third_party_id': _circuit_data['third_party_id'],
+                'related_services': get_related_services_for_third_party_type_circuit(_circuit_data)
+            }
+
+    def _get_formatted_related_service(_d):
+        for rs in _d['related-services']:
+            if rs['status'] == 'operational' and rs.get('third_party_id'):
+                yield {
+                    'id': rs['id'],
+                    'sid': rs.get('sid', ''),
+                    'status': rs['status'],
+                    'name': rs['name'],
+                    'service_type': rs['service_type'],
+                    'contacts': rs['contacts'],
+                    'planned_work_contacts': rs['planned_work_contacts'],
+                    'third_party_id': rs.get('third_party_id', '')
+                }
+
+    def _get_formatted_interface_service(_is):
+        if _is['status'] == 'operational' and _is.get('third_party_id'):
+            return {
+                'id': _is['id'],
+                'sid': _is.get('sid', ''),
+                'status': _is['status'],
+                'name': _is['name'],
+                'service_type': 'circuit_type',
+                'contacts': _is['contacts'],
+                'planned_work_contacts': _is['planned_work_contacts'],
+                'third_party_id': _is.get('third_party_id', '')
+            }
+
+    # get circuit hierarchy items that have third party id
+    third_party_circuit = {
+        k: v for k, v in hierarchy.items() if 'third_party_id' in v
+    }
+
+    # iterate over each third_party_circuit items to format and add operational once only
+    third_party_circuit_data = [_get_formatted_third_party_data(v)
+                                for k, v in third_party_circuit.items()
+                                if v and _get_formatted_third_party_data(v)]
+
+    # add third_party_circuit_data to the third_party_data map and add the map to cache
+    third_party_data['circuit_hierarchy'] = third_party_circuit_data
+
+    for services in interface_services.values():
+        if services:
+            current_interface_services = []
+            seen_ids = set()
+            for d in services:
+                if d.get('related-services'):
+                    for rs in _get_formatted_related_service(d):
+                        if rs['id'] not in seen_ids:
+                            current_interface_services.append(rs)
+                            seen_ids.add(rs['id'])
+                    if (str(d.get('id')) + 'circuit') not in seen_ids:
+                        _is = _get_formatted_interface_service(d)
+                        if _is:
+                            current_interface_services.append(_is)
+                        seen_ids.add(str(d.get('id')) + 'circuit')
+
+            if current_interface_services:
+                site = f'{services[0]["pop_name"]} ' \
+                       f'({services[0]["pop_abbreviation"]})'
+                eq_name = services[0]['equipment']
+                if_name = services[0]['port']
+                third_party_interface_data[site][eq_name][if_name] = current_interface_services
+    third_party_data['interface_services'] = third_party_interface_data
+
+    result = json.dumps(third_party_data)
+    r.set(cache_key, result.encode('utf-8'))
+
+
 def populate_mic_cache(interface_services, r):
     cache_key = "mic:impact:all-data"
     all_data = defaultdict(lambda: defaultdict(dict))

setup.py

@@ -2,7 +2,7 @@ from setuptools import setup, find_packages
 
 setup(
     name='inventory-provider',
-    version="0.114",
+    version="0.116",
     author='GEANT',
     author_email='swd@geant.org',
     description='Dashboard inventory provider',

test/test_nokia.py

@@ -2,7 +2,7 @@ import pathlib
 
 from lxml import etree
 
-from inventory_provider.nokia import get_lags, interface_addresses
+from inventory_provider.nokia import get_lags, interface_info, get_ports
 
 netconf_doc = etree.parse(pathlib.Path(__file__).parent.joinpath(
     'data/rt0.lon.uk.lab.office.geant.net-netconf-nokia.xml'))
@@ -16,10 +16,22 @@ def test_get_lags():
     assert found_names == expected_names
 
 
-def test_interface_addresses():
-    if_addresses = list(interface_addresses(netconf_doc))
-    assert len(if_addresses) == 9
-    found_names = {_interface['interface name'] for _interface in if_addresses}
+def test_interface_info():
+    if_addresses = list(interface_info(netconf_doc))
+    assert len(if_addresses) == 5
+    found_names = {_interface['interface-name'] for _interface in if_addresses}
     expected_names = {'lag-1.0', 'lag-2.0', 'system', 'to_rt0_ams_ZR-INFINERA',
                       'to_rt0_ams_ZR-NOKIA'}
     assert found_names == expected_names
+
+
+def test_get_ports():
+    ports = list(get_ports(netconf_doc))
+    assert len(ports) == 18
+    found_port_ids = {port['port-id'] for port in ports}
+    expected_ports = {
+        '1/1/c2', '1/1/c2/1', '1/1/c2/2', '1/1/c2/3', '1/1/c7', '1/1/c7/1',
+        '1/1/c8', '1/1/c8/1', '1/1/c9', '1/1/c9/1', '1/1/c13', '1/1/c13/1',
+        '2/1/c7', '2/1/c7/1', '2/1/c8', '2/1/c8/1', '2/1/c13', '2/1/c13/1',
+    }
+    assert found_port_ids == expected_ports

test/test_worker.py

@@ -860,12 +860,12 @@ def test_refresh_nokia_interface_list(mocked_redis, data_config):
     }
     assert bundles == expected_bundles
 
-    interfaces = json.loads(r.get(
+    interface_hosts = json.loads(r.get(
         'lab:netconf-interfaces-hosts:rt0.lon.uk.lab.office.geant.net'
     ).decode('utf-8'))
-    assert len(interfaces) == 9
+    assert len(interface_hosts) == 9
 
-    schema = {
+    interface_hosts_schema = {
       '$schema': 'http://json-schema.org/draft-07/schema#',
       'type': 'array',
       'items': {
@@ -885,4 +885,59 @@ def test_refresh_nokia_interface_list(mocked_redis, data_config):
         'additionalProperties': False
       }
     }
-    jsonschema.validate(interfaces, schema)
+    jsonschema.validate(interface_hosts, interface_hosts_schema)
+
+    interfaces_schema = {
+      '$schema': 'http://json-schema.org/draft-07/schema#',
+      'type': 'object',
+      'properties': {
+        'name': {
+          'type': 'string'
+        },
+        'description': {
+          'type': 'string'
+        },
+        'bundle': {
+          'type': 'array',
+          'items': {
+            'type': 'string'
+          }
+        },
+        'speed': {
+          'anyOf': [
+            {
+              'type': 'string',
+              'pattern': r'^\d+\wbps$'
+            },
+            {
+              'type': 'string',
+              'enum': ['']
+            }
+          ]
+        },
+        'ipv4': {
+          'type': 'array',
+          'items': {
+            'type': 'string',
+            'format': 'ipv6'
+          }
+        },
+        'ipv6': {
+          'type': 'array',
+          'items': {
+            'type': 'string',
+            'format': 'ipv6'
+          }
+        }
+      },
+      'required': ['name', 'description', 'bundle', 'speed', 'ipv4', 'ipv6']
+    }
+
+    keybase = 'lab:netconf-interfaces:rt0.lon.uk.lab.office.geant.net:*'
+    keys = r.keys(keybase)
+    interfaces = {}
+    for k in keys:
+        k = k.decode('utf-8')
+        interface = json.loads(r.get(k).decode('utf-8'))
+        interfaces[k] = interface
+        jsonschema.validate(interface, interfaces_schema)