Merge branch 'feature/DBOARD3-958' into develop

inventory_provider/juniper.py

@@ -11,6 +11,8 @@ from jnpr.junos import exception as EzErrors
 from lxml import etree
 import netifaces
 
+from inventory_provider.tasks.common import asn_to_int
+
 CONFIG_SCHEMA = """<?xml version="1.1" encoding="UTF-8" ?>
 <xs:schema xmlns:xs="http://www.w3.org/2001/XMLSchema">
 
@@ -280,36 +282,6 @@ def list_interfaces(netconf_config):
                 yield u
 
 
-def asn_to_int(asn_string: str) -> int:
-    """
-    Convert a possibly dotted ASN to an integer.
-
-    Args:
-    asn_string (str): ASN to be converted, can be in dot notation or not.
-
-    Returns:
-    int: ASN in integer format.
-
-    Raises:
-    ValueError: If the ASN string is not in the expected format or exceeds valid range.
-    """
-
-    dotted_asn_pattern = re.compile(r'^(\d+)\.(\d+)$')
-    match = dotted_asn_pattern.match(asn_string)
-
-    if match:
-        high_order, low_order = map(int, match.groups())
-
-        if high_order > 0xffff or low_order > 0xffff:
-            raise ValueError(f'Invalid ASN format: {asn_string}. Both components must be <= 0xffff.')
-
-        return (high_order << 16) | low_order
-    elif asn_string.isdigit():
-        return int(asn_string)
-    else:
-        raise ValueError(f'Unable to parse ASN string: {asn_string}. Expected either a pure integer or a dot notation.')
-
-
 def _system_bgp_peers(system_node):
     def _peering_params(neighbor_node):
         address = neighbor_node.find('name').text

inventory_provider/nokia.py

-from lxml import etree
+import ipaddress
 import logging
 import re
 
+from lxml import etree
 from ncclient import manager, xml_
 
+from inventory_provider.tasks.common import asn_to_int
+
 logger = logging.getLogger(__name__)
 
 BREAKOUT_PATTERN = re.compile(
@@ -88,6 +91,11 @@ def remove_xml_namespaces(etree_doc):
     return etree_doc
 
 
+def _get_admin_state_from_element(element):
+    admin_state_element = element.find('admin-state')
+    return admin_state_element.text if admin_state_element is not None else 'enable'
+
+
 def load_docs(hostname, ssh_params):
     """
     Load the running and state docs for the given hostname
@@ -168,10 +176,9 @@ def get_interfaces_state(state_doc):
 def get_ports_config(netconf_config):
     def _port_info(e):
         pi = {
-            'port-id': e.find('port-id').text
+            'port-id': e.find('port-id').text,
+            'admin-state': _get_admin_state_from_element(e)
         }
-        admin_state = e.find('admin-state')
-        pi['admin-state'] = admin_state.text if admin_state is not None else 'enabled'  # assume enabled if not present
 
         description = e.find('description')
         pi['description'] = description.text if description is not None else ''
@@ -211,14 +218,11 @@ def get_lags_config(netconf_config):
 
         port_elements = e.findall('./port')
         port_ids = (p.find('./port-id').text for p in port_elements)
-        admin_state = e.find('./admin-state')
         description_e = e.find('description')
         ifc = {
             'name': _name,
-            'description': (
-                description_e.text if description_e is not None else ''
-            ),
-            'admin-state': admin_state.text if admin_state is not None else 'enabled',  # assume enabled if not present
+            'description': description_e.text if description_e is not None else '',
+            'admin-state': _get_admin_state_from_element(e),
             'ports': [p_id for p_id in port_ids if p_id in enabled_ports],
         }
         return ifc
@@ -235,14 +239,11 @@ def get_interfaces_config(netconf_config):
             "interface-name": interface.find('interface-name').text,
             "ipv4": [],
             "ipv6": [],
+            'admin-state': _get_admin_state_from_element(interface)
         }
         description = interface.find('description')
         details["description"] = description.text if description is not None else ""
 
-        admin_state = interface.find('admin-state')
-        if admin_state is not None:
-            details["admin-state"] = admin_state.text
-
         for element in interface.xpath('ipv4/primary | ipv4/secondary'):
             address = element.find('address').text
             prefix_length = element.find('prefix-length').text
@@ -254,3 +255,59 @@ def get_interfaces_config(netconf_config):
             details["ipv6"].append(f'{address}/{prefix_length}')
 
         yield details
+
+
+def _get_neighbors_by_group(neighbor_elements):
+    neighbors_by_group = {}
+    for neighbor in neighbor_elements:
+        # admin_state = _get_admin_state_from_element(neighbor)  # do we want to do anything with this?
+        group_name = neighbor.find('group').text
+        address = neighbor.find('ip-address').text
+        info = {
+            'address': ipaddress.ip_address(address).exploded
+        }
+        peer_as = neighbor.find('peer-as')
+        if peer_as is not None:
+            info['remote-asn'] = asn_to_int(peer_as.text)
+        description = neighbor.find('description')
+        if description is not None:
+            info['description'] = description.text
+        neighbors_by_group.setdefault(group_name, []).append(info)
+    return neighbors_by_group
+
+
+def get_peer_info(neighbors_by_group, group_elements):
+    for bgp_group in group_elements:
+        # admin_state = _get_admin_state_from_element(bgp_group)  # do we want to do anything with this?
+        group_name = bgp_group.find('group-name').text
+        details = {
+            'group': group_name
+        }
+        local_as = bgp_group.find('local-as')
+        if local_as is not None:
+            asn_value_node = local_as.find('as-number')
+            details['local-asn'] = asn_to_int(asn_value_node.text)
+        for neighbor in neighbors_by_group.get(group_name, []):
+            neighbor.update(details)
+            yield neighbor
+
+
+def get_router_peers(netconf_config):
+    neighbors_by_group = _get_neighbors_by_group(netconf_config.xpath('configure/router/bgp/neighbor'))
+    group_elements = netconf_config.xpath('configure/router/bgp/group')
+    yield from get_peer_info(neighbors_by_group, group_elements)
+
+
+def get_all_vprn_peers(netconf_config):
+    for vprn in netconf_config.xpath('configure/service/vprn'):
+        service_name = vprn.find('service-name').text
+        neighbors_by_group = _get_neighbors_by_group(vprn.xpath('bgp/neighbor'))
+        group_elements = vprn.xpath('bgp/group')
+        for peer in get_peer_info(neighbors_by_group, group_elements):
+            peer['instance'] = service_name  # just to match the data from Juniper
+            yield peer
+
+
+def get_all_bgp_peers(netconf_config):
+    yield from get_router_peers(netconf_config)
+    yield from get_all_vprn_peers(netconf_config)

inventory_provider/tasks/common.py

@@ -255,3 +255,33 @@ def ims_sorted_service_type_key(service_type):
     # prettification ... e.g. no trailing _ for 'MD-VPN (NATIVE)'
     service_type = re.sub('_+$', '', service_type)
     return re.sub('^_+', '', service_type)
+
+
+def asn_to_int(asn_string: str) -> int:
+    """
+    Convert a possibly dotted ASN to an integer.
+
+    Args:
+    asn_string (str): ASN to be converted, can be in dot notation or not.
+
+    Returns:
+    int: ASN in integer format.
+
+    Raises:
+    ValueError: If the ASN string is not in the expected format or exceeds valid range.
+    """
+
+    dotted_asn_pattern = re.compile(r'^(\d+)\.(\d+)$')
+    match = dotted_asn_pattern.match(asn_string)
+
+    if match:
+        high_order, low_order = map(int, match.groups())
+
+        if high_order > 0xffff or low_order > 0xffff:
+            raise ValueError(f'Invalid ASN format: {asn_string}. Both components must be <= 0xffff.')
+
+        return (high_order << 16) | low_order
+    elif asn_string.isdigit():
+        return int(asn_string)
+    else:
+        raise ValueError(f'Unable to parse ASN string: {asn_string}. Expected either a pure integer or a dot notation.')

test/test_juniper_data_global.py

@@ -7,7 +7,7 @@ from lxml import etree
 import pytest
 
 from inventory_provider import juniper
-from inventory_provider.juniper import asn_to_int
+from inventory_provider.tasks.common import asn_to_int
 
 NETIFACES_TEST_DATA_STRING = """{
     'lo0': {{AF_INET}: [{'addr': '127.0.0.1', 'netmask': '255.0.0.0', 'peer': '127.0.0.1'}],

test/test_nokia.py

+import ipaddress
 import os
 import pathlib
+from copy import deepcopy
 from functools import lru_cache
 import pytest
 
@@ -23,6 +25,21 @@ PORT_SCHEMA = {
     'additionalProperties': True
 }
 
+PEERS_SCHEMA = {
+    '$schema': 'https://json-schema.org/draft/2020-12/schema',
+    'type': 'object',
+    'properties': {
+        'address': {'type': 'string'},
+        'remote-asn': {'type': 'integer'},
+        'local-asn': {'type': 'integer'},
+        'description': {'type': 'string'},
+        'group': {'type': 'string'},
+        'instance': {'type': 'string'}
+    },
+    'required': ['address', 'group'],
+    'additionalProperties': False
+}
+
 
 @lru_cache
 def _load_xml_doc(filename):
@@ -107,7 +124,7 @@ def test_get_lags(hostname, expected_bundles):
         )),
     ])
 def test_interface_info(hostname, all_expected_data):
-    netconf_doc = nokia.remove_xml_namespaces(_load_netconf_config(hostname=hostname))
+    netconf_doc = _load_netconf_config(hostname=hostname)
     interfaces_by_id = {ifc['interface-name']: ifc for ifc in
                         nokia.get_interfaces_config(netconf_doc)}
     assert len(interfaces_by_id) == len(all_expected_data)
@@ -127,7 +144,7 @@ def test_interface_info(hostname, all_expected_data):
     ('rt0.lon2.uk.geant.net', 139),
 ])
 def test_get_ports(hostname, port_count):
-    netconf_doc = nokia.remove_xml_namespaces(_load_netconf_config(hostname=hostname))
+    netconf_doc = _load_netconf_config(hostname=hostname)
     ports = list(nokia.get_ports_config(netconf_doc))
 
     assert len(ports) == port_count
@@ -186,3 +203,29 @@ def test_snmp_index():
         "management": 1280,
     }
     assert {ifc["interface-name"]: ifc["if-index"] for ifc in interfaces} == expected
+
+
+@pytest.mark.parametrize('hostname', [
+    'rt0.ams.nl.lab.office.geant.net',
+    'rt0.ams.nl.geant.net',
+    'rt0.fra.de.geant.net',
+    'rt0.gen.ch.geant.net',
+    'rt0.lon.uk.geant.net',
+    'rt0.lon2.uk.geant.net'
+])
+def test_get_peers(hostname):
+    netconf_doc = _load_netconf_config(hostname)
+    all_peers_from_doc = set()
+    all_peers_from_call = set()
+    for neighbor_element in netconf_doc.xpath('//bgp/neighbor'):
+        address = neighbor_element.find('ip-address').text
+        all_peers_from_doc.add(ipaddress.ip_address(address).exploded)
+    for peer in nokia.get_all_vprn_peers(netconf_doc):
+        jsonschema.validate(peer, PEERS_SCHEMA)
+        all_peers_from_call.add(peer['address'])
+    router_peers_schema = deepcopy(PEERS_SCHEMA)
+    del router_peers_schema['properties']['instance']
+    for peer in nokia.get_router_peers(netconf_doc):
+        jsonschema.validate(peer, PEERS_SCHEMA)
+        all_peers_from_call.add(peer['address'])
+    assert all_peers_from_doc == all_peers_from_call