added extraction and persistence of nokia bundles. RE. DBOARD3-893

inventory_provider/nokia.py

 import logging
+import re
 
 from ncclient import manager
 
@@ -6,11 +7,66 @@ from ncclient import manager
 logger = logging.getLogger(__name__)
 
 
+ROOT_NS = 'urn:ietf:params:xml:ns:netconf:base:1.0'
+NOKIA_NS = 'urn:nokia.com:sros:ns:yang:sr:conf'
+NS = {
+    'r': ROOT_NS,
+    'n': NOKIA_NS,
+}
+
+BREAKOUT_PATTERN = re.compile(
+    r'c(?P<count>\d+)-(?P<speed>\d+)(?P<unit>[a-zA-Z]+)'
+)
+'''
+For translating the breakout string to a speed
+example:
+    c1-400g - This has a single 400G port, groups would be as follows:
+        1 or 'count' - 1
+        2 or 'speed' - 400
+        3 or 'unit' - g
+    c4-100g - This has four 100G ports, not all of them may be used
+    groups would be as follows:
+        1 or 'count' - 4
+        2 or 'speed' - 100
+        3 or 'unit' - g
+'''
+SPEED_UNITS = {
+    'g': 'Gbs',
+    'G': 'Gbs',
+}
+
+
 def load_config(hostname, ssh_params, hostkey_verify=False):
-    logger.info(f"capturing netconf data for '{hostname}'")
+    logger.info(f'capturing netconf data for "{hostname}"')
     with manager.connect(
         host=hostname,
         hostkey_verify=hostkey_verify,
         **ssh_params
     ) as m:
         return m.get_config(source='running')
+
+
+def get_lags(config):
+    def _lag_info(e):
+        _name = e.find('./n:lag-name', namespaces=NS).text
+
+        def get_port(p):
+            return p.find('./n:port-id', namespaces=NS).text
+
+        port_elements = e.findall('./n:port', namespaces=NS)
+        ports = [get_port(p) for p in port_elements]
+        admin_state = e.find('./n:admin-state', namespaces=NS).text
+        description_e = e.find('n:description', namespaces=NS)
+        ifc = {
+            'name': _name,
+            'description': (
+                description_e.text if description_e is not None else ''
+            ),
+            'admin-status': admin_state,
+            'ports': ports,
+        }
+        return ifc
+
+    lags = config.findall('./n:configure/n:lag', namespaces=NS)
+    for lag in lags:
+        yield _lag_info(lag)

inventory_provider/tasks/worker.py

@@ -595,8 +595,9 @@ def reload_router_config_try_all(self, hostname, lab=False):
 @log_task_entry_and_exit
 def reload_router_config_nokia(self, hostname, lab=False):
     try:
-        _reload_router_config_nokia(
+        netconf_doc = _reload_router_config_nokia(
             hostname, lab, self.log_info, self.log_warning)
+        refresh_nokia_interface_list(hostname, netconf_doc)
     except Exception as e:
         errmsg = f'unhandled exception loading {hostname} info: {e}'
         logger.exception(errmsg)
@@ -614,7 +615,7 @@ def _reload_router_config_nokia(
 ):
     info_callback(
         f'loading netconf data for {"lab " if lab else ""} {hostname}')
-    retrieve_and_persist_netconf_config_nokia(
+    return retrieve_and_persist_netconf_config_nokia(
         hostname, lab, warning_callback)
 
 
@@ -650,6 +651,31 @@ def retrieve_and_persist_netconf_config_nokia(
     return netconf_str
 
 
+def refresh_nokia_interface_list(hostname, netconf):
+    r = get_next_redis(InventoryTask.config).pipeline()
+    _refresh_nokia_interface_list(hostname, netconf, r)
+
+
+def _refresh_nokia_interface_list(hostname, netconf, redis, lab=False):
+
+    bundles_keybase = f'netconf-interface-bundles:{hostname}'
+    if lab:
+        bundles_keybase = f'lab:{bundles_keybase}'
+
+    logger.debug(f'removing cached netconf-interfaces for {hostname}')
+    rp = redis.pipeline()
+    for k in redis.scan_iter(
+            f'{bundles_keybase}:*', count=1000):
+        rp.delete(k)
+    rp.execute()
+    rp = redis.pipeline()
+    for lag in nokia.get_lags(netconf):
+        rp.set(
+            f'{bundles_keybase}:{lag["name"]}',
+            json.dumps(lag['ports']))
+    rp.execute()
+
+
 @app.task(base=InventoryTask, bind=True, name='reload_lab_router_juniper')
 @log_task_entry_and_exit
 def reload_lab_router_config_juniper(self, hostname):

test/test_nokia.py

+import pathlib
+
+from lxml import etree
+
+from inventory_provider.nokia import get_lags
+
+netconf_doc = etree.parse(pathlib.Path(__file__).parent.joinpath(
+    'data/rt0.lon.uk.lab.office.geant.net-netconf-nokia.xml'))
+
+
+def test_get_lags():
+    lags = list(get_lags(netconf_doc))
+    assert len(lags) == 4
+    found_names = {lag['name'] for lag in lags}
+    expected_names = {'lag-1', 'lag-2', 'lag-3', 'lag-31'}
+    assert found_names == expected_names

test/test_worker.py

 import json
+import pathlib
+
+from lxml import etree
 
 from inventory_provider.tasks import common
 from inventory_provider.tasks.worker import transform_ims_data, \
     extract_ims_data, persist_ims_data, \
     retrieve_and_persist_neteng_managed_device_list, \
-    populate_poller_interfaces_cache
+    populate_poller_interfaces_cache, _refresh_nokia_interface_list
 
 
 def test_extract_ims_data(mocker):
@@ -834,3 +837,24 @@ def test_populate_poller_interfaces_cache(
     assert json.loads(no_lab) == no_lab_res
     all_res = no_lab_res + lab_res
     assert json.loads(all) == all_res
+
+
+def test_refresh_nokia_interface_list(mocked_redis, data_config):
+    netconf_doc = etree.parse(pathlib.Path(__file__).parent.joinpath(
+        'data/rt0.lon.uk.lab.office.geant.net-netconf-nokia.xml'))
+    r = common._get_redis(data_config)
+    _refresh_nokia_interface_list('rt0.lon.uk.lab.office.geant.net', netconf_doc, r, True)
+    keybase = 'lab:netconf-interface-bundles:rt0.lon.uk.lab.office.geant.net:'
+    keys = r.keys(f'{keybase}*')
+    bundles = {}
+    for k in keys:
+        k = k.decode('utf-8')
+        bundles[k] = json.loads(r.get(k).decode('utf-8'))
+
+    expected = {
+        f'{keybase}lag-1': ['1/1/c8/1', '1/1/c9/1'],
+        f'{keybase}lag-2': ['2/1/c8/1'],
+        f'{keybase}lag-3': ['1/1/c2/2'],
+        f'{keybase}lag-31': ['1/1/c2/1'],
+    }
+    assert bundles == expected