Draft: Feature/pol1 744 fix speed issues

• Fix interface-info timeouts
• Make the poller speed endpoint use the data retrieved without it disappearing into the ether never to be used by anything

inventory_provider/routes/poller.py

@@ -207,7 +207,6 @@ INTERFACE_LIST_SCHEMA = {
     'items': {'$ref': '#/definitions/interface'}
 }
 
-
 INTERFACE_SPEED_LIST_SCHEMA = {
     '$schema': 'https://json-schema.org/draft-07/schema#',
 
@@ -344,7 +343,6 @@ GWS_DIRECT_DATA_SCHEMA = {
     'items': {'$ref': '#/definitions/interface-counters'}
 }
 
-
 SERVICES_LIST_SCHEMA = {
     '$schema': 'https://json-schema.org/draft-07/schema#',
 
@@ -405,7 +403,6 @@ SERVICES_LIST_SCHEMA = {
     'items': {'$ref': '#/definitions/service'}
 }
 
-
 STRING_LIST_SCHEMA = {
     '$schema': 'https://json-schema.org/draft-07/schema#',
     'type': 'array',
@@ -490,7 +487,6 @@ def _get_dashboards(interface):
 
 
 def _get_dashboard_data(ifc, customers):
-
     def _get_interface_type(description):
         if re.match(r'^PHY', description):
             return INTERFACE_TYPES.PHYSICAL
@@ -573,7 +569,6 @@ def _load_interface_bundles(config, hostname=None, use_next_redis=False):
                 key_pattern=key_pattern,
                 num_threads=20,
                 use_next_redis=use_next_redis):
-
             m = re.match(
                 r'.*netconf-interface-bundles:([^:]+):(.+)', doc['key'])
             assert m
@@ -664,7 +659,6 @@ def _load_netconf_docs(
             key_pattern=filter_pattern,
             num_threads=10,
             use_next_redis=use_next_redis):
-
         yield {
             'router': doc['key'][key_prefix_len:],
             'netconf': doc['value']
@@ -681,6 +675,7 @@ def _load_interfaces(
     :param use_next_redis:
     :return:
     """
+
     def _load_docs(key_pattern):
 
         for doc in _load_netconf_docs(config, key_pattern, use_next_redis):
@@ -705,6 +700,28 @@ def _load_interfaces(
         yield from _load_docs(f'lab:{base_key_pattern}')
 
 
+def _add_speeds(config, interfaces):
+    netconf_interfaces_all = list(common.load_json_docs(config, 'netconf-interfaces:all'))
+    try:
+        all_netconf_interfaces = netconf_interfaces_all[0]
+    except IndexError:
+        all_netconf_interfaces = []
+
+    netconf_interface_index = {}
+    for netconf_interface_doc in all_netconf_interfaces:
+        nc_ifc = netconf_interface_doc['value']
+        if 'router' in nc_ifc and 'name' in nc_ifc:
+            netconf_interface_index[f"{nc_ifc['router']}---{nc_ifc['name']}"] = nc_ifc
+
+    for ifc in interfaces:
+        nc_ifc = netconf_interface_index.get(f"{ifc['router']}---{ifc['name']}", {})
+        if 'speed' in nc_ifc:
+            ifc['speed'] = nc_ifc['speed']
+        else:
+            ifc['speed'] = ''
+        yield ifc
+
+
 def _add_bundle_parents(interfaces, hostname=None):
     """
     generator that adds bundle-parents info to each interface.
@@ -715,12 +732,16 @@ def _add_bundle_parents(interfaces, hostname=None):
     """
     bundles = _load_interface_bundles(
         current_app.config['INVENTORY_PROVIDER_CONFIG'], hostname)
+    # create a quick look-up for interface details
+    interface_index = {f"{ifc['router']}---{ifc['name']}": ifc for ifc in interfaces}
 
     for ifc in interfaces:
         router_bundle = bundles.get(ifc['router'], None)
         if router_bundle:
             base_ifc = ifc['name'].split('.')[0]
-            ifc['bundle-parents'] = router_bundle.get(base_ifc, [])
+            bundle_parents = [interface_index.get(f"{ifc['router']}---{bundle_ifc}")
+                              for bundle_ifc in router_bundle.get(base_ifc, [])]
+            ifc['bundle-parents'] = bundle_parents
         yield ifc
 
 
@@ -780,6 +801,7 @@ def load_interfaces_to_poll(
                 yield ifc
             else:
                 continue
+
     return _get_populated_interfaces(basic_interfaces)
 
 
@@ -880,13 +902,14 @@ def interface_speed(ifc):
                 logger.warning(f'unrecognised speed: {speed}, using _name_to_speed fallback')
                 return _name_to_speed(ifc['name'])
         else:
+            logger.warning('no speed data for interface, using _name_to_speed fallback')
             return _name_to_speed(ifc['name'])
 
     if ifc['bundle-parents']:
         if not ifc['name'].startswith('ae'):
             logger.warning(
                 f'ifc has bundle-parents, but name is {ifc["name"]}')
-        return sum(_name_to_speed(name) for name in ifc['bundle-parents'])
+        return sum(_get_speed(parent_ifc) for parent_ifc in ifc['bundle-parents'])
 
     return _get_speed(ifc)
 
@@ -903,7 +926,10 @@ def _load_interfaces_and_speeds(hostname=None):
         current_app.config['INVENTORY_PROVIDER_CONFIG'],
         hostname,
         no_lab=no_lab)
-    with_bundles = _add_bundle_parents(basic_interfaces, hostname)
+    basic_interfaces_with_speeds = _add_speeds(
+        current_app.config['INVENTORY_PROVIDER_CONFIG'],
+        basic_interfaces)
+    with_bundles = _add_bundle_parents(list(basic_interfaces_with_speeds), hostname)
 
     def _result_ifc(ifc):
         return {
@@ -1007,9 +1033,9 @@ def eumetsat_multicast(hostname=None):
     } for idx in range(1, 73)]
 
     SUBSCRIPTIONS.append(
-        {'subscription': '232.223.223.1',  'endpoint': '193.17.9.7'})
+        {'subscription': '232.223.223.1', 'endpoint': '193.17.9.7'})
     SUBSCRIPTIONS.append(
-        {'subscription': '232.223.223.22',  'endpoint': '193.17.9.7'})
+        {'subscription': '232.223.223.22', 'endpoint': '193.17.9.7'})
 
     def _oid(sub):
         return ('1.3.6.1.2.1.83.1.1.2.1.16'

[Erik Reid]

in fact, it would be better to do this during the update

[Erik Reid]

this can't go to production - please create a doc with all interfaces

... or at least, there are the per-host records in "netconf-interfaces-hosts:*"

[Erik Reid]

what linter are you using? I don't think 2 spaces is any kind of linting best practice ...

[Sam Roberts]

the autoformat removed the 2 spaces and left 1 space instead