Feature/pol1 775 poller interfaces cache built from preparsed

Changes cache generation to use pre-parsed netconf caches in Redis. All unit tests pass as of writing.

inventory_provider/routes/poller.py

@@ -670,6 +670,23 @@ def _load_netconf_docs(
         }
 
 
+def _load_netconf_parsed_cache(config, cache_name, hostname=None, is_lab=False, use_next_redis=False):
+    hostname_suffix = hostname if hostname else ''
+    lab_prefix = 'lab:' if is_lab else ''
+    filter_pattern = f'{lab_prefix}{cache_name}:{hostname_suffix}*'
+    for doc in common.load_json_docs(
+            config_params=config,
+            key_pattern=filter_pattern,
+            num_threads=20,
+            use_next_redis=use_next_redis):
+        m = re.match(fr'(lab:)?{cache_name}:(.+:.+)', doc['key'])
+        if m:
+            # preparse the key as interfaces can include : in the name
+            key_parts = m.group(2).split(':')
+            key = f'{key_parts[0]}-----{":".join(key_parts[1:])}'
+            yield key, doc['value']
+
+
 def _load_interfaces(
         config, hostname=None, no_lab=False, use_next_redis=False):
     """
@@ -681,28 +698,32 @@ def _load_interfaces(
     :return:
     """
 
-    def _load_docs(key_pattern):
+    def _load_netconf_caches(is_lab=False):
 
-        for doc in _load_netconf_docs(config, key_pattern, use_next_redis):
+        interfaces = dict(_load_netconf_parsed_cache(config, 'netconf-interfaces',
+                                                     hostname, is_lab, use_next_redis))
+        interface_bundles = dict(_load_netconf_parsed_cache(config, 'netconf-interface-bundles',
+                                                            hostname, is_lab, use_next_redis))
 
-            for ifc in juniper.list_interfaces(doc['netconf']):
-                if not ifc['description']:
-                    continue
+        for key, ifc in interfaces.items():
+            router, interface_name = key.split('-----')
+            bundle = interface_bundles.get(key, [])
+            if not ifc['description']:
+                continue
 
-                yield {
-                    'router': doc['router'],
-                    'name': ifc['name'],
-                    'bundle': ifc['bundle'],
-                    'bundle-parents': [],
-                    'description': ifc['description'],
-                    'circuits': []
-                }
+            yield {
+                'router': router,
+                'name': interface_name,
+                'bundle': bundle,
+                'bundle-parents': [],
+                'description': ifc['description'],
+                'circuits': []
+            }
 
-    base_key_pattern = f'netconf:{hostname}*' if hostname else 'netconf:*'
-    yield from _load_docs(base_key_pattern)
+    yield from _load_netconf_caches()
     if not no_lab:
         logger.debug('lab')
-        yield from _load_docs(f'lab:{base_key_pattern}')
+        yield from _load_netconf_caches(is_lab=True)
 
 
 def _add_speeds(interfaces):
@@ -729,6 +750,7 @@ def _add_bundle_parents(interfaces, hostname=None):
     :param hostname: hostname or None for all
     :return: generator with bundle-parents populated in each element
     """
+
     def _get_base_name(name):
         return name.split('.')[0]