rewrote _load_interfaces to not parse the entire netconf docs

inventory_provider/routes/poller.py

@@ -670,6 +670,54 @@ def _load_netconf_docs(
         }
 
 
+def _get_filter_pattern(base_pattern, hostname=None, is_lab=False):
+    hostname_suffix = hostname if hostname else ''
+    lab_prefix = 'lab:' if is_lab else ''
+    return f'{lab_prefix}{base_pattern}:{hostname_suffix}*'
+
+
+def _load_netconf_interfaces(config, hostname=None, is_lab=False, use_next_redis=False):
+    filter_pattern = _get_filter_pattern('netconf-interfaces', hostname, is_lab)
+    for doc in common.load_json_docs(
+            config_params=config,
+            key_pattern=filter_pattern,
+            num_threads=20,
+            use_next_redis=use_next_redis):
+        m = re.match(r'^netconf-interfaces:(.+:.+)$', doc['key'])
+        if m:
+            key = m.group(1)
+            yield key, doc['value']
+
+
+# def _load_netconf_interface_hosts(config, hostname=None, is_lab=False, use_next_redis=False):
+#     filter_pattern = _get_filter_pattern('netconf-interfaces-hosts', hostname, is_lab)
+#     for doc in common.load_json_docs(
+#             config_params=config,
+#             key_pattern=filter_pattern,
+#             num_threads=20,
+#             use_next_redis=use_next_redis):
+#         m = re.match(r'^netconf-interfaces-hosts:(.+)$', doc['key'])
+#         if m:
+#             router = m.group(1)
+#             for host in doc['value']:
+#                 interface = host['interface name']
+#                 key = f'{router}:{interface}'
+#                 yield key, host
+
+
+def _load_netconf_interface_bundles(config, hostname=None, is_lab=False, use_next_redis=False):
+    filter_pattern = _get_filter_pattern('netconf-interface-bundles', hostname, is_lab)
+    for doc in common.load_json_docs(
+            config_params=config,
+            key_pattern=filter_pattern,
+            num_threads=20,
+            use_next_redis=use_next_redis):
+        m = re.match(r'^netconf-interface-bundles:(.+:.+)$', doc['key'])
+        if m:
+            key = m.group(1)
+            yield key, doc['value']
+
+
 def _load_interfaces(
         config, hostname=None, no_lab=False, use_next_redis=False):
     """
@@ -681,28 +729,31 @@ def _load_interfaces(
     :return:
     """
 
-    def _load_docs(key_pattern):
+    def _load_netconf_caches(is_lab=False):
 
-        for doc in _load_netconf_docs(config, key_pattern, use_next_redis):
+        interfaces = dict(_load_netconf_interfaces(config, hostname, is_lab, use_next_redis))
+        interface_bundles = dict(_load_netconf_interface_bundles(config, hostname, is_lab, use_next_redis))
+        # interface_hosts = dict(_load_netconf_interface_hosts(config, hostname, is_lab, use_next_redis))
 
-            for ifc in juniper.list_interfaces(doc['netconf']):
-                if not ifc['description']:
-                    continue
+        for key, ifc in interfaces.items():
+            router, interface_name = key.split(':')
+            bundle = interface_bundles[key]
+            if not ifc['description']:
+                continue
 
-                yield {
-                    'router': doc['router'],
-                    'name': ifc['name'],
-                    'bundle': ifc['bundle'],
-                    'bundle-parents': [],
-                    'description': ifc['description'],
-                    'circuits': []
-                }
+            yield {
+                'router': router,
+                'name': interface_name,
+                'bundle': bundle,
+                'bundle-parents': [],
+                'description': ifc['description'],
+                'circuits': []
+            }
 
-    base_key_pattern = f'netconf:{hostname}*' if hostname else 'netconf:*'
-    yield from _load_docs(base_key_pattern)
+    yield from _load_netconf_caches()
     if not no_lab:
         logger.debug('lab')
-        yield from _load_docs(f'lab:{base_key_pattern}')
+        yield from _load_netconf_caches(is_lab=True)
 
 
 def _add_speeds(interfaces):
@@ -729,6 +780,7 @@ def _add_bundle_parents(interfaces, hostname=None):
     :param hostname: hostname or None for all
     :return: generator with bundle-parents populated in each element
     """
+
     def _get_base_name(name):
         return name.split('.')[0]