keep lab netconf data separate

inventory_provider/routes/poller.py

@@ -134,17 +134,22 @@ def _load_snmp_indexes(hostname=None):
     return result
 
 
-def _load_interface_bundles(hostname=None):
+def _load_interface_bundles(hostname=None, lab=False):
     result = dict()
-    key_pattern = f'netconf-interface-bundles:{hostname}:*' \
-        if hostname else 'netconf-interface-bundles:*'
+
+    base_key = 'netconf-interface-bundles'
+    if lab:
+        base_key = f'lab:{base_key}'
+
+    key_pattern = f'{base_key}:{hostname}:*' \
+        if hostname else '{base_key}:*'
 
     for doc in common.load_json_docs(
             config_params=current_app.config['INVENTORY_PROVIDER_CONFIG'],
             key_pattern=key_pattern,
             num_threads=20):
 
-        m = re.match(r'^netconf-interface-bundles:([^:]+):(.+)', doc['key'])
+        m = re.match(r'.*netconf-interface-bundles:([^:]+):(.+)', doc['key'])
         assert m
 
         router = m.group(1)
@@ -219,6 +224,8 @@ def _add_bundle_parents(interfaces, hostname=None):
     :return: generator with bundle-parents populated in each element
     """
     bundles = _load_interface_bundles(hostname)
+    bundles.update(_load_interface_bundles(hostname, lab=True))
+
     for ifc in interfaces:
         router_bundle = bundles.get(ifc['router'], None)
         if router_bundle:

inventory_provider/tasks/worker.py

@@ -151,7 +151,21 @@ def snmp_refresh_interfaces(self, hostname, community, logical_systems):
 
 @app.task(base=InventoryTask, bind=True, name='netconf_refresh_config')
 @log_task_entry_and_exit
-def netconf_refresh_config(self, hostname):
+def netconf_refresh_config(self, hostname, lab=False):
+    """
+    load netconf and save
+
+    save under 'lab:...' if lab is true
+
+    :param self:
+    :param hostname:
+    :param lab:
+    :return:
+    """
+
+    redis_key = f'netconf:{hostname}'
+    if lab:
+        redis_key = f'lab:{redis_key}'
 
     try:
         netconf_doc = juniper.load_config(
@@ -162,7 +176,7 @@ def netconf_refresh_config(self, hostname):
         logger.exception(msg)
         self.log_warning(msg)
         r = get_current_redis(InventoryTask.config)
-        netconf_str = r.get(f'netconf:{hostname}')
+        netconf_str = r.get(redis_key)
         if not netconf_str:
             raise InventoryTaskError(
                 f'netconf error with {hostname}'
@@ -170,7 +184,7 @@ def netconf_refresh_config(self, hostname):
         self.log_warning(f'using cached netconf data for {hostname}')
 
     r = get_next_redis(InventoryTask.config)
-    r.set(f'netconf:{hostname}', netconf_str)
+    r.set(redis_key, netconf_str)
     self.log_info(f'netconf info loaded from {hostname}')
 
 
@@ -371,15 +385,20 @@ def update_neteng_managed_device_list(self):
     self.log_info(f'saved {len(routers)} managed routers')
 
 
-def load_netconf_data(hostname):
+def load_netconf_data(hostname, lab=False):
     """
     this method should only be called from a task
 
     :param hostname:
+    :param lab: True to look up under 'lab:...'
     :return:
     """
+    redis_key = f'netconf:{hostname}'
+    if lab:
+        redis_key = f'lab:{redis_key}'
+
     r = get_next_redis(InventoryTask.config)
-    netconf = r.get('netconf:' + hostname)
+    netconf = r.get(redis_key)
     if not netconf:
         raise InventoryTaskError('no netconf data found for %r' % hostname)
     return etree.fromstring(netconf.decode('utf-8'))
@@ -427,19 +446,37 @@ def refresh_juniper_bgp_peers(hostname, netconf):
 
 
 @log_task_entry_and_exit
-def refresh_juniper_interface_list(hostname, netconf):
+def refresh_juniper_interface_list(hostname, netconf, lab=False):
+    """
+    load all interfaces from the netconf doc
+
+    save under 'lab:...' if lab is true
+
+    :param hostname:
+    :param netconf:
+    :param lab:
+    :return:
+    """
     logger.debug(
         'removing cached netconf-interfaces for %r' % hostname)
 
     r = get_next_redis(InventoryTask.config)
 
+    interfaces_keybase = f'netconf-interfaces:{hostname}'
+    bundles_keybase = f'netconf-interface-bundles:{hostname}'
+    interfaces_all_key = f'netconf-interfaces-hosts:{hostname}'
+    if lab:
+        interfaces_keybase = f'lab:{interfaces_keybase}'
+        interfaces_all_key = f'lab:{interfaces_all_key}'
+        bundles_keybase = f'lab:{bundles_keybase}'
+
     rp = r.pipeline()
-    rp.delete(f'netconf-interfaces-hosts:{hostname}')
+    rp.delete(interfaces_all_key)
     # scan with bigger batches, to mitigate network latency effects
-    for k in r.scan_iter(f'netconf-interfaces:{hostname}:*', count=1000):
+    for k in r.scan_iter(f'{interfaces_keybase}:*', count=1000):
         rp.delete(k)
     for k in r.scan_iter(
-            f'netconf-interface-bundles:{hostname}:*', count=1000):
+            f'{bundles_keybase}:*', count=1000):
         rp.delete(k)
     rp.execute()
 
@@ -448,7 +485,7 @@ def refresh_juniper_interface_list(hostname, netconf):
     rp = r.pipeline()
 
     rp.set(
-        f'netconf-interfaces-hosts:{hostname}',
+        interfaces_all_key,
         json.dumps(list(juniper.interface_addresses(netconf))))
 
     for ifc in juniper.list_interfaces(netconf):
@@ -458,12 +495,12 @@ def refresh_juniper_interface_list(hostname, netconf):
             if bundle:
                 all_bundles[bundle].append(ifc['name'])
         rp.set(
-            f'netconf-interfaces:{hostname}:{ifc["name"]}',
+            f'{interfaces_keybase}:{ifc["name"]}',
             json.dumps(ifc))
 
     for k, v in all_bundles.items():
         rp.set(
-            f'netconf-interface-bundles:{hostname}:{k}',
+            f'{bundles_keybase}:{k}',
             json.dumps(v))
 
     rp.execute()
@@ -475,11 +512,11 @@ def reload_lab_router_config(self, hostname):
     self.log_info(f'loading netconf data for lab {hostname}')
 
     # load new netconf data, in this thread
-    netconf_refresh_config.apply(args=[hostname])
+    netconf_refresh_config.apply(args=[hostname, True])
 
-    netconf_doc = load_netconf_data(hostname)
+    netconf_doc = load_netconf_data(hostname, lab=True)
 
-    refresh_juniper_interface_list(hostname, netconf_doc)
+    refresh_juniper_interface_list(hostname, netconf_doc, lab=True)
 
     # load snmp indexes
     community = juniper.snmp_community_string(netconf_doc)
@@ -516,7 +553,7 @@ def reload_router_config(self, hostname):
         pass  # ok at this point if not found
 
     # load new netconf data, in this thread
-    netconf_refresh_config.apply(args=[hostname])
+    netconf_refresh_config.apply(args=[hostname, False])
 
     netconf_doc = load_netconf_data(hostname)