diff --git a/inventory_provider/routes/poller.py b/inventory_provider/routes/poller.py
index 4c9d998600cc19ab25dc09ee9a36c5e93d93bf81..f5468a2add8901c09605d0fde810e71ea6558a42 100644
--- a/inventory_provider/routes/poller.py
+++ b/inventory_provider/routes/poller.py
@@ -134,17 +134,22 @@ def _load_snmp_indexes(hostname=None):
return result
-def _load_interface_bundles(hostname=None):
+def _load_interface_bundles(hostname=None, lab=False):
result = dict()
- key_pattern = f'netconf-interface-bundles:{hostname}:*' \
- if hostname else 'netconf-interface-bundles:*'
+
+ base_key = 'netconf-interface-bundles'
+ if lab:
+ base_key = f'lab:{base_key}'
+
+ key_pattern = f'{base_key}:{hostname}:*' \
+ if hostname else '{base_key}:*'
for doc in common.load_json_docs(
config_params=current_app.config['INVENTORY_PROVIDER_CONFIG'],
key_pattern=key_pattern,
num_threads=20):
- m = re.match(r'^netconf-interface-bundles:([^:]+):(.+)', doc['key'])
+ m = re.match(r'.*netconf-interface-bundles:([^:]+):(.+)', doc['key'])
assert m
router = m.group(1)
@@ -219,6 +224,8 @@ def _add_bundle_parents(interfaces, hostname=None):
:return: generator with bundle-parents populated in each element
"""
bundles = _load_interface_bundles(hostname)
+ bundles.update(_load_interface_bundles(hostname, lab=True))
+
for ifc in interfaces:
router_bundle = bundles.get(ifc['router'], None)
if router_bundle:
diff --git a/inventory_provider/tasks/worker.py b/inventory_provider/tasks/worker.py
index 7bd4f5252e98f7465a199d4995d95b7c1a55457b..64b09bf9e018babf52ed3c56e687e5d4d15ae500 100644
--- a/inventory_provider/tasks/worker.py
+++ b/inventory_provider/tasks/worker.py
@@ -151,7 +151,21 @@ def snmp_refresh_interfaces(self, hostname, community, logical_systems):
@app.task(base=InventoryTask, bind=True, name='netconf_refresh_config')
@log_task_entry_and_exit
-def netconf_refresh_config(self, hostname):
+def netconf_refresh_config(self, hostname, lab=False):
+ """
+ load netconf and save
+
+ save under 'lab:...' if lab is true
+
+ :param self:
+ :param hostname:
+ :param lab:
+ :return:
+ """
+
+ redis_key = f'netconf:{hostname}'
+ if lab:
+ redis_key = f'lab:{redis_key}'
try:
netconf_doc = juniper.load_config(
@@ -162,7 +176,7 @@ def netconf_refresh_config(self, hostname):
logger.exception(msg)
self.log_warning(msg)
r = get_current_redis(InventoryTask.config)
- netconf_str = r.get(f'netconf:{hostname}')
+ netconf_str = r.get(redis_key)
if not netconf_str:
raise InventoryTaskError(
f'netconf error with {hostname}'
@@ -170,7 +184,7 @@ def netconf_refresh_config(self, hostname):
self.log_warning(f'using cached netconf data for {hostname}')
r = get_next_redis(InventoryTask.config)
- r.set(f'netconf:{hostname}', netconf_str)
+ r.set(redis_key, netconf_str)
self.log_info(f'netconf info loaded from {hostname}')
@@ -371,15 +385,20 @@ def update_neteng_managed_device_list(self):
self.log_info(f'saved {len(routers)} managed routers')
-def load_netconf_data(hostname):
+def load_netconf_data(hostname, lab=False):
"""
this method should only be called from a task
:param hostname:
+ :param lab: True to look up under 'lab:...'
:return:
"""
+ redis_key = f'netconf:{hostname}'
+ if lab:
+ redis_key = f'lab:{redis_key}'
+
r = get_next_redis(InventoryTask.config)
- netconf = r.get('netconf:' + hostname)
+ netconf = r.get(redis_key)
if not netconf:
raise InventoryTaskError('no netconf data found for %r' % hostname)
return etree.fromstring(netconf.decode('utf-8'))
@@ -427,19 +446,37 @@ def refresh_juniper_bgp_peers(hostname, netconf):
@log_task_entry_and_exit
-def refresh_juniper_interface_list(hostname, netconf):
+def refresh_juniper_interface_list(hostname, netconf, lab=False):
+ """
+ load all interfaces from the netconf doc
+
+ save under 'lab:...' if lab is true
+
+ :param hostname:
+ :param netconf:
+ :param lab:
+ :return:
+ """
logger.debug(
'removing cached netconf-interfaces for %r' % hostname)
r = get_next_redis(InventoryTask.config)
+ interfaces_keybase = f'netconf-interfaces:{hostname}'
+ bundles_keybase = f'netconf-interface-bundles:{hostname}'
+ interfaces_all_key = f'netconf-interfaces-hosts:{hostname}'
+ if lab:
+ interfaces_keybase = f'lab:{interfaces_keybase}'
+ interfaces_all_key = f'lab:{interfaces_all_key}'
+ bundles_keybase = f'lab:{bundles_keybase}'
+
rp = r.pipeline()
- rp.delete(f'netconf-interfaces-hosts:{hostname}')
+ rp.delete(interfaces_all_key)
# scan with bigger batches, to mitigate network latency effects
- for k in r.scan_iter(f'netconf-interfaces:{hostname}:*', count=1000):
+ for k in r.scan_iter(f'{interfaces_keybase}:*', count=1000):
rp.delete(k)
for k in r.scan_iter(
- f'netconf-interface-bundles:{hostname}:*', count=1000):
+ f'{bundles_keybase}:*', count=1000):
rp.delete(k)
rp.execute()
@@ -448,7 +485,7 @@ def refresh_juniper_interface_list(hostname, netconf):
rp = r.pipeline()
rp.set(
- f'netconf-interfaces-hosts:{hostname}',
+ interfaces_all_key,
json.dumps(list(juniper.interface_addresses(netconf))))
for ifc in juniper.list_interfaces(netconf):
@@ -458,12 +495,12 @@ def refresh_juniper_interface_list(hostname, netconf):
if bundle:
all_bundles[bundle].append(ifc['name'])
rp.set(
- f'netconf-interfaces:{hostname}:{ifc["name"]}',
+ f'{interfaces_keybase}:{ifc["name"]}',
json.dumps(ifc))
for k, v in all_bundles.items():
rp.set(
- f'netconf-interface-bundles:{hostname}:{k}',
+ f'{bundles_keybase}:{k}',
json.dumps(v))
rp.execute()
@@ -475,11 +512,11 @@ def reload_lab_router_config(self, hostname):
self.log_info(f'loading netconf data for lab {hostname}')
# load new netconf data, in this thread
- netconf_refresh_config.apply(args=[hostname])
+ netconf_refresh_config.apply(args=[hostname, True])
- netconf_doc = load_netconf_data(hostname)
+ netconf_doc = load_netconf_data(hostname, lab=True)
- refresh_juniper_interface_list(hostname, netconf_doc)
+ refresh_juniper_interface_list(hostname, netconf_doc, lab=True)
# load snmp indexes
community = juniper.snmp_community_string(netconf_doc)
@@ -516,7 +553,7 @@ def reload_router_config(self, hostname):
pass # ok at this point if not found
# load new netconf data, in this thread
- netconf_refresh_config.apply(args=[hostname])
+ netconf_refresh_config.apply(args=[hostname, False])
netconf_doc = load_netconf_data(hostname)