From 0b0c766fd38f2e1b7d57d27256e4c8aa69874d9e Mon Sep 17 00:00:00 2001
From: Erik Reid <erik.reid@geant.org>
Date: Tue, 19 Jan 2021 15:50:02 +0100
Subject: [PATCH] store & return all peerings for a remote
---
inventory_provider/routes/classifier.py | 12 ++++----
inventory_provider/tasks/worker.py | 40 ++++++++++++++++++-------
2 files changed, 36 insertions(+), 16 deletions(-)
diff --git a/inventory_provider/routes/classifier.py b/inventory_provider/routes/classifier.py
index ad3b5db0..9e9b4845 100644
--- a/inventory_provider/routes/classifier.py
+++ b/inventory_provider/routes/classifier.py
@@ -384,13 +384,15 @@ def peer_info(address):
_location_from_service_dict(s) for s in i['services']]
snmp_info = r.get(
- f'snmp-peerings:{address}')
+ f'snmp-peerings:remote:{address}')
if snmp_info:
snmp_info = json.loads(snmp_info.decode('utf-8'))
- result['snmp'] = {
- 'community': snmp_info['community'],
- 'oid': snmp_info['oid']
- }
+ result['snmp'] = [
+ {
+ 'hostname': h['hostname'],
+ 'community': h['community'],
+ 'oid': h['oid']
+ } for h in snmp_info]
result['locations'] = _remove_duplicates_from_list(result['locations'])
result = json.dumps(result)
diff --git a/inventory_provider/tasks/worker.py b/inventory_provider/tasks/worker.py
index 9c877975..f8b491e2 100644
--- a/inventory_provider/tasks/worker.py
+++ b/inventory_provider/tasks/worker.py
@@ -96,7 +96,7 @@ def snmp_refresh_peerings(self, hostname, community, logical_systems):
logger.exception(msg)
self.log_warning(msg)
r = get_current_redis(InventoryTask.config)
- peerings = r.get(f'snmp-peerings:{hostname}:all')
+ peerings = r.get(f'snmp-peerings:hosts:{hostname}')
if peerings is None:
raise InventoryTaskError(
f'snmp error with {peerings}'
@@ -106,16 +106,7 @@ def snmp_refresh_peerings(self, hostname, community, logical_systems):
self.log_warning(f'using cached snmp peering data for {hostname}')
r = get_next_redis(InventoryTask.config)
-
- rp = r.pipeline()
- rp.set(f'snmp-peerings:{hostname}:all', json.dumps(peerings))
-
- for session in peerings:
- rp.set(
- f'snmp-peerings:{hostname}:{session["remote"]}',
- json.dumps(session))
-
- rp.execute()
+ r.set(f'snmp-peerings:hosts:{hostname}', json.dumps(peerings))
self.log_info(f'snmp peering info loaded from {hostname}')
@@ -726,6 +717,7 @@ def refresh_finalizer(self, pending_task_ids_json):
_wait_for_tasks(task_ids, update_callback=self.log_info)
_build_subnet_db(update_callback=self.log_info)
+ _build_peering_db(update_callback=self.log_info)
except (jsonschema.ValidationError,
json.JSONDecodeError,
@@ -760,6 +752,32 @@ def _build_subnet_db(update_callback=lambda s: None):
rp.execute()
+def _build_peering_db(update_callback=lambda s: None):
+
+ r = get_next_redis(InventoryTask.config)
+
+ update_callback('loading all network peerings')
+ peerings = {}
+
+ # scan with bigger batches, to mitigate network latency effects
+ key_prefix = 'snmp-peerings:hosts:'
+ for k in r.scan_iter(f'{key_prefix}*', count=1000):
+ key_name = k.decode('utf-8')
+ hostname = key_name[len(key_prefix):]
+ host_peerings = r.get(key_name).decode('utf-8')
+ host_peerings = json.loads(host_peerings)
+ for p in host_peerings:
+ p['hostname'] = hostname
+ peerings.setdefault(p['remote'], []).append(p)
+
+ update_callback(f'saving {len(peerings)} remote peers')
+
+ rp = r.pipeline()
+ for k, v in peerings.items():
+ rp.set(f'snmp-peerings:remote:{k}', json.dumps(v))
+ rp.execute()
+
+
def check_task_status(task_id, parent=None, forget=False):
r = AsyncResult(task_id, app=app)
assert r.id == task_id # sanity
--
GitLab