From b62914d169d8b3585d3ca05426c2da5714d9be8e Mon Sep 17 00:00:00 2001
From: Erik Reid <erik.reid@geant.org>
Date: Sat, 23 Jan 2021 11:39:02 +0100
Subject: [PATCH] precompute ix peering groups
---
inventory_provider/tasks/worker.py | 40 +++++++++++++++++++++++++++---
1 file changed, 36 insertions(+), 4 deletions(-)
diff --git a/inventory_provider/tasks/worker.py b/inventory_provider/tasks/worker.py
index 9830ca03..93efc5ea 100644
--- a/inventory_provider/tasks/worker.py
+++ b/inventory_provider/tasks/worker.py
@@ -723,10 +723,24 @@ def _build_subnet_db(update_callback=lambda s: None):
def _build_juniper_peering_db(update_callback=lambda s: None):
+ def _is_ix(peering_info):
+ if peering_info.get('instance', '') != 'IAS':
+ return False
+ if not peering_info.get('group', '').startswith('GEANT-IX'):
+ return False
+
+ expected_keys = ('description', 'local-asn', 'remote-asn')
+ if any(peering_info.get(x, None) is None for x in expected_keys):
+ logger.error('internal data error, looks like ix peering but'
+ f'some expected keys are missing: {peering_info}')
+ return False
+ return True
+
r = get_next_redis(InventoryTask.config)
update_callback('loading all juniper network peerings')
- peerings = {}
+ peerings_per_address = {}
+ ix_peerings = []
# scan with bigger batches, to mitigate network latency effects
key_prefix = 'juniper-peerings:hosts:'
@@ -737,13 +751,31 @@ def _build_juniper_peering_db(update_callback=lambda s: None):
host_peerings = json.loads(host_peerings)
for p in host_peerings:
p['hostname'] = hostname
- peerings.setdefault(p['address'], []).append(p)
+ peerings_per_address.setdefault(p['address'], []).append(p)
+ if _is_ix(p):
+ ix_peerings.append(p)
- update_callback(f'saving {len(peerings)} remote peers')
+
+ # sort ix peerings by group
+ ix_groups = {}
+ for p in ix_peerings:
+ description = p['description']
+ keyword = description.split(' ')[0] # regex needed??? (e.g. tabs???)
+ ix_groups.setdefault(keyword, set()).add(p['address'])
rp = r.pipeline()
- for k, v in peerings.items():
+
+ # create peering entries, keyed by remote addresses
+ update_callback(f'saving {len(peerings_per_address)} remote peers')
+ for k, v in peerings_per_address.items():
rp.set(f'juniper-peerings:remote:{k}', json.dumps(v))
+
+ # create ix group lists, keyed by group name
+ update_callback(f'saving {len(ix_groups)} remote ix peering groups')
+ for k, v in ix_groups.items():
+ group_addresses = list(v)
+ rp.set(f'juniper-peerings:ix-groups:{k}', json.dumps(group_addresses))
+
rp.execute()
--
GitLab