precompute ix peering groups

inventory_provider/tasks/worker.py

@@ -723,10 +723,24 @@ def _build_subnet_db(update_callback=lambda s: None):
 
 def _build_juniper_peering_db(update_callback=lambda s: None):
 
+    def _is_ix(peering_info):
+        if peering_info.get('instance', '') != 'IAS':
+            return False
+        if not peering_info.get('group', '').startswith('GEANT-IX'):
+            return False
+
+        expected_keys = ('description', 'local-asn', 'remote-asn')
+        if any(peering_info.get(x, None) is None for x in expected_keys):
+            logger.error('internal data error, looks like ix peering but'
+                         f'some expected keys are missing: {peering_info}')
+            return False
+        return True
+
     r = get_next_redis(InventoryTask.config)
 
     update_callback('loading all juniper network peerings')
-    peerings = {}
+    peerings_per_address = {}
+    ix_peerings = []
 
     # scan with bigger batches, to mitigate network latency effects
     key_prefix = 'juniper-peerings:hosts:'
@@ -737,13 +751,31 @@ def _build_juniper_peering_db(update_callback=lambda s: None):
         host_peerings = json.loads(host_peerings)
         for p in host_peerings:
             p['hostname'] = hostname
-            peerings.setdefault(p['address'], []).append(p)
+            peerings_per_address.setdefault(p['address'], []).append(p)
+            if _is_ix(p):
+                ix_peerings.append(p)
 
-    update_callback(f'saving {len(peerings)} remote peers')
+
+    # sort ix peerings by group
+    ix_groups = {}
+    for p in ix_peerings:
+        description = p['description']
+        keyword = description.split(' ')[0]  # regex needed??? (e.g. tabs???)
+        ix_groups.setdefault(keyword, set()).add(p['address'])
 
     rp = r.pipeline()
-    for k, v in peerings.items():
+
+    # create peering entries, keyed by remote addresses
+    update_callback(f'saving {len(peerings_per_address)} remote peers')
+    for k, v in peerings_per_address.items():
         rp.set(f'juniper-peerings:remote:{k}', json.dumps(v))
+
+    # create ix group lists, keyed by group name
+    update_callback(f'saving {len(ix_groups)} remote ix peering groups')
+    for k, v in ix_groups.items():
+        group_addresses = list(v)
+        rp.set(f'juniper-peerings:ix-groups:{k}', json.dumps(group_addresses))
+
     rp.execute()