diff --git a/inventory_provider/tasks/worker.py b/inventory_provider/tasks/worker.py index 9830ca0316e005dfe9600d172949f8aa1c07d605..93efc5ea61eb72bf96a6953b52e7bddcb238055c 100644 --- a/inventory_provider/tasks/worker.py +++ b/inventory_provider/tasks/worker.py @@ -723,10 +723,24 @@ def _build_subnet_db(update_callback=lambda s: None): def _build_juniper_peering_db(update_callback=lambda s: None): + def _is_ix(peering_info): + if peering_info.get('instance', '') != 'IAS': + return False + if not peering_info.get('group', '').startswith('GEANT-IX'): + return False + + expected_keys = ('description', 'local-asn', 'remote-asn') + if any(peering_info.get(x, None) is None for x in expected_keys): + logger.error('internal data error, looks like ix peering but' + f'some expected keys are missing: {peering_info}') + return False + return True + r = get_next_redis(InventoryTask.config) update_callback('loading all juniper network peerings') - peerings = {} + peerings_per_address = {} + ix_peerings = [] # scan with bigger batches, to mitigate network latency effects key_prefix = 'juniper-peerings:hosts:' @@ -737,13 +751,31 @@ def _build_juniper_peering_db(update_callback=lambda s: None): host_peerings = json.loads(host_peerings) for p in host_peerings: p['hostname'] = hostname - peerings.setdefault(p['address'], []).append(p) + peerings_per_address.setdefault(p['address'], []).append(p) + if _is_ix(p): + ix_peerings.append(p) - update_callback(f'saving {len(peerings)} remote peers') + + # sort ix peerings by group + ix_groups = {} + for p in ix_peerings: + description = p['description'] + keyword = description.split(' ')[0] # regex needed??? (e.g. tabs???) + ix_groups.setdefault(keyword, set()).add(p['address']) rp = r.pipeline() - for k, v in peerings.items(): + + # create peering entries, keyed by remote addresses + update_callback(f'saving {len(peerings_per_address)} remote peers') + for k, v in peerings_per_address.items(): rp.set(f'juniper-peerings:remote:{k}', json.dumps(v)) + + # create ix group lists, keyed by group name + update_callback(f'saving {len(ix_groups)} remote ix peering groups') + for k, v in ix_groups.items(): + group_addresses = list(v) + rp.set(f'juniper-peerings:ix-groups:{k}', json.dumps(group_addresses)) + rp.execute()