diff --git a/inventory_provider/juniper.py b/inventory_provider/juniper.py
index 50ffc3d078cd0e6492d04f9724b6429b4b80e55f..740c258102bcd4f155834908c0c73e86717fe2f5 100644
--- a/inventory_provider/juniper.py
+++ b/inventory_provider/juniper.py
@@ -211,6 +211,7 @@ def list_interfaces(netconf_config):
for u in _units(name.text, i):
yield u
+
def all_bgp_peers(netconf_config):
def _peering_params(neighbor_node):
@@ -253,64 +254,64 @@ def all_bgp_peers(netconf_config):
yield peer
-def list_bgp_routes(netconf_config):
- for r in netconf_config.xpath(
- '//configuration/routing-instances/'
- 'instance[name/text()="IAS"]/protocols/bgp/'
- 'group[starts-with(name/text(), "GEANT-IX")]/'
- 'neighbor'):
- name = r.find('name')
- description = r.find('description')
- local_as = r.find('local-as')
- if local_as is not None:
- local_as = local_as.find('as-number')
- peer_as = r.find('peer-as')
- yield {
- 'name': name.text,
- 'description': description.text,
- 'as': {
- 'local': int(local_as.text),
- 'peer': int(peer_as.text)
- }
- }
-
-
-def ix_public_peers(netconf_config):
- for r in netconf_config.xpath(
- '//configuration/routing-instances/'
- 'instance[name/text()="IAS"]/protocols/bgp/'
- 'group[starts-with(name/text(), "GEANT-IX")]/'
- 'neighbor'):
- name = r.find('name')
- description = r.find('description')
- local_as = r.find('local-as')
- if local_as is not None:
- local_as = local_as.find('as-number')
- peer_as = r.find('peer-as')
- yield {
- 'name': ipaddress.ip_address(name.text).exploded,
- 'description': description.text,
- 'as': {
- 'local': int(local_as.text),
- 'peer': int(peer_as.text)
- }
- }
-
-
-def vpn_rr_peers(netconf_config):
- for r in netconf_config.xpath(
- '//configuration/logical-systems[name/text()="VRR"]/'
- '/protocols/bgp/'
- 'group[name/text()="VPN-RR" or name/text()="VPN-RR-INTERNAL"]/'
- 'neighbor'):
- neighbor = {
- 'name': ipaddress.ip_address(r.find('name').text).exploded,
- 'description': r.find('description').text,
- }
- peer_as = r.find('peer-as')
- if peer_as is not None:
- neighbor['peer-as'] = int(r.find('peer-as').text)
- yield neighbor
+# def list_bgp_routes(netconf_config):
+# for r in netconf_config.xpath(
+# '//configuration/routing-instances/'
+# 'instance[name/text()="IAS"]/protocols/bgp/'
+# 'group[starts-with(name/text(), "GEANT-IX")]/'
+# 'neighbor'):
+# name = r.find('name')
+# description = r.find('description')
+# local_as = r.find('local-as')
+# if local_as is not None:
+# local_as = local_as.find('as-number')
+# peer_as = r.find('peer-as')
+# yield {
+# 'name': name.text,
+# 'description': description.text,
+# 'as': {
+# 'local': int(local_as.text),
+# 'peer': int(peer_as.text)
+# }
+# }
+#
+#
+# def ix_public_peers(netconf_config):
+# for r in netconf_config.xpath(
+# '//configuration/routing-instances/'
+# 'instance[name/text()="IAS"]/protocols/bgp/'
+# 'group[starts-with(name/text(), "GEANT-IX")]/'
+# 'neighbor'):
+# name = r.find('name')
+# description = r.find('description')
+# local_as = r.find('local-as')
+# if local_as is not None:
+# local_as = local_as.find('as-number')
+# peer_as = r.find('peer-as')
+# yield {
+# 'name': ipaddress.ip_address(name.text).exploded,
+# 'description': description.text,
+# 'as': {
+# 'local': int(local_as.text),
+# 'peer': int(peer_as.text)
+# }
+# }
+#
+#
+# def vpn_rr_peers(netconf_config):
+# for r in netconf_config.xpath(
+# '//configuration/logical-systems[name/text()="VRR"]/'
+# '/protocols/bgp/'
+# 'group[name/text()="VPN-RR" or name/text()="VPN-RR-INTERNAL"]/'
+# 'neighbor'):
+# neighbor = {
+# 'name': ipaddress.ip_address(r.find('name').text).exploded,
+# 'description': r.find('description').text,
+# }
+# peer_as = r.find('peer-as')
+# if peer_as is not None:
+# neighbor['peer-as'] = int(r.find('peer-as').text)
+# yield neighbor
def interface_addresses(netconf_config):
diff --git a/inventory_provider/tasks/worker.py b/inventory_provider/tasks/worker.py
index f8b491e264a39ed4cdb8637339adb3722a8d7eb4..6be440bdaf38311651788f2a1f23a10e986d672c 100644
--- a/inventory_provider/tasks/worker.py
+++ b/inventory_provider/tasks/worker.py
@@ -434,54 +434,23 @@ def clear_cached_classifier_responses(hostname=None):
rp.execute()
-def _refresh_peers(hostname, key_base, peers):
- logger.debug(
- 'removing cached %s for %r' % (key_base, hostname))
- r = get_next_redis(InventoryTask.config)
- # WARNING (optimization): this is an expensive query if
- # the redis connection is slow, and we currently only
- # call this method during a full refresh
- # for k in r.scan_iter(key_base + ':*'):
- # # potential race condition: another proc could have
- # # delete this element between the time we read the
- # # keys and the next statement ... check for None below
- # value = r.get(k.decode('utf-8'))
- # if value:
- # value = json.loads(value.decode('utf-8'))
- # if value['router'] == hostname:
- # r.delete(k)
-
- rp = r.pipeline()
- for peer in peers:
- peer['router'] = hostname
- rp.set(
- '%s:%s' % (key_base, peer['name']),
- json.dumps(peer))
- rp.execute()
-
-
@log_task_entry_and_exit
-def refresh_ix_public_peers(hostname, netconf):
- _refresh_peers(
- hostname,
- 'ix_public_peer',
- juniper.ix_public_peers(netconf))
-
-
-@log_task_entry_and_exit
-def refresh_vpn_rr_peers(hostname, netconf):
- _refresh_peers(
- hostname,
- 'vpn_rr_peer',
- juniper.vpn_rr_peers(netconf))
+def refresh_juniper_bgp_peers(hostname, netconf):
+ host_peerings = juniper.all_bgp_peers(netconf)
+ r = get_next_redis(InventoryTask.config)
+ r.set(f'juniper-peerings:hosts:{hostname}', list(host_peerings))
@log_task_entry_and_exit
def refresh_interface_address_lookups(hostname, netconf):
- _refresh_peers(
- hostname,
- 'reverse_interface_addresses',
- juniper.interface_addresses(netconf))
+ r = get_next_redis(InventoryTask.config)
+ rp = r.pipeline()
+ for interface in juniper.interface_addresses(netconf):
+ interface['router'] = hostname
+ rp.set(
+ f'reverse_interface_addresses:{interface["name"]}',
+ json.dumps(interface))
+ rp.execute()
@log_task_entry_and_exit
@@ -492,10 +461,10 @@ def refresh_juniper_interface_list(hostname, netconf):
r = get_next_redis(InventoryTask.config)
rp = r.pipeline()
# scan with bigger batches, to mitigate network latency effects
- for k in r.scan_iter('netconf-interfaces:%s:*' % hostname, count=1000):
+ for k in r.scan_iter(f'netconf-interfaces:{hostname}:*', count=1000):
rp.delete(k)
for k in r.scan_iter(
- 'netconf-interface-bundles:%s:*' % hostname, count=1000):
+ f'netconf-interface-bundles:{hostname}:*', count=1000):
rp.delete(k)
rp.execute()
@@ -508,11 +477,11 @@ def refresh_juniper_interface_list(hostname, netconf):
if bundle:
all_bundles[bundle].append(ifc['name'])
rp.set(
- 'netconf-interfaces:%s:%s' % (hostname, ifc['name']),
+ f'netconf-interfaces:{hostname}:{ifc["name"]}',
json.dumps(ifc))
for k, v in all_bundles.items():
rp.set(
- 'netconf-interface-bundles:%s:%s' % (hostname, k),
+ f'netconf-interface-bundles:{hostname}:{k}',
json.dumps(v))
rp.execute()
@@ -551,8 +520,7 @@ def reload_router_config(self, hostname):
# clear cached classifier responses for this router, and
# refresh peering data
self.log_info(f'refreshing peers & clearing cache for {hostname}')
- refresh_ix_public_peers(hostname, netconf_doc)
- refresh_vpn_rr_peers(hostname, netconf_doc)
+ refresh_juniper_bgp_peers(hostname, netconf_doc)
refresh_interface_address_lookups(hostname, netconf_doc)
refresh_juniper_interface_list(hostname, netconf_doc)
# clear_cached_classifier_responses(hostname)
@@ -717,7 +685,8 @@ def refresh_finalizer(self, pending_task_ids_json):
_wait_for_tasks(task_ids, update_callback=self.log_info)
_build_subnet_db(update_callback=self.log_info)
- _build_peering_db(update_callback=self.log_info)
+ _build_snmp_peering_db(update_callback=self.log_info)
+ _build_juniper_peering_db(update_callback=self.log_info)
except (jsonschema.ValidationError,
json.JSONDecodeError,
@@ -752,11 +721,37 @@ def _build_subnet_db(update_callback=lambda s: None):
rp.execute()
-def _build_peering_db(update_callback=lambda s: None):
+def _build_juniper_peering_db(update_callback=lambda s: None):
+
+ r = get_next_redis(InventoryTask.config)
+
+ update_callback('loading all juniper network peerings')
+ peerings = {}
+
+ # scan with bigger batches, to mitigate network latency effects
+ key_prefix = 'juniper-peerings:hosts:'
+ for k in r.scan_iter(f'{key_prefix}*', count=1000):
+ key_name = k.decode('utf-8')
+ hostname = key_name[len(key_prefix):]
+ host_peerings = r.get(key_name).decode('utf-8')
+ host_peerings = json.loads(host_peerings)
+ for p in host_peerings:
+ p['hostname'] = hostname
+ peerings.setdefault(p['address'], []).append(p)
+
+ update_callback(f'saving {len(peerings)} remote peers')
+
+ rp = r.pipeline()
+ for k, v in peerings.items():
+ rp.set(f'juniper-peerings:remote:{k}', json.dumps(v))
+ rp.execute()
+
+
+def _build_snmp_peering_db(update_callback=lambda s: None):
r = get_next_redis(InventoryTask.config)
- update_callback('loading all network peerings')
+ update_callback('loading all snmp network peerings')
peerings = {}
# scan with bigger batches, to mitigate network latency effects