remove separate ix/vpnrr peering lists from redis

inventory_provider/juniper.py

@@ -211,6 +211,7 @@ def list_interfaces(netconf_config):
         for u in _units(name.text, i):
             yield u
 
+
 def all_bgp_peers(netconf_config):
 
     def _peering_params(neighbor_node):
@@ -253,64 +254,64 @@ def all_bgp_peers(netconf_config):
                 yield peer
 
 
-def list_bgp_routes(netconf_config):
-    for r in netconf_config.xpath(
-            '//configuration/routing-instances/'
-            'instance[name/text()="IAS"]/protocols/bgp/'
-            'group[starts-with(name/text(), "GEANT-IX")]/'
-            'neighbor'):
-        name = r.find('name')
-        description = r.find('description')
-        local_as = r.find('local-as')
-        if local_as is not None:
-            local_as = local_as.find('as-number')
-        peer_as = r.find('peer-as')
-        yield {
-            'name': name.text,
-            'description': description.text,
-            'as': {
-                'local': int(local_as.text),
-                'peer': int(peer_as.text)
-            }
-        }
-
-
-def ix_public_peers(netconf_config):
-    for r in netconf_config.xpath(
-            '//configuration/routing-instances/'
-            'instance[name/text()="IAS"]/protocols/bgp/'
-            'group[starts-with(name/text(), "GEANT-IX")]/'
-            'neighbor'):
-        name = r.find('name')
-        description = r.find('description')
-        local_as = r.find('local-as')
-        if local_as is not None:
-            local_as = local_as.find('as-number')
-        peer_as = r.find('peer-as')
-        yield {
-            'name': ipaddress.ip_address(name.text).exploded,
-            'description': description.text,
-            'as': {
-                'local': int(local_as.text),
-                'peer': int(peer_as.text)
-            }
-        }
-
-
-def vpn_rr_peers(netconf_config):
-    for r in netconf_config.xpath(
-            '//configuration/logical-systems[name/text()="VRR"]/'
-            '/protocols/bgp/'
-            'group[name/text()="VPN-RR" or name/text()="VPN-RR-INTERNAL"]/'
-            'neighbor'):
-        neighbor = {
-            'name': ipaddress.ip_address(r.find('name').text).exploded,
-            'description': r.find('description').text,
-        }
-        peer_as = r.find('peer-as')
-        if peer_as is not None:
-            neighbor['peer-as'] = int(r.find('peer-as').text)
-        yield neighbor
+# def list_bgp_routes(netconf_config):
+#     for r in netconf_config.xpath(
+#             '//configuration/routing-instances/'
+#             'instance[name/text()="IAS"]/protocols/bgp/'
+#             'group[starts-with(name/text(), "GEANT-IX")]/'
+#             'neighbor'):
+#         name = r.find('name')
+#         description = r.find('description')
+#         local_as = r.find('local-as')
+#         if local_as is not None:
+#             local_as = local_as.find('as-number')
+#         peer_as = r.find('peer-as')
+#         yield {
+#             'name': name.text,
+#             'description': description.text,
+#             'as': {
+#                 'local': int(local_as.text),
+#                 'peer': int(peer_as.text)
+#             }
+#         }
+#
+#
+# def ix_public_peers(netconf_config):
+#     for r in netconf_config.xpath(
+#             '//configuration/routing-instances/'
+#             'instance[name/text()="IAS"]/protocols/bgp/'
+#             'group[starts-with(name/text(), "GEANT-IX")]/'
+#             'neighbor'):
+#         name = r.find('name')
+#         description = r.find('description')
+#         local_as = r.find('local-as')
+#         if local_as is not None:
+#             local_as = local_as.find('as-number')
+#         peer_as = r.find('peer-as')
+#         yield {
+#             'name': ipaddress.ip_address(name.text).exploded,
+#             'description': description.text,
+#             'as': {
+#                 'local': int(local_as.text),
+#                 'peer': int(peer_as.text)
+#             }
+#         }
+#
+#
+# def vpn_rr_peers(netconf_config):
+#     for r in netconf_config.xpath(
+#             '//configuration/logical-systems[name/text()="VRR"]/'
+#             '/protocols/bgp/'
+#             'group[name/text()="VPN-RR" or name/text()="VPN-RR-INTERNAL"]/'
+#             'neighbor'):
+#         neighbor = {
+#             'name': ipaddress.ip_address(r.find('name').text).exploded,
+#             'description': r.find('description').text,
+#         }
+#         peer_as = r.find('peer-as')
+#         if peer_as is not None:
+#             neighbor['peer-as'] = int(r.find('peer-as').text)
+#         yield neighbor
 
 
 def interface_addresses(netconf_config):

inventory_provider/tasks/worker.py

@@ -434,54 +434,23 @@ def clear_cached_classifier_responses(hostname=None):
     rp.execute()
 
 
-def _refresh_peers(hostname, key_base, peers):
-    logger.debug(
-        'removing cached %s for %r' % (key_base, hostname))
-    r = get_next_redis(InventoryTask.config)
-    # WARNING (optimization): this is an expensive query if
-    #       the redis connection is slow, and we currently only
-    #       call this method during a full refresh
-    # for k in r.scan_iter(key_base + ':*'):
-    #     # potential race condition: another proc could have
-    #     # delete this element between the time we read the
-    #     # keys and the next statement ... check for None below
-    #     value = r.get(k.decode('utf-8'))
-    #     if value:
-    #         value = json.loads(value.decode('utf-8'))
-    #         if value['router'] == hostname:
-    #             r.delete(k)
-
-    rp = r.pipeline()
-    for peer in peers:
-        peer['router'] = hostname
-        rp.set(
-            '%s:%s' % (key_base, peer['name']),
-            json.dumps(peer))
-    rp.execute()
-
-
 @log_task_entry_and_exit
-def refresh_ix_public_peers(hostname, netconf):
-    _refresh_peers(
-        hostname,
-        'ix_public_peer',
-        juniper.ix_public_peers(netconf))
-
-
-@log_task_entry_and_exit
-def refresh_vpn_rr_peers(hostname, netconf):
-    _refresh_peers(
-        hostname,
-        'vpn_rr_peer',
-        juniper.vpn_rr_peers(netconf))
+def refresh_juniper_bgp_peers(hostname, netconf):
+    host_peerings = juniper.all_bgp_peers(netconf)
+    r = get_next_redis(InventoryTask.config)
+    r.set(f'juniper-peerings:hosts:{hostname}', list(host_peerings))
 
 
 @log_task_entry_and_exit
 def refresh_interface_address_lookups(hostname, netconf):
-    _refresh_peers(
-        hostname,
-        'reverse_interface_addresses',
-        juniper.interface_addresses(netconf))
+    r = get_next_redis(InventoryTask.config)
+    rp = r.pipeline()
+    for interface in juniper.interface_addresses(netconf):
+        interface['router'] = hostname
+        rp.set(
+            f'reverse_interface_addresses:{interface["name"]}',
+            json.dumps(interface))
+    rp.execute()
 
 
 @log_task_entry_and_exit
@@ -492,10 +461,10 @@ def refresh_juniper_interface_list(hostname, netconf):
     r = get_next_redis(InventoryTask.config)
     rp = r.pipeline()
     # scan with bigger batches, to mitigate network latency effects
-    for k in r.scan_iter('netconf-interfaces:%s:*' % hostname, count=1000):
+    for k in r.scan_iter(f'netconf-interfaces:{hostname}:*', count=1000):
         rp.delete(k)
     for k in r.scan_iter(
-            'netconf-interface-bundles:%s:*' % hostname, count=1000):
+            f'netconf-interface-bundles:{hostname}:*', count=1000):
         rp.delete(k)
     rp.execute()
 
@@ -508,11 +477,11 @@ def refresh_juniper_interface_list(hostname, netconf):
             if bundle:
                 all_bundles[bundle].append(ifc['name'])
         rp.set(
-            'netconf-interfaces:%s:%s' % (hostname, ifc['name']),
+            f'netconf-interfaces:{hostname}:{ifc["name"]}',
             json.dumps(ifc))
     for k, v in all_bundles.items():
         rp.set(
-            'netconf-interface-bundles:%s:%s' % (hostname, k),
+            f'netconf-interface-bundles:{hostname}:{k}',
             json.dumps(v))
     rp.execute()
 
@@ -551,8 +520,7 @@ def reload_router_config(self, hostname):
     # clear cached classifier responses for this router, and
     # refresh peering data
     self.log_info(f'refreshing peers & clearing cache for {hostname}')
-    refresh_ix_public_peers(hostname, netconf_doc)
-    refresh_vpn_rr_peers(hostname, netconf_doc)
+    refresh_juniper_bgp_peers(hostname, netconf_doc)
     refresh_interface_address_lookups(hostname, netconf_doc)
     refresh_juniper_interface_list(hostname, netconf_doc)
     # clear_cached_classifier_responses(hostname)
@@ -717,7 +685,8 @@ def refresh_finalizer(self, pending_task_ids_json):
 
         _wait_for_tasks(task_ids, update_callback=self.log_info)
         _build_subnet_db(update_callback=self.log_info)
-        _build_peering_db(update_callback=self.log_info)
+        _build_snmp_peering_db(update_callback=self.log_info)
+        _build_juniper_peering_db(update_callback=self.log_info)
 
     except (jsonschema.ValidationError,
             json.JSONDecodeError,
@@ -752,11 +721,37 @@ def _build_subnet_db(update_callback=lambda s: None):
     rp.execute()
 
 
-def _build_peering_db(update_callback=lambda s: None):
+def _build_juniper_peering_db(update_callback=lambda s: None):
+
+    r = get_next_redis(InventoryTask.config)
+
+    update_callback('loading all juniper network peerings')
+    peerings = {}
+
+    # scan with bigger batches, to mitigate network latency effects
+    key_prefix = 'juniper-peerings:hosts:'
+    for k in r.scan_iter(f'{key_prefix}*', count=1000):
+        key_name = k.decode('utf-8')
+        hostname = key_name[len(key_prefix):]
+        host_peerings = r.get(key_name).decode('utf-8')
+        host_peerings = json.loads(host_peerings)
+        for p in host_peerings:
+            p['hostname'] = hostname
+            peerings.setdefault(p['address'], []).append(p)
+
+    update_callback(f'saving {len(peerings)} remote peers')
+
+    rp = r.pipeline()
+    for k, v in peerings.items():
+        rp.set(f'juniper-peerings:remote:{k}', json.dumps(v))
+    rp.execute()
+
+
+def _build_snmp_peering_db(update_callback=lambda s: None):
 
     r = get_next_redis(InventoryTask.config)
 
-    update_callback('loading all network peerings')
+    update_callback('loading all snmp network peerings')
     peerings = {}
 
     # scan with bigger batches, to mitigate network latency effects