Merge branch 'feature/REPORTING-297-msr-mdvpn-endpoint' into...

Merge branch 'feature/REPORTING-297-msr-mdvpn-endpoint' into feature/REPORTING-307-msr-vpn-proxy-endpoint

Changelog.md

@@ -2,6 +2,9 @@
 
 All notable changes to this project will be documented in this file.
 
+## [0.89] - 2022-07-01
+- REPORTING-306: Add NREN/ASN map to inventory provider configuration
+
 ## [0.88] - 2022-06-22
 - DBOARD3-596: Only include contacts with populated mail value
 - LG-46: Showing 'in-service' routers only

inventory_provider/config.py

@@ -184,10 +184,16 @@ CONFIG_SCHEMA = {
             'items': {'$ref': '#/definitions/gws-direct-nren-isp'}
         },
         'nren-asn-map': {
-            'type': 'object',
-            'patternProperties': {
-                r'^\d+$': {'type': 'string'}
-            }
+            'type': 'array',
+            'items': {
+                'type': 'object',
+                'properties': {
+                  'nren': {'type': 'string'},
+                  'asn': {'type': 'integer'}
+                },
+                'required': ['nren', 'asn'],
+                'additionalProperties': False
+            },
         }
     },
 

inventory_provider/routes/msr.py

@@ -982,12 +982,6 @@ def mdvpn():
                 return description.replace(prefix, '')
         return description
 
-    # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-    # REVIEW COMMENT ONLY, DO NOT MERGE:
-    # _make_group_index is extremely, extremely generic,
-    # but is it worth extracting, will anything else use it?
-    # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-
     def _make_group_index(group, index_key):
         """
         Utility function to take a list and make it a dict based off a given
@@ -1002,66 +996,65 @@ def mdvpn():
         index = {}
         for peering in group:
             key = peering.get(index_key)
-            if key in index:
-                peering_list = index[key]
-                peering_list.append(peering)
-            else:
-                index[key] = [peering]
+            index.setdefault(key, []).append(peering)
         return index
 
     def _bgplu_peerings(asn, bgplu_index):
-        peerings = []
-        if asn in bgplu_index:
-            for peering in bgplu_index[asn]:
-                formatted_peering = {
-                    "name": peering['description'],
-                    "v4": peering['address'],
-                    "v6": '',
-                    "hostname": peering['hostname']
-                }
-                peerings.append(formatted_peering)
-        return peerings
+        if asn not in bgplu_index:
+            yield False
+        for peering in bgplu_index[asn]:
+            formatted_peering = {
+                "name": peering['description'],
+                "v4": peering['address'],
+                "v6": '',
+                "hostname": peering['hostname']
+            }
+            yield formatted_peering
 
     def _vpnrr_peerings(asn, vpnrr_index):
-        peerings = []
-        if asn in vpnrr_index:
-            vrr_peering_group = vpnrr_index[asn]
-            # rearrange into index using ipv4 as key
-            # this will collect related entries under the same ipv4
-            ip_index = _make_group_index(vrr_peering_group, 'address')
-            for ip in ip_index:
-                ip_details = ip_index[ip]  # a list of all info for given ipv4
-                hostnames = [item['hostname'] for item in ip_details]
-                description = ip_details[0]['description']
-
-                formatted_peering = {
-                    "description": _get_consistent_description(description),
-                    "v4": ip,
-                    "hostname": hostnames
-                }
-                peerings.append(formatted_peering)
-        return peerings
+        if asn not in vpnrr_index:
+            yield False
+        vrr_peering_group = vpnrr_index[asn]
+        # rearrange into index using ipv4 as key
+        # this will collect related entries under the same ipv4
+        ip_index = _make_group_index(vrr_peering_group, 'address')
+        for ip in ip_index:
+            ip_details = ip_index[ip]  # a list of all info for given ipv4
+            hostnames = [item['hostname'] for item in ip_details]
+            description = ip_details[0]['description']
+
+            formatted_peering = {
+                "description": _get_consistent_description(description),
+                "v4": ip,
+                "hostname": hostnames
+            }
+            yield formatted_peering
 
     def _peerings_for_nren(asn, bgplu_index, vpnrr_index):
         return {
             "asn": asn,
-            "AP": _bgplu_peerings(asn, bgplu_index),
-            "VRR": _vpnrr_peerings(asn, vpnrr_index)
+            "AP": list(filter(None, _bgplu_peerings(asn, bgplu_index))),
+            "VRR": list(filter(None, _vpnrr_peerings(asn, vpnrr_index)))
         }
 
     r = common.get_current_redis()
-    bgplu = json.loads(r.get('juniper-peerings:group:BGPLU').decode('utf-8'))
-    vpnrr = json.loads(r.get('juniper-peerings:group:VPN-RR').decode('utf-8'))
-    bgplu_index = _make_group_index(bgplu, 'remote-asn')
-    vpnrr_index = _make_group_index(vpnrr, 'remote-asn')
-    config = current_app.config['INVENTORY_PROVIDER_CONFIG']
-    nren_asn_map = config['nren-asn-map']
-    nren_details = [
-        _peerings_for_nren(int(asn),
-                           bgplu_index,
-                           vpnrr_index)
-        for asn in nren_asn_map]
-    response = json.dumps(nren_details)
+    cache_key = 'classifier-cache:msr:mdvpn'
+    response = _ignore_cache_or_retrieve(request, cache_key, r)
+    if not response:
+        bgplu = json.loads(
+            r.get('juniper-peerings:group:BGPLU').decode('utf-8'))
+        vpnrr = json.loads(
+            r.get('juniper-peerings:group:VPN-RR').decode('utf-8'))
+        bgplu_index = _make_group_index(bgplu, 'remote-asn')
+        vpnrr_index = _make_group_index(vpnrr, 'remote-asn')
+        config = current_app.config['INVENTORY_PROVIDER_CONFIG']
+        nren_asn_map = config['nren-asn-map']
+        nren_details = [
+            _peerings_for_nren(pair['asn'],
+                               bgplu_index,
+                               vpnrr_index)
+            for pair in nren_asn_map]
+        response = json.dumps(nren_details)
     return Response(response, mimetype='application/json')
 
 

setup.py

@@ -2,7 +2,7 @@ from setuptools import setup, find_packages
 
 setup(
     name='inventory-provider',
-    version="0.89",
+    version="0.90",
     author='GEANT',
     author_email='swd@geant.org',
     description='Dashboard inventory provider',

test/conftest.py

@@ -66,11 +66,20 @@ def data_config_filename():
                 }
             ],
             'gws-direct': {},
-            'nren-asn-map': {
-                "1930": "FOO",
-                "680": "BAR",
-                "2200": "BAT"
-            }
+            'nren-asn-map': [
+                {
+                    "nren": "FOO",
+                    "asn": 1930
+                },
+                {
+                    "nren": "BAR",
+                    "asn": 680
+                },
+                {
+                    "nren": "BAT",
+                    "asn": 2200
+                }
+            ]
         }
 
         with open(os.path.join(TEST_DATA_DIRNAME, 'gws-direct.json')) as gws: