Finished release 0.90.

Changelog.md

@@ -2,6 +2,13 @@
 
 All notable changes to this project will be documented in this file.
 
+## [0.90] - 2022-07-19
+- REPORTING-297: Add /msr/mdvpn endpoint
+- REPORTING-307: Add /msr/vpn-proxy endpoint
+
+## [0.89] - 2022-07-01
+- REPORTING-306: Add NREN/ASN map to inventory provider configuration
+
 ## [0.88] - 2022-06-22
 - DBOARD3-596: Only include contacts with populated mail value
 - LG-46: Showing 'in-service' routers only

inventory_provider/config.py

@@ -184,11 +184,16 @@ CONFIG_SCHEMA = {
             'items': {'$ref': '#/definitions/gws-direct-nren-isp'}
         },
         'nren-asn-map': {
-            'type': 'object',
-            'patternProperties': {
-                r'^\d+$': {'type': 'string'}
+            'type': 'array',
+            'items': {
+                'type': 'object',
+                'properties': {
+                  'nren': {'type': 'string'},
+                  'asn': {'type': 'integer'}
+                },
+                'required': ['nren', 'asn'],
+                'additionalProperties': False
             },
-            'additionalProperties': False
         }
     },
 
@@ -221,7 +226,6 @@ CONFIG_SCHEMA = {
     'oneOf': [
         {
             'required': [
-                'ops-db',
                 'ssh',
                 'redis',
                 'redis-databases',
@@ -232,7 +236,6 @@ CONFIG_SCHEMA = {
         },
         {
             'required': [
-                'ops-db',
                 'ssh',
                 'sentinel',
                 'redis-databases',

inventory_provider/routes/msr.py

@@ -59,12 +59,24 @@ These endpoints are intended for use by MSR.
 
 .. autofunction:: inventory_provider.routes.msr.bgp_all_peerings
 
+/msr/mdpvn
+--------------------------------------------
+
+.. autofunction:: inventory_provider.routes.msr.mdvpn
+
+
 /msr/services
 --------------------------------------------
 
 .. autofunction:: inventory_provider.routes.msr.get_system_correlation_services
 
 
+/msr/vpn-proxy
+--------------------------------------------
+
+.. autofunction:: inventory_provider.routes.msr.vpn-proxy
+
+
 helpers
 -------------------------------------
 
@@ -256,6 +268,94 @@ SYSTEM_CORRELATION_SERVICES_LIST_SCHEMA = {
     'minItems': 1  # otherwise the route should return 404
 }
 
+MDVPN_LIST_SCHEMA = {
+    '$schema': 'http://json-schema.org/draft-07/schema#',
+    'definitions': {
+        'mdvpn_group': {
+            'type': 'object',
+            'properties': {
+                'asn': {'type': 'integer'},
+                'AP': {'$ref': '#/definitions/ap_peerings'},
+                'VRR': {'$ref': '#/definitions/vrr_peerings'}
+            },
+            'required': [
+                'asn', 'AP', 'VRR'
+            ],
+            'additionalProperties': False
+        },
+        'ap_peerings': {
+            'type': 'array',
+            'items': {
+                '$ref': '#/definitions/bgplu_peering'
+            }
+        },
+        'bgplu_peering': {
+            'type': 'object',
+            'properties': {
+                'name': {'type': 'string'},
+                'v4': {'type': 'string'},
+                'v6': {'type': 'string'},
+                'hostname': {'type': 'string'}
+            },
+            'required': [
+                'name', 'v4', 'v6', 'hostname'
+            ],
+            'additionalProperties': False
+        },
+        'vrr_peerings': {
+            'type': 'array',
+            'items': {
+                '$ref': '#/definitions/vpn_peering'
+            }
+        },
+        'vpn_peering': {
+            'type': 'object',
+            'properties': {
+                'description': {'type': 'string'},
+                'v4': {'type': 'string'},
+                'hostname': {
+                    'type': 'array',
+                    'items': {
+                        'type': 'string'
+                    },
+                    'minItems': 1
+                }
+            },
+            'additionalProperties': False
+        }
+    },
+    'type': 'array',
+    'items': {'$ref': '#/definitions/mdvpn_group'}
+}
+
+VPN_PROXY_LIST_SCHEMA = {
+    '$schema': 'http://json-schema.org/draft-07/schema#',
+    'definitions': {
+        'vpn_proxy_peering': {
+            'type': 'object',
+            'properties': {
+                'pop': {'type': 'string'},
+                'nren': {'type': 'string'},
+                'group': {'type': 'string'},
+                'v4': {'type': 'string'}
+            },
+            'additionalProperties': False
+        }
+    },
+    'type': 'array',
+    'items': {'$ref': '#/definitions/vpn_proxy_peering'}
+}
+
+DOMAIN_TO_POP_MAPPING = {
+    "mad.es": "Madrid",
+    "bra.sk": "Bratislava",
+    "vie.at": "Vienna",
+    "gen.ch": "Geneva",
+    "fra.de": "Frankfurt",
+    "pra.cz": "Prague",
+    "ams.nl": "Amsterdam"
+}
+
 
 @routes.after_request
 def after_request(resp):
@@ -875,3 +975,201 @@ def bgp_all_peerings():
     r = common.get_current_redis()
     response = r.get('juniper-peerings:all')
     return Response(response.decode('utf-8'), mimetype="application/json")
+
+
+@routes.route('/mdvpn', methods=['GET', 'POST'])
+@common.require_accepts_json
+def mdvpn():
+    """
+    Handler for `/mdvpn`
+
+    This method returns a list of all BGP-LU peerings, and the VR peerings
+    for both Paris & Ljubljana.
+
+    The response will be formatted according to the following schema:
+
+    .. asjson::
+       inventory_provider.routes.msr.MDVPN_LIST_SCHEMA
+
+    :return:
+    """
+
+    def _get_consistent_description(description):
+        """
+        The same interface in VRR peerings can have multiple names.
+        These names are (currently) the same but with a different local prefix,
+        with no ordering guaranteed by the redis cache.
+        As only one description is returned by this endpoint for each
+        IPv4 address, this serves as a quick and dirty way of merging these
+        multiple descriptions into one an external user can use to identify
+        the peering reliably.
+
+        :param description: The raw description for a VRR peering
+        :return: The same description with location prefix removed
+        """
+        # it is incredibly likely this will need revision later down the line
+        expected_prefixes = [
+            "MD-VPN-VRR-PARIS-",
+            "MD-VPN-VRR-LJUBLJANA-"
+        ]
+        for prefix in expected_prefixes:
+            if description.startswith(prefix):
+                return description.replace(prefix, '')
+        return description
+
+    def _make_group_index(group, index_key):
+        """
+        Utility function to take a list and make it a dict based off a given
+        key, for fast lookup of a specific key field.
+
+        :param group: A list of dicts which should all have `index_key` as a
+        field
+        :param index_key: Name of the key to index on
+        :return: Dict with `index_key` as the key field and a list of all
+        matching dicts as the value
+        """
+        index = {}
+        for peering in group:
+            key = peering.get(index_key)
+            index.setdefault(key, []).append(peering)
+        return index
+
+    def _bgplu_peerings(asn, bgplu_index):
+        for peering in bgplu_index.get(asn, []):
+            formatted_peering = {
+                "name": peering['description'],
+                "v4": peering['address'],
+                "v6": '',
+                "hostname": peering['hostname']
+            }
+            yield formatted_peering
+
+    def _vpnrr_peerings(asn, vpnrr_index):
+        # rearrange into index using ipv4 as key
+        # this will collect related entries under the same ipv4
+        ip_index = _make_group_index(vpnrr_index.get(asn, []), 'address')
+        for ip, ip_details in ip_index.items():
+            hostnames = [item['hostname'] for item in ip_details]
+            description = ip_details[0]['description']
+
+            formatted_peering = {
+                "description": _get_consistent_description(description),
+                "v4": ip,
+                "hostname": hostnames
+            }
+            yield formatted_peering
+
+    def _peerings_for_nren(asn, bgplu_index, vpnrr_index):
+        return {
+            "asn": asn,
+            "AP": list(_bgplu_peerings(asn, bgplu_index)),
+            "VRR": list(_vpnrr_peerings(asn, vpnrr_index))
+        }
+
+    r = common.get_current_redis()
+    cache_key = 'classifier-cache:msr:mdvpn'
+    response = _ignore_cache_or_retrieve(request, cache_key, r)
+    if not response:
+        bgplu = json.loads(
+            r.get('juniper-peerings:group:BGPLU').decode('utf-8'))
+        vpnrr = json.loads(
+            r.get('juniper-peerings:group:VPN-RR').decode('utf-8'))
+        bgplu_index = _make_group_index(bgplu, 'remote-asn')
+        vpnrr_index = _make_group_index(vpnrr, 'remote-asn')
+        config = current_app.config['INVENTORY_PROVIDER_CONFIG']
+        nren_asn_map = config['nren-asn-map']
+        nren_details = [
+            _peerings_for_nren(pair['asn'],
+                               bgplu_index,
+                               vpnrr_index)
+            for pair in nren_asn_map]
+        response = json.dumps(nren_details)
+    return Response(response, mimetype='application/json')
+
+
+@routes.route('/vpn-proxy', methods=['GET', 'POST'])
+@common.require_accepts_json
+def vpn_proxy():
+    """
+        Handler for `/vpn-proxy`
+
+        This method returns a list of all L3VPN related VPN proxy peerings.
+
+        The response will be formatted according to the following schema:
+
+        .. asjson::
+           inventory_provider.routes.msr.VPN_PROXY_LIST_SCHEMA
+
+        :return:
+        """
+
+    def _is_relevant(item):
+        """
+        Determine if a given peering in the VPN-PROXY logical system is
+        relevant to this endpoint (whether it's related to L3VPN)
+        :param item: peering dict
+        :return: True if the peering is L3VPN relevant, False otherwise
+        """
+        desc = item.get("description")
+        if desc is None:
+            return False
+        return "L3VPN" in desc
+
+    def _look_up_city_from_hostname(hostname):
+        """
+        Get the city name for a peering from a partial hostname match.
+        This uses a hardcoded lookup table.
+        :param hostname: hostname for the peering
+        :return: city name if found, "Unknown" otherwise
+        """
+        for snippet in DOMAIN_TO_POP_MAPPING:
+            if snippet in hostname:
+                return DOMAIN_TO_POP_MAPPING[snippet]
+        return "Unknown"
+
+    def _extract_nren_from_description(desc, group):
+        """
+        Retrieve the relevant NREN from the peering description and group.
+        This approach is, by its nature, very fragile to any changes to
+        descriptions, and should be revisited when that happens.
+
+        :param desc: description of a VPN-Proxy peering
+        :param group: group of the same VPN-Proxy peering
+        :return: name of the NREN
+        """
+        if group == "PRACE":
+            # common trait: the NREN is the first word in the description
+            return desc.split(' ')[0]
+        else:
+            # only other group is XiFi, and only CESNet is relevant
+            return 'CESNet'
+
+    def _format_peerings(vpnproxy):
+        """
+        Generator that iterates through a list of peering dicts, yielding
+        appropriately reformatted peerings if they are relevant to L3VPN.
+        :param vpnproxy: list of peering dicts taken from current redis
+        :return: generator of reformated peerings
+        """
+        for peering in vpnproxy:
+            if _is_relevant(peering):
+                desc = peering["description"]
+                group = peering["group"]
+                hostname = peering["hostname"]
+                formatted_peering = {
+                    "pop": _look_up_city_from_hostname(hostname),
+                    "nren": _extract_nren_from_description(desc, group),
+                    "group": group,
+                    "v4": peering.get("address")
+                }
+                yield formatted_peering
+
+    r = common.get_current_redis()
+    cache_key = 'classifier-cache:msr:vpn-proxy'
+    response = _ignore_cache_or_retrieve(request, cache_key, r)
+    if not response:
+        vpnproxy = json.loads(
+            r.get('juniper-peerings:logical-system:VPN-PROXY').decode('utf-8'))
+        peerings = list(_format_peerings(vpnproxy))
+        response = json.dumps(peerings)
+    return Response(response, mimetype='application/json')

setup.py

@@ -2,7 +2,7 @@ from setuptools import setup, find_packages
 
 setup(
     name='inventory-provider',
-    version="0.89",
+    version="0.90",
     author='GEANT',
     author_email='swd@geant.org',
     description='Dashboard inventory provider',

test/conftest.py

@@ -28,12 +28,6 @@ def data_config_filename():
 
     with tempfile.NamedTemporaryFile() as f:
         config = {
-            "ops-db": {
-                "hostname": "xxxxxxx.yyyyy.zzz",
-                "dbname": "xxxxxx",
-                "username": "xxxxxx",
-                "password": "xxxxxxxx"
-            },
             "ssh": {
                 "username": "uSeR-NaMe",
                 "private-key": "private-key-filename",
@@ -66,11 +60,20 @@ def data_config_filename():
                 }
             ],
             'gws-direct': {},
-            'nren-asn-map': {
-                "100": "BogusNREN",
-                "200": "FoobarNREN",
-                "300": "AlsoNET"
-            }
+            'nren-asn-map': [
+                {
+                    "nren": "FOO",
+                    "asn": 1930
+                },
+                {
+                    "nren": "BAR",
+                    "asn": 680
+                },
+                {
+                    "nren": "BAT",
+                    "asn": 2200
+                }
+            ]
         }
 
         with open(os.path.join(TEST_DATA_DIRNAME, 'gws-direct.json')) as gws:

test/test_msr_routes.py

@@ -5,7 +5,8 @@ import pytest
 
 from inventory_provider.routes.msr import PEERING_LIST_SCHEMA, \
     PEERING_GROUP_LIST_SCHEMA, PEERING_ADDRESS_SERVICES_LIST, \
-    SYSTEM_CORRELATION_SERVICES_LIST_SCHEMA, _get_services_for_address
+    SYSTEM_CORRELATION_SERVICES_LIST_SCHEMA, _get_services_for_address, \
+    MDVPN_LIST_SCHEMA, VPN_PROXY_LIST_SCHEMA
 from inventory_provider.routes.poller import SERVICES_LIST_SCHEMA
 from inventory_provider.tasks.common import _get_redis
 
@@ -321,3 +322,27 @@ def test_get_all_peerings(client):
     response_data = json.loads(rv.data.decode('utf-8'))
     jsonschema.validate(response_data, PEERING_LIST_SCHEMA)
     assert response_data  # test data is non-empty
+
+
+def test_get_mdvpn_peerings(client, mocked_redis):
+    rv = client.get(
+        '/msr/mdvpn',
+        headers=DEFAULT_REQUEST_HEADERS
+    )
+    assert rv.status_code == 200
+    assert rv.is_json
+    response_data = json.loads(rv.data.decode('utf-8'))
+    jsonschema.validate(response_data, MDVPN_LIST_SCHEMA)
+    assert response_data  # test data is non-empty
+
+
+def test_get_vpn_proxy_peerings(client, mocked_redis):
+    rv = client.get(
+        '/msr/vpn-proxy',
+        headers=DEFAULT_REQUEST_HEADERS
+    )
+    assert rv.status_code == 200
+    assert rv.is_json
+    response_data = json.loads(rv.data.decode('utf-8'))
+    jsonschema.validate(response_data, VPN_PROXY_LIST_SCHEMA)
+    assert response_data  # test data is non-empty