Merge branch 'feature/REPORTING-297-msr-mdvpn-endpoint' into 'develop'

Feature/reporting 297 msr mdvpn endpoint

See merge request !1

inventory_provider/config.py

@@ -184,11 +184,16 @@ CONFIG_SCHEMA = {
             'items': {'$ref': '#/definitions/gws-direct-nren-isp'}
         },
         'nren-asn-map': {
+            'type': 'array',
+            'items': {
                 'type': 'object',
-            'patternProperties': {
-                r'^\d+$': {'type': 'string'}
+                'properties': {
+                  'nren': {'type': 'string'},
+                  'asn': {'type': 'integer'}
                 },
+                'required': ['nren', 'asn'],
                 'additionalProperties': False
+            },
         }
     },
 

inventory_provider/routes/msr.py

@@ -59,6 +59,11 @@ These endpoints are intended for use by MSR.
 
 .. autofunction:: inventory_provider.routes.msr.bgp_all_peerings
 
+/msr/mdpvn
+--------------------------------------------
+
+.. autofunction:: inventory_provider.routes.msr.mdvpn
+
 /msr/services
 --------------------------------------------
 
@@ -256,6 +261,66 @@ SYSTEM_CORRELATION_SERVICES_LIST_SCHEMA = {
     'minItems': 1  # otherwise the route should return 404
 }
 
+MDVPN_LIST_SCHEMA = {
+    '$schema': 'http://json-schema.org/draft-07/schema#',
+    'definitions': {
+        'mdvpn_group': {
+            'type': 'object',
+            'properties': {
+                'asn': {'type': 'integer'},
+                'AP': {'$ref': '#/definitions/ap_peerings'},
+                'VRR': {'$ref': '#/definitions/vrr_peerings'}
+            },
+            'required': [
+                'asn', 'AP', 'VRR'
+            ],
+            'additionalProperties': False
+        },
+        'ap_peerings': {
+            'type': 'array',
+            'items': {
+                '$ref': '#/definitions/bgplu_peering'
+            }
+        },
+        'bgplu_peering': {
+            'type': 'object',
+            'properties': {
+                'name': {'type': 'string'},
+                'v4': {'type': 'string'},
+                'v6': {'type': 'string'},
+                'hostname': {'type': 'string'}
+            },
+            'required': [
+                'name', 'v4', 'v6', 'hostname'
+            ],
+            'additionalProperties': False
+        },
+        'vrr_peerings': {
+            'type': 'array',
+            'items': {
+                '$ref': '#/definitions/vpn_peering'
+            }
+        },
+        'vpn_peering': {
+            'type': 'object',
+            'properties': {
+                'description': {'type': 'string'},
+                'v4': {'type': 'string'},
+                'hostname': {
+                    'type': 'array',
+                    'items': {
+                        'type': 'string'
+                    },
+                    'minItems': 1
+                }
+            },
+            'additionalProperties': False
+        }
+    },
+    'type': 'array',
+    'items': {'$ref': '#/definitions/mdvpn_group'}
+}
+
 
 @routes.after_request
 def after_request(resp):
@@ -875,3 +940,116 @@ def bgp_all_peerings():
     r = common.get_current_redis()
     response = r.get('juniper-peerings:all')
     return Response(response.decode('utf-8'), mimetype="application/json")
+
+
+@routes.route('/mdvpn', methods=['GET', 'POST'])
+@common.require_accepts_json
+def mdvpn():
+    """
+    Handler for `/mdvpn`
+
+    This method returns a list of all BGP-LU peerings, and the VR peerings
+    for both Paris & Ljubljana.
+
+    The response will be formatted according to the following schema:
+
+    .. asjson::
+       inventory_provider.routes.msr.MDVPN_LIST_SCHEMA
+
+    :return:
+    """
+
+    def _get_consistent_description(description):
+        """
+        The same interface in VRR peerings can have multiple names.
+        These names are (currently) the same but with a different local prefix,
+        with no ordering guaranteed by the redis cache.
+        As only one description is returned by this endpoint for each
+        IPv4 address, this serves as a quick and dirty way of merging these
+        multiple descriptions into one an external user can use to identify
+        the peering reliably.
+
+        :param description: The raw description for a VRR peering
+        :return: The same description with location prefix removed
+        """
+        # it is incredibly likely this will need revision later down the line
+        expected_prefixes = [
+            "MD-VPN-VRR-PARIS-",
+            "MD-VPN-VRR-LJUBLJANA-"
+        ]
+        for prefix in expected_prefixes:
+            if description.startswith(prefix):
+                return description.replace(prefix, '')
+        return description
+
+    def _make_group_index(group, index_key):
+        """
+        Utility function to take a list and make it a dict based off a given
+        key, for fast lookup of a specific key field.
+
+        :param group: A list of dicts which should all have `index_key` as a
+        field
+        :param index_key: Name of the key to index on
+        :return: Dict with `index_key` as the key field and a list of all
+        matching dicts as the value
+        """
+        index = {}
+        for peering in group:
+            key = peering.get(index_key)
+            index.setdefault(key, []).append(peering)
+        return index
+
+    def _bgplu_peerings(asn, bgplu_index):
+        for peering in bgplu_index.get(asn, []):
+            formatted_peering = {
+                "name": peering['description'],
+                "v4": peering['address'],
+                "v6": '',
+                "hostname": peering['hostname']
+            }
+            yield formatted_peering
+
+    def _vpnrr_peerings(asn, vpnrr_index):
+        if asn in vpnrr_index:
+            vrr_peering_group = vpnrr_index[asn]
+            # rearrange into index using ipv4 as key
+            # this will collect related entries under the same ipv4
+            ip_index = _make_group_index(vrr_peering_group, 'address')
+            for ip in ip_index:
+                ip_details = ip_index[ip]  # a list of all info for given ipv4
+                hostnames = [item['hostname'] for item in ip_details]
+                description = ip_details[0]['description']
+
+                formatted_peering = {
+                    "description": _get_consistent_description(description),
+                    "v4": ip,
+                    "hostname": hostnames
+                }
+                yield formatted_peering
+
+    def _peerings_for_nren(asn, bgplu_index, vpnrr_index):
+        return {
+            "asn": asn,
+            "AP": list(_bgplu_peerings(asn, bgplu_index)),
+            "VRR": list(_vpnrr_peerings(asn, vpnrr_index))
+        }
+
+    r = common.get_current_redis()
+    cache_key = 'classifier-cache:msr:mdvpn'
+    response = _ignore_cache_or_retrieve(request, cache_key, r)
+    if not response:
+        bgplu = json.loads(
+            r.get('juniper-peerings:group:BGPLU').decode('utf-8'))
+        vpnrr = json.loads(
+            r.get('juniper-peerings:group:VPN-RR').decode('utf-8'))
+        bgplu_index = _make_group_index(bgplu, 'remote-asn')
+        vpnrr_index = _make_group_index(vpnrr, 'remote-asn')
+        config = current_app.config['INVENTORY_PROVIDER_CONFIG']
+        nren_asn_map = config['nren-asn-map']
+        nren_details = [
+            _peerings_for_nren(pair['asn'],
+                               bgplu_index,
+                               vpnrr_index)
+            for pair in nren_asn_map]
+        response = json.dumps(nren_details)
+    return Response(response, mimetype='application/json')

test/conftest.py

@@ -60,11 +60,20 @@ def data_config_filename():
                 }
             ],
             'gws-direct': {},
-            'nren-asn-map': {
-                "100": "BogusNREN",
-                "200": "FoobarNREN",
-                "300": "AlsoNET"
+            'nren-asn-map': [
+                {
+                    "nren": "FOO",
+                    "asn": 1930
+                },
+                {
+                    "nren": "BAR",
+                    "asn": 680
+                },
+                {
+                    "nren": "BAT",
+                    "asn": 2200
                 }
+            ]
         }
 
         with open(os.path.join(TEST_DATA_DIRNAME, 'gws-direct.json')) as gws:

test/test_msr_routes.py

@@ -5,7 +5,8 @@ import pytest
 
 from inventory_provider.routes.msr import PEERING_LIST_SCHEMA, \
     PEERING_GROUP_LIST_SCHEMA, PEERING_ADDRESS_SERVICES_LIST, \
-    SYSTEM_CORRELATION_SERVICES_LIST_SCHEMA, _get_services_for_address
+    SYSTEM_CORRELATION_SERVICES_LIST_SCHEMA, _get_services_for_address, \
+    MDVPN_LIST_SCHEMA
 from inventory_provider.routes.poller import SERVICES_LIST_SCHEMA
 from inventory_provider.tasks.common import _get_redis
 
@@ -321,3 +322,15 @@ def test_get_all_peerings(client):
     response_data = json.loads(rv.data.decode('utf-8'))
     jsonschema.validate(response_data, PEERING_LIST_SCHEMA)
     assert response_data  # test data is non-empty
+
+
+def test_get_mdvpn_peerings(client, mocked_redis):
+    rv = client.get(
+        '/msr/mdvpn',
+        headers=DEFAULT_REQUEST_HEADERS
+    )
+    assert rv.status_code == 200
+    assert rv.is_json
+    response_data = json.loads(rv.data.decode('utf-8'))
+    jsonschema.validate(response_data, MDVPN_LIST_SCHEMA)
+    assert response_data  # test data is non-empty