implemented msr/services endpoint

inventory_provider/routes/msr.py

@@ -77,6 +77,8 @@ import ipaddress
 import logging
 import re
 import threading
+from collections import defaultdict
+from typing import Dict
 
 from flask import Blueprint, Response, request, current_app
 import jsonschema
@@ -84,7 +86,8 @@ import jsonschema
 from inventory_provider.routes import common
 from inventory_provider.routes.classifier import \
     get_ims_equipment_name, get_ims_interface, get_interface_services_and_loc
-from inventory_provider.routes.common import _ignore_cache_or_retrieve
+from inventory_provider.routes.common import _ignore_cache_or_retrieve, \
+    ims_equipment_to_hostname
 from inventory_provider.routes.poller import get_services
 from inventory_provider.tasks import common as tasks_common
 
@@ -199,7 +202,7 @@ SYSTEM_CORRELATION_SERVICES_LIST_SCHEMA = {
                     'additionalProperties': False
                 }
             },
-            'required': ['hostname', 'interface', 'addresses'],
+            'required': ['hostname', 'interface'],
             'additionalProperties': False
         },
         'optical-endpoint': {
@@ -211,23 +214,16 @@ SYSTEM_CORRELATION_SERVICES_LIST_SCHEMA = {
             'required': ['equipment', 'port'],
             'additionalProperties': False
         },
-        'ip-endpoints': {
-            'type': 'array',
-            'items': {'$ref': '#/definitions/ip-endpoint'},
-            'minItems': 1
-        },
-        'optical-endpoints': {
+        'endpoints': {
             'type': 'array',
-            'items': {'$ref': '#/definitions/optical-endpoint'},
+            'items': {
+                'oneOf': [
+                    {'$ref': '#/definitions/ip-endpoint'},
+                    {'$ref': '#/definitions/optical-endpoint'}
+                ]
+            },
             'minItems': 1
         },
-
-        'endpoints': {
-            'oneOf': [
-                {'$ref': '#/definitions/ip-endpoints'},
-                {'$ref': '#/definitions/optical-endpoints'}
-            ]
-        },
         'service': {
             'type': 'object',
             'properties': {
@@ -248,7 +244,6 @@ SYSTEM_CORRELATION_SERVICES_LIST_SCHEMA = {
             'additionalProperties': False
         }
     },
-
     'type': 'array',
     'items': {'$ref': '#/definitions/service'},
     'minItems': 1  # otherwise the route should return 404
@@ -758,62 +753,98 @@ def get_system_correlation_services():
     :return:
     """
 
-    dummy_data = [
-        {
-            'circuit_id': 123,
-            'sid': 'AABBCC',
-            'name': 'BOGUS IP #1',
-            'speed': 1 << 20,
-            'service_type': 'GEANT IP',
-            'customer': 'SURF',
-            'endpoints': [
-                {
-                    'hostname': 'mx1.ams.nl.geant.net',
-                    'interface': 'xe-0/0/1',
-                    'addresses': {
-                        'v4': '10.0.0.1/30',
-                        'v6': '2620:0000:1cff:dead:beee:0000:0000:02d9/127'
-                    }
-                }
-            ]
-        },
-        {
-            'circuit_id': 234,
-            'sid': 'DDEEFF',
-            'name': 'BOGUS PLUS SERVICE',
-            'speed': 1 << 20,
-            'service_type': 'GEANT PLUS',
-            'customer': 'SOMEBODY',
-            'endpoints': [
-                {
-                    'hostname': 'mx1.ams.nl.geant.net',
-                    'interface': 'xe-0/0/2',
-                    'addresses': {
-                        'v4': '10.0.1.1/30',
-                        'v6': '2620:0000:1cff:dead:beef:0000:0000:02d9/127'
-                    }
-                },
-                {
-                    'hostname': 'mx1.lon.uk.geant.net',
-                    'interface': 'xe-4/3/2',
-                    'addresses': {
-                        'v4': '10.0.1.2/30',
-                        'v6': '2620:0000:1cff:dead:beef:0000:0000:02da/127'
-                    }
+    cache_key = 'classifier-cache:msr:services'
+
+    r = common.get_current_redis()
+    response = _ignore_cache_or_retrieve(request, cache_key, r)
+    if not response:
+        peering_info = defaultdict(defaultdict)
+
+        key_pattern = 'netconf-interfaces:*'
+
+        host_if_extraction_re = re.compile(
+            r'^netconf-interfaces:(.+?):')
+        for doc in common.load_json_docs(
+                config_params=current_app.config['INVENTORY_PROVIDER_CONFIG'],
+                key_pattern=key_pattern,
+                num_threads=20):
+            matches = host_if_extraction_re.match(doc['key'])
+            if matches:
+                peering_info[matches[1]][doc['value']['name']] = doc['value']
+
+        logical_interface_re = re.compile(r'.*\.\d+$')
+
+        def _ip_endpoint_extractor(endpoint_details: Dict):
+            if logical_interface_re.match(endpoint_details['port']):
+                hostname = ims_equipment_to_hostname(
+                    endpoint_details['equipment'])
+                interface = endpoint_details['port'].lower()
+
+                ip_endpoint = {
+                    'hostname': hostname,
+                    'interface': interface,
                 }
-            ]
-        },
-        {
-            'circuit_id': 123,
-            'sid': 'GGHHIIJJ',
-            'name': 'BOGUS LAMBDA SERVICE',
-            'speed': 1 << 20,
-            'service_type': 'GEANT LAMBDA',
-            'customer': 'JISC',
-            'endpoints': [
-                {'equipment': 'LON01-DTNX10-1', 'port': 'B-2-T7-1'}
-            ]
-        }
-    ]
+                addresses = {}
+                host_info = peering_info.get(hostname, {})
+                interface_info = host_info.get(interface, {})
+                ipv4 = interface_info.get('ipv4')
+                ipv6 = interface_info.get('ipv6')
+                if ipv4:
+                    addresses['v4'] = ipv4[0]
+                if ipv6:
+                    addresses['v6'] = ipv6[0]
+                if ipv4 or ipv6:
+                    ip_endpoint['addresses'] = addresses
+
+                return ip_endpoint
+
+        def _optical_endpoint_extractor(endpoint_details: Dict):
+            return {
+                'equipment': endpoint_details['equipment'],
+                'port': endpoint_details['port']
+            }
+
+        def _endpoint_extractor(endpoint_details: Dict):
+            if not endpoint_details['geant_equipment']:
+                return
+            potential_hostname = ims_equipment_to_hostname(
+                    endpoint_details['equipment'])
+            if potential_hostname in peering_info.keys():
+                return _ip_endpoint_extractor(endpoint_details)
+            else:
+                return _optical_endpoint_extractor(endpoint_details)
+
+        sid_services = json.loads(r.get('ims:sid_services').decode('utf-8'))
+
+        response = []
+        for sid, details in sid_services.items():
+            service_info = {'endpoints': []}
+            for d in details:
+                if not service_info.get('sid'):
+                    service_info['circuit_id'] = d['circuit_id']
+                    service_info['sid'] = d['sid']
+                    service_info['name'] = d['name']
+                    service_info['speed'] = d['speed']
+                    service_info['service_type'] = d['service_type']
+                    service_info['customer'] = d['customer']
+
+                endpoint = _endpoint_extractor(d)
+                if endpoint:
+                    service_info['endpoints'].append(endpoint)
+            if service_info.get('endpoints'):
+                response.append(service_info)
+
+        jsonschema.validate(response,
+                            SYSTEM_CORRELATION_SERVICES_LIST_SCHEMA)
 
-    return Response(json.dumps(dummy_data), mimetype="application/json")
+        if response:
+            response = json.dumps(response, indent=2)
+            r.set(cache_key, response.encode('utf-8'))
+
+    if not response:
+        return Response(
+            response='no services found',
+            status=404,
+            mimetype="text/html")
+
+    return Response(response, mimetype="application/json")