Finished release 0.92.

Changelog.md

@@ -2,6 +2,9 @@
 
 All notable changes to this project will be documented in this file.
 
+## [0.92] - 2022-08-12
+- REPORTING-312: Added NREN asn's to /msr/services response
+
 ## [0.91] - 2022-08-03
 - REPORTING-311: Added /msr/asn-peers endpoint
 

inventory_provider/routes/msr.py

@@ -260,7 +260,8 @@ SYSTEM_CORRELATION_SERVICES_LIST_SCHEMA = {
                 'service_type': {'type': 'string'},  # TODO: enum?
                 'project': {'type': 'string'},  # TODO: remove this?
                 'customer': {'type': 'string'},
-                'endpoints': {'$ref': '#/definitions/endpoints'}
+                'endpoints': {'$ref': '#/definitions/endpoints'},
+                'redundant_asn': {'type': 'integer'}
             },
             'required': [
                 'circuit_id', 'sid', 'name', 'speed', 'status', 'monitored',
@@ -881,6 +882,129 @@ def get_peering_services():
     return Response(response, mimetype="application/json")
 
 
+# TODO: @functools.cache is only available in py3.9
+@functools.lru_cache(maxsize=None)
+def _load_all_interfaces():
+    """
+    loads all ip interfaces in the network and returns as a dict
+    of dicts:
+      hostname -> interface name -> interface info
+
+    :return: dict of dicts
+    """
+    # dict of dicts:
+    #   peering_info[hostname][interface_name] = dict of ifc details
+    result = defaultdict(dict)
+
+    host_if_extraction_re = re.compile(r'^netconf-interfaces:(.+?):')
+    for doc in common.load_json_docs(
+            config_params=current_app.config['INVENTORY_PROVIDER_CONFIG'],
+            key_pattern='netconf-interfaces:*',
+            num_threads=20):
+        matches = host_if_extraction_re.match(doc['key'])
+        if matches:
+            hostname = matches[1]
+            interface_name = doc['value']['name']
+            result[hostname][interface_name] = doc['value']
+
+    return result
+
+
+# TODO: @functools.cache is only available in py3.9
+@functools.lru_cache(maxsize=None)
+def _load_redundant_access_peers():
+    """
+    load all peers that should be considered
+    redundant for access services
+
+    that is, all peers for services like NREN-APx
+
+    :return: dict of [peer address] -> [remote asn]
+    """
+    r = common.get_current_redis()
+    result = {}
+
+    # cf. REPORTING-312: limited to eGEANT group,
+    # but this can be expanded here in future
+    redundant_access_groups = ['eGEANT']
+    for g in redundant_access_groups:
+        doc = r.get(f'juniper-peerings:group:{g}')
+        for peer in json.loads(doc.decode('utf-8')):
+            result[peer['address']] = peer['remote-asn']
+
+    return result
+
+
+def _ip_endpoint_extractor(endpoint_details: dict):
+    """
+    special-purpose method used only by _endpoint_extractor
+
+    operates on a dictionary formatted as in worker.transform_ims_data
+    (cf sid_services)
+
+    WARNING: assumes hostname is a router in _load_all_interfaces,
+             asserts if not
+
+    :param endpoint_details: dict formatted as in worker.transform_ims_data
+    :return: dict formatted
+             as SYSTEM_CORRELATION_SERVICES_LIST_SCHEMA.ip-endpoint
+    """
+    hostname = ims_equipment_to_hostname(endpoint_details['equipment'])
+    interface = endpoint_details['port'].lower()
+
+    ip_endpoint = {
+        'hostname': hostname,
+        'interface': interface,
+    }
+
+    all_interfaces = _load_all_interfaces()
+    # sanity: should have already been checked
+    assert hostname in all_interfaces
+
+    host_info = all_interfaces[hostname]
+    interface_info = host_info.get(interface, {})
+
+    addresses = {}
+    ipv4 = interface_info.get('ipv4')
+    ipv6 = interface_info.get('ipv6')
+    if ipv4:
+        addresses['v4'] = ipv4[0]
+    if ipv6:
+        addresses['v6'] = ipv6[0]
+    if addresses:
+        ip_endpoint['addresses'] = addresses
+
+    return ip_endpoint
+
+
+def _endpoint_extractor(endpoint_details: Dict):
+    """
+    special-purpose method used only by get_system_correlation_services
+
+    operates on a dictionary formatted as in worker.transform_ims_data
+    (cf sid_services)
+
+    WARNING: assumes hostname is a router in _load_all_interfaces,
+             asserts if not
+
+    :param endpoint_details: dict formatted as in worker.transform_ims_data
+    :return: dict formatted as one element of the array
+             SYSTEM_CORRELATION_SERVICES_LIST_SCHEMA.endpoints
+    """
+    if not endpoint_details['geant_equipment']:
+        return
+    potential_hostname = ims_equipment_to_hostname(
+        endpoint_details['equipment'])
+    all_routers = _load_all_interfaces().keys()
+    if potential_hostname in all_routers:
+        return _ip_endpoint_extractor(endpoint_details)
+    else:
+        return {
+            'equipment': endpoint_details['equipment'],
+            'port': endpoint_details['port']
+        }
+
+
 @routes.route('/services', methods=['GET', 'POST'])
 @common.require_accepts_json
 def get_system_correlation_services():
@@ -900,63 +1024,26 @@ def get_system_correlation_services():
     :return:
     """
 
+    def _get_redundancy_asn(endpoints):
+        # endpoints should be a list formatted as
+        # SYSTEM_CORRELATION_SERVICES_LIST_SCHEMA.endpoints
+
+        redundant_peerings = _load_redundant_access_peers()
+        for ep in endpoints:
+            addresses = ep.get('addresses', {})
+            for ifc_address in addresses.values():
+                for p, asn in redundant_peerings.items():
+                    peer = ipaddress.ip_address(p)
+                    ifc = ipaddress.ip_interface(ifc_address)
+                    if peer in ifc.network:
+                        return asn
+        return None
+
     cache_key = 'classifier-cache:msr:services'
 
     r = common.get_current_redis()
     response = _ignore_cache_or_retrieve(request, cache_key, r)
     if not response:
-        peering_info = defaultdict(defaultdict)
-
-        key_pattern = 'netconf-interfaces:*'
-
-        host_if_extraction_re = re.compile(
-            r'^netconf-interfaces:(.+?):')
-        for doc in common.load_json_docs(
-                config_params=current_app.config['INVENTORY_PROVIDER_CONFIG'],
-                key_pattern=key_pattern,
-                num_threads=20):
-            matches = host_if_extraction_re.match(doc['key'])
-            if matches:
-                peering_info[matches[1]][doc['value']['name']] = doc['value']
-
-        def _ip_endpoint_extractor(endpoint_details: Dict):
-            hostname = ims_equipment_to_hostname(
-                endpoint_details['equipment'])
-            interface = endpoint_details['port'].lower()
-
-            ip_endpoint = {
-                'hostname': hostname,
-                'interface': interface,
-            }
-            addresses = {}
-            host_info = peering_info.get(hostname, {})
-            interface_info = host_info.get(interface, {})
-            ipv4 = interface_info.get('ipv4')
-            ipv6 = interface_info.get('ipv6')
-            if ipv4:
-                addresses['v4'] = ipv4[0]
-            if ipv6:
-                addresses['v6'] = ipv6[0]
-            if ipv4 or ipv6:
-                ip_endpoint['addresses'] = addresses
-
-            return ip_endpoint
-
-        def _optical_endpoint_extractor(endpoint_details: Dict):
-            return {
-                'equipment': endpoint_details['equipment'],
-                'port': endpoint_details['port']
-            }
-
-        def _endpoint_extractor(endpoint_details: Dict):
-            if not endpoint_details['geant_equipment']:
-                return
-            potential_hostname = ims_equipment_to_hostname(
-                    endpoint_details['equipment'])
-            if potential_hostname in peering_info.keys():
-                return _ip_endpoint_extractor(endpoint_details)
-            else:
-                return _optical_endpoint_extractor(endpoint_details)
 
         sid_services = json.loads(r.get('ims:sid_services').decode('utf-8'))
 
@@ -977,15 +1064,19 @@ def get_system_correlation_services():
                 endpoint = _endpoint_extractor(d)
                 if endpoint:
                     service_info['endpoints'].append(endpoint)
+
             if service_info.get('endpoints'):
+                asn = _get_redundancy_asn(service_info['endpoints'])
+                if asn:
+                    service_info['redundant_asn'] = asn
                 response.append(service_info)
 
-        jsonschema.validate(response,
-                            SYSTEM_CORRELATION_SERVICES_LIST_SCHEMA)
+        jsonschema.validate(
+            response, SYSTEM_CORRELATION_SERVICES_LIST_SCHEMA)
 
         if response:
             response = json.dumps(response, indent=2)
-            r.set(cache_key, response.encode('utf-8'))
+            # r.set(cache_key, response.encode('utf-8'))
 
     if not response:
         return Response(
@@ -1225,10 +1316,10 @@ def asn_peers(asn):
     This method returns a list of all peers filtered by `group` and `instance`,
     which can be passed either as URL query parameters or as entries in a
     POST request with a JSON body that matches this schema:
-    `{
-        "group": "group to filter by",
-        "instance": "instance to filter by"
-    }`
+        `{
+            "group": "group to filter by",
+            "instance": "instance to filter by"
+        }`
     Results are returned where all filters given are true, and exact string
     matches.
 

inventory_provider/tasks/worker.py

@@ -722,11 +722,24 @@ def ims_task(self, use_current=False):
 
 
 def extract_ims_data():
-
     c = InventoryTask.config["ims"]
+    return _extract_ims_data(
+        ims_api_url=c['api'],
+        ims_username=c['username'],
+        ims_password=c['password'])
+
+
+def _extract_ims_data(ims_api_url, ims_username, ims_password):
+    """
+    convenient entry point for testing ...
 
+    :param ims_api_url:
+    :param ims_username:
+    :param ims_password:
+    :return:
+    """
     def _ds() -> IMS:
-        return IMS(c['api'], c['username'], c['password'])
+        return IMS(ims_api_url, ims_username, ims_password)
 
     locations = {}
     lg_routers = []

setup.py

@@ -2,7 +2,7 @@ from setuptools import setup, find_packages
 
 setup(
     name='inventory-provider',
-    version="0.91",
+    version="0.92",
     author='GEANT',
     author_email='swd@geant.org',
     description='Dashboard inventory provider',