initial skeleton

inventory_provider/routes/msr.py

@@ -83,6 +83,11 @@ These endpoints are intended for use by MSR.
 .. autofunction:: inventory_provider.routes.msr.asn_peers
 
 
+/msr/ip-services
+--------------------------------------------
+
+.. autofunction:: inventory_provider.routes.msr.ip_services
+
 helpers
 -------------------------------------
 
@@ -363,6 +368,46 @@ DOMAIN_TO_POP_MAPPING = {
     "ams.nl": "Amsterdam"
 }
 
+IP_SERVICES_LIST_SCHEMA = {
+    '$schema': 'http://json-schema.org/draft-07/schema#',
+    'definitions': {
+        'service': {
+            'type': 'object',
+            'properties': {
+                'name': {'type': 'string'},
+                'customer': {'type': 'string'},
+                'type': {'type': 'string'},
+                'pop': {
+                    'type': 'object',
+                    'properties': {
+                        'name': {'type': 'string'},
+                        'abbrev': {'type': 'string'},
+                    },
+                    'required': ['name', 'abbrev'],
+                    'additionalProperties': False
+                }
+            },
+            'required': ['name', 'customer', 'type', 'pop'],
+            'additionalProperties': False
+        },
+        'interface-address': {
+            'properties': {
+                'hostname': {'type': 'string', 'format': 'hostname'},
+                'port': {'type': 'string'},
+                'address': {'type': 'string'},
+                'services': {
+                    'type': 'array',
+                    'items': {'$ref': '#/definitions/service'}
+                }
+            },
+            'required': ['hostname', 'port', 'address', 'services'],
+            'additionalProperties': False
+        }
+    },
+    'type': 'array',
+    'items': {'$ref': '#/definitions/interface-address'}
+}
+
 # very similar to PEERING_LIST_SCHEMA but
 # with a field for NREN, which is required
 ASN_PEER_LIST_SCHEMA = {
@@ -1389,3 +1434,114 @@ def asn_peers(asn):
         r.set(cache_key, response.encode('utf-8'))
 
     return Response(response, mimetype='application/json')
+
+
+def _dedupe(iter_of_objs):
+    """
+    remove duplicates from the input iterable
+
+    the elements of the input iterable must be json-serializable
+
+    :param iter_of_objs: iterable
+    :return: a new iterable containing unique elements from the original
+    """
+    iter_of_json = (json.dumps(_o, sort_keys=True) for _o in iter_of_objs)
+    return (json.loads(_s) for _s in set(iter_of_json))
+
+
+def _load_ip_services():
+    """
+    yields items that
+    :return:
+    """
+
+    ims_interface_services = {}
+    equip_port_re = re.compile(r'^ims:interface_services:([^:]+):(.+)')
+    for doc in common.load_json_docs(
+            config_params=current_app.config['INVENTORY_PROVIDER_CONFIG'],
+            key_pattern='ims:interface_services:*',
+            num_threads=40):
+
+        m = equip_port_re.match(doc['key'])
+        assert m  # sanity
+        equipment = m.group(1)
+        port = m.group(2)
+        assert all([_s['equipment'] == equipment for _s in doc['value']])  # sanity
+        assert all([_s['port'] == port for _s in doc['value']])  # sanity
+        hostname = ims_equipment_to_hostname(equipment)
+        ims_interface_services[f'{hostname}:{port.lower()}'] = doc['value']
+
+    netconf_interfaces = {}
+    host_if_extraction_re = re.compile(r'^netconf-interfaces:(.+?):')
+    for doc in common.load_json_docs(
+            config_params=current_app.config['INVENTORY_PROVIDER_CONFIG'],
+            key_pattern='netconf-interfaces:*',
+            num_threads=20):
+        matches = host_if_extraction_re.match(doc['key'])
+        if matches:
+            hostname = matches[1]
+            doc['value']['hostname'] = hostname
+            port = doc['value']['name']
+            netconf_interfaces[f'{hostname}:{port}'] = doc['value']
+
+    def _service_info(_s):
+        return {
+            'name': _s['name'],
+            'customer': _s['customer'],
+            'type': _s['service_type'],
+            'pop': {
+                'name': _s['pop_name'],
+                'abbrev': _s['pop_abbreviation']
+            }
+        }
+
+    def _merged_result():
+        for key, ifc in netconf_interfaces.items():
+            # for _s in map(_service_info, ims_interface_services.get(key, [])):
+            _services1 = list(map(_service_info, ims_interface_services.get(key, [])))
+            _services = list(_dedupe(_services1))
+            if len(_services) != len(_services1):
+                print('here')
+            if len(_services) > 1:
+                print('here')
+
+            for address in ifc['ipv4'] + ifc['ipv6']:
+                try:
+                    yield {
+                        'hostname': ifc['hostname'],
+                        'port': ifc['name'],
+                        'address': address,
+                        'services': _services
+                    }
+                except TypeError:
+                    raise
+
+    yield from _merged_result()
+
+
+@routes.route('/ip-services', methods=['GET', 'POST'])
+@common.require_accepts_json
+def ip_services():
+    """
+    This method will return a list of all interface addresses
+    with any operational services defined for that interface.
+
+    The response will be formatted according to the following schema:
+
+    .. asjson::
+       inventory_provider.routes.msr.IP_SERVICES_LIST_SCHEMA
+
+    :return: a json list, formatted as above
+    """
+
+    cache_key = 'classifier-cache:msr:ip-services'
+
+    r = common.get_current_redis()
+    result = _ignore_cache_or_retrieve(request, cache_key, r)
+
+    if not result:
+        result = list(_load_ip_services())
+        result = json.dumps(result)
+        r.set(cache_key, result.encode('utf-8'))
+
+    return Response(result.encode('utf-8'), mimetype='application/json')

test/test_msr_routes.py

@@ -6,7 +6,8 @@ import pytest
 from inventory_provider.routes.msr import PEERING_LIST_SCHEMA, \
     PEERING_GROUP_LIST_SCHEMA, PEERING_ADDRESS_SERVICES_LIST, \
     SYSTEM_CORRELATION_SERVICES_LIST_SCHEMA, _get_services_for_address, \
-    MDVPN_LIST_SCHEMA, VPN_PROXY_LIST_SCHEMA, ASN_PEER_LIST_SCHEMA
+    MDVPN_LIST_SCHEMA, VPN_PROXY_LIST_SCHEMA, ASN_PEER_LIST_SCHEMA, \
+    IP_SERVICES_LIST_SCHEMA, _dedupe
 from inventory_provider.routes.poller import SERVICES_LIST_SCHEMA
 from inventory_provider.tasks.common import _get_redis
 
@@ -391,3 +392,27 @@ def test_get_asn_peers_post(endpoint_variant, post_body, client, mocked_redis):
     response_data = json.loads(rv.data.decode('utf-8'))
     jsonschema.validate(response_data, ASN_PEER_LIST_SCHEMA)
     assert response_data  # test data is non-empty
+
+
+def test_dedupe():
+    data = [
+        {'a': 1, 'b': {'a': 1, 'b': 2}, 'c': 3},
+        {'a': 1, 'b': {'a': 1, 'b': 2}, 'c': 3, 'd': 4},
+        {'a': 1, 'b': {'a': 1, 'b': 2}, 'c': 3},
+        {'a': 1, 'b': {'a': 1, 'b': 2}, 'c': 3},
+    ]
+    result = list(_dedupe(data))
+    assert len(result) == 2
+
+
+def test_ip_services(client):
+
+    rv = client.get(
+        '/msr/ip-services',
+        headers=DEFAULT_REQUEST_HEADERS)
+    assert rv.status_code == 200
+    assert rv.is_json
+    response_data = json.loads(rv.data.decode('utf-8'))
+    jsonschema.validate(response_data, IP_SERVICES_LIST_SCHEMA)
+
+    assert response_data  # test data is non-empty