diff --git a/Changelog.md b/Changelog.md
index 00dbccbc6e5fd395636f5506fd3475d920c92050..0b78aed3f2a96828a31e568adabf0f8c51dcfafb 100644
--- a/Changelog.md
+++ b/Changelog.md
@@ -2,6 +2,9 @@
All notable changes to this project will be documented in this file.
+## [0.92] - 2022-08-12
+- REPORTING-312: Added NREN asn's to /msr/services response
+
## [0.91] - 2022-08-03
- REPORTING-311: Added /msr/asn-peers endpoint
diff --git a/inventory_provider/routes/msr.py b/inventory_provider/routes/msr.py
index 8c052a67f30d3f540375a2666c9b5ffd460fb3d9..a1828a2082edb8f23454ddb32a72f7ccd564eac1 100644
--- a/inventory_provider/routes/msr.py
+++ b/inventory_provider/routes/msr.py
@@ -260,7 +260,8 @@ SYSTEM_CORRELATION_SERVICES_LIST_SCHEMA = {
'service_type': {'type': 'string'}, # TODO: enum?
'project': {'type': 'string'}, # TODO: remove this?
'customer': {'type': 'string'},
- 'endpoints': {'$ref': '#/definitions/endpoints'}
+ 'endpoints': {'$ref': '#/definitions/endpoints'},
+ 'redundant_asn': {'type': 'integer'}
},
'required': [
'circuit_id', 'sid', 'name', 'speed', 'status', 'monitored',
@@ -881,6 +882,129 @@ def get_peering_services():
return Response(response, mimetype="application/json")
+# TODO: @functools.cache is only available in py3.9
+@functools.lru_cache(maxsize=None)
+def _load_all_interfaces():
+ """
+ loads all ip interfaces in the network and returns as a dict
+ of dicts:
+ hostname -> interface name -> interface info
+
+ :return: dict of dicts
+ """
+ # dict of dicts:
+ # peering_info[hostname][interface_name] = dict of ifc details
+ result = defaultdict(dict)
+
+ host_if_extraction_re = re.compile(r'^netconf-interfaces:(.+?):')
+ for doc in common.load_json_docs(
+ config_params=current_app.config['INVENTORY_PROVIDER_CONFIG'],
+ key_pattern='netconf-interfaces:*',
+ num_threads=20):
+ matches = host_if_extraction_re.match(doc['key'])
+ if matches:
+ hostname = matches[1]
+ interface_name = doc['value']['name']
+ result[hostname][interface_name] = doc['value']
+
+ return result
+
+
+# TODO: @functools.cache is only available in py3.9
+@functools.lru_cache(maxsize=None)
+def _load_redundant_access_peers():
+ """
+ load all peers that should be considered
+ redundant for access services
+
+ that is, all peers for services like NREN-APx
+
+ :return: dict of [peer address] -> [remote asn]
+ """
+ r = common.get_current_redis()
+ result = {}
+
+ # cf. REPORTING-312: limited to eGEANT group,
+ # but this can be expanded here in future
+ redundant_access_groups = ['eGEANT']
+ for g in redundant_access_groups:
+ doc = r.get(f'juniper-peerings:group:{g}')
+ for peer in json.loads(doc.decode('utf-8')):
+ result[peer['address']] = peer['remote-asn']
+
+ return result
+
+
+def _ip_endpoint_extractor(endpoint_details: dict):
+ """
+ special-purpose method used only by _endpoint_extractor
+
+ operates on a dictionary formatted as in worker.transform_ims_data
+ (cf sid_services)
+
+ WARNING: assumes hostname is a router in _load_all_interfaces,
+ asserts if not
+
+ :param endpoint_details: dict formatted as in worker.transform_ims_data
+ :return: dict formatted
+ as SYSTEM_CORRELATION_SERVICES_LIST_SCHEMA.ip-endpoint
+ """
+ hostname = ims_equipment_to_hostname(endpoint_details['equipment'])
+ interface = endpoint_details['port'].lower()
+
+ ip_endpoint = {
+ 'hostname': hostname,
+ 'interface': interface,
+ }
+
+ all_interfaces = _load_all_interfaces()
+ # sanity: should have already been checked
+ assert hostname in all_interfaces
+
+ host_info = all_interfaces[hostname]
+ interface_info = host_info.get(interface, {})
+
+ addresses = {}
+ ipv4 = interface_info.get('ipv4')
+ ipv6 = interface_info.get('ipv6')
+ if ipv4:
+ addresses['v4'] = ipv4[0]
+ if ipv6:
+ addresses['v6'] = ipv6[0]
+ if addresses:
+ ip_endpoint['addresses'] = addresses
+
+ return ip_endpoint
+
+
+def _endpoint_extractor(endpoint_details: Dict):
+ """
+ special-purpose method used only by get_system_correlation_services
+
+ operates on a dictionary formatted as in worker.transform_ims_data
+ (cf sid_services)
+
+ WARNING: assumes hostname is a router in _load_all_interfaces,
+ asserts if not
+
+ :param endpoint_details: dict formatted as in worker.transform_ims_data
+ :return: dict formatted as one element of the array
+ SYSTEM_CORRELATION_SERVICES_LIST_SCHEMA.endpoints
+ """
+ if not endpoint_details['geant_equipment']:
+ return
+ potential_hostname = ims_equipment_to_hostname(
+ endpoint_details['equipment'])
+ all_routers = _load_all_interfaces().keys()
+ if potential_hostname in all_routers:
+ return _ip_endpoint_extractor(endpoint_details)
+ else:
+ return {
+ 'equipment': endpoint_details['equipment'],
+ 'port': endpoint_details['port']
+ }
+
+
@routes.route('/services', methods=['GET', 'POST'])
@common.require_accepts_json
def get_system_correlation_services():
@@ -900,63 +1024,26 @@ def get_system_correlation_services():
:return:
"""
+ def _get_redundancy_asn(endpoints):
+ # endpoints should be a list formatted as
+ # SYSTEM_CORRELATION_SERVICES_LIST_SCHEMA.endpoints
+
+ redundant_peerings = _load_redundant_access_peers()
+ for ep in endpoints:
+ addresses = ep.get('addresses', {})
+ for ifc_address in addresses.values():
+ for p, asn in redundant_peerings.items():
+ peer = ipaddress.ip_address(p)
+ ifc = ipaddress.ip_interface(ifc_address)
+ if peer in ifc.network:
+ return asn
+ return None
+
cache_key = 'classifier-cache:msr:services'
r = common.get_current_redis()
response = _ignore_cache_or_retrieve(request, cache_key, r)
if not response:
- peering_info = defaultdict(defaultdict)
-
- key_pattern = 'netconf-interfaces:*'
-
- host_if_extraction_re = re.compile(
- r'^netconf-interfaces:(.+?):')
- for doc in common.load_json_docs(
- config_params=current_app.config['INVENTORY_PROVIDER_CONFIG'],
- key_pattern=key_pattern,
- num_threads=20):
- matches = host_if_extraction_re.match(doc['key'])
- if matches:
- peering_info[matches[1]][doc['value']['name']] = doc['value']
-
- def _ip_endpoint_extractor(endpoint_details: Dict):
- hostname = ims_equipment_to_hostname(
- endpoint_details['equipment'])
- interface = endpoint_details['port'].lower()
-
- ip_endpoint = {
- 'hostname': hostname,
- 'interface': interface,
- }
- addresses = {}
- host_info = peering_info.get(hostname, {})
- interface_info = host_info.get(interface, {})
- ipv4 = interface_info.get('ipv4')
- ipv6 = interface_info.get('ipv6')
- if ipv4:
- addresses['v4'] = ipv4[0]
- if ipv6:
- addresses['v6'] = ipv6[0]
- if ipv4 or ipv6:
- ip_endpoint['addresses'] = addresses
-
- return ip_endpoint
-
- def _optical_endpoint_extractor(endpoint_details: Dict):
- return {
- 'equipment': endpoint_details['equipment'],
- 'port': endpoint_details['port']
- }
-
- def _endpoint_extractor(endpoint_details: Dict):
- if not endpoint_details['geant_equipment']:
- return
- potential_hostname = ims_equipment_to_hostname(
- endpoint_details['equipment'])
- if potential_hostname in peering_info.keys():
- return _ip_endpoint_extractor(endpoint_details)
- else:
- return _optical_endpoint_extractor(endpoint_details)
sid_services = json.loads(r.get('ims:sid_services').decode('utf-8'))
@@ -977,15 +1064,19 @@ def get_system_correlation_services():
endpoint = _endpoint_extractor(d)
if endpoint:
service_info['endpoints'].append(endpoint)
+
if service_info.get('endpoints'):
+ asn = _get_redundancy_asn(service_info['endpoints'])
+ if asn:
+ service_info['redundant_asn'] = asn
response.append(service_info)
- jsonschema.validate(response,
- SYSTEM_CORRELATION_SERVICES_LIST_SCHEMA)
+ jsonschema.validate(
+ response, SYSTEM_CORRELATION_SERVICES_LIST_SCHEMA)
if response:
response = json.dumps(response, indent=2)
- r.set(cache_key, response.encode('utf-8'))
+ # r.set(cache_key, response.encode('utf-8'))
if not response:
return Response(
@@ -1225,10 +1316,10 @@ def asn_peers(asn):
This method returns a list of all peers filtered by `group` and `instance`,
which can be passed either as URL query parameters or as entries in a
POST request with a JSON body that matches this schema:
- `{
- "group": "group to filter by",
- "instance": "instance to filter by"
- }`
+ `{
+ "group": "group to filter by",
+ "instance": "instance to filter by"
+ }`
Results are returned where all filters given are true, and exact string
matches.
diff --git a/inventory_provider/tasks/worker.py b/inventory_provider/tasks/worker.py
index 3517bb0377fa4b6acf5e1310223de180fd01bba6..dda87815227f1f467aa6d0932f9ab3a863f2d980 100644
--- a/inventory_provider/tasks/worker.py
+++ b/inventory_provider/tasks/worker.py
@@ -722,11 +722,24 @@ def ims_task(self, use_current=False):
def extract_ims_data():
-
c = InventoryTask.config["ims"]
+ return _extract_ims_data(
+ ims_api_url=c['api'],
+ ims_username=c['username'],
+ ims_password=c['password'])
+
+
+def _extract_ims_data(ims_api_url, ims_username, ims_password):
+ """
+ convenient entry point for testing ...
+ :param ims_api_url:
+ :param ims_username:
+ :param ims_password:
+ :return:
+ """
def _ds() -> IMS:
- return IMS(c['api'], c['username'], c['password'])
+ return IMS(ims_api_url, ims_username, ims_password)
locations = {}
lg_routers = []
diff --git a/setup.py b/setup.py
index 4da31e2bd8816da258d745c106d778803380c1a4..7334603bb525aa4215e804b13b20b74e509bff2b 100644
--- a/setup.py
+++ b/setup.py
@@ -2,7 +2,7 @@ from setuptools import setup, find_packages
setup(
name='inventory-provider',
- version="0.91",
+ version="0.92",
author='GEANT',
author_email='swd@geant.org',
description='Dashboard inventory provider',