diff --git a/inventory_provider/config.py b/inventory_provider/config.py index d62dbb82e23c4277824a1c2e19d74f10d765f1e7..3fa03b9de584b4783a4305ad9f4e9ee7ec014ba3 100644 --- a/inventory_provider/config.py +++ b/inventory_provider/config.py @@ -201,13 +201,36 @@ CONFIG_SCHEMA = { 'items': { 'type': 'object', 'properties': { - 'nren': {'type': 'string'}, - 'asn': {'type': 'integer'} + 'nren': {'type': 'string'}, + 'asn': {'type': 'integer'} }, 'required': ['nren', 'asn'], 'additionalProperties': False }, - } + }, + 'aai': { + 'type': 'object', + 'properties': { + 'discovery_endpoint_url': {'type': 'string'}, + 'inventory_provider': {'type': 'object', + 'properties': { + 'client_id': {'type': 'string'}, + 'secret': {'type': 'string'} + }, + 'required': ['client_id', 'secret'], + 'additionalProperties': False} + }, + 'required': ['discovery_endpoint_url', 'inventory_provider'], + 'additionalProperties': False + }, + 'orchestrator': { + 'type': 'object', + 'properties': { + 'url': {'type': 'string'}, + }, + 'required': ['url'], + 'additionalProperties': False + }, }, 'type': 'object', @@ -223,10 +246,14 @@ CONFIG_SCHEMA = { 'minItems': 1, 'items': {'type': 'integer'} }, - 'managed-routers': {'type': 'string'}, 'lab-routers': { - 'type': 'array', - 'items': {'type': 'string'} + 'type': 'object', + 'properties': { + '^[a-zA-Z0-9.-]+$': { + 'type': 'string', + 'enum': ['nokia', 'juniper'] + } + } }, 'unmanaged-interfaces': { 'type': 'array', @@ -237,7 +264,8 @@ CONFIG_SCHEMA = { 'nokia-community-inventory-provider': {'type': 'string'}, 'nokia-community-dashboard': {'type': 'string'}, 'nokia-community-brian': {'type': 'string'}, - + 'aai': {'$ref': '#/definitions/aai'}, + 'orchestrator': {'$ref': '#/definitions/orchestrator'} }, 'oneOf': [ { @@ -247,9 +275,10 @@ CONFIG_SCHEMA = { 'redis', 'redis-databases', 'ims', - 'managed-routers', 'gws-direct', - 'nren-asn-map'] + 'nren-asn-map', + 'aai', + 'orchestrator'] }, { 'required': [ @@ -258,9 +287,10 @@ CONFIG_SCHEMA = { 'sentinel', 'redis-databases', 'ims', - 'managed-routers', 'gws-direct', - 'nren-asn-map'] + 'nren-asn-map', + 'aai', + 'orchestrator'] } ], 'additionalProperties': False diff --git a/inventory_provider/gap.py b/inventory_provider/gap.py new file mode 100644 index 0000000000000000000000000000000000000000..31816ea5a67c36e6b7d947c6d1f4b0ab7692f59a --- /dev/null +++ b/inventory_provider/gap.py @@ -0,0 +1,172 @@ +import concurrent.futures +import logging +import socket + +import requests +from requests.adapters import HTTPAdapter +from urllib3.poolmanager import PoolManager + +logger = logging.getLogger(__name__) + +GRANT_TYPE = 'client_credentials' +SCOPE = 'openid profile email aarc' + + +class IPv4Adapter(HTTPAdapter): + """A custom adapter that forces the use of IPv4. + The reason for this is that the orchestrator does not support IPv6. We use this adapter to force the use of IPv4 as + a temporary workaround. This adapter should be removed once the orchestrator supports IPv6.""" + + def init_poolmanager(self, connections, maxsize, block=False, **pool_kwargs): + pool_kwargs['socket_options'] = [(socket.IPPROTO_IP, socket.IP_TOS, 0)] + self.poolmanager = PoolManager( + num_pools=connections, + maxsize=maxsize, + block=block, + **pool_kwargs + ) + + def proxy_manager_for(self, proxy, **proxy_kwargs): + proxy_kwargs['socket_options'] = [(socket.IPPROTO_IP, socket.IP_TOS, 0)] + return super().proxy_manager_for(proxy, **proxy_kwargs) + + +def get_token_endpoint(discovery_endpoint_url: str) -> str: + response = requests.get(discovery_endpoint_url) + response.raise_for_status() + return response.json()['token_endpoint'] + + +def get_token(aai_config: dict) -> str: + """Get an access token using the given configuration.""" + response = requests.post( + get_token_endpoint(aai_config['discovery_endpoint_url']), + data={ + 'grant_type': GRANT_TYPE, + 'scope': SCOPE, + 'client_id': aai_config['inventory_provider']['client_id'], + 'client_secret': aai_config['inventory_provider']['secret'] + } + ) + response.raise_for_status() + return response.json()['access_token'] + + +def make_request(body: dict, token: str, app_config: dict) -> dict: + """Make a request to the orchestrator using the given body.""" + api_url = f'{app_config["orchestrator"]["url"]}/api/graphql' + headers = {'Authorization': f'Bearer {token}'} + session = requests.Session() + # Mount the adapter to force IPv4 + # This should be removed once the orchestrator supports IPv6 + # See the docstring of the IPv4Adapter class for more info + adapter = IPv4Adapter() + session.mount('http://', adapter) + session.mount('https://', adapter) + response = session.post(api_url, headers=headers, json=body) + response.raise_for_status() + # The graphql API returns a 200 status code even if there are errors in the response + if errors := response.json().get('errors'): + err_msg = f'GraphQL query returned errors: {errors}' + logger.error(err_msg) + raise ValueError(err_msg) + return response.json() + + +def extract_router_info(device: dict, token: str, app_config: dict) -> dict or None: + tag_to_key_map = { + "RTR": "router", + "OFFICE_ROUTER": "officeRouter", + "Super_POP_SWITCH": "superPopSwitch" + } + + tag = device.get("product", {}).get("tag") + key = tag_to_key_map.get(tag) + subscription_id = device.get("subscriptionId") + + if key is None or subscription_id is None: + logger.warning(f"Skipping device with invalid tag or subscription ID: {device}") + return None + + query = f""" + query {{ + subscriptions( + filterBy: {{ field: "subscriptionId", value: "{subscription_id}" }} + ) {{ + page {{ + subscriptionId + productBlockInstances {{ + productBlockInstanceValues + }} + }} + }} + }} + """ + + response = make_request(body={'query': query}, token=token, app_config=app_config) + page_data = response.get('data', {}).get('subscriptions', {}).get('page') + + if not page_data: + logger.warning(f"No data for subscription ID: {subscription_id}") + return None + + instance_values = page_data[0].get('productBlockInstances', [{}])[0].get('productBlockInstanceValues', []) + + fqdn = next((item.get('value') for item in instance_values if item.get('field') == f'{key}Fqdn'), None) + vendor = next((item.get('value') for item in instance_values if item.get('field') == 'vendor'), None) + + if fqdn and vendor: + return {'fqdn': fqdn, 'vendor': vendor} + else: + logger.warning(f"Skipping device with missing FQDN or vendor: {device}") + return None + + +def load_routers_from_orchestrator(app_config: dict) -> dict: + """Gets devices from the orchestrator and returns a dictionary of FQDNs and vendors.""" + token = get_token(app_config['aai']) + routers = {} + end_cursor = 0 + has_next_page = True + + while has_next_page: + query = f""" + {{ + subscriptions( + filterBy: {{field: "status", value: "PROVISIONING|ACTIVE"}}, + first: 100, + after: {end_cursor}, + query: "tag:(RTR|OFFICE_ROUTER|Super_POP_SWITCH)" + ) {{ + pageInfo {{ + hasNextPage + endCursor + }} + page {{ + subscriptionId + product {{ + tag + }} + }} + }} + }} + """ + + response = make_request(body={'query': query}, token=token, app_config=app_config) + try: + devices = response['data']['subscriptions']['page'] + page_info = response['data']['subscriptions']['pageInfo'] + end_cursor = page_info['endCursor'] + has_next_page = page_info['hasNextPage'] + except (TypeError, KeyError): + devices = [] + has_next_page = False + + with concurrent.futures.ThreadPoolExecutor() as executor: + futures = [executor.submit(extract_router_info, device, token, app_config) for device in devices] + for future in concurrent.futures.as_completed(futures): + router_info = future.result() + if router_info is not None: + routers[router_info['fqdn']] = router_info['vendor'] + + return routers diff --git a/inventory_provider/juniper.py b/inventory_provider/juniper.py index 47e1d1e302156d83371b0ca6b549289d0fc1d953..3d846358c595c040153c4dbfc42a2e1cf49f18b0 100644 --- a/inventory_provider/juniper.py +++ b/inventory_provider/juniper.py @@ -10,7 +10,6 @@ from jnpr.junos import Device from jnpr.junos import exception as EzErrors from lxml import etree import netifaces -import requests CONFIG_SCHEMA = """<?xml version="1.1" encoding="UTF-8" ?> <xs:schema xmlns:xs="http://www.w3.org/2001/XMLSchema"> @@ -394,20 +393,6 @@ def interface_addresses(netconf_config): } -def load_routers_from_netdash(url): - """ - Query url for a linefeed-delimitted list of managed router hostnames. - - :param url: url of alldevices.txt file - :return: list of router hostnames - """ - r = requests.get(url=url) - r.raise_for_status() - return [ - ln.strip() for ln in r.text.splitlines() if ln.strip() - ] - - def local_interfaces( type=netifaces.AF_INET, omit_link_local=True, diff --git a/inventory_provider/routes/poller.py b/inventory_provider/routes/poller.py index 45634297e78c08effbb9aafa804da806816d4681..0bbbc3b8d98a8194576507c5a71a872705bfdc86 100644 --- a/inventory_provider/routes/poller.py +++ b/inventory_provider/routes/poller.py @@ -914,10 +914,17 @@ def interfaces(hostname=None): return Response(result, mimetype="application/json") +def get_netdash_equipment() -> dict[str, str]: + """Get the netdash equipment mapping from redis.""" + current_redis = common.get_current_redis() + return json.loads(current_redis.get('netdash').decode('utf-8')) + + def load_error_report_interfaces( config, hostname=None, use_next_redis=False ): interfaces = _load_interfaces(config, hostname, use_next_redis=use_next_redis) + netdash_equipment = get_netdash_equipment() def filter_interface(interface: dict): return all( @@ -932,14 +939,12 @@ def load_error_report_interfaces( ) ) - def transform_interface(interface: dict): + def transform_interface(interface: dict) -> dict: return { "router": interface["router"], "name": interface["name"], "description": interface["description"], - # TODO: This is a complete hack until we have a proper way to determine - # router vendor - "vendor": "nokia" if interface["router"].startswith("rt0") else "juniper" + "vendor": netdash_equipment.get(interface["router"]), } return sorted( diff --git a/inventory_provider/routes/testing.py b/inventory_provider/routes/testing.py index 41e806034d6ace22d17dd7e890487c629f87cecb..5fcb7cce47a48eaa86ac4794d098448929ff69e4 100644 --- a/inventory_provider/routes/testing.py +++ b/inventory_provider/routes/testing.py @@ -55,7 +55,8 @@ def juniper_addresses(): routers = r.get('netdash') assert routers # sanity: value shouldn't be empty routers = json.loads(routers.decode('utf-8')) - return jsonify(routers) + juniper_routers = [k for k, v in routers.items() if v == 'juniper'] + return jsonify(juniper_routers) @routes.route("bgp/<hostname>", methods=['GET']) diff --git a/inventory_provider/tasks/worker.py b/inventory_provider/tasks/worker.py index 82c0699cd5b44e5453e3cec646a776e5f92fcb1f..fd0d4f5d0c270803b4e3d135869b4e5f52666dd4 100644 --- a/inventory_provider/tasks/worker.py +++ b/inventory_provider/tasks/worker.py @@ -27,7 +27,7 @@ from inventory_provider.tasks.common \ latch_db, get_latch, set_latch, update_latch_status, \ ims_sorted_service_type_key, set_single_latch from inventory_provider.tasks import monitor -from inventory_provider import config, nokia +from inventory_provider import config, nokia, gap from inventory_provider import environment from inventory_provider import snmp from inventory_provider import juniper @@ -455,13 +455,6 @@ def check_task_status(task_id, parent=None, forget=False): yield result -def get_router_vendors(): - c = InventoryTask.config["ims"] - ds = \ - IMS(c['api'], c['username'], c['password'], c.get('verify-ssl', False)) - return {r.lower(): v.lower() for r, v in ims_data.get_router_vendors(ds)} - - @app.task(base=InventoryTask, bind=True, name='update_entry_point') @log_task_entry_and_exit def update_entry_point(self): @@ -476,52 +469,37 @@ def update_entry_point(self): warning_callback=self.log_warning ) lab_routers = InventoryTask.config.get('lab-routers', []) - - ims_rv = get_router_vendors() - - def _get_router_vendor(router): - return ims_rv.get(router.lower().split('.geant.')[0], 'unknown') - - def _get_lab_router_vendor(router): - _rv = ims_rv.get(router.lower().split('.geant.')[0]) - if not _rv: - _rv = ims_rv.get(router.lower().split('.office.')[0], - 'unknown') - return _rv - - rv = {r: _get_router_vendor(r) for r in routers} - lab_rv = {r: _get_lab_router_vendor(r) for r in lab_routers} chord( ( ims_task.s().on_error(task_error_handler.s()), chord( - (reload_router_config_juniper.s(r) for r, v in rv.items() + (reload_router_config_juniper.s(r) for r, v in routers.items() if v == 'juniper'), empty_task.si('juniper router tasks complete') ), chord( (reload_lab_router_config_juniper.s(r) - for r, v in lab_rv.items() if v == 'juniper'), + for r, v in lab_routers.items() if v == 'juniper'), empty_task.si('juniper lab router tasks complete') ), chord( - (reload_router_config_nokia.s(r) for r, v in rv.items() + (reload_router_config_nokia.s(r) for r, v in routers.items() if v == 'nokia'), empty_task.si('nokia router tasks complete') ), chord( (reload_router_config_nokia.s(r, True) - for r, v in lab_rv.items() if v == 'nokia'), + for r, v in lab_routers.items() if v == 'nokia'), empty_task.si('nokia lab router tasks complete') ), chord( - (reload_router_config_try_all.s(r) for r, v in rv.items() + (reload_router_config_try_all.s(r) for r, v in routers.items() if v == 'unknown'), empty_task.si('unknown router tasks complete') ), chord( (reload_router_config_try_all.s(r, True) - for r, v in lab_rv.items() if v == 'unknown'), + for r, v in lab_routers.items() if v == 'unknown'), empty_task.si('unknown lab router tasks complete') ) ), @@ -552,8 +530,7 @@ def retrieve_and_persist_neteng_managed_device_list( netdash_equipment = None try: info_callback('querying netdash for managed routers') - netdash_equipment = list(juniper.load_routers_from_netdash( - InventoryTask.config['managed-routers'])) + netdash_equipment = gap.load_routers_from_orchestrator(InventoryTask.config) except Exception as e: warning_callback(f'Error retrieving device list: {e}') diff --git a/test/conftest.py b/test/conftest.py index 47a350147bfd1b244e5742b15db437e891cb2c3b..bd976e63f94394dc9e9207c0583ac5476c72aa95 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -69,7 +69,6 @@ def data_config_filename(): "username": "ims_username", "password": "ims_password" }, - "managed-routers": "bogus url", "unmanaged-interfaces": [ { "address": "99.99.99.99", @@ -102,7 +101,17 @@ def data_config_filename(): "nren": "BAZ", "asn": 1853 } - ] + ], + "aai": { + "discovery_endpoint_url": "https://smaple.discovery.endpoint", + "inventory_provider": { + "client_id": "sample-client-id", + "secret": "sample-secret" + } + }, + "orchestrator": { + "url": "https://orchestrator.url" + } } config['gws-direct'] = read_json_test_data('gws-direct.json') diff --git a/test/data/router-info.json b/test/data/router-info.json index c5d6758fdbe6dd4be250a18fc25afa3c04386104..5352029a65d23d054d1e445be633de72732417e2 100644 Binary files a/test/data/router-info.json and b/test/data/router-info.json differ diff --git a/test/test_flask_config.py b/test/test_flask_config.py index 262b2a84d1bf5dd1e5479e54f4f0ea98f00ddbe1..7ba83ec54c3e2e8389924e14b3cff0bad6aea29b 100644 --- a/test/test_flask_config.py +++ b/test/test_flask_config.py @@ -27,9 +27,18 @@ def config(): 'username': 'ims_username', 'password': 'ims_password', }, - 'managed-routers': 'router_list', 'gws-direct': [], 'nren-asn-map': [], + 'aai': { + 'discovery_endpoint_url': 'https://smaple.discovery.endpoint', + 'inventory_provider': { + 'client_id': 'sample-client-id', + 'secret': 'sample-secret' + } + }, + 'orchestrator': { + 'url': 'https://orchestrator.url' + } } diff --git a/test/test_gap.py b/test/test_gap.py new file mode 100644 index 0000000000000000000000000000000000000000..27b5a60aaf7bdea15befcb691f82352af1636410 --- /dev/null +++ b/test/test_gap.py @@ -0,0 +1,104 @@ +from unittest.mock import patch, MagicMock + +from inventory_provider import gap + + +def test_get_token_endpoint(mocker): + mock_response = MagicMock() + mock_response.json.return_value = {'token_endpoint': 'http://example.com/token'} + mock_response.raise_for_status = MagicMock() + mocker.patch('inventory_provider.gap.requests.get', return_value=mock_response) + + discovery_endpoint_url = 'http://example.com/aai' + token_endpoint = gap.get_token_endpoint(discovery_endpoint_url) + assert token_endpoint == 'http://example.com/token' + + +def test_get_token(mocker): + mocker.patch('inventory_provider.gap.get_token_endpoint', return_value='http://example.com/token') + mock_response = MagicMock() + mock_response.json.return_value = {'access_token': 'test_token'} + mock_response.raise_for_status = MagicMock() + mocker.patch('inventory_provider.gap.requests.post', return_value=mock_response) + + aai_config = { + 'discovery_endpoint_url': 'http://example.com/aai', + 'inventory_provider': {'client_id': 'test_id', 'secret': 'test_secret'} + } + token = gap.get_token(aai_config) + assert token == 'test_token' + + +def test_extract_router_info(mocker): + device = { + 'product': {'tag': 'RTR'}, + 'subscriptionId': 'test_subscription_id' + } + mocker.patch('inventory_provider.gap.make_request', return_value={ + 'data': { + 'subscriptions': { + 'page': [{ + 'productBlockInstances': [{ + 'productBlockInstanceValues': [ + {'field': 'routerFqdn', 'value': 'test_fqdn'}, + {'field': 'vendor', 'value': 'test_vendor'} + ] + }] + }] + } + } + }) + config = { + "aai": { + "discovery_endpoint_url": "https://sample-url", + "inventory_provider": { + "client_id": "test_id", + "secret": "test_secret" + } + }, + "orchestrator": { + "url": "orchestrator-url" + } + } + router_info = gap.extract_router_info(device, 'test_token', config) + assert router_info == {'fqdn': 'test_fqdn', 'vendor': 'test_vendor'} + + +def test_load_routers_from_orchestrator(mocker): + mocker.patch('inventory_provider.gap.make_request', return_value={ + 'data': { + 'subscriptions': { + 'page': [ + {'subscriptionId': '1', 'product': {'tag': 'RTR'}}, + {'subscriptionId': '2', 'product': {'tag': 'OFFICE_ROUTER'}} + ], + "pageInfo": { + "totalItems": 2, + "startCursor": 0, + "hasPreviousPage": False, + "hasNextPage": False, + "endCursor": 2 + } + } + } + }) + mocker.patch('inventory_provider.gap.extract_router_info', side_effect=[ + {'fqdn': 'fqdn1', 'vendor': 'vendor1'}, + {'fqdn': 'fqdn2', 'vendor': 'vendor2'} + ]) + mocker.patch('inventory_provider.gap.get_token', return_value='test_token') + config = { + "aai": { + "discovery_endpoint_url": "https://sample-url", + "inventory_provider": { + "client_id": "test_id", + "secret": "test_secret" + } + }, + "orchestrator": { + "url": "orchestrator-url" + } + } + + routers = gap.load_routers_from_orchestrator(config) + assert routers == {'fqdn1': 'vendor1', 'fqdn2': 'vendor2'} diff --git a/test/test_junos_devices_query.py b/test/test_junos_devices_query.py deleted file mode 100644 index b1ef7ee2b1691c056eb139e163b52d8c977a0866..0000000000000000000000000000000000000000 --- a/test/test_junos_devices_query.py +++ /dev/null @@ -1,25 +0,0 @@ -import os - -import responses - -from inventory_provider import juniper - -TEST_DATA_FILENAME = os.path.realpath(os.path.join( - os.path.dirname(__file__), - 'data', - 'netdash-alldevices.txt')) - - -@responses.activate -def test_junosspace_devices_parsing(data_config): - data_config['managed-routers'] = 'http://sabababa' - with open(TEST_DATA_FILENAME) as f: - responses.add( - method=responses.GET, - url=data_config['managed-routers'], - body=f.read()) - - hostnames = juniper.load_routers_from_netdash( - data_config['managed-routers']) - - assert 'mx1.ams.nl.geant.net' in hostnames diff --git a/test/test_worker.py b/test/test_worker.py index dcc32853304b2b88fefa0f7a7c47396fdd1173ff..0cad8a59947a86c0a438629d4bfff52741f57a57 100644 --- a/test/test_worker.py +++ b/test/test_worker.py @@ -637,19 +637,16 @@ def test_persist_ims_data(mocker, data_config, mocked_redis): def test_retrieve_and_persist_neteng_managed_device_list( mocker, data_config, mocked_redis): - device_list = ['abc', 'def'] + device_list = [{'abc': 'juniper'}, {'def': 'nokia'}] r = common._get_redis(data_config) mocker.patch( 'inventory_provider.tasks.worker.InventoryTask.config' ) - mocker.patch('inventory_provider.tasks.worker.get_next_redis', - return_value=r) + mocker.patch('inventory_provider.tasks.worker.get_next_redis', return_value=r) r.delete('netdash') - mocked_j = mocker.patch( - 'inventory_provider.tasks.worker.juniper.load_routers_from_netdash' - ) - mocked_j.return_value = device_list + mocker.patch('inventory_provider.tasks.worker.get_current_redis', return_value=r) + mocker.patch('inventory_provider.gap.load_routers_from_orchestrator', return_value=device_list) result = retrieve_and_persist_neteng_managed_device_list() assert result == device_list assert json.loads(r.get('netdash')) == device_list @@ -1020,6 +1017,13 @@ def test_populate_error_report_interfaces_cache(mocker, data_config, mocked_redi mocker.patch( 'inventory_provider.tasks.worker.InventoryTask.config' ) + + netdash_equipment = { + "router_a.geant.net": "juniper", + "rt0.geant.net": "nokia" + } + + mocker.patch('inventory_provider.routes.poller.get_netdash_equipment', return_value=netdash_equipment) exp_router_a_interfaces = [ { "router": "router_a.geant.net",