diff --git a/inventory_provider/config.py b/inventory_provider/config.py index 950ad31076a818bf869949f58a2a1c84c562b0f1..3fa03b9de584b4783a4305ad9f4e9ee7ec014ba3 100644 --- a/inventory_provider/config.py +++ b/inventory_provider/config.py @@ -246,7 +246,6 @@ CONFIG_SCHEMA = { 'minItems': 1, 'items': {'type': 'integer'} }, - 'managed-routers': {'type': 'string'}, 'lab-routers': { 'type': 'object', 'properties': { @@ -276,7 +275,6 @@ CONFIG_SCHEMA = { 'redis', 'redis-databases', 'ims', - 'managed-routers', 'gws-direct', 'nren-asn-map', 'aai', @@ -289,7 +287,6 @@ CONFIG_SCHEMA = { 'sentinel', 'redis-databases', 'ims', - 'managed-routers', 'gws-direct', 'nren-asn-map', 'aai', diff --git a/inventory_provider/gap.py b/inventory_provider/gap.py index 7e2da432682c58a280c5bf6e66f3f707f5ff0499..2f728c31caca91b4f835a8950ac5b36a89aa1b39 100644 --- a/inventory_provider/gap.py +++ b/inventory_provider/gap.py @@ -1,124 +1,122 @@ import logging import requests - -from inventory_provider.tasks.config import inventory_provider_config +from flask import current_app logger = logging.getLogger(__name__) +GRANT_TYPE = 'client_credentials' +SCOPE = 'openid profile email aarc' -class Orchestrator: - GRANT_TYPE = 'client_credentials' - SCOPE = 'openid profile email aarc' - - def __init__(self): - self.config = inventory_provider_config['aai']['inventory_provider'] - self.base_url = f'{inventory_provider_config["orchestrator"]["url"]}/api/graphql' - - @staticmethod - def _get_token_endpoint() -> str: - response = requests.get(inventory_provider_config['aai']['discovery_endpoint_url']) - response.raise_for_status() - return response.json()['token_endpoint'] - - def _get_token(self) -> str: - response = requests.post( - self._get_token_endpoint(), - data={ - 'grant_type': self.GRANT_TYPE, - 'scope': self.SCOPE, - 'client_id': self.config['client_id'], - 'client_secret': self.config['secret'] - } - ) - response.raise_for_status() - return response.json()['access_token'] - - def _get_headers(self) -> dict: - token = self._get_token() - return { - 'Authorization': f'Bearer {token}' - } - def _make_request(self, url: str, body: dict) -> dict: - headers = self._get_headers() - response = requests.post(url, headers=headers, json=body) - response.raise_for_status() - return response.json() - - def _extract_router_info(self, device: dict) -> dict or None: - tag_to_key_map = { - "RTR": "router", - "OFFICE_ROUTER": "office_router", - "Super_POP_SWITCH": "super_pop_switch" - } +def get_token_endpoint(discovery_endpoint_url: str) -> str: + response = requests.get(discovery_endpoint_url) + response.raise_for_status() + return response.json()['token_endpoint'] - tag = device.get("product", {}).get("tag") - key = tag_to_key_map.get(tag) - subscription_id = device.get("subscriptionId") - - if key is None or subscription_id is None: - logger.warning(f"Skipping device with invalid tag or subscription ID: {device}") - return None - - query = f""" - query {{ - subscriptions( - filterBy: {{ field: "subscriptionId", value: "{subscription_id}" }} - ) {{ - page {{ - subscriptionId - productBlockInstances {{ - productBlockInstanceValues - }} + +def get_token(aai_config: dict) -> str: + """Get an access token using the given configuration.""" + response = requests.post( + get_token_endpoint(aai_config['discovery_endpoint_url']), + data={ + 'grant_type': GRANT_TYPE, + 'scope': SCOPE, + 'client_id': aai_config['inventory_provider']['client_id'], + 'client_secret': aai_config['inventory_provider']['secret'] + } + ) + response.raise_for_status() + return response.json()['access_token'] + + +def make_request(body: dict) -> dict: + """Make a request to the orchestrator using the given body.""" + config = current_app.config['INVENTORY_PROVIDER_CONFIG'] + api_url = f'{config["orchestrator"]["url"]}/api/graphql' + headers = {'Authorization': f'Bearer {get_token(config["aai"])}'} + response = requests.post(api_url, headers=headers, json=body) + response.raise_for_status() + return response.json() + + +def extract_router_info(device: dict) -> dict or None: + tag_to_key_map = { + "RTR": "router", + "OFFICE_ROUTER": "office_router", + "Super_POP_SWITCH": "super_pop_switch" + } + + tag = device.get("product", {}).get("tag") + key = tag_to_key_map.get(tag) + subscription_id = device.get("subscriptionId") + + if key is None or subscription_id is None: + logger.warning(f"Skipping device with invalid tag or subscription ID: {device}") + return None + + query = f""" + query {{ + subscriptions( + filterBy: {{ field: "subscriptionId", value: "{subscription_id}" }} + ) {{ + page {{ + subscriptionId + productBlockInstances {{ + productBlockInstanceValues }} }} }} - """ + }} + """ - response = self._make_request(self.base_url, body={'query': query}) - page_data = response.get('data', {}).get('subscriptions', {}).get('page') + response = make_request(body={'query': query}) + page_data = response.get('data', {}).get('subscriptions', {}).get('page') - if not page_data: - logger.warning(f"No data for subscription ID: {subscription_id}") - return None + if not page_data: + logger.warning(f"No data for subscription ID: {subscription_id}") + return None - instance_values = page_data[0].get('productBlockInstances', [{}])[0].get('productBlockInstanceValues', []) + instance_values = page_data[0].get('productBlockInstances', [{}])[0].get('productBlockInstanceValues', []) - fqdn = next((item.get('value') for item in instance_values if item.get('field') == f'{key}Fqdn'), None) - vendor = next((item.get('value') for item in instance_values if item.get('field') == 'vendor'), None) + fqdn = next((item.get('value') for item in instance_values if item.get('field') == f'{key}Fqdn'), None) + vendor = next((item.get('value') for item in instance_values if item.get('field') == 'vendor'), None) - if fqdn and vendor: - return {'fqdn': fqdn, 'vendor': vendor} - else: - logger.warning(f"Skipping device with missing FQDN or vendor: {device}") - return None + if fqdn and vendor: + return {'fqdn': fqdn, 'vendor': vendor} + else: + logger.warning(f"Skipping device with missing FQDN or vendor: {device}") + return None - def load_routers_from_orchestrator(self) -> dict: - query = """ - { - subscriptions( - filterBy: {field: "status", value: "ACTIVE"}, - query: "tag:(RTR|OFFICE_ROUTER|Super_POP_SWITCH)" - - ) { - page { - subscriptionId - product { - tag - } + +def load_routers_from_orchestrator() -> dict: + """Gets devices from the orchestrator and returns a dictionary of FQDNs and vendors.""" + + query = """ + { + subscriptions( + filterBy: {field: "status", value: "ACTIVE"}, + query: "tag:(RTR|OFFICE_ROUTER|Super_POP_SWITCH)" + ) { + page { + subscriptionId + product { + tag } } } - """ - routers = {} - response = self._make_request(self.base_url, body={'query': query}) - try: - devices = response['data']['subscriptions']['page'] - except TypeError: - devices = [] - for device in devices: - router_info = self._extract_router_info(device) - if router_info is not None: - routers[router_info['fqdn']] = router_info['vendor'] - return routers + } + """ + routers = {} + response = make_request(body={'query': query}) + + try: + devices = response['data']['subscriptions']['page'] + except (TypeError, KeyError): + devices = [] + + for device in devices: + router_info = extract_router_info(device) + if router_info is not None: + routers[router_info['fqdn']] = router_info['vendor'] + return routers diff --git a/inventory_provider/juniper.py b/inventory_provider/juniper.py index 47e1d1e302156d83371b0ca6b549289d0fc1d953..3ca41b1c17d1a8849967f1c3432523209e5d4ee5 100644 --- a/inventory_provider/juniper.py +++ b/inventory_provider/juniper.py @@ -393,21 +393,6 @@ def interface_addresses(netconf_config): "interface name": ifc['name'] } - -def load_routers_from_netdash(url): - """ - Query url for a linefeed-delimitted list of managed router hostnames. - - :param url: url of alldevices.txt file - :return: list of router hostnames - """ - r = requests.get(url=url) - r.raise_for_status() - return [ - ln.strip() for ln in r.text.splitlines() if ln.strip() - ] - - def local_interfaces( type=netifaces.AF_INET, omit_link_local=True, diff --git a/inventory_provider/routes/testing.py b/inventory_provider/routes/testing.py index 41e806034d6ace22d17dd7e890487c629f87cecb..5fcb7cce47a48eaa86ac4794d098448929ff69e4 100644 --- a/inventory_provider/routes/testing.py +++ b/inventory_provider/routes/testing.py @@ -55,7 +55,8 @@ def juniper_addresses(): routers = r.get('netdash') assert routers # sanity: value shouldn't be empty routers = json.loads(routers.decode('utf-8')) - return jsonify(routers) + juniper_routers = [k for k, v in routers.items() if v == 'juniper'] + return jsonify(juniper_routers) @routes.route("bgp/<hostname>", methods=['GET']) diff --git a/inventory_provider/tasks/worker.py b/inventory_provider/tasks/worker.py index 813af6f0e39db275872535697e318e6aa3a89207..7bd89612eb642de294ac2a9e058265f869a6b80e 100644 --- a/inventory_provider/tasks/worker.py +++ b/inventory_provider/tasks/worker.py @@ -20,7 +20,6 @@ from ncclient.transport import TransportError from inventory_provider.db import ims_data from inventory_provider.db.ims import IMS -from inventory_provider.gap import Orchestrator from inventory_provider.routes.poller import load_error_report_interfaces, load_interfaces_to_poll from inventory_provider.tasks.app import app from inventory_provider.tasks.common \ @@ -28,7 +27,7 @@ from inventory_provider.tasks.common \ latch_db, get_latch, set_latch, update_latch_status, \ ims_sorted_service_type_key, set_single_latch from inventory_provider.tasks import monitor -from inventory_provider import config, nokia +from inventory_provider import config, nokia, gap from inventory_provider import environment from inventory_provider import snmp from inventory_provider import juniper @@ -530,7 +529,7 @@ def retrieve_and_persist_neteng_managed_device_list( netdash_equipment = None try: info_callback('querying netdash for managed routers') - netdash_equipment = Orchestrator().load_routers_from_orchestrator() + netdash_equipment = gap.load_routers_from_orchestrator() except Exception as e: warning_callback(f'Error retrieving device list: {e}') diff --git a/test/data/router-info.json b/test/data/router-info.json index c5d6758fdbe6dd4be250a18fc25afa3c04386104..5352029a65d23d054d1e445be633de72732417e2 100644 Binary files a/test/data/router-info.json and b/test/data/router-info.json differ diff --git a/test/test_flask_config.py b/test/test_flask_config.py index 262b2a84d1bf5dd1e5479e54f4f0ea98f00ddbe1..7ba83ec54c3e2e8389924e14b3cff0bad6aea29b 100644 --- a/test/test_flask_config.py +++ b/test/test_flask_config.py @@ -27,9 +27,18 @@ def config(): 'username': 'ims_username', 'password': 'ims_password', }, - 'managed-routers': 'router_list', 'gws-direct': [], 'nren-asn-map': [], + 'aai': { + 'discovery_endpoint_url': 'https://smaple.discovery.endpoint', + 'inventory_provider': { + 'client_id': 'sample-client-id', + 'secret': 'sample-secret' + } + }, + 'orchestrator': { + 'url': 'https://orchestrator.url' + } } diff --git a/test/test_junos_devices_query.py b/test/test_junos_devices_query.py deleted file mode 100644 index b1ef7ee2b1691c056eb139e163b52d8c977a0866..0000000000000000000000000000000000000000 --- a/test/test_junos_devices_query.py +++ /dev/null @@ -1,25 +0,0 @@ -import os - -import responses - -from inventory_provider import juniper - -TEST_DATA_FILENAME = os.path.realpath(os.path.join( - os.path.dirname(__file__), - 'data', - 'netdash-alldevices.txt')) - - -@responses.activate -def test_junosspace_devices_parsing(data_config): - data_config['managed-routers'] = 'http://sabababa' - with open(TEST_DATA_FILENAME) as f: - responses.add( - method=responses.GET, - url=data_config['managed-routers'], - body=f.read()) - - hostnames = juniper.load_routers_from_netdash( - data_config['managed-routers']) - - assert 'mx1.ams.nl.geant.net' in hostnames diff --git a/test/test_worker.py b/test/test_worker.py index dcc32853304b2b88fefa0f7a7c47396fdd1173ff..d8628ff40da93f2d95d813e38ed78ff84fb30e39 100644 --- a/test/test_worker.py +++ b/test/test_worker.py @@ -637,19 +637,16 @@ def test_persist_ims_data(mocker, data_config, mocked_redis): def test_retrieve_and_persist_neteng_managed_device_list( mocker, data_config, mocked_redis): - device_list = ['abc', 'def'] + device_list = [{'abc': 'juniper'}, {'def': 'nokia'}] r = common._get_redis(data_config) mocker.patch( 'inventory_provider.tasks.worker.InventoryTask.config' ) - mocker.patch('inventory_provider.tasks.worker.get_next_redis', - return_value=r) + mocker.patch('inventory_provider.tasks.worker.get_next_redis', return_value=r) r.delete('netdash') - mocked_j = mocker.patch( - 'inventory_provider.tasks.worker.juniper.load_routers_from_netdash' - ) - mocked_j.return_value = device_list + mocker.patch('inventory_provider.tasks.worker.get_current_redis', return_value=r) + mocker.patch('inventory_provider.gap.load_routers_from_orchestrator', return_value=device_list) result = retrieve_and_persist_neteng_managed_device_list() assert result == device_list assert json.loads(r.get('netdash')) == device_list @@ -1020,6 +1017,7 @@ def test_populate_error_report_interfaces_cache(mocker, data_config, mocked_redi mocker.patch( 'inventory_provider.tasks.worker.InventoryTask.config' ) + mocker.patch('inventory_provider.routes.poller.load_error_report_interfaces.get_vendor', return_value='juniper') exp_router_a_interfaces = [ { "router": "router_a.geant.net",