diff --git a/brian_dashboard_manager/grafana/provision.py b/brian_dashboard_manager/grafana/provision.py index 2e363e146594bf8d310d1c0a95c4fcd30fa75b63..8c0c443b70ec9ba554c4b017d4ba0d5b4799bb67 100644 --- a/brian_dashboard_manager/grafana/provision.py +++ b/brian_dashboard_manager/grafana/provision.py @@ -27,7 +27,7 @@ from brian_dashboard_manager.grafana.folder import find_folder, \ delete_folder, delete_unknown_folders from brian_dashboard_manager.inventory_provider.interfaces import \ get_gws_direct, get_gws_indirect, get_interfaces, \ - get_eumetsat_multicast_subscriptions + get_eumetsat_multicast_subscriptions, get_nren_regions from brian_dashboard_manager.templating.helpers import \ get_aggregate_dashboard_data, get_interface_data, \ @@ -203,7 +203,7 @@ NREN_CATEGORIES = { } -def provision_folder(token_request, folder_name, dash, services, +def provision_folder(token_request, folder_name, dash, services, regions, ds_name, excluded_dashboards): """ Function to provision dashboards within a folder. @@ -213,6 +213,7 @@ def provision_folder(token_request, folder_name, dash, services, :param dash: the dashboards to provision, with interface data to generate the dashboards from :param services: service data from reporting provider for service-based dashboards + :param regions: region data from inventory provider to indicate what regions NRENs belong to :param ds_name: the name of the datasource to query in the dashboard panels :param excluded_dashboards: list of dashboards to exclude from provisioning for the organisation @@ -243,6 +244,10 @@ def provision_folder(token_request, folder_name, dash, services, # dashboard should include error panels errors = dash.get('errors', False) + nren_regions = { + region['nren']: region['region'] for region in regions + } + is_nren_legacy = folder_name == "NREN Access LEGACY" is_nren = folder_name == "NREN Access" nren_category = _get_nren_category(folder_name) # will be None if the NREN is not in a category @@ -251,7 +256,7 @@ def provision_folder(token_request, folder_name, dash, services, has_aggregate_panels = is_nren or is_nren_legacy or nren_category or is_re_peer or is_service def _filter_nren_interface_data_by_tag(tag=None): - raw_data = get_nren_interface_data(services, interfaces, excluded_dashboards) + raw_data = get_nren_interface_data(services, nren_regions, interfaces, excluded_dashboards) all_tags = NREN_CATEGORIES.keys() if tag is None: # only return interfaces that DO NOT have a category tag (uncategorized NRENs) @@ -425,6 +430,7 @@ def _provision_interfaces(config, org_config, ds_name, token): interfaces = get_interfaces(config['inventory_provider']) services = fetch_services(config['reporting_provider']) + regions = get_nren_regions(config['inventory_provider']) excluded_nrens = org_config['excluded_nrens'] excluded_folders = org_config.get('excluded_folders', {}) @@ -481,7 +487,7 @@ def _provision_interfaces(config, org_config, ds_name, token): f'Provisioning {org_config["name"]}/{folder_name} dashboards') res = executor.submit( provision_folder, token, - folder_name, folder, services, ds_name, + folder_name, folder, services, regions, ds_name, excluded_folder_dashboards(org_config, folder_name)) provisioned.append(res) @@ -652,7 +658,7 @@ def _provision_service_dashboards(config, org_config, ds_name, token): :return: generator of UIDs of dashboards that were created """ services = fetch_services(config['reporting_provider']) - + regions = get_nren_regions(config['inventory_provider']) excluded_folders = org_config.get('excluded_folders', {}) logger.info('Provisioning service-specific dashboards') @@ -684,7 +690,7 @@ def _provision_service_dashboards(config, org_config, ds_name, token): f'Provisioning {org_config["name"]}/{folder_name} dashboards') res = executor.submit( provision_folder, token, - folder_name, folder, services, ds_name, + folder_name, folder, services, regions, ds_name, excluded_folder_dashboards(org_config, folder_name)) provisioned.append(res) diff --git a/brian_dashboard_manager/inventory_provider/interfaces.py b/brian_dashboard_manager/inventory_provider/interfaces.py index f37adb6df8964fcf508601eb885c6476e929f8d6..03b21870b065439b2cff66ecfc5e89fae512c503 100644 --- a/brian_dashboard_manager/inventory_provider/interfaces.py +++ b/brian_dashboard_manager/inventory_provider/interfaces.py @@ -242,6 +242,22 @@ MULTICAST_SUBSCRIPTION_LIST_SCHEMA = { 'items': {'$ref': '#/definitions/subscription'} } +NREN_REGION_LIST_SCHEMA = { + '$schema': 'https://json-schema.org/draft-07/schema#', + 'definitions': { + 'nren_region': { + 'type': 'object', + 'properties': { + 'name': {'type': 'string'}, + 'region': {'type': 'string'} + } + } + }, + + 'type': 'array', + 'items': {'$ref': '#/definitions/nren_region'} +} + def _get_ip_info(host): """ @@ -410,3 +426,22 @@ def get_eumetsat_multicast_subscriptions(host): jsonschema.validate(data, MULTICAST_SUBSCRIPTION_LIST_SCHEMA) return data + + +def get_nren_regions(host): + """ + Get all NREN regions, where specified. + + :param host: Hostname to perform the request to. + :return: A list of NRENs and regions. + """ + try: + r = requests.get(f'{host}/poller/regions') + r.raise_for_status() + data = r.json() + except HTTPError: + logger.exception('Failed to get NREN regions') + data = [] + + jsonschema.validate(data, NREN_REGION_LIST_SCHEMA) + return data diff --git a/brian_dashboard_manager/templating/helpers.py b/brian_dashboard_manager/templating/helpers.py index e49063c8244fc014cad4a747e6b572f2908ee098..f9f940303392be4796e6f5ecd94f8b64ab5d248b 100644 --- a/brian_dashboard_manager/templating/helpers.py +++ b/brian_dashboard_manager/templating/helpers.py @@ -216,7 +216,7 @@ def get_re_peer_interface_data(interfaces): return result -def get_nren_interface_data(services, interfaces, excluded_dashboards): +def get_nren_interface_data(services, regions, interfaces, excluded_dashboards): """ Helper for grouping interface data to be used for generating dashboards for NRENs. @@ -224,6 +224,7 @@ def get_nren_interface_data(services, interfaces, excluded_dashboards): Extracts information from interfaces to be used in panels. :param services: list of services + :param regions: dict of NRENs to regions :param interfaces: list of interfaces :param excluded_dashboards: list of dashboards to exclude for the organization we are generating dashboards for @@ -359,14 +360,20 @@ def get_nren_interface_data(services, interfaces, excluded_dashboards): 'hostname': host, 'interface': interface_name }) + # add dashboard tags from interface dashboards_info for tag in interface.get('dashboards', []): dashboard['TAGS'].add(tag) result[dashboard_name] = dashboard for customer in list(result.keys()): + # add region tags + if customer in regions: + result[customer]['TAGS'].add(regions[customer]) + lengths = [len(val) for val in result[customer].values()] - if sum(lengths) == 0: + tags_length = len(result[customer]['TAGS']) # exclude tags from count + if sum(lengths) - tags_length == 0: # no services/interfaces, so remove it del result[customer] return result diff --git a/test/data/regions.json b/test/data/regions.json new file mode 100644 index 0000000000000000000000000000000000000000..a712bbe80d1eda3bed0449df1d5a4ae0b9b2f329 --- /dev/null +++ b/test/data/regions.json @@ -0,0 +1,22 @@ +[ + { + "nren": "URAN", + "region": "EAP" + }, + { + "nren": "ASNET-AM", + "region": "EAP" + }, + { + "nren": "AZSCIENCENET", + "region": "EAP" + }, + { + "nren": "GRENA", + "region": "EAP" + }, + { + "nren": "RENAM", + "region": "EAP" + } +] \ No newline at end of file diff --git a/test/test_update.py b/test/test_update.py index 222635387acb1d726b877ba42fc40333820e2311..ce93afc1ebc7e6d02a3c24ac1c7b2bec63e0d999 100644 --- a/test/test_update.py +++ b/test/test_update.py @@ -2,6 +2,7 @@ import pytest import responses from brian_dashboard_manager.grafana.provision import provision_folder, provision +from brian_dashboard_manager.inventory_provider.interfaces import get_nren_regions from brian_dashboard_manager.services.api import fetch_services TEST_INTERFACES = [ @@ -647,7 +648,7 @@ def reporting_provider(get_test_data, data_config): @pytest.fixture -def populate_inventory(data_config): +def populate_inventory(get_test_data, data_config): """function-fixture for provisioning inventory provider. Call it with a dictionary {url_path: contents}. ie {"/poller/interfaces": [...]} """ @@ -660,6 +661,12 @@ def populate_inventory(data_config): json=contents, ) + responses.add( + method=responses.GET, + url=f"{data_config['inventory_provider']}/poller/regions", + json=get_test_data("regions.json"), + ) + return _populate @@ -716,12 +723,14 @@ def test_provision_nren_folder( ) services = fetch_services(data_config['reporting_provider']) + regions = [] result = provision_folder( mock_grafana.request, folder_name, dashboards["NREN"], services, + regions, "testdatasource", excluded_nrens, ) @@ -827,7 +836,7 @@ def test_provision_re_peer_dashboard( @pytest.mark.parametrize( "folder_name, dashboard_id, expected_dashboard_count", [ - ('NREN Access', 'NREN', 6), # uncategorized interfaces + ('NREN Access', 'NREN', 5), # uncategorized interfaces ('EAP Access', 'EAP', 1) # EAP categorized interfaces ] ) @@ -864,12 +873,14 @@ def test_provision_nren_category( ) services = fetch_services(data_config['reporting_provider']) + regions = get_nren_regions(data_config['inventory_provider']) result = provision_folder( mock_grafana.request, folder_name, dashboards[dashboard_id], services, + regions, "testdatasource", [], )