diff --git a/brian_dashboard_manager/dashboards/region_eap.json b/brian_dashboard_manager/dashboards/region_eap.json index 78e9aa932a2d59a4aaf0cc88ef7d3918599ce8d2..3e2933c88e731ed86d620620fb28f97cbc320305 100644 --- a/brian_dashboard_manager/dashboards/region_eap.json +++ b/brian_dashboard_manager/dashboards/region_eap.json @@ -114,6 +114,6 @@ ] }, "timezone": "", - "title": "EAP Access", + "title": "EAP Dashboard", "version": 1 } diff --git a/brian_dashboard_manager/grafana/provision.py b/brian_dashboard_manager/grafana/provision.py index 55faf80ed1dd47562b06d1b44667f3379b80d8fb..fe4b3d13edf4eeced15e655fb2293f55e50f0ca1 100644 --- a/brian_dashboard_manager/grafana/provision.py +++ b/brian_dashboard_manager/grafana/provision.py @@ -46,7 +46,7 @@ from brian_dashboard_manager.templating.render import ( logger = logging.getLogger(__name__) -MAX_WORKERS = 4 +MAX_WORKERS = 1 DASHBOARDS = { 'NRENLEGACY': { 'tag': ['customerslegacy'], @@ -60,7 +60,7 @@ DASHBOARDS = { }, 'EAP': { 'tag': ['eap'], - 'folder_name': 'EAP NRENs', + 'folder_name': 'EAP NREN Access', 'interfaces': [] }, 'RE_PEER': { @@ -202,12 +202,6 @@ AGG_DASHBOARDS = { } } -NREN_CATEGORIES = { - 'EAP': { - 'folder_name': 'EAP NRENs' - } -} - def provision_folder(token_request, folder_name, dash, services, regions, ds_name, excluded_dashboards): @@ -241,53 +235,33 @@ def provision_folder(token_request, folder_name, dash, services, regions, ) ) - def _get_nren_category(_folder_name): - for category in NREN_CATEGORIES: - if _folder_name == NREN_CATEGORIES[category]['folder_name']: - return category - return None + def _get_customers_for_region(region=None): + customers = [] + region_lookup = {region['nren']: region['region'] for region in regions} + for service in services: + service_customers = service.get('customers') + for cust in service_customers: + cust_region = region_lookup.get(cust, None) + if cust_region == region: + customers.append(cust) + return customers # dashboard should include error panels errors = dash.get('errors', False) - nren_regions = { - region['nren']: region['region'] for region in regions - } - is_nren_legacy = folder_name == "NREN Access LEGACY" is_nren = folder_name == "NREN Access" - nren_category = _get_nren_category(folder_name) # will be None if the NREN is not in a category + is_eap = folder_name == "EAP NREN Access" is_re_peer = folder_name == "RE Peer" is_service = 'service_type' in dash - has_aggregate_panels = is_nren or is_nren_legacy or nren_category or is_re_peer or is_service - - def _filter_nren_interface_data_by_tag(tag=None): - raw_data = get_nren_interface_data(services, nren_regions, interfaces, excluded_dashboards) - all_tags = NREN_CATEGORIES.keys() - if tag is None: - # only return interfaces that DO NOT have a category tag (uncategorized NRENs) - for interface_key, interface in raw_data.items(): - tags = interface.get('TAGS', set()) - not_parented = True - for interface_tag in tags: - if interface_tag in all_tags: - not_parented = False - break - if not_parented: - yield interface_key, interface - else: - for interface_key, interface in raw_data.items(): - if tag in interface.get('TAGS', set()): - yield interface_key, interface + has_aggregate_panels = is_nren or is_eap or is_nren_legacy or is_re_peer or is_service if is_nren_legacy: data = get_nren_interface_data_old(interfaces) dash_data = get_nren_dashboard_data(data, ds_name, tag) - elif is_nren: - data = dict(_filter_nren_interface_data_by_tag(None)) - dash_data = get_nren_dashboard_data(data, ds_name, tag) - elif nren_category: - data = dict(_filter_nren_interface_data_by_tag(nren_category)) + elif is_nren or is_eap: + region_customers = _get_customers_for_region("EAP" if is_eap else None) + data = get_nren_interface_data(services, interfaces, excluded_dashboards, region_customers) dash_data = get_nren_dashboard_data(data, ds_name, tag) elif is_re_peer: data = get_re_peer_interface_data(interfaces) @@ -963,7 +937,8 @@ def provision(config, raise_exceptions=False): 'GWS Indirect', 'GWS Direct', 'Aggregates', - 'EUMETSAT Multicast' + 'EUMETSAT Multicast', + 'EAP Dashboard' } folders_to_keep.update({dash['folder_name'] for dash in DASHBOARDS.values()}) diff --git a/brian_dashboard_manager/inventory_provider/interfaces.py b/brian_dashboard_manager/inventory_provider/interfaces.py index 03b21870b065439b2cff66ecfc5e89fae512c503..7a74d987b4601a87dd7588f722dd6cc3c4dc2f9a 100644 --- a/brian_dashboard_manager/inventory_provider/interfaces.py +++ b/brian_dashboard_manager/inventory_provider/interfaces.py @@ -248,7 +248,7 @@ NREN_REGION_LIST_SCHEMA = { 'nren_region': { 'type': 'object', 'properties': { - 'name': {'type': 'string'}, + 'nren': {'type': 'string'}, 'region': {'type': 'string'} } } diff --git a/brian_dashboard_manager/templating/helpers.py b/brian_dashboard_manager/templating/helpers.py index db75f5ec0236901f808465aaeef0af9d5df5cb49..3621734e1681e276bae9e2f990cecef93f03ab9b 100644 --- a/brian_dashboard_manager/templating/helpers.py +++ b/brian_dashboard_manager/templating/helpers.py @@ -9,7 +9,6 @@ from itertools import product from functools import partial, reduce from string import ascii_uppercase -from brian_dashboard_manager.grafana import provision from brian_dashboard_manager.templating.render import create_panel, \ create_panel_target, create_dropdown_panel @@ -218,7 +217,7 @@ def get_re_peer_interface_data(interfaces): return result -def get_nren_interface_data(services, regions, interfaces, excluded_dashboards): +def get_nren_interface_data(services, interfaces, excluded_dashboards, region_customers): """ Helper for grouping interface data to be used for generating dashboards for NRENs. @@ -226,10 +225,11 @@ def get_nren_interface_data(services, regions, interfaces, excluded_dashboards): Extracts information from interfaces to be used in panels. :param services: list of services - :param regions: dict of NRENs to regions :param interfaces: list of interfaces :param excluded_dashboards: list of dashboards to exclude for the organization we are generating dashboards for + :param region_customers: list of customers in the region we are + generating dashboards for (currently used to separate EAP NRENs) :return: dictionary of dashboards and their service/interface data """ @@ -244,6 +244,8 @@ def get_nren_interface_data(services, regions, interfaces, excluded_dashboards): for cust in _customers: if cust.lower() in excluded_dashboards: continue + if cust not in region_customers: + continue customers[cust].append(service) for customer, services in customers.items(): @@ -254,8 +256,7 @@ def get_nren_interface_data(services, regions, interfaces, excluded_dashboards): dashboard = result.setdefault(customer, { 'AGGREGATES': [], 'SERVICES': [], - 'PHYSICAL': [], - 'TAGS': set() + 'PHYSICAL': [] }) for service in services: @@ -331,14 +332,11 @@ def get_nren_interface_data(services, regions, interfaces, excluded_dashboards): for info in dashboards_info: dashboard_name = info['name'] - if dashboard_name in provision.NREN_CATEGORIES: # do not create panels for the category itself - continue dashboard = result.get(dashboard_name, { 'AGGREGATES': [], 'SERVICES': [], - 'PHYSICAL': [], - 'TAGS': set() + 'PHYSICAL': [] }) _covered_by_service = _check_in_aggregate(router, interface_name) @@ -364,20 +362,12 @@ def get_nren_interface_data(services, regions, interfaces, excluded_dashboards): 'hostname': host, 'interface': interface_name }) - # add dashboard tags from interface dashboards_info - for tag in interface.get('dashboards', []): - dashboard['TAGS'].add(tag) result[dashboard_name] = dashboard for customer in list(result.keys()): - # add region tags - if customer in regions: - result[customer]['TAGS'].add(regions[customer]) - lengths = [len(val) for val in result[customer].values()] - tags_length = len(result[customer]['TAGS']) # exclude tags from count - if sum(lengths) - tags_length == 0: + if sum(lengths) == 0: # no services/interfaces, so remove it del result[customer] return result diff --git a/test/data/regions.json b/test/data/regions.json deleted file mode 100644 index a712bbe80d1eda3bed0449df1d5a4ae0b9b2f329..0000000000000000000000000000000000000000 --- a/test/data/regions.json +++ /dev/null @@ -1,22 +0,0 @@ -[ - { - "nren": "URAN", - "region": "EAP" - }, - { - "nren": "ASNET-AM", - "region": "EAP" - }, - { - "nren": "AZSCIENCENET", - "region": "EAP" - }, - { - "nren": "GRENA", - "region": "EAP" - }, - { - "nren": "RENAM", - "region": "EAP" - } -] \ No newline at end of file diff --git a/test/test_update.py b/test/test_update.py index 0d14a71c7bca25594cd87f385f0ee70031e9bfa4..5cbd8e38f5fa10d5e9554919cf5ec0fc3460da4f 100644 --- a/test/test_update.py +++ b/test/test_update.py @@ -300,7 +300,6 @@ TEST_INTERFACES = [ "snmp-index": 1033, "dashboards": [ "EAP", - "NREN", "RE_CUST" ], "dashboard_info": { @@ -569,6 +568,40 @@ NREN_INTERFACES = [ }], "ipv4": [], "ipv6": [] + }, + { + + "router": "rt1.fra.de.geant.net", + "name": "xe-11/2/5.300", + "bundle": [], + "bundle-parents": [], + "description": "SRV_GLOBAL CUSTOMER AZSCIENCENET #AZSCIENCENET-AP1 $GS-00433 | ASN202993 |", + "circuits": [ + { + "id": 723799, + "name": "AZSCIENCENET-AP1", + "type": "GEANT IP", + "status": "operational" + } + ], + "snmp-index": 1033, + "dashboards": [ + "EAP", + "RE_CUST" + ], + "dashboard_info": { + "name": "AZSCIENCENET", + "interface_type": "LOGICAL" + }, + "dashboards_info": [ + { + "name": "AZSCIENCENET", + "interface_type": "LOGICAL" + } + ], + "port_type": "SERVICE", + "ipv4": [], + "ipv6": [] } ] @@ -617,6 +650,29 @@ EUMETSAT_MULTICAST = [ } ] +NREN_REGIONS = [ + { + "nren": "URAN", + "region": "EAP" + }, + { + "nren": "ASNET-AM", + "region": "EAP" + }, + { + "nren": "AZSCIENCENET", + "region": "EAP" + }, + { + "nren": "GRENA", + "region": "EAP" + }, + { + "nren": "RENAM", + "region": "EAP" + } +] + def generate_folder(data): return { @@ -659,12 +715,6 @@ def populate_inventory(get_test_data, data_config): json=contents, ) - responses.add( - method=responses.GET, - url=f"{data_config['inventory_provider']}/poller/regions", - json=get_test_data("regions.json"), - ) - return _populate @@ -672,19 +722,19 @@ def populate_inventory(get_test_data, data_config): @pytest.mark.parametrize( "folder_name, excluded_nrens, expected_nrens", [ - ("NREN Access", [], ['ASNET-AM', 'CESNET', 'GEANT', 'KIAE', 'LITNET', 'SWITCH']), - ("NREN Access", ["GEANT", "KIAE"], ['ASNET-AM', 'CESNET', 'LITNET', 'SWITCH']), + ("NREN Access", [], ['CESNET', 'GEANT', 'KIAE', 'LITNET', 'SWITCH']), + ("NREN Access", ["GEANT", "KIAE"], ['CESNET', 'LITNET', 'SWITCH']), ( "NREN Access", [], - ["ASNET-AM", "LITNET", "CESNET", "GEANT", "KIAE", "SWITCH"], + ["LITNET", "CESNET", "GEANT", "KIAE", "SWITCH"], ), ( "NREN Access", - ["ASNET-AM", "GEANT"], + ["GEANT"], ["LITNET", "CESNET", "KIAE", "SWITCH"], ), - ("testfolder", ["GEANT"], ["KIAE", "SWITCH", "AZSCIENCENET"]), + ("testfolder", ["GEANT"], ["KIAE", "SWITCH"]), ], ) def test_provision_nren_folder( @@ -717,11 +767,12 @@ def test_provision_nren_folder( "/poller/interfaces": NREN_INTERFACES, "/data/interfaces": NREN_INTERFACES, "/poller/eumetsat-multicast": EUMETSAT_MULTICAST, + "/poller/regions": NREN_REGIONS, } ) services = fetch_services(data_config['reporting_provider']) - regions = [] + regions = get_nren_regions(data_config['inventory_provider']) result = provision_folder( mock_grafana.request, @@ -762,6 +813,7 @@ def test_provision( "/poller/interfaces": NREN_INTERFACES, "/data/interfaces": NREN_INTERFACES, "/poller/eumetsat-multicast": EUMETSAT_MULTICAST, + "/poller/regions": NREN_REGIONS, } ) for org in data_config["organizations"][1:]: @@ -801,6 +853,7 @@ def test_provision_re_peer_dashboard( "/poller/interfaces": interfaces, "/data/interfaces": interfaces, "/poller/eumetsat-multicast": EUMETSAT_MULTICAST, + "/poller/regions": NREN_REGIONS, } ) _mocked_gws = mocker.patch( @@ -834,8 +887,8 @@ def test_provision_re_peer_dashboard( @pytest.mark.parametrize( "folder_name, dashboard_id, expected_dashboard_count", [ - ('NREN Access', 'NREN', 5), # uncategorized interfaces - ('EAP Access', 'EAP', 1) # EAP categorized interfaces + ('NREN Access', 'NREN', 5), + ('EAP NREN Access', 'EAP', 1) ] ) def test_provision_nren_category( @@ -850,6 +903,9 @@ def test_provision_nren_category( nren_interfaces = [ iface for iface in TEST_INTERFACES if "NREN" in iface["dashboards"] ] + eap_interfaces = [ + iface for iface in TEST_INTERFACES if "EAP" in iface["dashboards"] + ] dashboards = { "NREN": { "tag": ["customers"], @@ -858,8 +914,8 @@ def test_provision_nren_category( }, "EAP": { "tag": ["eap"], - "folder_name": "EAP Access", - "interfaces": nren_interfaces, + "folder_name": "EAP NREN Access", + "interfaces": eap_interfaces, }, } populate_inventory( @@ -867,6 +923,7 @@ def test_provision_nren_category( "/poller/interfaces": NREN_INTERFACES, "/data/interfaces": NREN_INTERFACES, "/poller/eumetsat-multicast": EUMETSAT_MULTICAST, + "/poller/regions": NREN_REGIONS, } )