diff --git a/brian_dashboard_manager/dashboards/region_eap.json b/brian_dashboard_manager/dashboards/region_eap.json new file mode 100644 index 0000000000000000000000000000000000000000..3e2933c88e731ed86d620620fb28f97cbc320305 --- /dev/null +++ b/brian_dashboard_manager/dashboards/region_eap.json @@ -0,0 +1,119 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 454, + "links": [], + "panels": [ + { + "datasource": null, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "folderId": null, + "gridPos": { + "h": 25, + "w": 24, + "x": 0, + "y": 0 + }, + "headings": false, + "id": 2, + "limit": 100, + "pluginVersion": "7.1.4", + "query": "", + "recent": false, + "search": true, + "starred": false, + "tags": [ + "eap" + ], + "targets": [ + { + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [] + } + ], + "timeFrom": null, + "timeShift": null, + "title": "", + "type": "dashlist" + } + ], + "schemaVersion": 26, + "style": "dark", + "tags": [ + "customers" + ], + "templating": { + "list": [] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "EAP Dashboard", + "version": 1 +} diff --git a/brian_dashboard_manager/grafana/provision.py b/brian_dashboard_manager/grafana/provision.py index b9ce5bf75380851e942d6df1864780127818d410..24e96aad08eb5ac94880a7295158ad30d1083b79 100644 --- a/brian_dashboard_manager/grafana/provision.py +++ b/brian_dashboard_manager/grafana/provision.py @@ -28,7 +28,7 @@ from brian_dashboard_manager.grafana.folder import find_folder, \ delete_folder, delete_unknown_folders from brian_dashboard_manager.inventory_provider.interfaces import \ get_gws_direct, get_gws_indirect, get_interfaces, \ - get_eumetsat_multicast_subscriptions + get_eumetsat_multicast_subscriptions, get_nren_regions from brian_dashboard_manager.templating.helpers import \ get_aggregate_dashboard_data, get_interface_data, \ @@ -58,6 +58,11 @@ DASHBOARDS = { 'folder_name': 'NREN Access', 'interfaces': [] }, + 'EAP': { + 'tag': ['eap'], + 'folder_name': 'EAP NREN Access', + 'interfaces': [] + }, 'RE_PEER': { 'tag': 'RE_PEER', 'folder_name': 'RE Peer', @@ -190,10 +195,15 @@ AGG_DASHBOARDS = { 'dashboard_name': 'ANA', 'interfaces': [] }, + 'EAP': { + 'tag': 'eap', + 'dashboard_name': 'EAP Aggregate', + 'interfaces': [] + } } -def provision_folder(token_request, folder_name, dash, services, +def provision_folder(token_request, folder_name, dash, services, regions, ds_name, excluded_dashboards): """ Function to provision dashboards within a folder. @@ -203,6 +213,7 @@ def provision_folder(token_request, folder_name, dash, services, :param dash: the dashboards to provision, with interface data to generate the dashboards from :param services: service data from reporting provider for service-based dashboards + :param regions: region data from inventory provider to indicate what regions NRENs belong to :param ds_name: the name of the datasource to query in the dashboard panels :param excluded_dashboards: list of dashboards to exclude from provisioning for the organisation @@ -224,21 +235,33 @@ def provision_folder(token_request, folder_name, dash, services, ) ) + def _get_customers_for_region(region=None): + customers = [] + region_lookup = {region['nren']: region['region'] for region in regions} + for service in services: + service_customers = service.get('customers', []) + for cust in service_customers: + cust_region = region_lookup.get(cust) + if cust_region == region: + customers.append(cust) + return customers + # dashboard should include error panels errors = dash.get('errors', False) is_nren_legacy = folder_name == "NREN Access LEGACY" is_nren = folder_name == "NREN Access" + is_eap = folder_name == "EAP NREN Access" is_re_peer = folder_name == "RE Peer" is_service = 'service_type' in dash - has_aggregate_panels = is_nren or is_nren_legacy or is_re_peer or is_service + has_aggregate_panels = is_nren or is_eap or is_nren_legacy or is_re_peer or is_service if is_nren_legacy: data = get_nren_interface_data_old(interfaces) dash_data = get_nren_dashboard_data(data, ds_name, tag) - elif is_nren: - data = get_nren_interface_data( - services, interfaces, excluded_dashboards) + elif is_nren or is_eap: + region_customers = _get_customers_for_region("EAP" if is_eap else None) + data = get_nren_interface_data(services, interfaces, excluded_dashboards, region_customers) dash_data = get_nren_dashboard_data(data, ds_name, tag) elif is_re_peer: data = get_re_peer_interface_data(interfaces) @@ -387,6 +410,7 @@ def _provision_interfaces(config, org_config, ds_name, token): interfaces = get_interfaces(config['inventory_provider']) services = fetch_services(config['reporting_provider']) + regions = get_nren_regions(config['inventory_provider']) excluded_nrens = org_config['excluded_nrens'] excluded_folders = org_config.get('excluded_folders', {}) @@ -443,7 +467,7 @@ def _provision_interfaces(config, org_config, ds_name, token): f'Provisioning {org_config["name"]}/{folder_name} dashboards') res = executor.submit( provision_folder, token, - folder_name, folder, services, ds_name, + folder_name, folder, services, regions, ds_name, excluded_folder_dashboards(org_config, folder_name)) provisioned.append(res) @@ -614,7 +638,7 @@ def _provision_service_dashboards(config, org_config, ds_name, token): :return: generator of UIDs of dashboards that were created """ services = fetch_services(config['reporting_provider']) - + regions = get_nren_regions(config['inventory_provider']) excluded_folders = org_config.get('excluded_folders', {}) logger.info('Provisioning service-specific dashboards') @@ -646,7 +670,7 @@ def _provision_service_dashboards(config, org_config, ds_name, token): f'Provisioning {org_config["name"]}/{folder_name} dashboards') res = executor.submit( provision_folder, token, - folder_name, folder, services, ds_name, + folder_name, folder, services, regions, ds_name, excluded_folder_dashboards(org_config, folder_name)) provisioned.append(res) @@ -902,7 +926,8 @@ def provision(config, raise_exceptions=False): 'GWS Indirect', 'GWS Direct', 'Aggregates', - 'EUMETSAT Multicast' + 'EUMETSAT Multicast', + 'EAP Dashboard' } folders_to_keep.update({dash['folder_name'] for dash in DASHBOARDS.values()}) diff --git a/brian_dashboard_manager/inventory_provider/interfaces.py b/brian_dashboard_manager/inventory_provider/interfaces.py index f37adb6df8964fcf508601eb885c6476e929f8d6..7a74d987b4601a87dd7588f722dd6cc3c4dc2f9a 100644 --- a/brian_dashboard_manager/inventory_provider/interfaces.py +++ b/brian_dashboard_manager/inventory_provider/interfaces.py @@ -242,6 +242,22 @@ MULTICAST_SUBSCRIPTION_LIST_SCHEMA = { 'items': {'$ref': '#/definitions/subscription'} } +NREN_REGION_LIST_SCHEMA = { + '$schema': 'https://json-schema.org/draft-07/schema#', + 'definitions': { + 'nren_region': { + 'type': 'object', + 'properties': { + 'nren': {'type': 'string'}, + 'region': {'type': 'string'} + } + } + }, + + 'type': 'array', + 'items': {'$ref': '#/definitions/nren_region'} +} + def _get_ip_info(host): """ @@ -410,3 +426,22 @@ def get_eumetsat_multicast_subscriptions(host): jsonschema.validate(data, MULTICAST_SUBSCRIPTION_LIST_SCHEMA) return data + + +def get_nren_regions(host): + """ + Get all NREN regions, where specified. + + :param host: Hostname to perform the request to. + :return: A list of NRENs and regions. + """ + try: + r = requests.get(f'{host}/poller/regions') + r.raise_for_status() + data = r.json() + except HTTPError: + logger.exception('Failed to get NREN regions') + data = [] + + jsonschema.validate(data, NREN_REGION_LIST_SCHEMA) + return data diff --git a/brian_dashboard_manager/templating/helpers.py b/brian_dashboard_manager/templating/helpers.py index a5aa9e9ed6d8058af115767aaa4144e041a25b8c..01d1528aef000d486a659d66025ce6ada6084b2d 100644 --- a/brian_dashboard_manager/templating/helpers.py +++ b/brian_dashboard_manager/templating/helpers.py @@ -8,6 +8,7 @@ import logging from itertools import product from functools import partial, reduce from string import ascii_uppercase + from brian_dashboard_manager.templating.render import create_panel, \ create_panel_target, create_dropdown_panel @@ -216,7 +217,7 @@ def get_re_peer_interface_data(interfaces): return result -def get_nren_interface_data(services, interfaces, excluded_dashboards): +def get_nren_interface_data(services, interfaces, excluded_dashboards, region_customers): """ Helper for grouping interface data to be used for generating dashboards for NRENs. @@ -227,6 +228,8 @@ def get_nren_interface_data(services, interfaces, excluded_dashboards): :param interfaces: list of interfaces :param excluded_dashboards: list of dashboards to exclude for the organization we are generating dashboards for + :param region_customers: list of customers in the region we are + generating dashboards for (currently used to separate EAP NRENs) :return: dictionary of dashboards and their service/interface data """ @@ -237,10 +240,12 @@ def get_nren_interface_data(services, interfaces, excluded_dashboards): aggregate_interfaces = dict() for service in services: - _customers = service.get('customers') + _customers = service.get('customers', []) for cust in _customers: if cust.lower() in excluded_dashboards: continue + if cust not in region_customers: + continue customers[cust].append(service) for customer, services in customers.items(): diff --git a/changelog.md b/changelog.md index 6eb8973c76359b0f674658976bcbbf618644d9e9..b1f465b75ac4a4f8daefad021cb10a9accfbebc6 100644 --- a/changelog.md +++ b/changelog.md @@ -2,6 +2,9 @@ All notable changes to this project will be documented in this file. +## [0.66] - 2024-10-14 +- POL1-430 - Add EAP NREN Access dashboard to access EAP NRENs and aggregate dashboard + ## [0.65] - 2024-09-23 - Support Grafana v11.3 layout & API changes diff --git a/setup.py b/setup.py index be5db1ef6741eb993ae6a217bcbe2aafb3baaa7a..07237d5a6535a8b2f8b19f31c6cce1f60e7560a0 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import setup, find_packages setup( name='brian-dashboard-manager', - version="0.65", + version="0.66", author='GEANT', author_email='swd@geant.org', description='', diff --git a/test/test_update.py b/test/test_update.py index 16800bf946b492b745e0d13ddabf1f365620e25a..5cbd8e38f5fa10d5e9554919cf5ec0fc3460da4f 100644 --- a/test/test_update.py +++ b/test/test_update.py @@ -2,6 +2,7 @@ import pytest import responses from brian_dashboard_manager.grafana.provision import provision_folder, provision +from brian_dashboard_manager.inventory_provider.interfaces import get_nren_regions from brian_dashboard_manager.services.api import fetch_services TEST_INTERFACES = [ @@ -280,6 +281,40 @@ TEST_INTERFACES = [ }], "ipv4": [], "ipv6": [] + }, + { + + "router": "rt1.fra.de.geant.net", + "name": "xe-11/2/5.300", + "bundle": [], + "bundle-parents": [], + "description": "SRV_GLOBAL CUSTOMER AZSCIENCENET #AZSCIENCENET-AP1 $GS-00433 | ASN202993 |", + "circuits": [ + { + "id": 723799, + "name": "AZSCIENCENET-AP1", + "type": "GEANT IP", + "status": "operational" + } + ], + "snmp-index": 1033, + "dashboards": [ + "EAP", + "RE_CUST" + ], + "dashboard_info": { + "name": "AZSCIENCENET", + "interface_type": "LOGICAL" + }, + "dashboards_info": [ + { + "name": "AZSCIENCENET", + "interface_type": "LOGICAL" + } + ], + "port_type": "SERVICE", + "ipv4": [], + "ipv6": [] } ] @@ -288,7 +323,7 @@ NREN_INTERFACES = [ "router": "mx1.dub2.ie.geant.net", "name": "xe-0/0/0", "bundle": [ - "ae10" + "ae10" ], "bundle-parents": [], "description": "PHY CUSTOMER HEANET P_AE10 SRF9948758 | HEANET-AP2-LL3", # noqa: E501 @@ -313,7 +348,7 @@ NREN_INTERFACES = [ "router": "mx1.dub2.ie.geant.net", "name": "xe-1/0/1", "bundle": [ - "ae10" + "ae10" ], "bundle-parents": [], "description": "PHY CUSTOMER HEANET P_AE10 SRF0000001 | HEANET-AP2-LL2", # noqa: E501 @@ -338,7 +373,7 @@ NREN_INTERFACES = [ "router": "mx1.dub2.ie.geant.net", "name": "xe-1/1/0", "bundle": [ - "ae10" + "ae10" ], "bundle-parents": [], "description": "PHY CUSTOMER HEANET P_AE10 SRF9925903 | HEANET-AP2-LL1", # noqa: E501 @@ -364,9 +399,9 @@ NREN_INTERFACES = [ "name": "ae10", "bundle": [], "bundle-parents": [ - "xe-0/0/0", - "xe-1/0/1", - "xe-1/1/0" + "xe-0/0/0", + "xe-1/0/1", + "xe-1/1/0" ], "description": "LAG CUSTOMER HEANET SRF9925909 | HEANET-AP2-LAG", "circuits": [], @@ -391,9 +426,9 @@ NREN_INTERFACES = [ "name": "ae10.12", "bundle": [], "bundle-parents": [ - "xe-0/0/0", - "xe-1/0/1", - "xe-1/1/0" + "xe-0/0/0", + "xe-1/0/1", + "xe-1/1/0" ], "description": "SRV_GLOBAL CUSTOMER HEANET #HEANET-AP2 | ASN1213 | ", "circuits": [ @@ -429,9 +464,9 @@ NREN_INTERFACES = [ "name": "ae10.30", "bundle": [], "bundle-parents": [ - "xe-0/0/0", - "xe-1/0/1", - "xe-1/1/0" + "xe-0/0/0", + "xe-1/0/1", + "xe-1/1/0" ], "description": "SRV_MDVPN CUSTOMER HEANET AP2 #HEANET-BGP-LU-CoC-1 |", # noqa: E501 "circuits": [ @@ -465,9 +500,9 @@ NREN_INTERFACES = [ "name": "ae10.333", "bundle": [], "bundle-parents": [ - "xe-0/0/0", - "xe-1/0/1", - "xe-1/1/0" + "xe-0/0/0", + "xe-1/0/1", + "xe-1/1/0" ], "description": "SRV_IAS CUSTOMER HEANET #HEANET-AP2-IAS IASPS | ASN1213 ", # noqa: E501 "circuits": [ @@ -503,9 +538,9 @@ NREN_INTERFACES = [ "name": "ae10.1214", "bundle": [], "bundle-parents": [ - "xe-0/0/0", - "xe-1/0/1", - "xe-1/1/0" + "xe-0/0/0", + "xe-1/0/1", + "xe-1/1/0" ], "description": "SRV_L2CIRCUIT CUSTOMER HEANET GEANT #ams-dub2-HEANET-RARE-21061 |", # noqa: E501 "circuits": [ @@ -527,13 +562,46 @@ NREN_INTERFACES = [ "dashboards_info": [{ "name": "GEANT", "interface_type": "LOGICAL" - }, - { + }, { "name": "HEANET", "interface_type": "LOGICAL" }], "ipv4": [], "ipv6": [] + }, + { + + "router": "rt1.fra.de.geant.net", + "name": "xe-11/2/5.300", + "bundle": [], + "bundle-parents": [], + "description": "SRV_GLOBAL CUSTOMER AZSCIENCENET #AZSCIENCENET-AP1 $GS-00433 | ASN202993 |", + "circuits": [ + { + "id": 723799, + "name": "AZSCIENCENET-AP1", + "type": "GEANT IP", + "status": "operational" + } + ], + "snmp-index": 1033, + "dashboards": [ + "EAP", + "RE_CUST" + ], + "dashboard_info": { + "name": "AZSCIENCENET", + "interface_type": "LOGICAL" + }, + "dashboards_info": [ + { + "name": "AZSCIENCENET", + "interface_type": "LOGICAL" + } + ], + "port_type": "SERVICE", + "ipv4": [], + "ipv6": [] } ] @@ -582,6 +650,29 @@ EUMETSAT_MULTICAST = [ } ] +NREN_REGIONS = [ + { + "nren": "URAN", + "region": "EAP" + }, + { + "nren": "ASNET-AM", + "region": "EAP" + }, + { + "nren": "AZSCIENCENET", + "region": "EAP" + }, + { + "nren": "GRENA", + "region": "EAP" + }, + { + "nren": "RENAM", + "region": "EAP" + } +] + def generate_folder(data): return { @@ -611,7 +702,7 @@ def reporting_provider(get_test_data, data_config): @pytest.fixture -def populate_inventory(data_config): +def populate_inventory(get_test_data, data_config): """function-fixture for provisioning inventory provider. Call it with a dictionary {url_path: contents}. ie {"/poller/interfaces": [...]} """ @@ -631,29 +722,29 @@ def populate_inventory(data_config): @pytest.mark.parametrize( "folder_name, excluded_nrens, expected_nrens", [ - ("NREN Access", [], ['ASNET-AM', 'CESNET', 'GEANT', 'KIAE', 'LITNET', 'SWITCH']), - ("NREN Access", ["GEANT", "KIAE"], ['ASNET-AM', 'CESNET', 'LITNET', 'SWITCH']), + ("NREN Access", [], ['CESNET', 'GEANT', 'KIAE', 'LITNET', 'SWITCH']), + ("NREN Access", ["GEANT", "KIAE"], ['CESNET', 'LITNET', 'SWITCH']), ( - "NREN Access", - [], - ["ASNET-AM", "LITNET", "CESNET", "GEANT", "KIAE", "SWITCH"], + "NREN Access", + [], + ["LITNET", "CESNET", "GEANT", "KIAE", "SWITCH"], ), ( - "NREN Access", - ["ASNET-AM", "GEANT"], - ["LITNET", "CESNET", "KIAE", "SWITCH"], + "NREN Access", + ["GEANT"], + ["LITNET", "CESNET", "KIAE", "SWITCH"], ), ("testfolder", ["GEANT"], ["KIAE", "SWITCH"]), ], ) def test_provision_nren_folder( - folder_name, - excluded_nrens, - expected_nrens, - data_config, - mock_grafana, - reporting_provider, - populate_inventory, + folder_name, + excluded_nrens, + expected_nrens, + data_config, + mock_grafana, + reporting_provider, + populate_inventory, ): dashboards = { "NREN": { @@ -676,16 +767,19 @@ def test_provision_nren_folder( "/poller/interfaces": NREN_INTERFACES, "/data/interfaces": NREN_INTERFACES, "/poller/eumetsat-multicast": EUMETSAT_MULTICAST, + "/poller/regions": NREN_REGIONS, } ) services = fetch_services(data_config['reporting_provider']) + regions = get_nren_regions(data_config['inventory_provider']) result = provision_folder( mock_grafana.request, folder_name, dashboards["NREN"], services, + regions, "testdatasource", excluded_nrens, ) @@ -700,7 +794,7 @@ def test_provision_nren_folder( @responses.activate def test_provision( - data_config, mocker, mock_grafana, reporting_provider, populate_inventory + data_config, mocker, mock_grafana, reporting_provider, populate_inventory ): mock_grafana.create_datasource( { @@ -719,6 +813,7 @@ def test_provision( "/poller/interfaces": NREN_INTERFACES, "/data/interfaces": NREN_INTERFACES, "/poller/eumetsat-multicast": EUMETSAT_MULTICAST, + "/poller/regions": NREN_REGIONS, } ) for org in data_config["organizations"][1:]: @@ -739,7 +834,7 @@ def test_provision( @responses.activate def test_provision_re_peer_dashboard( - mocker, data_config, mock_grafana, reporting_provider, populate_inventory + mocker, data_config, mock_grafana, reporting_provider, populate_inventory ): interfaces = [ { @@ -758,6 +853,7 @@ def test_provision_re_peer_dashboard( "/poller/interfaces": interfaces, "/data/interfaces": interfaces, "/poller/eumetsat-multicast": EUMETSAT_MULTICAST, + "/poller/regions": NREN_REGIONS, } ) _mocked_gws = mocker.patch( @@ -785,3 +881,62 @@ def test_provision_re_peer_dashboard( assert "traffic" in panels[4]["title"] assert "IPv6" in panels[5]["title"] assert "Interfaces" in panels[6]["title"] + + +@responses.activate +@pytest.mark.parametrize( + "folder_name, dashboard_id, expected_dashboard_count", + [ + ('NREN Access', 'NREN', 5), + ('EAP NREN Access', 'EAP', 1) + ] +) +def test_provision_nren_category( + folder_name, + dashboard_id, + expected_dashboard_count, + data_config, + mock_grafana, + reporting_provider, + populate_inventory, +): + nren_interfaces = [ + iface for iface in TEST_INTERFACES if "NREN" in iface["dashboards"] + ] + eap_interfaces = [ + iface for iface in TEST_INTERFACES if "EAP" in iface["dashboards"] + ] + dashboards = { + "NREN": { + "tag": ["customers"], + "folder_name": "NREN Access", + "interfaces": nren_interfaces, + }, + "EAP": { + "tag": ["eap"], + "folder_name": "EAP NREN Access", + "interfaces": eap_interfaces, + }, + } + populate_inventory( + { + "/poller/interfaces": NREN_INTERFACES, + "/data/interfaces": NREN_INTERFACES, + "/poller/eumetsat-multicast": EUMETSAT_MULTICAST, + "/poller/regions": NREN_REGIONS, + } + ) + + services = fetch_services(data_config['reporting_provider']) + regions = get_nren_regions(data_config['inventory_provider']) + + result = provision_folder( + mock_grafana.request, + folder_name, + dashboards[dashboard_id], + services, + regions, + "testdatasource", + [], + ) + assert len(result) == expected_dashboard_count diff --git a/tox.ini b/tox.ini index 6b749b99e29ebfbdbe4980c958b1845fba1e7ae5..b5793fc9f433c662460444f24b58a19d8e0d5289 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py38, py311 +envlist = py311 [flake8] exclude = venv,.tox