diff --git a/brian_dashboard_manager/dashboards/services_mws.json b/brian_dashboard_manager/dashboards/services_mws.json
new file mode 100755
index 0000000000000000000000000000000000000000..39c9a333c0d644e486f673d7ccd14557842852c7
--- /dev/null
+++ b/brian_dashboard_manager/dashboards/services_mws.json
@@ -0,0 +1,119 @@
+{
+ "annotations": {
+ "list": [
+ {
+ "builtIn": 1,
+ "datasource": "-- Grafana --",
+ "enable": true,
+ "hide": true,
+ "iconColor": "rgba(0, 211, 255, 1)",
+ "name": "Annotations & Alerts",
+ "type": "dashboard"
+ }
+ ]
+ },
+ "editable": true,
+ "gnetId": null,
+ "graphTooltip": 0,
+ "id": 454,
+ "links": [],
+ "panels": [
+ {
+ "datasource": null,
+ "fieldConfig": {
+ "defaults": {
+ "custom": {}
+ },
+ "overrides": []
+ },
+ "folderId": null,
+ "gridPos": {
+ "h": 25,
+ "w": 24,
+ "x": 0,
+ "y": 0
+ },
+ "headings": false,
+ "id": 2,
+ "limit": 100,
+ "pluginVersion": "7.1.4",
+ "query": "",
+ "recent": false,
+ "search": true,
+ "starred": false,
+ "tags": [
+ "mws"
+ ],
+ "targets": [
+ {
+ "groupBy": [
+ {
+ "params": [
+ "$__interval"
+ ],
+ "type": "time"
+ },
+ {
+ "params": [
+ "null"
+ ],
+ "type": "fill"
+ }
+ ],
+ "orderByTime": "ASC",
+ "policy": "default",
+ "refId": "A",
+ "resultFormat": "time_series",
+ "select": [
+ [
+ {
+ "params": [
+ "value"
+ ],
+ "type": "field"
+ },
+ {
+ "params": [],
+ "type": "mean"
+ }
+ ]
+ ],
+ "tags": []
+ }
+ ],
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "",
+ "type": "dashlist"
+ }
+ ],
+ "schemaVersion": 26,
+ "style": "dark",
+ "tags": [
+ "services"
+ ],
+ "templating": {
+ "list": []
+ },
+ "time": {
+ "from": "now-6h",
+ "to": "now"
+ },
+ "timepicker": {
+ "refresh_intervals": [
+ "5s",
+ "10s",
+ "30s",
+ "1m",
+ "5m",
+ "15m",
+ "30m",
+ "1h",
+ "2h",
+ "1d"
+ ]
+ },
+ "timezone": "",
+ "title": "GÉANT Managed Wavelength Service",
+ "version": 1
+}
diff --git a/brian_dashboard_manager/grafana/provision.py b/brian_dashboard_manager/grafana/provision.py
index 0259d48414956946ce6b2f990af970e0a195e1ed..1eb6614792e4c51edd483f94ff09fba05a8c1262 100644
--- a/brian_dashboard_manager/grafana/provision.py
+++ b/brian_dashboard_manager/grafana/provision.py
@@ -33,7 +33,8 @@ from brian_dashboard_manager.templating.helpers import \
get_aggregate_dashboard_data, get_interface_data, \
get_nren_interface_data, get_dashboard_data, \
get_nren_dashboard_data, get_aggregate_interface_data, \
- get_nren_interface_data_old, get_re_peer_dashboard_data, get_re_peer_interface_data
+ get_nren_interface_data_old, get_re_peer_dashboard_data, get_re_peer_interface_data, get_service_data, \
+ get_service_dashboard_data
from brian_dashboard_manager.templating.gws import generate_gws, generate_indirect
from brian_dashboard_manager.templating.eumetsat import generate_eumetsat_multicast
@@ -136,6 +137,16 @@ DASHBOARDS = {
}
}
+SERVICE_DASHBOARDS = {
+ 'MWS': {
+ 'tag': ['mws'],
+ 'service_type': 'GEANT MANAGED WAVELENGTH SERVICE',
+ 'folder_name': 'Managed Wavelength Service',
+ 'interfaces': [],
+ 'services': []
+ }
+}
+
AGG_DASHBOARDS = {
'CLS_PEERS': {
'tag': 'cls_peers',
@@ -181,8 +192,8 @@ AGG_DASHBOARDS = {
}
-def provision_folder(token_request, folder_name, dash,
- config, ds_name, excluded_dashboards):
+def provision_folder(token_request, folder_name, dash, services,
+ ds_name, excluded_dashboards):
"""
Function to provision dashboards within a folder.
@@ -190,7 +201,7 @@ def provision_folder(token_request, folder_name, dash,
:param folder_name: Name of the folder to provision dashboards in
:param dash: the dashboards to provision, with interface data to generate
the dashboards from
- :param config: the application config
+ :param services: service data from reporting provider for service-based dashboards
:param ds_name: the name of the datasource to query in the dashboard panels
:param excluded_dashboards: list of dashboards to exclude from provisioning
for the organisation
@@ -218,19 +229,24 @@ def provision_folder(token_request, folder_name, dash,
is_nren_beta = folder_name == "NREN Access BETA" # needed for POL1-642 BETA
is_nren = folder_name == "NREN Access"
is_re_peer = folder_name == "RE Peer"
+ is_service = 'service_type' in dash
+ # todo: figure out a neater way to do this?
+ is_complex = is_nren or is_nren_beta or is_re_peer or is_service
if is_nren:
data = get_nren_interface_data_old(interfaces)
dash_data = get_nren_dashboard_data(data, ds_name, tag)
elif is_nren_beta:
# needed for POL1-642 BETA
- services = fetch_services(config['reporting_provider'])
data = get_nren_interface_data(
services, interfaces, excluded_dashboards)
dash_data = get_nren_dashboard_data(data, ds_name, tag)
elif is_re_peer:
data = get_re_peer_interface_data(interfaces)
dash_data = get_re_peer_dashboard_data(data, ds_name, tag)
+ elif is_service:
+ data = get_service_data(dash['service_type'], services, interfaces, excluded_dashboards)
+ dash_data = get_service_dashboard_data(data, ds_name, tag)
else:
data = get_interface_data(interfaces)
dash_data = get_dashboard_data(
@@ -243,7 +259,7 @@ def provision_folder(token_request, folder_name, dash,
with ThreadPoolExecutor(max_workers=MAX_WORKERS) as executor:
for dashboard in dash_data:
- if is_nren or is_nren_beta or is_re_peer:
+ if is_complex:
rendered = render_complex_dashboard(**dashboard)
else:
rendered = render_simple_dashboard(**dashboard)
@@ -346,6 +362,20 @@ def excluded_folder_dashboards(org_config, folder_name):
return excluded if isinstance(excluded, list) else []
+def _interfaces_to_keep(interface, excluded_nrens):
+ dash_info = interface.get('dashboards_info')
+ if dash_info is None:
+ logger.info(f'No "dashboards_info" for '
+ f'{interface["router"]}:{interface["name"]}')
+ # throw it away
+ return False
+ dashboards = {nren['name'].lower() for nren in dash_info}
+ is_lab_router = 'lab.office' in interface['router'].lower()
+ should_keep = not (is_lab_router or any(
+ nren.lower() in dashboards for nren in excluded_nrens))
+ return should_keep
+
+
def _provision_interfaces(config, org_config, ds_name, token):
"""
This function is used to provision most dashboards,
@@ -359,24 +389,12 @@ def _provision_interfaces(config, org_config, ds_name, token):
"""
interfaces = get_interfaces(config['inventory_provider'])
+ services = fetch_services(config['reporting_provider'])
excluded_nrens = org_config['excluded_nrens']
excluded_folders = org_config.get('excluded_folders', {})
- def interfaces_to_keep(interface):
- dash_info = interface.get('dashboards_info')
- if dash_info is None:
- logger.info(f'No "dashboards_info" for '
- f'{interface["router"]}:{interface["name"]}')
- # throw it away
- return False
- dashboards = {nren['name'].lower() for nren in dash_info}
- is_lab_router = 'lab.office' in interface['router'].lower()
- should_keep = not (is_lab_router or any(
- nren.lower() in dashboards for nren in excluded_nrens))
- return should_keep
-
- relevant_interfaces = list(filter(interfaces_to_keep, interfaces))
+ relevant_interfaces = list(filter(lambda x: _interfaces_to_keep(x, excluded_nrens), interfaces))
for interface in relevant_interfaces:
interface['dashboards_info'] = list(filter(
lambda x: x['name'] != '',
@@ -429,7 +447,7 @@ def _provision_interfaces(config, org_config, ds_name, token):
f'Provisioning {org_config["name"]}/{folder_name} dashboards')
res = executor.submit(
provision_folder, token,
- folder_name, folder, config, ds_name,
+ folder_name, folder, services, ds_name,
excluded_folder_dashboards(org_config, folder_name))
provisioned.append(res)
@@ -588,6 +606,61 @@ def _provision_aggregates(config, org_config, ds_name, token):
yield from provisioned
+def _provision_service_dashboards(config, org_config, ds_name, token):
+ """
+ This function is used to provision service-specific dashboards,
+ overwriting existing ones.
+
+ :param config: the application config
+ :param org_config: the organisation config
+ :param ds_name: the name of the datasource to query in the dashboards
+ :param token: a token_request object
+ :return: generator of UIDs of dashboards that were created
+ """
+ services = fetch_services(config['reporting_provider'])
+
+ excluded_folders = org_config.get('excluded_folders', {})
+
+ logger.info('Provisioning service-specific dashboards')
+
+ # loop over service dashboards and get service types we care about
+ dash_service_types = {SERVICE_DASHBOARDS[dash]['service_type']: dash for dash in SERVICE_DASHBOARDS}
+ # loop over services and append to dashboards
+ for service in services:
+ if service['service_type'] in dash_service_types:
+ dash = dash_service_types[service['service_type']]
+ svcs = SERVICE_DASHBOARDS[dash]['services']
+ svcs.append(service)
+
+ # provision dashboards and their folders
+ with ThreadPoolExecutor(max_workers=MAX_WORKERS) as executor:
+ provisioned = []
+ for folder in SERVICE_DASHBOARDS.values():
+ folder_name = folder['folder_name']
+
+ # boolean True means entire folder excluded
+ # if list, it is specific dashboard names not to provision
+ # so is handled at provision time.
+ if is_excluded_folder(excluded_folders, folder_name):
+ executor.submit(
+ delete_folder, token, title=folder_name)
+ continue
+
+ logger.info(
+ f'Provisioning {org_config["name"]}/{folder_name} dashboards')
+ res = executor.submit(
+ provision_folder, token,
+ folder_name, folder, services, ds_name,
+ excluded_folder_dashboards(org_config, folder_name))
+ provisioned.append(res)
+
+ for result in provisioned:
+ folder = result.result()
+ if folder is None:
+ continue
+ yield from folder
+
+
def _provision_static_dashboards(config, org_config, ds_name, token):
"""
This function is used to provision static dashboards from json files,
@@ -795,6 +868,8 @@ def provision(config, raise_exceptions=False):
config, org_config, ds_name, token_request),
_provision_aggregates(
config, org_config, ds_name, token_request),
+ _provision_service_dashboards(
+ config, org_config, ds_name, token_request),
_provision_static_dashboards(
config, org_config, ds_name, token_request),
_get_ignored_dashboards(
@@ -825,6 +900,8 @@ def provision(config, raise_exceptions=False):
}
folders_to_keep.update({dash['folder_name']
for dash in DASHBOARDS.values()})
+ folders_to_keep.update({dash['folder_name']
+ for dash in SERVICE_DASHBOARDS.values()})
ignored_folders = config.get('ignored_folders', [])
folders_to_keep.update(ignored_folders)
diff --git a/brian_dashboard_manager/templating/helpers.py b/brian_dashboard_manager/templating/helpers.py
index 31c47cf52313f91c65792d1aee8b030f3754e038..a5aa9e9ed6d8058af115767aaa4144e041a25b8c 100644
--- a/brian_dashboard_manager/templating/helpers.py
+++ b/brian_dashboard_manager/templating/helpers.py
@@ -340,7 +340,7 @@ def get_nren_interface_data(services, interfaces, excluded_dashboards):
'interface': interface_name,
'hostname': host,
'alias':
- f"{router} - {interface_name} - {dashboard_name} "
+ f"{router} - {interface_name} - {dashboard_name} "
})
if info['interface_type'] == 'AGGREGATE':
@@ -368,6 +368,74 @@ def get_nren_interface_data(services, interfaces, excluded_dashboards):
return result
+def get_service_data(service_type, services, interfaces, excluded_dashboards):
+ """
+ Helper for grouping interface data to be used for generating
+ dashboards for specific service types, grouped by NRENs.
+
+ Extracts information from interfaces to be used in panels.
+
+ :param services: list of services
+ :param interfaces: list of interfaces
+ :param excluded_dashboards: list of dashboards to exclude for
+ the organization we are generating dashboards for
+
+ :return: dictionary of dashboards and their service/interface data
+ """
+ result = {}
+
+ customers = defaultdict(list)
+
+ for service in services:
+ _customers = service.get('customers')
+ _service_type = service.get('service_type')
+ for cust in _customers:
+ if cust.lower() in excluded_dashboards:
+ continue
+ if _service_type != service_type:
+ continue
+ customers[cust].append(service)
+
+ for customer, services in customers.items():
+ dashboard = result.setdefault(customer, {
+ 'SERVICES': []
+ })
+
+ for service in services:
+ _interfaces = service.get('endpoints')
+ name = service.get('name')
+ sid = service.get('sid')
+ scid = service.get('scid')
+
+ measurement = 'scid_rates'
+
+ if len(_interfaces) == 0:
+ continue
+
+ if 'interface' in _interfaces[0]:
+ if_name = _interfaces[0].get('interface')
+ router = _interfaces[0].get('hostname')
+ else:
+ if_name = _interfaces[0].get('port')
+ router = _interfaces[0].get('equipment')
+ router = router.replace('.geant.net', '')
+ title = f'{router} - {{}} - {if_name} - {name} ({sid})'
+
+ dashboard['SERVICES'].append({
+ 'measurement': measurement,
+ 'title': title,
+ 'scid': scid,
+ 'sort': (sid[:2], name)
+ })
+
+ for customer in list(result.keys()):
+ lengths = [len(val) for val in result[customer].values()]
+ if sum(lengths) == 0:
+ # no services/interfaces, so remove it
+ del result[customer]
+ return result
+
+
def get_interface_data(interfaces):
"""
Helper for grouping interface data to be used for generating
@@ -438,6 +506,7 @@ def get_aggregate_interface_data(interfaces, agg_name, group_field):
prev[curr[field]] = groups
prev['EVERYSINGLETARGET'] = all_agg
return prev
+
return reduce_func
for interface in interfaces:
@@ -612,6 +681,31 @@ def default_interface_panel_generator(gridPos, use_all_traffic=True, use_ipv6=Tr
return get_panel_definitions
+def _get_dashboard_data(data, datasource, tag, single_data_func):
+ """
+ Helper for generating dashboard definitions.
+ Uses multiprocessing to speed up generation.
+
+ :param data: the dashboard names and the panel data for each dashboard
+ :param datasource: datasource to use for the panels
+ :param tag: tag to use for the dashboard, used for dashboard dropdowns on
+ the home dashboard.
+ :param single_data_func: function that gets data for one definition
+
+ :return: generator for dashboard definitions for each dashboard
+ """
+
+ with ProcessPoolExecutor(max_workers=NUM_PROCESSES) as executor:
+ for dash in executor.map(
+ partial(
+ single_data_func,
+ datasource=datasource,
+ tag=tag),
+ data.items()
+ ):
+ yield dash
+
+
def get_nren_dashboard_data_single(data, datasource, tag):
"""
Helper for generating dashboard definitions for a single NREN.
@@ -649,7 +743,7 @@ def get_nren_dashboard_data_single(data, datasource, tag):
def sort_key(panel):
sort = panel.get('sort')
if not sort:
- return 'ZZZ'+panel.get('hostname') # sort to end
+ return 'ZZZ' + panel.get('hostname') # sort to end
return sort
service_panels = panel_gen(
@@ -692,28 +786,7 @@ def get_nren_dashboard_data_single(data, datasource, tag):
def get_nren_dashboard_data(data, datasource, tag):
- """
- Helper for generating dashboard definitions for all NRENs.
- Uses multiprocessing to speed up generation.
-
- :param data: the NREN names and the panel data for each NREN
- :param datasource: datasource to use for the panels
- :param tag: tag to use for the dashboard, used for dashboard dropdowns on
- the home dashboard.
-
- :return: generator for dashboard definitions for each NREN
- """
-
- with ProcessPoolExecutor(max_workers=NUM_PROCESSES) as executor:
- for dash in executor.map(
- partial(
- get_nren_dashboard_data_single,
- datasource=datasource,
- tag=tag),
- data.items()
- ):
-
- yield dash
+ yield from _get_dashboard_data(data, datasource, tag, get_nren_dashboard_data_single)
def get_re_peer_dashboard_data_single(data, datasource, tag):
@@ -752,7 +825,7 @@ def get_re_peer_dashboard_data_single(data, datasource, tag):
def sort_key(panel):
sort = panel.get('sort')
if not sort:
- return 'ZZZ'+panel.get('hostname') # sort to end
+ return 'ZZZ' + panel.get('hostname') # sort to end
return sort
service_panels = panel_gen(
@@ -786,28 +859,61 @@ def get_re_peer_dashboard_data_single(data, datasource, tag):
def get_re_peer_dashboard_data(data, datasource, tag):
+ yield from _get_dashboard_data(data, datasource, tag, get_re_peer_dashboard_data_single)
+
+
+def get_service_dashboard_data_single(data, datasource, tag):
"""
- Helper for generating dashboard definitions for all R&E Peers.
- Uses multiprocessing to speed up generation.
+ Helper for generating dashboard definitions for a single service.
- :param data: the names and the panel data for each R&E Peer
+ :param data: data for the dashboard, including the service name and
+ the panel data
:param datasource: datasource to use for the panels
:param tag: tag to use for the dashboard, used for dashboard dropdowns on
the home dashboard.
- :return: generator for dashboard definitions for each R&E Peer
+ :return: dashboard definition for the service dashboard
"""
- with ProcessPoolExecutor(max_workers=NUM_PROCESSES) as executor:
- for dash in executor.map(
- partial(
- get_re_peer_dashboard_data_single,
- datasource=datasource,
- tag=tag),
- data.items()
- ):
+ service_name, dash = data
+ id_gen = num_generator()
- yield dash
+ gridPos = gridPos_generator(id_gen)
+
+ panel_gen = default_interface_panel_generator(gridPos, use_all_traffic=True, use_ipv6=False)
+
+ services_dropdown = create_dropdown_panel('Services', **next(gridPos))
+
+ def sort_key(panel):
+ sort = panel.get('sort')
+ if not sort:
+ return 'ZZZ' + panel.get('hostname') # sort to end
+ return sort
+
+ service_panels = panel_gen(
+ sorted(dash['SERVICES'], key=sort_key), datasource)
+
+ dropdown_groups = [{
+ 'dropdown': services_dropdown,
+ 'panels': service_panels,
+ }]
+
+ result = {
+ 'nren_name': service_name,
+ 'datasource': datasource,
+ 'aggregate_panels': [],
+ 'dropdown_groups': dropdown_groups
+ }
+ if isinstance(tag, list):
+ result['tags'] = tag
+ else:
+ result['tag'] = tag
+
+ return result
+
+
+def get_service_dashboard_data(data, datasource, tag):
+ yield from _get_dashboard_data(data, datasource, tag, get_service_dashboard_data_single)
def get_dashboard_data_single(
@@ -867,15 +973,14 @@ def get_dashboard_data(
with ProcessPoolExecutor(max_workers=NUM_PROCESSES) as executor:
for dash in executor.map(
- partial(
- get_dashboard_data_single,
- datasource=datasource,
- tag=tag,
- panel_generator=panel_generator,
- errors=errors),
- data.items()
+ partial(
+ get_dashboard_data_single,
+ datasource=datasource,
+ tag=tag,
+ panel_generator=panel_generator,
+ errors=errors),
+ data.items()
):
-
yield dash
diff --git a/changelog.md b/changelog.md
index 8f58d66691485adf969c3df859e9cfd4fbcddb38..75c99a064a66b8284707d6b04b017fec7f7357e9 100644
--- a/changelog.md
+++ b/changelog.md
@@ -2,6 +2,10 @@
All notable changes to this project will be documented in this file.
+## [0.63] - 2024-07-18
+- POL1-700 - Managed Wavelength Service dashboard
+- Service-type based dashboard support
+
## [0.62] - 2024-07-11
- POL1-579: New layout for R&E Peer dashboards
- Migrate from jinja2 to just plain python dictionaries
diff --git a/setup.py b/setup.py
index 9b630b2b1f3658bb3cc9882a3e6887a286e526ab..598fa7359f3060337f8dbc076e69b828dcffbe17 100644
--- a/setup.py
+++ b/setup.py
@@ -2,7 +2,7 @@ from setuptools import setup, find_packages
setup(
name='brian-dashboard-manager',
- version="0.62",
+ version="0.63",
author='GEANT',
author_email='swd@geant.org',
description='',
diff --git a/test/test_update.py b/test/test_update.py
index e407e600de890cfc03178033a42fdb6df6e1486e..9e5dfc380a7c20d6ddc8840122309fe8fe0df12c 100644
--- a/test/test_update.py
+++ b/test/test_update.py
@@ -2,6 +2,7 @@ import pytest
import responses
from brian_dashboard_manager.grafana.provision import provision_folder, provision
+from brian_dashboard_manager.services.api import fetch_services
TEST_INTERFACES = [
{
@@ -678,11 +679,13 @@ def test_provision_nren_folder(
}
)
+ services = fetch_services(data_config['reporting_provider'])
+
result = provision_folder(
mock_grafana.request,
folder_name,
dashboards["NREN"],
- data_config,
+ services,
"testdatasource",
excluded_nrens,
)