diff --git a/brian_dashboard_manager/grafana/provision.py b/brian_dashboard_manager/grafana/provision.py
index 0259d48414956946ce6b2f990af970e0a195e1ed..51ff6ee4bd8726c770f760afb511aff3ce56264a 100644
--- a/brian_dashboard_manager/grafana/provision.py
+++ b/brian_dashboard_manager/grafana/provision.py
@@ -33,7 +33,8 @@ from brian_dashboard_manager.templating.helpers import \
get_aggregate_dashboard_data, get_interface_data, \
get_nren_interface_data, get_dashboard_data, \
get_nren_dashboard_data, get_aggregate_interface_data, \
- get_nren_interface_data_old, get_re_peer_dashboard_data, get_re_peer_interface_data
+ get_nren_interface_data_old, get_re_peer_dashboard_data, get_re_peer_interface_data, get_service_data, \
+ get_service_dashboard_data
from brian_dashboard_manager.templating.gws import generate_gws, generate_indirect
from brian_dashboard_manager.templating.eumetsat import generate_eumetsat_multicast
@@ -136,6 +137,16 @@ DASHBOARDS = {
}
}
+SERVICE_DASHBOARDS = {
+ 'MWS': {
+ 'tag': ['mws'],
+ 'service_type': 'GEANT MANAGED WAVELENGTH SERVICE',
+ 'folder_name': 'Managed Wavelength Service',
+ 'interfaces': [],
+ 'services': []
+ }
+}
+
AGG_DASHBOARDS = {
'CLS_PEERS': {
'tag': 'cls_peers',
@@ -181,8 +192,8 @@ AGG_DASHBOARDS = {
}
-def provision_folder(token_request, folder_name, dash,
- config, ds_name, excluded_dashboards):
+def provision_folder(token_request, folder_name, dash, services,
+ ds_name, excluded_dashboards):
"""
Function to provision dashboards within a folder.
@@ -190,7 +201,7 @@ def provision_folder(token_request, folder_name, dash,
:param folder_name: Name of the folder to provision dashboards in
:param dash: the dashboards to provision, with interface data to generate
the dashboards from
- :param config: the application config
+ :param services: service data from reporting provider for service-based dashboards
:param ds_name: the name of the datasource to query in the dashboard panels
:param excluded_dashboards: list of dashboards to exclude from provisioning
for the organisation
@@ -218,19 +229,22 @@ def provision_folder(token_request, folder_name, dash,
is_nren_beta = folder_name == "NREN Access BETA" # needed for POL1-642 BETA
is_nren = folder_name == "NREN Access"
is_re_peer = folder_name == "RE Peer"
+ is_service = 'service_type' in dash
if is_nren:
data = get_nren_interface_data_old(interfaces)
dash_data = get_nren_dashboard_data(data, ds_name, tag)
elif is_nren_beta:
# needed for POL1-642 BETA
- services = fetch_services(config['reporting_provider'])
data = get_nren_interface_data(
services, interfaces, excluded_dashboards)
dash_data = get_nren_dashboard_data(data, ds_name, tag)
elif is_re_peer:
data = get_re_peer_interface_data(interfaces)
dash_data = get_re_peer_dashboard_data(data, ds_name, tag)
+ elif is_service:
+ data = get_service_data(dash['service_type'], services, interfaces, excluded_dashboards)
+ dash_data = get_service_dashboard_data(data, ds_name, tag)
else:
data = get_interface_data(interfaces)
dash_data = get_dashboard_data(
@@ -243,7 +257,7 @@ def provision_folder(token_request, folder_name, dash,
with ThreadPoolExecutor(max_workers=MAX_WORKERS) as executor:
for dashboard in dash_data:
- if is_nren or is_nren_beta or is_re_peer:
+ if is_nren or is_nren_beta or is_re_peer or is_service:
rendered = render_complex_dashboard(**dashboard)
else:
rendered = render_simple_dashboard(**dashboard)
@@ -346,6 +360,20 @@ def excluded_folder_dashboards(org_config, folder_name):
return excluded if isinstance(excluded, list) else []
+def _interfaces_to_keep(interface, excluded_nrens):
+ dash_info = interface.get('dashboards_info')
+ if dash_info is None:
+ logger.info(f'No "dashboards_info" for '
+ f'{interface["router"]}:{interface["name"]}')
+ # throw it away
+ return False
+ dashboards = {nren['name'].lower() for nren in dash_info}
+ is_lab_router = 'lab.office' in interface['router'].lower()
+ should_keep = not (is_lab_router or any(
+ nren.lower() in dashboards for nren in excluded_nrens))
+ return should_keep
+
+
def _provision_interfaces(config, org_config, ds_name, token):
"""
This function is used to provision most dashboards,
@@ -359,24 +387,12 @@ def _provision_interfaces(config, org_config, ds_name, token):
"""
interfaces = get_interfaces(config['inventory_provider'])
+ services = fetch_services(config['reporting_provider'])
excluded_nrens = org_config['excluded_nrens']
excluded_folders = org_config.get('excluded_folders', {})
- def interfaces_to_keep(interface):
- dash_info = interface.get('dashboards_info')
- if dash_info is None:
- logger.info(f'No "dashboards_info" for '
- f'{interface["router"]}:{interface["name"]}')
- # throw it away
- return False
- dashboards = {nren['name'].lower() for nren in dash_info}
- is_lab_router = 'lab.office' in interface['router'].lower()
- should_keep = not (is_lab_router or any(
- nren.lower() in dashboards for nren in excluded_nrens))
- return should_keep
-
- relevant_interfaces = list(filter(interfaces_to_keep, interfaces))
+ relevant_interfaces = list(filter(lambda x: _interfaces_to_keep(x, excluded_nrens), interfaces))
for interface in relevant_interfaces:
interface['dashboards_info'] = list(filter(
lambda x: x['name'] != '',
@@ -429,7 +445,7 @@ def _provision_interfaces(config, org_config, ds_name, token):
f'Provisioning {org_config["name"]}/{folder_name} dashboards')
res = executor.submit(
provision_folder, token,
- folder_name, folder, config, ds_name,
+ folder_name, folder, services, ds_name,
excluded_folder_dashboards(org_config, folder_name))
provisioned.append(res)
@@ -588,6 +604,83 @@ def _provision_aggregates(config, org_config, ds_name, token):
yield from provisioned
+def _provision_service_dashboards(config, org_config, ds_name, token):
+ """
+ This function is used to provision service-specific dashboards,
+ overwriting existing ones.
+
+ :param config: the application config
+ :param org_config: the organisation config
+ :param ds_name: the name of the datasource to query in the dashboards
+ :param token: a token_request object
+ :return: generator of UIDs of dashboards that were created
+ """
+ interfaces = get_interfaces(config['inventory_provider'])
+ services = fetch_services(config['reporting_provider'])
+
+ excluded_nrens = org_config['excluded_nrens']
+ excluded_folders = org_config.get('excluded_folders', {})
+
+ logger.info('Provisioning service-specific dashboards')
+
+ relevant_interfaces = list(filter(lambda x: _interfaces_to_keep(x, excluded_nrens), interfaces))
+ for interface in relevant_interfaces:
+ interface['dashboards_info'] = list(filter(
+ lambda x: x['name'] != '',
+ interface['dashboards_info']
+ ))
+
+ # create a lookup index for interface data
+ interfaces_index = {f'{interface["router"]}###{interface["name"]}': interface for interface in relevant_interfaces}
+ # loop over service dashboards and get service types we care about
+ dash_service_types = {SERVICE_DASHBOARDS[dash]['service_type']: dash for dash in SERVICE_DASHBOARDS}
+ # loop over services and get relevant interfaces, append to dashboards
+ for service in services:
+ if service['service_type'] in dash_service_types:
+ dash = dash_service_types[service['service_type']]
+ svcs = SERVICE_DASHBOARDS[dash]['services']
+ svcs.append(service)
+ # attach service interfaces
+ if len(service['endpoints']) == 0:
+ continue
+ ifaces = SERVICE_DASHBOARDS[dash]['interfaces']
+ for endpoint in service['endpoints']:
+ if 'interface' in endpoint:
+ if_name = endpoint.get('interface')
+ router = endpoint.get('hostname')
+ ifc = interfaces_index.get(f'{router}###{if_name}')
+ if ifc:
+ ifaces.append(ifc)
+
+ # provision dashboards and their folders
+ with ThreadPoolExecutor(max_workers=MAX_WORKERS) as executor:
+ provisioned = []
+ for folder in SERVICE_DASHBOARDS.values():
+ folder_name = folder['folder_name']
+
+ # boolean True means entire folder excluded
+ # if list, it is specific dashboard names not to provision
+ # so is handled at provision time.
+ if is_excluded_folder(excluded_folders, folder_name):
+ executor.submit(
+ delete_folder, token, title=folder_name)
+ continue
+
+ logger.info(
+ f'Provisioning {org_config["name"]}/{folder_name} dashboards')
+ res = executor.submit(
+ provision_folder, token,
+ folder_name, folder, services, ds_name,
+ excluded_folder_dashboards(org_config, folder_name))
+ provisioned.append(res)
+
+ for result in provisioned:
+ folder = result.result()
+ if folder is None:
+ continue
+ yield from folder
+
+
def _provision_static_dashboards(config, org_config, ds_name, token):
"""
This function is used to provision static dashboards from json files,
@@ -795,6 +888,8 @@ def provision(config, raise_exceptions=False):
config, org_config, ds_name, token_request),
_provision_aggregates(
config, org_config, ds_name, token_request),
+ _provision_service_dashboards(
+ config, org_config, ds_name, token_request),
_provision_static_dashboards(
config, org_config, ds_name, token_request),
_get_ignored_dashboards(
@@ -821,7 +916,8 @@ def provision(config, raise_exceptions=False):
'GWS Indirect',
'GWS Direct',
'Aggregates',
- 'EUMETSAT Multicast'
+ 'EUMETSAT Multicast',
+ 'Managed Wavelength Service',
}
folders_to_keep.update({dash['folder_name']
for dash in DASHBOARDS.values()})
diff --git a/brian_dashboard_manager/templating/helpers.py b/brian_dashboard_manager/templating/helpers.py
index 31c47cf52313f91c65792d1aee8b030f3754e038..7061ec6ff303182f4ac0d4140c2d0f68140ff4f9 100644
--- a/brian_dashboard_manager/templating/helpers.py
+++ b/brian_dashboard_manager/templating/helpers.py
@@ -340,7 +340,7 @@ def get_nren_interface_data(services, interfaces, excluded_dashboards):
'interface': interface_name,
'hostname': host,
'alias':
- f"{router} - {interface_name} - {dashboard_name} "
+ f"{router} - {interface_name} - {dashboard_name} "
})
if info['interface_type'] == 'AGGREGATE':
@@ -368,6 +368,120 @@ def get_nren_interface_data(services, interfaces, excluded_dashboards):
return result
+def get_service_data(service_type, services, interfaces, excluded_dashboards):
+ """
+ Helper for grouping interface data to be used for generating
+ dashboards for specific service types, grouped by NRENs.
+
+ Extracts information from interfaces to be used in panels.
+
+ :param services: list of services
+ :param interfaces: list of interfaces
+ :param excluded_dashboards: list of dashboards to exclude for
+ the organization we are generating dashboards for
+
+ :return: dictionary of dashboards and their service/interface data
+ """
+ result = {}
+
+ customers = defaultdict(list)
+
+ for service in services:
+ _customers = service.get('customers')
+ _service_type = service.get('service_type')
+ for cust in _customers:
+ if cust.lower() in excluded_dashboards:
+ continue
+ if _service_type != service_type:
+ continue
+ customers[cust].append(service)
+
+ for customer, services in customers.items():
+ dashboard = result.setdefault(customer, {
+ 'SERVICES': [],
+ 'PHYSICAL': []
+ })
+
+ for service in services:
+ _interfaces = service.get('endpoints')
+ name = service.get('name')
+ sid = service.get('sid')
+ scid = service.get('scid')
+
+ measurement = 'scid_rates'
+
+ if len(_interfaces) == 0:
+ continue
+
+ if 'interface' in _interfaces[0]:
+ if_name = _interfaces[0].get('interface')
+ router = _interfaces[0].get('hostname')
+ else:
+ if_name = _interfaces[0].get('port')
+ router = _interfaces[0].get('equipment')
+ router = router.replace('.geant.net', '')
+ title = f'{router} - {{}} - {if_name} - {name} ({sid})'
+
+ has_v6_interface = False
+ for interface in _interfaces:
+ if 'addresses' in interface:
+ for address in interface['addresses']:
+ if address.find(':') > 0:
+ has_v6_interface = True
+ break
+
+ dashboard['SERVICES'].append({
+ 'measurement': measurement,
+ 'title': title,
+ 'scid': scid,
+ 'sort': (sid[:2], name),
+ 'has_v6': has_v6_interface
+ })
+
+ for interface in interfaces:
+
+ description = interface['description'].strip()
+ interface_name = interface['name']
+ host = interface['router']
+
+ router = host.replace('.geant.net', '')
+ panel_title = f"{router} - {{}} - {interface_name} - {description}"
+
+ dashboards_info = interface['dashboards_info']
+
+ for info in dashboards_info:
+ dashboard_name = info['name']
+
+ dashboard = result.get(dashboard_name, {
+ 'SERVICES': [],
+ 'PHYSICAL': []
+ })
+
+ if info['interface_type'] == 'AGGREGATE':
+ # link aggregates are also shown
+ # under the physical dropdown
+ dashboard['PHYSICAL'].append({
+ 'title': panel_title,
+ 'hostname': host,
+ 'interface': interface_name
+ })
+ elif info['interface_type'] == 'PHYSICAL':
+ dashboard['PHYSICAL'].append({
+ 'title': panel_title,
+ 'hostname': host,
+ 'interface': interface_name
+ })
+
+ result[dashboard_name] = dashboard
+
+ for customer in list(result.keys()):
+ lengths = [len(val) for val in result[customer].values()]
+ if sum(lengths) == 0:
+ # no services/interfaces, so remove it
+ del result[customer]
+ return result
+
+
def get_interface_data(interfaces):
"""
Helper for grouping interface data to be used for generating
@@ -438,6 +552,7 @@ def get_aggregate_interface_data(interfaces, agg_name, group_field):
prev[curr[field]] = groups
prev['EVERYSINGLETARGET'] = all_agg
return prev
+
return reduce_func
for interface in interfaces:
@@ -612,6 +727,31 @@ def default_interface_panel_generator(gridPos, use_all_traffic=True, use_ipv6=Tr
return get_panel_definitions
+def _get_dashboard_data(data, datasource, tag, single_data_func):
+ """
+ Helper for generating dashboard definitions.
+ Uses multiprocessing to speed up generation.
+
+ :param data: the dashboard names and the panel data for each dashboard
+ :param datasource: datasource to use for the panels
+ :param tag: tag to use for the dashboard, used for dashboard dropdowns on
+ the home dashboard.
+ :param single_data_func: function that gets data for one definition
+
+ :return: generator for dashboard definitions for each dashboard
+ """
+
+ with ProcessPoolExecutor(max_workers=NUM_PROCESSES) as executor:
+ for dash in executor.map(
+ partial(
+ single_data_func,
+ datasource=datasource,
+ tag=tag),
+ data.items()
+ ):
+ yield dash
+
+
def get_nren_dashboard_data_single(data, datasource, tag):
"""
Helper for generating dashboard definitions for a single NREN.
@@ -649,7 +789,7 @@ def get_nren_dashboard_data_single(data, datasource, tag):
def sort_key(panel):
sort = panel.get('sort')
if not sort:
- return 'ZZZ'+panel.get('hostname') # sort to end
+ return 'ZZZ' + panel.get('hostname') # sort to end
return sort
service_panels = panel_gen(
@@ -692,28 +832,7 @@ def get_nren_dashboard_data_single(data, datasource, tag):
def get_nren_dashboard_data(data, datasource, tag):
- """
- Helper for generating dashboard definitions for all NRENs.
- Uses multiprocessing to speed up generation.
-
- :param data: the NREN names and the panel data for each NREN
- :param datasource: datasource to use for the panels
- :param tag: tag to use for the dashboard, used for dashboard dropdowns on
- the home dashboard.
-
- :return: generator for dashboard definitions for each NREN
- """
-
- with ProcessPoolExecutor(max_workers=NUM_PROCESSES) as executor:
- for dash in executor.map(
- partial(
- get_nren_dashboard_data_single,
- datasource=datasource,
- tag=tag),
- data.items()
- ):
-
- yield dash
+ yield from _get_dashboard_data(data, datasource, tag, get_nren_dashboard_data_single)
def get_re_peer_dashboard_data_single(data, datasource, tag):
@@ -752,7 +871,7 @@ def get_re_peer_dashboard_data_single(data, datasource, tag):
def sort_key(panel):
sort = panel.get('sort')
if not sort:
- return 'ZZZ'+panel.get('hostname') # sort to end
+ return 'ZZZ' + panel.get('hostname') # sort to end
return sort
service_panels = panel_gen(
@@ -786,28 +905,68 @@ def get_re_peer_dashboard_data_single(data, datasource, tag):
def get_re_peer_dashboard_data(data, datasource, tag):
+ yield from _get_dashboard_data(data, datasource, tag, get_re_peer_dashboard_data_single)
+
+
+def get_service_dashboard_data_single(data, datasource, tag):
"""
- Helper for generating dashboard definitions for all R&E Peers.
- Uses multiprocessing to speed up generation.
+ Helper for generating dashboard definitions for a single service.
- :param data: the names and the panel data for each R&E Peer
+ :param data: data for the dashboard, including the service name and
+ the panel data
:param datasource: datasource to use for the panels
:param tag: tag to use for the dashboard, used for dashboard dropdowns on
the home dashboard.
- :return: generator for dashboard definitions for each R&E Peer
+ :return: dashboard definition for the service dashboard
"""
- with ProcessPoolExecutor(max_workers=NUM_PROCESSES) as executor:
- for dash in executor.map(
- partial(
- get_re_peer_dashboard_data_single,
- datasource=datasource,
- tag=tag),
- data.items()
- ):
+ service_name, dash = data
+ id_gen = num_generator()
- yield dash
+ gridPos = gridPos_generator(id_gen)
+
+ panel_gen = default_interface_panel_generator(gridPos, use_all_traffic=True, use_ipv6=False)
+
+ services_dropdown = create_dropdown_panel('Services', **next(gridPos))
+
+ def sort_key(panel):
+ sort = panel.get('sort')
+ if not sort:
+ return 'ZZZ' + panel.get('hostname') # sort to end
+ return sort
+
+ service_panels = panel_gen(
+ sorted(dash['SERVICES'], key=sort_key), datasource)
+
+ iface_dropdown = create_dropdown_panel('Interfaces', **next(gridPos))
+ phys_panels = panel_gen(dash['PHYSICAL'], datasource, True)
+
+ dropdown_groups = [{
+ 'dropdown': services_dropdown,
+ 'panels': service_panels,
+ }]
+ dropdown_groups.append({
+ 'dropdown': iface_dropdown,
+ 'panels': phys_panels,
+ })
+
+ result = {
+ 'nren_name': service_name,
+ 'datasource': datasource,
+ 'aggregate_panels': [],
+ 'dropdown_groups': dropdown_groups
+ }
+ if isinstance(tag, list):
+ result['tags'] = tag
+ else:
+ result['tag'] = tag
+
+ return result
+
+
+def get_service_dashboard_data(data, datasource, tag):
+ yield from _get_dashboard_data(data, datasource, tag, get_service_dashboard_data_single)
def get_dashboard_data_single(
@@ -867,15 +1026,14 @@ def get_dashboard_data(
with ProcessPoolExecutor(max_workers=NUM_PROCESSES) as executor:
for dash in executor.map(
- partial(
- get_dashboard_data_single,
- datasource=datasource,
- tag=tag,
- panel_generator=panel_generator,
- errors=errors),
- data.items()
+ partial(
+ get_dashboard_data_single,
+ datasource=datasource,
+ tag=tag,
+ panel_generator=panel_generator,
+ errors=errors),
+ data.items()
):
-
yield dash
diff --git a/brian_dashboard_manager/templating/render.py b/brian_dashboard_manager/templating/render.py
index 19b4cdfab4d7ac3d84fb0de47ad6d1d135f598ea..65dc9e41dd321714528e8d42985c02f72a216ca6 100644
--- a/brian_dashboard_manager/templating/render.py
+++ b/brian_dashboard_manager/templating/render.py
@@ -276,7 +276,8 @@ def render_complex_dashboard(
):
assert tag or tags
panels = [create_infobox()]
- panels.extend(aggregate_panels)
+ if len(aggregate_panels) > 0:
+ panels.extend(aggregate_panels)
for group in dropdown_groups:
panels.append(group["dropdown"])
panels.extend(group["panels"])