managed wavelength service (and service dashboards) implemented

brian_dashboard_manager/grafana/provision.py

@@ -33,7 +33,8 @@ from brian_dashboard_manager.templating.helpers import \
     get_aggregate_dashboard_data, get_interface_data, \
     get_nren_interface_data, get_dashboard_data, \
     get_nren_dashboard_data, get_aggregate_interface_data, \
-    get_nren_interface_data_old, get_re_peer_dashboard_data, get_re_peer_interface_data
+    get_nren_interface_data_old, get_re_peer_dashboard_data, get_re_peer_interface_data, get_service_data, \
+    get_service_dashboard_data
 
 from brian_dashboard_manager.templating.gws import generate_gws, generate_indirect
 from brian_dashboard_manager.templating.eumetsat import generate_eumetsat_multicast
@@ -136,6 +137,16 @@ DASHBOARDS = {
     }
 }
 
+SERVICE_DASHBOARDS = {
+    'MWS': {
+        'tag': ['mws'],
+        'service_type': 'GEANT MANAGED WAVELENGTH SERVICE',
+        'folder_name': 'Managed Wavelength Service',
+        'interfaces': [],
+        'services': []
+    }
+}
+
 AGG_DASHBOARDS = {
     'CLS_PEERS': {
         'tag': 'cls_peers',
@@ -181,8 +192,8 @@ AGG_DASHBOARDS = {
 }
 
 
-def provision_folder(token_request, folder_name, dash,
-                     config, ds_name, excluded_dashboards):
+def provision_folder(token_request, folder_name, dash, services,
+                     ds_name, excluded_dashboards):
     """
     Function to provision dashboards within a folder.
 
@@ -190,7 +201,7 @@ def provision_folder(token_request, folder_name, dash,
     :param folder_name: Name of the folder to provision dashboards in
     :param dash: the dashboards to provision, with interface data to generate
     the dashboards from
-    :param config: the application config
+    :param services: service data from reporting provider for service-based dashboards
     :param ds_name: the name of the datasource to query in the dashboard panels
     :param excluded_dashboards: list of dashboards to exclude from provisioning
     for the organisation
@@ -218,19 +229,22 @@ def provision_folder(token_request, folder_name, dash,
     is_nren_beta = folder_name == "NREN Access BETA"  # needed for POL1-642 BETA
     is_nren = folder_name == "NREN Access"
     is_re_peer = folder_name == "RE Peer"
+    is_service = 'service_type' in dash
 
     if is_nren:
         data = get_nren_interface_data_old(interfaces)
         dash_data = get_nren_dashboard_data(data, ds_name, tag)
     elif is_nren_beta:
         # needed for POL1-642 BETA
-        services = fetch_services(config['reporting_provider'])
         data = get_nren_interface_data(
             services, interfaces, excluded_dashboards)
         dash_data = get_nren_dashboard_data(data, ds_name, tag)
     elif is_re_peer:
         data = get_re_peer_interface_data(interfaces)
         dash_data = get_re_peer_dashboard_data(data, ds_name, tag)
+    elif is_service:
+        data = get_service_data(dash['service_type'], services, interfaces, excluded_dashboards)
+        dash_data = get_service_dashboard_data(data, ds_name, tag)
     else:
         data = get_interface_data(interfaces)
         dash_data = get_dashboard_data(
@@ -243,7 +257,7 @@ def provision_folder(token_request, folder_name, dash,
 
     with ThreadPoolExecutor(max_workers=MAX_WORKERS) as executor:
         for dashboard in dash_data:
-            if is_nren or is_nren_beta or is_re_peer:
+            if is_nren or is_nren_beta or is_re_peer or is_service:
                 rendered = render_complex_dashboard(**dashboard)
             else:
                 rendered = render_simple_dashboard(**dashboard)
@@ -346,6 +360,20 @@ def excluded_folder_dashboards(org_config, folder_name):
     return excluded if isinstance(excluded, list) else []
 
 
+def _interfaces_to_keep(interface, excluded_nrens):
+    dash_info = interface.get('dashboards_info')
+    if dash_info is None:
+        logger.info(f'No "dashboards_info" for '
+                    f'{interface["router"]}:{interface["name"]}')
+        # throw it away
+        return False
+    dashboards = {nren['name'].lower() for nren in dash_info}
+    is_lab_router = 'lab.office' in interface['router'].lower()
+    should_keep = not (is_lab_router or any(
+        nren.lower() in dashboards for nren in excluded_nrens))
+    return should_keep
+
+
 def _provision_interfaces(config, org_config, ds_name, token):
     """
     This function is used to provision most dashboards,
@@ -359,24 +387,12 @@ def _provision_interfaces(config, org_config, ds_name, token):
     """
 
     interfaces = get_interfaces(config['inventory_provider'])
+    services = fetch_services(config['reporting_provider'])
 
     excluded_nrens = org_config['excluded_nrens']
     excluded_folders = org_config.get('excluded_folders', {})
 
-    def interfaces_to_keep(interface):
-        dash_info = interface.get('dashboards_info')
-        if dash_info is None:
-            logger.info(f'No "dashboards_info" for '
-                        f'{interface["router"]}:{interface["name"]}')
-            # throw it away
-            return False
-        dashboards = {nren['name'].lower() for nren in dash_info}
-        is_lab_router = 'lab.office' in interface['router'].lower()
-        should_keep = not (is_lab_router or any(
-            nren.lower() in dashboards for nren in excluded_nrens))
-        return should_keep
-
-    relevant_interfaces = list(filter(interfaces_to_keep, interfaces))
+    relevant_interfaces = list(filter(lambda x: _interfaces_to_keep(x, excluded_nrens), interfaces))
     for interface in relevant_interfaces:
         interface['dashboards_info'] = list(filter(
             lambda x: x['name'] != '',
@@ -429,7 +445,7 @@ def _provision_interfaces(config, org_config, ds_name, token):
                 f'Provisioning {org_config["name"]}/{folder_name} dashboards')
             res = executor.submit(
                 provision_folder, token,
-                folder_name, folder, config, ds_name,
+                folder_name, folder, services, ds_name,
                 excluded_folder_dashboards(org_config, folder_name))
             provisioned.append(res)
 
@@ -588,6 +604,83 @@ def _provision_aggregates(config, org_config, ds_name, token):
             yield from provisioned
 
 
+def _provision_service_dashboards(config, org_config, ds_name, token):
+    """
+    This function is used to provision service-specific dashboards,
+    overwriting existing ones.
+
+    :param config: the application config
+    :param org_config: the organisation config
+    :param ds_name: the name of the datasource to query in the dashboards
+    :param token: a token_request object
+    :return: generator of UIDs of dashboards that were created
+    """
+    interfaces = get_interfaces(config['inventory_provider'])
+    services = fetch_services(config['reporting_provider'])
+
+    excluded_nrens = org_config['excluded_nrens']
+    excluded_folders = org_config.get('excluded_folders', {})
+
+    logger.info('Provisioning service-specific dashboards')
+
+    relevant_interfaces = list(filter(lambda x: _interfaces_to_keep(x, excluded_nrens), interfaces))
+    for interface in relevant_interfaces:
+        interface['dashboards_info'] = list(filter(
+            lambda x: x['name'] != '',
+            interface['dashboards_info']
+        ))
+
+    # create a lookup index for interface data
+    interfaces_index = {f'{interface["router"]}###{interface["name"]}': interface for interface in relevant_interfaces}
+    # loop over service dashboards and get service types we care about
+    dash_service_types = {SERVICE_DASHBOARDS[dash]['service_type']: dash for dash in SERVICE_DASHBOARDS}
+    # loop over services and get relevant interfaces, append to dashboards
+    for service in services:
+        if service['service_type'] in dash_service_types:
+            dash = dash_service_types[service['service_type']]
+            svcs = SERVICE_DASHBOARDS[dash]['services']
+            svcs.append(service)
+            # attach service interfaces
+            if len(service['endpoints']) == 0:
+                continue
+            ifaces = SERVICE_DASHBOARDS[dash]['interfaces']
+            for endpoint in service['endpoints']:
+                if 'interface' in endpoint:
+                    if_name = endpoint.get('interface')
+                    router = endpoint.get('hostname')
+                    ifc = interfaces_index.get(f'{router}###{if_name}')
+                    if ifc:
+                        ifaces.append(ifc)
+
+    # provision dashboards and their folders
+    with ThreadPoolExecutor(max_workers=MAX_WORKERS) as executor:
+        provisioned = []
+        for folder in SERVICE_DASHBOARDS.values():
+            folder_name = folder['folder_name']
+
+            # boolean True means entire folder excluded
+            # if list, it is specific dashboard names not to provision
+            # so is handled at provision time.
+            if is_excluded_folder(excluded_folders, folder_name):
+                executor.submit(
+                    delete_folder, token, title=folder_name)
+                continue
+
+            logger.info(
+                f'Provisioning {org_config["name"]}/{folder_name} dashboards')
+            res = executor.submit(
+                provision_folder, token,
+                folder_name, folder, services, ds_name,
+                excluded_folder_dashboards(org_config, folder_name))
+            provisioned.append(res)
+
+        for result in provisioned:
+            folder = result.result()
+            if folder is None:
+                continue
+            yield from folder
+
+
 def _provision_static_dashboards(config, org_config, ds_name, token):
     """
     This function is used to provision static dashboards from json files,
@@ -795,6 +888,8 @@ def provision(config, raise_exceptions=False):
                     config, org_config, ds_name, token_request),
                 _provision_aggregates(
                     config, org_config, ds_name, token_request),
+                _provision_service_dashboards(
+                    config, org_config, ds_name, token_request),
                 _provision_static_dashboards(
                     config, org_config, ds_name, token_request),
                 _get_ignored_dashboards(
@@ -821,7 +916,8 @@ def provision(config, raise_exceptions=False):
                 'GWS Indirect',
                 'GWS Direct',
                 'Aggregates',
-                'EUMETSAT Multicast'
+                'EUMETSAT Multicast',
+                'Managed Wavelength Service',
             }
             folders_to_keep.update({dash['folder_name']
                                     for dash in DASHBOARDS.values()})

brian_dashboard_manager/templating/helpers.py

@@ -340,7 +340,7 @@ def get_nren_interface_data(services, interfaces, excluded_dashboards):
                     'interface': interface_name,
                     'hostname': host,
                     'alias':
-                    f"{router} - {interface_name} - {dashboard_name} "
+                        f"{router} - {interface_name} - {dashboard_name} "
                 })
 
             if info['interface_type'] == 'AGGREGATE':
@@ -368,6 +368,120 @@ def get_nren_interface_data(services, interfaces, excluded_dashboards):
     return result
 
 
+def get_service_data(service_type, services, interfaces, excluded_dashboards):
+    """
+    Helper for grouping interface data to be used for generating
+    dashboards for specific service types, grouped by NRENs.
+
+    Extracts information from interfaces to be used in panels.
+
+    :param services: list of services
+    :param interfaces: list of interfaces
+    :param excluded_dashboards: list of dashboards to exclude for
+    the organization we are generating dashboards for
+
+    :return: dictionary of dashboards and their service/interface data
+    """
+    result = {}
+
+    customers = defaultdict(list)
+
+    for service in services:
+        _customers = service.get('customers')
+        _service_type = service.get('service_type')
+        for cust in _customers:
+            if cust.lower() in excluded_dashboards:
+                continue
+            if _service_type != service_type:
+                continue
+            customers[cust].append(service)
+
+    for customer, services in customers.items():
+        dashboard = result.setdefault(customer, {
+            'SERVICES': [],
+            'PHYSICAL': []
+        })
+
+        for service in services:
+            _interfaces = service.get('endpoints')
+            name = service.get('name')
+            sid = service.get('sid')
+            scid = service.get('scid')
+
+            measurement = 'scid_rates'
+
+            if len(_interfaces) == 0:
+                continue
+
+            if 'interface' in _interfaces[0]:
+                if_name = _interfaces[0].get('interface')
+                router = _interfaces[0].get('hostname')
+            else:
+                if_name = _interfaces[0].get('port')
+                router = _interfaces[0].get('equipment')
+            router = router.replace('.geant.net', '')
+            title = f'{router} - {{}} - {if_name} - {name} ({sid})'
+
+            has_v6_interface = False
+            for interface in _interfaces:
+                if 'addresses' in interface:
+                    for address in interface['addresses']:
+                        if address.find(':') > 0:
+                            has_v6_interface = True
+                            break
+
+            dashboard['SERVICES'].append({
+                'measurement': measurement,
+                'title': title,
+                'scid': scid,
+                'sort': (sid[:2], name),
+                'has_v6': has_v6_interface
+            })
+
+    for interface in interfaces:
+
+        description = interface['description'].strip()
+        interface_name = interface['name']
+        host = interface['router']
+
+        router = host.replace('.geant.net', '')
+        panel_title = f"{router} - {{}} - {interface_name} - {description}"
+
+        dashboards_info = interface['dashboards_info']
+
+        for info in dashboards_info:
+            dashboard_name = info['name']
+
+            dashboard = result.get(dashboard_name, {
+                'SERVICES': [],
+                'PHYSICAL': []
+            })
+
+            if info['interface_type'] == 'AGGREGATE':
+                # link aggregates are also shown
+                # under the physical dropdown
+                dashboard['PHYSICAL'].append({
+                    'title': panel_title,
+                    'hostname': host,
+                    'interface': interface_name
+                })
+            elif info['interface_type'] == 'PHYSICAL':
+                dashboard['PHYSICAL'].append({
+                    'title': panel_title,
+                    'hostname': host,
+                    'interface': interface_name
+                })
+
+            result[dashboard_name] = dashboard
+
+    for customer in list(result.keys()):
+        lengths = [len(val) for val in result[customer].values()]
+        if sum(lengths) == 0:
+            # no services/interfaces, so remove it
+            del result[customer]
+    return result
+
+
 def get_interface_data(interfaces):
     """
     Helper for grouping interface data to be used for generating
@@ -438,6 +552,7 @@ def get_aggregate_interface_data(interfaces, agg_name, group_field):
             prev[curr[field]] = groups
             prev['EVERYSINGLETARGET'] = all_agg
             return prev
+
         return reduce_func
 
     for interface in interfaces:
@@ -612,6 +727,31 @@ def default_interface_panel_generator(gridPos, use_all_traffic=True, use_ipv6=Tr
     return get_panel_definitions
 
 
+def _get_dashboard_data(data, datasource, tag, single_data_func):
+    """
+    Helper for generating dashboard definitions.
+    Uses multiprocessing to speed up generation.
+
+    :param data: the dashboard names and the panel data for each dashboard
+    :param datasource: datasource to use for the panels
+    :param tag: tag to use for the dashboard, used for dashboard dropdowns on
+    the home dashboard.
+    :param single_data_func: function that gets data for one definition
+
+    :return: generator for dashboard definitions for each dashboard
+    """
+
+    with ProcessPoolExecutor(max_workers=NUM_PROCESSES) as executor:
+        for dash in executor.map(
+                partial(
+                    single_data_func,
+                    datasource=datasource,
+                    tag=tag),
+                data.items()
+        ):
+            yield dash
+
+
 def get_nren_dashboard_data_single(data, datasource, tag):
     """
     Helper for generating dashboard definitions for a single NREN.
@@ -649,7 +789,7 @@ def get_nren_dashboard_data_single(data, datasource, tag):
     def sort_key(panel):
         sort = panel.get('sort')
         if not sort:
-            return 'ZZZ'+panel.get('hostname')  # sort to end
+            return 'ZZZ' + panel.get('hostname')  # sort to end
         return sort
 
     service_panels = panel_gen(
@@ -692,28 +832,7 @@ def get_nren_dashboard_data_single(data, datasource, tag):
 
 
 def get_nren_dashboard_data(data, datasource, tag):
-    """
-    Helper for generating dashboard definitions for all NRENs.
-    Uses multiprocessing to speed up generation.
-
-    :param data: the NREN names and the panel data for each NREN
-    :param datasource: datasource to use for the panels
-    :param tag: tag to use for the dashboard, used for dashboard dropdowns on
-    the home dashboard.
-
-    :return: generator for dashboard definitions for each NREN
-    """
-
-    with ProcessPoolExecutor(max_workers=NUM_PROCESSES) as executor:
-        for dash in executor.map(
-            partial(
-                get_nren_dashboard_data_single,
-                datasource=datasource,
-                tag=tag),
-            data.items()
-        ):
-
-            yield dash
+    yield from _get_dashboard_data(data, datasource, tag, get_nren_dashboard_data_single)
 
 
 def get_re_peer_dashboard_data_single(data, datasource, tag):
@@ -752,7 +871,7 @@ def get_re_peer_dashboard_data_single(data, datasource, tag):
     def sort_key(panel):
         sort = panel.get('sort')
         if not sort:
-            return 'ZZZ'+panel.get('hostname')  # sort to end
+            return 'ZZZ' + panel.get('hostname')  # sort to end
         return sort
 
     service_panels = panel_gen(
@@ -786,28 +905,68 @@ def get_re_peer_dashboard_data_single(data, datasource, tag):
 
 
 def get_re_peer_dashboard_data(data, datasource, tag):
+    yield from _get_dashboard_data(data, datasource, tag, get_re_peer_dashboard_data_single)
+
+
+def get_service_dashboard_data_single(data, datasource, tag):
     """
-    Helper for generating dashboard definitions for all R&E Peers.
-    Uses multiprocessing to speed up generation.
+    Helper for generating dashboard definitions for a single service.
 
-    :param data: the names and the panel data for each R&E Peer
+    :param data: data for the dashboard, including the service name and
+    the panel data
     :param datasource: datasource to use for the panels
     :param tag: tag to use for the dashboard, used for dashboard dropdowns on
     the home dashboard.
 
-    :return: generator for dashboard definitions for each R&E Peer
+    :return: dashboard definition for the service dashboard
     """
 
-    with ProcessPoolExecutor(max_workers=NUM_PROCESSES) as executor:
-        for dash in executor.map(
-            partial(
-                get_re_peer_dashboard_data_single,
-                datasource=datasource,
-                tag=tag),
-            data.items()
-        ):
+    service_name, dash = data
+    id_gen = num_generator()
 
-            yield dash
+    gridPos = gridPos_generator(id_gen)
+
+    panel_gen = default_interface_panel_generator(gridPos, use_all_traffic=True, use_ipv6=False)
+
+    services_dropdown = create_dropdown_panel('Services', **next(gridPos))
+
+    def sort_key(panel):
+        sort = panel.get('sort')
+        if not sort:
+            return 'ZZZ' + panel.get('hostname')  # sort to end
+        return sort
+
+    service_panels = panel_gen(
+        sorted(dash['SERVICES'], key=sort_key), datasource)
+
+    iface_dropdown = create_dropdown_panel('Interfaces', **next(gridPos))
+    phys_panels = panel_gen(dash['PHYSICAL'], datasource, True)
+
+    dropdown_groups = [{
+        'dropdown': services_dropdown,
+        'panels': service_panels,
+    }]
+    dropdown_groups.append({
+        'dropdown': iface_dropdown,
+        'panels': phys_panels,
+    })
+
+    result = {
+        'nren_name': service_name,
+        'datasource': datasource,
+        'aggregate_panels': [],
+        'dropdown_groups': dropdown_groups
+    }
+    if isinstance(tag, list):
+        result['tags'] = tag
+    else:
+        result['tag'] = tag
+
+    return result
+
+
+def get_service_dashboard_data(data, datasource, tag):
+    yield from _get_dashboard_data(data, datasource, tag, get_service_dashboard_data_single)
 
 
 def get_dashboard_data_single(
@@ -867,15 +1026,14 @@ def get_dashboard_data(
 
     with ProcessPoolExecutor(max_workers=NUM_PROCESSES) as executor:
         for dash in executor.map(
-            partial(
-                get_dashboard_data_single,
-                datasource=datasource,
-                tag=tag,
-                panel_generator=panel_generator,
-                errors=errors),
-            data.items()
+                partial(
+                    get_dashboard_data_single,
+                    datasource=datasource,
+                    tag=tag,
+                    panel_generator=panel_generator,
+                    errors=errors),
+                data.items()
         ):
-
             yield dash
 
 

brian_dashboard_manager/templating/render.py

@@ -276,7 +276,8 @@ def render_complex_dashboard(
 ):
     assert tag or tags
     panels = [create_infobox()]
-    panels.extend(aggregate_panels)
+    if len(aggregate_panels) > 0:
+        panels.extend(aggregate_panels)
     for group in dropdown_groups:
         panels.append(group["dropdown"])
         panels.extend(group["panels"])