Add GWS indirect dashboards

brian_dashboard_manager/grafana/provision.py

@@ -8,6 +8,7 @@ import time
 import json
 import datetime
 from functools import reduce
+from concurrent.futures import Future
 from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor
 from brian_dashboard_manager.config import DEFAULT_ORGANIZATIONS, STATE_PATH
 from brian_dashboard_manager.grafana.utils.request import \
@@ -23,7 +24,7 @@ from brian_dashboard_manager.grafana.datasource import \
 from brian_dashboard_manager.grafana.folder import find_folder, \
     delete_folder
 from brian_dashboard_manager.inventory_provider.interfaces import \
-    get_gws_direct, get_interfaces
+    get_gws_direct, get_gws_indirect, get_interfaces
 from brian_dashboard_manager.templating.nren_access import generate_nrens
 
 from brian_dashboard_manager.templating.helpers import is_re_customer, \
@@ -34,7 +35,8 @@ from brian_dashboard_manager.templating.helpers import is_re_customer, \
     get_interface_data, parse_backbone_name, parse_phy_upstream_name, \
     get_dashboard_data, get_aggregate_interface_data
 
-from brian_dashboard_manager.templating.gws import generate_gws
+from brian_dashboard_manager.templating.gws import generate_gws, \
+    generate_indirect
 
 from brian_dashboard_manager.templating.render import render_dashboard
 
@@ -119,6 +121,9 @@ def provision_maybe(config):
             now = datetime.datetime.now()
             write_timestamp(now.timestamp(), val)
             provision(config)
+        except Exception as e:
+            logger.exception('Uncaught Exception:')
+            raise e
         finally:
             now = datetime.datetime.now()
             write_timestamp(now.timestamp(), False)
@@ -182,7 +187,10 @@ def provision(config):
 
         def excluded(interface):
             desc = interface.get('description', '').lower()
-            return not any(nren.lower() in desc for nren in excluded_nrens)
+            lab = 'lab.office' in interface.get('router', '').lower()
+            excluded_desc = any(
+                nren.lower() in desc for nren in excluded_nrens)
+            return not (excluded_desc or lab)
 
         excluded_interfaces = list(filter(excluded, interfaces))
 
@@ -265,6 +273,14 @@ def provision(config):
         dash_list = find_dashboard(token_request) or []
         dash_list = reduce(get_uid, dash_list, {})
 
+        def update_dash_list(dashboards):
+            for dashboard in dashboards:
+                if isinstance(dashboard, Future):
+                    dashboard = dashboard.result()
+                if dashboard is None:
+                    continue
+                dash_list[dashboard.get('uid')] = True
+
         with ProcessPoolExecutor(max_workers=4) as executor:
             provisioned = []
             for folder_name, dash in dashboards.items():
@@ -289,10 +305,35 @@ def provision(config):
                 folder = result.result()
                 if folder is None:
                     continue
-                for dashboard in folder:
-                    if dashboard is None:
+                update_dash_list(folder)
+
+        # fetch GWS direct data and provision related dashboards
+        logger.info('Provisioning GWS Indirect dashboards')
+        folder_name = 'GWS Indirect'
+        exclude_indirect = excluded_folders.get(folder_name, [])
+        exclude_indirect = list(map(lambda f: f.lower(), exclude_indirect))
+        if isinstance(exclude_indirect, bool) and exclude_indirect:
+            # don't provision GWS Direct folder
+            delete_folder(token_request, folder_name)
+        else:
+            folder = find_folder(token_request, folder_name)
+            with ProcessPoolExecutor(max_workers=4) as executor:
+                gws_indirect_data = get_gws_indirect(
+                    config['inventory_provider'])
+                provisioned = []
+
+                dashes = generate_indirect(gws_indirect_data, datasource_name)
+                for dashboard in dashes:
+                    rendered = render_dashboard(dashboard)
+                    if rendered.get('title').lower() in exclude_indirect:
+                        executor.submit(delete_dashboard, token_request,
+                                        rendered, folder['id'])
                         continue
-                    dash_list[dashboard.get('uid')] = True
+                    provisioned.append(executor.submit(create_dashboard,
+                                                       token_request,
+                                                       rendered, folder['id']))
+
+                update_dash_list(provisioned)
 
         # fetch GWS direct data and provision related dashboards
         logger.info('Provisioning GWS Direct dashboards')
@@ -318,11 +359,7 @@ def provision(config):
                                                        token_request,
                                                        rendered, folder['id']))
 
-                for result in provisioned:
-                    dashboard = result.result()
-                    if dashboard is None:
-                        continue
-                    dash_list[dashboard.get('uid')] = True
+                update_dash_list(provisioned)
 
         aggregate_dashboards = {
             'CLS PEERS': {
@@ -370,11 +407,7 @@ def provision(config):
                                           excluded_interfaces, datasource_name)
                     provisioned.append(res)
 
-                for result in provisioned:
-                    dashboard = result.result()
-                    if dashboard is None:
-                        continue
-                    dash_list[dashboard.get('uid')] = True
+                update_dash_list(provisioned)
 
         # NREN Access dashboards
         # uses a different template than the above.

brian_dashboard_manager/templating/gws.py

@@ -2,7 +2,7 @@ from typing import DefaultDict
 from brian_dashboard_manager.templating.helpers import get_dashboard_data
 
 
-def get_interface_data(interfaces):
+def get_panel_data(interfaces):
     result = DefaultDict(list)
 
     for interface in interfaces:
@@ -39,8 +39,36 @@ def get_interface_data(interfaces):
     return result
 
 
+def get_gws_indirect_panel_data(interfaces):
+    result = DefaultDict(list)
+
+    for interface in interfaces:
+
+        hostname = interface.get('hostname').replace('.geant.net', '')
+        interface_name = interface.get('interface')
+        service_name = interface.get('name')
+        customer = interface.get('customer')
+
+        measurement = 'dscp32_rates'
+        panel_title = f'{hostname} - {{}} - {interface_name} - #{service_name}'
+        result[f'GWS Indirect - {customer}'].append({
+            'measurement': measurement,
+            'title': panel_title,
+            'interface': interface_name,
+            'hostname': interface.get('hostname'),
+            'has_v6': False
+        })
+    return result
+
+
 def generate_gws(gws_data, datasource):
 
-    panel_data = get_interface_data(gws_data)
-    for dashboard in get_dashboard_data(panel_data, datasource, 'GWS_DIRECT'):
-        yield dashboard
+    panel_data = get_panel_data(gws_data)
+    for dash in get_dashboard_data(panel_data, datasource, 'GWS_DIRECT'):
+        yield dash
+
+
+def generate_indirect(gws_data, datasource):
+    panel_data = get_gws_indirect_panel_data(gws_data)
+    for dash in get_dashboard_data(panel_data, datasource, 'GWS_INDIRECT'):
+        yield dash

test/test_gws_indirect.py

+import responses
+import json
+from brian_dashboard_manager.templating.gws import generate_indirect
+from brian_dashboard_manager.inventory_provider.interfaces import \
+    get_gws_indirect
+
+
+TEST_DATA = [
+    {
+        "id": 712361,
+        "name": "FCCN-AP3-IAS",
+        "customer": "FCCN",
+        "speed": 107374182400,
+        "pop": "PORTO",
+        "hostname": "rt1.por.pt.geant.net",
+        "interface": "ae10.333",
+        "type": "GWS - INDIRECT",
+        "status": "operational"
+    },
+    {
+        "id": 661222,
+        "name": "FCCN-AP2-IAS",
+        "customer": "FCCN",
+        "speed": 42949672960,
+        "pop": "LISBON 2",
+        "hostname": "mx1.lis.pt.geant.net",
+        "interface": "ae10.333",
+        "type": "GWS - INDIRECT",
+        "status": "operational"
+    },
+    {
+        "id": 661500,
+        "name": "IUCC-AP1-IAS",
+        "customer": "IUCC",
+        "speed": 32212254720,
+        "pop": "LONDON",
+        "hostname": "mx1.lon.uk.geant.net",
+        "interface": "ae21.333",
+        "type": "GWS - INDIRECT",
+        "status": "operational"
+    },
+    {
+        "id": 663112,
+        "name": "ROEDUNET_AP1_IAS",
+        "customer": "ROEDUNET",
+        "speed": 42949672960,
+        "pop": "BUCHAREST",
+        "hostname": "mx1.buc.ro.geant.net",
+        "interface": "ae11.333",
+        "type": "GWS - INDIRECT",
+        "status": "operational"
+    },
+    {
+        "id": 663228,
+        "name": "IUCC-AP2-IAS",
+        "customer": "IUCC",
+        "speed": 32212254720,
+        "pop": "FRANKFURT",
+        "hostname": "mx1.fra.de.geant.net",
+        "interface": "ae21.333",
+        "type": "GWS - INDIRECT",
+        "status": "operational"
+    },
+    {
+        "id": 661641,
+        "name": "FCCN-AP1-IAS",
+        "customer": "FCCN",
+        "speed": 42949672960,
+        "pop": "LISBON",
+        "hostname": "mx2.lis.pt.geant.net",
+        "interface": "ae10.333",
+        "type": "GWS - INDIRECT",
+        "status": "operational"
+    }
+]
+
+
+@responses.activate
+def test_gws(data_config, mocker, client):
+
+    def get_callback(request):
+        return 200, {}, json.dumps(TEST_DATA)
+
+    responses.add_callback(
+        method=responses.GET,
+        url=f"{data_config['inventory_provider']}/poller/gws/indirect",
+        callback=get_callback)
+
+    gws_data = get_gws_indirect(data_config['inventory_provider'])
+
+    dashboards = list(generate_indirect(gws_data, 'testdatasource'))
+
+    assert len(dashboards) == 3
+
+    assert dashboards[0]['title'] == 'GWS Indirect - FCCN'
+    assert len(dashboards[0]['panels']) == 3
+
+    assert dashboards[1]['title'] == 'GWS Indirect - IUCC'
+    assert len(dashboards[1]['panels']) == 2
+
+    assert dashboards[2]['title'] == 'GWS Indirect - ROEDUNET'
+    assert len(dashboards[2]['panels']) == 1

test/test_update.py

@@ -464,6 +464,10 @@ def test_provision(data_config, mocker, client):
         'brian_dashboard_manager.grafana.provision.get_gws_direct')
     _mocked_gws.return_value = []
 
+    _mocked_gws_indirect = mocker.patch(
+        'brian_dashboard_manager.grafana.provision.get_gws_indirect')
+    _mocked_gws_indirect.return_value = []
+
     UID = 1
     ID = 1
     VERSION = 1