diff --git a/brian_dashboard_manager/grafana/provision.py b/brian_dashboard_manager/grafana/provision.py
index d0a21f35dc694acbbe6c207c0aeee37cc3212932..dca966d756ace3a2808984a56c6800fedb0b5ec6 100644
--- a/brian_dashboard_manager/grafana/provision.py
+++ b/brian_dashboard_manager/grafana/provision.py
@@ -8,6 +8,7 @@ import time
import json
import datetime
from functools import reduce
+from concurrent.futures import Future
from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor
from brian_dashboard_manager.config import DEFAULT_ORGANIZATIONS, STATE_PATH
from brian_dashboard_manager.grafana.utils.request import \
@@ -23,7 +24,7 @@ from brian_dashboard_manager.grafana.datasource import \
from brian_dashboard_manager.grafana.folder import find_folder, \
delete_folder
from brian_dashboard_manager.inventory_provider.interfaces import \
- get_gws_direct, get_interfaces
+ get_gws_direct, get_gws_indirect, get_interfaces
from brian_dashboard_manager.templating.nren_access import generate_nrens
from brian_dashboard_manager.templating.helpers import is_re_customer, \
@@ -34,7 +35,8 @@ from brian_dashboard_manager.templating.helpers import is_re_customer, \
get_interface_data, parse_backbone_name, parse_phy_upstream_name, \
get_dashboard_data, get_aggregate_interface_data
-from brian_dashboard_manager.templating.gws import generate_gws
+from brian_dashboard_manager.templating.gws import generate_gws, \
+ generate_indirect
from brian_dashboard_manager.templating.render import render_dashboard
@@ -119,6 +121,9 @@ def provision_maybe(config):
now = datetime.datetime.now()
write_timestamp(now.timestamp(), val)
provision(config)
+ except Exception as e:
+ logger.exception('Uncaught Exception:')
+ raise e
finally:
now = datetime.datetime.now()
write_timestamp(now.timestamp(), False)
@@ -182,7 +187,10 @@ def provision(config):
def excluded(interface):
desc = interface.get('description', '').lower()
- return not any(nren.lower() in desc for nren in excluded_nrens)
+ lab = 'lab.office' in interface.get('router', '').lower()
+ excluded_desc = any(
+ nren.lower() in desc for nren in excluded_nrens)
+ return not (excluded_desc or lab)
excluded_interfaces = list(filter(excluded, interfaces))
@@ -265,6 +273,14 @@ def provision(config):
dash_list = find_dashboard(token_request) or []
dash_list = reduce(get_uid, dash_list, {})
+ def update_dash_list(dashboards):
+ for dashboard in dashboards:
+ if isinstance(dashboard, Future):
+ dashboard = dashboard.result()
+ if dashboard is None:
+ continue
+ dash_list[dashboard.get('uid')] = True
+
with ProcessPoolExecutor(max_workers=4) as executor:
provisioned = []
for folder_name, dash in dashboards.items():
@@ -289,10 +305,35 @@ def provision(config):
folder = result.result()
if folder is None:
continue
- for dashboard in folder:
- if dashboard is None:
+ update_dash_list(folder)
+
+ # fetch GWS direct data and provision related dashboards
+ logger.info('Provisioning GWS Indirect dashboards')
+ folder_name = 'GWS Indirect'
+ exclude_indirect = excluded_folders.get(folder_name, [])
+ exclude_indirect = list(map(lambda f: f.lower(), exclude_indirect))
+ if isinstance(exclude_indirect, bool) and exclude_indirect:
+ # don't provision GWS Direct folder
+ delete_folder(token_request, folder_name)
+ else:
+ folder = find_folder(token_request, folder_name)
+ with ProcessPoolExecutor(max_workers=4) as executor:
+ gws_indirect_data = get_gws_indirect(
+ config['inventory_provider'])
+ provisioned = []
+
+ dashes = generate_indirect(gws_indirect_data, datasource_name)
+ for dashboard in dashes:
+ rendered = render_dashboard(dashboard)
+ if rendered.get('title').lower() in exclude_indirect:
+ executor.submit(delete_dashboard, token_request,
+ rendered, folder['id'])
continue
- dash_list[dashboard.get('uid')] = True
+ provisioned.append(executor.submit(create_dashboard,
+ token_request,
+ rendered, folder['id']))
+
+ update_dash_list(provisioned)
# fetch GWS direct data and provision related dashboards
logger.info('Provisioning GWS Direct dashboards')
@@ -318,11 +359,7 @@ def provision(config):
token_request,
rendered, folder['id']))
- for result in provisioned:
- dashboard = result.result()
- if dashboard is None:
- continue
- dash_list[dashboard.get('uid')] = True
+ update_dash_list(provisioned)
aggregate_dashboards = {
'CLS PEERS': {
@@ -370,11 +407,7 @@ def provision(config):
excluded_interfaces, datasource_name)
provisioned.append(res)
- for result in provisioned:
- dashboard = result.result()
- if dashboard is None:
- continue
- dash_list[dashboard.get('uid')] = True
+ update_dash_list(provisioned)
# NREN Access dashboards
# uses a different template than the above.
diff --git a/brian_dashboard_manager/templating/gws.py b/brian_dashboard_manager/templating/gws.py
index e9f4b1353244bb31b67dc399769882afe20daa35..6a321a6156acccec5ce5afbc172c4258e6f06875 100644
--- a/brian_dashboard_manager/templating/gws.py
+++ b/brian_dashboard_manager/templating/gws.py
@@ -2,7 +2,7 @@ from typing import DefaultDict
from brian_dashboard_manager.templating.helpers import get_dashboard_data
-def get_interface_data(interfaces):
+def get_panel_data(interfaces):
result = DefaultDict(list)
for interface in interfaces:
@@ -39,8 +39,36 @@ def get_interface_data(interfaces):
return result
+def get_gws_indirect_panel_data(interfaces):
+ result = DefaultDict(list)
+
+ for interface in interfaces:
+
+ hostname = interface.get('hostname').replace('.geant.net', '')
+ interface_name = interface.get('interface')
+ service_name = interface.get('name')
+ customer = interface.get('customer')
+
+ measurement = 'dscp32_rates'
+ panel_title = f'{hostname} - {{}} - {interface_name} - #{service_name}'
+ result[f'GWS Indirect - {customer}'].append({
+ 'measurement': measurement,
+ 'title': panel_title,
+ 'interface': interface_name,
+ 'hostname': interface.get('hostname'),
+ 'has_v6': False
+ })
+ return result
+
+
def generate_gws(gws_data, datasource):
- panel_data = get_interface_data(gws_data)
- for dashboard in get_dashboard_data(panel_data, datasource, 'GWS_DIRECT'):
- yield dashboard
+ panel_data = get_panel_data(gws_data)
+ for dash in get_dashboard_data(panel_data, datasource, 'GWS_DIRECT'):
+ yield dash
+
+
+def generate_indirect(gws_data, datasource):
+ panel_data = get_gws_indirect_panel_data(gws_data)
+ for dash in get_dashboard_data(panel_data, datasource, 'GWS_INDIRECT'):
+ yield dash
diff --git a/test/test_gws_indirect.py b/test/test_gws_indirect.py
new file mode 100644
index 0000000000000000000000000000000000000000..9edd611152bbd469f5ba3deec6a38e4069ed86e4
--- /dev/null
+++ b/test/test_gws_indirect.py
@@ -0,0 +1,102 @@
+import responses
+import json
+from brian_dashboard_manager.templating.gws import generate_indirect
+from brian_dashboard_manager.inventory_provider.interfaces import \
+ get_gws_indirect
+
+
+TEST_DATA = [
+ {
+ "id": 712361,
+ "name": "FCCN-AP3-IAS",
+ "customer": "FCCN",
+ "speed": 107374182400,
+ "pop": "PORTO",
+ "hostname": "rt1.por.pt.geant.net",
+ "interface": "ae10.333",
+ "type": "GWS - INDIRECT",
+ "status": "operational"
+ },
+ {
+ "id": 661222,
+ "name": "FCCN-AP2-IAS",
+ "customer": "FCCN",
+ "speed": 42949672960,
+ "pop": "LISBON 2",
+ "hostname": "mx1.lis.pt.geant.net",
+ "interface": "ae10.333",
+ "type": "GWS - INDIRECT",
+ "status": "operational"
+ },
+ {
+ "id": 661500,
+ "name": "IUCC-AP1-IAS",
+ "customer": "IUCC",
+ "speed": 32212254720,
+ "pop": "LONDON",
+ "hostname": "mx1.lon.uk.geant.net",
+ "interface": "ae21.333",
+ "type": "GWS - INDIRECT",
+ "status": "operational"
+ },
+ {
+ "id": 663112,
+ "name": "ROEDUNET_AP1_IAS",
+ "customer": "ROEDUNET",
+ "speed": 42949672960,
+ "pop": "BUCHAREST",
+ "hostname": "mx1.buc.ro.geant.net",
+ "interface": "ae11.333",
+ "type": "GWS - INDIRECT",
+ "status": "operational"
+ },
+ {
+ "id": 663228,
+ "name": "IUCC-AP2-IAS",
+ "customer": "IUCC",
+ "speed": 32212254720,
+ "pop": "FRANKFURT",
+ "hostname": "mx1.fra.de.geant.net",
+ "interface": "ae21.333",
+ "type": "GWS - INDIRECT",
+ "status": "operational"
+ },
+ {
+ "id": 661641,
+ "name": "FCCN-AP1-IAS",
+ "customer": "FCCN",
+ "speed": 42949672960,
+ "pop": "LISBON",
+ "hostname": "mx2.lis.pt.geant.net",
+ "interface": "ae10.333",
+ "type": "GWS - INDIRECT",
+ "status": "operational"
+ }
+]
+
+
+@responses.activate
+def test_gws(data_config, mocker, client):
+
+ def get_callback(request):
+ return 200, {}, json.dumps(TEST_DATA)
+
+ responses.add_callback(
+ method=responses.GET,
+ url=f"{data_config['inventory_provider']}/poller/gws/indirect",
+ callback=get_callback)
+
+ gws_data = get_gws_indirect(data_config['inventory_provider'])
+
+ dashboards = list(generate_indirect(gws_data, 'testdatasource'))
+
+ assert len(dashboards) == 3
+
+ assert dashboards[0]['title'] == 'GWS Indirect - FCCN'
+ assert len(dashboards[0]['panels']) == 3
+
+ assert dashboards[1]['title'] == 'GWS Indirect - IUCC'
+ assert len(dashboards[1]['panels']) == 2
+
+ assert dashboards[2]['title'] == 'GWS Indirect - ROEDUNET'
+ assert len(dashboards[2]['panels']) == 1
diff --git a/test/test_update.py b/test/test_update.py
index 29b955c71b2b95aa5118a13d78bec06d4087c379..55050de6be23493d7b201cdecb1dfaa0f36ad556 100644
--- a/test/test_update.py
+++ b/test/test_update.py
@@ -464,6 +464,10 @@ def test_provision(data_config, mocker, client):
'brian_dashboard_manager.grafana.provision.get_gws_direct')
_mocked_gws.return_value = []
+ _mocked_gws_indirect = mocker.patch(
+ 'brian_dashboard_manager.grafana.provision.get_gws_indirect')
+ _mocked_gws_indirect.return_value = []
+
UID = 1
ID = 1
VERSION = 1