diff --git a/brian_dashboard_manager/grafana/provision.py b/brian_dashboard_manager/grafana/provision.py
index a0c9ba67f807c857e09f168ff66583cd82e2ed23..c8df565d908f02223e532753f77c84c268cc329f 100644
--- a/brian_dashboard_manager/grafana/provision.py
+++ b/brian_dashboard_manager/grafana/provision.py
@@ -33,11 +33,14 @@ from brian_dashboard_manager.templating.helpers import \
from brian_dashboard_manager.templating.gws import generate_gws, \
generate_indirect
-from brian_dashboard_manager.templating.eumetsat import generate_eumetsat_multicast
+from brian_dashboard_manager.templating.eumetsat \
+ import generate_eumetsat_multicast
from brian_dashboard_manager.templating.render import render_dashboard
logger = logging.getLogger(__name__)
+DASHBOARD_CHANGES = None # will be an instance of DashboardChanges
+
MAX_WORKERS = 1
DASHBOARDS = {
'NREN': {
@@ -153,6 +156,28 @@ AGG_DASHBOARDS = {
}
+class DashboardChanges(object):
+ def __init__(self, token):
+ # Map of dashboard UID -> whether it has been updated.
+ # This is used to remove stale dashboards at the end.
+ all_dashboards = find_dashboard(token) or []
+ self.updated = {d['uid']: False for d in all_dashboards}
+
+ def update_dash_list(self, dashboards):
+ for dashboard in dashboards:
+ if isinstance(dashboard, Future):
+ dashboard = dashboard.result()
+ if dashboard is None:
+ continue
+ self.updated[dashboard.get('uid')] = True
+
+ def delete_untouched(self, token):
+ for uid, provisioned in self.updated.items():
+ if not provisioned:
+ logger.info(f'Deleting stale dashboard with UID {uid}')
+ delete_dashboard(token, {'uid': uid})
+
+
def provision_folder(token_request, folder_name, dash,
ds_name, excluded_dashboards):
"""
@@ -252,31 +277,9 @@ def excluded_folder_dashboards(org_config, folder_name):
return excluded if isinstance(excluded, list) else []
-class DashboardChanges(object):
- def __init__(self, token):
- # Map of dashboard UID -> whether it has been updated.
- # This is used to remove stale dashboards at the end.
- all_dashboards = find_dashboard(token) or []
- self.updated = {d['uid']: False for d in all_dashboards}
-
- def update_dash_list(self, dashboards):
- for dashboard in dashboards:
- if isinstance(dashboard, Future):
- dashboard = dashboard.result()
- if dashboard is None:
- continue
- self.updated[dashboard.get('uid')] = True
-
- def delete_untouched(self, token):
- for uid, provisioned in self.updated.items():
- if not provisioned:
- logger.info(f'Deleting stale dashboard with UID {uid}')
- delete_dashboard(token, {'uid': uid})
-
-
-DASHBOARD_CHANGES = None # will be an instance of DashboardChanges
-
def _provision_interfaces(config, org_config, ds_name, token):
+ # Provision dashboards, overwriting existing ones.
+
interfaces = get_interfaces(config['inventory_provider'])
excluded_nrens = org_config['excluded_nrens']
@@ -287,14 +290,8 @@ def _provision_interfaces(config, org_config, ds_name, token):
to_exclude = any(nren.lower() in desc for nren in excluded_nrens)
return not (to_exclude or lab)
-
relevant_interfaces = list(filter(excluded, interfaces))
- # Provision dashboards, overwriting existing ones.
-
-
-
-
# loop over interfaces and add them to the dashboard_name
# -> folder mapping structure `dashboards` above, for convenience.
for iface in relevant_interfaces:
@@ -326,9 +323,10 @@ def _provision_interfaces(config, org_config, ds_name, token):
logger.info(
f'Provisioning {org_config["name"]}/{folder_name} dashboards')
- res = executor.submit(provision_folder, token,
- folder_name, folder, ds_name,
- excluded_folder_dashboards(org_config, folder_name))
+ res = executor.submit(
+ provision_folder, token,
+ folder_name, folder, ds_name,
+ excluded_folder_dashboards(org_config, folder_name))
provisioned.append(res)
for result in provisioned:
@@ -393,14 +391,19 @@ def _provision_eumetsat_multicast(config, org_config, ds_name, token):
else:
folder = find_folder(token, folder_name)
with ThreadPoolExecutor(max_workers=MAX_WORKERS) as executor:
- subscriptions = get_eumetsat_multicast_subscriptions(config['inventory_provider'])
+ subscriptions = get_eumetsat_multicast_subscriptions(
+ config['inventory_provider'])
provisioned = []
- for dashboard in generate_eumetsat_multicast(subscriptions, ds_name):
+ for dashboard in generate_eumetsat_multicast(
+ subscriptions, ds_name):
rendered = render_dashboard(dashboard)
- provisioned.append(executor.submit(create_dashboard,
- token,
- rendered, folder['id']))
+ provisioned.append(
+ executor.submit(
+ create_dashboard,
+ token,
+ rendered,
+ folder['id']))
DASHBOARD_CHANGES.update_dash_list(provisioned)
@@ -414,7 +417,8 @@ def _provision_aggregates(config, org_config, ds_name, token):
provisioned = []
agg_folder = find_folder(token, 'Aggregates')
for dash in AGG_DASHBOARDS.values():
- excluded_dashboards = excluded_folder_dashboards(org_config, 'Aggregates')
+ excluded_dashboards = excluded_folder_dashboards(
+ org_config, 'Aggregates')
if dash['dashboard_name'] in excluded_dashboards:
dash_name = {
'title': f'Aggregate - {dash["dashboard_name"]}'}
@@ -423,7 +427,7 @@ def _provision_aggregates(config, org_config, ds_name, token):
agg_folder['id'])
continue
logger.info(f'Provisioning {org_config["name"]}' +
- f'/Aggregate {dash["dashboard_name"]} dashboards') # noqa: E501
+ f'/Aggregate {dash["dashboard_name"]} dashboards')
res = executor.submit(
provision_aggregate, token,
agg_folder, dash, ds_name)
@@ -458,7 +462,8 @@ def _set_ignored_folders_as_updated(config, org_config, token):
ignored_folders = config.get('ignored_folders', [])
for name in ignored_folders:
logger.info(
- f'Ignoring dashboards under the folder {org_config["name"]}/{name}')
+ 'Ignoring dashboards under '
+ f'the folder {org_config["name"]}/{name}')
folder = find_folder(token, name, create=False)
if folder is None:
continue
@@ -548,7 +553,6 @@ def provision(config):
org['info'] = 'Org exists in grafana but is not configured'
return None
-
for org in all_orgs:
org_id = org['id']
delete_expired_api_tokens(request, org_id)
@@ -572,9 +576,11 @@ def provision(config):
_provision_interfaces(config, org_config, ds_name, token_request)
_provision_gws_indirect(config, org_config, ds_name, token_request)
_provision_gws_direct(config, org_config, ds_name, token_request)
- _provision_eumetsat_multicast(config, org_config, ds_name, token_request)
+ _provision_eumetsat_multicast(
+ config, org_config, ds_name, token_request)
_provision_aggregates(config, org_config, ds_name, token_request)
- _provision_static_dashboards(config, org_config, ds_name, token_request)
+ _provision_static_dashboards(
+ config, org_config, ds_name, token_request)
_set_ignored_folders_as_updated(config, org_config, token_request)
DASHBOARD_CHANGES.delete_untouched(token_request)
diff --git a/brian_dashboard_manager/templating/eumetsat.py b/brian_dashboard_manager/templating/eumetsat.py
index 2c28bfb684e9595ac82f609a33a09d7db216ed4f..17a417b5db0cb409ef1551f20991dc4157fc0560 100644
--- a/brian_dashboard_manager/templating/eumetsat.py
+++ b/brian_dashboard_manager/templating/eumetsat.py
@@ -21,16 +21,16 @@ def get_panel_data(all_subscriptions):
# make the panels sorted deterministically
for name in result.keys():
- result[name] = sorted(result[name], key=operator.itemgetter('subscription'))
+ result[name] = sorted(
+ result[name],
+ key=operator.itemgetter('subscription'))
return result
-
def get_panel_fields(panel, panel_type, datasource):
"""
- Helper for generating a single panel,
- with ingress/egress and percentile targets
+ Helper for generating a single multicast panel
"""
letters = letter_generator()
@@ -56,13 +56,10 @@ def get_panel_fields(panel, panel_type, datasource):
'y_axis_type': 'bits',
})
+
def subscription_panel_generator(gridPos):
"""
- Shared wrapper for shorter calls without
- gridPos to generate panels.
-
- Generates panels used in a normal dashboard
- for all traffic + (conditionally) IPv6 + Errors
+ Generates panels used for multicast traffic dashboards
"""
def get_panel_definitions(panels, datasource, errors=False):
result = []
@@ -97,4 +94,3 @@ def generate_eumetsat_multicast(subscriptions, datasource):
panel_generator=subscription_panel_generator):
yield dash
-
diff --git a/test/test_eumetsat_multicast.py b/test/test_eumetsat_multicast.py
index 8811d11f0d163e55cf189ace57dd0d65bc2ffdb0..dd088bd1bec130ace2ee468667d8362a4b2ef405 100644
--- a/test/test_eumetsat_multicast.py
+++ b/test/test_eumetsat_multicast.py
@@ -1,6 +1,4 @@
import responses
-import json
-from brian_dashboard_manager.templating.gws import generate_gws
from brian_dashboard_manager.inventory_provider.interfaces import \
get_eumetsat_multicast_subscriptions
@@ -8,42 +6,42 @@ from brian_dashboard_manager.inventory_provider.interfaces import \
TEST_DATA = [
{
'router': 'mx1.ams.nl.geant.net',
- 'oid': '1.3.6.1.2.1.83.1.1.2.1.16.232.223.222.1.193.17.9.3.255.255.255.255',
+ 'oid': '1.3.6.1.2.1.83.1.1.2.1.16.232.223.222.1.193.17.9.3.255.255.255.255', # noqa: E501
'community': '0pBiFbD',
'subscription': '232.223.222.1',
'endpoint': '193.17.9.3'
},
{
'router': 'mx1.ams.nl.geant.net',
- 'oid': '1.3.6.1.2.1.83.1.1.2.1.16.232.223.222.2.193.17.9.3.255.255.255.255',
+ 'oid': '1.3.6.1.2.1.83.1.1.2.1.16.232.223.222.2.193.17.9.3.255.255.255.255', # noqa: E501
'community': '0pBiFbD',
'subscription': '232.223.222.2',
'endpoint': '193.17.9.3'
},
{
'router': 'mx1.lon.uk.geant.net',
- 'oid': '1.3.6.1.2.1.83.1.1.2.1.16.232.223.222.1.193.17.9.3.255.255.255.255',
+ 'oid': '1.3.6.1.2.1.83.1.1.2.1.16.232.223.222.1.193.17.9.3.255.255.255.255', # noqa: E501
'community': '0pBiFbD',
'subscription': '232.223.222.1',
'endpoint': '193.17.9.3'
},
{
'router': 'mx1.lon.uk.geant.net',
- 'oid': '1.3.6.1.2.1.83.1.1.2.1.16.232.223.222.2.193.17.9.3.255.255.255.255',
+ 'oid': '1.3.6.1.2.1.83.1.1.2.1.16.232.223.222.2.193.17.9.3.255.255.255.255', # noqa: E501
'community': '0pBiFbD',
'subscription': '232.223.222.2',
'endpoint': '193.17.9.3'
},
{
'router': 'mx1.fra.de.geant.net',
- 'oid': '1.3.6.1.2.1.83.1.1.2.1.16.232.223.222.1.193.17.9.3.255.255.255.255',
+ 'oid': '1.3.6.1.2.1.83.1.1.2.1.16.232.223.222.1.193.17.9.3.255.255.255.255', # noqa: E501
'community': '0pBiFbD',
'subscription': '232.223.222.1',
'endpoint': '193.17.9.3'
},
{
'router': 'mx1.fra.de.geant.net',
- 'oid': '1.3.6.1.2.1.83.1.1.2.1.16.232.223.222.2.193.17.9.3.255.255.255.255',
+ 'oid': '1.3.6.1.2.1.83.1.1.2.1.16.232.223.222.2.193.17.9.3.255.255.255.255', # noqa: E501
'community': '0pBiFbD',
'subscription': '232.223.222.2',
'endpoint': '193.17.9.3'
@@ -59,7 +57,8 @@ def test_eumetsat_subscriptions(data_config, client):
url=f'{data_config["inventory_provider"]}/poller/eumetsat-multicast',
json=TEST_DATA)
- subscriptions = get_eumetsat_multicast_subscriptions(data_config['inventory_provider'])
+ subscriptions = get_eumetsat_multicast_subscriptions(
+ data_config['inventory_provider'])
print(subscriptions)
# dashboards = list(generate_gws(gws_data, 'testdatasource'))
#
diff --git a/test/test_gws_direct.py b/test/test_gws_direct.py
index 46caa180006e1273de075de9485f43b91a9130f5..cbbd2380cc47cdd9f07af637096191d6e08c5b79 100644
--- a/test/test_gws_direct.py
+++ b/test/test_gws_direct.py
@@ -1,5 +1,4 @@
import responses
-import json
from brian_dashboard_manager.templating.gws import generate_gws
from brian_dashboard_manager.inventory_provider.interfaces import \
get_gws_direct
diff --git a/test/test_gws_indirect.py b/test/test_gws_indirect.py
index 7e5a78ba1de95f71a47d66e456031572de42dce2..ba26780233982a47877ccc72f4e8d60234044067 100644
--- a/test/test_gws_indirect.py
+++ b/test/test_gws_indirect.py
@@ -1,5 +1,4 @@
import responses
-import json
from brian_dashboard_manager.templating.gws import generate_indirect
from brian_dashboard_manager.inventory_provider.interfaces import \
get_gws_indirect
diff --git a/test/test_update.py b/test/test_update.py
index ad61acbd22b92ad2845f4473db8eeaaa402de831..f12d2c60cb74dacb20518f05bee91af97b89dbb7 100644
--- a/test/test_update.py
+++ b/test/test_update.py
@@ -1,17 +1,9 @@
import responses
import json
-import re
from brian_dashboard_manager.grafana.provision import provision_folder, \
provision
-
-DEFAULT_REQUEST_HEADERS = {
- "Content-type": "application/json",
- "Accept": ["application/json"]
-}
-
-
TEST_INTERFACES = [
{
"router": "srx2.ch.office.geant.net",
@@ -469,6 +461,51 @@ NREN_INTERFACES = [
}
]
+EUMETSAT_MULTICAST = [
+ {
+ 'router': 'mx1.ams.nl.geant.net',
+ 'oid': '1.3.6.1.2.1.83.1.1.2.1.16.232.223.222.1.193.17.9.3.255.255.255.255', # noqa: E501
+ 'community': '0pBiFbD',
+ 'subscription': '232.223.222.1',
+ 'endpoint': '193.17.9.3'
+ },
+ {
+ 'router': 'mx1.ams.nl.geant.net',
+ 'oid': '1.3.6.1.2.1.83.1.1.2.1.16.232.223.222.2.193.17.9.3.255.255.255.255', # noqa: E501
+ 'community': '0pBiFbD',
+ 'subscription': '232.223.222.2',
+ 'endpoint': '193.17.9.3'
+ },
+ {
+ 'router': 'mx1.lon.uk.geant.net',
+ 'oid': '1.3.6.1.2.1.83.1.1.2.1.16.232.223.222.1.193.17.9.3.255.255.255.255', # noqa: E501
+ 'community': '0pBiFbD',
+ 'subscription': '232.223.222.1',
+ 'endpoint': '193.17.9.3'
+ },
+ {
+ 'router': 'mx1.lon.uk.geant.net',
+ 'oid': '1.3.6.1.2.1.83.1.1.2.1.16.232.223.222.2.193.17.9.3.255.255.255.255', # noqa: E501
+ 'community': '0pBiFbD',
+ 'subscription': '232.223.222.2',
+ 'endpoint': '193.17.9.3'
+ },
+ {
+ 'router': 'mx1.fra.de.geant.net',
+ 'oid': '1.3.6.1.2.1.83.1.1.2.1.16.232.223.222.1.193.17.9.3.255.255.255.255', # noqa: E501
+ 'community': '0pBiFbD',
+ 'subscription': '232.223.222.1',
+ 'endpoint': '193.17.9.3'
+ },
+ {
+ 'router': 'mx1.fra.de.geant.net',
+ 'oid': '1.3.6.1.2.1.83.1.1.2.1.16.232.223.222.2.193.17.9.3.255.255.255.255', # noqa: E501
+ 'community': '0pBiFbD',
+ 'subscription': '232.223.222.2',
+ 'endpoint': '193.17.9.3'
+ }
+]
+
def generate_folder(data):
return {
@@ -494,12 +531,16 @@ def test_provision_folder(data_config, mocker):
'NREN': {
'tag': ['customers'],
'folder_name': 'NREN Access',
- 'interfaces': [iface for iface in TEST_INTERFACES if 'NREN' in iface['dashboards']] # noqa: E501
+ 'interfaces': [
+ iface for iface in TEST_INTERFACES
+ if 'NREN' in iface['dashboards']]
},
'RE_CUST': {
'tag': 'RE_CUST',
'folder_name': 'RE Customer',
- 'interfaces': [iface for iface in TEST_INTERFACES if 'RE_CUST' in iface['dashboards']] # noqa: E501
+ 'interfaces': [
+ iface for iface in TEST_INTERFACES
+ if 'RE_CUST' in iface['dashboards']]
},
}
@@ -554,15 +595,25 @@ def test_provision(data_config, mocker, client):
url=f"{data_config['inventory_provider']}/data/interfaces",
json=NREN_INTERFACES)
+ responses.add(
+ method=responses.GET,
+ url=f'{data_config["inventory_provider"]}/poller/eumetsat-multicast',
+ json=EUMETSAT_MULTICAST)
+
+ responses.add(
+ method=responses.DELETE,
+ url=f"http://{data_config['hostname']}/api/folders",
+ json={"message": "Deleted folder"})
+
responses.add(
method=responses.GET,
url=f"http://{data_config['hostname']}/api/folders",
json=[])
responses.add(
- method=responses.DELETE,
- url=re.compile(f"http://{data_config['hostname']}/api/folders"),
- json={"message": "Deleted folder"} )
+ method='get',
+ url=f"http://{data_config['hostname']}/api/folders",
+ json=[])
def folder_post(request):
data = json.loads(request.body)
@@ -573,26 +624,31 @@ def test_provision(data_config, mocker, client):
url=f"http://{data_config['hostname']}/api/folders",
callback=folder_post)
- responses.add(
- method=responses.GET,
- url=f"http://{data_config['hostname']}/api/search?query=Home",
- json=[])
+ def search_responses(request):
+ if request.params.get('query', None) == 'Home':
+ return 200, {}, json.dumps([])
+ if request.params.get('type', None) == 'dash-db':
+ return 200, {}, json.dumps([])
+ assert False # no other queries expected
- TEST_DATASOURCE = [{
- "name": "brian-influx-datasource",
- "type": "influxdb",
- "access": "proxy",
- "url": "http://test-brian-datasource.geant.org:8086",
- "database": "test-db",
- "basicAuth": False,
- "isDefault": True,
- "readOnly": False
- }]
+ responses.add_callback(
+ method=responses.GET,
+ url=f"http://{data_config['hostname']}/api/search",
+ callback=search_responses)
responses.add(
method=responses.GET,
url=f"http://{data_config['hostname']}/api/datasources",
- json=TEST_DATASOURCE)
+ json=[{
+ "name": "brian-influx-datasource",
+ "type": "influxdb",
+ "access": "proxy",
+ "url": "http://test-brian-datasource.geant.org:8086",
+ "database": "test-db",
+ "basicAuth": False,
+ "isDefault": True,
+ "readOnly": False
+ }])
responses.add(
method=responses.POST,