diff --git a/brian_dashboard_manager/grafana/provision.py b/brian_dashboard_manager/grafana/provision.py
index 8acabc7fa8ad6e54cdad6bd0a5c0036e414c43e7..57b85d05b4502012bedd2b11fa4866be583568ab 100644
--- a/brian_dashboard_manager/grafana/provision.py
+++ b/brian_dashboard_manager/grafana/provision.py
@@ -18,11 +18,12 @@ from brian_dashboard_manager.inventory_provider.interfaces import \
from brian_dashboard_manager.templating.nren_access import generate_nrens
from brian_dashboard_manager.templating.helpers import is_re_customer, \
- is_cls, is_ias_customer, is_ias_private, is_ias_public, is_ias_upstream, \
- is_lag_backbone, is_nren, is_phy_upstream, is_re_peer, is_gcs, \
- is_geantopen, is_l2circuit, is_lhcone_peer, is_lhcone_customer, is_mdvpn,\
+ is_cls_peer, is_cls, is_ias_customer, is_ias_private, is_ias_public, \
+ is_ias_upstream, is_ias_peer, is_lag_backbone, is_nren, is_phy_upstream, \
+ is_re_peer, is_gcs, is_geantopen, is_l2circuit, is_lhcone_peer, \
+ is_lhcone_customer, is_lhcone, is_mdvpn, get_aggregate_dashboard_data, \
get_interface_data, parse_backbone_name, parse_phy_upstream_name, \
- get_dashboard_data
+ get_dashboard_data, get_aggregate_interface_data
from brian_dashboard_manager.templating.render import render_dashboard
@@ -61,6 +62,21 @@ def provision_folder(token_request, folder_name,
rendered, folder['id'])
+def provision_aggregate(token_request, agg_type, aggregate_folder,
+ dash, excluded_interfaces, datasource_name):
+ predicate = dash['predicate']
+ tag = dash['tag']
+
+ relevant_interfaces = filter(predicate, excluded_interfaces)
+ data = get_aggregate_interface_data(relevant_interfaces, agg_type)
+
+ dashboard = get_aggregate_dashboard_data(
+ f'Aggregate - {agg_type}', data, datasource_name, tag)
+
+ rendered = render_dashboard(dashboard)
+ create_dashboard(token_request, rendered, aggregate_folder['id'])
+
+
def provision(config):
request = AdminRequest(**config)
@@ -200,6 +216,39 @@ def provision(config):
folder_name, dash,
excluded_interfaces, datasource_name)
+ aggregate_dashboards = {
+ 'CLS PEERS': {
+ 'predicate': is_cls_peer,
+ 'tag': 'cls_peers',
+ },
+ 'IAS PEERS': {
+ 'predicate': is_ias_peer,
+ 'tag': 'ias_peers',
+ },
+ 'GWS UPSTREAMS': {
+ 'predicate': is_ias_upstream,
+ 'tag': 'gws_upstreams',
+ },
+ 'LHCONE': {
+ 'predicate': is_lhcone,
+ 'tag': 'lhcone',
+ },
+ # 'CAE1': {
+ # 'predicate': is_cae1,
+ # 'tag': 'cae',
+ # }
+ }
+
+ with ProcessPoolExecutor(max_workers=4) as executor:
+ aggregate_folder = find_folder(token_request, 'Aggregates')
+ for agg_type, dash in aggregate_dashboards.items():
+ logger.info(
+ f'Provisioning {org["name"]}' +
+ f'/Aggregate {agg_type} dashboards')
+ executor.submit(provision_aggregate, token_request, agg_type,
+ aggregate_folder, dash,
+ excluded_interfaces, datasource_name)
+
# NREN Access dashboards
# uses a different template than the above.
logger.info('Provisioning NREN Access dashboards')
diff --git a/brian_dashboard_manager/templating/helpers.py b/brian_dashboard_manager/templating/helpers.py
index 46e141c5eb2282444c7458f45bdfc231f051945c..9f9adf29cb3d2094dcaa2943b5e72bf729a2d5c7 100644
--- a/brian_dashboard_manager/templating/helpers.py
+++ b/brian_dashboard_manager/templating/helpers.py
@@ -1,8 +1,11 @@
import re
import logging
+import json
from itertools import product
+from functools import reduce
from string import ascii_uppercase
-from brian_dashboard_manager.templating.render import create_panel
+from brian_dashboard_manager.templating.render import create_panel, \
+ create_panel_target
PANEL_HEIGHT = 12
PANEL_WIDTH = 24
@@ -35,6 +38,10 @@ def is_cls(interface):
return 'SRV_CLS' in get_description(interface)
+def is_cls_peer(interface):
+ return 'SRV_CLS PRIVATE' in get_description(interface)
+
+
def is_ias_public(interface):
return 'SRV_IAS PUBLIC' in get_description(interface)
@@ -51,6 +58,10 @@ def is_ias_upstream(interface):
return 'SRV_IAS UPSTREAM' in get_description(interface)
+def is_ias_peer(interface):
+ return is_ias_public(interface) or is_ias_private(interface)
+
+
def is_re_peer(interface):
return 'SRV_GLOBAL RE_INTERCONNECT' in get_description(interface)
@@ -82,6 +93,11 @@ def is_lhcone_customer(interface):
return 'LHCONE' in description and 'SRV_L3VPN CUSTOMER' in description
+def is_lhcone(interface):
+ regex = 'SRV_L3VPN (CUSTOMER|RE_INTERCONNECT)'
+ return re.match(regex, get_description(interface))
+
+
def is_mdvpn(interface):
return re.match('^SRV_MDVPN CUSTOMER', get_description(interface))
@@ -96,6 +112,10 @@ def is_lag_backbone(interface):
return is_infrastructure_backbone(interface) and is_lag
+def is_cae1(interface):
+ return interface.get('router', '').lower() == 'mx1.lon.uk.geant.net'
+
+
def parse_backbone_name(description, *args, **kwargs):
link = description.split('|')[1].strip()
link = link.replace('( ', '(')
@@ -119,16 +139,24 @@ def num_generator(start=1):
num += 1
-def gridPos_generator(id_generator, start=0):
+def gridPos_generator(id_generator, start=0, agg=False):
num = start
while True:
yield {
"height": PANEL_HEIGHT,
- "width": PANEL_WIDTH,
+ "width": PANEL_WIDTH if not agg else PANEL_WIDTH // 2,
"x": 0,
"y": num * PANEL_HEIGHT,
"id": next(id_generator)
}
+ if agg:
+ yield {
+ "height": PANEL_HEIGHT,
+ "width": PANEL_WIDTH // 2,
+ "x": PANEL_WIDTH // 2,
+ "y": num * PANEL_HEIGHT,
+ "id": next(id_generator)
+ }
num += 1
@@ -155,12 +183,14 @@ def letter_generator():
# parse_func receives interface information and returns a peer name.
def get_interface_data(interfaces, name_parse_func=None):
result = {}
+
+ if not name_parse_func:
+ # Most (but not all) descriptions use a format
+ # which has the peer name as the third element.
+ def name_parse_func(desc, *args, **kwargs):
+ return desc.split(' ')[2].upper()
+
for interface in interfaces:
- if not name_parse_func:
- # Most (but not all) descriptions use a format
- # which has the peer name as the third element.
- def name_parse_func(desc, *args, **kwargs):
- return desc.split(' ')[2].upper()
description = interface.get('description', '').strip()
interface_name = interface.get('name')
@@ -182,6 +212,67 @@ def get_interface_data(interfaces, name_parse_func=None):
return result
+def get_aggregate_interface_data(interfaces, agg_type):
+ result = []
+
+ def reduce_func(prev, curr):
+ remotes = prev.get(curr['remote'], [])
+ remotes.append(curr)
+ all_agg = prev.get('EVERYSINGLEPANEL', [])
+ all_agg.append(curr)
+ prev[curr['remote']] = remotes
+ prev['EVERYSINGLEPANEL'] = all_agg
+ return prev
+
+ for interface in interfaces:
+
+ description = interface.get('description', '').strip()
+ interface_name = interface.get('name')
+ host = interface.get('router', '')
+
+ remote = description.split(' ')[2].upper()
+
+ result.append({
+ 'type': agg_type,
+ 'interface': interface_name,
+ 'hostname': host,
+ 'remote': remote,
+ 'alias': f"{host.split('.')[1].upper()} - {remote}"
+ })
+ return reduce(reduce_func, result, {})
+
+
+# Helper used for generating stacked aggregate panels
+# with multiple target fields (ingress/egress)
+def get_aggregate_targets(targets):
+ ingress = []
+ egress = []
+
+ # used to generate refIds
+ letters = letter_generator()
+
+ for target in targets:
+ ref_id = next(letters)
+ in_data = {
+ **target,
+ 'alias': f"{target['alias']} - Ingress Traffic",
+ 'refId': ref_id,
+ 'select_field': 'ingress'
+ }
+ out_data = {
+ **target,
+ 'alias': f"{target['alias']} - Egress Traffic",
+ 'refId': ref_id,
+ 'select_field': 'egress'
+ }
+ ingress_target = create_panel_target(in_data)
+ egress_target = create_panel_target(out_data)
+ ingress.append(ingress_target)
+ egress.append(egress_target)
+
+ return ingress, egress
+
+
# Helper used for generating all traffic/error panels
# with a single target field (ingress/egress or err/s)
def get_panel_fields(panel, panel_type, datasource):
@@ -214,6 +305,7 @@ def get_panel_fields(panel, panel_type, datasource):
return create_panel({
**panel,
'datasource': datasource,
+ 'linewidth': 1,
'title': panel['title'].format(panel_type),
'panel_targets': [get_target_data(*target) for target in targets],
'y_axis_type': 'errors' if is_error else 'bits',
@@ -244,3 +336,82 @@ def get_dashboard_data(data, datasource, tag, errors=False):
'panels': get_panel_definitions(panels, datasource),
'tag': tag
}
+
+
+def create_aggregate_panel(title, gridpos, targets, datasource):
+
+ ingress_targets, egress_targets = get_aggregate_targets(targets)
+ result = []
+
+ ingress_pos = next(gridpos)
+ egress_pos = next(gridpos)
+
+ is_total = 'totals' in title.lower()
+
+ def reduce_alias(prev, curr):
+ d = json.loads(curr)
+ alias = d['alias']
+ if 'egress' in alias.lower():
+ prev[alias] = '#0000FF'
+ else:
+ prev[alias] = '#00FF00'
+ return prev
+
+ ingress_colors = reduce(reduce_alias, ingress_targets, {})
+ egress_colors = reduce(reduce_alias, egress_targets, {})
+
+ result.append(create_panel({
+ **ingress_pos,
+ 'stack': True,
+ 'linewidth': 0 if is_total else 1,
+ 'datasource': datasource,
+ 'title': title + ' - ingress',
+ 'targets': ingress_targets,
+ 'y_axis_type': 'bits',
+ 'alias_colors': json.dumps(ingress_colors) if is_total else {}
+ }))
+
+ result.append(create_panel({
+ **egress_pos,
+ 'stack': True,
+ 'linewidth': 0 if is_total else 1,
+ 'datasource': datasource,
+ 'title': title + ' - egress',
+ 'targets': egress_targets,
+ 'y_axis_type': 'bits',
+ 'alias_colors': json.dumps(egress_colors) if is_total else {}
+ }))
+
+ return result
+
+
+def get_aggregate_dashboard_data(title, targets, datasource, tag):
+ id_gen = num_generator()
+ gridPos = gridPos_generator(id_gen, agg=True)
+
+ panels = []
+ all_targets = targets.get('EVERYSINGLEPANEL', [])
+
+ ingress, egress = create_aggregate_panel(
+ title, gridPos, all_targets, datasource)
+ panels.extend([ingress, egress])
+
+ totals_title = title + ' - Totals'
+ t_in, t_eg = create_aggregate_panel(
+ totals_title, gridPos, all_targets, datasource)
+ panels.extend([t_in, t_eg])
+
+ if 'EVERYSINGLEPANEL' in targets:
+ del targets['EVERYSINGLEPANEL']
+
+ for target in targets:
+ _in, _out = create_aggregate_panel(
+ title + f' - {target}', gridPos, targets[target], datasource)
+ panels.extend([_in, _out])
+
+ return {
+ 'title': title,
+ 'datasource': datasource,
+ 'panels': panels,
+ 'tag': tag
+ }
diff --git a/brian_dashboard_manager/templating/nren_access.py b/brian_dashboard_manager/templating/nren_access.py
index 1f0a8194ca74f124f730342d978d732a2448526a..bf258ea240618a9127d8d184a8808ac9162fa27d 100644
--- a/brian_dashboard_manager/templating/nren_access.py
+++ b/brian_dashboard_manager/templating/nren_access.py
@@ -2,11 +2,10 @@ import json
import os
import jinja2
from concurrent.futures import ProcessPoolExecutor
-from brian_dashboard_manager.templating.render import create_dropdown_panel, \
- create_panel_target
+from brian_dashboard_manager.templating.render import create_dropdown_panel
from brian_dashboard_manager.templating.helpers import \
is_aggregate_interface, is_logical_interface, is_physical_interface, \
- num_generator, gridPos_generator, letter_generator, \
+ num_generator, gridPos_generator, get_aggregate_targets, \
get_panel_fields
@@ -66,37 +65,6 @@ id_gen = num_generator(start=3)
gridPos = gridPos_generator(id_gen, start=1)
-# Aggregate panels have unique targets,
-# handle those here.
-def get_aggregate_targets(aggregates):
- ingress = []
- egress = []
-
- # used to generate refIds
- letters = letter_generator()
-
- for target in aggregates:
- ref_id = next(letters)
- in_data = {
- **target,
- 'alias': f"{target['alias']} - Ingress Traffic",
- 'refId': ref_id,
- 'select_field': 'ingress'
- }
- out_data = {
- **target,
- 'alias': f"{target['alias']} - Egress Traffic",
- 'refId': ref_id,
- 'select_field': 'egress'
- }
- ingress_target = create_panel_target(in_data)
- egress_target = create_panel_target(out_data)
- ingress.append(ingress_target)
- egress.append(egress_target)
-
- return ingress, egress
-
-
def get_panel_definitions(panels, datasource, errors=False):
result = []
for panel in panels:
diff --git a/brian_dashboard_manager/templating/render.py b/brian_dashboard_manager/templating/render.py
index cf263decc3f2facc483164588158a874ebd52eda..dede30c87a9d9869551a1680e183e73a2060d1e4 100644
--- a/brian_dashboard_manager/templating/render.py
+++ b/brian_dashboard_manager/templating/render.py
@@ -46,7 +46,7 @@ def create_panel(data):
with open(file) as f:
template = jinja2.Template(f.read())
yaxes = create_yaxes(data.get('y_axis_type', 'bits'))
- targets = []
+ targets = data.get('targets', [])
for target in data.get('panel_targets', []):
targets.append(create_panel_target(target))
return template.render({**data, 'yaxes': yaxes, 'targets': targets})
diff --git a/brian_dashboard_manager/templating/templates/shared/panel.json.j2 b/brian_dashboard_manager/templating/templates/shared/panel.json.j2
index f3e2389ffbf47896e1bb04afa3a3d5c806964098..005e069176752e00082ad58c14348d5edd185b09 100644
--- a/brian_dashboard_manager/templating/templates/shared/panel.json.j2
+++ b/brian_dashboard_manager/templating/templates/shared/panel.json.j2
@@ -1,10 +1,15 @@
-{
+{
+ {% if alias_colors %}
+ "aliasColors": {{ alias_colors }},
+ {% else %}
"aliasColors": {},
+ {% endif %}
"bars": false,
"collapsed": null,
"dashLength": 10,
"dashes": false,
"datasource": "{{ datasource }}",
+ "decimals": 2,
"fieldConfig": {
"defaults": {
"custom": {}
@@ -12,15 +17,20 @@
"overrides": []
},
"fill": 1,
- "fillGradient": 5,
+ "fillGradient": 10,
"gridPos": {
"h": {{ height }},
"w": {{ width }},
+ {% if x %}
+ "x": {{ x }},
+ {% else %}
"x": 0,
+ {% endif %}
"y": {{ y }}
},
"hiddenSeries": false,
"id": {{ id }},
+ {% if not disable_legend %}
"legend": {
"alignAsTable": true,
"avg": true,
@@ -32,8 +42,9 @@
"total": false,
"values": true
},
+ {% endif %}
"lines": true,
- "linewidth": 1,
+ "linewidth": {{ linewidth }},
"nullPointMode": "null",
"options": {
"alertThreshold": true
@@ -45,7 +56,11 @@
"search": null,
"seriesOverrides": [],
"spaceLength": 10,
- "stack": null,
+ {% if stack %}
+ "stack": true,
+ {% else %}
+ "stack": false,
+ {% endif %}
"steppedLine": false,
"tags": null,
"thresholds": [],
diff --git a/changelog.md b/changelog.md
index dfb7dd9d0ab9c8015752ccbd89f42350c6ffc50f..be7cc37267d0b6c649279409a661a15be5800f05 100644
--- a/changelog.md
+++ b/changelog.md
@@ -2,6 +2,9 @@
All notable changes to this project will be documented in this file.
+## [0.5] - 2021-03-10
+- Added provisioning of aggregate dashboards with many targets
+
## [0.4] - 2021-03-03
- Provisioning is now limited to a single org at once.
diff --git a/setup.py b/setup.py
index 297888f8d6a7cc8c9a5567591ea8146306314cee..a3584f43e836f93f6269a0714ee08f177725d994 100644
--- a/setup.py
+++ b/setup.py
@@ -2,7 +2,7 @@ from setuptools import setup, find_packages
setup(
name='brian-dashboard-manager',
- version="0.4",
+ version="0.5",
author='GEANT',
author_email='swd@geant.org',
description='',
diff --git a/test/test_aggregrate.py b/test/test_aggregrate.py
new file mode 100644
index 0000000000000000000000000000000000000000..ac0df399fbcda6c2997770980f2df137db06261a
--- /dev/null
+++ b/test/test_aggregrate.py
@@ -0,0 +1,177 @@
+from brian_dashboard_manager.grafana.utils.request import TokenRequest
+import responses
+from brian_dashboard_manager.grafana.provision import provision_aggregate, \
+ is_cls_peer
+
+DEFAULT_REQUEST_HEADERS = {
+ "Content-type": "application/json",
+ "Accept": ["application/json"]
+}
+
+
+TEST_INTERFACES = [
+ {
+ "router": "mx1.ath2.gr.geant.net",
+ "name": "xe-1/0/1",
+ "bundle": [],
+ "bundle-parents": [],
+ "snmp-index": 569,
+ "description": "PHY RESERVED | New OTEGLOBE ATH2-VIE 10Gb LS",
+ "circuits": []
+ },
+ {
+ "router": "mx1.ath2.gr.geant.net",
+ "name": "ge-1/3/7",
+ "bundle": [],
+ "bundle-parents": [],
+ "snmp-index": 543,
+ "description": "PHY SPARE",
+ "circuits": []
+ },
+ {
+ "router": "mx1.ham.de.geant.net",
+ "name": "xe-2/2/0.13",
+ "bundle": [],
+ "bundle-parents": [],
+ "snmp-index": 721,
+ "description": "SRV_L2CIRCUIT CUSTOMER WP6T3 WP6T3 #ham_lon2-WP6-GTS_20063 |", # noqa: E501
+ "circuits": [
+ {
+ "id": 52382,
+ "name": "ham_lon2-WP6-GTS_20063_L2c",
+ "type": "",
+ "status": "operational"
+ }
+ ]
+ },
+ {
+ "router": "mx1.fra.de.geant.net",
+ "name": "ae27",
+ "bundle": [],
+ "bundle-parents": [
+ "xe-10/0/2",
+ "xe-10/3/2",
+ "xe-10/3/3"
+ ],
+ "snmp-index": 760,
+ "description": "LAG CUSTOMER ULAKBIM SRF9940983 |",
+ "circuits": [
+ {
+ "id": 40983,
+ "name": "ULAKBIM AP2 LAG",
+ "type": "",
+ "status": "operational"
+ }
+ ]
+ },
+ {
+ "router": "mx2.zag.hr.geant.net",
+ "name": "xe-2/1/0",
+ "bundle": [],
+ "bundle-parents": [],
+ "snmp-index": 739,
+ "description": "PHY SPARE",
+ "circuits": []
+ },
+ {
+ "router": "rt1.rig.lv.geant.net",
+ "name": "xe-0/1/5",
+ "bundle": [],
+ "bundle-parents": [],
+ "snmp-index": 539,
+ "description": "PHY SPARE",
+ "circuits": []
+ },
+ {
+ "router": "srx1.ch.office.geant.net",
+ "name": "ge-0/0/0",
+ "bundle": [],
+ "bundle-parents": [],
+ "snmp-index": 513,
+ "description": "Reserved for GEANT OC to test Virgin Media link",
+ "circuits": []
+ },
+ {
+ "router": "mx1.par.fr.geant.net",
+ "name": "xe-4/1/4.1",
+ "bundle": [],
+ "bundle-parents": [],
+ "snmp-index": 1516,
+ "description": "SRV_L2CIRCUIT INFRASTRUCTURE JRA1 JRA1 | #SDX-L2_PILOT-Br52 OF-P3_par ", # noqa: E501
+ "circuits": []
+ },
+ {
+ "router": "mx1.lon.uk.geant.net",
+ "name": "lt-1/3/0.61",
+ "bundle": [],
+ "bundle-parents": [],
+ "snmp-index": 1229,
+ "description": "SRV_IAS INFRASTRUCTURE ACCESS GLOBAL #LON-IAS-RE-Peering | BGP Peering - IAS Side", # noqa: E501
+ "circuits": []
+ },
+ {
+ "router": "mx1.sof.bg.geant.net",
+ "name": "xe-2/0/5",
+ "bundle": [],
+ "bundle-parents": [],
+ "snmp-index": 694,
+ "description": "PHY RESERVED | Prime Telecom Sofia-Bucharest 3_4",
+ "circuits": []
+ },
+ {
+ "router": "mx1.sof.bg.geant.net",
+ "name": "xe-2/0/5",
+ "bundle": [],
+ "bundle-parents": [],
+ "snmp-index": 694,
+ "description": "SRV_GLOBAL CUSTOMER HEANET TESTDESCRIPTION |",
+ "circuits": []
+ }
+]
+
+
+def generate_folder(data):
+ return {
+ "id": 555,
+ "uid": data['uid'],
+ "title": data['title'],
+ "url": f"/dashboards/f/{data['uid']}/{data['title'].lower()}",
+ "hasAcl": False,
+ "canSave": True,
+ "canEdit": True,
+ "canAdmin": True,
+ "createdBy": "Anonymous",
+ "created": "2021-02-23T15:33:46Z",
+ "updatedBy": "Anonymous",
+ "updated": "2021-02-23T15:33:46Z",
+ "version": 1
+ }
+
+
+@responses.activate
+def test_provision_aggregate(data_config, mocker, client):
+
+ TEST_DATASOURCE = [{
+ "name": "brian-influx-datasource",
+ "type": "influxdb",
+ "access": "proxy",
+ "url": "http://test-brian-datasource.geant.org:8086",
+ "database": "test-db",
+ "basicAuth": False,
+ "isDefault": True,
+ "readOnly": False
+ }]
+
+ _mocked_create_dashboard = mocker.patch(
+ 'brian_dashboard_manager.grafana.provision.create_dashboard')
+ # we dont care about this, tested separately
+ _mocked_create_dashboard.return_value = None
+
+ request = TokenRequest(**data_config, token='test')
+ fake_folder = generate_folder({'uid': 'aggtest', 'title': 'aggtest'})
+ dash = {
+ 'predicate': is_cls_peer,
+ 'tag': 'cls_peers',
+ }
+ provision_aggregate(request, 'MY FAKE PEERS', fake_folder,
+ dash, TEST_INTERFACES, TEST_DATASOURCE[0]['name'])