Merge branch 'feature/aggregate-dashboards' into 'develop'

Feature/aggregate dashboards

See merge request live-projects/brian-dashboard-manager!4

brian_dashboard_manager/grafana/provision.py

@@ -18,11 +18,12 @@ from brian_dashboard_manager.inventory_provider.interfaces import \
 from brian_dashboard_manager.templating.nren_access import generate_nrens
 
 from brian_dashboard_manager.templating.helpers import is_re_customer, \
-    is_cls, is_ias_customer, is_ias_private, is_ias_public, is_ias_upstream, \
-    is_lag_backbone, is_nren, is_phy_upstream, is_re_peer, is_gcs, \
-    is_geantopen, is_l2circuit, is_lhcone_peer, is_lhcone_customer, is_mdvpn,\
+    is_cls_peer, is_cls, is_ias_customer, is_ias_private, is_ias_public, \
+    is_ias_upstream, is_ias_peer, is_lag_backbone, is_nren, is_phy_upstream, \
+    is_re_peer, is_gcs, is_geantopen, is_l2circuit, is_lhcone_peer, \
+    is_lhcone_customer, is_lhcone, is_mdvpn, get_aggregate_dashboard_data, \
     get_interface_data, parse_backbone_name, parse_phy_upstream_name, \
-    get_dashboard_data
+    get_dashboard_data, get_aggregate_interface_data
 
 from brian_dashboard_manager.templating.render import render_dashboard
 
@@ -61,6 +62,21 @@ def provision_folder(token_request, folder_name,
                             rendered, folder['id'])
 
 
+def provision_aggregate(token_request, agg_type, aggregate_folder,
+                        dash, excluded_interfaces, datasource_name):
+    predicate = dash['predicate']
+    tag = dash['tag']
+
+    relevant_interfaces = filter(predicate, excluded_interfaces)
+    data = get_aggregate_interface_data(relevant_interfaces, agg_type)
+
+    dashboard = get_aggregate_dashboard_data(
+        f'Aggregate - {agg_type}', data, datasource_name, tag)
+
+    rendered = render_dashboard(dashboard)
+    create_dashboard(token_request, rendered, aggregate_folder['id'])
+
+
 def provision(config):
 
     request = AdminRequest(**config)
@@ -200,6 +216,39 @@ def provision(config):
                                 folder_name, dash,
                                 excluded_interfaces, datasource_name)
 
+        aggregate_dashboards = {
+            'CLS PEERS': {
+                'predicate': is_cls_peer,
+                'tag': 'cls_peers',
+            },
+            'IAS PEERS': {
+                'predicate': is_ias_peer,
+                'tag': 'ias_peers',
+            },
+            'GWS UPSTREAMS': {
+                'predicate': is_ias_upstream,
+                'tag': 'gws_upstreams',
+            },
+            'LHCONE': {
+                'predicate': is_lhcone,
+                'tag': 'lhcone',
+            },
+            # 'CAE1': {
+            #     'predicate': is_cae1,
+            #     'tag': 'cae',
+            # }
+        }
+
+        with ProcessPoolExecutor(max_workers=4) as executor:
+            aggregate_folder = find_folder(token_request, 'Aggregates')
+            for agg_type, dash in aggregate_dashboards.items():
+                logger.info(
+                    f'Provisioning {org["name"]}' +
+                    f'/Aggregate {agg_type} dashboards')
+                executor.submit(provision_aggregate, token_request, agg_type,
+                                aggregate_folder, dash,
+                                excluded_interfaces, datasource_name)
+
         # NREN Access dashboards
         # uses a different template than the above.
         logger.info('Provisioning NREN Access dashboards')

brian_dashboard_manager/templating/helpers.py

 import re
 import logging
+import json
 from itertools import product
+from functools import reduce
 from string import ascii_uppercase
-from brian_dashboard_manager.templating.render import create_panel
+from brian_dashboard_manager.templating.render import create_panel, \
+    create_panel_target
 
 PANEL_HEIGHT = 12
 PANEL_WIDTH = 24
@@ -35,6 +38,10 @@ def is_cls(interface):
     return 'SRV_CLS' in get_description(interface)
 
 
+def is_cls_peer(interface):
+    return 'SRV_CLS PRIVATE' in get_description(interface)
+
+
 def is_ias_public(interface):
     return 'SRV_IAS PUBLIC' in get_description(interface)
 
@@ -51,6 +58,10 @@ def is_ias_upstream(interface):
     return 'SRV_IAS UPSTREAM' in get_description(interface)
 
 
+def is_ias_peer(interface):
+    return is_ias_public(interface) or is_ias_private(interface)
+
+
 def is_re_peer(interface):
     return 'SRV_GLOBAL RE_INTERCONNECT' in get_description(interface)
 
@@ -82,6 +93,11 @@ def is_lhcone_customer(interface):
     return 'LHCONE' in description and 'SRV_L3VPN CUSTOMER' in description
 
 
+def is_lhcone(interface):
+    regex = 'SRV_L3VPN (CUSTOMER|RE_INTERCONNECT)'
+    return re.match(regex, get_description(interface))
+
+
 def is_mdvpn(interface):
     return re.match('^SRV_MDVPN CUSTOMER', get_description(interface))
 
@@ -96,6 +112,10 @@ def is_lag_backbone(interface):
     return is_infrastructure_backbone(interface) and is_lag
 
 
+def is_cae1(interface):
+    return interface.get('router', '').lower() == 'mx1.lon.uk.geant.net'
+
+
 def parse_backbone_name(description, *args, **kwargs):
     link = description.split('|')[1].strip()
     link = link.replace('( ', '(')
@@ -119,16 +139,24 @@ def num_generator(start=1):
         num += 1
 
 
-def gridPos_generator(id_generator, start=0):
+def gridPos_generator(id_generator, start=0, agg=False):
     num = start
     while True:
         yield {
             "height": PANEL_HEIGHT,
-            "width": PANEL_WIDTH,
+            "width": PANEL_WIDTH if not agg else PANEL_WIDTH // 2,
             "x": 0,
             "y": num * PANEL_HEIGHT,
             "id": next(id_generator)
         }
+        if agg:
+            yield {
+                "height": PANEL_HEIGHT,
+                "width": PANEL_WIDTH // 2,
+                "x": PANEL_WIDTH // 2,
+                "y": num * PANEL_HEIGHT,
+                "id": next(id_generator)
+            }
         num += 1
 
 
@@ -155,12 +183,14 @@ def letter_generator():
 # parse_func receives interface information and returns a peer name.
 def get_interface_data(interfaces, name_parse_func=None):
     result = {}
+
+    if not name_parse_func:
+        # Most (but not all) descriptions use a format
+        # which has the peer name as the third element.
+        def name_parse_func(desc, *args, **kwargs):
+            return desc.split(' ')[2].upper()
+
     for interface in interfaces:
-        if not name_parse_func:
-            # Most (but not all) descriptions use a format
-            # which has the peer name as the third element.
-            def name_parse_func(desc, *args, **kwargs):
-                return desc.split(' ')[2].upper()
 
         description = interface.get('description', '').strip()
         interface_name = interface.get('name')
@@ -182,6 +212,67 @@ def get_interface_data(interfaces, name_parse_func=None):
     return result
 
 
+def get_aggregate_interface_data(interfaces, agg_type):
+    result = []
+
+    def reduce_func(prev, curr):
+        remotes = prev.get(curr['remote'], [])
+        remotes.append(curr)
+        all_agg = prev.get('EVERYSINGLEPANEL', [])
+        all_agg.append(curr)
+        prev[curr['remote']] = remotes
+        prev['EVERYSINGLEPANEL'] = all_agg
+        return prev
+
+    for interface in interfaces:
+
+        description = interface.get('description', '').strip()
+        interface_name = interface.get('name')
+        host = interface.get('router', '')
+
+        remote = description.split(' ')[2].upper()
+
+        result.append({
+            'type': agg_type,
+            'interface': interface_name,
+            'hostname': host,
+            'remote': remote,
+            'alias': f"{host.split('.')[1].upper()} - {remote}"
+        })
+    return reduce(reduce_func, result, {})
+
+
+# Helper used for generating stacked aggregate panels
+# with multiple target fields (ingress/egress)
+def get_aggregate_targets(targets):
+    ingress = []
+    egress = []
+
+    # used to generate refIds
+    letters = letter_generator()
+
+    for target in targets:
+        ref_id = next(letters)
+        in_data = {
+            **target,
+            'alias': f"{target['alias']} - Ingress Traffic",
+            'refId': ref_id,
+            'select_field': 'ingress'
+        }
+        out_data = {
+            **target,
+            'alias': f"{target['alias']} - Egress Traffic",
+            'refId': ref_id,
+            'select_field': 'egress'
+        }
+        ingress_target = create_panel_target(in_data)
+        egress_target = create_panel_target(out_data)
+        ingress.append(ingress_target)
+        egress.append(egress_target)
+
+    return ingress, egress
+
+
 # Helper used for generating all traffic/error panels
 # with a single target field (ingress/egress or err/s)
 def get_panel_fields(panel, panel_type, datasource):
@@ -214,6 +305,7 @@ def get_panel_fields(panel, panel_type, datasource):
     return create_panel({
         **panel,
         'datasource': datasource,
+        'linewidth': 1,
         'title': panel['title'].format(panel_type),
         'panel_targets': [get_target_data(*target) for target in targets],
         'y_axis_type': 'errors' if is_error else 'bits',
@@ -244,3 +336,82 @@ def get_dashboard_data(data, datasource, tag, errors=False):
             'panels': get_panel_definitions(panels, datasource),
             'tag': tag
         }
+
+
+def create_aggregate_panel(title, gridpos, targets, datasource):
+
+    ingress_targets, egress_targets = get_aggregate_targets(targets)
+    result = []
+
+    ingress_pos = next(gridpos)
+    egress_pos = next(gridpos)
+
+    is_total = 'totals' in title.lower()
+
+    def reduce_alias(prev, curr):
+        d = json.loads(curr)
+        alias = d['alias']
+        if 'egress' in alias.lower():
+            prev[alias] = '#0000FF'
+        else:
+            prev[alias] = '#00FF00'
+        return prev
+
+    ingress_colors = reduce(reduce_alias, ingress_targets, {})
+    egress_colors = reduce(reduce_alias, egress_targets, {})
+
+    result.append(create_panel({
+        **ingress_pos,
+        'stack': True,
+        'linewidth': 0 if is_total else 1,
+        'datasource': datasource,
+        'title': title + ' - ingress',
+        'targets': ingress_targets,
+        'y_axis_type': 'bits',
+        'alias_colors': json.dumps(ingress_colors) if is_total else {}
+    }))
+
+    result.append(create_panel({
+        **egress_pos,
+        'stack': True,
+        'linewidth': 0 if is_total else 1,
+        'datasource': datasource,
+        'title': title + ' - egress',
+        'targets': egress_targets,
+        'y_axis_type': 'bits',
+        'alias_colors': json.dumps(egress_colors) if is_total else {}
+    }))
+
+    return result
+
+
+def get_aggregate_dashboard_data(title, targets, datasource, tag):
+    id_gen = num_generator()
+    gridPos = gridPos_generator(id_gen, agg=True)
+
+    panels = []
+    all_targets = targets.get('EVERYSINGLEPANEL', [])
+
+    ingress, egress = create_aggregate_panel(
+        title, gridPos, all_targets, datasource)
+    panels.extend([ingress, egress])
+
+    totals_title = title + ' - Totals'
+    t_in, t_eg = create_aggregate_panel(
+        totals_title, gridPos, all_targets, datasource)
+    panels.extend([t_in, t_eg])
+
+    if 'EVERYSINGLEPANEL' in targets:
+        del targets['EVERYSINGLEPANEL']
+
+    for target in targets:
+        _in, _out = create_aggregate_panel(
+            title + f' - {target}', gridPos, targets[target], datasource)
+        panels.extend([_in, _out])
+
+    return {
+        'title': title,
+        'datasource': datasource,
+        'panels': panels,
+        'tag': tag
+    }

brian_dashboard_manager/templating/nren_access.py

@@ -2,11 +2,10 @@ import json
 import os
 import jinja2
 from concurrent.futures import ProcessPoolExecutor
-from brian_dashboard_manager.templating.render import create_dropdown_panel, \
-    create_panel_target
+from brian_dashboard_manager.templating.render import create_dropdown_panel
 from brian_dashboard_manager.templating.helpers import \
     is_aggregate_interface, is_logical_interface, is_physical_interface, \
-    num_generator, gridPos_generator, letter_generator, \
+    num_generator, gridPos_generator, get_aggregate_targets, \
     get_panel_fields
 
 
@@ -66,37 +65,6 @@ id_gen = num_generator(start=3)
 gridPos = gridPos_generator(id_gen, start=1)
 
 
-# Aggregate panels have unique targets,
-# handle those here.
-def get_aggregate_targets(aggregates):
-    ingress = []
-    egress = []
-
-    # used to generate refIds
-    letters = letter_generator()
-
-    for target in aggregates:
-        ref_id = next(letters)
-        in_data = {
-            **target,
-            'alias': f"{target['alias']} - Ingress Traffic",
-            'refId': ref_id,
-            'select_field': 'ingress'
-        }
-        out_data = {
-            **target,
-            'alias': f"{target['alias']} - Egress Traffic",
-            'refId': ref_id,
-            'select_field': 'egress'
-        }
-        ingress_target = create_panel_target(in_data)
-        egress_target = create_panel_target(out_data)
-        ingress.append(ingress_target)
-        egress.append(egress_target)
-
-    return ingress, egress
-
-
 def get_panel_definitions(panels, datasource, errors=False):
     result = []
     for panel in panels:

brian_dashboard_manager/templating/render.py

@@ -46,7 +46,7 @@ def create_panel(data):
     with open(file) as f:
         template = jinja2.Template(f.read())
     yaxes = create_yaxes(data.get('y_axis_type', 'bits'))
-    targets = []
+    targets = data.get('targets', [])
     for target in data.get('panel_targets', []):
         targets.append(create_panel_target(target))
     return template.render({**data, 'yaxes': yaxes, 'targets': targets})

brian_dashboard_manager/templating/templates/shared/panel.json.j2

-{
+{   
+    {% if alias_colors %}
+    "aliasColors": {{ alias_colors }},
+    {% else %}
     "aliasColors": {},
+    {% endif %}
     "bars": false,
     "collapsed": null,
     "dashLength": 10,
     "dashes": false,
     "datasource": "{{ datasource }}",
+    "decimals": 2,
     "fieldConfig": {
         "defaults": {
             "custom": {}
@@ -12,15 +17,20 @@
         "overrides": []
     },
     "fill": 1,
-    "fillGradient": 5,
+    "fillGradient": 10,
     "gridPos": {
         "h": {{ height }},
         "w": {{ width }},
+        {% if x %}
+        "x": {{ x }},
+        {% else %}
         "x": 0,
+        {% endif %}
         "y": {{ y }}
     },
     "hiddenSeries": false,
     "id": {{ id }},
+    {% if not disable_legend %}
     "legend": {
         "alignAsTable": true,
         "avg": true,
@@ -32,8 +42,9 @@
         "total": false,
         "values": true
     },
+    {% endif %}
     "lines": true,
-    "linewidth": 1,
+    "linewidth": {{ linewidth }},
     "nullPointMode": "null",
     "options": {
         "alertThreshold": true
@@ -45,7 +56,11 @@
     "search": null,
     "seriesOverrides": [],
     "spaceLength": 10,
-    "stack": null,
+    {% if stack %}
+    "stack": true,
+    {% else %}
+    "stack": false,
+    {% endif %}
     "steppedLine": false,
     "tags": null,
     "thresholds": [],

test/test_aggregrate.py

+from brian_dashboard_manager.grafana.utils.request import TokenRequest
+import responses
+from brian_dashboard_manager.grafana.provision import provision_aggregate, \
+    is_cls_peer
+
+DEFAULT_REQUEST_HEADERS = {
+    "Content-type": "application/json",
+    "Accept": ["application/json"]
+}
+
+
+TEST_INTERFACES = [
+    {
+        "router": "mx1.ath2.gr.geant.net",
+        "name": "xe-1/0/1",
+        "bundle": [],
+        "bundle-parents": [],
+        "snmp-index": 569,
+        "description": "PHY RESERVED | New OTEGLOBE ATH2-VIE 10Gb LS",
+        "circuits": []
+    },
+    {
+        "router": "mx1.ath2.gr.geant.net",
+        "name": "ge-1/3/7",
+        "bundle": [],
+        "bundle-parents": [],
+        "snmp-index": 543,
+        "description": "PHY SPARE",
+        "circuits": []
+    },
+    {
+        "router": "mx1.ham.de.geant.net",
+        "name": "xe-2/2/0.13",
+        "bundle": [],
+        "bundle-parents": [],
+        "snmp-index": 721,
+        "description": "SRV_L2CIRCUIT CUSTOMER WP6T3 WP6T3 #ham_lon2-WP6-GTS_20063 |",  # noqa: E501
+        "circuits": [
+            {
+                "id": 52382,
+                "name": "ham_lon2-WP6-GTS_20063_L2c",
+                "type": "",
+                "status": "operational"
+            }
+        ]
+    },
+    {
+        "router": "mx1.fra.de.geant.net",
+        "name": "ae27",
+        "bundle": [],
+        "bundle-parents": [
+            "xe-10/0/2",
+            "xe-10/3/2",
+            "xe-10/3/3"
+        ],
+        "snmp-index": 760,
+        "description": "LAG CUSTOMER ULAKBIM SRF9940983 |",
+        "circuits": [
+            {
+                "id": 40983,
+                "name": "ULAKBIM AP2 LAG",
+                "type": "",
+                "status": "operational"
+            }
+        ]
+    },
+    {
+        "router": "mx2.zag.hr.geant.net",
+        "name": "xe-2/1/0",
+        "bundle": [],
+        "bundle-parents": [],
+        "snmp-index": 739,
+        "description": "PHY SPARE",
+        "circuits": []
+    },
+    {
+        "router": "rt1.rig.lv.geant.net",
+        "name": "xe-0/1/5",
+        "bundle": [],
+        "bundle-parents": [],
+        "snmp-index": 539,
+        "description": "PHY SPARE",
+        "circuits": []
+    },
+    {
+        "router": "srx1.ch.office.geant.net",
+        "name": "ge-0/0/0",
+        "bundle": [],
+        "bundle-parents": [],
+        "snmp-index": 513,
+        "description": "Reserved for GEANT OC to test Virgin Media link",
+        "circuits": []
+    },
+    {
+        "router": "mx1.par.fr.geant.net",
+        "name": "xe-4/1/4.1",
+        "bundle": [],
+        "bundle-parents": [],
+        "snmp-index": 1516,
+        "description": "SRV_L2CIRCUIT INFRASTRUCTURE JRA1 JRA1 | #SDX-L2_PILOT-Br52 OF-P3_par   ",  # noqa: E501
+        "circuits": []
+    },
+    {
+        "router": "mx1.lon.uk.geant.net",
+        "name": "lt-1/3/0.61",
+        "bundle": [],
+        "bundle-parents": [],
+        "snmp-index": 1229,
+        "description": "SRV_IAS INFRASTRUCTURE ACCESS GLOBAL #LON-IAS-RE-Peering | BGP Peering - IAS Side",  # noqa: E501
+        "circuits": []
+    },
+    {
+        "router": "mx1.sof.bg.geant.net",
+        "name": "xe-2/0/5",
+        "bundle": [],
+        "bundle-parents": [],
+        "snmp-index": 694,
+        "description": "PHY RESERVED | Prime Telecom Sofia-Bucharest 3_4",
+        "circuits": []
+    },
+    {
+        "router": "mx1.sof.bg.geant.net",
+        "name": "xe-2/0/5",
+        "bundle": [],
+        "bundle-parents": [],
+        "snmp-index": 694,
+        "description": "SRV_GLOBAL CUSTOMER HEANET TESTDESCRIPTION |",
+        "circuits": []
+    }
+]
+
+
+def generate_folder(data):
+    return {
+        "id": 555,
+        "uid": data['uid'],
+        "title": data['title'],
+        "url": f"/dashboards/f/{data['uid']}/{data['title'].lower()}",
+        "hasAcl": False,
+        "canSave": True,
+        "canEdit": True,
+        "canAdmin": True,
+        "createdBy": "Anonymous",
+        "created": "2021-02-23T15:33:46Z",
+        "updatedBy": "Anonymous",
+        "updated": "2021-02-23T15:33:46Z",
+        "version": 1
+    }
+
+
+@responses.activate
+def test_provision_aggregate(data_config, mocker, client):
+
+    TEST_DATASOURCE = [{
+        "name": "brian-influx-datasource",
+        "type": "influxdb",
+        "access": "proxy",
+        "url": "http://test-brian-datasource.geant.org:8086",
+        "database": "test-db",
+        "basicAuth": False,
+        "isDefault": True,
+        "readOnly": False
+    }]
+
+    _mocked_create_dashboard = mocker.patch(
+        'brian_dashboard_manager.grafana.provision.create_dashboard')
+    # we dont care about this, tested separately
+    _mocked_create_dashboard.return_value = None
+
+    request = TokenRequest(**data_config, token='test')
+    fake_folder = generate_folder({'uid': 'aggtest', 'title': 'aggtest'})
+    dash = {
+        'predicate': is_cls_peer,
+        'tag': 'cls_peers',
+    }
+    provision_aggregate(request, 'MY FAKE PEERS', fake_folder,
+                        dash, TEST_INTERFACES, TEST_DATASOURCE[0]['name'])