diff --git a/brian_dashboard_manager/templating/__init__.py b/brian_dashboard_manager/templating/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/brian_dashboard_manager/templating/helpers.py b/brian_dashboard_manager/templating/helpers.py new file mode 100644 index 0000000000000000000000000000000000000000..f808f8be95c4d177f3b152ade487ae8800935f55 --- /dev/null +++ b/brian_dashboard_manager/templating/helpers.py @@ -0,0 +1,242 @@ +import re +from itertools import product +from string import ascii_uppercase +from brian_dashboard_manager.templating.render import create_panel + +PANEL_HEIGHT = 12 +PANEL_WIDTH = 24 + + +def get_description(interface): + return interface.get('description', '').strip() + + +def is_physical_interface(interface): + return re.match('^PHY', get_description(interface)) + + +def is_aggregate_interface(interface): + return re.match('^LAG', get_description(interface)) + + +def is_logical_interface(interface): + return re.match('^SRV_', get_description(interface)) + + +def is_nren(interface): + regex = '(PHY|LAG|(SRV_(GLOBAL|LHCONE|MDVPN|IAS|CLS|L3VPN))) CUSTOMER' + return re.match(regex, get_description(interface)) + + +def is_cls(interface): + return 'SRV_CLS' in get_description(interface) + + +def is_ias_public(interface): + return 'SRV_IAS PUBLIC' in get_description(interface) + + +def is_ias_private(interface): + return 'SRV_IAS PRIVATE' in get_description(interface) + + +def is_ias_customer(interface): + return 'SRV_IAS CUSTOMER' in get_description(interface) + + +def is_ias_upstream(interface): + return 'SRV_IAS UPSTREAM' in get_description(interface) + + +def is_re_peer(interface): + return 'SRV_GLOBAL RE_INTERCONNECT' in get_description(interface) + + +def is_re_customer(interface): + regex = '(PHY|LAG|SRV_GLOBAL) CUSTOMER' + return re.match(regex, get_description(interface)) + + +def is_gcs(interface): + return re.match('^SRV_GCS', get_description(interface)) + + +def is_geantopen(interface): + return 'GEANTOPEN' in get_description(interface) + + +def is_l2circuit(interface): + return 'SRV_L2CIRCUIT' in get_description(interface) + + +def is_lhcone_peer(interface): + description = get_description(interface) + return 'LHCONE' in description and 'SRV_L3VPN RE' in description + + +def is_lhcone_customer(interface): + description = get_description(interface) + return 'LHCONE' in description and 'SRV_L3VPN CUSTOMER' in description + + +def is_mdvpn(interface): + return re.match('^SRV_MDVPN CUSTOMER', get_description(interface)) + + +def is_infrastructure_backbone(interface): + regex = '(SRV_GLOBAL|LAG|PHY) INFRASTRUCTURE BACKBONE' + return re.match(regex, get_description(interface)) + + +def is_lag_backbone(interface): + is_lag = 'LAG' in get_description(interface) + return is_infrastructure_backbone(interface) and is_lag + + +def parse_backbone_name(description, *args, **kwargs): + link = description.split('|')[1].strip() + link = link.replace('( ', '(') + return link + + +def is_phy_upstream(interface): + return re.match('^PHY UPSTREAM', get_description(interface)) + + +def parse_phy_upstream_name(description, host): + name = description.split(' ')[2].strip().upper() + location = host.split('.')[1].upper() + return f'{name} - {location}' + + +def num_generator(start=1): + num = start + while True: + yield num + num += 1 + + +def gridPos_generator(id_generator, start=0): + num = start + while True: + yield { + "height": PANEL_HEIGHT, + "width": PANEL_WIDTH, + "x": 0, + "y": num * PANEL_HEIGHT, + "id": next(id_generator) + } + num += 1 + + +def letter_generator(): + i = 0 + j = 0 + num_letters = len(ascii_uppercase) + while True: + result = ascii_uppercase[i % num_letters] + + # tack on an extra letter if we are out of them + if (i >= num_letters): + result += ascii_uppercase[j % num_letters] + j += 1 + if (j != 0 and j % num_letters == 0): + i += 1 + else: + i += 1 + + yield result + + +# peer_predicate is a function that is used to filter the interfaces +# parse_func receives interface information and returns a peer name. +def get_interface_data(interfaces, name_parse_func=None): + result = {} + for interface in interfaces: + if not name_parse_func: + # Most (but not all) descriptions use a format + # which has the peer name as the third element. + def name_parse_func(desc, *args, **kwargs): + return desc.split(' ')[2].upper() + + description = interface.get('description', '').strip() + interface_name = interface.get('name') + host = interface.get('router', '') + + dashboard_name = name_parse_func(description, host) + + peer = result.get(dashboard_name, []) + + router = host.replace('.geant.net', '') + panel_title = f"{router} - {{}} - {interface_name} - {description}" + + peer.append({ + 'title': panel_title, + 'interface': interface_name, + 'hostname': host + }) + result[dashboard_name] = peer + return result + + +# Helper used for generating all traffic/error panels +# with a single target field (ingress/egress or err/s) +def get_panel_fields(panel, panel_type, datasource): + letters = letter_generator() + + def get_target_data(alias, field): + return { + **panel, # panel has target hostname and interface + 'alias': alias, + 'refId': next(letters), + 'select_field': field, + 'percentile': 'percentile' in alias.lower(), + } + + error_fields = [('Ingress Errors', 'errorsIn'), + ('Egress Errors', 'errorsOut')] + + ingress = ['Ingress Traffic', 'Ingress 95th Percentile'] + egress = ['Egress Traffic', 'Egress 95th Percentile'] + + is_v6 = panel_type == 'IPv6' + is_error = panel_type == 'errors' + in_field = 'ingressv6' if is_v6 else 'ingress' + out_field = 'egressv6' if is_v6 else 'egress' + + fields = [*product(ingress, [in_field]), *product(egress, [out_field])] + + targets = error_fields if is_error else fields + + return create_panel({ + **panel, + 'datasource': datasource, + 'title': panel['title'].format(panel_type), + 'panel_targets': [get_target_data(*target) for target in targets], + 'y_axis_type': 'errors' if is_error else 'bits', + }) + + +def get_dashboard_data(data, datasource, tag, errors=False): + id_gen = num_generator() + gridPos = gridPos_generator(id_gen) + + def get_panel_definitions(panels, datasource): + result = [] + for panel in panels: + result.append(get_panel_fields( + {**panel, **next(gridPos)}, 'traffic', datasource)) + result.append(get_panel_fields( + {**panel, **next(gridPos)}, 'IPv6', datasource)) + if errors: + result.append(get_panel_fields( + {**panel, **next(gridPos)}, 'errors', datasource)) + return result + + for peer, panels in data.items(): + yield { + 'title': peer, + 'datasource': datasource, + 'panels': get_panel_definitions(panels, datasource), + 'tag': tag + } diff --git a/brian_dashboard_manager/templating/nren_access.py b/brian_dashboard_manager/templating/nren_access.py new file mode 100644 index 0000000000000000000000000000000000000000..4430515addc4c0b3ccb79da89a86038797e69da5 --- /dev/null +++ b/brian_dashboard_manager/templating/nren_access.py @@ -0,0 +1,155 @@ +import json +import os +import jinja2 +from brian_dashboard_manager.templating.render import create_dropdown_panel, \ + create_panel_target +from brian_dashboard_manager.templating.helpers import \ + is_aggregate_interface, is_logical_interface, is_physical_interface, \ + num_generator, gridPos_generator, letter_generator, get_panel_fields + + +def get_nrens(interfaces): + result = {} + for interface in interfaces: + + description = interface.get('description', '').strip() + + nren_name = description.split(' ')[2].upper() + + nren = result.get( + nren_name, {'AGGREGATES': [], 'SERVICES': [], 'PHYSICAL': []}) + + interface_name = interface.get('name') + host = interface.get('router', '') + router = host.replace('.geant.net', '') + panel_title = f"{router} - {{}} - {interface_name} - {description}" + + if is_aggregate_interface(interface): + nren['AGGREGATES'].append({ + 'interface': interface_name, + 'hostname': host, + 'alias': f"{host.split('.')[1].upper()} - {nren_name}" + }) + + # link aggregates are also shown + # under the physical dropdown + nren['PHYSICAL'].append({ + 'title': panel_title, + 'hostname': host, + 'interface': interface_name + }) + + elif is_logical_interface(interface): + nren['SERVICES'].append({ + 'title': panel_title, + 'hostname': host, + 'interface': interface_name + }) + elif is_physical_interface(interface): + nren['PHYSICAL'].append({ + 'title': panel_title, + 'hostname': host, + 'interface': interface_name + }) + + result[nren_name] = nren + return result + + +# start IDs from 3 since aggregate +# panels have hardcoded IDs (1, 2). +id_gen = num_generator(start=3) + +# aggregate panels have y=0, start generating at 1*height +gridPos = gridPos_generator(id_gen, start=1) + + +# Aggregate panels have unique targets, +# handle those here. +def get_aggregate_targets(aggregates): + ingress = [] + egress = [] + + # used to generate refIds + letters = letter_generator() + + for target in aggregates: + ref_id = next(letters) + in_data = { + **target, + 'alias': f"{target['alias']} - Ingress Traffic", + 'refId': ref_id, + 'select_field': 'ingress' + } + out_data = { + **target, + 'alias': f"{target['alias']} - Egress Traffic", + 'refId': ref_id, + 'select_field': 'egress' + } + ingress_target = create_panel_target(in_data) + egress_target = create_panel_target(out_data) + ingress.append(ingress_target) + egress.append(egress_target) + + return ingress, egress + + +def get_panel_definitions(panels, datasource, errors=False): + result = [] + for panel in panels: + result.append(get_panel_fields( + {**panel, **next(gridPos)}, 'traffic', datasource)) + result.append(get_panel_fields( + {**panel, **next(gridPos)}, 'IPv6', datasource)) + if errors: + result.append(get_panel_fields( + {**panel, **next(gridPos)}, 'errors', datasource)) + return result + + +def get_dashboard_data(interfaces, datasource): + + nren_data = get_nrens(interfaces) + for nren, data in nren_data.items(): + + agg_ingress, agg_egress = get_aggregate_targets(data['AGGREGATES']) + services_dropdown = create_dropdown_panel('Services', **next(gridPos)) + service_panels = get_panel_definitions(data['SERVICES'], datasource) + iface_dropdown = create_dropdown_panel('Interfaces', **next(gridPos)) + phys_panels = get_panel_definitions(data['PHYSICAL'], datasource, True) + + yield { + 'nren_name': nren, + 'datasource': datasource, + 'ingress_aggregate_targets': agg_ingress, + 'egress_aggregate_targets': agg_egress, + 'dropdown_groups': [ + { + 'dropdown': services_dropdown, + 'panels': service_panels, + }, + { + 'dropdown': iface_dropdown, + 'panels': phys_panels, + } + ] + } + + +def generate_nrens(interfaces, datasource): + file = os.path.abspath(os.path.join( + os.path.dirname(__file__), + 'templates', + 'nren_access', + 'nren-dashboard.json.j2')) + + with open(file) as f: + template = jinja2.Template(f.read()) + + for dashboard in get_dashboard_data(interfaces, datasource): + rendered = template.render(dashboard) + rendered = json.loads(rendered) + rendered['uid'] = None + rendered['id'] = None + yield rendered diff --git a/brian_dashboard_manager/templating/render.py b/brian_dashboard_manager/templating/render.py new file mode 100644 index 0000000000000000000000000000000000000000..cf263decc3f2facc483164588158a874ebd52eda --- /dev/null +++ b/brian_dashboard_manager/templating/render.py @@ -0,0 +1,68 @@ +import os +import json +import jinja2 + + +def create_dropdown_panel(title, **kwargs): + TEMPLATE_FILENAME = os.path.abspath(os.path.join( + os.path.dirname(__file__), + 'templates', + 'shared', + 'dropdown.json.j2')) + with open(TEMPLATE_FILENAME) as f: + template = jinja2.Template(f.read()) + return template.render({**kwargs, 'title': title}) + + +# wrapper around bits/s and err/s panel labels +def create_yaxes(type): + file = os.path.abspath(os.path.join( + os.path.dirname(__file__), + 'templates', + 'shared', + 'yaxes.json.j2')) + with open(file) as f: + template = jinja2.Template(f.read()) + return template.render({'type': type}) + + +def create_panel_target(data): + file = os.path.abspath(os.path.join( + os.path.dirname(__file__), + 'templates', + 'shared', + 'panel_target.json.j2')) + with open(file) as f: + template = jinja2.Template(f.read()) + return template.render(data) + + +def create_panel(data): + file = os.path.abspath(os.path.join( + os.path.dirname(__file__), + 'templates', + 'shared', + 'panel.json.j2')) + with open(file) as f: + template = jinja2.Template(f.read()) + yaxes = create_yaxes(data.get('y_axis_type', 'bits')) + targets = [] + for target in data.get('panel_targets', []): + targets.append(create_panel_target(target)) + return template.render({**data, 'yaxes': yaxes, 'targets': targets}) + + +def render_dashboard(dashboard): + file = os.path.abspath(os.path.join( + os.path.dirname(__file__), + 'templates', + 'shared', + 'dashboard.json.j2')) + + with open(file) as f: + template = jinja2.Template(f.read()) + rendered = template.render(dashboard) + rendered = json.loads(rendered) + rendered['uid'] = None + rendered['id'] = None + return rendered diff --git a/brian_dashboard_manager/templating/templates/nren_access/nren-dashboard.json.j2 b/brian_dashboard_manager/templating/templates/nren_access/nren-dashboard.json.j2 new file mode 100644 index 0000000000000000000000000000000000000000..bca5b473718ce97f5aa71b771cb6a1c374ad2c59 --- /dev/null +++ b/brian_dashboard_manager/templating/templates/nren_access/nren-dashboard.json.j2 @@ -0,0 +1,237 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": false, + "gnetId": null, + "graphTooltip": 0, + "schemaVersion": 27, + "style": "dark", + "tags": ["customers"], + "templating": { + "list": [] + }, + "time": { + "from": "now-24h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "{{ nren_name }}", + "version": 1, + "links": [], + "panels": [ + { + "aliasColors": {}, + "bars": false, + "collapsed": null, + "dashLength": 10, + "dashes": false, + "datasource": "{{ datasource }}", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 10, + "gridPos": { + "h": 12, + "w": 12, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 1, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "search": null, + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "tags": null, + "targets": [ + {% for target in ingress_aggregate_targets %} + {{ target }}{{ "," if not loop.last }} + {% endfor %} + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Aggregate - {{ nren_name }} - ingress", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": null + }, + "yaxes": [ + { + "format": "bps", + "label": "bits per second", + "logBase": 1, + "max": null, + "min": "", + "show": true + }, + { + "format": "bps", + "label": "bits per second", + "logBase": 1, + "max": null, + "min": "", + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "collapsed": null, + "dashLength": 10, + "dashes": false, + "datasource": "{{ datasource }}", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 10, + "gridPos": { + "h": 12, + "w": 12, + "x": 12, + "y": 0 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "search": null, + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "tags": null, + "targets": [ + {% for target in egress_aggregate_targets %} + {{ target }}{{ "," if not loop.last }} + {% endfor %} + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Aggregate - {{ nren_name }} - egress", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": null + }, + "yaxes": [ + { + "format": "bps", + "label": "bits per second", + "logBase": 1, + "max": null, + "min": "", + "show": true + }, + { + "format": "bps", + "label": "bits per second", + "logBase": 1, + "max": null, + "min": "", + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + {% for group in dropdown_groups %} + {{ group.dropdown }} + {% if group.panels|length > 0 %} + , + {% endif %} + {% for panel in group.panels %} + {{ panel }}{{ "," if not loop.last }} + {% endfor %} + {{ "," if not loop.last }} + {% endfor %} + ] +} \ No newline at end of file diff --git a/brian_dashboard_manager/templating/templates/shared/dashboard.json.j2 b/brian_dashboard_manager/templating/templates/shared/dashboard.json.j2 new file mode 100644 index 0000000000000000000000000000000000000000..3201b6fc060a005e093810a2336acd90eabb340f --- /dev/null +++ b/brian_dashboard_manager/templating/templates/shared/dashboard.json.j2 @@ -0,0 +1,38 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": false, + "gnetId": null, + "graphTooltip": 0, + "schemaVersion": 27, + "style": "dark", + "tags": ["{{ tag }}"], + "templating": { + "list": [] + }, + "time": { + "from": "now-24h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "{{ title }}", + "version": 1, + "links": [], + "panels": [ + {% for panel in panels %} + {{ panel }}{{ "," if not loop.last }} + {% endfor %} + ] +} \ No newline at end of file diff --git a/brian_dashboard_manager/templating/templates/shared/panel.json.j2 b/brian_dashboard_manager/templating/templates/shared/panel.json.j2 new file mode 100644 index 0000000000000000000000000000000000000000..f3e2389ffbf47896e1bb04afa3a3d5c806964098 --- /dev/null +++ b/brian_dashboard_manager/templating/templates/shared/panel.json.j2 @@ -0,0 +1,81 @@ +{ + "aliasColors": {}, + "bars": false, + "collapsed": null, + "dashLength": 10, + "dashes": false, + "datasource": "{{ datasource }}", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 5, + "gridPos": { + "h": {{ height }}, + "w": {{ width }}, + "x": 0, + "y": {{ y }} + }, + "hiddenSeries": false, + "id": {{ id }}, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": false, + "rightSide": null, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "search": null, + "seriesOverrides": [], + "spaceLength": 10, + "stack": null, + "steppedLine": false, + "tags": null, + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "{{ title }}", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": null + }, + "yaxes": [ + {{ yaxes }} + ], + "yaxis": { + "align": false, + "alignLevel": null + }, + "targets": [ + {% for target in targets %} + {{ target }}{{ "," if not loop.last }} + {% endfor %} + ] +} \ No newline at end of file diff --git a/brian_dashboard_manager/templating/templates/shared/panel_target.json.j2 b/brian_dashboard_manager/templating/templates/shared/panel_target.json.j2 new file mode 100644 index 0000000000000000000000000000000000000000..b98bd3c3b5e335c6c7f798ee1ac2ad8eead80143 --- /dev/null +++ b/brian_dashboard_manager/templating/templates/shared/panel_target.json.j2 @@ -0,0 +1,57 @@ +{ + "alias": "{{ alias }}", + "groupBy": [ + {% if not percentile %} + { + "params": ["5m"], + "type": "time" + }, + { + "params": ["linear"], + "type": "fill" + } + {% endif %} + ], + "measurement": "interface_rates", + "orderByTime": null, + "policy": null, + "refId": "{{ refId }}", + "resultFormat": "time_series", + "select": [ + [ + { + "params": ["{{ select_field }}"], + "type": "field" + }, + {% if not percentile %} + { + "params": [], + "type": "mean" + }, + {% else %} + { + "params": [95], + "type": "percentile" + }, + {% endif %} + { + "params": ["*8"], + "type": "math" + } + ] + ], + "tags": [ + { + "condition": null, + "key": "hostname", + "operator": "=", + "value": "{{ hostname }}" + }, + { + "condition": "AND", + "key": "interface_name", + "operator": "=", + "value": "{{ interface }}" + } + ] +} \ No newline at end of file