Add jinja templating for generated dashboards

brian_dashboard_manager/templating/helpers.py

+import re
+from itertools import product
+from string import ascii_uppercase
+from brian_dashboard_manager.templating.render import create_panel
+
+PANEL_HEIGHT = 12
+PANEL_WIDTH = 24
+
+
+def get_description(interface):
+    return interface.get('description', '').strip()
+
+
+def is_physical_interface(interface):
+    return re.match('^PHY', get_description(interface))
+
+
+def is_aggregate_interface(interface):
+    return re.match('^LAG', get_description(interface))
+
+
+def is_logical_interface(interface):
+    return re.match('^SRV_', get_description(interface))
+
+
+def is_nren(interface):
+    regex = '(PHY|LAG|(SRV_(GLOBAL|LHCONE|MDVPN|IAS|CLS|L3VPN))) CUSTOMER'
+    return re.match(regex, get_description(interface))
+
+
+def is_cls(interface):
+    return 'SRV_CLS' in get_description(interface)
+
+
+def is_ias_public(interface):
+    return 'SRV_IAS PUBLIC' in get_description(interface)
+
+
+def is_ias_private(interface):
+    return 'SRV_IAS PRIVATE' in get_description(interface)
+
+
+def is_ias_customer(interface):
+    return 'SRV_IAS CUSTOMER' in get_description(interface)
+
+
+def is_ias_upstream(interface):
+    return 'SRV_IAS UPSTREAM' in get_description(interface)
+
+
+def is_re_peer(interface):
+    return 'SRV_GLOBAL RE_INTERCONNECT' in get_description(interface)
+
+
+def is_re_customer(interface):
+    regex = '(PHY|LAG|SRV_GLOBAL) CUSTOMER'
+    return re.match(regex, get_description(interface))
+
+
+def is_gcs(interface):
+    return re.match('^SRV_GCS', get_description(interface))
+
+
+def is_geantopen(interface):
+    return 'GEANTOPEN' in get_description(interface)
+
+
+def is_l2circuit(interface):
+    return 'SRV_L2CIRCUIT' in get_description(interface)
+
+
+def is_lhcone_peer(interface):
+    description = get_description(interface)
+    return 'LHCONE' in description and 'SRV_L3VPN RE' in description
+
+
+def is_lhcone_customer(interface):
+    description = get_description(interface)
+    return 'LHCONE' in description and 'SRV_L3VPN CUSTOMER' in description
+
+
+def is_mdvpn(interface):
+    return re.match('^SRV_MDVPN CUSTOMER', get_description(interface))
+
+
+def is_infrastructure_backbone(interface):
+    regex = '(SRV_GLOBAL|LAG|PHY) INFRASTRUCTURE BACKBONE'
+    return re.match(regex, get_description(interface))
+
+
+def is_lag_backbone(interface):
+    is_lag = 'LAG' in get_description(interface)
+    return is_infrastructure_backbone(interface) and is_lag
+
+
+def parse_backbone_name(description, *args, **kwargs):
+    link = description.split('|')[1].strip()
+    link = link.replace('( ', '(')
+    return link
+
+
+def is_phy_upstream(interface):
+    return re.match('^PHY UPSTREAM', get_description(interface))
+
+
+def parse_phy_upstream_name(description, host):
+    name = description.split(' ')[2].strip().upper()
+    location = host.split('.')[1].upper()
+    return f'{name} - {location}'
+
+
+def num_generator(start=1):
+    num = start
+    while True:
+        yield num
+        num += 1
+
+
+def gridPos_generator(id_generator, start=0):
+    num = start
+    while True:
+        yield {
+            "height": PANEL_HEIGHT,
+            "width": PANEL_WIDTH,
+            "x": 0,
+            "y": num * PANEL_HEIGHT,
+            "id": next(id_generator)
+        }
+        num += 1
+
+
+def letter_generator():
+    i = 0
+    j = 0
+    num_letters = len(ascii_uppercase)
+    while True:
+        result = ascii_uppercase[i % num_letters]
+
+        # tack on an extra letter if we are out of them
+        if (i >= num_letters):
+            result += ascii_uppercase[j % num_letters]
+            j += 1
+            if (j != 0 and j % num_letters == 0):
+                i += 1
+        else:
+            i += 1
+
+        yield result
+
+
+# peer_predicate is a function that is used to filter the interfaces
+# parse_func receives interface information and returns a peer name.
+def get_interface_data(interfaces, name_parse_func=None):
+    result = {}
+    for interface in interfaces:
+        if not name_parse_func:
+            # Most (but not all) descriptions use a format
+            # which has the peer name as the third element.
+            def name_parse_func(desc, *args, **kwargs):
+                return desc.split(' ')[2].upper()
+
+        description = interface.get('description', '').strip()
+        interface_name = interface.get('name')
+        host = interface.get('router', '')
+
+        dashboard_name = name_parse_func(description, host)
+
+        peer = result.get(dashboard_name, [])
+
+        router = host.replace('.geant.net', '')
+        panel_title = f"{router} - {{}} - {interface_name} - {description}"
+
+        peer.append({
+            'title': panel_title,
+            'interface': interface_name,
+            'hostname': host
+        })
+        result[dashboard_name] = peer
+    return result
+
+
+# Helper used for generating all traffic/error panels
+# with a single target field (ingress/egress or err/s)
+def get_panel_fields(panel, panel_type, datasource):
+    letters = letter_generator()
+
+    def get_target_data(alias, field):
+        return {
+            **panel,  # panel has target hostname and interface
+            'alias': alias,
+            'refId': next(letters),
+            'select_field': field,
+            'percentile': 'percentile' in alias.lower(),
+        }
+
+    error_fields = [('Ingress Errors', 'errorsIn'),
+                    ('Egress Errors', 'errorsOut')]
+
+    ingress = ['Ingress Traffic', 'Ingress 95th Percentile']
+    egress = ['Egress Traffic', 'Egress 95th Percentile']
+
+    is_v6 = panel_type == 'IPv6'
+    is_error = panel_type == 'errors'
+    in_field = 'ingressv6' if is_v6 else 'ingress'
+    out_field = 'egressv6' if is_v6 else 'egress'
+
+    fields = [*product(ingress, [in_field]), *product(egress, [out_field])]
+
+    targets = error_fields if is_error else fields
+
+    return create_panel({
+        **panel,
+        'datasource': datasource,
+        'title': panel['title'].format(panel_type),
+        'panel_targets': [get_target_data(*target) for target in targets],
+        'y_axis_type': 'errors' if is_error else 'bits',
+    })
+
+
+def get_dashboard_data(data, datasource, tag, errors=False):
+    id_gen = num_generator()
+    gridPos = gridPos_generator(id_gen)
+
+    def get_panel_definitions(panels, datasource):
+        result = []
+        for panel in panels:
+            result.append(get_panel_fields(
+                {**panel, **next(gridPos)}, 'traffic', datasource))
+            result.append(get_panel_fields(
+                {**panel, **next(gridPos)}, 'IPv6', datasource))
+            if errors:
+                result.append(get_panel_fields(
+                    {**panel, **next(gridPos)}, 'errors', datasource))
+        return result
+
+    for peer, panels in data.items():
+        yield {
+            'title': peer,
+            'datasource': datasource,
+            'panels': get_panel_definitions(panels, datasource),
+            'tag': tag
+        }

brian_dashboard_manager/templating/nren_access.py

+import json
+import os
+import jinja2
+from brian_dashboard_manager.templating.render import create_dropdown_panel, \
+    create_panel_target
+from brian_dashboard_manager.templating.helpers import \
+    is_aggregate_interface, is_logical_interface, is_physical_interface, \
+    num_generator, gridPos_generator, letter_generator, get_panel_fields
+
+
+def get_nrens(interfaces):
+    result = {}
+    for interface in interfaces:
+
+        description = interface.get('description', '').strip()
+
+        nren_name = description.split(' ')[2].upper()
+
+        nren = result.get(
+            nren_name, {'AGGREGATES': [], 'SERVICES': [], 'PHYSICAL': []})
+
+        interface_name = interface.get('name')
+        host = interface.get('router', '')
+        router = host.replace('.geant.net', '')
+        panel_title = f"{router} - {{}} - {interface_name} - {description}"
+
+        if is_aggregate_interface(interface):
+            nren['AGGREGATES'].append({
+                'interface': interface_name,
+                'hostname': host,
+                'alias': f"{host.split('.')[1].upper()} - {nren_name}"
+            })
+
+            # link aggregates are also shown
+            # under the physical dropdown
+            nren['PHYSICAL'].append({
+                'title': panel_title,
+                'hostname': host,
+                'interface': interface_name
+            })
+
+        elif is_logical_interface(interface):
+            nren['SERVICES'].append({
+                'title': panel_title,
+                'hostname': host,
+                'interface': interface_name
+            })
+        elif is_physical_interface(interface):
+            nren['PHYSICAL'].append({
+                'title': panel_title,
+                'hostname': host,
+                'interface': interface_name
+            })
+
+        result[nren_name] = nren
+    return result
+
+
+# start IDs from 3 since aggregate
+# panels have hardcoded IDs (1, 2).
+id_gen = num_generator(start=3)
+
+# aggregate panels have y=0, start generating at 1*height
+gridPos = gridPos_generator(id_gen, start=1)
+
+
+# Aggregate panels have unique targets,
+# handle those here.
+def get_aggregate_targets(aggregates):
+    ingress = []
+    egress = []
+
+    # used to generate refIds
+    letters = letter_generator()
+
+    for target in aggregates:
+        ref_id = next(letters)
+        in_data = {
+            **target,
+            'alias': f"{target['alias']} - Ingress Traffic",
+            'refId': ref_id,
+            'select_field': 'ingress'
+        }
+        out_data = {
+            **target,
+            'alias': f"{target['alias']} - Egress Traffic",
+            'refId': ref_id,
+            'select_field': 'egress'
+        }
+        ingress_target = create_panel_target(in_data)
+        egress_target = create_panel_target(out_data)
+        ingress.append(ingress_target)
+        egress.append(egress_target)
+
+    return ingress, egress
+
+
+def get_panel_definitions(panels, datasource, errors=False):
+    result = []
+    for panel in panels:
+        result.append(get_panel_fields(
+            {**panel, **next(gridPos)}, 'traffic', datasource))
+        result.append(get_panel_fields(
+            {**panel, **next(gridPos)}, 'IPv6', datasource))
+        if errors:
+            result.append(get_panel_fields(
+                {**panel, **next(gridPos)}, 'errors', datasource))
+    return result
+
+
+def get_dashboard_data(interfaces, datasource):
+
+    nren_data = get_nrens(interfaces)
+    for nren, data in nren_data.items():
+
+        agg_ingress, agg_egress = get_aggregate_targets(data['AGGREGATES'])
+        services_dropdown = create_dropdown_panel('Services', **next(gridPos))
+        service_panels = get_panel_definitions(data['SERVICES'], datasource)
+        iface_dropdown = create_dropdown_panel('Interfaces', **next(gridPos))
+        phys_panels = get_panel_definitions(data['PHYSICAL'], datasource, True)
+
+        yield {
+            'nren_name': nren,
+            'datasource': datasource,
+            'ingress_aggregate_targets': agg_ingress,
+            'egress_aggregate_targets': agg_egress,
+            'dropdown_groups': [
+                {
+                    'dropdown': services_dropdown,
+                    'panels': service_panels,
+                },
+                {
+                    'dropdown': iface_dropdown,
+                    'panels': phys_panels,
+                }
+            ]
+        }
+
+
+def generate_nrens(interfaces, datasource):
+    file = os.path.abspath(os.path.join(
+        os.path.dirname(__file__),
+        'templates',
+        'nren_access',
+        'nren-dashboard.json.j2'))
+
+    with open(file) as f:
+        template = jinja2.Template(f.read())
+
+    for dashboard in get_dashboard_data(interfaces, datasource):
+        rendered = template.render(dashboard)
+        rendered = json.loads(rendered)
+        rendered['uid'] = None
+        rendered['id'] = None
+        yield rendered

brian_dashboard_manager/templating/render.py

+import os
+import json
+import jinja2
+
+
+def create_dropdown_panel(title, **kwargs):
+    TEMPLATE_FILENAME = os.path.abspath(os.path.join(
+        os.path.dirname(__file__),
+        'templates',
+        'shared',
+        'dropdown.json.j2'))
+    with open(TEMPLATE_FILENAME) as f:
+        template = jinja2.Template(f.read())
+    return template.render({**kwargs, 'title': title})
+
+
+# wrapper around bits/s and err/s panel labels
+def create_yaxes(type):
+    file = os.path.abspath(os.path.join(
+        os.path.dirname(__file__),
+        'templates',
+        'shared',
+        'yaxes.json.j2'))
+    with open(file) as f:
+        template = jinja2.Template(f.read())
+    return template.render({'type': type})
+
+
+def create_panel_target(data):
+    file = os.path.abspath(os.path.join(
+        os.path.dirname(__file__),
+        'templates',
+        'shared',
+        'panel_target.json.j2'))
+    with open(file) as f:
+        template = jinja2.Template(f.read())
+    return template.render(data)
+
+
+def create_panel(data):
+    file = os.path.abspath(os.path.join(
+        os.path.dirname(__file__),
+        'templates',
+        'shared',
+        'panel.json.j2'))
+    with open(file) as f:
+        template = jinja2.Template(f.read())
+    yaxes = create_yaxes(data.get('y_axis_type', 'bits'))
+    targets = []
+    for target in data.get('panel_targets', []):
+        targets.append(create_panel_target(target))
+    return template.render({**data, 'yaxes': yaxes, 'targets': targets})
+
+
+def render_dashboard(dashboard):
+    file = os.path.abspath(os.path.join(
+        os.path.dirname(__file__),
+        'templates',
+        'shared',
+        'dashboard.json.j2'))
+
+    with open(file) as f:
+        template = jinja2.Template(f.read())
+    rendered = template.render(dashboard)
+    rendered = json.loads(rendered)
+    rendered['uid'] = None
+    rendered['id'] = None
+    return rendered

brian_dashboard_manager/templating/templates/nren_access/nren-dashboard.json.j2

+{
+  "annotations": {
+    "list": [
+      {
+        "builtIn": 1,
+        "datasource": "-- Grafana --",
+        "enable": true,
+        "hide": true,
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "name": "Annotations & Alerts",
+        "type": "dashboard"
+      }
+    ]
+  },
+  "editable": false,
+  "gnetId": null,
+  "graphTooltip": 0,
+  "schemaVersion": 27,
+  "style": "dark",
+  "tags": ["customers"],
+  "templating": {
+    "list": []
+  },
+  "time": {
+    "from": "now-24h",
+    "to": "now"
+  },
+  "timepicker": {},
+  "timezone": "",
+  "title": "{{ nren_name }}",
+  "version": 1,
+  "links": [],
+  "panels": [
+      {
+      "aliasColors": {},
+      "bars": false,
+      "collapsed": null,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": "{{ datasource }}",
+      "fieldConfig": {
+        "defaults": {
+          "custom": {}
+        },
+        "overrides": []
+      },
+      "fill": 1,
+      "fillGradient": 10,
+      "gridPos": {
+        "h": 12,
+        "w": 12,
+        "x": 0,
+        "y": 0
+      },
+      "hiddenSeries": false,
+      "id": 1,
+      "legend": {
+        "alignAsTable": true,
+        "avg": true,
+        "current": true,
+        "max": true,
+        "min": false,
+        "rightSide": false,
+        "show": true,
+        "total": false,
+        "values": true
+      },
+      "lines": true,
+      "linewidth": 1,
+      "nullPointMode": "null",
+      "options": {
+        "alertThreshold": true
+      },
+      "percentage": false,
+      "pointradius": 2,
+      "points": false,
+      "renderer": "flot",
+      "search": null,
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": true,
+      "steppedLine": false,
+      "tags": null,
+      "targets": [
+        {% for target in ingress_aggregate_targets %}
+          {{ target }}{{ "," if not loop.last }}
+        {% endfor %}
+      ],
+      "thresholds": [],
+      "timeFrom": null,
+      "timeRegions": [],
+      "timeShift": null,
+      "title": "Aggregate - {{ nren_name }} - ingress",
+      "tooltip": {
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "buckets": null,
+        "mode": "time",
+        "name": null,
+        "show": true,
+        "values": null
+      },
+      "yaxes": [
+        {
+          "format": "bps",
+          "label": "bits per second",
+          "logBase": 1,
+          "max": null,
+          "min": "",
+          "show": true
+        },
+        {
+          "format": "bps",
+          "label": "bits per second",
+          "logBase": 1,
+          "max": null,
+          "min": "",
+          "show": true
+        }
+      ],
+      "yaxis": {
+        "align": false,
+        "alignLevel": null
+      }
+    },
+    {
+      "aliasColors": {},
+      "bars": false,
+      "collapsed": null,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": "{{ datasource }}",
+      "fieldConfig": {
+        "defaults": {
+          "custom": {}
+        },
+        "overrides": []
+      },
+      "fill": 1,
+      "fillGradient": 10,
+      "gridPos": {
+        "h": 12,
+        "w": 12,
+        "x": 12,
+        "y": 0
+      },
+      "hiddenSeries": false,
+      "id": 2,
+      "legend": {
+        "alignAsTable": true,
+        "avg": true,
+        "current": true,
+        "max": true,
+        "min": false,
+        "rightSide": false,
+        "show": true,
+        "total": false,
+        "values": true
+      },
+      "lines": true,
+      "linewidth": 1,
+      "nullPointMode": "null",
+      "options": {
+        "alertThreshold": true
+      },
+      "percentage": false,
+      "pointradius": 2,
+      "points": false,
+      "renderer": "flot",
+      "search": null,
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": true,
+      "steppedLine": false,
+      "tags": null,
+      "targets": [
+        {% for target in egress_aggregate_targets %}
+          {{ target }}{{ "," if not loop.last }}
+        {% endfor %}
+      ],
+      "thresholds": [],
+      "timeFrom": null,
+      "timeRegions": [],
+      "timeShift": null,
+      "title": "Aggregate - {{ nren_name }} - egress",
+      "tooltip": {
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "buckets": null,
+        "mode": "time",
+        "name": null,
+        "show": true,
+        "values": null
+      },
+      "yaxes": [
+        {
+          "format": "bps",
+          "label": "bits per second",
+          "logBase": 1,
+          "max": null,
+          "min": "",
+          "show": true
+        },
+        {
+          "format": "bps",
+          "label": "bits per second",
+          "logBase": 1,
+          "max": null,
+          "min": "",
+          "show": true
+        }
+      ],
+      "yaxis": {
+        "align": false,
+        "alignLevel": null
+      }
+    },
+    {% for group in dropdown_groups %}
+      {{ group.dropdown }}
+      {% if group.panels|length > 0 %}
+        ,
+      {% endif %}
+      {% for panel in group.panels %}
+        {{ panel }}{{ "," if not loop.last }}
+      {% endfor %}
+      {{ "," if not loop.last }}
+    {% endfor %}
+  ]
+}
\ No newline at end of file

brian_dashboard_manager/templating/templates/shared/dashboard.json.j2

+{
+  "annotations": {
+    "list": [
+      {
+        "builtIn": 1,
+        "datasource": "-- Grafana --",
+        "enable": true,
+        "hide": true,
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "name": "Annotations & Alerts",
+        "type": "dashboard"
+      }
+    ]
+  },
+  "editable": false,
+  "gnetId": null,
+  "graphTooltip": 0,
+  "schemaVersion": 27,
+  "style": "dark",
+  "tags": ["{{ tag }}"],
+  "templating": {
+    "list": []
+  },
+  "time": {
+    "from": "now-24h",
+    "to": "now"
+  },
+  "timepicker": {},
+  "timezone": "",
+  "title": "{{ title }}",
+  "version": 1,
+  "links": [],
+  "panels": [
+    {% for panel in panels %}
+      {{ panel }}{{ "," if not loop.last }}
+    {% endfor %}
+  ]
+}
\ No newline at end of file

brian_dashboard_manager/templating/templates/shared/panel.json.j2

+{
+    "aliasColors": {},
+    "bars": false,
+    "collapsed": null,
+    "dashLength": 10,
+    "dashes": false,
+    "datasource": "{{ datasource }}",
+    "fieldConfig": {
+        "defaults": {
+            "custom": {}
+        },
+        "overrides": []
+    },
+    "fill": 1,
+    "fillGradient": 5,
+    "gridPos": {
+        "h": {{ height }},
+        "w": {{ width }},
+        "x": 0,
+        "y": {{ y }}
+    },
+    "hiddenSeries": false,
+    "id": {{ id }},
+    "legend": {
+        "alignAsTable": true,
+        "avg": true,
+        "current": true,
+        "max": true,
+        "min": false,
+        "rightSide": null,
+        "show": true,
+        "total": false,
+        "values": true
+    },
+    "lines": true,
+    "linewidth": 1,
+    "nullPointMode": "null",
+    "options": {
+        "alertThreshold": true
+    },
+    "percentage": false,
+    "pointradius": 2,
+    "points": false,
+    "renderer": "flot",
+    "search": null,
+    "seriesOverrides": [],
+    "spaceLength": 10,
+    "stack": null,
+    "steppedLine": false,
+    "tags": null,
+    "thresholds": [],
+    "timeFrom": null,
+    "timeRegions": [],
+    "timeShift": null,
+    "title": "{{ title }}",
+    "tooltip": {
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+    },
+    "type": "graph",
+    "xaxis": {
+        "buckets": null,
+        "mode": "time",
+        "name": null,
+        "show": true,
+        "values": null
+    },
+    "yaxes": [
+        {{ yaxes }}
+    ],
+    "yaxis": {
+        "align": false,
+        "alignLevel": null
+    },
+    "targets": [
+    {% for target in targets %}
+        {{ target }}{{ "," if not loop.last }}
+    {% endfor %}
+    ]
+}
\ No newline at end of file

brian_dashboard_manager/templating/templates/shared/panel_target.json.j2

+{
+    "alias": "{{ alias }}",
+    "groupBy": [
+        {% if not percentile %}
+        {
+            "params": ["5m"],
+            "type": "time"
+        },
+        {
+            "params": ["linear"],
+            "type": "fill"
+        }
+        {% endif %}
+    ],
+    "measurement": "interface_rates",
+    "orderByTime": null,
+    "policy": null,
+    "refId": "{{ refId }}",
+    "resultFormat": "time_series",
+    "select": [
+        [
+            {
+                "params": ["{{ select_field }}"],
+                "type": "field"
+            },
+        {% if not percentile %}
+            {
+                "params": [],
+                "type": "mean"
+            },
+        {% else %}
+            {
+                "params": [95],
+                "type": "percentile"
+            },
+        {% endif %}
+            {
+                "params": ["*8"],
+                "type": "math"
+            }
+        ]
+    ],
+    "tags": [
+        {
+            "condition": null,
+            "key": "hostname",
+            "operator": "=",
+            "value": "{{ hostname }}"
+        },
+        {
+            "condition": "AND",
+            "key": "interface_name",
+            "operator": "=",
+            "value": "{{ interface }}"
+        }
+    ]
+}
\ No newline at end of file