Add GWS direct dashboard generation

brian_dashboard_manager/grafana/provision.py

@@ -23,7 +23,7 @@ from brian_dashboard_manager.grafana.datasource import \
 from brian_dashboard_manager.grafana.folder import find_folder, \
     delete_folder
 from brian_dashboard_manager.inventory_provider.interfaces import \
-    get_interfaces
+    get_gws_direct, get_interfaces
 from brian_dashboard_manager.templating.nren_access import generate_nrens
 
 from brian_dashboard_manager.templating.helpers import is_re_customer, \
@@ -34,6 +34,8 @@ from brian_dashboard_manager.templating.helpers import is_re_customer, \
     get_interface_data, parse_backbone_name, parse_phy_upstream_name, \
     get_dashboard_data, get_aggregate_interface_data
 
+from brian_dashboard_manager.templating.gws import generate_gws
+
 from brian_dashboard_manager.templating.render import render_dashboard
 
 logger = logging.getLogger(__name__)
@@ -282,6 +284,7 @@ def provision(config):
                                       excluded_interfaces, datasource_name,
                                       exclude)
                 provisioned.append(res)
+
             for result in provisioned:
                 folder = result.result()
                 if folder is None:
@@ -291,6 +294,36 @@ def provision(config):
                         continue
                     dash_list[dashboard.get('uid')] = True
 
+        # fetch GWS direct data and provision related dashboards
+        logger.info('Provisioning GWS Direct dashboards')
+        folder_name = 'GWS Direct'
+        exclude_gws = excluded_folders.get(folder_name, [])
+        exclude_gws = list(map(lambda f: f.lower(), exclude_gws))
+        if isinstance(exclude_gws, bool) and exclude_gws:
+            # don't provision GWS Direct folder
+            delete_folder(token_request, folder_name)
+        else:
+            folder = find_folder(token_request, folder_name)
+            with ProcessPoolExecutor(max_workers=4) as executor:
+                gws_data = get_gws_direct(config['inventory_provider'])
+                provisioned = []
+
+                for dashboard in generate_gws(gws_data, datasource_name):
+                    rendered = render_dashboard(dashboard)
+                    if rendered.get('title').lower() in exclude_gws:
+                        executor.submit(delete_dashboard, token_request,
+                                        rendered, folder['id'])
+                        continue
+                    provisioned.append(executor.submit(create_dashboard,
+                                                       token_request,
+                                                       rendered, folder['id']))
+
+                for result in provisioned:
+                    dashboard = result.result()
+                    if dashboard is None:
+                        continue
+                    dash_list[dashboard.get('uid')] = True
+
         aggregate_dashboards = {
             'CLS PEERS': {
                 'predicate': is_cls_peer,
@@ -319,7 +352,6 @@ def provision(config):
         if isinstance(exclude_agg, bool) and exclude_agg:
             # don't provision aggregate folder
             delete_folder(token_request, 'Aggregates')
-            pass
         else:
             with ProcessPoolExecutor(max_workers=4) as executor:
                 provisioned = []

brian_dashboard_manager/inventory_provider/interfaces.py

@@ -47,3 +47,10 @@ def get_interfaces(host):  # pragma: no cover
         return interface
     enriched = list(map(enrich, interfaces))
     return enriched
+
+
+def get_gws_direct(host):
+    r = requests.get(f'{host}/poller/gws/direct')
+    r.raise_for_status()
+    interfaces = r.json()
+    return interfaces

brian_dashboard_manager/templating/gws.py

+from typing import DefaultDict
+from brian_dashboard_manager.templating.helpers import get_dashboard_data
+
+
+def get_interface_data(interfaces):
+    result = DefaultDict(list)
+
+    for interface in interfaces:
+
+        isp = interface.get('isp')
+        nren = interface.get('nren')
+
+        # identifier for interface, as we don't have their names
+        interface_tag = interface.get('tag')
+
+        counters = interface.get('counters')
+
+        skip = True
+
+        for counter in counters:
+            if counter.get('field') in ['traffic_in', 'traffic_out']:
+                skip = False
+
+        if skip:
+            # only process interfaces where we are polling traffic data
+            continue
+
+        gws_measurement = 'gwsd_rates'
+        panel_title = f'{nren} - {interface_tag} - {{}}'
+        result[f'GWS Direct - {isp}'].append({
+            'isp': isp,
+            'nren': nren,
+            'measurement': gws_measurement,
+            'title': panel_title,
+            'interface_tag': interface_tag,
+            'hostname': interface.get('hostname'),
+            'has_v6': False
+        })
+    return result
+
+
+def generate_gws(gws_data, datasource):
+
+    panel_data = get_interface_data(gws_data)
+    for dashboard in get_dashboard_data(panel_data, datasource, 'GWS_DIRECT'):
+        yield dashboard

brian_dashboard_manager/templating/helpers.py

@@ -294,7 +294,9 @@ def get_panel_fields(panel, panel_type, datasource):
 
     def get_target_data(alias, field):
         return {
-            **panel,  # panel has target hostname and interface
+            # panel includes identifying information
+            # such as hostname, interface, etc.
+            **panel,
             'alias': alias,
             'refId': next(letters),
             'select_field': field,
@@ -344,9 +346,9 @@ def get_dashboard_data(data, datasource, tag, errors=False):
                     {**panel, **next(gridPos)}, 'errors', datasource))
         return result
 
-    for peer, panels in data.items():
+    for dashboard_name, panels in data.items():
         result = {
-            'title': peer,
+            'title': dashboard_name,
             'datasource': datasource,
             'panels': get_panel_definitions(panels, datasource),
         }

brian_dashboard_manager/templating/templates/shared/panel_target.json.j2

@@ -12,7 +12,11 @@
         }
         {% endif %}
     ],
+    {% if measurement %}
+    "measurement": "{{ measurement }}",
+    {% else %}
     "measurement": "interface_rates",
+    {% endif %}
     "orderByTime": null,
     "policy": null,
     "refId": "{{ refId }}",
@@ -41,6 +45,7 @@
         ]
     ],
     "tags": [
+        {% if not isp %}
         {
             "condition": null,
             "key": "hostname",
@@ -53,5 +58,25 @@
             "operator": "=",
             "value": "{{ interface }}"
         }
+        {% else %}
+        {
+            "condition": null,
+            "key": "tag",
+            "operator": "=",
+            "value": "{{ interface_tag }}"
+        },
+        {
+            "condition": "AND",
+            "key": "isp",
+            "operator": "=",
+            "value": "{{ isp }}"
+        },
+        {
+            "condition": "AND",
+            "key": "nren",
+            "operator": "=",
+            "value": "{{ nren }}"
+        }
+        {% endif %}
     ]
 }
\ No newline at end of file

test/test_gws_direct.py

+import responses
+import json
+from brian_dashboard_manager.templating.gws import generate_gws
+from brian_dashboard_manager.inventory_provider.interfaces import \
+    get_gws_direct
+
+
+TEST_DATA = [
+    {
+        "nren": "ARNES",
+        "isp": "Cogent",
+        "hostname": "88.200.0.63",
+        "tag": "a",
+        "counters": [
+            {
+                "field": "discards_in",
+                "oid": "1.3.6.1.2.1.2.2.1.13.533",
+                "community": "gn2nocT3st"
+            },
+            {
+                "field": "discards_out",
+                "oid": "1.3.6.1.2.1.2.2.1.19.533",
+                "community": "gn2nocT3st"
+            },
+            {
+                "field": "errors_in",
+                "oid": "1.3.6.1.2.1.2.2.1.14.533",
+                "community": "gn2nocT3st"
+            },
+            {
+                "field": "errors_out",
+                "oid": "1.3.6.1.2.1.2.2.1.20.533",
+                "community": "gn2nocT3st"
+            }
+        ]
+    },
+    {
+        "nren": "ARNES",
+        "isp": "Cogent",
+        "hostname": "88.200.0.63",
+        "tag": "b",
+        "counters": [
+            {
+                "field": "traffic_in",
+                "oid": "1.3.6.1.2.1.31.1.1.1.6.531",
+                "community": "gn2nocT3st"
+            },
+            {
+                "field": "traffic_out",
+                "oid": "1.3.6.1.2.1.31.1.1.1.10.531",
+                "community": "gn2nocT3st"
+            }
+        ]
+    },
+    {
+        "nren": "ARNES",
+        "isp": "Cogent",
+        "hostname": "88.200.0.63",
+        "tag": "c",
+        "counters": [
+            {
+                "field": "traffic_in",
+                "oid": "1.3.6.1.2.1.31.1.1.1.6.525",
+                "community": "gn2nocT3st"
+            },
+            {
+                "field": "traffic_out",
+                "oid": "1.3.6.1.2.1.31.1.1.1.10.525",
+                "community": "gn2nocT3st"
+            }
+        ]
+    },
+    {
+        "nren": "ARNES",
+        "isp": "Cogent",
+        "hostname": "88.200.0.63",
+        "tag": "d",
+        "counters": [
+            {
+                "field": "traffic_in",
+                "oid": "1.3.6.1.2.1.31.1.1.1.6.553",
+                "community": "gn2nocT3st"
+            },
+            {
+                "field": "traffic_out",
+                "oid": "1.3.6.1.2.1.31.1.1.1.10.553",
+                "community": "gn2nocT3st"
+            }
+        ]
+    },
+    {
+        "nren": "ARNES",
+        "isp": "Telia",
+        "hostname": "62.40.124.6",
+        "tag": "a",
+        "counters": [
+            {
+                "field": "traffic_in",
+                "oid": "1.3.6.1.2.1.31.1.1.1.6.611",
+                "community": "gn2nocT3st"
+            },
+            {
+                "field": "traffic_out",
+                "oid": "1.3.6.1.2.1.31.1.1.1.10.611",
+                "community": "gn2nocT3st"
+            }
+        ]
+    }
+]
+
+
+@responses.activate
+def test_gws(data_config, mocker, client):
+
+    def get_callback(request):
+        return 200, {}, json.dumps(TEST_DATA)
+
+    responses.add_callback(
+        method=responses.GET,
+        url=f"{data_config['inventory_provider']}/poller/gws/direct",
+        callback=get_callback)
+
+    gws_data = get_gws_direct(data_config['inventory_provider'])
+
+    dashboards = list(generate_gws(gws_data, 'testdatasource'))
+
+    assert len(dashboards) == 2
+
+    assert dashboards[0]['title'] == 'GWS Direct - Cogent'
+    assert len(dashboards[0]['panels']) == 3
+
+    assert dashboards[1]['title'] == 'GWS Direct - Telia'
+    assert len(dashboards[1]['panels']) == 1

test/test_update.py

@@ -460,6 +460,10 @@ def test_provision(data_config, mocker, client):
     _mocked_get_dashboard_definitions = mocker.patch(
         'brian_dashboard_manager.grafana.provision.get_dashboard_definitions')
 
+    _mocked_gws = mocker.patch(
+        'brian_dashboard_manager.grafana.provision.get_gws_direct')
+    _mocked_gws.return_value = []
+
     UID = 1
     ID = 1
     VERSION = 1