diff --git a/MANIFEST.in b/MANIFEST.in index 1842a72e0920e258d37be3cfd645fda7a4f9da39..cff7aadeb2cf4ea2296ae9e5e8c3b22045952df7 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -2,5 +2,4 @@ include brian_dashboard_manager/logging_default_config.json include brian_dashboard_manager/dashboards/* include brian_dashboard_manager/datasources/* include config.json.example -recursive-include brian_dashboard_manager/templating/templates * recursive-exclude test * \ No newline at end of file diff --git a/brian_dashboard_manager/app.py b/brian_dashboard_manager/app.py index 5bc14b921b5f95fb5c68b08e83b5491952de018e..347dbeea0a7460e6274685db7417ca2b0176f041 100644 --- a/brian_dashboard_manager/app.py +++ b/brian_dashboard_manager/app.py @@ -2,9 +2,8 @@ default app creation """ import brian_dashboard_manager -from brian_dashboard_manager import environment, CONFIG_KEY +from brian_dashboard_manager import CONFIG_KEY -environment.setup_logging() app = brian_dashboard_manager.create_app() if __name__ == "__main__": diff --git a/brian_dashboard_manager/config.py b/brian_dashboard_manager/config.py index b22ef4e4d11e7ce2764cd9b04ffdfb2f1791685b..81e47202158b056c85f355861cedfa0d41e4e429 100644 --- a/brian_dashboard_manager/config.py +++ b/brian_dashboard_manager/config.py @@ -148,7 +148,7 @@ DEFAULT_ORGANIZATIONS = [ ] CONFIG_SCHEMA = { - "$schema": "https://json-schema.org/draft-07/schema#", + "$schema": "http://json-schema.org/draft-07/schema#", "definitions": { "influx-datasource": { diff --git a/brian_dashboard_manager/grafana/dashboard.py b/brian_dashboard_manager/grafana/dashboard.py index 37b67163e27fc33c19fe325801b4781e7489d772..74723ee0cc89d306a81bf3e1120debbe971f3192 100644 --- a/brian_dashboard_manager/grafana/dashboard.py +++ b/brian_dashboard_manager/grafana/dashboard.py @@ -55,7 +55,7 @@ def delete_dashboard(request: TokenRequest, dashboard: dict, folder_id=None): dash = _search_dashboard(request, dashboard, folder_id) if dash is None: return True - uid = dash.get('dashboard', {}).get('uid', '') + uid = dash.get('uid', '') if uid: return _delete_dashboard(request, uid) else: @@ -188,7 +188,7 @@ def _get_dashboard(request: TokenRequest, uid): r = request.get(f'api/dashboards/uid/{uid}') except HTTPError: return None - return r.json() + return r.json()['dashboard'] def create_dashboard(request: TokenRequest, dashboard: dict, folder_id=None): @@ -211,7 +211,7 @@ def create_dashboard(request: TokenRequest, dashboard: dict, folder_id=None): # The title might not match the one that's provisioned with that UID. # Try to find it by searching for the title instead. if existing_dashboard is not None: - grafana_title = existing_dashboard['dashboard']['title'] + grafana_title = existing_dashboard['title'] different = grafana_title != title else: different = False @@ -220,12 +220,12 @@ def create_dashboard(request: TokenRequest, dashboard: dict, folder_id=None): existing_dashboard = _search_dashboard(request, dashboard, folder_id) if existing_dashboard: - dashboard['uid'] = existing_dashboard['dashboard']['uid'] - dashboard['id'] = existing_dashboard['dashboard']['id'] - dashboard['version'] = existing_dashboard['dashboard']['version'] + dashboard['uid'] = existing_dashboard['uid'] + dashboard['id'] = existing_dashboard['id'] + dashboard['version'] = existing_dashboard['version'] else: # We are creating a new dashboard, delete ID if it exists. - del dashboard['id'] + dashboard.pop('id', None) payload = { 'dashboard': dashboard, @@ -241,14 +241,17 @@ def create_dashboard(request: TokenRequest, dashboard: dict, folder_id=None): return r.json() except HTTPError as e: message = '' - if e.response is not None and e.response.status_code < 500: + if e.response is not None: # log the error message from Grafana - message = e.response.json() + try: + message = e.response.json() + except json.JSONDecodeError: + message = e.response.text + logger.exception(f"Error when provisioning dashboard {title}: {message}") # only retry on server side errors if e.response is not None and e.response.status_code < 500: break - logger.exception(f'Error when provisioning dashboard {title}: {message}') time.sleep(1) # sleep for 1 second before retrying return None diff --git a/brian_dashboard_manager/grafana/organization.py b/brian_dashboard_manager/grafana/organization.py index 7115b83ff00c0407a8f54fe9a51217cd295c9b4d..629fe8a956187f021470fa3a610a3116e63b0246 100644 --- a/brian_dashboard_manager/grafana/organization.py +++ b/brian_dashboard_manager/grafana/organization.py @@ -2,17 +2,16 @@ Grafana Organization management helpers. """ + +import logging import random import string -import logging -import jinja2 -import json -import os -from typing import Dict, List, Union from datetime import datetime -from brian_dashboard_manager.grafana.utils.request import AdminRequest, \ - TokenRequest +from typing import Dict, List, Union + from brian_dashboard_manager.grafana.dashboard import create_dashboard +from brian_dashboard_manager.grafana.utils.request import AdminRequest, TokenRequest +from brian_dashboard_manager.templating.homedashboard import render_homedashboard logger = logging.getLogger(__name__) @@ -105,7 +104,7 @@ def delete_api_token(request: AdminRequest, token_id: int, org_id=None): :return: delete response """ - assert token_id + assert token_id is not None if org_id: switch_active_organization(request, org_id) result = request.delete(f'api/auth/keys/{token_id}') @@ -146,19 +145,8 @@ def set_home_dashboard(request: TokenRequest, is_staff): :param is_staff: True if the organization is the staff organization :return: True if successful """ - - file = os.path.abspath(os.path.join( - os.path.dirname(__file__), - '..', - 'templating', - 'templates', - 'homedashboard.json.j2')) - - with open(file) as f: - template = jinja2.Template(f.read()) - rendered = template.render({'staff': is_staff}) - rendered = json.loads(rendered) - dashboard = create_dashboard(request, rendered) + payload = render_homedashboard(staff=is_staff) + dashboard = create_dashboard(request, payload) r = request.put('api/org/preferences', json={ 'homeDashboardId': dashboard.get('id') }).json() diff --git a/brian_dashboard_manager/grafana/provision.py b/brian_dashboard_manager/grafana/provision.py index 8087026120e9a57352257a8bce0474353fe72493..0259d48414956946ce6b2f990af970e0a195e1ed 100644 --- a/brian_dashboard_manager/grafana/provision.py +++ b/brian_dashboard_manager/grafana/provision.py @@ -33,13 +33,14 @@ from brian_dashboard_manager.templating.helpers import \ get_aggregate_dashboard_data, get_interface_data, \ get_nren_interface_data, get_dashboard_data, \ get_nren_dashboard_data, get_aggregate_interface_data, \ - get_nren_interface_data_old + get_nren_interface_data_old, get_re_peer_dashboard_data, get_re_peer_interface_data -from brian_dashboard_manager.templating.gws import generate_gws, \ - generate_indirect -from brian_dashboard_manager.templating.eumetsat \ - import generate_eumetsat_multicast -from brian_dashboard_manager.templating.render import render_dashboard +from brian_dashboard_manager.templating.gws import generate_gws, generate_indirect +from brian_dashboard_manager.templating.eumetsat import generate_eumetsat_multicast +from brian_dashboard_manager.templating.render import ( + render_complex_dashboard, + render_simple_dashboard, +) logger = logging.getLogger(__name__) @@ -214,10 +215,10 @@ def provision_folder(token_request, folder_name, dash, # dashboard should include error panels errors = dash.get('errors', False) - # needed for POL1-642 BETA - is_nren_beta = folder_name == 'NREN Access BETA' + is_nren_beta = folder_name == "NREN Access BETA" # needed for POL1-642 BETA + is_nren = folder_name == "NREN Access" + is_re_peer = folder_name == "RE Peer" - is_nren = folder_name == 'NREN Access' if is_nren: data = get_nren_interface_data_old(interfaces) dash_data = get_nren_dashboard_data(data, ds_name, tag) @@ -227,6 +228,9 @@ def provision_folder(token_request, folder_name, dash, data = get_nren_interface_data( services, interfaces, excluded_dashboards) dash_data = get_nren_dashboard_data(data, ds_name, tag) + elif is_re_peer: + data = get_re_peer_interface_data(interfaces) + dash_data = get_re_peer_dashboard_data(data, ds_name, tag) else: data = get_interface_data(interfaces) dash_data = get_dashboard_data( @@ -239,11 +243,13 @@ def provision_folder(token_request, folder_name, dash, with ThreadPoolExecutor(max_workers=MAX_WORKERS) as executor: for dashboard in dash_data: - rendered = render_dashboard( - dashboard, nren=is_nren or is_nren_beta) - if rendered.get('title').lower() in excluded_dashboards: - executor.submit(delete_dashboard, token_request, - rendered, folder['id']) + if is_nren or is_nren_beta or is_re_peer: + rendered = render_complex_dashboard(**dashboard) + else: + rendered = render_simple_dashboard(**dashboard) + + if rendered.get("title").lower() in excluded_dashboards: + executor.submit(delete_dashboard, token_request, rendered, folder["id"]) continue provisioned.append(executor.submit(create_dashboard, token_request, rendered, folder['id'])) @@ -273,7 +279,7 @@ def provision_aggregate(token_request, folder, dashboard = get_aggregate_dashboard_data( f'Aggregate - {name}', data, ds_name, tag) - rendered = render_dashboard(dashboard) + rendered = render_simple_dashboard(**dashboard) return create_dashboard(token_request, rendered, folder['id']) @@ -460,7 +466,7 @@ def _provision_gws_indirect(config, org_config, ds_name, token): provisioned = [] dashes = generate_indirect(gws_indirect_data, ds_name) for dashboard in dashes: - rendered = render_dashboard(dashboard) + rendered = render_simple_dashboard(**dashboard) provisioned.append(executor.submit(create_dashboard, token, rendered, folder['id'])) @@ -495,7 +501,7 @@ def _provision_gws_direct(config, org_config, ds_name, token): provisioned = [] for dashboard in generate_gws(gws_data, ds_name): - rendered = render_dashboard(dashboard) + rendered = render_simple_dashboard(**dashboard) provisioned.append(executor.submit(create_dashboard, token, rendered, folder['id'])) @@ -530,7 +536,7 @@ def _provision_eumetsat_multicast(config, org_config, ds_name, token): for dashboard in generate_eumetsat_multicast( subscriptions, ds_name): - rendered = render_dashboard(dashboard) + rendered = render_simple_dashboard(**dashboard) provisioned.append( executor.submit( create_dashboard, @@ -725,7 +731,7 @@ def provision_maybe(config): write_timestamp(now.timestamp(), False) -def provision(config): +def provision(config, raise_exceptions=False): """ The entrypoint for the provisioning process. @@ -827,6 +833,8 @@ def provision(config): delete_api_token(request, token['id'], org_id=org_id) except Exception: logger.exception(f'Error when provisioning org {org["name"]}') + if raise_exceptions: + raise break logger.info(f'Time to complete: {time.time() - start}') diff --git a/brian_dashboard_manager/inventory_provider/interfaces.py b/brian_dashboard_manager/inventory_provider/interfaces.py index 429969405e9b63fbe53c71b40a6be96e8dbabfdf..f37adb6df8964fcf508601eb885c6476e929f8d6 100644 --- a/brian_dashboard_manager/inventory_provider/interfaces.py +++ b/brian_dashboard_manager/inventory_provider/interfaces.py @@ -62,7 +62,7 @@ _PORT_TYPES = [t.name for t in list(PORT_TYPES)] _INTERFACE_TYPES = [i.name for i in list(INTERFACE_TYPES)] ROUTER_INTERFACES_SCHEMA = { - "$schema": "https://json-schema.org/draft-07/schema#", + "$schema": "http://json-schema.org/draft-07/schema#", "type": "array", "items": { "type": "object", @@ -89,7 +89,7 @@ ROUTER_INTERFACES_SCHEMA = { } INTERFACE_LIST_SCHEMA = { - '$schema': 'https://json-schema.org/draft-07/schema#', + '$schema': 'http://json-schema.org/draft-07/schema#', 'definitions': { 'service': { @@ -137,7 +137,7 @@ INTERFACE_LIST_SCHEMA = { } GWS_DIRECT_DATA_SCHEMA = { - '$schema': 'https://json-schema.org/draft-07/schema#', + '$schema': 'http://json-schema.org/draft-07/schema#', 'definitions': { 'oid': { @@ -214,7 +214,7 @@ GWS_DIRECT_DATA_SCHEMA = { } MULTICAST_SUBSCRIPTION_LIST_SCHEMA = { - '$schema': 'https://json-schema.org/draft-07/schema#', + '$schema': 'http://json-schema.org/draft-07/schema#', 'definitions': { 'ipv4-address': { diff --git a/brian_dashboard_manager/routes/update.py b/brian_dashboard_manager/routes/update.py index 56ada82117d92c40804f694e7240983ab73733c6..4674d82c85041886e04c692e35cc8debe58c72b2 100644 --- a/brian_dashboard_manager/routes/update.py +++ b/brian_dashboard_manager/routes/update.py @@ -13,7 +13,7 @@ from brian_dashboard_manager.config import STATE_PATH routes = Blueprint("update", __name__) UPDATE_RESPONSE_SCHEMA = { - '$schema': 'https://json-schema.org/draft-07/schema#', + '$schema': 'http://json-schema.org/draft-07/schema#', 'type': 'object', 'properties': { 'message': { diff --git a/brian_dashboard_manager/templating/eumetsat.py b/brian_dashboard_manager/templating/eumetsat.py index 45cf2472a3eaa0327165948f1f581de99edf7283..e2717f2578566523a5795b37ecd5f88e588461f9 100644 --- a/brian_dashboard_manager/templating/eumetsat.py +++ b/brian_dashboard_manager/templating/eumetsat.py @@ -57,16 +57,16 @@ def get_panel_fields(panel, panel_type, datasource): # 'percentile': 'percentile' in alias.lower(), } - targets = [('Multicast Traffic', 'octets')] - - return create_panel({ + targets = [("Multicast Traffic", "octets")] + title = panel.pop("title").format(panel_type) + return create_panel( **panel, - 'datasource': datasource, - 'linewidth': 1, - 'title': panel['title'].format(panel_type), - 'panel_targets': [get_target_data(*target) for target in targets], - 'y_axis_type': 'bits', - }) + datasource=datasource, + linewidth=1, + title=title, + panel_targets=[get_target_data(*target) for target in targets], + y_axis_type="bits", + ) def subscription_panel_generator(gridPos): diff --git a/brian_dashboard_manager/templating/helpers.py b/brian_dashboard_manager/templating/helpers.py index c9bba091cf88b506a33d3ba4db7e1cf65057c3d2..31c47cf52313f91c65792d1aee8b030f3754e038 100644 --- a/brian_dashboard_manager/templating/helpers.py +++ b/brian_dashboard_manager/templating/helpers.py @@ -5,7 +5,6 @@ necessary data to generate the dashboards from templates. from collections import defaultdict from concurrent.futures import ProcessPoolExecutor import logging -import json from itertools import product from functools import partial, reduce from string import ascii_uppercase @@ -153,6 +152,70 @@ def get_nren_interface_data_old(interfaces): return result +def get_re_peer_interface_data(interfaces): + """ + Helper for grouping interfaces into groups of R&E Peers + See POL1-579 + Aggregate (AGGREGATES) all service interfaces (logical) (ipv4 only) + Services (SERVICES) contain all logical interfaces (both ipv4 and ipv6) + Interfaces (PHYSICAL) contain physical interfaces and LAGs (AGGREGATE) + """ + result = {} + + for interface in interfaces: + + description = interface['description'].strip() + interface_name = interface['name'] + host = interface['router'] + + router = host.replace('.geant.net', '') + location = host.split('.')[1].upper() + panel_title = f"{router} - {{}} - {interface_name} - {description}" + + dashboards_info = interface['dashboards_info'] + + for info in dashboards_info: + dashboard_name = info['name'] + + dashboard = result.get(dashboard_name, { + 'AGGREGATES': [], + 'SERVICES': [], + 'PHYSICAL': [] + }) + + if info['interface_type'] == 'AGGREGATE': + # link aggregates are shown under the physical dropdown + dashboard['PHYSICAL'].append({ + 'title': panel_title, + 'hostname': host, + 'interface': interface_name + }) + + elif info['interface_type'] == 'LOGICAL': + dashboard['AGGREGATES'].append({ + 'interface': interface_name, + 'hostname': host, + 'alias': + f"{location} - {dashboard_name} ({interface_name})" + }) + + dashboard['SERVICES'].append({ + 'title': panel_title, + 'hostname': host, + 'interface': interface_name, + 'has_v6': len(interface.get('ipv6', [])) > 0 + }) + elif info['interface_type'] == 'PHYSICAL': + dashboard['PHYSICAL'].append({ + 'title': panel_title, + 'hostname': host, + 'interface': interface_name + }) + + result[dashboard_name] = dashboard + return result + + def get_nren_interface_data(services, interfaces, excluded_dashboards): """ Helper for grouping interface data to be used for generating @@ -432,8 +495,8 @@ def get_aggregate_targets(targets): 'refId': ref_id, 'select_field': 'egress' } - ingress_target = create_panel_target(in_data) - egress_target = create_panel_target(out_data) + ingress_target = create_panel_target(**in_data) + egress_target = create_panel_target(**out_data) ingress.append(ingress_target) egress.append(egress_target) @@ -484,15 +547,16 @@ def get_panel_fields(panel, panel_type, datasource): fields = [*product(ingress, [in_field]), *product(egress, [out_field])] targets = error_fields if is_error else fields + title = panel.pop("title").format(panel_type) - return create_panel({ + return create_panel( **panel, - 'datasource': datasource, - 'linewidth': 1, - 'title': panel['title'].format(panel_type), - 'panel_targets': [get_target_data(*target) for target in targets], - 'y_axis_type': 'errors' if is_error else 'bits', - }) + datasource=datasource, + linewidth=1, + title=title, + panel_targets=[get_target_data(*target) for target in targets], + y_axis_type="errors" if is_error else "bits", + ) def default_interface_panel_generator(gridPos, use_all_traffic=True, use_ipv6=True): @@ -652,6 +716,100 @@ def get_nren_dashboard_data(data, datasource, tag): yield dash +def get_re_peer_dashboard_data_single(data, datasource, tag): + """ + Helper for generating dashboard definitions for a single R&E Peer. + + NREN dashboards have two aggregate panels (ingress and egress), + and two dropdown panels for services and interfaces. + + :param data: data for the dashboard, including the R&E Peer name and + the panel data + :param datasource: datasource to use for the panels + :param tag: tag to use for the dashboard, used for dashboard dropdowns on + the home dashboard. + + :return: dashboard definition for the R&E Peer dashboard + """ + + peer, dash = data + id_gen = num_generator() + + if len(dash['AGGREGATES']) > 0: + agg_panels = create_aggregate_panel( + f'Aggregate - {peer}', + gridPos_generator(id_gen, agg=True), + dash['AGGREGATES'], datasource) + gridPos = gridPos_generator(id_gen, start=2) + else: + gridPos = gridPos_generator(id_gen) + agg_panels = [] + + panel_gen = default_interface_panel_generator(gridPos, use_all_traffic=True, use_ipv6=True) + + services_dropdown = create_dropdown_panel('Services', **next(gridPos)) + + def sort_key(panel): + sort = panel.get('sort') + if not sort: + return 'ZZZ'+panel.get('hostname') # sort to end + return sort + + service_panels = panel_gen( + sorted(dash['SERVICES'], key=sort_key), datasource) + + iface_dropdown = create_dropdown_panel('Interfaces', **next(gridPos)) + phys_panels = panel_gen(dash['PHYSICAL'], datasource, True) + + dropdown_groups = [{ + 'dropdown': services_dropdown, + 'panels': service_panels, + }] + + dropdown_groups.append({ + 'dropdown': iface_dropdown, + 'panels': phys_panels, + }) + + result = { + 'nren_name': peer, + 'datasource': datasource, + 'aggregate_panels': agg_panels, + 'dropdown_groups': dropdown_groups + } + if isinstance(tag, list): + result['tags'] = tag + else: + result['tag'] = tag + + return result + + +def get_re_peer_dashboard_data(data, datasource, tag): + """ + Helper for generating dashboard definitions for all R&E Peers. + Uses multiprocessing to speed up generation. + + :param data: the names and the panel data for each R&E Peer + :param datasource: datasource to use for the panels + :param tag: tag to use for the dashboard, used for dashboard dropdowns on + the home dashboard. + + :return: generator for dashboard definitions for each R&E Peer + """ + + with ProcessPoolExecutor(max_workers=NUM_PROCESSES) as executor: + for dash in executor.map( + partial( + get_re_peer_dashboard_data_single, + datasource=datasource, + tag=tag), + data.items() + ): + + yield dash + + def get_dashboard_data_single( data, datasource, tag, panel_generator=default_interface_panel_generator, @@ -743,8 +901,7 @@ def create_aggregate_panel(title, gridpos, targets, datasource): is_total = 'totals' in title.lower() def reduce_alias(prev, curr): - d = json.loads(curr) - alias = d['alias'] + alias = curr['alias'] if 'egress' in alias.lower(): prev[alias] = '#0000FF' else: @@ -754,27 +911,27 @@ def create_aggregate_panel(title, gridpos, targets, datasource): ingress_colors = reduce(reduce_alias, ingress_targets, {}) egress_colors = reduce(reduce_alias, egress_targets, {}) - ingress = create_panel({ + ingress = create_panel( **ingress_pos, - 'stack': True, - 'linewidth': 0 if is_total else 1, - 'datasource': datasource, - 'title': title + ' - ingress', - 'targets': ingress_targets, - 'y_axis_type': 'bits', - 'alias_colors': json.dumps(ingress_colors) if is_total else {} - }) + stack=True, + linewidth=0 if is_total else 1, + datasource=datasource, + title=title + " - ingress", + targets=ingress_targets, + y_axis_type="bits", + alias_colors=ingress_colors if is_total else {}, + ) - egress = create_panel({ + egress = create_panel( **egress_pos, - 'stack': True, - 'linewidth': 0 if is_total else 1, - 'datasource': datasource, - 'title': title + ' - egress', - 'targets': egress_targets, - 'y_axis_type': 'bits', - 'alias_colors': json.dumps(egress_colors) if is_total else {} - }) + stack=True, + linewidth=0 if is_total else 1, + datasource=datasource, + title=title + " - egress", + targets=egress_targets, + y_axis_type="bits", + alias_colors=egress_colors if is_total else {}, + ) return ingress, egress diff --git a/brian_dashboard_manager/templating/homedashboard.py b/brian_dashboard_manager/templating/homedashboard.py new file mode 100644 index 0000000000000000000000000000000000000000..d28e732e52f434c9672911917563c5a16ef69778 --- /dev/null +++ b/brian_dashboard_manager/templating/homedashboard.py @@ -0,0 +1,1154 @@ +def render_homedashboard(staff): + return { + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": True, + "hide": True, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard", + } + ] + }, + "editable": True, + "gnetId": None, + "graphTooltip": 0, + "id": 49, + "uid": "home", + "iteration": 1595947519970, + "links": _render_links(staff), + "panels": _render_panels(staff), + "schemaVersion": 26, + "style": "dark", + "tags": [], + "templating": { + "list": ( + [ + { + "allValue": None, + "datasource": "PollerInfluxDB", + "definition": "SHOW TAG VALUES WITH KEY=hostname", + "hide": 0, + "includeAll": False, + "label": "Router:", + "multi": False, + "name": "hostname", + "options": [], + "query": "SHOW TAG VALUES WITH KEY=hostname", + "refresh": 1, + "regex": "", + "skipUrlSync": False, + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": False, + }, + { + "allValue": None, + "datasource": "PollerInfluxDB", + "definition": "SHOW TAG VALUES WITH KEY IN (interface_name) WHERE hostname =~ /$hostname/ ", + "hide": 0, + "includeAll": False, + "label": "Interface :", + "multi": False, + "name": "interface_name", + "options": [], + "query": "SHOW TAG VALUES WITH KEY IN (interface_name) WHERE hostname =~ /$hostname/ ", + "refresh": 1, + "regex": "", + "skipUrlSync": False, + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": False, + }, + ] + if staff + else [] + ) + }, + "time": {"from": "now-6h", "to": "now"}, + "timepicker": { + "refresh_intervals": [ + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d", + ] + }, + "timezone": "", + "title": "Home", + "version": 1, + } + + +def _render_links(staff): + result = [ + { + "asDropdown": True, + "icon": "external link", + "tags": ["services"], + "targetBlank": True, + "title": "Services", + "type": "dashboards", + }, + { + "asDropdown": True, + "icon": "external link", + "tags": ["infrastructure"], + "targetBlank": True, + "title": "Infrastructure", + "type": "dashboards", + }, + { + "asDropdown": True, + "icon": "external link", + "tags": ["customers"], + "targetBlank": True, + "title": "NREN Access", + "type": "dashboards", + }, + ] + if staff: + result.append( + { + "asDropdown": True, + "icon": "external link", + "tags": ["customersbeta"], + "targetBlank": True, + "title": "NREN Access BETA", + "type": "dashboards", + } + ) + result.append( + { + "asDropdown": True, + "icon": "external link", + "tags": ["peers"], + "targetBlank": True, + "title": "Peers", + "type": "dashboards", + } + ) + return result + + +def _render_panels(staff): + if staff: + return [ + { + "aliasColors": {}, + "bars": False, + "dashLength": 10, + "dashes": False, + "datasource": "PollerInfluxDB", + "fieldConfig": {"defaults": {"custom": {}}, "overrides": []}, + "fill": 1, + "fillGradient": 3, + "gridPos": {"h": 14, "w": 12, "x": 0, "y": 0}, + "hiddenSeries": False, + "id": 2, + "legend": { + "alignAsTable": True, + "avg": True, + "current": True, + "max": True, + "min": False, + "rightSide": False, + "show": True, + "total": False, + "values": True, + }, + "lines": True, + "linewidth": 1, + "nullPointMode": "null", + "percentage": False, + "pluginVersion": "7.1.1", + "pointradius": 2, + "points": False, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": False, + "steppedLine": False, + "targets": [ + { + "alias": "Ingress Traffic", + "groupBy": [ + {"params": ["5m"], "type": "time"}, + {"params": ["linear"], "type": "fill"}, + ], + "measurement": "interface_rates", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + {"params": ["ingress"], "type": "field"}, + {"params": [], "type": "mean"}, + {"params": ["*8"], "type": "math"}, + ] + ], + "tags": [ + { + "key": "hostname", + "operator": "=~", + "value": "/^$hostname$/", + }, + { + "condition": "AND", + "key": "interface_name", + "operator": "=~", + "value": "/^$interface_name$/", + }, + ], + }, + { + "alias": "Egress Traffic", + "groupBy": [ + {"params": ["5m"], "type": "time"}, + {"params": ["linear"], "type": "fill"}, + ], + "measurement": "interface_rates", + "orderByTime": "ASC", + "policy": "default", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + {"params": ["egress"], "type": "field"}, + {"params": [], "type": "mean"}, + {"params": ["*8"], "type": "math"}, + ] + ], + "tags": [ + { + "key": "hostname", + "operator": "=~", + "value": "/^$hostname$/", + }, + { + "condition": "AND", + "key": "interface_name", + "operator": "=~", + "value": "/^$interface_name$/", + }, + ], + }, + { + "alias": "Ingress 95th Percentile", + "groupBy": [], + "measurement": "interface_rates", + "orderByTime": "ASC", + "policy": "default", + "refId": "C", + "resultFormat": "time_series", + "select": [ + [ + {"params": ["ingress"], "type": "field"}, + {"params": [95], "type": "percentile"}, + {"params": ["*8"], "type": "math"}, + ] + ], + "tags": [ + { + "condition": None, + "key": "hostname", + "operator": "=~", + "value": "/^$hostname$/", + }, + { + "condition": "AND", + "key": "interface_name", + "operator": "=~", + "value": "/^$interface_name$/", + }, + ], + }, + { + "alias": "Egress 95th Percentile", + "groupBy": [], + "measurement": "interface_rates", + "orderByTime": "ASC", + "policy": "default", + "refId": "D", + "resultFormat": "time_series", + "select": [ + [ + {"params": ["egress"], "type": "field"}, + {"params": [95], "type": "percentile"}, + {"params": ["*8"], "type": "math"}, + ] + ], + "tags": [ + { + "condition": None, + "key": "hostname", + "operator": "=~", + "value": "/^$hostname$/", + }, + { + "condition": "AND", + "key": "interface_name", + "operator": "=~", + "value": "/^$interface_name$/", + }, + ], + }, + ], + "thresholds": [], + "timeFrom": None, + "timeRegions": [], + "timeShift": None, + "title": "$hostname - $interface_name - Traffic", + "tooltip": {"shared": True, "sort": 0, "value_type": "individual"}, + "type": "graph", + "xaxis": { + "buckets": None, + "mode": "time", + "name": None, + "show": True, + "values": [], + }, + "yaxes": [ + { + "format": "bps", + "label": "bits per second", + "logBase": 1, + "max": None, + "min": None, + "show": True, + }, + { + "format": "bps", + "label": "bits per second", + "logBase": 1, + "max": None, + "min": None, + "show": True, + }, + ], + "yaxis": {"align": False, "alignLevel": None}, + }, + { + "aliasColors": {}, + "bars": False, + "dashLength": 10, + "dashes": False, + "datasource": "PollerInfluxDB", + "fieldConfig": {"defaults": {"custom": {}}, "overrides": []}, + "fill": 1, + "fillGradient": 3, + "gridPos": {"h": 14, "w": 12, "x": 12, "y": 0}, + "hiddenSeries": False, + "id": 3, + "legend": { + "alignAsTable": True, + "avg": True, + "current": True, + "max": True, + "min": False, + "rightSide": False, + "show": True, + "total": False, + "values": True, + }, + "lines": True, + "linewidth": 1, + "nullPointMode": "null", + "percentage": False, + "pluginVersion": "7.1.1", + "pointradius": 2, + "points": False, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": False, + "steppedLine": False, + "targets": [ + { + "alias": "Inbound", + "groupBy": [ + {"params": ["5m"], "type": "time"}, + {"params": ["linear"], "type": "fill"}, + ], + "measurement": "interface_rates", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + {"params": ["ingressv6"], "type": "field"}, + {"params": [], "type": "mean"}, + {"params": ["*8"], "type": "math"}, + ] + ], + "tags": [ + { + "key": "hostname", + "operator": "=~", + "value": "/^$hostname$/", + }, + { + "condition": "AND", + "key": "interface_name", + "operator": "=~", + "value": "/^$interface_name$/", + }, + ], + }, + { + "alias": "Outbound", + "groupBy": [ + {"params": ["5m"], "type": "time"}, + {"params": ["linear"], "type": "fill"}, + ], + "measurement": "interface_rates", + "orderByTime": "ASC", + "policy": "default", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + {"params": ["egressv6"], "type": "field"}, + {"params": [], "type": "mean"}, + {"params": ["*8"], "type": "math"}, + ] + ], + "tags": [ + { + "key": "hostname", + "operator": "=~", + "value": "/^$hostname$/", + }, + { + "condition": "AND", + "key": "interface_name", + "operator": "=~", + "value": "/^$interface_name$/", + }, + ], + }, + { + "alias": "Ingress 95th Percentile", + "groupBy": [], + "measurement": "interface_rates", + "orderByTime": "ASC", + "policy": "default", + "refId": "C", + "resultFormat": "time_series", + "select": [ + [ + {"params": ["ingressv6"], "type": "field"}, + {"params": [95], "type": "percentile"}, + {"params": ["*8"], "type": "math"}, + ] + ], + "tags": [ + { + "condition": None, + "key": "hostname", + "operator": "=~", + "value": "/^$hostname$/", + }, + { + "condition": "AND", + "key": "interface_name", + "operator": "=~", + "value": "/^$interface_name$/", + }, + ], + }, + { + "alias": "Egress 95th Percentile", + "groupBy": [], + "measurement": "interface_rates", + "orderByTime": "ASC", + "policy": "default", + "refId": "D", + "resultFormat": "time_series", + "select": [ + [ + {"params": ["egressv6"], "type": "field"}, + {"params": [95], "type": "percentile"}, + {"params": ["*8"], "type": "math"}, + ] + ], + "tags": [ + { + "condition": None, + "key": "hostname", + "operator": "=~", + "value": "/^$hostname$/", + }, + { + "condition": "AND", + "key": "interface_name", + "operator": "=~", + "value": "/^$interface_name$/", + }, + ], + }, + ], + "thresholds": [], + "timeFrom": None, + "timeRegions": [], + "timeShift": None, + "title": "$hostname - $interface_name - IPv6 Traffic", + "tooltip": {"shared": True, "sort": 0, "value_type": "individual"}, + "type": "graph", + "xaxis": { + "buckets": None, + "mode": "time", + "name": None, + "show": True, + "values": [], + }, + "yaxes": [ + { + "format": "bps", + "label": "bits per second", + "logBase": 1, + "max": None, + "min": None, + "show": True, + }, + { + "format": "bps", + "label": "bits per second", + "logBase": 1, + "max": None, + "min": None, + "show": True, + }, + ], + "yaxis": {"align": False, "alignLevel": None}, + }, + { + "aliasColors": {}, + "bars": False, + "dashLength": 10, + "dashes": False, + "datasource": "PollerInfluxDB", + "decimals": 2, + "fill": 1, + "fillGradient": 10, + "gridPos": {"h": 14, "w": 12, "x": 0, "y": 14}, + "hiddenSeries": False, + "id": 4, + "legend": { + "alignAsTable": True, + "avg": True, + "current": True, + "max": True, + "min": False, + "rightSide": False, + "show": True, + "total": False, + "values": True, + }, + "lines": True, + "linewidth": 1, + "nullPointMode": "null", + "options": {"alertThreshold": True}, + "percentage": False, + "pluginVersion": "8.2.5", + "pointradius": 2, + "points": False, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": False, + "steppedLine": False, + "targets": [ + { + "alias": "Ingress Errors", + "groupBy": [ + {"params": ["5m"], "type": "time"}, + {"params": ["linear"], "type": "fill"}, + ], + "measurement": "interface_rates", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + {"params": ["errorsIn"], "type": "field"}, + {"params": [], "type": "mean"}, + ] + ], + "tags": [ + { + "key": "hostname", + "operator": "=~", + "value": "/^$hostname$/", + }, + { + "condition": "AND", + "key": "interface_name", + "operator": "=~", + "value": "/^$interface_name$/", + }, + ], + }, + { + "alias": "Egress Errors", + "groupBy": [ + {"params": ["5m"], "type": "time"}, + {"params": ["linear"], "type": "fill"}, + ], + "measurement": "interface_rates", + "orderByTime": "ASC", + "policy": "default", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + {"params": ["errorsOut"], "type": "field"}, + {"params": [], "type": "mean"}, + ] + ], + "tags": [ + { + "key": "hostname", + "operator": "=~", + "value": "/^$hostname$/", + }, + { + "condition": "AND", + "key": "interface_name", + "operator": "=~", + "value": "/^$interface_name$/", + }, + ], + }, + { + "alias": "Ingress Discards", + "groupBy": [ + {"params": ["5m"], "type": "time"}, + {"params": ["linear"], "type": "fill"}, + ], + "measurement": "interface_rates", + "orderByTime": "ASC", + "policy": "default", + "refId": "C", + "resultFormat": "time_series", + "select": [ + [ + {"params": ["discardsIn"], "type": "field"}, + {"params": [], "type": "mean"}, + ] + ], + "tags": [ + { + "condition": None, + "key": "hostname", + "operator": "=~", + "value": "/^$hostname$/", + }, + { + "condition": "AND", + "key": "interface_name", + "operator": "=~", + "value": "/^$interface_name$/", + }, + ], + }, + { + "alias": "Egress Discards", + "groupBy": [ + {"params": ["5m"], "type": "time"}, + {"params": ["linear"], "type": "fill"}, + ], + "hide": False, + "measurement": "interface_rates", + "orderByTime": "ASC", + "policy": "default", + "refId": "D", + "resultFormat": "time_series", + "select": [ + [ + {"params": ["discardsOut"], "type": "field"}, + {"params": [], "type": "mean"}, + ] + ], + "tags": [ + { + "condition": None, + "key": "hostname", + "operator": "=~", + "value": "/^$hostname$/", + }, + { + "condition": "AND", + "key": "interface_name", + "operator": "=~", + "value": "/^$interface_name$/", + }, + ], + }, + ], + "thresholds": [], + "timeFrom": None, + "timeRegions": [], + "timeShift": None, + "title": "$hostname - $interface_name - errors", + "tooltip": {"shared": True, "sort": 0, "value_type": "individual"}, + "type": "graph", + "xaxis": { + "buckets": None, + "mode": "time", + "name": None, + "show": True, + "values": [], + }, + "yaxes": [ + { + "$$hashKey": "object:45", + "format": "none", + "label": "errors and discards per second", + "logBase": 1, + "max": None, + "min": "0", + "show": True, + }, + { + "$$hashKey": "object:46", + "format": "none", + "label": "errors and discards per second", + "logBase": 1, + "max": None, + "min": "0", + "show": True, + }, + ], + "yaxis": {"align": False, "alignLevel": None}, + }, + ] + return [ + { + "aliasColors": {}, + "bars": False, + "dashLength": 10, + "dashes": False, + "datasource": None, + "fieldConfig": {"defaults": {"custom": {}}, "overrides": []}, + "fill": 1, + "fillGradient": 0, + "gridPos": {"h": 8, "w": 12, "x": 0, "y": 0}, + "hiddenSeries": False, + "id": 6, + "legend": { + "avg": False, + "current": False, + "max": False, + "min": False, + "show": True, + "total": False, + "values": False, + }, + "lines": True, + "linewidth": 1, + "nullPointMode": "null", + "options": {"alertThreshold": True}, + "percentage": False, + "pluginVersion": "7.2.1", + "pointradius": 2, + "points": False, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": True, + "steppedLine": False, + "targets": [ + { + "alias": "Private", + "groupBy": [ + {"params": ["15m"], "type": "time"}, + {"params": ["linear"], "type": "fill"}, + ], + "measurement": "interface_rates", + "orderByTime": "ASC", + "policy": "default", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + {"params": ["ingress"], "type": "field"}, + {"params": [], "type": "mean"}, + {"params": ["*8"], "type": "math"}, + ] + ], + "tags": [ + { + "key": "interface_name", + "operator": "=", + "value": "PHY_PRIVATE", + } + ], + }, + { + "alias": "R&E Interconnect", + "groupBy": [ + {"params": ["15m"], "type": "time"}, + {"params": ["null"], "type": "fill"}, + ], + "measurement": "interface_rates", + "orderByTime": "ASC", + "policy": "default", + "refId": "C", + "resultFormat": "time_series", + "select": [ + [ + {"params": ["ingress"], "type": "field"}, + {"params": [], "type": "mean"}, + {"params": ["* 8"], "type": "math"}, + ] + ], + "tags": [ + { + "key": "interface_name", + "operator": "=", + "value": "PHY_RE_INTERCONNECT", + } + ], + }, + { + "alias": "Public", + "groupBy": [ + {"params": ["15m"], "type": "time"}, + {"params": ["null"], "type": "fill"}, + ], + "measurement": "interface_rates", + "orderByTime": "ASC", + "policy": "default", + "refId": "D", + "resultFormat": "time_series", + "select": [ + [ + {"params": ["ingress"], "type": "field"}, + {"params": [], "type": "mean"}, + {"params": ["* 8"], "type": "math"}, + ] + ], + "tags": [ + { + "key": "interface_name", + "operator": "=", + "value": "PHY_PUBLIC", + } + ], + }, + { + "alias": "Upstream", + "groupBy": [ + {"params": ["15m"], "type": "time"}, + {"params": ["null"], "type": "fill"}, + ], + "measurement": "interface_rates", + "orderByTime": "ASC", + "policy": "default", + "refId": "E", + "resultFormat": "time_series", + "select": [ + [ + {"params": ["ingress"], "type": "field"}, + {"params": [], "type": "mean"}, + {"params": ["* 8"], "type": "math"}, + ] + ], + "tags": [ + { + "key": "interface_name", + "operator": "=", + "value": "PHY_UPSTREAM", + } + ], + }, + { + "alias": "Customer", + "groupBy": [ + {"params": ["15m"], "type": "time"}, + {"params": ["linear"], "type": "fill"}, + ], + "measurement": "interface_rates", + "orderByTime": "ASC", + "policy": "default", + "query": ( + 'SELECT mean("ingress") *8 FROM "interface_rates" WHERE ("interface_name" =' + " 'PHY_CUSTOMER') AND $timeFilter GROUP BY time($__interval) fill(linear)" + ), + "rawQuery": False, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + {"params": ["ingress"], "type": "field"}, + {"params": [], "type": "mean"}, + {"params": ["*8"], "type": "math"}, + ] + ], + "tags": [ + { + "key": "interface_name", + "operator": "=", + "value": "PHY_CUSTOMER", + } + ], + }, + ], + "thresholds": [], + "timeFrom": None, + "timeRegions": [], + "timeShift": None, + "title": "Network Aggregate (Ingress)", + "tooltip": {"shared": True, "sort": 0, "value_type": "individual"}, + "type": "graph", + "xaxis": { + "buckets": None, + "mode": "time", + "name": None, + "show": True, + "values": [], + }, + "yaxes": [ + { + "format": "bps", + "label": None, + "logBase": 1, + "max": None, + "min": None, + "show": True, + }, + { + "format": "short", + "label": None, + "logBase": 1, + "max": None, + "min": None, + "show": True, + }, + ], + "yaxis": {"align": False, "alignLevel": None}, + }, + { + "aliasColors": {}, + "bars": False, + "dashLength": 10, + "dashes": False, + "datasource": None, + "fieldConfig": {"defaults": {"custom": {}}, "overrides": []}, + "fill": 1, + "fillGradient": 0, + "gridPos": {"h": 8, "w": 12, "x": 12, "y": 0}, + "hiddenSeries": False, + "id": 7, + "legend": { + "avg": False, + "current": False, + "max": False, + "min": False, + "show": True, + "total": False, + "values": False, + }, + "lines": True, + "linewidth": 1, + "nullPointMode": "null", + "options": {"alertThreshold": True}, + "percentage": False, + "pluginVersion": "7.2.1", + "pointradius": 2, + "points": False, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": True, + "steppedLine": False, + "targets": [ + { + "alias": "Private", + "groupBy": [ + {"params": ["15m"], "type": "time"}, + {"params": ["linear"], "type": "fill"}, + ], + "measurement": "interface_rates", + "orderByTime": "ASC", + "policy": "default", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + {"params": ["egress"], "type": "field"}, + {"params": [], "type": "mean"}, + {"params": ["*8"], "type": "math"}, + ] + ], + "tags": [ + { + "key": "interface_name", + "operator": "=", + "value": "PHY_PRIVATE", + } + ], + }, + { + "alias": "R&E Interconnect", + "groupBy": [ + {"params": ["15m"], "type": "time"}, + {"params": ["null"], "type": "fill"}, + ], + "measurement": "interface_rates", + "orderByTime": "ASC", + "policy": "default", + "refId": "C", + "resultFormat": "time_series", + "select": [ + [ + {"params": ["egress"], "type": "field"}, + {"params": [], "type": "mean"}, + {"params": ["* 8"], "type": "math"}, + ] + ], + "tags": [ + { + "key": "interface_name", + "operator": "=", + "value": "PHY_RE_INTERCONNECT", + } + ], + }, + { + "alias": "Public", + "groupBy": [ + {"params": ["15m"], "type": "time"}, + {"params": ["null"], "type": "fill"}, + ], + "measurement": "interface_rates", + "orderByTime": "ASC", + "policy": "default", + "refId": "D", + "resultFormat": "time_series", + "select": [ + [ + {"params": ["egress"], "type": "field"}, + {"params": [], "type": "mean"}, + {"params": ["* 8"], "type": "math"}, + ] + ], + "tags": [ + { + "key": "interface_name", + "operator": "=", + "value": "PHY_PUBLIC", + } + ], + }, + { + "alias": "Upstream", + "groupBy": [ + {"params": ["15m"], "type": "time"}, + {"params": ["null"], "type": "fill"}, + ], + "measurement": "interface_rates", + "orderByTime": "ASC", + "policy": "default", + "refId": "E", + "resultFormat": "time_series", + "select": [ + [ + {"params": ["egress"], "type": "field"}, + {"params": [], "type": "mean"}, + {"params": ["* 8"], "type": "math"}, + ] + ], + "tags": [ + { + "key": "interface_name", + "operator": "=", + "value": "PHY_UPSTREAM", + } + ], + }, + { + "alias": "Customer", + "groupBy": [ + {"params": ["15m"], "type": "time"}, + {"params": ["linear"], "type": "fill"}, + ], + "measurement": "interface_rates", + "orderByTime": "ASC", + "policy": "default", + "query": ( + 'SELECT mean("ingress") *8 FROM "interface_rates" WHERE ("interface_name" =' + " 'PHY_CUSTOMER') AND $timeFilter GROUP BY time($__interval) fill(linear)" + ), + "rawQuery": False, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + {"params": ["egress"], "type": "field"}, + {"params": [], "type": "mean"}, + {"params": ["*8"], "type": "math"}, + ] + ], + "tags": [ + { + "key": "interface_name", + "operator": "=", + "value": "PHY_CUSTOMER", + } + ], + }, + ], + "thresholds": [], + "timeFrom": None, + "timeRegions": [], + "timeShift": None, + "title": "Network Aggregate (Egress)", + "tooltip": {"shared": True, "sort": 0, "value_type": "individual"}, + "type": "graph", + "xaxis": { + "buckets": None, + "mode": "time", + "name": None, + "show": True, + "values": [], + }, + "yaxes": [ + { + "format": "bps", + "label": None, + "logBase": 1, + "max": None, + "min": None, + "show": True, + }, + { + "format": "short", + "label": None, + "logBase": 1, + "max": None, + "min": None, + "show": True, + }, + ], + "yaxis": {"align": False, "alignLevel": None}, + }, + ] diff --git a/brian_dashboard_manager/templating/render.py b/brian_dashboard_manager/templating/render.py index 1885f3ed488452de1f580aa63e4bc5de66f145a7..19b4cdfab4d7ac3d84fb0de47ad6d1d135f598ea 100644 --- a/brian_dashboard_manager/templating/render.py +++ b/brian_dashboard_manager/templating/render.py @@ -1,80 +1,35 @@ -""" -Methods for rendering of the -various Jinja templates from the given data. -""" -import os -import json -import jinja2 - - -def _read_template(filename): - """ - Reads the template from the given filename. - - :param filename: path to the template file - - :return: template - """ - with open(filename) as f: - return jinja2.Template(f.read()) - - -dropdown_template_file = os.path.abspath(os.path.join( - os.path.dirname(__file__), - 'templates', - 'shared', - 'dropdown.json.j2')) - -yaxes_template_file = os.path.abspath(os.path.join( - os.path.dirname(__file__), - 'templates', - 'shared', - 'yaxes.json.j2')) - -panel_template_file = file = os.path.abspath(os.path.join( - os.path.dirname(__file__), - 'templates', - 'shared', - 'panel.json.j2')) - -panel_target_template_file = os.path.abspath(os.path.join( - os.path.dirname(__file__), - 'templates', - 'shared', - 'panel_target.json.j2')) - -nren_dashboard_template_file = os.path.abspath(os.path.join( - os.path.dirname(__file__), - 'templates', - 'nren_access', - 'nren-dashboard.json.j2')) - -dashboard_template_file = os.path.abspath(os.path.join( - os.path.dirname(__file__), - 'templates', - 'shared', - 'dashboard.json.j2')) - - -DROPDOWN_TEMPLATE = _read_template(dropdown_template_file) -YAXES_TEMPLATE = _read_template(yaxes_template_file) -PANEL_TEMPLATE = _read_template(panel_template_file) -PANEL_TARGET_TEMPLATE = _read_template(panel_target_template_file) -NREN_DASHBOARD_TEMPLATE = _read_template(nren_dashboard_template_file) -DASHBOARD_TEMPLATE = _read_template(dashboard_template_file) - - -def create_dropdown_panel(title, **kwargs): +def create_dropdown_panel(title, id, y, **kwargs): """ Creates a dropdown panel from the given data. :param title: title of the dropdown panel + :param id: id of the dropdown panel + :param y: y of the dropdown panel :param kwargs: data to be used in the template :return: rendered dropdown panel JSON """ - - return DROPDOWN_TEMPLATE.render({**kwargs, 'title': title}) + return { + "aliasColors": {}, + "collapsed": False, + "datasource": None, + "fill": None, + "fillGradient": None, + "gridPos": {"h": 1, "w": 24, "x": 0, "y": y}, + "id": id, + "legend": None, + "lines": None, + "linewidth": None, + "search": None, + "stack": None, + "tags": None, + "targets": None, + "title": title, + "type": "row", + "xaxis": None, + "yaxes": None, + "yaxis": None, + } def create_yaxes(type): @@ -85,11 +40,62 @@ def create_yaxes(type): :return: rendered yaxes JSON """ - - return YAXES_TEMPLATE.render({'type': type}) - - -def create_panel_target(data): + if type == "errors": + return [ + { + "format": "none", + "label": "errors and discards per second", + "logBase": 1, + "max": None, + "min": 0, + "show": True, + }, + { + "format": "none", + "label": "errors and discards per second", + "logBase": 1, + "max": None, + "min": 0, + "show": True, + }, + ] + else: + return [ + { + "format": "bps", + "label": "bits per second", + "logBase": 1, + "max": None, + "min": "", + "show": True, + }, + { + "format": "bps", + "label": "bits per second", + "logBase": 1, + "max": None, + "min": "", + "show": True, + }, + ] + + +def create_panel_target( + alias, + select_field, + refId, + percentile=False, + measurement="interface_rates", + errors=False, + isp=None, + subscription=None, + scid=None, + interface_tag=None, + nren=None, + hostname=None, + interface=None, + **_ +): """ Creates a panel target from the given data. A panel target defines how to query data for a single timeseries. @@ -98,11 +104,85 @@ def create_panel_target(data): :return: rendered panel target JSON """ - - return PANEL_TARGET_TEMPLATE.render(data) - - -def create_panel(data): + select = [{"params": [select_field], "type": "field"}] + select.append( + {"params": [95], "type": "percentile"} + if percentile + else {"params": [], "type": "max"} + ) + if not errors: + select.append({"params": ["*8"], "type": "math"}) + if isp: + tags = [ + {"condition": None, "key": "tag", "operator": "=", "value": interface_tag}, + {"condition": "AND", "key": "isp", "operator": "=", "value": isp}, + {"condition": "AND", "key": "nren", "operator": "=", "value": nren}, + ] + elif subscription: + tags = [ + {"condition": None, "key": "hostname", "operator": "=", "value": hostname}, + { + "condition": "AND", + "key": "subscription", + "operator": "=", + "value": subscription, + }, + ] + elif scid: + tags = [{"condition": None, "key": "scid", "operator": "=", "value": scid}] + else: + tags = [ + { + "condition": None, + "key": "hostname", + "operator": "=", + "value": hostname, + }, + { + "condition": "AND", + "key": "interface_name", + "operator": "=", + "value": interface, + }, + ] + result = { + "alias": alias, + "groupBy": ( + [] + if percentile + else [ + {"params": ["$__interval"], "type": "time"}, + {"params": ["null"], "type": "fill"}, + ] + ), + "measurement": measurement, + "orderByTime": None, + "policy": None, + "refId": refId, + "resultFormat": "time_series", + "select": [select], + "tags": tags, + } + return result + + +def create_panel( + title, + height, + width, + linewidth, + y, + id, + datasource, + x=0, + alias_colors=None, + disable_legend=False, + stack=False, + y_axis_type="bits", + targets=None, + panel_targets=None, + **_ +): """ Creates a panel from the given data. Constructs the yaxes and panel targets and renders the panel template using these. @@ -111,33 +191,162 @@ def create_panel(data): :return: rendered panel JSON """ - - yaxes = create_yaxes(data.get('y_axis_type', 'bits')) - targets = data.get('targets', []) - for target in data.get('panel_targets', []): - targets.append(create_panel_target(target)) - return PANEL_TEMPLATE.render({**data, 'yaxes': yaxes, 'targets': targets}) - - -def render_dashboard(dashboard, nren=False): - """ - Renders the dashboard template using the given data. - NREN dashboards are rendered using a different template that uses - a different layout than other dashboards. - - :param dashboard: data to be used in the template - :param nren: whether the dashboard is an NREN dashboard - - :return: rendered dashboard JSON - """ - - if nren: - template = NREN_DASHBOARD_TEMPLATE - else: - template = DASHBOARD_TEMPLATE - - rendered = template.render(dashboard) - rendered = json.loads(rendered) - rendered['uid'] = None - rendered['id'] = None - return rendered + yaxes = create_yaxes(y_axis_type) + + result = { + "aliasColors": alias_colors or {}, + "bars": False, + "collapsed": None, + "dashLength": 10, + "dashes": False, + "datasource": datasource, + "decimals": 2, + "fieldConfig": {"defaults": {"custom": {}}, "overrides": []}, + "fill": 1, + "fillGradient": 10, + "gridPos": {"h": height, "w": width, "x": x, "y": y}, + "hiddenSeries": False, + "id": id, + "lines": True, + "linewidth": linewidth, + "nullPointMode": "null", + "options": {"alertThreshold": True}, + "percentage": False, + "pointradius": 2, + "points": False, + "renderer": "flot", + "search": None, + "seriesOverrides": [], + "spaceLength": 10, + "stack": stack, + "steppedLine": False, + "tags": None, + "thresholds": [], + "timeFrom": None, + "timeRegions": [], + "timeShift": None, + "title": title, + "tooltip": {"shared": True, "sort": 0, "value_type": "individual"}, + "type": "graph", + "xaxis": { + "buckets": None, + "mode": "time", + "name": None, + "show": True, + "values": None, + }, + "yaxes": yaxes, + "yaxis": {"align": False, "alignLevel": None}, + "targets": [], + } + if not disable_legend: + result["legend"] = { + "alignAsTable": True, + "avg": True, + "current": True, + "max": True, + "min": False, + "rightSide": None, + "show": True, + "total": False, + "values": True, + } + + targets = targets or [] + for target in panel_targets or []: + targets.append(create_panel_target(**target)) + result["targets"] = targets + return result + + +def create_infobox(): + return { + "datasource": None, + "gridPos": {"h": 1, "w": 24, "x": 0, "y": 0}, + "id": 1, + "options": {"content": "", "mode": "html"}, + "pluginVersion": "8.2.5", + "title": "INFO: The average values displayed are only mean values for timescales of 2 days or less", + "type": "text", + } + + +def render_complex_dashboard( + nren_name, aggregate_panels, dropdown_groups, tag=None, tags=None, **_ +): + assert tag or tags + panels = [create_infobox()] + panels.extend(aggregate_panels) + for group in dropdown_groups: + panels.append(group["dropdown"]) + panels.extend(group["panels"]) + + return { + "id": None, + "uid": None, + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": True, + "hide": True, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard", + } + ] + }, + "editable": False, + "gnetId": None, + "graphTooltip": 0, + "schemaVersion": 27, + "style": "dark", + "tags": tags or [tag], + "templating": {"list": []}, + "time": {"from": "now-24h", "to": "now"}, + "timepicker": {}, + "timezone": "", + "title": nren_name, + "version": 1, + "links": [], + "panels": panels, + } + + +def render_simple_dashboard(title, tag=None, tags=None, panels=None, **_): + assert tag or tags + return { + "id": None, + "uid": None, + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": True, + "hide": True, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard", + } + ] + }, + "editable": False, + "gnetId": None, + "graphTooltip": 0, + "schemaVersion": 27, + "style": "dark", + "tags": tags or [tag], + "templating": {"list": []}, + "time": {"from": "now-24h", "to": "now"}, + "timepicker": {}, + "timezone": "", + "title": title, + "version": 1, + "links": [], + "panels": [ + create_infobox(), + *(panels or []), + ], + } diff --git a/brian_dashboard_manager/templating/templates/homedashboard.json.j2 b/brian_dashboard_manager/templating/templates/homedashboard.json.j2 deleted file mode 100644 index d29f738a29a70e797cf521ce556be9a88f0a2e76..0000000000000000000000000000000000000000 --- a/brian_dashboard_manager/templating/templates/homedashboard.json.j2 +++ /dev/null @@ -1,1686 +0,0 @@ -{ - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": "-- Grafana --", - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": true, - "gnetId": null, - "graphTooltip": 0, - "id": 49, - "uid": "home", - "iteration": 1595947519970, - "links": [ - { - "asDropdown": true, - "icon": "external link", - "tags": [ - "services" - ], - "targetBlank": true, - "title": "Services", - "type": "dashboards" - }, - { - "asDropdown": true, - "icon": "external link", - "tags": [ - "infrastructure" - ], - "targetBlank": true, - "title": "Infrastructure", - "type": "dashboards" - }, - { - "asDropdown": true, - "icon": "external link", - "tags": [ - "customers" - ], - "targetBlank": true, - "title": "NREN Access", - "type": "dashboards" - }, - {% if staff %} - { - "asDropdown": true, - "icon": "external link", - "tags": [ - "customersbeta" - ], - "targetBlank": true, - "title": "NREN Access BETA", - "type": "dashboards" - }, - {% endif %} - { - "asDropdown": true, - "icon": "external link", - "tags": [ - "peers" - ], - "targetBlank": true, - "title": "Peers", - "type": "dashboards" - } - ], - "panels": [ - {% if staff %} - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "PollerInfluxDB", - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 3, - "gridPos": { - "h": 14, - "w": 12, - "x": 0, - "y": 0 - }, - "hiddenSeries": false, - "id": 2, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "max": true, - "min": false, - "rightSide": false, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "percentage": false, - "pluginVersion": "7.1.1", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "Ingress Traffic", - "groupBy": [ - { - "params": [ - "5m" - ], - "type": "time" - }, - { - "params": [ - "linear" - ], - "type": "fill" - } - ], - "measurement": "interface_rates", - "orderByTime": "ASC", - "policy": "default", - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "ingress" - ], - "type": "field" - }, - { - "params": [], - "type": "mean" - }, - { - "params": [ - "*8" - ], - "type": "math" - } - ] - ], - "tags": [ - { - "key": "hostname", - "operator": "=~", - "value": "/^$hostname$/" - }, - { - "condition": "AND", - "key": "interface_name", - "operator": "=~", - "value": "/^$interface_name$/" - } - ] - }, - { - "alias": "Egress Traffic", - "groupBy": [ - { - "params": [ - "5m" - ], - "type": "time" - }, - { - "params": [ - "linear" - ], - "type": "fill" - } - ], - "measurement": "interface_rates", - "orderByTime": "ASC", - "policy": "default", - "refId": "B", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "egress" - ], - "type": "field" - }, - { - "params": [], - "type": "mean" - }, - { - "params": [ - "*8" - ], - "type": "math" - } - ] - ], - "tags": [ - { - "key": "hostname", - "operator": "=~", - "value": "/^$hostname$/" - }, - { - "condition": "AND", - "key": "interface_name", - "operator": "=~", - "value": "/^$interface_name$/" - } - ] - }, - { - "alias": "Ingress 95th Percentile", - "groupBy": [], - "measurement": "interface_rates", - "orderByTime": "ASC", - "policy": "default", - "refId": "C", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "ingress" - ], - "type": "field" - }, - { - "params": [ - 95 - ], - "type": "percentile" - }, - { - "params": [ - "*8" - ], - "type": "math" - } - ] - ], - "tags": [ - { - "condition": null, - "key": "hostname", - "operator": "=~", - "value": "/^$hostname$/" - }, - { - "condition": "AND", - "key": "interface_name", - "operator": "=~", - "value": "/^$interface_name$/" - } - ] - }, - { - "alias": "Egress 95th Percentile", - "groupBy": [], - "measurement": "interface_rates", - "orderByTime": "ASC", - "policy": "default", - "refId": "D", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "egress" - ], - "type": "field" - }, - { - "params": [ - 95 - ], - "type": "percentile" - }, - { - "params": [ - "*8" - ], - "type": "math" - } - ] - ], - "tags": [ - { - "condition": null, - "key": "hostname", - "operator": "=~", - "value": "/^$hostname$/" - }, - { - "condition": "AND", - "key": "interface_name", - "operator": "=~", - "value": "/^$interface_name$/" - } - ] - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "$hostname - $interface_name - Traffic", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "bps", - "label": "bits per second", - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "bps", - "label": "bits per second", - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "PollerInfluxDB", - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 3, - "gridPos": { - "h": 14, - "w": 12, - "x": 12, - "y": 0 - }, - "hiddenSeries": false, - "id": 3, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "max": true, - "min": false, - "rightSide": false, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "percentage": false, - "pluginVersion": "7.1.1", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "Inbound", - "groupBy": [ - { - "params": [ - "5m" - ], - "type": "time" - }, - { - "params": [ - "linear" - ], - "type": "fill" - } - ], - "measurement": "interface_rates", - "orderByTime": "ASC", - "policy": "default", - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "ingressv6" - ], - "type": "field" - }, - { - "params": [], - "type": "mean" - }, - { - "params": [ - "*8" - ], - "type": "math" - } - ] - ], - "tags": [ - { - "key": "hostname", - "operator": "=~", - "value": "/^$hostname$/" - }, - { - "condition": "AND", - "key": "interface_name", - "operator": "=~", - "value": "/^$interface_name$/" - } - ] - }, - { - "alias": "Outbound", - "groupBy": [ - { - "params": [ - "5m" - ], - "type": "time" - }, - { - "params": [ - "linear" - ], - "type": "fill" - } - ], - "measurement": "interface_rates", - "orderByTime": "ASC", - "policy": "default", - "refId": "B", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "egressv6" - ], - "type": "field" - }, - { - "params": [], - "type": "mean" - }, - { - "params": [ - "*8" - ], - "type": "math" - } - ] - ], - "tags": [ - { - "key": "hostname", - "operator": "=~", - "value": "/^$hostname$/" - }, - { - "condition": "AND", - "key": "interface_name", - "operator": "=~", - "value": "/^$interface_name$/" - } - ] - }, - { - "alias": "Ingress 95th Percentile", - "groupBy": [], - "measurement": "interface_rates", - "orderByTime": "ASC", - "policy": "default", - "refId": "C", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "ingressv6" - ], - "type": "field" - }, - { - "params": [ - 95 - ], - "type": "percentile" - }, - { - "params": [ - "*8" - ], - "type": "math" - } - ] - ], - "tags": [ - { - "condition": null, - "key": "hostname", - "operator": "=~", - "value": "/^$hostname$/" - }, - { - "condition": "AND", - "key": "interface_name", - "operator": "=~", - "value": "/^$interface_name$/" - } - ] - }, - { - "alias": "Egress 95th Percentile", - "groupBy": [], - "measurement": "interface_rates", - "orderByTime": "ASC", - "policy": "default", - "refId": "D", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "egressv6" - ], - "type": "field" - }, - { - "params": [ - 95 - ], - "type": "percentile" - }, - { - "params": [ - "*8" - ], - "type": "math" - } - ] - ], - "tags": [ - { - "condition": null, - "key": "hostname", - "operator": "=~", - "value": "/^$hostname$/" - }, - { - "condition": "AND", - "key": "interface_name", - "operator": "=~", - "value": "/^$interface_name$/" - } - ] - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "$hostname - $interface_name - IPv6 Traffic", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "bps", - "label": "bits per second", - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "bps", - "label": "bits per second", - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "PollerInfluxDB", - "decimals": 2, - "fill": 1, - "fillGradient": 10, - "gridPos": { - "h": 14, - "w": 12, - "x": 0, - "y": 14 - }, - "hiddenSeries": false, - "id": 4, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "max": true, - "min": false, - "rightSide": false, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "8.2.5", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "Ingress Errors", - "groupBy": [ - { - "params": [ - "5m" - ], - "type": "time" - }, - { - "params": [ - "linear" - ], - "type": "fill" - } - ], - "measurement": "interface_rates", - "orderByTime": "ASC", - "policy": "default", - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "errorsIn" - ], - "type": "field" - }, - { - "params": [], - "type": "mean" - } - ] - ], - "tags": [ - { - "key": "hostname", - "operator": "=~", - "value": "/^$hostname$/" - }, - { - "condition": "AND", - "key": "interface_name", - "operator": "=~", - "value": "/^$interface_name$/" - } - ] - }, - { - "alias": "Egress Errors", - "groupBy": [ - { - "params": [ - "5m" - ], - "type": "time" - }, - { - "params": [ - "linear" - ], - "type": "fill" - } - ], - "measurement": "interface_rates", - "orderByTime": "ASC", - "policy": "default", - "refId": "B", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "errorsOut" - ], - "type": "field" - }, - { - "params": [], - "type": "mean" - } - ] - ], - "tags": [ - { - "key": "hostname", - "operator": "=~", - "value": "/^$hostname$/" - }, - { - "condition": "AND", - "key": "interface_name", - "operator": "=~", - "value": "/^$interface_name$/" - } - ] - }, - { - "alias": "Ingress Discards", - "groupBy": [ - { - "params": [ - "5m" - ], - "type": "time" - }, - { - "params": [ - "linear" - ], - "type": "fill" - } - ], - "measurement": "interface_rates", - "orderByTime": "ASC", - "policy": "default", - "refId": "C", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "discardsIn" - ], - "type": "field" - }, - { - "params": [], - "type": "mean" - } - ] - ], - "tags": [ - { - "condition": null, - "key": "hostname", - "operator": "=~", - "value": "/^$hostname$/" - }, - { - "condition": "AND", - "key": "interface_name", - "operator": "=~", - "value": "/^$interface_name$/" - } - ] - }, - { - "alias": "Egress Discards", - "groupBy": [ - { - "params": [ - "5m" - ], - "type": "time" - }, - { - "params": [ - "linear" - ], - "type": "fill" - } - ], - "hide": false, - "measurement": "interface_rates", - "orderByTime": "ASC", - "policy": "default", - "refId": "D", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "discardsOut" - ], - "type": "field" - }, - { - "params": [], - "type": "mean" - } - ] - ], - "tags": [ - { - "condition": null, - "key": "hostname", - "operator": "=~", - "value": "/^$hostname$/" - }, - { - "condition": "AND", - "key": "interface_name", - "operator": "=~", - "value": "/^$interface_name$/" - } - ] - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "$hostname - $interface_name - errors", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:45", - "format": "none", - "label": "errors and discards per second", - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "$$hashKey": "object:46", - "format": "none", - "label": "errors and discards per second", - "logBase": 1, - "max": null, - "min": "0", - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - } - {% else %} - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 0 - }, - "hiddenSeries": false, - "id": 6, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.2.1", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": true, - "steppedLine": false, - "targets": [ - { - "alias": "Private", - "groupBy": [ - { - "params": [ - "15m" - ], - "type": "time" - }, - { - "params": [ - "linear" - ], - "type": "fill" - } - ], - "measurement": "interface_rates", - "orderByTime": "ASC", - "policy": "default", - "refId": "B", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "ingress" - ], - "type": "field" - }, - { - "params": [], - "type": "mean" - }, - { - "params": [ - "*8" - ], - "type": "math" - } - ] - ], - "tags": [ - { - "key": "interface_name", - "operator": "=", - "value": "PHY_PRIVATE" - } - ] - }, - { - "alias": "R&E Interconnect", - "groupBy": [ - { - "params": [ - "15m" - ], - "type": "time" - }, - { - "params": [ - "null" - ], - "type": "fill" - } - ], - "measurement": "interface_rates", - "orderByTime": "ASC", - "policy": "default", - "refId": "C", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "ingress" - ], - "type": "field" - }, - { - "params": [], - "type": "mean" - }, - { - "params": [ - "* 8" - ], - "type": "math" - } - ] - ], - "tags": [ - { - "key": "interface_name", - "operator": "=", - "value": "PHY_RE_INTERCONNECT" - } - ] - }, - { - "alias": "Public", - "groupBy": [ - { - "params": [ - "15m" - ], - "type": "time" - }, - { - "params": [ - "null" - ], - "type": "fill" - } - ], - "measurement": "interface_rates", - "orderByTime": "ASC", - "policy": "default", - "refId": "D", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "ingress" - ], - "type": "field" - }, - { - "params": [], - "type": "mean" - }, - { - "params": [ - "* 8" - ], - "type": "math" - } - ] - ], - "tags": [ - { - "key": "interface_name", - "operator": "=", - "value": "PHY_PUBLIC" - } - ] - }, - { - "alias": "Upstream", - "groupBy": [ - { - "params": [ - "15m" - ], - "type": "time" - }, - { - "params": [ - "null" - ], - "type": "fill" - } - ], - "measurement": "interface_rates", - "orderByTime": "ASC", - "policy": "default", - "refId": "E", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "ingress" - ], - "type": "field" - }, - { - "params": [], - "type": "mean" - }, - { - "params": [ - "* 8" - ], - "type": "math" - } - ] - ], - "tags": [ - { - "key": "interface_name", - "operator": "=", - "value": "PHY_UPSTREAM" - } - ] - }, - { - "alias": "Customer", - "groupBy": [ - { - "params": [ - "15m" - ], - "type": "time" - }, - { - "params": [ - "linear" - ], - "type": "fill" - } - ], - "measurement": "interface_rates", - "orderByTime": "ASC", - "policy": "default", - "query": "SELECT mean(\"ingress\") *8 FROM \"interface_rates\" WHERE (\"interface_name\" = 'PHY_CUSTOMER') AND $timeFilter GROUP BY time($__interval) fill(linear)", - "rawQuery": false, - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "ingress" - ], - "type": "field" - }, - { - "params": [], - "type": "mean" - }, - { - "params": [ - "*8" - ], - "type": "math" - } - ] - ], - "tags": [ - { - "key": "interface_name", - "operator": "=", - "value": "PHY_CUSTOMER" - } - ] - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Network Aggregate (Ingress)", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "bps", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 0 - }, - "hiddenSeries": false, - "id": 7, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.2.1", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": true, - "steppedLine": false, - "targets": [ - { - "alias": "Private", - "groupBy": [ - { - "params": [ - "15m" - ], - "type": "time" - }, - { - "params": [ - "linear" - ], - "type": "fill" - } - ], - "measurement": "interface_rates", - "orderByTime": "ASC", - "policy": "default", - "refId": "B", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "egress" - ], - "type": "field" - }, - { - "params": [], - "type": "mean" - }, - { - "params": [ - "*8" - ], - "type": "math" - } - ] - ], - "tags": [ - { - "key": "interface_name", - "operator": "=", - "value": "PHY_PRIVATE" - } - ] - }, - { - "alias": "R&E Interconnect", - "groupBy": [ - { - "params": [ - "15m" - ], - "type": "time" - }, - { - "params": [ - "null" - ], - "type": "fill" - } - ], - "measurement": "interface_rates", - "orderByTime": "ASC", - "policy": "default", - "refId": "C", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "egress" - ], - "type": "field" - }, - { - "params": [], - "type": "mean" - }, - { - "params": [ - "* 8" - ], - "type": "math" - } - ] - ], - "tags": [ - { - "key": "interface_name", - "operator": "=", - "value": "PHY_RE_INTERCONNECT" - } - ] - }, - { - "alias": "Public", - "groupBy": [ - { - "params": [ - "15m" - ], - "type": "time" - }, - { - "params": [ - "null" - ], - "type": "fill" - } - ], - "measurement": "interface_rates", - "orderByTime": "ASC", - "policy": "default", - "refId": "D", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "egress" - ], - "type": "field" - }, - { - "params": [], - "type": "mean" - }, - { - "params": [ - "* 8" - ], - "type": "math" - } - ] - ], - "tags": [ - { - "key": "interface_name", - "operator": "=", - "value": "PHY_PUBLIC" - } - ] - }, - { - "alias": "Upstream", - "groupBy": [ - { - "params": [ - "15m" - ], - "type": "time" - }, - { - "params": [ - "null" - ], - "type": "fill" - } - ], - "measurement": "interface_rates", - "orderByTime": "ASC", - "policy": "default", - "refId": "E", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "egress" - ], - "type": "field" - }, - { - "params": [], - "type": "mean" - }, - { - "params": [ - "* 8" - ], - "type": "math" - } - ] - ], - "tags": [ - { - "key": "interface_name", - "operator": "=", - "value": "PHY_UPSTREAM" - } - ] - }, - { - "alias": "Customer", - "groupBy": [ - { - "params": [ - "15m" - ], - "type": "time" - }, - { - "params": [ - "linear" - ], - "type": "fill" - } - ], - "measurement": "interface_rates", - "orderByTime": "ASC", - "policy": "default", - "query": "SELECT mean(\"ingress\") *8 FROM \"interface_rates\" WHERE (\"interface_name\" = 'PHY_CUSTOMER') AND $timeFilter GROUP BY time($__interval) fill(linear)", - "rawQuery": false, - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "egress" - ], - "type": "field" - }, - { - "params": [], - "type": "mean" - }, - { - "params": [ - "*8" - ], - "type": "math" - } - ] - ], - "tags": [ - { - "key": "interface_name", - "operator": "=", - "value": "PHY_CUSTOMER" - } - ] - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Network Aggregate (Egress)", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "bps", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - } - {% endif %} - ], - "schemaVersion": 26, - "style": "dark", - "tags": [], - "templating": { - "list": [ - {% if staff %} - { - "allValue": null, - "datasource": "PollerInfluxDB", - "definition": "SHOW TAG VALUES WITH KEY=hostname", - "hide": 0, - "includeAll": false, - "label": "Router:", - "multi": false, - "name": "hostname", - "options": [], - "query": "SHOW TAG VALUES WITH KEY=hostname", - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "sort": 0, - "tagValuesQuery": "", - "tags": [], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": null, - "datasource": "PollerInfluxDB", - "definition": "SHOW TAG VALUES WITH KEY IN (interface_name) WHERE hostname =~ /$hostname/ ", - "hide": 0, - "includeAll": false, - "label": "Interface :", - "multi": false, - "name": "interface_name", - "options": [], - "query": "SHOW TAG VALUES WITH KEY IN (interface_name) WHERE hostname =~ /$hostname/ ", - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "sort": 0, - "tagValuesQuery": "", - "tags": [], - "tagsQuery": "", - "type": "query", - "useTags": false - } - {% endif %} - ] - }, - "time": { - "from": "now-6h", - "to": "now" - }, - "timepicker": { - "refresh_intervals": [ - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ] - }, - "timezone": "", - "title": "Home", - "version": 1 -} \ No newline at end of file diff --git a/brian_dashboard_manager/templating/templates/nren_access/nren-dashboard.json.j2 b/brian_dashboard_manager/templating/templates/nren_access/nren-dashboard.json.j2 deleted file mode 100644 index eeb9dbb9fb0151e4f25a0f100e495796bad1e2ff..0000000000000000000000000000000000000000 --- a/brian_dashboard_manager/templating/templates/nren_access/nren-dashboard.json.j2 +++ /dev/null @@ -1,73 +0,0 @@ -{ - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": "-- Grafana --", - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": false, - "gnetId": null, - "graphTooltip": 0, - "schemaVersion": 27, - "style": "dark", - {% if not tags %} - "tags": ["{{ tag }}"], - {% else %} - "tags": [ - {% for tag in tags %} - "{{ tag }}"{{ "," if not loop.last }} - {% endfor %} - ], - {% endif %} - "templating": { - "list": [] - }, - "time": { - "from": "now-24h", - "to": "now" - }, - "timepicker": {}, - "timezone": "", - "title": "{{ nren_name }}", - "version": 1, - "links": [], - "panels": [ - { - "datasource": null, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 1, - "options": { - "content": "", - "mode": "html" - }, - "pluginVersion": "8.2.5", - "title": "INFO: The average values displayed are only mean values for timescales of 2 days or less", - "type": "text" - }, - {% for panel in aggregate_panels %} - {{ panel }}, - {% endfor %} - {% for group in dropdown_groups %} - {{ group.dropdown }} - {% if group.panels|length > 0 %} - , - {% endif %} - {% for panel in group.panels %} - {{ panel }}{{ "," if not loop.last }} - {% endfor %} - {{ "," if not loop.last }} - {% endfor %} - ] -} \ No newline at end of file diff --git a/brian_dashboard_manager/templating/templates/shared/dashboard.json.j2 b/brian_dashboard_manager/templating/templates/shared/dashboard.json.j2 deleted file mode 100644 index bbefccc38a500ec4bd4656e586cbf7d11b7aab4f..0000000000000000000000000000000000000000 --- a/brian_dashboard_manager/templating/templates/shared/dashboard.json.j2 +++ /dev/null @@ -1,63 +0,0 @@ -{ - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": "-- Grafana --", - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": false, - "gnetId": null, - "graphTooltip": 0, - "schemaVersion": 27, - "style": "dark", - {% if not tags %} - "tags": ["{{ tag }}"], - {% else %} - "tags": [ - {% for tag in tags %} - "{{ tag }}"{{ "," if not loop.last }} - {% endfor %} - ], - {% endif %} - "templating": { - "list": [] - }, - "time": { - "from": "now-24h", - "to": "now" - }, - "timepicker": {}, - "timezone": "", - "title": "{{ title }}", - "version": 1, - "links": [], - "panels": [ - { - "datasource": null, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 1, - "options": { - "content": "", - "mode": "html" - }, - "pluginVersion": "8.2.5", - "title": "INFO: The average values displayed are only mean values for timescales of 2 days or less", - "type": "text" - }{{ "," if panels }} - {% for panel in panels %} - {{ panel }}{{ "," if not loop.last }} - {% endfor %} - ] -} \ No newline at end of file diff --git a/brian_dashboard_manager/templating/templates/shared/dropdown.json.j2 b/brian_dashboard_manager/templating/templates/shared/dropdown.json.j2 deleted file mode 100644 index ec3c1b8e7b17644c3a4caa60b6d3a95f9af48ea1..0000000000000000000000000000000000000000 --- a/brian_dashboard_manager/templating/templates/shared/dropdown.json.j2 +++ /dev/null @@ -1,26 +0,0 @@ -{ - "aliasColors": {}, - "collapsed": false, - "datasource": null, - "fill": null, - "fillGradient": null, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": {{ y }} - }, - "id": {{ id }}, - "legend": null, - "lines": null, - "linewidth": null, - "search": null, - "stack": null, - "tags": null, - "targets": null, - "title": "{{ title }}", - "type": "row", - "xaxis": null, - "yaxes": null, - "yaxis": null -} \ No newline at end of file diff --git a/brian_dashboard_manager/templating/templates/shared/panel.json.j2 b/brian_dashboard_manager/templating/templates/shared/panel.json.j2 deleted file mode 100644 index 005e069176752e00082ad58c14348d5edd185b09..0000000000000000000000000000000000000000 --- a/brian_dashboard_manager/templating/templates/shared/panel.json.j2 +++ /dev/null @@ -1,96 +0,0 @@ -{ - {% if alias_colors %} - "aliasColors": {{ alias_colors }}, - {% else %} - "aliasColors": {}, - {% endif %} - "bars": false, - "collapsed": null, - "dashLength": 10, - "dashes": false, - "datasource": "{{ datasource }}", - "decimals": 2, - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 10, - "gridPos": { - "h": {{ height }}, - "w": {{ width }}, - {% if x %} - "x": {{ x }}, - {% else %} - "x": 0, - {% endif %} - "y": {{ y }} - }, - "hiddenSeries": false, - "id": {{ id }}, - {% if not disable_legend %} - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "max": true, - "min": false, - "rightSide": null, - "show": true, - "total": false, - "values": true - }, - {% endif %} - "lines": true, - "linewidth": {{ linewidth }}, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pointradius": 2, - "points": false, - "renderer": "flot", - "search": null, - "seriesOverrides": [], - "spaceLength": 10, - {% if stack %} - "stack": true, - {% else %} - "stack": false, - {% endif %} - "steppedLine": false, - "tags": null, - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "{{ title }}", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": null - }, - "yaxes": [ - {{ yaxes }} - ], - "yaxis": { - "align": false, - "alignLevel": null - }, - "targets": [ - {% for target in targets %} - {{ target }}{{ "," if not loop.last }} - {% endfor %} - ] -} \ No newline at end of file diff --git a/brian_dashboard_manager/templating/templates/shared/panel_target.json.j2 b/brian_dashboard_manager/templating/templates/shared/panel_target.json.j2 deleted file mode 100644 index b140cc06b9c64f79114c06cccc081b507e32b9a9..0000000000000000000000000000000000000000 --- a/brian_dashboard_manager/templating/templates/shared/panel_target.json.j2 +++ /dev/null @@ -1,104 +0,0 @@ -{ - "alias": "{{ alias }}", - "groupBy": [ - {% if not percentile %} - { - "params": ["$__interval"], - "type": "time" - }, - { - "params": ["null"], - "type": "fill" - } - {% endif %} - ], - {% if measurement %} - "measurement": "{{ measurement }}", - {% else %} - "measurement": "interface_rates", - {% endif %} - "orderByTime": null, - "policy": null, - "refId": "{{ refId }}", - "resultFormat": "time_series", - "select": [ - [ - { - "params": ["{{ select_field }}"], - "type": "field" - }, - {% if not percentile %} - { - "params": [], - "type": "max" - } - {% else %} - { - "params": [95], - "type": "percentile" - } - {% endif %} - {% if not errors %} - ,{ - "params": ["*8"], - "type": "math" - } - {% endif %} - ] - ], - "tags": [ - {% if isp %} - { - "condition": null, - "key": "tag", - "operator": "=", - "value": "{{ interface_tag }}" - }, - { - "condition": "AND", - "key": "isp", - "operator": "=", - "value": "{{ isp }}" - }, - { - "condition": "AND", - "key": "nren", - "operator": "=", - "value": "{{ nren }}" - } - {% elif subscription %} - { - "condition": null, - "key": "hostname", - "operator": "=", - "value": "{{ hostname }}" - }, - { - "condition": "AND", - "key": "subscription", - "operator": "=", - "value": "{{ subscription }}" - } - {% elif scid %} - { - "condition": null, - "key": "scid", - "operator": "=", - "value": "{{ scid }}" - } - {% else %} - { - "condition": null, - "key": "hostname", - "operator": "=", - "value": "{{ hostname }}" - }, - { - "condition": "AND", - "key": "interface_name", - "operator": "=", - "value": "{{ interface }}" - } - {% endif %} - ] -} \ No newline at end of file diff --git a/brian_dashboard_manager/templating/templates/shared/yaxes.json.j2 b/brian_dashboard_manager/templating/templates/shared/yaxes.json.j2 deleted file mode 100644 index 94e4555f92e15d1cb3bd91f7c988b73255a74ffe..0000000000000000000000000000000000000000 --- a/brian_dashboard_manager/templating/templates/shared/yaxes.json.j2 +++ /dev/null @@ -1,35 +0,0 @@ -{% if type == 'errors' %} -{ - "format": "none", - "label": "errors and discards per second", - "logBase": 1, - "max": null, - "min": 0, - "show": true -}, -{ - "format": "none", - "label": "errors and discards per second", - "logBase": 1, - "max": null, - "min": 0, - "show": true -} -{% else %} -{ - "format": "bps", - "label": "bits per second", - "logBase": 1, - "max": null, - "min": "", - "show": true -}, -{ - "format": "bps", - "label": "bits per second", - "logBase": 1, - "max": null, - "min": "", - "show": true -} -{% endif %} diff --git a/changelog.md b/changelog.md index e53263caacab2ba3aef37d08fed6bee9d6d6a780..8f58d66691485adf969c3df859e9cfd4fbcddb38 100644 --- a/changelog.md +++ b/changelog.md @@ -2,6 +2,10 @@ All notable changes to this project will be documented in this file. +## [0.62] - 2024-07-11 +- POL1-579: New layout for R&E Peer dashboards +- Migrate from jinja2 to just plain python dictionaries + ## [0.61] - 2024-05-27 - Remove traffic type format string in aggregate panels diff --git a/config.json.example b/config.json.example index c0071e0bbe1d16b355a7d108b7eca2df1982f2dc..900a5308a2c2cdacf02088ca1e1ef9d75d8f35b9 100644 --- a/config.json.example +++ b/config.json.example @@ -2,8 +2,8 @@ "admin_username": "admin", "admin_password": "admin", "hostname": "localhost:3000", - "inventory_provider": "http://inventory-provider01.geant.org:8080", - "reporting_provider": "http://prod-tableau-wdc.geant.org:9090", + "inventory_provider": "https://prod-inprov01.geant.org", + "reporting_provider": "https://prod-tableau-wdc.geant.org", "datasources": { "influxdb": { "name": "PollerInfluxDB", diff --git a/requirements.txt b/requirements.txt index b650084178d47fa18e888ba4ae3ffe970e1d9712..d7ef2533a16044f57822ee6681251830dc753982 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,6 @@ requests jsonschema flask -jinja2 pytest pytest-mock diff --git a/setup.py b/setup.py index 6ef7bf0c75ce876b9c518d8a922c465e8699c005..9b630b2b1f3658bb3cc9882a3e6887a286e526ab 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import setup, find_packages setup( name='brian-dashboard-manager', - version="0.61", + version="0.62", author='GEANT', author_email='swd@geant.org', description='', @@ -12,7 +12,6 @@ setup( 'requests', 'jsonschema', 'flask', - 'jinja2', 'sentry-sdk[flask]' ], include_package_data=True, diff --git a/test/conftest.py b/test/conftest.py index 81a61e687660f713df69414c20a69004898483b9..dbc0f52723b405677db6d72a7fef7ecf827d9981 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -1,8 +1,17 @@ +import copy +import datetime +import itertools import json import os -import tempfile +import pathlib +import re +import string +import threading +from brian_dashboard_manager import environment +from brian_dashboard_manager.grafana.utils.request import TokenRequest import pytest import brian_dashboard_manager +import responses @pytest.fixture @@ -51,25 +60,367 @@ def data_config(): } -def get_test_data(filename): - data_filename = os.path.join( - os.path.dirname(__file__), - 'data', - filename) - with open(data_filename) as f: - return json.loads(f.read()) +DATA_DIR = pathlib.Path(__file__).parent / "data" @pytest.fixture -def data_config_filename(data_config): - with tempfile.NamedTemporaryFile() as f: - f.write(json.dumps(data_config).encode('utf-8')) - f.flush() - yield f.name +def get_test_data(): + def _get_test_data(filename): + return json.loads((DATA_DIR / filename).read_text()) + + return _get_test_data + + +@pytest.fixture +def data_config_filename(data_config, tmp_path): + file = tmp_path / "data_config.json" + file.write_text(json.dumps(data_config)) + return str(file) @pytest.fixture -def client(data_config_filename): - os.environ['CONFIG_FILENAME'] = data_config_filename +def client(mocker, data_config_filename): + mocker.patch.object(environment, "setup_logging") + os.environ["CONFIG_FILENAME"] = data_config_filename with brian_dashboard_manager.create_app().test_client() as c: yield c + + +@pytest.fixture +def mock_grafana(data_config): + def uid_generator(): + return ( + "".join(result) + for result in itertools.permutations(string.ascii_lowercase, 8) + ) + + lock = threading.RLock() + + def synchronized(fun): + def _sync(*args, **kwargs): + with lock: + return fun(*args, **kwargs) + + return _sync + + class MockGrafana: + def __init__(self) -> None: + self.folders = {} + self.dashboards = {} + self.dashboards_by_folder_uid = {} + self.datasources = {} + self.organizations = {} + self.api_tokens = {} + self.uids = uid_generator() + self.ids = itertools.count(start=1) + self.request = TokenRequest(data_config["hostname"], token="abc") + + @synchronized + def list_api_tokens(self): + return list(copy.copy(val) for val in self.api_tokens.values()) + + @synchronized + def create_api_token(self, payload): + lifetime = payload.get("secondsToLive") + return self._create_object( + { + **payload, + "key": "key", + "expiration": ( + ( + datetime.datetime.utcnow() + datetime.timedelta(lifetime) + ).strftime("%Y-%m-%dT%H:%M:%SZ") + if lifetime is not None + else None + ), + }, + self.api_tokens, + ) + + @synchronized + def delete_api_token(self, uid): + return self._delete_object(uid, self.api_tokens, name="api token") + + @synchronized + def list_organizations(self): + return list(copy.copy(val) for val in self.organizations.values()) + + @synchronized + def create_organization(self, organization): + organization = { + **organization, + "id": next(self.ids), + "uid": str(organization.get("uid") or next(self.uids)), + } + self.organizations[organization["uid"]] = organization + return { + "orgId": organization["id"], + "message": "Organization created", + } + + @synchronized + def delete_organization(self, uid): + return self._delete_object(uid, self.organizations, name="organization") + + @synchronized + def list_datasources(self): + return list(copy.copy(val) for val in self.datasources.values()) + + @synchronized + def create_datasource(self, datasource): + return self._create_object(datasource, self.datasources) + + @synchronized + def delete_datasource(self, uid): + return self._delete_object(uid, self.datasources, name="datasource") + + @synchronized + def list_folders(self): + return list(self.folders.values()) + + @synchronized + def create_folder(self, folder): + return self._create_object(folder, self.folders) + + @synchronized + def delete_folder(self, uid): + return self._delete_object(uid, self.folders, name="folder") + + @synchronized + def list_dashboards(self, title=None, folder_id=None): + return [ + copy.copy(db) + for db in self.dashboards.values() + if (title is None or db["title"] == title) + and (folder_id is None or db.get("folderId") == folder_id) + ] + + @synchronized + def create_dashboard(self, dashboard, folder_id=None): + result = self._create_object(dashboard, self.dashboards) + folder_uid = next( + iter(f["uid"] for f in self.folders.values() if f["id"] == folder_id), + None, + ) + for idx, db in enumerate(self.dashboards_by_folder_uid.get(folder_uid, [])): + if db["uid"] == result["uid"]: + self.dashboards_by_folder_uid[folder_uid].pop(idx) + self.dashboards_by_folder_uid[folder_uid].insert(idx, result) + break + else: + self.dashboards_by_folder_uid.setdefault(folder_uid, []).append(result) + return result + + @synchronized + def delete_dashboard(self, uid): + result = self._delete_object(uid, self.dashboards, name="dashboard") + for dashboards in self.dashboards_by_folder_uid.values(): + for idx, db in enumerate(dashboards): + if db["uid"] == result["uid"]: + dashboards.pop(idx) + break + return result + + def _create_object(self, obj, all_objects): + id = obj.get("id") + uid = obj.get("uid") + obj = { + **obj, + "id": id if id is not None else next(self.ids), + "uid": str(uid if uid is not None else next(self.uids)), + } + all_objects[obj["uid"]] = obj + return obj + + def _delete_object(self, uid, all_objects, name="object"): + del all_objects[uid] + return {"message": f"deleted {name}"} + + grafana = MockGrafana() + + url_prefix = f".*{data_config['hostname']}/" + + def json_request_cb(fun): + def _wrapped(request): + payload = json.loads(request.body) if request.body else None + status, result = fun(payload, request) + return (status, {}, json.dumps(result)) + + return _wrapped + + # --- Api Tokens --- + responses.add_callback( + method=responses.GET, + url=re.compile(url_prefix + r"api/auth/keys?.+$"), + callback=json_request_cb(lambda p, b: (200, grafana.list_api_tokens())), + ) + + @json_request_cb + def create_api_token_callback(payload, request): + return (200, grafana.create_api_token(payload)) + + responses.add_callback( + method=responses.POST, + url=re.compile(url_prefix + "api/auth/keys"), + callback=create_api_token_callback, + ) + + @json_request_cb + def delete_api_token_callback(payload, request): + id = int(request.path_url.split("/")[-1]) + for uid, ds in grafana.api_tokens.items(): + if ds["id"] == id: + return (200, grafana.delete_api_token(uid)) + return (404, "") + + responses.add_callback( + method=responses.DELETE, + url=re.compile(url_prefix + r"api/auth/keys/.+$"), + callback=delete_api_token_callback, + ) + # --- Organizations --- + responses.add_callback( + method=responses.GET, + url=re.compile(url_prefix + "api/orgs"), + callback=json_request_cb(lambda p, b: (200, grafana.list_organizations())), + ) + + @json_request_cb + def create_organization_callback(body, _): + return (200, grafana.create_organization(body)) + + responses.add_callback( + method=responses.POST, + url=re.compile(url_prefix + "api/orgs"), + callback=create_organization_callback, + ) + + # --- Datasources --- + responses.add_callback( + method=responses.GET, + url=re.compile(url_prefix + "api/datasources"), + callback=json_request_cb(lambda p, b: (200, grafana.list_datasources())), + ) + + @json_request_cb + def create_datasource_callback(payload, _): + return (200, grafana.create_datasource(payload)) + + responses.add_callback( + method=responses.POST, + url=re.compile(url_prefix + "api/datasources"), + callback=create_datasource_callback, + ) + + @json_request_cb + def delete_datasource_callback(payload, request): + name = request.path_url.split("/")[-1] + for uid, ds in grafana.datasources.items(): + if ds["name"] == name: + return (200, {}, grafana.delete_datasource(uid)) + return (404, {}, "") + + responses.add_callback( + method=responses.DELETE, + url=re.compile(url_prefix + r"api/datasources/name/.+$"), + callback=delete_datasource_callback, + ) + # --- Folders --- + responses.add_callback( + method=responses.GET, + url=re.compile(url_prefix + "api/folders"), + callback=json_request_cb(lambda p, b: (200, grafana.list_folders())), + ) + + @json_request_cb + def create_folder_callback(payload, _): + return (200, grafana.create_folder(payload)) + + responses.add_callback( + method=responses.POST, + url=re.compile(url_prefix + "api/folders"), + callback=create_folder_callback, + ) + + @json_request_cb + def delete_folder_callback(payload, request): + uid = request.path_url.split("/")[-1] + try: + return (200, grafana.delete_folder(uid)) + except KeyError: + return (404, {}) + + responses.add_callback( + method=responses.DELETE, + url=re.compile(url_prefix + r"api/folders/.+$"), + callback=delete_folder_callback, + ) + + # --- Dashboards --- + @json_request_cb + def get_dashboard_callback(payload, request): + uid = request.path_url.split("/")[-1] + try: + return (200, {"dashboard": grafana.dashboards[uid]}) + except KeyError: + return (404, {}) + + responses.add_callback( + method=responses.GET, + url=re.compile(url_prefix + r"api/dashboards/uid/.+$"), + callback=get_dashboard_callback, + ) + + @json_request_cb + def search_dashboard_callback(_, request): + query = request.params + return ( + 200, + grafana.list_dashboards(query.get("title"), query.get("folderIds")), + ) + + responses.add_callback( + method=responses.GET, + url=re.compile(url_prefix + "api/search"), + callback=search_dashboard_callback, + ) + + @json_request_cb + def create_dashboard_callback(payload, request): + dashboard = payload["dashboard"] + folder_id = payload.get("folderId") + + return (200, grafana.create_dashboard(dashboard, folder_id=folder_id)) + + responses.add_callback( + method=responses.POST, + url=re.compile(url_prefix + "api/dashboards/db"), + callback=create_dashboard_callback, + ) + + @json_request_cb + def delete_dashboard_callback(payload, request): + uid = request.path_url.split("/")[-1] + try: + return (200, grafana.delete_dashboard(uid)) + except KeyError: + return (404, {}) + + responses.add_callback( + method=responses.DELETE, + url=re.compile(url_prefix + r"api/dashboards/uid/.+$"), + callback=delete_dashboard_callback, + ) + + # --- Other --- + responses.add( + method=responses.PUT, + url=re.compile(url_prefix + "api/org/preferences"), + json={"message": "Preferences updated"}, + ) + responses.add( + method=responses.POST, + url=re.compile(url_prefix + r"api/user/using/.+$"), + json={"message": "ok"}, + ) + return grafana diff --git a/test/test_grafana_dashboard.py b/test/test_grafana_dashboard.py index e3da83f02c59e21ca0d49348e396d11e8b1aec7b..c0534103b0ccdad86cdb82c4dea29d3f5d3da31f 100644 --- a/test/test_grafana_dashboard.py +++ b/test/test_grafana_dashboard.py @@ -1,130 +1,73 @@ -import pytest -import json -import requests import responses from brian_dashboard_manager.grafana import dashboard, provision from brian_dashboard_manager.grafana.utils.request import TokenRequest @responses.activate -def test_get_dashboard(data_config): +def test_get_dashboard(mock_grafana): + mock_grafana.create_dashboard({"uid": "1"}) - UID = 1 + request = mock_grafana.request - request = TokenRequest(**data_config, token='test') - - responses.add_callback( - method=responses.GET, - url=request.BASE_URL + - f'api/dashboards/uid/{UID}', - callback=lambda f: (404, {}, '')) + data = dashboard._get_dashboard(request, "1") + assert data["uid"] == "1" - data = dashboard._get_dashboard(request, UID) + data = dashboard._get_dashboard(request, "2") assert data is None - responses.add( - method=responses.GET, - url=request.BASE_URL + f'api/dashboards/uid/{UID+1}', - json={'uid': 1}) - - data = dashboard._get_dashboard(request, UID + 1) - assert data['uid'] == 1 - @responses.activate -def test_delete_dashboards(data_config): - UID = 1 - dashboards = [{'uid': UID}] +def test_delete_dashboards_new(mock_grafana): + mock_grafana.create_dashboard({"uid": "1"}) + mock_grafana.create_dashboard({"uid": "2"}) - request = TokenRequest(**data_config, token='test') + request = mock_grafana.request + assert dashboard.delete_dashboards(request) is True + assert not mock_grafana.dashboards - responses.add( - method=responses.GET, - url=request.BASE_URL + f'api/dashboards/uid/{UID}', - json=dashboards[0]) - - responses.add( - method=responses.GET, - url=request.BASE_URL + 'api/search', - json=dashboards) - - def delete_callback(request): - uid = request.path_url.split('/')[-1] - assert int(uid) == UID - return 200, {}, json.dumps({'message': 'Dashboard has been deleted.'}) - - responses.add_callback( - method=responses.DELETE, - url=request.BASE_URL + - f'api/dashboards/uid/{UID}', - callback=delete_callback) - data = dashboard.delete_dashboards(request) - assert data is True - - responses.add_callback( - method=responses.DELETE, - url=request.BASE_URL + - f'api/dashboards/uid/{UID+1}', - callback=lambda f: (400, {}, '')) - - with pytest.raises(requests.HTTPError): - data = dashboard._delete_dashboard(request, UID + 1) +@responses.activate +def test_delete_nonexsiting_dashboard(mock_grafana): + assert not mock_grafana.dashboards + assert dashboard._delete_dashboard(mock_grafana.request, "2") is True @responses.activate -def test_delete_dashboard(data_config): - UID = 1 - ID = 1 +def test_delete_dashboard_by_uid(mock_grafana): + UID = "1" VERSION = 1 - FOLDER_ID = 1 - TITLE = 'testdashboard' - dash = {'id': ID, 'uid': UID, 'title': TITLE, 'version': VERSION} - request = TokenRequest(**data_config, token='test') - - responses.add( - method=responses.DELETE, - url=request.BASE_URL + f'api/dashboards/uid/{UID}', - json={'message': 'deleted dashboard'}) - - responses.add( - method=responses.GET, - url=request.BASE_URL + f'api/dashboards/uid/{UID}', - json={}) - - responses.add( - method=responses.GET, - url=request.BASE_URL + 'api/search', - json=[dash]) + TITLE = "testdashboard" + dash = {"uid": UID, "title": TITLE, "version": VERSION} + mock_grafana.create_dashboard(dash) + request = mock_grafana.request - deleted = dashboard.delete_dashboard(request, dash) - assert deleted - del dash['uid'] - deleted = dashboard.delete_dashboard(request, dash, FOLDER_ID) - assert deleted + assert mock_grafana.dashboards + assert dashboard.delete_dashboard(request, dash) + assert not mock_grafana.dashboards @responses.activate -def test_search_dashboard(data_config): - UID = 1 - TITLE = 'testdashboard' - dashboards = [{'uid': UID, 'title': TITLE}] +def test_delete_dashboard_by_title_and_folder(mock_grafana): + FOLDER_ID = 1 + TITLE = "testdashboard" + dash = {"title": TITLE, "folderId": FOLDER_ID} + mock_grafana.create_dashboard(dash) + request = mock_grafana.request - request = TokenRequest(**data_config, token='test') + assert mock_grafana.dashboards + assert dashboard.delete_dashboard(request, dash) + assert not mock_grafana.dashboards - responses.add( - method=responses.GET, - url=request.BASE_URL + 'api/search', - json=dashboards) - responses.add( - method=responses.GET, - url=request.BASE_URL + f'api/dashboards/uid/{UID}', - json=dashboards[0]) +@responses.activate +def test_search_dashboard(mock_grafana): + UID = "1" + TITLE = "testdashboard" + mock_grafana.create_dashboard({"uid": UID, "title": TITLE}) + request = mock_grafana.request - data = dashboard._search_dashboard( - request, {'title': dashboards[0]['title']}) - assert data['uid'] == UID + data = dashboard._search_dashboard(request, {"title": TITLE}) + assert data["uid"] == UID data = dashboard._search_dashboard(request, {'title': 'DoesNotExist'}) assert data is None @@ -144,40 +87,48 @@ def test_search_dashboard_error(data_config): @responses.activate -def test_create_dashboard(data_config): - UID = 1 - ID = 1 +def test_create_dashboard(mock_grafana): + UID = "1" VERSION = 1 - TITLE = 'testdashboard' - dashboard = {'id': ID, 'uid': UID, 'title': TITLE, 'version': VERSION} - request = TokenRequest(**data_config, token='test') + TITLE = "testdashboard" + dashboard = {"uid": UID, "title": TITLE, "version": VERSION} + request = mock_grafana.request + assert not mock_grafana.dashboards - responses.add( - method=responses.GET, - url=request.BASE_URL + f'api/dashboards/uid/{UID}', - json={'dashboard': dashboard}) + provision.create_dashboard(request, dashboard) + assert UID in mock_grafana.dashboards - responses.add_callback( - method=responses.GET, - url=request.BASE_URL + 'api/search', - callback=lambda f: (400, {}, '')) - def post_callback(request): - body = json.loads(request.body) - return 200, {}, json.dumps(body['dashboard']) +@responses.activate +def test_create_dashboard_updates_with_existing_uid(mock_grafana): + dashboard = {'title': 'testdashboard', 'version': 1} + dash = mock_grafana.create_dashboard(dashboard) + request = mock_grafana.request - responses.add_callback( - method=responses.POST, - url=request.BASE_URL + 'api/dashboards/db', - callback=post_callback) + dashboard.update(**dash) + dashboard['data'] = 'data' + + provision.create_dashboard(request, dashboard) + in_grafana = mock_grafana.dashboards[dashboard['uid']] + assert in_grafana['data'] == 'data' + assert in_grafana['id'] == dashboard['id'] + + +@responses.activate +def test_create_dashboard_no_uid_does_not_send_id(mock_grafana): + ID = 1042 + VERSION = 1 + TITLE = 'testdashboard' + dashboard = {'id': ID, 'title': TITLE, 'version': VERSION} + request = mock_grafana.request data = provision.create_dashboard(request, dashboard) - assert data == dashboard + assert data['id'] != ID @responses.activate -def test_create_dashboard_no_uid_error(data_config): - ID = 1 +def test_create_dashboard_error(data_config): + ID = 1042 VERSION = 1 TITLE = 'testdashboard' dashboard = {'id': ID, 'title': TITLE, 'version': VERSION} @@ -189,13 +140,7 @@ def test_create_dashboard_no_uid_error(data_config): callback=lambda f: (400, {}, '')) def post_callback(request): - body = json.loads(request.body) - # if a dashboard doesn't have an UID, the ID should not be sent to - # grafana. - assert 'id' not in body['dashboard'] - - # have already tested a successful response, respond with error here. - return 400, {}, '{}' + return 400, {}, '' responses.add_callback( method=responses.POST, diff --git a/test/test_grafana_datasource.py b/test/test_grafana_datasource.py index fe939950629514c56ec85862ef3d0e7eda4ad8b1..858c63081bd66243a2f94c80b8605ad38b5c3b5f 100644 --- a/test/test_grafana_datasource.py +++ b/test/test_grafana_datasource.py @@ -5,35 +5,23 @@ from brian_dashboard_manager.grafana.utils.request import AdminRequest @responses.activate -def test_get_datasources(data_config): - - BODY = [] - - request = AdminRequest(**data_config) - - responses.add( - method=responses.GET, - url=request.BASE_URL + 'api/datasources', json=BODY) +def test_get_datasources(mock_grafana): + mock_grafana.create_datasource({"some": "data"}) + request = mock_grafana.request data = datasource.get_datasources(request) - assert data == BODY + assert data[0]["some"] == "data" @responses.activate -def test_get_missing_datasource_definitions(data_config): - # this only retrieves data from the filesystem and checks against - # what's configured in grafana.. just make sure - # we cover the part for fetching datasources +def test_get_missing_datasource_definitions(mock_grafana, tmp_path): + request = mock_grafana.request - request = AdminRequest(**data_config) - - responses.add(method=responses.GET, url=request.BASE_URL + - 'api/datasources', json={}) - - dir = '/tmp/dirthatreallyshouldnotexistsousealonganduniquestring' - # it returns a generator, so iterate :) - for data in datasource.get_missing_datasource_definitions(request, dir): - pass + source = {"some": "data"} + (tmp_path / "datasource.json").write_text(json.dumps(source)) + assert list( + datasource.get_missing_datasource_definitions(request, str(tmp_path)) + ) == [source] def test_datasource_provisioned(): @@ -61,9 +49,7 @@ def test_datasource_provisioned(): @responses.activate -def test_create_prod_datasource(data_config): - ORG_ID = 1 - +def test_create_prod_datasource(mock_grafana): BODY = { "name": "brian-influx-datasource", "type": "influxdb", @@ -75,43 +61,12 @@ def test_create_prod_datasource(data_config): "readOnly": False } - request = AdminRequest(**data_config) - - def post_callback(request): - body = json.loads(request.body) - result = { - 'datasource': { - 'id': 1, - 'orgId': ORG_ID, - 'type': 'graphite', - 'typeLogoUrl': '', - 'password': '', - 'user': '', - 'basicAuthUser': '', - 'basicAuthPassword': '', - 'withCredentials': False, - 'jsonData': {}, - 'secureJsonFields': {}, - 'version': 1 - }, - 'id': 1, - 'message': 'Datasource added', - 'name': body['name'] - } - result['datasource'].update(body) - return 200, {}, json.dumps(result) - - responses.add_callback( - method=responses.POST, - url=request.BASE_URL + 'api/datasources', - callback=post_callback) + request = mock_grafana.request - data = provision.create_datasource( - request, BODY) + assert not mock_grafana.datasources - datasource_type = data['datasource']['type'] - datasource_config_url = data_config['datasources'][datasource_type]['url'] - assert data['datasource']['url'] != datasource_config_url + result = provision.create_datasource(request, BODY) + assert result["uid"] in mock_grafana.datasources @responses.activate diff --git a/test/test_grafana_folder.py b/test/test_grafana_folder.py index e9baedc9e356078094bbd1b71b7b2db5f5af74ea..1fff9f194af2e94a3cf9b029e588f6059ec83681 100644 --- a/test/test_grafana_folder.py +++ b/test/test_grafana_folder.py @@ -1,8 +1,6 @@ -import json import responses from brian_dashboard_manager.grafana.folder import find_folder -from brian_dashboard_manager.grafana.utils.request import TokenRequest def generate_folder(data): @@ -24,26 +22,21 @@ def generate_folder(data): @responses.activate -def test_find_folder(data_config): +def test_find_folder(data_config, mock_grafana): + TITLE = "testfolder123" + request = mock_grafana.request + assert not mock_grafana.folders + folder = find_folder(request, TITLE, create=True) + assert folder["title"] == TITLE + assert folder["uid"] in mock_grafana.folders - TITLE = 'testfolder123' - - request = TokenRequest(**data_config, token='test') - - responses.add( - method=responses.GET, - url=f"http://{data_config['hostname']}/api/folders", - json=[]) - def folder_post(request): - data = json.loads(request.body) - return 200, {}, json.dumps(generate_folder(data)) - - responses.add_callback( - method=responses.POST, - url=f"http://{data_config['hostname']}/api/folders", - callback=folder_post) +@responses.activate +def test_find_folder_no_create(data_config, mock_grafana): + TITLE = 'testfolder123' + request = mock_grafana.request + assert not mock_grafana.folders - folder = find_folder(request, TITLE) - assert folder['id'] == 555 - assert folder['title'] == TITLE + folder = find_folder(request, TITLE, create=False) + assert folder is None + assert not mock_grafana.folders diff --git a/test/test_grafana_organization.py b/test/test_grafana_organization.py index c54b38182e2af9427d3289a8515f4fc174513fc0..e9a62280fc6b4a02f1ccd34e0b927e5b8d48cb8a 100644 --- a/test/test_grafana_organization.py +++ b/test/test_grafana_organization.py @@ -1,111 +1,51 @@ -import json import responses -from datetime import datetime, timedelta from brian_dashboard_manager.grafana import provision -from brian_dashboard_manager.grafana.utils.request import AdminRequest @responses.activate -def test_get_organizations(data_config): - request = AdminRequest(**data_config) +def test_get_organizations(mock_grafana): + request = mock_grafana.request - responses.add(method=responses.GET, - url=request.BASE_URL + 'api/orgs', - json=[{'id': 91, - 'name': 'Testorg1'}, - {'id': 92, - 'name': 'GÉANT Testorg2'}, - {'id': 93, - 'name': 'NRENsTestorg3'}, - {'id': 94, - 'name': 'General Public'}]) + mock_grafana.create_organization({"id": 91, "name": "Testorg1"}) + mock_grafana.create_organization({"id": 91, "name": "Testorg2"}) data = provision.get_organizations(request) - assert data is not None + assert [o["name"] for o in data] == ["Testorg1", "Testorg2"] @responses.activate -def test_create_organization(data_config): +def test_create_organization(mock_grafana): ORG_NAME = 'fakeorg123' + request = mock_grafana.request - def post_callback(request): - body = json.loads(request.body) - assert body['name'] == ORG_NAME - return 200, {}, json.dumps( - {'orgId': 1, 'message': 'Organization created'}) - - request = AdminRequest(**data_config) - - responses.add_callback( - method=responses.POST, - url=request.BASE_URL + 'api/orgs', - callback=post_callback) + assert not len(mock_grafana.organizations) data = provision.create_organization(request, ORG_NAME) - assert data is not None + assert len(mock_grafana.organizations) == 1 + assert next(iter(mock_grafana.organizations.values()))["id"] == data["id"] + assert data["name"] == ORG_NAME @responses.activate -def test_delete_expired_api_tokens(data_config): - ORG_ID = 1 - KEY_ID = 1 - - def post_callback(request): - assert request.params['includeExpired'] == 'True' - time = (datetime.now() - timedelta(seconds=60) - ).strftime('%Y-%m-%dT%H:%M:%SZ') - return 200, {}, json.dumps([{'expiration': time, 'id': KEY_ID}]) - - request = AdminRequest(**data_config) - responses.add_callback( - method=responses.GET, - url=request.BASE_URL + 'api/auth/keys', - callback=post_callback) - - responses.add( - method=responses.POST, - url=request.BASE_URL + - f'api/user/using/{ORG_ID}', - json={ - "message": "Active organization changed"}) - responses.add( - method=responses.DELETE, - url=request.BASE_URL + - f'api/auth/keys/{KEY_ID}', - json={ - "message": "API key deleted"}) +def test_delete_expired_api_tokens(mock_grafana): + mock_grafana.create_api_token({"secondsToLive": -1}) # an expired token + request = mock_grafana.request + assert len(mock_grafana.api_tokens) == 1 provision.delete_expired_api_tokens(request) + assert not mock_grafana.api_tokens @responses.activate -def test_create_api_token(data_config): +def test_create_api_token(mock_grafana): ORG_ID = 1 - TOKEN_ID = 1 BODY = { 'name': 'test-token', 'role': 'Admin', 'secondsToLive': 3600 } - - request = AdminRequest(**data_config) - - def post_callback(request): - body = json.loads(request.body) - assert body == BODY - return 200, {}, json.dumps({'id': TOKEN_ID}) - - responses.add_callback( - method=responses.POST, - url=request.BASE_URL + 'api/auth/keys', - callback=post_callback) - - responses.add( - method=responses.POST, - url=request.BASE_URL + - f'api/user/using/{ORG_ID}', - json={ - "message": "Active organization changed"}) - - data = provision.create_api_token(request, ORG_ID, BODY) - assert data['id'] == TOKEN_ID + request = mock_grafana.request + assert not mock_grafana.api_tokens + provision.create_api_token(request, ORG_ID, BODY) + assert len(mock_grafana.api_tokens) == 1 + assert next(iter(mock_grafana.api_tokens.values()))["name"] == "test-token" diff --git a/test/test_gws_direct.py b/test/test_gws_direct.py index 0a263c0f33a2c07ae2ec3fcd4c98020c42c1ca78..c8712279c937f760e2b9724a225f2abc04d2a36d 100644 --- a/test/test_gws_direct.py +++ b/test/test_gws_direct.py @@ -1,22 +1,17 @@ import responses -from test.conftest import get_test_data from brian_dashboard_manager.templating.gws import generate_gws -from brian_dashboard_manager.inventory_provider.interfaces import \ - get_gws_direct - -TEST_DATA = get_test_data('gws-direct-data.json') +from brian_dashboard_manager.inventory_provider.interfaces import get_gws_direct @responses.activate -def test_gws(data_config, client): - +def test_gws(data_config, get_test_data): responses.add( method=responses.GET, url=f"{data_config['inventory_provider']}/poller/gws/direct", - json=TEST_DATA) + json=get_test_data("gws-direct-data.json"), + ) gws_data = get_gws_direct(data_config['inventory_provider']) - dashboards = list(generate_gws(gws_data, 'testdatasource')) assert len(dashboards) == 4 diff --git a/test/test_helpers.py b/test/test_helpers.py new file mode 100644 index 0000000000000000000000000000000000000000..05c96559636c3aa0f1455ee0dbb94a9a69288b41 --- /dev/null +++ b/test/test_helpers.py @@ -0,0 +1,70 @@ +from brian_dashboard_manager.templating.helpers import get_re_peer_interface_data +import pytest + + +@pytest.mark.parametrize( + "interface_type, expected", + [ + ( + "LOGICAL", + { + "AGGREGATES": [ + { + "interface": "xe-0/0/0.1", + "hostname": "mx1.dub2.ie.geant.net", + "alias": "DUB2 - ESNET (xe-0/0/0.1)", + } + ], + "SERVICES": [ + { + "title": "mx1.dub2.ie - {} - xe-0/0/0.1 - description", + "interface": "xe-0/0/0.1", + "hostname": "mx1.dub2.ie.geant.net", + "has_v6": True, + } + ], + }, + ), + ( + "PHYSICAL", + { + "PHYSICAL": [ + { + "title": "mx1.dub2.ie - {} - xe-0/0/0.1 - description", + "interface": "xe-0/0/0.1", + "hostname": "mx1.dub2.ie.geant.net", + } + ], + }, + ), + ( + "AGGREGATE", + { + "PHYSICAL": [ + { + "title": "mx1.dub2.ie - {} - xe-0/0/0.1 - description", + "interface": "xe-0/0/0.1", + "hostname": "mx1.dub2.ie.geant.net", + } + ], + }, + ), + ], +) +def test_re_peer_interface_data(interface_type, expected): + interfaces = [ + { + "router": "mx1.dub2.ie.geant.net", + "name": "xe-0/0/0.1", + "description": "description", + "dashboards": ["RE_PEER"], + "dashboard_info": {"name": "ESNET", "interface_type": interface_type}, + "dashboards_info": [{"name": "ESNET", "interface_type": interface_type}], + "ipv4": ["1.1.1.1"], + "ipv6": ["::2"], + }, + ] + + assert get_re_peer_interface_data(interfaces) == { + "ESNET": {"AGGREGATES": [], "SERVICES": [], "PHYSICAL": [], **expected} + } diff --git a/test/test_update.py b/test/test_update.py index 57b09f2057e5dfb234d0de0d59d8749d05104c7e..e407e600de890cfc03178033a42fdb6df6e1486e 100644 --- a/test/test_update.py +++ b/test/test_update.py @@ -1,9 +1,7 @@ +import pytest import responses -import json -from brian_dashboard_manager.grafana.provision import provision_folder, \ - provision -from test.conftest import get_test_data +from brian_dashboard_manager.grafana.provision import provision_folder, provision TEST_INTERFACES = [ { @@ -602,166 +600,107 @@ def generate_folder(data): } -@responses.activate -def test_provision_folder(data_config, mocker): - dashboards = { - 'NREN': { - 'tag': ['customers'], - 'folder_name': 'NREN Access', - 'interfaces': [ - iface for iface in TEST_INTERFACES - if 'NREN' in iface['dashboards']] - }, - 'RE_CUST': { - 'tag': 'RE_CUST', - 'folder_name': 'RE Customer', - 'interfaces': [ - iface for iface in TEST_INTERFACES - if 'RE_CUST' in iface['dashboards']] - }, - - } - +@pytest.fixture +def reporting_provider(get_test_data, data_config): responses.add( method=responses.GET, url=f"{data_config['reporting_provider']}/scid/current", - json=get_test_data('services.json')) - - # just return a generated folder - _mocked_find_folder = mocker.patch( - 'brian_dashboard_manager.grafana.provision.find_folder') - _mocked_find_folder.return_value = generate_folder( - {'uid': 'testfolderuid', 'title': 'testfolder'}) - - def create_dashboard(request, dashboard, folder_id=None): - return dashboard - - mocker.patch( - 'brian_dashboard_manager.grafana.provision.create_dashboard', - create_dashboard) - - def _search_dashboard(request, dashboard, folder_id=None): - return None - - mocker.patch( - 'brian_dashboard_manager.grafana.dashboard._search_dashboard', - _search_dashboard) - - def delete_dashboard(request, dashboard, folder_id=None): - return True - - mocker.patch( - 'brian_dashboard_manager.grafana.dashboard.delete_dashboard', - delete_dashboard) - - excluded_dashboards = [] - nren_result = provision_folder( - None, 'NREN Access', dashboards['NREN'], - data_config, 'testdatasource', excluded_dashboards) - - assert len(nren_result) == 3 - assert nren_result[0]['title'] == 'GEANT' - assert nren_result[1]['title'] == 'KIAE' - assert nren_result[2]['title'] == 'SWITCH' - - excluded_dashboards = ['KIAE', 'GEANT'] - nren_excluded = provision_folder( - None, 'NREN Access', dashboards['NREN'], - data_config, 'testdatasource', excluded_dashboards) - - assert len(nren_excluded) == 1 - assert nren_excluded[0]['title'] == 'SWITCH' - - excluded_dashboards = [] - nren_result_beta = provision_folder( - None, 'NREN Access BETA', dashboards['NREN'], - data_config, 'testdatasource', excluded_dashboards) + json=get_test_data("services.json"), + ) - assert len(nren_result_beta) == 6 - assert nren_result_beta[0]['title'] == 'ASNET-AM' - assert nren_result_beta[1]['title'] == 'LITNET' - assert nren_result_beta[2]['title'] == 'CESNET' - assert nren_result_beta[3]['title'] == 'GEANT' - assert nren_result_beta[4]['title'] == 'KIAE' - assert nren_result_beta[5]['title'] == 'SWITCH' - excluded_dashboards = ['ASNET-AM', 'GEANT'] - nren_excluded_beta = provision_folder( - None, 'NREN Access BETA', dashboards['NREN'], - data_config, 'testdatasource', excluded_dashboards) +@pytest.fixture +def populate_inventory(data_config): + """function-fixture for provisioning inventory provider. Call it with a dictionary + {url_path: contents}. ie {"/poller/interfaces": [...]} + """ - assert len(nren_excluded_beta) == 4 - assert nren_excluded_beta[0]['title'] == 'LITNET' - assert nren_excluded_beta[1]['title'] == 'CESNET' - assert nren_excluded_beta[2]['title'] == 'KIAE' - assert nren_excluded_beta[3]['title'] == 'SWITCH' + def _populate(contents_dict): + for path, contents in contents_dict.items(): + responses.add( + method=responses.GET, + url=f"{data_config['inventory_provider']}{path}", + json=contents, + ) - cust_result = provision_folder(None, 'testfolder', dashboards['RE_CUST'], - data_config, 'testdatasource', ['GEANT']) - assert len(cust_result) == 2 - assert cust_result[0]['title'] == 'KIAE' - assert cust_result[1]['title'] == 'SWITCH' + return _populate @responses.activate -def test_provision(data_config, mocker, client): - - responses.add( - method=responses.GET, - url=f"{data_config['reporting_provider']}/scid/current", - json=get_test_data('services.json')) - - responses.add( - method=responses.GET, - url=f"{data_config['inventory_provider']}/poller/interfaces", - json=NREN_INTERFACES) - - responses.add( - method=responses.GET, - url=f"{data_config['inventory_provider']}/data/interfaces", - json=NREN_INTERFACES) - - responses.add( - method=responses.GET, - url=f'{data_config["inventory_provider"]}/poller/eumetsat-multicast', - json=EUMETSAT_MULTICAST) - - responses.add( - method=responses.DELETE, - url=f"http://{data_config['hostname']}/api/folders", - json={"message": "Deleted folder"}) - - responses.add( - method=responses.GET, - url=f"http://{data_config['hostname']}/api/folders", - json=[ - generate_folder({'uid': 'fakeuid', 'title': 'fakefolder'})]) - - def folder_post(request): - data = json.loads(request.body) - return 200, {}, json.dumps(generate_folder(data)) - - responses.add_callback( - method=responses.POST, - url=f"http://{data_config['hostname']}/api/folders", - callback=folder_post) - - def search_responses(request): - if request.params.get('query', None) == 'Home': - return 200, {}, json.dumps([]) - if request.params.get('type', None) == 'dash-db': - return 200, {}, json.dumps([]) - assert False # no other queries expected +@pytest.mark.parametrize( + "folder_name, excluded_nrens, expected_nrens", + [ + ("NREN Access", [], ["GEANT", "KIAE", "SWITCH"]), + ("NREN Access", ["GEANT", "KIAE"], ["SWITCH"]), + ( + "NREN Access BETA", + [], + ["ASNET-AM", "LITNET", "CESNET", "GEANT", "KIAE", "SWITCH"], + ), + ( + "NREN Access BETA", + ["ASNET-AM", "GEANT"], + ["LITNET", "CESNET", "KIAE", "SWITCH"], + ), + ("testfolder", ["GEANT"], ["KIAE", "SWITCH"]), + ], +) +def test_provision_nren_folder( + folder_name, + excluded_nrens, + expected_nrens, + data_config, + mock_grafana, + reporting_provider, + populate_inventory, +): + dashboards = { + "NREN": { + "tag": ["customers"], + "folder_name": "NREN Access", + "interfaces": [ + iface for iface in TEST_INTERFACES if "NREN" in iface["dashboards"] + ], + }, + "RE_CUST": { + "tag": "RE_CUST", + "folder_name": "RE Customer", + "interfaces": [ + iface for iface in TEST_INTERFACES if "RE_CUST" in iface["dashboards"] + ], + }, + } + populate_inventory( + { + "/poller/interfaces": NREN_INTERFACES, + "/data/interfaces": NREN_INTERFACES, + "/poller/eumetsat-multicast": EUMETSAT_MULTICAST, + } + ) + + result = provision_folder( + mock_grafana.request, + folder_name, + dashboards["NREN"], + data_config, + "testdatasource", + excluded_nrens, + ) + assert len(result) == len(expected_nrens) + for i, nren in enumerate(expected_nrens): + assert result[i]["title"] == nren + if "NREN" in folder_name: + # Every NREN dashboard must have at least 4 panels + # (3 default panels and 1 per ifc) + assert len(result[i]["panels"]) > 3 - responses.add_callback( - method=responses.GET, - url=f"http://{data_config['hostname']}/api/search", - callback=search_responses) - responses.add( - method=responses.GET, - url=f"http://{data_config['hostname']}/api/datasources", - json=[{ +@responses.activate +def test_provision( + data_config, mocker, mock_grafana, reporting_provider, populate_inventory +): + mock_grafana.create_datasource( + { "name": "brian-influx-datasource", "type": "influxdb", "access": "proxy", @@ -769,63 +708,20 @@ def test_provision(data_config, mocker, client): "database": "test-db", "basicAuth": False, "isDefault": True, - "readOnly": False - }]) - - responses.add( - method=responses.POST, - url=f"http://{data_config['hostname']}/api/dashboards/db", - json={'uid': '999', 'id': 666}) - - responses.add( - method=responses.PUT, - url=f"http://{data_config['hostname']}/api/org/preferences", - json={'message': 'Preferences updated'}) - - def homedashboard(request): - return 404, {}, '' - - responses.add_callback( - method=responses.GET, - url=f"http://{data_config['hostname']}/api/dashboards/uid/home", - callback=homedashboard) - - PROVISIONED_ORGANIZATION = { - 'name': data_config['organizations'][0], - 'id': 0 - } - - EXISTING_ORGS = [{**org, 'id': i + 1} - for i, org in enumerate(data_config['organizations'][1:])] - - _mocked_get_organizations = mocker.patch( - 'brian_dashboard_manager.grafana.provision.get_organizations') - # all organizations are provisioned except the first one. - _mocked_get_organizations.return_value = EXISTING_ORGS.copy() - - _mocked_create_organization = mocker.patch( - 'brian_dashboard_manager.grafana.provision.create_organization') - - # spoof creating first organization - _mocked_create_organization.return_value = PROVISIONED_ORGANIZATION - - _mocked_delete_expired_api_tokens = mocker.patch( - 'brian_dashboard_manager.grafana.provision.delete_expired_api_tokens') - # we dont care about this, , tested separately - _mocked_delete_expired_api_tokens.return_value = None - - _mocked_create_api_token = mocker.patch( - 'brian_dashboard_manager.grafana.provision.create_api_token') - _mocked_create_api_token.return_value = { - 'key': 'testtoken', 'id': 0} # api token - - _mocked_create_datasource = mocker.patch( - 'brian_dashboard_manager.grafana.provision.create_datasource') - # we dont care about this, just mark it created - _mocked_create_datasource.return_value = True - - _mocked_get_dashboard_definitions = mocker.patch( - 'brian_dashboard_manager.grafana.provision.get_dashboard_definitions') + "readOnly": False, + } + ) + populate_inventory( + { + "/poller/interfaces": NREN_INTERFACES, + "/data/interfaces": NREN_INTERFACES, + "/poller/eumetsat-multicast": EUMETSAT_MULTICAST, + } + ) + for org in data_config["organizations"][1:]: + mock_grafana.create_organization(org) + + mock_grafana.create_dashboard({"title": "testdashboard", "version": 1}) _mocked_gws = mocker.patch( 'brian_dashboard_manager.grafana.provision.get_gws_direct') @@ -835,23 +731,54 @@ def test_provision(data_config, mocker, client): 'brian_dashboard_manager.grafana.provision.get_gws_indirect') _mocked_gws_indirect.return_value = [] - UID = 1 - ID = 1 - VERSION = 1 - TITLE = 'testdashboard' - dashboard = {'id': ID, 'uid': UID, 'title': TITLE, 'version': VERSION} - _mocked_get_dashboard_definitions.return_value = [ - dashboard # test dashboard - ] + provision(data_config, raise_exceptions=True) - _mocked_create_dashboard = mocker.patch( - 'brian_dashboard_manager.grafana.provision.create_dashboard') - # we dont care about this, just mark it created - # we dont care about this, tested separately - _mocked_create_dashboard.return_value = {'uid': '999', 'id': 666} - _mocked_delete_api_token = mocker.patch( - 'brian_dashboard_manager.grafana.provision.delete_api_token') - # we dont care about this, tested separately - _mocked_delete_api_token.return_value = None - provision(data_config) +@responses.activate +def test_provision_re_peer_dashboard( + mocker, data_config, mock_grafana, reporting_provider, populate_inventory +): + interfaces = [ + { + "router": "mx1.dub2.ie.geant.net", + "name": "xe-0/0/0.1", + "description": "PHY SVC P_AE10 SRF9948758 | HEANET-AP2-LL3", # noqa: E501 + "dashboards": ["RE_PEER"], + "dashboard_info": {"name": "ESNET", "interface_type": "LOGICAL"}, + "dashboards_info": [{"name": "ESNET", "interface_type": "LOGICAL"}], + "ipv4": ["1.1.1.1"], + "ipv6": ["::2"], + }, + ] + populate_inventory( + { + "/poller/interfaces": interfaces, + "/data/interfaces": interfaces, + "/poller/eumetsat-multicast": EUMETSAT_MULTICAST, + } + ) + _mocked_gws = mocker.patch( + "brian_dashboard_manager.grafana.provision.get_gws_direct" + ) + _mocked_gws.return_value = [] + + _mocked_gws_indirect = mocker.patch( + "brian_dashboard_manager.grafana.provision.get_gws_indirect" + ) + _mocked_gws_indirect.return_value = [] + data_config["organizations"] = [ + {"name": "Testorg1", "excluded_nrens": ["GEANT"], "excluded_dashboards": []}, + ] + provision(data_config, raise_exceptions=True) + folder_uid = "RE_Peer" + assert len(mock_grafana.dashboards_by_folder_uid[folder_uid]) == 1 + panels = mock_grafana.dashboards_by_folder_uid[folder_uid][0]["panels"] + expected_types = ["text", "graph", "graph", "row", "graph", "graph", "row"] + assert [p["type"] for p in panels] == expected_types + assert "INFO" in panels[0]["title"] + assert "ingress" in panels[1]["title"] + assert "egress" in panels[2]["title"] + assert "Services" in panels[3]["title"] + assert "traffic" in panels[4]["title"] + assert "IPv6" in panels[5]["title"] + assert "Interfaces" in panels[6]["title"] diff --git a/tox.ini b/tox.ini index dc78bc46551ca55171192633bc42e0c3eaaa935f..6b749b99e29ebfbdbe4980c958b1845fba1e7ae5 100644 --- a/tox.ini +++ b/tox.ini @@ -10,13 +10,12 @@ concurrency = multiprocessing,thread [testenv] deps = - pytest-xdist pytest-cov flake8 -r requirements.txt commands = coverage erase - pytest -n auto --cov brian_dashboard_manager --cov-fail-under=80 --cov-report html --cov-report xml --cov-report term -p no:checkdocs + pytest --cov brian_dashboard_manager --cov-fail-under=80 --cov-report html --cov-report xml --cov-report term -p no:checkdocs flake8 sphinx-build -M html docs/source docs/build