From d22d8970b0136da72951eb66c636b2489ffb30ff Mon Sep 17 00:00:00 2001
From: Bjarke Madsen <bjarke.madsen@geant.org>
Date: Wed, 24 Feb 2021 11:49:53 +0100
Subject: [PATCH] Provision dashboards generated with jinja
---
brian_dashboard_manager/grafana/provision.py | 177 ++++++++++++++++---
1 file changed, 153 insertions(+), 24 deletions(-)
diff --git a/brian_dashboard_manager/grafana/provision.py b/brian_dashboard_manager/grafana/provision.py
index b258101..bafbbd1 100644
--- a/brian_dashboard_manager/grafana/provision.py
+++ b/brian_dashboard_manager/grafana/provision.py
@@ -1,20 +1,29 @@
import logging
+from brian_dashboard_manager.config import DEFAULT_ORGANIZATIONS
from brian_dashboard_manager.grafana.utils.request import \
AdminRequest, \
TokenRequest
from brian_dashboard_manager.grafana.organization import \
- get_organizations, \
- create_organization, \
- create_api_token, \
- delete_api_token, \
- delete_expired_api_tokens
+ get_organizations, create_organization, create_api_token, \
+ delete_api_token, delete_expired_api_tokens, set_home_dashboard
from brian_dashboard_manager.grafana.dashboard import \
- get_dashboard_definitions, \
- create_dashboard
+ get_dashboard_definitions, create_dashboard, find_dashboard
from brian_dashboard_manager.grafana.datasource import \
- get_missing_datasource_definitions,\
- create_datasource
+ check_provisioned, create_datasource
+from brian_dashboard_manager.grafana.folder import \
+ get_folders, create_folder
+from brian_dashboard_manager.inventory_provider.interfaces import \
+ get_interfaces
+from brian_dashboard_manager.templating.nren_access import generate_nrens
+from brian_dashboard_manager.templating.helpers import is_re_customer, \
+ is_cls, is_ias_customer, is_ias_private, is_ias_public, is_ias_upstream, \
+ is_lag_backbone, is_nren, is_phy_upstream, is_re_peer, is_gcs, \
+ is_geantopen, is_l2circuit, is_lhcone_peer, is_lhcone_customer, is_mdvpn,\
+ get_interface_data, parse_backbone_name, parse_phy_upstream_name, \
+ get_dashboard_data
+
+from brian_dashboard_manager.templating.render import render_dashboard
logger = logging.getLogger(__name__)
@@ -24,36 +33,156 @@ def provision(config):
request = AdminRequest(**config)
all_orgs = get_organizations(request)
- organizations_to_provision = config.get('organizations', [])
+ orgs_to_provision = config.get('organizations', DEFAULT_ORGANIZATIONS)
- missing = (name for name in organizations_to_provision
- if name not in [org['name'] for org in all_orgs])
+ missing = (org['name'] for org in orgs_to_provision
+ if org['name'] not in [org['name'] for org in all_orgs])
- for org in missing:
- org_data = create_organization(request, org)
+ for org_name in missing:
+ org_data = create_organization(request, org_name)
all_orgs.append(org_data)
+ interfaces = get_interfaces(config['inventory_provider'])
for org in all_orgs:
org_id = org['id']
delete_expired_api_tokens(request, org_id)
token = create_api_token(request, org_id)
token_request = TokenRequest(token=token['key'], **config)
- logger.info(f'--- Provisioning org {org["name"]} (ID #{org_id}) ---')
+ folders = get_folders(token_request)
+
+ def find_folder(title):
+
+ try:
+ folder = next(
+ f for f in folders if f['title'].lower() == title.lower())
+ except StopIteration:
+ folder = None
+
+ if not folder:
+ logger.info(f'Created folder: {title}')
+ folder = create_folder(token_request, title)
+ folders.append(folder)
+ return folder
+
+ logger.info(
+ f'--- Provisioning org {org["name"]} (ID #{org_id}) ---')
+
+ try:
+ org_config = next(
+ o for o in orgs_to_provision if o['name'] == org['name'])
+ except StopIteration:
+ org_config = None
+
+ if not org_config:
+ logger.error(
+ f'Org {org["name"]} does not have valid configuration.')
+ org['info'] = 'Org exists in grafana but is not configured'
+ continue
+
+ # Only provision influxdb datasource for now
+ datasource = config.get('datasources').get('influxdb')
# Provision missing data sources
- for datasource in get_missing_datasource_definitions(token_request):
- if datasource:
- ds = create_datasource(token_request,
- datasource,
- config.get('datasources'))
- if ds:
- logger.info(
- f'Provisioned datasource: {datasource["name"]}')
+ if not check_provisioned(token_request, datasource):
+ ds = create_datasource(token_request,
+ datasource,
+ config.get('datasources'))
+ if ds:
+ logger.info(
+ f'Provisioned datasource: {datasource["name"]}')
+
+ excluded_nrens = org_config.get('excluded_nrens', [])
+ excluded_nrens = list(map(lambda f: f.lower(), excluded_nrens))
+
+ def excluded(interface):
+ desc = interface.get('description', '').lower()
+ return not any(nren.lower() in desc for nren in excluded_nrens)
+ excluded_interfaces = list(filter(excluded, interfaces))
+
+ dashboards = {
+ 'CLS': {'predicate': is_cls, 'tag': 'CLS'},
+ 'RE PEER': {'predicate': is_re_peer, 'tag': 'RE_PEER'},
+ 'RE CUST': {'predicate': is_re_customer, 'tag': 'RE_CUST'},
+ 'GEANTOPEN': {'predicate': is_geantopen, 'tag': 'GEANTOPEN'},
+ 'GCS': {'predicate': is_gcs, 'tag': 'AUTOMATED_L2_CIRCUITS'},
+ 'L2 CIRCUIT': {'predicate': is_l2circuit, 'tag': 'L2_CIRCUITS'},
+ 'LHCONE PEER': {'predicate': is_lhcone_peer, 'tag': 'LHCONE_PEER'},
+ 'LHCONE CUST': {
+ 'predicate': is_lhcone_customer,
+ 'tag': 'LHCONE_CUST'
+ },
+ 'MDVPN Customers': {'predicate': is_mdvpn, 'tag': 'MDVPN'},
+ 'Infrastructure Backbone': {
+ 'predicate': is_lag_backbone,
+ 'tag': 'BACKBONE',
+ 'errors': True,
+ 'parse_func': parse_backbone_name
+ },
+ 'IAS PRIVATE': {'predicate': is_ias_private, 'tag': 'IAS_PRIVATE'},
+ 'IAS PUBLIC': {'predicate': is_ias_public, 'tag': 'IAS_PUBLIC'},
+ 'IAS CUSTOMER': {
+ 'predicate': is_ias_customer,
+ 'tag': 'IAS_CUSTOMER'
+ },
+ 'IAS UPSTREAM': {
+ 'predicate': is_ias_upstream,
+ 'tag': 'IAS_UPSTREAM'
+ },
+ 'GWS PHY Upstream': {
+ 'predicate': is_phy_upstream,
+ 'tag': 'GWS_UPSTREAM',
+ 'errors': True,
+ 'parse_func': parse_phy_upstream_name
+ }
+
+ }
# Provision dashboards, overwriting existing ones.
+
+ datasource_name = datasource.get('name', 'PollerInfluxDB')
+ for folder_name, dash in dashboards.items():
+ folder = find_folder(folder_name)
+ predicate = dash['predicate']
+ tag = dash['tag']
+
+ # dashboard will include error panel
+ errors = dash.get('errors')
+
+ # custom parsing function for description to dashboard name
+ parse_func = dash.get('parse_func')
+
+ logger.info(f'Provisioning {folder_name} dashboards')
+
+ relevant_interfaces = filter(predicate, excluded_interfaces)
+ data = get_interface_data(relevant_interfaces, parse_func)
+ dash_data = get_dashboard_data(data, datasource_name, tag, errors)
+ for dashboard in dash_data:
+ rendered = render_dashboard(dashboard)
+ create_dashboard(token_request, rendered, folder['id'])
+
+ # NREN Access dashboards
+ # uses a different template than the above.
+ logger.info('Provisioning NREN Access dashboards')
+ folder = find_folder('NREN Access')
+ nrens = filter(is_nren, excluded_interfaces)
+ for dashboard in generate_nrens(nrens, datasource_name):
+ create_dashboard(token_request, dashboard, folder['id'])
+
+ # Non-generated dashboards
+ excluded_dashboards = org_config.get('excluded_dashboards', [])
+ logger.info('Provisioning static dashboards')
for dashboard in get_dashboard_definitions():
- create_dashboard(token_request, dashboard)
+ if dashboard['title'] not in excluded_dashboards:
+ if dashboard['title'].lower() == 'home':
+ dashboard['uid'] = 'home'
+ create_dashboard(token_request, dashboard)
+
+ # Home dashboard is always called "Home"
+ # Make sure it's set for the organization
+ home_dashboard = find_dashboard(token_request, 'Home')
+ if home_dashboard:
+ set_home_dashboard(token_request, home_dashboard['id'])
delete_api_token(request, org_id, token['id'])
--
GitLab