"""
This module is responsible for the
entire provisioning lifecycle.
"""
import itertools
import logging
import time
from concurrent.futures import Future
from concurrent.futures import ThreadPoolExecutor
from brian_dashboard_manager.config import DEFAULT_ORGANIZATIONS
from brian_dashboard_manager.grafana.utils.request import AdminRequest, \
TokenRequest, MAX_THREADS
from brian_dashboard_manager.services.api import fetch_services
from brian_dashboard_manager.grafana.organization import \
get_organizations, create_organization, create_api_token, \
delete_api_token, delete_expired_api_tokens, set_home_dashboard, \
get_or_create_service_account, delete_service_account, create_service_account_token
from brian_dashboard_manager.grafana.dashboard import list_dashboards, \
get_dashboard_definitions, create_dashboard, delete_dashboard
from brian_dashboard_manager.grafana.datasource import \
datasource_exists, create_datasource
from brian_dashboard_manager.grafana.folder import find_folder, \
delete_folder, delete_unknown_folders, list_folder_dashboards
from brian_dashboard_manager.inventory_provider.interfaces import \
get_gws_direct, get_gws_indirect, get_interfaces, \
get_eumetsat_multicast_subscriptions, get_nren_regions
from brian_dashboard_manager.templating.helpers import \
get_aggregate_dashboard_data, get_interface_data, \
get_nren_interface_data, get_dashboard_data, \
get_nren_dashboard_data, get_aggregate_interface_data, \
get_nren_interface_data_old, get_re_peer_dashboard_data, get_re_peer_interface_data, get_service_data, \
get_service_dashboard_data
from brian_dashboard_manager.templating.gws import generate_gws, generate_indirect
from brian_dashboard_manager.templating.eumetsat import generate_eumetsat_multicast
from brian_dashboard_manager.templating.render import (
render_with_aggregate_dashboard,
render_simple_dashboard,
)
logger = logging.getLogger(__name__)
DASHBOARDS = {
'NRENLEGACY': {
'tag': ['customerslegacy'],
'folder_name': 'NREN Access LEGACY',
'interfaces': []
},
'NREN': {
'tag': ['customers'],
'folder_name': 'NREN Access',
'interfaces': []
},
'EAP': {
'tag': ['eap'],
'folder_name': 'EAP NREN Access',
'interfaces': []
},
'RE_PEER': {
'tag': 'RE_PEER',
'folder_name': 'RE Peer',
'interfaces': []
},
'RE_CUST': {
'tag': 'RE_CUST',
'folder_name': 'RE Customer',
'interfaces': []
},
'GEANTOPEN': {
'tag': 'GEANTOPEN',
'folder_name': 'GEANTOPEN',
'interfaces': []
},
'GCS': {
'tag': 'AUTOMATED_L2_CIRCUITS',
'folder_name': 'GCS',
'interfaces': []
},
'L2_CIRCUIT': {
'tag': 'L2_CIRCUITS',
'folder_name': 'L2 Circuit',
'interfaces': []
},
'LHCONE_PEER': {
'tag': 'LHCONE_PEER',
'folder_name': 'LHCONE Peer',
'interfaces': []
},
'LHCONE_CUST': {
'tag': 'LHCONE_CUST',
'folder_name': 'LHCONE Customer',
'interfaces': []
},
'MDVPN_CUSTOMERS': {
'tag': 'MDVPN',
'folder_name': 'MDVPN Customers',
'interfaces': []
},
'INFRASTRUCTURE_BACKBONE': {
'tag': 'BACKBONE',
'errors': True,
'folder_name': 'Infrastructure Backbone',
'interfaces': []
},
'IAS_PRIVATE': {
'tag': 'IAS_PRIVATE',
'folder_name': 'IAS Private',
'interfaces': []
},
'IAS_PUBLIC': {
'tag': 'IAS_PUBLIC',
'folder_name': 'IAS Public',
'interfaces': []
},
'IAS_CUSTOMER': {
'tag': 'IAS_CUSTOMER',
'folder_name': 'IAS Customer',
'interfaces': []
},
'IAS_UPSTREAM': {
'tag': ['IAS_UPSTREAM', 'UPSTREAM'],
'folder_name': 'IAS Upstream',
'interfaces': []
},
'GWS_PHY_UPSTREAM': {
'tag': ['GWS_UPSTREAM', 'UPSTREAM'],
'errors': True,
'folder_name': 'GWS PHY Upstream',
'interfaces': []
},
'GBS_10G': {
'tag': 'GBS_10G',
'errors': True,
'folder_name': '10G Guaranteed Bandwidth Service',
'interfaces': []
}
}
SERVICE_DASHBOARDS = {
'MWS': {
'tag': ['mws'],
'service_type': 'GEANT MANAGED WAVELENGTH SERVICE',
'folder_name': 'Managed Wavelength Service',
'interfaces': [],
'services': []
}
}
AGG_DASHBOARDS = {
'CLS_PEERS': {
'tag': 'cls_peers',
'dashboard_name': 'CLS Peers',
'interfaces': []
},
'IAS_PEERS': {
'tag': 'ias_peers',
'dashboard_name': 'IAS Peers',
'interfaces': []
},
'IAS_UPSTREAM': {
'tag': 'gws_upstreams',
'dashboard_name': 'GWS Upstreams',
'interfaces': []
},
'LHCONE': {
'tag': 'lhcone',
'dashboard_name': 'LHCONE',
'interfaces': []
},
'CAE1': {
'tag': 'cae',
'dashboard_name': 'CAE1',
'interfaces': []
},
'IC1': {
'tag': ['ic1', 'peer-aggregate'],
'dashboard_name': 'IC-1',
'interfaces': []
},
'COPERNICUS': {
'tag': ['copernicus', 'services', 'peer-aggregate'],
'dashboard_name': 'COPERNICUS',
'group_by': 'location',
'interfaces': []
},
'ANA': {
'tag': ['ana', 'peer-aggregate'],
'dashboard_name': 'ANA',
'interfaces': []
},
'EAP': {
'tag': 'eap',
'dashboard_name': 'EAP Aggregate',
'interfaces': []
}
}
def provision_folder(thread_executor: ThreadPoolExecutor, token_request, folder_name, dash, services, regions,
ds_name, excluded_dashboards):
"""
Function to provision dashboards within a folder.
:param token_request: TokenRequest object
:param folder_name: Name of the folder to provision dashboards in
:param dash: the dashboards to provision, with interface data to generate
the dashboards from
:param services: service data from reporting provider for service-based dashboards
:param regions: region data from inventory provider to indicate what regions NRENs belong to
:param ds_name: the name of the datasource to query in the dashboard panels
:param excluded_dashboards: list of dashboards to exclude from provisioning
for the organisation
:return: list of dashboard definitions for the created dashboards
"""
if not isinstance(excluded_dashboards, (list, set)):
excluded_dashboards = set()
else:
excluded_dashboards = set([s.lower() for s in excluded_dashboards])
folder = find_folder(token_request, folder_name)
if not folder:
raise ValueError(f'Folder {folder_name} not found')
folder_dashboards_by_name = list_folder_dashboards(token_request, folder['uid'])
tag = dash['tag']
interfaces = list(
filter(
lambda x: x['dashboards_info'],
dash['interfaces']
)
)
def _get_customers_for_region(region=None):
customers = []
region_lookup = {region['nren']: region['region'] for region in regions}
for service in services:
service_customers = service.get('customers', [])
for cust in service_customers:
cust_region = region_lookup.get(cust)
if cust_region == region:
customers.append(cust)
return customers
# dashboard should include error panels
errors = dash.get('errors', False)
is_nren_legacy = folder_name == "NREN Access LEGACY"
is_nren = folder_name == "NREN Access"
is_eap = folder_name == "EAP NREN Access"
is_re_peer = folder_name == "RE Peer"
is_service = 'service_type' in dash
has_aggregate_panels = is_nren or is_eap or is_nren_legacy or is_re_peer or is_service
if is_nren_legacy:
data = get_nren_interface_data_old(interfaces)
dash_data = get_nren_dashboard_data(data, ds_name, tag)
elif is_nren or is_eap:
region_customers = _get_customers_for_region("EAP" if is_eap else None)
data = get_nren_interface_data(services, interfaces, excluded_dashboards, region_customers)
dash_data = get_nren_dashboard_data(data, ds_name, tag)
elif is_re_peer:
data = get_re_peer_interface_data(interfaces)
dash_data = get_re_peer_dashboard_data(data, ds_name, tag)
elif is_service:
data = get_service_data(dash['service_type'], services, interfaces, excluded_dashboards)
dash_data = get_service_dashboard_data(data, ds_name, tag)
else:
data = get_interface_data(interfaces)
dash_data = get_dashboard_data(data=data, datasource=ds_name, tag=tag, errors=errors)
for dashboard in dash_data:
if has_aggregate_panels:
rendered = render_with_aggregate_dashboard(**dashboard)
else:
rendered = render_simple_dashboard(**dashboard)
dash_title = rendered.get("title").lower()
if dash_title in excluded_dashboards:
if dash_title in folder_dashboards_by_name:
delete_dashboard(token_request, rendered, folder['id'])
continue
yield thread_executor.submit(create_dashboard, token_request, rendered, folder['id'], folder_dashboards_by_name)
def provision_aggregate(token_request, folder,
dash, ds_name, folder_dashboards_by_name):
"""
Function to provision an aggregate dashboard within a folder.
:param token_request: TokenRequest object
:param folder: the folder to provision dashboards in
:param dash: the dashboards to provision, with interface data to generate
the dashboards from
:param ds_name: the name of the datasource to query in the dashboard panels
:return: dashboard definition for the created dashboard
"""
name = dash['dashboard_name']
tag = dash['tag']
interfaces = dash['interfaces']
group_field = dash.get('group_by', 'remote')
data = get_aggregate_interface_data(interfaces, name, group_field)
dashboard = get_aggregate_dashboard_data(
f'Aggregate - {name}', data, ds_name, tag)
rendered = render_simple_dashboard(**dashboard)
return create_dashboard(token_request, rendered, folder['id'], folder_dashboards_by_name)
def is_excluded_folder(excluded_folders, folder_name):
"""
Function to determine if a folder should be excluded from provisioning.
:param excluded_folders: dict of excluded folders and dashboards
within them, it looks like this:
{
"Aggregates": ["GWS UPSTREAMS", "IAS PEERS"],
"IAS CUSTOMER": True,
"IAS PRIVATE": True,
"IAS PUBLIC": True,
"IAS UPSTREAM": True,
"GWS PHY Upstream": True,
"EUMETSAT Multicast": True,
"NREN Access BETA": True
}
If the value is True, the entire folder is excluded.
If the value is a list, the list contains the names of the dashboards
within the folder that should be excluded.
The case of a boolean `True` value is handled by this function.
The case of a list is handled at provision time by the
excluded_folder_dashboards and provision_folder functions.
:param folder_name: the name of the folder to check against the
excluded_folders
:return: True if the folder should be excluded, False otherwise
"""
excluded = excluded_folders.get(folder_name, False)
return isinstance(excluded, bool) and excluded
def excluded_folder_dashboards(org_config, folder_name):
"""
Function to get the list of dashboards to exclude from provisioning
for a given folder.
If the folder is the NREN Access folder, the list of excluded NRENs
is also added to the list of excluded dashboards.
:param org_config: the organisation config
:param folder_name: the name of the folder to check against the
excluded_folders
:return: list of dashboard names to exclude from provisioning for the
organisation
"""
excluded_folders = org_config.get('excluded_folders', {})
excluded = excluded_folders.get(folder_name, [])
if 'NREN Access' in folder_name:
excluded_nrens = org_config.get('excluded_nrens', [])
excluded = list(set(excluded).union(set(excluded_nrens)))
return excluded if isinstance(excluded, list) else []
def _interfaces_to_keep(interface, excluded_nrens):
dash_info = interface.get('dashboards_info')
if dash_info is None:
logger.info(f'No "dashboards_info" for '
f'{interface["router"]}:{interface["name"]}')
# throw it away
return False
dashboards = {nren['name'].lower() for nren in dash_info}
is_lab_router = 'lab.office' in interface['router'].lower()
should_keep = not (is_lab_router or any(
nren.lower() in dashboards for nren in excluded_nrens))
return should_keep
def _provision_interfaces(thread_executor: ThreadPoolExecutor, config,
org_config, ds_name, token, interfaces, services, regions):
"""
This function is used to provision most dashboards,
overwriting existing ones.
:param thread_executor: a ThreadPoolExecutor for concurrent requests
:param config: the application config
:param org_config: the organisation config
:param ds_name: the name of the datasource to query in the dashboards
:param token: a token_request object
:return: generator of dashboards that were created
"""
excluded_nrens = org_config['excluded_nrens']
excluded_folders = org_config.get('excluded_folders', {})
relevant_interfaces = list(filter(lambda x: _interfaces_to_keep(x, excluded_nrens), interfaces))
for interface in relevant_interfaces:
interface['dashboards_info'] = list(filter(
lambda x: x['name'] != '',
interface['dashboards_info']
))
# loop over interfaces and add them to the dashboard_name
# -> folder mapping structure `dashboards` above, for convenience.
for dash in DASHBOARDS:
DASHBOARDS[dash]['interfaces'] = []
for dash in AGG_DASHBOARDS:
AGG_DASHBOARDS[dash]['interfaces'] = []
for iface in relevant_interfaces:
for dash_name in iface['dashboards']:
# add interface to matched dashboard
if dash_name in DASHBOARDS:
ifaces = DASHBOARDS[dash_name]['interfaces']
ifaces.append(iface)
if dash_name == 'NREN':
# add to NRENLEGACY list of interfaces, used for legacy NREN dashboards (not service-based)
ifaces = DASHBOARDS['NRENLEGACY']['interfaces']
ifaces.append(iface)
# add to matched aggregate dashboard
if dash_name in AGG_DASHBOARDS:
ifaces = AGG_DASHBOARDS[dash_name]['interfaces']
ifaces.append(iface)
# provision dashboards and their folders
for folder in DASHBOARDS.values():
folder_name = folder['folder_name']
# boolean True means entire folder excluded
# if list, it is specific dashboard names not to provision
# so is handled at provision time.
if is_excluded_folder(excluded_folders, folder_name):
delete_folder(token, title=folder_name)
continue
logger.info(
f'Provisioning {org_config["name"]}/{folder_name} dashboards')
yield from provision_folder(thread_executor, token, folder_name, folder, services, regions, ds_name,
excluded_folder_dashboards(org_config, folder_name))
def _provision_gws_indirect(thread_executor: ThreadPoolExecutor, config, org_config, ds_name, token):
"""
This function is used to provision GWS Indirect dashboards,
overwriting existing ones.
:param thread_executor: a ThreadPoolExecutor for concurrent requests
:param config: the application config
:param org_config: the organisation config
:param ds_name: the name of the datasource to query in the dashboards
:param token: a token_request object
:return: generator of futures of dashboards that were created
"""
logger.info('Provisioning GWS Indirect dashboards')
folder_name = 'GWS Indirect'
excluded_folders = org_config.get('excluded_folders', {})
if is_excluded_folder(excluded_folders, folder_name):
# don't provision GWS Direct folder
delete_folder(token, title=folder_name)
else:
folder = find_folder(token, folder_name)
if not folder:
raise ValueError(f'Folder {folder_name} not found')
folder_dashboards_by_name = list_folder_dashboards(token, folder['uid'])
gws_indirect_data = get_gws_indirect(
config['inventory_provider'])
provisioned = []
dashes = generate_indirect(gws_indirect_data, ds_name)
for dashboard in dashes:
rendered = render_simple_dashboard(**dashboard)
provisioned.append(thread_executor.submit(create_dashboard,
token,
rendered, folder['id'], folder_dashboards_by_name))
yield from provisioned
def _provision_gws_direct(thread_executor: ThreadPoolExecutor, config, org_config, ds_name, token):
"""
This function is used to provision GWS Direct dashboards,
overwriting existing ones.
:param thread_executor: a ThreadPoolExecutor for concurrent requests
:param config: the application config
:param org_config: the organisation config
:param ds_name: the name of the datasource to query in the dashboards
:param token: a token_request object
:return: generator of futures of dashboards that were created
"""
logger.info('Provisioning GWS Direct dashboards')
folder_name = 'GWS Direct'
excluded_folders = org_config.get('excluded_folders', {})
if is_excluded_folder(excluded_folders, folder_name):
# don't provision GWS Direct folder
delete_folder(token, title=folder_name)
else:
folder = find_folder(token, folder_name)
if not folder:
raise ValueError(f'Folder {folder_name} not found')
folder_dashboards_by_name = list_folder_dashboards(token, folder['uid'])
gws_data = get_gws_direct(config['inventory_provider'])
provisioned = []
for dashboard in generate_gws(gws_data, ds_name):
rendered = render_simple_dashboard(**dashboard)
provisioned.append(
thread_executor.submit(create_dashboard, token, rendered, folder['id'], folder_dashboards_by_name)
)
yield from provisioned
def _provision_eumetsat_multicast(thread_executor: ThreadPoolExecutor, config, org_config, ds_name, token):
"""
This function is used to provision EUMETSAT Multicast dashboards,
overwriting existing ones.
:param thread_executor: a ThreadPoolExecutor for concurrent requests
:param config: the application config
:param org_config: the organisation config
:param ds_name: the name of the datasource to query in the dashboards
:param token: a token_request object
:return: generator of futures of dashboards that were created
"""
logger.info('Provisioning EUMETSAT Multicast dashboards')
folder_name = 'EUMETSAT Multicast'
excluded_folders = org_config.get('excluded_folders', {})
if is_excluded_folder(excluded_folders, folder_name):
# don't provision EUMETSAT Multicast folder
delete_folder(token, title=folder_name)
else:
folder = find_folder(token, folder_name)
if not folder:
raise ValueError(f'Folder {folder_name} not found')
folder_dashboards_by_name = list_folder_dashboards(token, folder['uid'])
subscriptions = get_eumetsat_multicast_subscriptions(
config['inventory_provider'])
provisioned = []
for dashboard in generate_eumetsat_multicast(subscriptions, ds_name):
rendered = render_simple_dashboard(**dashboard)
provisioned.append(
thread_executor.submit(create_dashboard, token, rendered, folder['id'], folder_dashboards_by_name)
)
yield from provisioned
def _provision_aggregates(thread_executor: ThreadPoolExecutor, config, org_config, ds_name, token):
"""
This function is used to provision Aggregate dashboards,
overwriting existing ones.
:param thread_executor: a ThreadPoolExecutor for concurrent requests
:param config: the application config
:param org_config: the organisation config
:param ds_name: the name of the datasource to query in the dashboards
:param token: a token_request object
:return: generator of futures of dashboards that were created
"""
excluded_folders = org_config.get('excluded_folders', {})
folder_name = 'Aggregates'
if is_excluded_folder(excluded_folders, folder_name):
# don't provision aggregate folder
delete_folder(token, title=folder_name)
else:
provisioned = []
agg_folder = find_folder(token, folder_name)
if not agg_folder:
raise ValueError(f'Folder {folder_name} not found')
folder_dashboards_by_name = list_folder_dashboards(token, agg_folder['uid'])
for dash in AGG_DASHBOARDS.values():
excluded_dashboards = excluded_folder_dashboards(org_config, folder_name)
if dash['dashboard_name'] in excluded_dashboards:
dash_name = {'title': f'Aggregate - {dash["dashboard_name"]}'}
delete_dashboard(token, dash_name, agg_folder['id'])
continue
logger.info(f'Provisioning {org_config["name"]}/Aggregate {dash["dashboard_name"]} dashboards')
provisioned.append(
thread_executor.submit(provision_aggregate, token, agg_folder, dash, ds_name, folder_dashboards_by_name)
)
yield from provisioned
def _provision_service_dashboards(thread_executor: ThreadPoolExecutor, config, org_config, ds_name, token):
"""
This function is used to provision service-specific dashboards,
overwriting existing ones.
:param thread_executor: a ThreadPoolExecutor for concurrent requests
:param config: the application config
:param org_config: the organisation config
:param ds_name: the name of the datasource to query in the dashboards
:param token: a token_request object
:return: generator of UIDs of dashboards that were created
"""
services = fetch_services(config['reporting_provider'])
regions = get_nren_regions(config['inventory_provider'])
excluded_folders = org_config.get('excluded_folders', {})
logger.info('Provisioning service-specific dashboards')
# loop over service dashboards and get service types we care about
dash_service_types = {SERVICE_DASHBOARDS[dash]['service_type']: dash for dash in SERVICE_DASHBOARDS}
# loop over services and append to dashboards
for service in services:
if service['service_type'] in dash_service_types:
dash = dash_service_types[service['service_type']]
svcs = SERVICE_DASHBOARDS[dash]['services']
svcs.append(service)
# provision dashboards and their folders
provisioned = []
for folder in SERVICE_DASHBOARDS.values():
folder_name = folder['folder_name']
# boolean True means entire folder excluded
# if list, it is specific dashboard names not to provision
# so is handled at provision time.
if is_excluded_folder(excluded_folders, folder_name):
delete_folder(token, title=folder_name)
continue
logger.info(
f'Provisioning {org_config["name"]}/{folder_name} dashboards')
res = thread_executor.submit(
provision_folder, thread_executor, token,
folder_name, folder, services, regions, ds_name,
excluded_folder_dashboards(org_config, folder_name))
provisioned.append(res)
for result in provisioned:
folder = result.result()
if folder is None:
continue
yield from folder
def _provision_static_dashboards(thread_executor: ThreadPoolExecutor, config, org_config, ds_name, token):
"""
This function is used to provision static dashboards from json files,
overwriting existing ones.
:param thread_executor: a ThreadPoolExecutor for concurrent requests
:param config: the application config
:param org_config: the organisation config
:param ds_name: the name of the datasource to query in the dashboards
:param token: a token_request object
:return: generator of UIDs of dashboards that were created
"""
# Statically defined dashboards from json files
excluded_dashboards = org_config.get('excluded_dashboards', [])
logger.info('Provisioning static dashboards')
provisioned = []
for dashboard in get_dashboard_definitions():
if dashboard['title'] not in excluded_dashboards:
logger.info(f'Provisioning static {dashboard["title"]} dashboard')
provisioned.append(thread_executor.submit(create_dashboard, token, dashboard))
else:
logger.info(f'Ensuring {dashboard["title"]} static dashboard is deleted')
delete_dashboard(token, dashboard)
yield from provisioned
# Home dashboard is always called "Home"
# Make sure it's set for the organization
logger.info('Configuring Home dashboard')
yield thread_executor.submit(set_home_dashboard, token, is_staff=org_config['name'] == 'GÉANT Staff')
def _get_ignored_dashboards(thread_executor: ThreadPoolExecutor, config, org_config, ds_name, token):
"""
This function is used to get a list of dashboards that should not be
touched by the provisioning process.
:param thread_executor: a ThreadPoolExecutor for concurrent requests
:param config: the application config
:param org_config: the organisation config
:param ds_name: the name of the datasource to query in the dashboards
:param token: a token_request object
:return: generator of UIDs of dashboards that should not be touched
"""
ignored_folders = config.get('ignored_folders', [])
for name in ignored_folders:
logger.info(
'Ignoring dashboards under '
f'the folder {org_config["name"]}/{name}')
folder = find_folder(token, name, create=False)
if folder is None:
continue
to_ignore = list_dashboards(token, folder_id=folder['id'])
if not to_ignore:
continue
for dash in to_ignore:
yield dash
def _provision_datasource(config, token):
"""
This function is used to provision the datasource from the config.
:param config: the application config
:param token: a token_request object
:return: the datasource config
"""
datasource = config.get('datasources').get('influxdb')
# Provision missing data sources
if not datasource_exists(token, datasource):
create_datasource(token, datasource)
return datasource
def _provision_orgs(config):
"""
This function is used to provision the organisations from the config.
:param config: the application config
:return: a list of all organisations
"""
request = AdminRequest(**config)
all_orgs = get_organizations(request)
orgs_to_provision = config.get('organizations', DEFAULT_ORGANIZATIONS)
missing = (org['name'] for org in orgs_to_provision
if org['name'] not in [org['name'] for org in all_orgs])
for org_name in missing:
org_data = create_organization(request, org_name)
all_orgs.append(org_data)
return all_orgs
def _provision_org(config, org, org_config, interfaces, services, regions):
try:
request = AdminRequest(**config)
org_id = org['id']
accounts = []
logger.info(f'--- Provisioning org {org["name"]} (ID #{org_id}) ---')
try:
# create a service account for provisioning (>grafana 11.0)
account = get_or_create_service_account(request, org_id)
token = create_service_account_token(request, account['id'])
accounts.append((org_id, account))
except Exception:
# we're on a older version of grafana
token = create_api_token(request, org_id)
accounts.append((org_id, token))
token_request = TokenRequest(token=token['key'], **config)
logger.debug(accounts)
all_original_dashboards = list_dashboards(token_request)
all_original_dashboard_uids = {d['uid']: d.get('folderUrl', '') + d['url'] for d in all_original_dashboards}
datasource = _provision_datasource(config, token_request)
ds_name = datasource.get('name', 'PollerInfluxDB')
with ThreadPoolExecutor(max_workers=MAX_THREADS) as thread_executor:
args = (thread_executor, config, org_config, ds_name, token_request)
# call to list is needed to queue up the futures
managed_dashboards = list(itertools.chain(
_provision_interfaces(*args, interfaces, services, regions),
_provision_gws_indirect(*args),
_provision_gws_direct(*args),
_provision_eumetsat_multicast(*args),
_provision_aggregates(*args),
_provision_service_dashboards(*args),
_provision_static_dashboards(*args),
_get_ignored_dashboards(*args)
))
managed_dashboard_uids = {}
for dashboard in managed_dashboards:
if isinstance(dashboard, Future):
dashboard = dashboard.result()
if dashboard is None:
continue
assert dashboard['uid'] not in managed_dashboard_uids, \
f'Dashboard with UID {dashboard["uid"]} already exists: {dashboard}'
managed_dashboard_uids[dashboard['uid']] = dashboard['url']
difference = set(all_original_dashboard_uids.keys()) - set(managed_dashboard_uids.keys())
for uid in difference:
info = all_original_dashboard_uids[uid]
# delete unmanaged dashboards
logger.info(f'Deleting stale dashboard {info} with UID {uid}')
delete_dashboard(token_request, {'uid': uid})
folders_to_keep = {
# General is a base folder present in Grafana
'General',
# other folders, created outside of the DASHBOARDS list
'GWS Indirect',
'GWS Direct',
'Aggregates',
'EUMETSAT Multicast',
'EAP Dashboard'
}
folders_to_keep.update({dash['folder_name']
for dash in DASHBOARDS.values()})
folders_to_keep.update({dash['folder_name']
for dash in SERVICE_DASHBOARDS.values()})
ignored_folders = config.get('ignored_folders', [])
folders_to_keep.update(ignored_folders)
delete_unknown_folders(token_request, folders_to_keep)
try:
delete_service_account(request, account['id'])
except Exception:
# we're on a older version of grafana
delete_api_token(request, token['id'], org_id=org_id)
except Exception:
logger.exception(f'Error when provisioning org {org["name"]}')
def provision(config):
"""
The entrypoint for the provisioning process.
Provisions organisations, datasources, and dashboards within Grafana.
Removes dashboards and folders not controlled by the provisioning process.
:param config: the application config
:return:
"""
start = time.time()
all_orgs = _provision_orgs(config)
request = AdminRequest(**config)
try:
# needed for older versions of grafana (<11.0)
delete_expired_api_tokens(request)
except Exception:
pass
def _find_org_config(org):
orgs_to_provision = config.get('organizations', DEFAULT_ORGANIZATIONS)
try:
return next(
o for o in orgs_to_provision if o['name'] == org['name'])
except StopIteration:
logger.info(
f'Org {org["name"]} does not have valid configuration.')
return None
orgs = list(filter(lambda t: t[1] is not None, [(org, _find_org_config(org)) for org in all_orgs]))
interfaces = get_interfaces(config['inventory_provider'])
services = fetch_services(config['reporting_provider'])
regions = get_nren_regions(config['inventory_provider'])
for org, org_config in orgs:
_provision_org(config, org, org_config, interfaces, services, regions)
logger.info(f'Time to complete: {time.time() - start}')
return all_orgs