Skip to content
Snippets Groups Projects
Commit 88751ac8 authored by Bjarke Madsen's avatar Bjarke Madsen
Browse files

Delete stale dashboards after provisioning.

parent 66608ca0
No related branches found
No related tags found
No related merge requests found
......@@ -74,12 +74,27 @@ def delete_dashboards(request: TokenRequest):
# Searches for a dashboard with given title
def find_dashboard(request: TokenRequest, title):
r = request.get('api/search', params={
'query': title
})
def find_dashboard(request: TokenRequest, title=None):
param = {
**({'query': title} if title else {}),
'type': 'dash-db',
'limit': 5000,
'page': 1
}
r = request.get('api/search', params=param)
if r and len(r) > 0:
return r[0]
if title:
return r[0]
else:
while True:
param['page'] += 1
page = request.get('api/search', params=param)
if len(page) > 0:
r.extend(page)
else:
break
return r
return None
......
......@@ -7,6 +7,7 @@ import logging
import time
import json
import datetime
from functools import reduce
from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor
from brian_dashboard_manager.config import DEFAULT_ORGANIZATIONS, STATE_PATH
from brian_dashboard_manager.grafana.utils.request import \
......@@ -15,7 +16,7 @@ from brian_dashboard_manager.grafana.utils.request import \
from brian_dashboard_manager.grafana.organization import \
get_organizations, create_organization, create_api_token, \
delete_api_token, delete_expired_api_tokens, set_home_dashboard
from brian_dashboard_manager.grafana.dashboard import \
from brian_dashboard_manager.grafana.dashboard import find_dashboard, \
get_dashboard_definitions, create_dashboard, delete_dashboard
from brian_dashboard_manager.grafana.datasource import \
check_provisioned, create_datasource
......@@ -39,10 +40,13 @@ logger = logging.getLogger(__name__)
def generate_all_nrens(token_request, nrens, folder_id, datasource_name):
provisioned = []
with ThreadPoolExecutor(max_workers=8) as executor:
for dashboard in generate_nrens(nrens, datasource_name):
executor.submit(create_dashboard, token_request,
dashboard, folder_id)
res = executor.submit(create_dashboard, token_request,
dashboard, folder_id)
provisioned.append(res)
return [r.result() for r in provisioned]
def provision_folder(token_request, folder_name,
......@@ -70,6 +74,8 @@ def provision_folder(token_request, folder_name,
excluded_dashboards = list(
map(lambda s: s.lower(), excluded_dashboards))
provisioned = []
with ThreadPoolExecutor(max_workers=4) as executor:
for dashboard in dash_data:
rendered = render_dashboard(dashboard)
......@@ -77,8 +83,9 @@ def provision_folder(token_request, folder_name,
executor.submit(delete_dashboard, token_request,
rendered, folder['id'])
continue
executor.submit(create_dashboard, token_request,
rendered, folder['id'])
provisioned.append(executor.submit(create_dashboard, token_request,
rendered, folder['id']))
return [r.result() for r in provisioned]
def provision_aggregate(token_request, agg_type, aggregate_folder,
......@@ -93,7 +100,7 @@ def provision_aggregate(token_request, agg_type, aggregate_folder,
f'Aggregate - {agg_type}', data, datasource_name, tag)
rendered = render_dashboard(dashboard)
create_dashboard(token_request, rendered, aggregate_folder['id'])
return create_dashboard(token_request, rendered, aggregate_folder['id'])
def provision_maybe(config):
......@@ -247,7 +254,17 @@ def provision(config):
datasource_name = datasource.get('name', 'PollerInfluxDB')
excluded_folders = org_config.get('excluded_folders', {})
def get_uid(prev, curr):
prev[curr.get('uid')] = False
return prev
# Map of dashboard UID -> whether it has been updated.
# This is used to remove stale dashboards at the end.
dash_list = find_dashboard(token_request) or []
dash_list = reduce(get_uid, dash_list, {})
with ProcessPoolExecutor(max_workers=4) as executor:
provisioned = []
for folder_name, dash in dashboards.items():
exclude = excluded_folders.get(folder_name)
if exclude:
......@@ -260,10 +277,19 @@ def provision(config):
logger.info(
f'Provisioning {org["name"]}/{folder_name} dashboards')
executor.submit(provision_folder, token_request,
folder_name, dash,
excluded_interfaces, datasource_name,
exclude)
res = executor.submit(provision_folder, token_request,
folder_name, dash,
excluded_interfaces, datasource_name,
exclude)
provisioned.append(res)
for result in provisioned:
folder = result.result()
if folder is None:
continue
for dashboard in folder:
if dashboard is None:
continue
dash_list[dashboard.get('uid')] = True
aggregate_dashboards = {
'CLS PEERS': {
......@@ -296,6 +322,7 @@ def provision(config):
pass
else:
with ProcessPoolExecutor(max_workers=4) as executor:
provisioned = []
agg_folder = find_folder(token_request, 'Aggregates')
for agg_type, dash in aggregate_dashboards.items():
if agg_type in exclude_agg:
......@@ -306,28 +333,40 @@ def provision(config):
continue
logger.info(f'Provisioning {org["name"]}' +
f'/Aggregate {agg_type} dashboards')
executor.submit(provision_aggregate, token_request,
agg_type, agg_folder, dash,
excluded_interfaces, datasource_name)
res = executor.submit(provision_aggregate, token_request,
agg_type, agg_folder, dash,
excluded_interfaces, datasource_name)
provisioned.append(res)
for result in provisioned:
dashboard = result.result()
if dashboard is None:
continue
dash_list[dashboard.get('uid')] = True
# NREN Access dashboards
# uses a different template than the above.
logger.info('Provisioning NREN Access dashboards')
# always recreate NREN folder
delete_folder(token_request, 'NREN Access')
folder = find_folder(token_request, 'NREN Access')
nrens = filter(is_nren, excluded_interfaces)
generate_all_nrens(token_request,
nrens, folder['id'], datasource_name)
provisioned = generate_all_nrens(
token_request, nrens, folder['id'], datasource_name)
for dashboard in provisioned:
if dashboard is None:
continue
dash_list[dashboard.get('uid')] = True
# Non-generated dashboards
excluded_dashboards = org_config.get('excluded_dashboards', [])
logger.info('Provisioning static dashboards')
for dashboard in get_dashboard_definitions():
if dashboard['title'] not in excluded_dashboards:
create_dashboard(token_request, dashboard)
res = create_dashboard(token_request, dashboard)
if res:
dash_list[res.get('uid')] = True
else:
delete_dashboard(token_request, dashboard)
......@@ -336,6 +375,13 @@ def provision(config):
logger.info('Configuring Home dashboard')
is_staff = org['name'] == 'GÉANT Staff'
set_home_dashboard(token_request, is_staff)
# just hardcode that we updated home dashboard
dash_list['home'] = True
for dash, provisioned in dash_list.items():
if not provisioned:
logger.info(f'Deleting stale dashboard with UID {dash}')
delete_dashboard(token_request, {'uid': dash})
logger.info(f'Time to complete: {time.time() - start}')
for org_id, token in tokens:
......
import responses
import json
import re
from brian_dashboard_manager.grafana.utils.request import TokenRequest
from brian_dashboard_manager.templating.nren_access import get_nrens
from brian_dashboard_manager.grafana.provision import provision_folder, \
generate_all_nrens, provision
......@@ -241,6 +242,7 @@ def test_provision_folder(data_config, mocker):
'testdatasource', ['CLS TESTDASHBOARD'])
@responses.activate
def test_provision_nrens(data_config, mocker):
NREN_INTERFACES = [
# physical
......@@ -299,12 +301,34 @@ def test_provision_nrens(data_config, mocker):
}
]
UID = '1337'
def get_callback(request):
query = request.params.get('query')
return 200, {}, json.dumps({'uid': UID, 'title': query})
responses.add_callback(
method=responses.GET,
url=re.compile(f"http://{data_config['hostname']}/api/search"),
callback=get_callback)
def post_callback(request):
dashboard = json.loads(request.body).get('dashboard')
title = dashboard.get('title')
return 200, {}, json.dumps({'uid': UID, 'title': title})
responses.add_callback(
method=responses.POST,
url=re.compile(f"http://{data_config['hostname']}/api/dashboards/db"),
callback=post_callback)
nrens = get_nrens(NREN_INTERFACES)
assert len(nrens) == 1 and nrens.get('HEANET') is not None
assert len(nrens.get('HEANET').get('AGGREGATES')) == 1
assert len(nrens.get('HEANET').get('SERVICES')) == 1
assert len(nrens.get('HEANET').get('PHYSICAL')) == 2
generate_all_nrens(None, NREN_INTERFACES, 1, 'testdatasource')
token_request = TokenRequest(token='testtoken', **data_config)
generate_all_nrens(token_request, NREN_INTERFACES, 1, 'testdatasource')
@responses.activate
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment