diff --git a/brian_dashboard_manager/grafana/dashboard.py b/brian_dashboard_manager/grafana/dashboard.py
index 5b11704a675e52d603064b025e7ac589fe8a100c..012ccad4b8b783393c7d8162b1bee2d606fd6a4e 100644
--- a/brian_dashboard_manager/grafana/dashboard.py
+++ b/brian_dashboard_manager/grafana/dashboard.py
@@ -74,12 +74,27 @@ def delete_dashboards(request: TokenRequest):
# Searches for a dashboard with given title
-def find_dashboard(request: TokenRequest, title):
- r = request.get('api/search', params={
- 'query': title
- })
+def find_dashboard(request: TokenRequest, title=None):
+ param = {
+ **({'query': title} if title else {}),
+ 'type': 'dash-db',
+ 'limit': 5000,
+ 'page': 1
+ }
+ r = request.get('api/search', params=param)
if r and len(r) > 0:
- return r[0]
+ if title:
+ return r[0]
+ else:
+ while True:
+ param['page'] += 1
+ page = request.get('api/search', params=param)
+ if len(page) > 0:
+ r.extend(page)
+ else:
+ break
+ return r
+
return None
diff --git a/brian_dashboard_manager/grafana/provision.py b/brian_dashboard_manager/grafana/provision.py
index 80b5aae2c1ddae3e4df5c21017a117b0b17abe24..54b5bf90391a38e9cf48f87e5610f0bbba0ee793 100644
--- a/brian_dashboard_manager/grafana/provision.py
+++ b/brian_dashboard_manager/grafana/provision.py
@@ -7,6 +7,7 @@ import logging
import time
import json
import datetime
+from functools import reduce
from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor
from brian_dashboard_manager.config import DEFAULT_ORGANIZATIONS, STATE_PATH
from brian_dashboard_manager.grafana.utils.request import \
@@ -15,7 +16,7 @@ from brian_dashboard_manager.grafana.utils.request import \
from brian_dashboard_manager.grafana.organization import \
get_organizations, create_organization, create_api_token, \
delete_api_token, delete_expired_api_tokens, set_home_dashboard
-from brian_dashboard_manager.grafana.dashboard import \
+from brian_dashboard_manager.grafana.dashboard import find_dashboard, \
get_dashboard_definitions, create_dashboard, delete_dashboard
from brian_dashboard_manager.grafana.datasource import \
check_provisioned, create_datasource
@@ -39,10 +40,13 @@ logger = logging.getLogger(__name__)
def generate_all_nrens(token_request, nrens, folder_id, datasource_name):
+ provisioned = []
with ThreadPoolExecutor(max_workers=8) as executor:
for dashboard in generate_nrens(nrens, datasource_name):
- executor.submit(create_dashboard, token_request,
- dashboard, folder_id)
+ res = executor.submit(create_dashboard, token_request,
+ dashboard, folder_id)
+ provisioned.append(res)
+ return [r.result() for r in provisioned]
def provision_folder(token_request, folder_name,
@@ -70,6 +74,8 @@ def provision_folder(token_request, folder_name,
excluded_dashboards = list(
map(lambda s: s.lower(), excluded_dashboards))
+ provisioned = []
+
with ThreadPoolExecutor(max_workers=4) as executor:
for dashboard in dash_data:
rendered = render_dashboard(dashboard)
@@ -77,8 +83,9 @@ def provision_folder(token_request, folder_name,
executor.submit(delete_dashboard, token_request,
rendered, folder['id'])
continue
- executor.submit(create_dashboard, token_request,
- rendered, folder['id'])
+ provisioned.append(executor.submit(create_dashboard, token_request,
+ rendered, folder['id']))
+ return [r.result() for r in provisioned]
def provision_aggregate(token_request, agg_type, aggregate_folder,
@@ -93,7 +100,7 @@ def provision_aggregate(token_request, agg_type, aggregate_folder,
f'Aggregate - {agg_type}', data, datasource_name, tag)
rendered = render_dashboard(dashboard)
- create_dashboard(token_request, rendered, aggregate_folder['id'])
+ return create_dashboard(token_request, rendered, aggregate_folder['id'])
def provision_maybe(config):
@@ -247,7 +254,17 @@ def provision(config):
datasource_name = datasource.get('name', 'PollerInfluxDB')
excluded_folders = org_config.get('excluded_folders', {})
+ def get_uid(prev, curr):
+ prev[curr.get('uid')] = False
+ return prev
+
+ # Map of dashboard UID -> whether it has been updated.
+ # This is used to remove stale dashboards at the end.
+ dash_list = find_dashboard(token_request) or []
+ dash_list = reduce(get_uid, dash_list, {})
+
with ProcessPoolExecutor(max_workers=4) as executor:
+ provisioned = []
for folder_name, dash in dashboards.items():
exclude = excluded_folders.get(folder_name)
if exclude:
@@ -260,10 +277,19 @@ def provision(config):
logger.info(
f'Provisioning {org["name"]}/{folder_name} dashboards')
- executor.submit(provision_folder, token_request,
- folder_name, dash,
- excluded_interfaces, datasource_name,
- exclude)
+ res = executor.submit(provision_folder, token_request,
+ folder_name, dash,
+ excluded_interfaces, datasource_name,
+ exclude)
+ provisioned.append(res)
+ for result in provisioned:
+ folder = result.result()
+ if folder is None:
+ continue
+ for dashboard in folder:
+ if dashboard is None:
+ continue
+ dash_list[dashboard.get('uid')] = True
aggregate_dashboards = {
'CLS PEERS': {
@@ -296,6 +322,7 @@ def provision(config):
pass
else:
with ProcessPoolExecutor(max_workers=4) as executor:
+ provisioned = []
agg_folder = find_folder(token_request, 'Aggregates')
for agg_type, dash in aggregate_dashboards.items():
if agg_type in exclude_agg:
@@ -306,28 +333,40 @@ def provision(config):
continue
logger.info(f'Provisioning {org["name"]}' +
f'/Aggregate {agg_type} dashboards')
- executor.submit(provision_aggregate, token_request,
- agg_type, agg_folder, dash,
- excluded_interfaces, datasource_name)
+ res = executor.submit(provision_aggregate, token_request,
+ agg_type, agg_folder, dash,
+ excluded_interfaces, datasource_name)
+ provisioned.append(res)
+
+ for result in provisioned:
+ dashboard = result.result()
+ if dashboard is None:
+ continue
+ dash_list[dashboard.get('uid')] = True
# NREN Access dashboards
# uses a different template than the above.
logger.info('Provisioning NREN Access dashboards')
- # always recreate NREN folder
- delete_folder(token_request, 'NREN Access')
folder = find_folder(token_request, 'NREN Access')
nrens = filter(is_nren, excluded_interfaces)
- generate_all_nrens(token_request,
- nrens, folder['id'], datasource_name)
+ provisioned = generate_all_nrens(
+ token_request, nrens, folder['id'], datasource_name)
+
+ for dashboard in provisioned:
+ if dashboard is None:
+ continue
+ dash_list[dashboard.get('uid')] = True
# Non-generated dashboards
excluded_dashboards = org_config.get('excluded_dashboards', [])
logger.info('Provisioning static dashboards')
for dashboard in get_dashboard_definitions():
if dashboard['title'] not in excluded_dashboards:
- create_dashboard(token_request, dashboard)
+ res = create_dashboard(token_request, dashboard)
+ if res:
+ dash_list[res.get('uid')] = True
else:
delete_dashboard(token_request, dashboard)
@@ -336,6 +375,13 @@ def provision(config):
logger.info('Configuring Home dashboard')
is_staff = org['name'] == 'GÉANT Staff'
set_home_dashboard(token_request, is_staff)
+ # just hardcode that we updated home dashboard
+ dash_list['home'] = True
+
+ for dash, provisioned in dash_list.items():
+ if not provisioned:
+ logger.info(f'Deleting stale dashboard with UID {dash}')
+ delete_dashboard(token_request, {'uid': dash})
logger.info(f'Time to complete: {time.time() - start}')
for org_id, token in tokens:
diff --git a/changelog.md b/changelog.md
index 4ffeb7a57a474be8a1dd380b65ef7e33d6d5935a..ce622e4126e6cd8b35f3823e6ff3a6639eae63bb 100644
--- a/changelog.md
+++ b/changelog.md
@@ -2,6 +2,9 @@
All notable changes to this project will be documented in this file.
+## [0.14] - 2021-04-19
+- [POL1-420] Delete stale dashboards after provisioning.
+
## [0.13] - 2021-04-15
- Delete NREN folder when provisioning, stale dashboards could otherwise never be removed.
diff --git a/setup.py b/setup.py
index eb68a217263496bee12adb255450881ad5686e2e..7a3fea90c63b1e8ce54e678688035d1c4fc5a2bd 100644
--- a/setup.py
+++ b/setup.py
@@ -2,7 +2,7 @@ from setuptools import setup, find_packages
setup(
name='brian-dashboard-manager',
- version="0.13",
+ version="0.14",
author='GEANT',
author_email='swd@geant.org',
description='',
diff --git a/test/test_update.py b/test/test_update.py
index 04851d0a157642b4be79ff0ead7c1581c8e72463..0512ebcebc934133fb3f53fadcc2ad46f0b990e0 100644
--- a/test/test_update.py
+++ b/test/test_update.py
@@ -1,6 +1,7 @@
import responses
import json
import re
+from brian_dashboard_manager.grafana.utils.request import TokenRequest
from brian_dashboard_manager.templating.nren_access import get_nrens
from brian_dashboard_manager.grafana.provision import provision_folder, \
generate_all_nrens, provision
@@ -241,6 +242,7 @@ def test_provision_folder(data_config, mocker):
'testdatasource', ['CLS TESTDASHBOARD'])
+@responses.activate
def test_provision_nrens(data_config, mocker):
NREN_INTERFACES = [
# physical
@@ -299,12 +301,34 @@ def test_provision_nrens(data_config, mocker):
}
]
+ UID = '1337'
+
+ def get_callback(request):
+ query = request.params.get('query')
+ return 200, {}, json.dumps({'uid': UID, 'title': query})
+
+ responses.add_callback(
+ method=responses.GET,
+ url=re.compile(f"http://{data_config['hostname']}/api/search"),
+ callback=get_callback)
+
+ def post_callback(request):
+ dashboard = json.loads(request.body).get('dashboard')
+ title = dashboard.get('title')
+ return 200, {}, json.dumps({'uid': UID, 'title': title})
+
+ responses.add_callback(
+ method=responses.POST,
+ url=re.compile(f"http://{data_config['hostname']}/api/dashboards/db"),
+ callback=post_callback)
+
nrens = get_nrens(NREN_INTERFACES)
assert len(nrens) == 1 and nrens.get('HEANET') is not None
assert len(nrens.get('HEANET').get('AGGREGATES')) == 1
assert len(nrens.get('HEANET').get('SERVICES')) == 1
assert len(nrens.get('HEANET').get('PHYSICAL')) == 2
- generate_all_nrens(None, NREN_INTERFACES, 1, 'testdatasource')
+ token_request = TokenRequest(token='testtoken', **data_config)
+ generate_all_nrens(token_request, NREN_INTERFACES, 1, 'testdatasource')
@responses.activate