Skip to content
Snippets Groups Projects
Commit 70c5ac5e authored by Erik Reid's avatar Erik Reid
Browse files

pep8

parent d8fc6385
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
brian_dashboard_manager/grafana/provision.py
+ 51
− 45
View file @ 70c5ac5e
......@@ -33,11 +33,14 @@ from brian_dashboard_manager.templating.helpers import \
from brian_dashboard_manager.templating.gws import generate_gws, \
generate_indirect
from brian_dashboard_manager.templating.eumetsat import generate_eumetsat_multicast
from brian_dashboard_manager.templating.eumetsat \
import generate_eumetsat_multicast
from brian_dashboard_manager.templating.render import render_dashboard
logger = logging.getLogger(__name__)
DASHBOARD_CHANGES = None # will be an instance of DashboardChanges
MAX_WORKERS = 1
DASHBOARDS = {
'NREN': {
......@@ -153,6 +156,28 @@ AGG_DASHBOARDS = {
}
class DashboardChanges(object):
def __init__(self, token):
# Map of dashboard UID -> whether it has been updated.
# This is used to remove stale dashboards at the end.
all_dashboards = find_dashboard(token) or []
self.updated = {d['uid']: False for d in all_dashboards}
def update_dash_list(self, dashboards):
for dashboard in dashboards:
if isinstance(dashboard, Future):
dashboard = dashboard.result()
if dashboard is None:
continue
self.updated[dashboard.get('uid')] = True
def delete_untouched(self, token):
for uid, provisioned in self.updated.items():
if not provisioned:
logger.info(f'Deleting stale dashboard with UID {uid}')
delete_dashboard(token, {'uid': uid})
def provision_folder(token_request, folder_name, dash,
ds_name, excluded_dashboards):
"""
......@@ -252,31 +277,9 @@ def excluded_folder_dashboards(org_config, folder_name):
return excluded if isinstance(excluded, list) else []
class DashboardChanges(object):
def __init__(self, token):
# Map of dashboard UID -> whether it has been updated.
# This is used to remove stale dashboards at the end.
all_dashboards = find_dashboard(token) or []
self.updated = {d['uid']: False for d in all_dashboards}
def update_dash_list(self, dashboards):
for dashboard in dashboards:
if isinstance(dashboard, Future):
dashboard = dashboard.result()
if dashboard is None:
continue
self.updated[dashboard.get('uid')] = True
def delete_untouched(self, token):
for uid, provisioned in self.updated.items():
if not provisioned:
logger.info(f'Deleting stale dashboard with UID {uid}')
delete_dashboard(token, {'uid': uid})
DASHBOARD_CHANGES = None # will be an instance of DashboardChanges
def _provision_interfaces(config, org_config, ds_name, token):
# Provision dashboards, overwriting existing ones.
interfaces = get_interfaces(config['inventory_provider'])
excluded_nrens = org_config['excluded_nrens']
......@@ -287,14 +290,8 @@ def _provision_interfaces(config, org_config, ds_name, token):
to_exclude = any(nren.lower() in desc for nren in excluded_nrens)
return not (to_exclude or lab)
relevant_interfaces = list(filter(excluded, interfaces))
# Provision dashboards, overwriting existing ones.
# loop over interfaces and add them to the dashboard_name
# -> folder mapping structure `dashboards` above, for convenience.
for iface in relevant_interfaces:
......@@ -326,9 +323,10 @@ def _provision_interfaces(config, org_config, ds_name, token):
logger.info(
f'Provisioning {org_config["name"]}/{folder_name} dashboards')
res = executor.submit(provision_folder, token,
folder_name, folder, ds_name,
excluded_folder_dashboards(org_config, folder_name))
res = executor.submit(
provision_folder, token,
folder_name, folder, ds_name,
excluded_folder_dashboards(org_config, folder_name))
provisioned.append(res)
for result in provisioned:
......@@ -393,14 +391,19 @@ def _provision_eumetsat_multicast(config, org_config, ds_name, token):
else:
folder = find_folder(token, folder_name)
with ThreadPoolExecutor(max_workers=MAX_WORKERS) as executor:
subscriptions = get_eumetsat_multicast_subscriptions(config['inventory_provider'])
subscriptions = get_eumetsat_multicast_subscriptions(
config['inventory_provider'])
provisioned = []
for dashboard in generate_eumetsat_multicast(subscriptions, ds_name):
for dashboard in generate_eumetsat_multicast(
subscriptions, ds_name):
rendered = render_dashboard(dashboard)
provisioned.append(executor.submit(create_dashboard,
token,
rendered, folder['id']))
provisioned.append(
executor.submit(
create_dashboard,
token,
rendered,
folder['id']))
DASHBOARD_CHANGES.update_dash_list(provisioned)
......@@ -414,7 +417,8 @@ def _provision_aggregates(config, org_config, ds_name, token):
provisioned = []
agg_folder = find_folder(token, 'Aggregates')
for dash in AGG_DASHBOARDS.values():
excluded_dashboards = excluded_folder_dashboards(org_config, 'Aggregates')
excluded_dashboards = excluded_folder_dashboards(
org_config, 'Aggregates')
if dash['dashboard_name'] in excluded_dashboards:
dash_name = {
'title': f'Aggregate - {dash["dashboard_name"]}'}
......@@ -423,7 +427,7 @@ def _provision_aggregates(config, org_config, ds_name, token):
agg_folder['id'])
continue
logger.info(f'Provisioning {org_config["name"]}' +
f'/Aggregate {dash["dashboard_name"]} dashboards') # noqa: E501
f'/Aggregate {dash["dashboard_name"]} dashboards')
res = executor.submit(
provision_aggregate, token,
agg_folder, dash, ds_name)
......@@ -458,7 +462,8 @@ def _set_ignored_folders_as_updated(config, org_config, token):
ignored_folders = config.get('ignored_folders', [])
for name in ignored_folders:
logger.info(
f'Ignoring dashboards under the folder {org_config["name"]}/{name}')
'Ignoring dashboards under '
f'the folder {org_config["name"]}/{name}')
folder = find_folder(token, name, create=False)
if folder is None:
continue
......@@ -548,7 +553,6 @@ def provision(config):
org['info'] = 'Org exists in grafana but is not configured'
return None
for org in all_orgs:
org_id = org['id']
delete_expired_api_tokens(request, org_id)
......@@ -572,9 +576,11 @@ def provision(config):
_provision_interfaces(config, org_config, ds_name, token_request)
_provision_gws_indirect(config, org_config, ds_name, token_request)
_provision_gws_direct(config, org_config, ds_name, token_request)
_provision_eumetsat_multicast(config, org_config, ds_name, token_request)
_provision_eumetsat_multicast(
config, org_config, ds_name, token_request)
_provision_aggregates(config, org_config, ds_name, token_request)
_provision_static_dashboards(config, org_config, ds_name, token_request)
_provision_static_dashboards(
config, org_config, ds_name, token_request)
_set_ignored_folders_as_updated(config, org_config, token_request)
DASHBOARD_CHANGES.delete_untouched(token_request)
......
brian_dashboard_manager/templating/eumetsat.py
+ 6
− 10
View file @ 70c5ac5e
......@@ -21,16 +21,16 @@ def get_panel_data(all_subscriptions):
# make the panels sorted deterministically
for name in result.keys():
result[name] = sorted(result[name], key=operator.itemgetter('subscription'))
result[name] = sorted(
result[name],
key=operator.itemgetter('subscription'))
return result
def get_panel_fields(panel, panel_type, datasource):
"""
Helper for generating a single panel,
with ingress/egress and percentile targets
Helper for generating a single multicast panel
"""
letters = letter_generator()
......@@ -56,13 +56,10 @@ def get_panel_fields(panel, panel_type, datasource):
'y_axis_type': 'bits',
})
def subscription_panel_generator(gridPos):
"""
Shared wrapper for shorter calls without
gridPos to generate panels.
Generates panels used in a normal dashboard
for all traffic + (conditionally) IPv6 + Errors
Generates panels used for multicast traffic dashboards
"""
def get_panel_definitions(panels, datasource, errors=False):
result = []
......@@ -97,4 +94,3 @@ def generate_eumetsat_multicast(subscriptions, datasource):
panel_generator=subscription_panel_generator):
yield dash
test/test_eumetsat_multicast.py
+ 8
− 9
View file @ 70c5ac5e
import responses
import json
from brian_dashboard_manager.templating.gws import generate_gws
from brian_dashboard_manager.inventory_provider.interfaces import \
get_eumetsat_multicast_subscriptions
......@@ -8,42 +6,42 @@ from brian_dashboard_manager.inventory_provider.interfaces import \
TEST_DATA = [
{
'router': 'mx1.ams.nl.geant.net',
'oid': '1.3.6.1.2.1.83.1.1.2.1.16.232.223.222.1.193.17.9.3.255.255.255.255',
'oid': '1.3.6.1.2.1.83.1.1.2.1.16.232.223.222.1.193.17.9.3.255.255.255.255', # noqa: E501
'community': '0pBiFbD',
'subscription': '232.223.222.1',
'endpoint': '193.17.9.3'
},
{
'router': 'mx1.ams.nl.geant.net',
'oid': '1.3.6.1.2.1.83.1.1.2.1.16.232.223.222.2.193.17.9.3.255.255.255.255',
'oid': '1.3.6.1.2.1.83.1.1.2.1.16.232.223.222.2.193.17.9.3.255.255.255.255', # noqa: E501
'community': '0pBiFbD',
'subscription': '232.223.222.2',
'endpoint': '193.17.9.3'
},
{
'router': 'mx1.lon.uk.geant.net',
'oid': '1.3.6.1.2.1.83.1.1.2.1.16.232.223.222.1.193.17.9.3.255.255.255.255',
'oid': '1.3.6.1.2.1.83.1.1.2.1.16.232.223.222.1.193.17.9.3.255.255.255.255', # noqa: E501
'community': '0pBiFbD',
'subscription': '232.223.222.1',
'endpoint': '193.17.9.3'
},
{
'router': 'mx1.lon.uk.geant.net',
'oid': '1.3.6.1.2.1.83.1.1.2.1.16.232.223.222.2.193.17.9.3.255.255.255.255',
'oid': '1.3.6.1.2.1.83.1.1.2.1.16.232.223.222.2.193.17.9.3.255.255.255.255', # noqa: E501
'community': '0pBiFbD',
'subscription': '232.223.222.2',
'endpoint': '193.17.9.3'
},
{
'router': 'mx1.fra.de.geant.net',
'oid': '1.3.6.1.2.1.83.1.1.2.1.16.232.223.222.1.193.17.9.3.255.255.255.255',
'oid': '1.3.6.1.2.1.83.1.1.2.1.16.232.223.222.1.193.17.9.3.255.255.255.255', # noqa: E501
'community': '0pBiFbD',
'subscription': '232.223.222.1',
'endpoint': '193.17.9.3'
},
{
'router': 'mx1.fra.de.geant.net',
'oid': '1.3.6.1.2.1.83.1.1.2.1.16.232.223.222.2.193.17.9.3.255.255.255.255',
'oid': '1.3.6.1.2.1.83.1.1.2.1.16.232.223.222.2.193.17.9.3.255.255.255.255', # noqa: E501
'community': '0pBiFbD',
'subscription': '232.223.222.2',
'endpoint': '193.17.9.3'
......@@ -59,7 +57,8 @@ def test_eumetsat_subscriptions(data_config, client):
url=f'{data_config["inventory_provider"]}/poller/eumetsat-multicast',
json=TEST_DATA)
subscriptions = get_eumetsat_multicast_subscriptions(data_config['inventory_provider'])
subscriptions = get_eumetsat_multicast_subscriptions(
data_config['inventory_provider'])
print(subscriptions)
# dashboards = list(generate_gws(gws_data, 'testdatasource'))
#
......
test/test_gws_direct.py
+ 0
− 1
View file @ 70c5ac5e
import responses
import json
from brian_dashboard_manager.templating.gws import generate_gws
from brian_dashboard_manager.inventory_provider.interfaces import \
get_gws_direct
......
test/test_gws_indirect.py
+ 0
− 1
View file @ 70c5ac5e
import responses
import json
from brian_dashboard_manager.templating.gws import generate_indirect
from brian_dashboard_manager.inventory_provider.interfaces import \
get_gws_indirect
......
test/test_update.py
+ 84
− 28
View file @ 70c5ac5e
import responses
import json
import re
from brian_dashboard_manager.grafana.provision import provision_folder, \
provision
DEFAULT_REQUEST_HEADERS = {
"Content-type": "application/json",
"Accept": ["application/json"]
}
TEST_INTERFACES = [
{
"router": "srx2.ch.office.geant.net",
......@@ -469,6 +461,51 @@ NREN_INTERFACES = [
}
]
EUMETSAT_MULTICAST = [
{
'router': 'mx1.ams.nl.geant.net',
'oid': '1.3.6.1.2.1.83.1.1.2.1.16.232.223.222.1.193.17.9.3.255.255.255.255', # noqa: E501
'community': '0pBiFbD',
'subscription': '232.223.222.1',
'endpoint': '193.17.9.3'
},
{
'router': 'mx1.ams.nl.geant.net',
'oid': '1.3.6.1.2.1.83.1.1.2.1.16.232.223.222.2.193.17.9.3.255.255.255.255', # noqa: E501
'community': '0pBiFbD',
'subscription': '232.223.222.2',
'endpoint': '193.17.9.3'
},
{
'router': 'mx1.lon.uk.geant.net',
'oid': '1.3.6.1.2.1.83.1.1.2.1.16.232.223.222.1.193.17.9.3.255.255.255.255', # noqa: E501
'community': '0pBiFbD',
'subscription': '232.223.222.1',
'endpoint': '193.17.9.3'
},
{
'router': 'mx1.lon.uk.geant.net',
'oid': '1.3.6.1.2.1.83.1.1.2.1.16.232.223.222.2.193.17.9.3.255.255.255.255', # noqa: E501
'community': '0pBiFbD',
'subscription': '232.223.222.2',
'endpoint': '193.17.9.3'
},
{
'router': 'mx1.fra.de.geant.net',
'oid': '1.3.6.1.2.1.83.1.1.2.1.16.232.223.222.1.193.17.9.3.255.255.255.255', # noqa: E501
'community': '0pBiFbD',
'subscription': '232.223.222.1',
'endpoint': '193.17.9.3'
},
{
'router': 'mx1.fra.de.geant.net',
'oid': '1.3.6.1.2.1.83.1.1.2.1.16.232.223.222.2.193.17.9.3.255.255.255.255', # noqa: E501
'community': '0pBiFbD',
'subscription': '232.223.222.2',
'endpoint': '193.17.9.3'
}
]
def generate_folder(data):
return {
......@@ -494,12 +531,16 @@ def test_provision_folder(data_config, mocker):
'NREN': {
'tag': ['customers'],
'folder_name': 'NREN Access',
'interfaces': [iface for iface in TEST_INTERFACES if 'NREN' in iface['dashboards']] # noqa: E501
'interfaces': [
iface for iface in TEST_INTERFACES
if 'NREN' in iface['dashboards']]
},
'RE_CUST': {
'tag': 'RE_CUST',
'folder_name': 'RE Customer',
'interfaces': [iface for iface in TEST_INTERFACES if 'RE_CUST' in iface['dashboards']] # noqa: E501
'interfaces': [
iface for iface in TEST_INTERFACES
if 'RE_CUST' in iface['dashboards']]
},
}
......@@ -554,15 +595,25 @@ def test_provision(data_config, mocker, client):
url=f"{data_config['inventory_provider']}/data/interfaces",
json=NREN_INTERFACES)
responses.add(
method=responses.GET,
url=f'{data_config["inventory_provider"]}/poller/eumetsat-multicast',
json=EUMETSAT_MULTICAST)
responses.add(
method=responses.DELETE,
url=f"http://{data_config['hostname']}/api/folders",
json={"message": "Deleted folder"})
responses.add(
method=responses.GET,
url=f"http://{data_config['hostname']}/api/folders",
json=[])
responses.add(
method=responses.DELETE,
url=re.compile(f"http://{data_config['hostname']}/api/folders"),
json={"message": "Deleted folder"} )
method='get',
url=f"http://{data_config['hostname']}/api/folders",
json=[])
def folder_post(request):
data = json.loads(request.body)
......@@ -573,26 +624,31 @@ def test_provision(data_config, mocker, client):
url=f"http://{data_config['hostname']}/api/folders",
callback=folder_post)
responses.add(
method=responses.GET,
url=f"http://{data_config['hostname']}/api/search?query=Home",
json=[])
def search_responses(request):
if request.params.get('query', None) == 'Home':
return 200, {}, json.dumps([])
if request.params.get('type', None) == 'dash-db':
return 200, {}, json.dumps([])
assert False # no other queries expected
TEST_DATASOURCE = [{
"name": "brian-influx-datasource",
"type": "influxdb",
"access": "proxy",
"url": "http://test-brian-datasource.geant.org:8086",
"database": "test-db",
"basicAuth": False,
"isDefault": True,
"readOnly": False
}]
responses.add_callback(
method=responses.GET,
url=f"http://{data_config['hostname']}/api/search",
callback=search_responses)
responses.add(
method=responses.GET,
url=f"http://{data_config['hostname']}/api/datasources",
json=TEST_DATASOURCE)
json=[{
"name": "brian-influx-datasource",
"type": "influxdb",
"access": "proxy",
"url": "http://test-brian-datasource.geant.org:8086",
"database": "test-db",
"basicAuth": False,
"isDefault": True,
"readOnly": False
}])
responses.add(
method=responses.POST,
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment