Skip to content
Snippets Groups Projects
Commit 70c5ac5e authored by Erik Reid's avatar Erik Reid
Browse files

pep8

parent d8fc6385
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
brian_dashboard_manager/grafana/provision.py
+ 51
− 45
View file @ 70c5ac5e
...@@ -33,11 +33,14 @@ from brian_dashboard_manager.templating.helpers import \ ...@@ -33,11 +33,14 @@ from brian_dashboard_manager.templating.helpers import \
from brian_dashboard_manager.templating.gws import generate_gws, \ from brian_dashboard_manager.templating.gws import generate_gws, \
generate_indirect generate_indirect
from brian_dashboard_manager.templating.eumetsat import generate_eumetsat_multicast from brian_dashboard_manager.templating.eumetsat \
import generate_eumetsat_multicast
from brian_dashboard_manager.templating.render import render_dashboard from brian_dashboard_manager.templating.render import render_dashboard
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
DASHBOARD_CHANGES = None # will be an instance of DashboardChanges
MAX_WORKERS = 1 MAX_WORKERS = 1
DASHBOARDS = { DASHBOARDS = {
'NREN': { 'NREN': {
...@@ -153,6 +156,28 @@ AGG_DASHBOARDS = { ...@@ -153,6 +156,28 @@ AGG_DASHBOARDS = {
} }
class DashboardChanges(object):
def __init__(self, token):
# Map of dashboard UID -> whether it has been updated.
# This is used to remove stale dashboards at the end.
all_dashboards = find_dashboard(token) or []
self.updated = {d['uid']: False for d in all_dashboards}
def update_dash_list(self, dashboards):
for dashboard in dashboards:
if isinstance(dashboard, Future):
dashboard = dashboard.result()
if dashboard is None:
continue
self.updated[dashboard.get('uid')] = True
def delete_untouched(self, token):
for uid, provisioned in self.updated.items():
if not provisioned:
logger.info(f'Deleting stale dashboard with UID {uid}')
delete_dashboard(token, {'uid': uid})
def provision_folder(token_request, folder_name, dash, def provision_folder(token_request, folder_name, dash,
ds_name, excluded_dashboards): ds_name, excluded_dashboards):
""" """
...@@ -252,31 +277,9 @@ def excluded_folder_dashboards(org_config, folder_name): ...@@ -252,31 +277,9 @@ def excluded_folder_dashboards(org_config, folder_name):
return excluded if isinstance(excluded, list) else [] return excluded if isinstance(excluded, list) else []
class DashboardChanges(object):
def __init__(self, token):
# Map of dashboard UID -> whether it has been updated.
# This is used to remove stale dashboards at the end.
all_dashboards = find_dashboard(token) or []
self.updated = {d['uid']: False for d in all_dashboards}
def update_dash_list(self, dashboards):
for dashboard in dashboards:
if isinstance(dashboard, Future):
dashboard = dashboard.result()
if dashboard is None:
continue
self.updated[dashboard.get('uid')] = True
def delete_untouched(self, token):
for uid, provisioned in self.updated.items():
if not provisioned:
logger.info(f'Deleting stale dashboard with UID {uid}')
delete_dashboard(token, {'uid': uid})
DASHBOARD_CHANGES = None # will be an instance of DashboardChanges
def _provision_interfaces(config, org_config, ds_name, token): def _provision_interfaces(config, org_config, ds_name, token):
# Provision dashboards, overwriting existing ones.
interfaces = get_interfaces(config['inventory_provider']) interfaces = get_interfaces(config['inventory_provider'])
excluded_nrens = org_config['excluded_nrens'] excluded_nrens = org_config['excluded_nrens']
...@@ -287,14 +290,8 @@ def _provision_interfaces(config, org_config, ds_name, token): ...@@ -287,14 +290,8 @@ def _provision_interfaces(config, org_config, ds_name, token):
to_exclude = any(nren.lower() in desc for nren in excluded_nrens) to_exclude = any(nren.lower() in desc for nren in excluded_nrens)
return not (to_exclude or lab) return not (to_exclude or lab)
relevant_interfaces = list(filter(excluded, interfaces)) relevant_interfaces = list(filter(excluded, interfaces))
# Provision dashboards, overwriting existing ones.
# loop over interfaces and add them to the dashboard_name # loop over interfaces and add them to the dashboard_name
# -> folder mapping structure `dashboards` above, for convenience. # -> folder mapping structure `dashboards` above, for convenience.
for iface in relevant_interfaces: for iface in relevant_interfaces:
...@@ -326,9 +323,10 @@ def _provision_interfaces(config, org_config, ds_name, token): ...@@ -326,9 +323,10 @@ def _provision_interfaces(config, org_config, ds_name, token):
logger.info( logger.info(
f'Provisioning {org_config["name"]}/{folder_name} dashboards') f'Provisioning {org_config["name"]}/{folder_name} dashboards')
res = executor.submit(provision_folder, token, res = executor.submit(
folder_name, folder, ds_name, provision_folder, token,
excluded_folder_dashboards(org_config, folder_name)) folder_name, folder, ds_name,
excluded_folder_dashboards(org_config, folder_name))
provisioned.append(res) provisioned.append(res)
for result in provisioned: for result in provisioned:
...@@ -393,14 +391,19 @@ def _provision_eumetsat_multicast(config, org_config, ds_name, token): ...@@ -393,14 +391,19 @@ def _provision_eumetsat_multicast(config, org_config, ds_name, token):
else: else:
folder = find_folder(token, folder_name) folder = find_folder(token, folder_name)
with ThreadPoolExecutor(max_workers=MAX_WORKERS) as executor: with ThreadPoolExecutor(max_workers=MAX_WORKERS) as executor:
subscriptions = get_eumetsat_multicast_subscriptions(config['inventory_provider']) subscriptions = get_eumetsat_multicast_subscriptions(
config['inventory_provider'])
provisioned = [] provisioned = []
for dashboard in generate_eumetsat_multicast(subscriptions, ds_name): for dashboard in generate_eumetsat_multicast(
subscriptions, ds_name):
rendered = render_dashboard(dashboard) rendered = render_dashboard(dashboard)
provisioned.append(executor.submit(create_dashboard, provisioned.append(
token, executor.submit(
rendered, folder['id'])) create_dashboard,
token,
rendered,
folder['id']))
DASHBOARD_CHANGES.update_dash_list(provisioned) DASHBOARD_CHANGES.update_dash_list(provisioned)
...@@ -414,7 +417,8 @@ def _provision_aggregates(config, org_config, ds_name, token): ...@@ -414,7 +417,8 @@ def _provision_aggregates(config, org_config, ds_name, token):
provisioned = [] provisioned = []
agg_folder = find_folder(token, 'Aggregates') agg_folder = find_folder(token, 'Aggregates')
for dash in AGG_DASHBOARDS.values(): for dash in AGG_DASHBOARDS.values():
excluded_dashboards = excluded_folder_dashboards(org_config, 'Aggregates') excluded_dashboards = excluded_folder_dashboards(
org_config, 'Aggregates')
if dash['dashboard_name'] in excluded_dashboards: if dash['dashboard_name'] in excluded_dashboards:
dash_name = { dash_name = {
'title': f'Aggregate - {dash["dashboard_name"]}'} 'title': f'Aggregate - {dash["dashboard_name"]}'}
...@@ -423,7 +427,7 @@ def _provision_aggregates(config, org_config, ds_name, token): ...@@ -423,7 +427,7 @@ def _provision_aggregates(config, org_config, ds_name, token):
agg_folder['id']) agg_folder['id'])
continue continue
logger.info(f'Provisioning {org_config["name"]}' + logger.info(f'Provisioning {org_config["name"]}' +
f'/Aggregate {dash["dashboard_name"]} dashboards') # noqa: E501 f'/Aggregate {dash["dashboard_name"]} dashboards')
res = executor.submit( res = executor.submit(
provision_aggregate, token, provision_aggregate, token,
agg_folder, dash, ds_name) agg_folder, dash, ds_name)
...@@ -458,7 +462,8 @@ def _set_ignored_folders_as_updated(config, org_config, token): ...@@ -458,7 +462,8 @@ def _set_ignored_folders_as_updated(config, org_config, token):
ignored_folders = config.get('ignored_folders', []) ignored_folders = config.get('ignored_folders', [])
for name in ignored_folders: for name in ignored_folders:
logger.info( logger.info(
f'Ignoring dashboards under the folder {org_config["name"]}/{name}') 'Ignoring dashboards under '
f'the folder {org_config["name"]}/{name}')
folder = find_folder(token, name, create=False) folder = find_folder(token, name, create=False)
if folder is None: if folder is None:
continue continue
...@@ -548,7 +553,6 @@ def provision(config): ...@@ -548,7 +553,6 @@ def provision(config):
org['info'] = 'Org exists in grafana but is not configured' org['info'] = 'Org exists in grafana but is not configured'
return None return None
for org in all_orgs: for org in all_orgs:
org_id = org['id'] org_id = org['id']
delete_expired_api_tokens(request, org_id) delete_expired_api_tokens(request, org_id)
...@@ -572,9 +576,11 @@ def provision(config): ...@@ -572,9 +576,11 @@ def provision(config):
_provision_interfaces(config, org_config, ds_name, token_request) _provision_interfaces(config, org_config, ds_name, token_request)
_provision_gws_indirect(config, org_config, ds_name, token_request) _provision_gws_indirect(config, org_config, ds_name, token_request)
_provision_gws_direct(config, org_config, ds_name, token_request) _provision_gws_direct(config, org_config, ds_name, token_request)
_provision_eumetsat_multicast(config, org_config, ds_name, token_request) _provision_eumetsat_multicast(
config, org_config, ds_name, token_request)
_provision_aggregates(config, org_config, ds_name, token_request) _provision_aggregates(config, org_config, ds_name, token_request)
_provision_static_dashboards(config, org_config, ds_name, token_request) _provision_static_dashboards(
config, org_config, ds_name, token_request)
_set_ignored_folders_as_updated(config, org_config, token_request) _set_ignored_folders_as_updated(config, org_config, token_request)
DASHBOARD_CHANGES.delete_untouched(token_request) DASHBOARD_CHANGES.delete_untouched(token_request)
......
brian_dashboard_manager/templating/eumetsat.py
+ 6
− 10
View file @ 70c5ac5e
...@@ -21,16 +21,16 @@ def get_panel_data(all_subscriptions): ...@@ -21,16 +21,16 @@ def get_panel_data(all_subscriptions):
# make the panels sorted deterministically # make the panels sorted deterministically
for name in result.keys(): for name in result.keys():
result[name] = sorted(result[name], key=operator.itemgetter('subscription')) result[name] = sorted(
result[name],
key=operator.itemgetter('subscription'))
return result return result
def get_panel_fields(panel, panel_type, datasource): def get_panel_fields(panel, panel_type, datasource):
""" """
Helper for generating a single panel, Helper for generating a single multicast panel
with ingress/egress and percentile targets
""" """
letters = letter_generator() letters = letter_generator()
...@@ -56,13 +56,10 @@ def get_panel_fields(panel, panel_type, datasource): ...@@ -56,13 +56,10 @@ def get_panel_fields(panel, panel_type, datasource):
'y_axis_type': 'bits', 'y_axis_type': 'bits',
}) })
def subscription_panel_generator(gridPos): def subscription_panel_generator(gridPos):
""" """
Shared wrapper for shorter calls without Generates panels used for multicast traffic dashboards
gridPos to generate panels.
Generates panels used in a normal dashboard
for all traffic + (conditionally) IPv6 + Errors
""" """
def get_panel_definitions(panels, datasource, errors=False): def get_panel_definitions(panels, datasource, errors=False):
result = [] result = []
...@@ -97,4 +94,3 @@ def generate_eumetsat_multicast(subscriptions, datasource): ...@@ -97,4 +94,3 @@ def generate_eumetsat_multicast(subscriptions, datasource):
panel_generator=subscription_panel_generator): panel_generator=subscription_panel_generator):
yield dash yield dash
test/test_eumetsat_multicast.py
+ 8
− 9
View file @ 70c5ac5e
import responses import responses
import json
from brian_dashboard_manager.templating.gws import generate_gws
from brian_dashboard_manager.inventory_provider.interfaces import \ from brian_dashboard_manager.inventory_provider.interfaces import \
get_eumetsat_multicast_subscriptions get_eumetsat_multicast_subscriptions
...@@ -8,42 +6,42 @@ from brian_dashboard_manager.inventory_provider.interfaces import \ ...@@ -8,42 +6,42 @@ from brian_dashboard_manager.inventory_provider.interfaces import \
TEST_DATA = [ TEST_DATA = [
{ {
'router': 'mx1.ams.nl.geant.net', 'router': 'mx1.ams.nl.geant.net',
'oid': '1.3.6.1.2.1.83.1.1.2.1.16.232.223.222.1.193.17.9.3.255.255.255.255', 'oid': '1.3.6.1.2.1.83.1.1.2.1.16.232.223.222.1.193.17.9.3.255.255.255.255', # noqa: E501
'community': '0pBiFbD', 'community': '0pBiFbD',
'subscription': '232.223.222.1', 'subscription': '232.223.222.1',
'endpoint': '193.17.9.3' 'endpoint': '193.17.9.3'
}, },
{ {
'router': 'mx1.ams.nl.geant.net', 'router': 'mx1.ams.nl.geant.net',
'oid': '1.3.6.1.2.1.83.1.1.2.1.16.232.223.222.2.193.17.9.3.255.255.255.255', 'oid': '1.3.6.1.2.1.83.1.1.2.1.16.232.223.222.2.193.17.9.3.255.255.255.255', # noqa: E501
'community': '0pBiFbD', 'community': '0pBiFbD',
'subscription': '232.223.222.2', 'subscription': '232.223.222.2',
'endpoint': '193.17.9.3' 'endpoint': '193.17.9.3'
}, },
{ {
'router': 'mx1.lon.uk.geant.net', 'router': 'mx1.lon.uk.geant.net',
'oid': '1.3.6.1.2.1.83.1.1.2.1.16.232.223.222.1.193.17.9.3.255.255.255.255', 'oid': '1.3.6.1.2.1.83.1.1.2.1.16.232.223.222.1.193.17.9.3.255.255.255.255', # noqa: E501
'community': '0pBiFbD', 'community': '0pBiFbD',
'subscription': '232.223.222.1', 'subscription': '232.223.222.1',
'endpoint': '193.17.9.3' 'endpoint': '193.17.9.3'
}, },
{ {
'router': 'mx1.lon.uk.geant.net', 'router': 'mx1.lon.uk.geant.net',
'oid': '1.3.6.1.2.1.83.1.1.2.1.16.232.223.222.2.193.17.9.3.255.255.255.255', 'oid': '1.3.6.1.2.1.83.1.1.2.1.16.232.223.222.2.193.17.9.3.255.255.255.255', # noqa: E501
'community': '0pBiFbD', 'community': '0pBiFbD',
'subscription': '232.223.222.2', 'subscription': '232.223.222.2',
'endpoint': '193.17.9.3' 'endpoint': '193.17.9.3'
}, },
{ {
'router': 'mx1.fra.de.geant.net', 'router': 'mx1.fra.de.geant.net',
'oid': '1.3.6.1.2.1.83.1.1.2.1.16.232.223.222.1.193.17.9.3.255.255.255.255', 'oid': '1.3.6.1.2.1.83.1.1.2.1.16.232.223.222.1.193.17.9.3.255.255.255.255', # noqa: E501
'community': '0pBiFbD', 'community': '0pBiFbD',
'subscription': '232.223.222.1', 'subscription': '232.223.222.1',
'endpoint': '193.17.9.3' 'endpoint': '193.17.9.3'
}, },
{ {
'router': 'mx1.fra.de.geant.net', 'router': 'mx1.fra.de.geant.net',
'oid': '1.3.6.1.2.1.83.1.1.2.1.16.232.223.222.2.193.17.9.3.255.255.255.255', 'oid': '1.3.6.1.2.1.83.1.1.2.1.16.232.223.222.2.193.17.9.3.255.255.255.255', # noqa: E501
'community': '0pBiFbD', 'community': '0pBiFbD',
'subscription': '232.223.222.2', 'subscription': '232.223.222.2',
'endpoint': '193.17.9.3' 'endpoint': '193.17.9.3'
...@@ -59,7 +57,8 @@ def test_eumetsat_subscriptions(data_config, client): ...@@ -59,7 +57,8 @@ def test_eumetsat_subscriptions(data_config, client):
url=f'{data_config["inventory_provider"]}/poller/eumetsat-multicast', url=f'{data_config["inventory_provider"]}/poller/eumetsat-multicast',
json=TEST_DATA) json=TEST_DATA)
subscriptions = get_eumetsat_multicast_subscriptions(data_config['inventory_provider']) subscriptions = get_eumetsat_multicast_subscriptions(
data_config['inventory_provider'])
print(subscriptions) print(subscriptions)
# dashboards = list(generate_gws(gws_data, 'testdatasource')) # dashboards = list(generate_gws(gws_data, 'testdatasource'))
# #
......
test/test_gws_direct.py
+ 0
− 1
View file @ 70c5ac5e
import responses import responses
import json
from brian_dashboard_manager.templating.gws import generate_gws from brian_dashboard_manager.templating.gws import generate_gws
from brian_dashboard_manager.inventory_provider.interfaces import \ from brian_dashboard_manager.inventory_provider.interfaces import \
get_gws_direct get_gws_direct
......
test/test_gws_indirect.py
+ 0
− 1
View file @ 70c5ac5e
import responses import responses
import json
from brian_dashboard_manager.templating.gws import generate_indirect from brian_dashboard_manager.templating.gws import generate_indirect
from brian_dashboard_manager.inventory_provider.interfaces import \ from brian_dashboard_manager.inventory_provider.interfaces import \
get_gws_indirect get_gws_indirect
......
test/test_update.py
+ 84
− 28
View file @ 70c5ac5e
import responses import responses
import json import json
import re
from brian_dashboard_manager.grafana.provision import provision_folder, \ from brian_dashboard_manager.grafana.provision import provision_folder, \
provision provision
DEFAULT_REQUEST_HEADERS = {
"Content-type": "application/json",
"Accept": ["application/json"]
}
TEST_INTERFACES = [ TEST_INTERFACES = [
{ {
"router": "srx2.ch.office.geant.net", "router": "srx2.ch.office.geant.net",
...@@ -469,6 +461,51 @@ NREN_INTERFACES = [ ...@@ -469,6 +461,51 @@ NREN_INTERFACES = [
} }
] ]
EUMETSAT_MULTICAST = [
{
'router': 'mx1.ams.nl.geant.net',
'oid': '1.3.6.1.2.1.83.1.1.2.1.16.232.223.222.1.193.17.9.3.255.255.255.255', # noqa: E501
'community': '0pBiFbD',
'subscription': '232.223.222.1',
'endpoint': '193.17.9.3'
},
{
'router': 'mx1.ams.nl.geant.net',
'oid': '1.3.6.1.2.1.83.1.1.2.1.16.232.223.222.2.193.17.9.3.255.255.255.255', # noqa: E501
'community': '0pBiFbD',
'subscription': '232.223.222.2',
'endpoint': '193.17.9.3'
},
{
'router': 'mx1.lon.uk.geant.net',
'oid': '1.3.6.1.2.1.83.1.1.2.1.16.232.223.222.1.193.17.9.3.255.255.255.255', # noqa: E501
'community': '0pBiFbD',
'subscription': '232.223.222.1',
'endpoint': '193.17.9.3'
},
{
'router': 'mx1.lon.uk.geant.net',
'oid': '1.3.6.1.2.1.83.1.1.2.1.16.232.223.222.2.193.17.9.3.255.255.255.255', # noqa: E501
'community': '0pBiFbD',
'subscription': '232.223.222.2',
'endpoint': '193.17.9.3'
},
{
'router': 'mx1.fra.de.geant.net',
'oid': '1.3.6.1.2.1.83.1.1.2.1.16.232.223.222.1.193.17.9.3.255.255.255.255', # noqa: E501
'community': '0pBiFbD',
'subscription': '232.223.222.1',
'endpoint': '193.17.9.3'
},
{
'router': 'mx1.fra.de.geant.net',
'oid': '1.3.6.1.2.1.83.1.1.2.1.16.232.223.222.2.193.17.9.3.255.255.255.255', # noqa: E501
'community': '0pBiFbD',
'subscription': '232.223.222.2',
'endpoint': '193.17.9.3'
}
]
def generate_folder(data): def generate_folder(data):
return { return {
...@@ -494,12 +531,16 @@ def test_provision_folder(data_config, mocker): ...@@ -494,12 +531,16 @@ def test_provision_folder(data_config, mocker):
'NREN': { 'NREN': {
'tag': ['customers'], 'tag': ['customers'],
'folder_name': 'NREN Access', 'folder_name': 'NREN Access',
'interfaces': [iface for iface in TEST_INTERFACES if 'NREN' in iface['dashboards']] # noqa: E501 'interfaces': [
iface for iface in TEST_INTERFACES
if 'NREN' in iface['dashboards']]
}, },
'RE_CUST': { 'RE_CUST': {
'tag': 'RE_CUST', 'tag': 'RE_CUST',
'folder_name': 'RE Customer', 'folder_name': 'RE Customer',
'interfaces': [iface for iface in TEST_INTERFACES if 'RE_CUST' in iface['dashboards']] # noqa: E501 'interfaces': [
iface for iface in TEST_INTERFACES
if 'RE_CUST' in iface['dashboards']]
}, },
} }
...@@ -554,15 +595,25 @@ def test_provision(data_config, mocker, client): ...@@ -554,15 +595,25 @@ def test_provision(data_config, mocker, client):
url=f"{data_config['inventory_provider']}/data/interfaces", url=f"{data_config['inventory_provider']}/data/interfaces",
json=NREN_INTERFACES) json=NREN_INTERFACES)
responses.add(
method=responses.GET,
url=f'{data_config["inventory_provider"]}/poller/eumetsat-multicast',
json=EUMETSAT_MULTICAST)
responses.add(
method=responses.DELETE,
url=f"http://{data_config['hostname']}/api/folders",
json={"message": "Deleted folder"})
responses.add( responses.add(
method=responses.GET, method=responses.GET,
url=f"http://{data_config['hostname']}/api/folders", url=f"http://{data_config['hostname']}/api/folders",
json=[]) json=[])
responses.add( responses.add(
method=responses.DELETE, method='get',
url=re.compile(f"http://{data_config['hostname']}/api/folders"), url=f"http://{data_config['hostname']}/api/folders",
json={"message": "Deleted folder"} ) json=[])
def folder_post(request): def folder_post(request):
data = json.loads(request.body) data = json.loads(request.body)
...@@ -573,26 +624,31 @@ def test_provision(data_config, mocker, client): ...@@ -573,26 +624,31 @@ def test_provision(data_config, mocker, client):
url=f"http://{data_config['hostname']}/api/folders", url=f"http://{data_config['hostname']}/api/folders",
callback=folder_post) callback=folder_post)
responses.add( def search_responses(request):
method=responses.GET, if request.params.get('query', None) == 'Home':
url=f"http://{data_config['hostname']}/api/search?query=Home", return 200, {}, json.dumps([])
json=[]) if request.params.get('type', None) == 'dash-db':
return 200, {}, json.dumps([])
assert False # no other queries expected
TEST_DATASOURCE = [{ responses.add_callback(
"name": "brian-influx-datasource", method=responses.GET,
"type": "influxdb", url=f"http://{data_config['hostname']}/api/search",
"access": "proxy", callback=search_responses)
"url": "http://test-brian-datasource.geant.org:8086",
"database": "test-db",
"basicAuth": False,
"isDefault": True,
"readOnly": False
}]
responses.add( responses.add(
method=responses.GET, method=responses.GET,
url=f"http://{data_config['hostname']}/api/datasources", url=f"http://{data_config['hostname']}/api/datasources",
json=TEST_DATASOURCE) json=[{
"name": "brian-influx-datasource",
"type": "influxdb",
"access": "proxy",
"url": "http://test-brian-datasource.geant.org:8086",
"database": "test-db",
"basicAuth": False,
"isDefault": True,
"readOnly": False
}])
responses.add( responses.add(
method=responses.POST, method=responses.POST,
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment