Skip to content
Snippets Groups Projects
Commit 717f9c20 authored by Sam Roberts's avatar Sam Roberts
Browse files

reworked to be much, much simpler

parent d3d947e8
No related branches found
No related tags found
1 merge request!13Feature/pol1 430 EAP NRENs
Hide whitespace changes
Inline Side-by-side
brian_dashboard_manager/dashboards/region_eap.json
+ 1
− 1
View file @ 717f9c20
......@@ -114,6 +114,6 @@
]
},
"timezone": "",
"title": "EAP Access",
"title": "EAP Dashboard",
"version": 1
}
brian_dashboard_manager/grafana/provision.py
+ 19
− 44
View file @ 717f9c20
......@@ -46,7 +46,7 @@ from brian_dashboard_manager.templating.render import (
logger = logging.getLogger(__name__)
MAX_WORKERS = 4
MAX_WORKERS = 1
DASHBOARDS = {
'NRENLEGACY': {
'tag': ['customerslegacy'],
......@@ -60,7 +60,7 @@ DASHBOARDS = {
},
'EAP': {
'tag': ['eap'],
'folder_name': 'EAP NRENs',
'folder_name': 'EAP NREN Access',
'interfaces': []
},
'RE_PEER': {
......@@ -202,12 +202,6 @@ AGG_DASHBOARDS = {
}
}
NREN_CATEGORIES = {
'EAP': {
'folder_name': 'EAP NRENs'
}
}
def provision_folder(token_request, folder_name, dash, services, regions,
ds_name, excluded_dashboards):
......@@ -241,53 +235,33 @@ def provision_folder(token_request, folder_name, dash, services, regions,
)
)
def _get_nren_category(_folder_name):
for category in NREN_CATEGORIES:
if _folder_name == NREN_CATEGORIES[category]['folder_name']:
return category
return None
def _get_customers_for_region(region=None):
customers = []
region_lookup = {region['nren']: region['region'] for region in regions}
for service in services:
service_customers = service.get('customers')
for cust in service_customers:
cust_region = region_lookup.get(cust, None)
if cust_region == region:
customers.append(cust)
return customers
# dashboard should include error panels
errors = dash.get('errors', False)
nren_regions = {
region['nren']: region['region'] for region in regions
}
is_nren_legacy = folder_name == "NREN Access LEGACY"
is_nren = folder_name == "NREN Access"
nren_category = _get_nren_category(folder_name) # will be None if the NREN is not in a category
is_eap = folder_name == "EAP NREN Access"
is_re_peer = folder_name == "RE Peer"
is_service = 'service_type' in dash
has_aggregate_panels = is_nren or is_nren_legacy or nren_category or is_re_peer or is_service
def _filter_nren_interface_data_by_tag(tag=None):
raw_data = get_nren_interface_data(services, nren_regions, interfaces, excluded_dashboards)
all_tags = NREN_CATEGORIES.keys()
if tag is None:
# only return interfaces that DO NOT have a category tag (uncategorized NRENs)
for interface_key, interface in raw_data.items():
tags = interface.get('TAGS', set())
not_parented = True
for interface_tag in tags:
if interface_tag in all_tags:
not_parented = False
break
if not_parented:
yield interface_key, interface
else:
for interface_key, interface in raw_data.items():
if tag in interface.get('TAGS', set()):
yield interface_key, interface
has_aggregate_panels = is_nren or is_eap or is_nren_legacy or is_re_peer or is_service
if is_nren_legacy:
data = get_nren_interface_data_old(interfaces)
dash_data = get_nren_dashboard_data(data, ds_name, tag)
elif is_nren:
data = dict(_filter_nren_interface_data_by_tag(None))
dash_data = get_nren_dashboard_data(data, ds_name, tag)
elif nren_category:
data = dict(_filter_nren_interface_data_by_tag(nren_category))
elif is_nren or is_eap:
region_customers = _get_customers_for_region("EAP" if is_eap else None)
data = get_nren_interface_data(services, interfaces, excluded_dashboards, region_customers)
dash_data = get_nren_dashboard_data(data, ds_name, tag)
elif is_re_peer:
data = get_re_peer_interface_data(interfaces)
......@@ -963,7 +937,8 @@ def provision(config, raise_exceptions=False):
'GWS Indirect',
'GWS Direct',
'Aggregates',
'EUMETSAT Multicast'
'EUMETSAT Multicast',
'EAP Dashboard'
}
folders_to_keep.update({dash['folder_name']
for dash in DASHBOARDS.values()})
......
brian_dashboard_manager/inventory_provider/interfaces.py
+ 1
− 1
View file @ 717f9c20
......@@ -248,7 +248,7 @@ NREN_REGION_LIST_SCHEMA = {
'nren_region': {
'type': 'object',
'properties': {
'name': {'type': 'string'},
'nren': {'type': 'string'},
'region': {'type': 'string'}
}
}
......
brian_dashboard_manager/templating/helpers.py
+ 8
− 18
View file @ 717f9c20
......@@ -9,7 +9,6 @@ from itertools import product
from functools import partial, reduce
from string import ascii_uppercase
from brian_dashboard_manager.grafana import provision
from brian_dashboard_manager.templating.render import create_panel, \
create_panel_target, create_dropdown_panel
......@@ -218,7 +217,7 @@ def get_re_peer_interface_data(interfaces):
return result
def get_nren_interface_data(services, regions, interfaces, excluded_dashboards):
def get_nren_interface_data(services, interfaces, excluded_dashboards, region_customers):
"""
Helper for grouping interface data to be used for generating
dashboards for NRENs.
......@@ -226,10 +225,11 @@ def get_nren_interface_data(services, regions, interfaces, excluded_dashboards):
Extracts information from interfaces to be used in panels.
:param services: list of services
:param regions: dict of NRENs to regions
:param interfaces: list of interfaces
:param excluded_dashboards: list of dashboards to exclude for
the organization we are generating dashboards for
:param region_customers: list of customers in the region we are
generating dashboards for (currently used to separate EAP NRENs)
:return: dictionary of dashboards and their service/interface data
"""
......@@ -244,6 +244,8 @@ def get_nren_interface_data(services, regions, interfaces, excluded_dashboards):
for cust in _customers:
if cust.lower() in excluded_dashboards:
continue
if cust not in region_customers:
continue
customers[cust].append(service)
for customer, services in customers.items():
......@@ -254,8 +256,7 @@ def get_nren_interface_data(services, regions, interfaces, excluded_dashboards):
dashboard = result.setdefault(customer, {
'AGGREGATES': [],
'SERVICES': [],
'PHYSICAL': [],
'TAGS': set()
'PHYSICAL': []
})
for service in services:
......@@ -331,14 +332,11 @@ def get_nren_interface_data(services, regions, interfaces, excluded_dashboards):
for info in dashboards_info:
dashboard_name = info['name']
if dashboard_name in provision.NREN_CATEGORIES: # do not create panels for the category itself
continue
dashboard = result.get(dashboard_name, {
'AGGREGATES': [],
'SERVICES': [],
'PHYSICAL': [],
'TAGS': set()
'PHYSICAL': []
})
_covered_by_service = _check_in_aggregate(router, interface_name)
......@@ -364,20 +362,12 @@ def get_nren_interface_data(services, regions, interfaces, excluded_dashboards):
'hostname': host,
'interface': interface_name
})
# add dashboard tags from interface dashboards_info
for tag in interface.get('dashboards', []):
dashboard['TAGS'].add(tag)
result[dashboard_name] = dashboard
for customer in list(result.keys()):
# add region tags
if customer in regions:
result[customer]['TAGS'].add(regions[customer])
lengths = [len(val) for val in result[customer].values()]
tags_length = len(result[customer]['TAGS']) # exclude tags from count
if sum(lengths) - tags_length == 0:
if sum(lengths) == 0:
# no services/interfaces, so remove it
del result[customer]
return result
......
test/data/regions.json deleted 100644 → 0
+ 0
− 22
View file @ d3d947e8
[
{
"nren": "URAN",
"region": "EAP"
},
{
"nren": "ASNET-AM",
"region": "EAP"
},
{
"nren": "AZSCIENCENET",
"region": "EAP"
},
{
"nren": "GRENA",
"region": "EAP"
},
{
"nren": "RENAM",
"region": "EAP"
}
]
\ No newline at end of file
test/test_update.py
+ 74
− 17
View file @ 717f9c20
......@@ -300,7 +300,6 @@ TEST_INTERFACES = [
"snmp-index": 1033,
"dashboards": [
"EAP",
"NREN",
"RE_CUST"
],
"dashboard_info": {
......@@ -569,6 +568,40 @@ NREN_INTERFACES = [
}],
"ipv4": [],
"ipv6": []
},
{
"router": "rt1.fra.de.geant.net",
"name": "xe-11/2/5.300",
"bundle": [],
"bundle-parents": [],
"description": "SRV_GLOBAL CUSTOMER AZSCIENCENET #AZSCIENCENET-AP1 $GS-00433 | ASN202993 |",
"circuits": [
{
"id": 723799,
"name": "AZSCIENCENET-AP1",
"type": "GEANT IP",
"status": "operational"
}
],
"snmp-index": 1033,
"dashboards": [
"EAP",
"RE_CUST"
],
"dashboard_info": {
"name": "AZSCIENCENET",
"interface_type": "LOGICAL"
},
"dashboards_info": [
{
"name": "AZSCIENCENET",
"interface_type": "LOGICAL"
}
],
"port_type": "SERVICE",
"ipv4": [],
"ipv6": []
}
]
......@@ -617,6 +650,29 @@ EUMETSAT_MULTICAST = [
}
]
NREN_REGIONS = [
{
"nren": "URAN",
"region": "EAP"
},
{
"nren": "ASNET-AM",
"region": "EAP"
},
{
"nren": "AZSCIENCENET",
"region": "EAP"
},
{
"nren": "GRENA",
"region": "EAP"
},
{
"nren": "RENAM",
"region": "EAP"
}
]
def generate_folder(data):
return {
......@@ -659,12 +715,6 @@ def populate_inventory(get_test_data, data_config):
json=contents,
)
responses.add(
method=responses.GET,
url=f"{data_config['inventory_provider']}/poller/regions",
json=get_test_data("regions.json"),
)
return _populate
......@@ -672,19 +722,19 @@ def populate_inventory(get_test_data, data_config):
@pytest.mark.parametrize(
"folder_name, excluded_nrens, expected_nrens",
[
("NREN Access", [], ['ASNET-AM', 'CESNET', 'GEANT', 'KIAE', 'LITNET', 'SWITCH']),
("NREN Access", ["GEANT", "KIAE"], ['ASNET-AM', 'CESNET', 'LITNET', 'SWITCH']),
("NREN Access", [], ['CESNET', 'GEANT', 'KIAE', 'LITNET', 'SWITCH']),
("NREN Access", ["GEANT", "KIAE"], ['CESNET', 'LITNET', 'SWITCH']),
(
"NREN Access",
[],
["ASNET-AM", "LITNET", "CESNET", "GEANT", "KIAE", "SWITCH"],
["LITNET", "CESNET", "GEANT", "KIAE", "SWITCH"],
),
(
"NREN Access",
["ASNET-AM", "GEANT"],
["GEANT"],
["LITNET", "CESNET", "KIAE", "SWITCH"],
),
("testfolder", ["GEANT"], ["KIAE", "SWITCH", "AZSCIENCENET"]),
("testfolder", ["GEANT"], ["KIAE", "SWITCH"]),
],
)
def test_provision_nren_folder(
......@@ -717,11 +767,12 @@ def test_provision_nren_folder(
"/poller/interfaces": NREN_INTERFACES,
"/data/interfaces": NREN_INTERFACES,
"/poller/eumetsat-multicast": EUMETSAT_MULTICAST,
"/poller/regions": NREN_REGIONS,
}
)
services = fetch_services(data_config['reporting_provider'])
regions = []
regions = get_nren_regions(data_config['inventory_provider'])
result = provision_folder(
mock_grafana.request,
......@@ -762,6 +813,7 @@ def test_provision(
"/poller/interfaces": NREN_INTERFACES,
"/data/interfaces": NREN_INTERFACES,
"/poller/eumetsat-multicast": EUMETSAT_MULTICAST,
"/poller/regions": NREN_REGIONS,
}
)
for org in data_config["organizations"][1:]:
......@@ -801,6 +853,7 @@ def test_provision_re_peer_dashboard(
"/poller/interfaces": interfaces,
"/data/interfaces": interfaces,
"/poller/eumetsat-multicast": EUMETSAT_MULTICAST,
"/poller/regions": NREN_REGIONS,
}
)
_mocked_gws = mocker.patch(
......@@ -834,8 +887,8 @@ def test_provision_re_peer_dashboard(
@pytest.mark.parametrize(
"folder_name, dashboard_id, expected_dashboard_count",
[
('NREN Access', 'NREN', 5), # uncategorized interfaces
('EAP Access', 'EAP', 1) # EAP categorized interfaces
('NREN Access', 'NREN', 5),
('EAP NREN Access', 'EAP', 1)
]
)
def test_provision_nren_category(
......@@ -850,6 +903,9 @@ def test_provision_nren_category(
nren_interfaces = [
iface for iface in TEST_INTERFACES if "NREN" in iface["dashboards"]
]
eap_interfaces = [
iface for iface in TEST_INTERFACES if "EAP" in iface["dashboards"]
]
dashboards = {
"NREN": {
"tag": ["customers"],
......@@ -858,8 +914,8 @@ def test_provision_nren_category(
},
"EAP": {
"tag": ["eap"],
"folder_name": "EAP Access",
"interfaces": nren_interfaces,
"folder_name": "EAP NREN Access",
"interfaces": eap_interfaces,
},
}
populate_inventory(
......@@ -867,6 +923,7 @@ def test_provision_nren_category(
"/poller/interfaces": NREN_INTERFACES,
"/data/interfaces": NREN_INTERFACES,
"/poller/eumetsat-multicast": EUMETSAT_MULTICAST,
"/poller/regions": NREN_REGIONS,
}
)
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment