Newer
Older
Helper functions used to group interfaces together and generate the
necessary data to generate the dashboards from templates.
from concurrent.futures import ProcessPoolExecutor
from itertools import product
from functools import partial, reduce
from string import ascii_uppercase
from brian_dashboard_manager.templating.render import create_panel, \
create_panel_target, create_dropdown_panel
NUM_PROCESSES = 4
PANEL_HEIGHT = 12
PANEL_WIDTH = 24
logger = logging.getLogger(__file__)
def num_generator(start=1):
num = start
while True:
yield num
num += 1
def gridPos_generator(id_generator, start=0, agg=False):
num = start
while True:
yield {
"height": PANEL_HEIGHT,
"width": PANEL_WIDTH if not agg else PANEL_WIDTH // 2,
"x": 0,
"y": num * PANEL_HEIGHT,
"id": next(id_generator)
}
if agg:
yield {
"height": PANEL_HEIGHT,
"width": PANEL_WIDTH // 2,
"x": PANEL_WIDTH // 2,
"y": num * PANEL_HEIGHT,
"id": next(id_generator)
}
num += 1
def letter_generator():
i = 0
j = 0
num_letters = len(ascii_uppercase)
while True:
result = ascii_uppercase[i % num_letters]
# tack on an extra letter if we are out of them
if (i >= num_letters):
result += ascii_uppercase[j % num_letters]
j += 1
if (j != 0 and j % num_letters == 0):
i += 1
else:
i += 1
yield result
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
def get_nren_interface_data_old(interfaces):
"""
Helper for grouping interfaces into groups of NRENs
Extracts information from interfaces to be used in panels.
NREN dashboards have aggregate panels at the top and
dropdowns for services / physical interfaces.
"""
result = {}
for interface in interfaces:
description = interface['description'].strip()
interface_name = interface['name']
host = interface['router']
router = host.replace('.geant.net', '')
location = host.split('.')[1].upper()
panel_title = f"{router} - {{}} - {interface_name} - {description}"
dashboards_info = interface['dashboards_info']
for info in dashboards_info:
dashboard_name = info['name']
dashboard = result.get(dashboard_name, {
'AGGREGATES': [],
'SERVICES': [],
'PHYSICAL': []
})
if info['interface_type'] == 'AGGREGATE':
dashboard['AGGREGATES'].append({
'interface': interface_name,
'hostname': host,
'alias':
f"{location} - {dashboard_name} ({interface_name})"
})
# link aggregates are also shown
# under the physical dropdown
dashboard['PHYSICAL'].append({
'title': panel_title,
'hostname': host,
'interface': interface_name
})
elif info['interface_type'] == 'LOGICAL':
dashboard['SERVICES'].append({
'title': panel_title,
'hostname': host,
'interface': interface_name
})
elif info['interface_type'] == 'PHYSICAL':
dashboard['PHYSICAL'].append({
'title': panel_title,
'hostname': host,
'interface': interface_name
})
result[dashboard_name] = dashboard
return result
def get_nren_interface_data(services, interfaces, excluded_dashboards):
"""
Helper for grouping interfaces into groups of NRENs
Extracts information from interfaces to be used in panels.
NREN dashboards have aggregate panels at the top and
dropdowns for services / physical interfaces.
"""
aggregate_interfaces = dict()
for service in services:
_customers = service.get('customers')
for cust in _customers:
if cust.lower() in excluded_dashboards:
continue
customers[cust].append(service)
for customer, services in customers.items():
if not any([s['service_type'] == 'GEANT IP' for s in services]):
# NREN access customers must have at least one GEANT IP service
continue
dashboard = result.setdefault(customer, {
'AGGREGATES': [],
'SERVICES': [],
'PHYSICAL': []
})
for service in services:
_interfaces = service.get('endpoints')
name = service.get('name')
sid = service.get('sid')
scid = service.get('scid')
measurement = 'scid_rates'
if len(_interfaces) == 0:
continue
if len(_interfaces) > 1:
logger.info(
f'{sid} {name} aggregate service has > 1 interface')
continue
if 'interface' in _interfaces[0]:
if_name = _interfaces[0].get('interface')
router = _interfaces[0].get('hostname')
else:
if_name = _interfaces[0].get('port')
router = _interfaces[0].get('equipment')
router = router.replace('.geant.net', '')
location = router.split('.')[1].upper()
title = f'{location} - {customer} ({if_name}) | {name}'
aggregate_interfaces[f'{router}:::{if_name}'] = True
dashboard['AGGREGATES'].append({
'measurement': measurement,
'alias': title,
'scid': scid
})
if 'MDVPN' in service['service_type']:
# MDVPN type services don't have data in BRIAN
continue
title = f'{name} ({sid})'
dashboard['SERVICES'].append({
'measurement': measurement,
'title': title,
'scid': scid
})
def _check_in_aggregate(router, interface):
return f'{router}:::{interface}' in aggregate_interfaces
for interface in interfaces:
description = interface['description'].strip()
interface_name = interface['name']
host = interface['router']
port_type = interface.get('port_type', 'unknown').lower()
router = host.replace('.geant.net', '')
location = host.split('.')[1].upper()
panel_title = f"{router} - {{}} - {interface_name} - {description}"
dashboards_info = interface['dashboards_info']
for info in dashboards_info:
dashboard_name = info['name']
dashboard = result.get(dashboard_name, {
'AGGREGATES': [],
'SERVICES': [],
'PHYSICAL': []
_covered_by_service = _check_in_aggregate(router, interface_name)
if port_type == 'access' and not _covered_by_service:
dashboard['AGGREGATES'].append({
'interface': interface_name,
'hostname': host,
'alias':
f"{location} - {dashboard_name} ({interface_name})"
})
if info['interface_type'] == 'AGGREGATE':
# link aggregates are also shown
# under the physical dropdown
dashboard['PHYSICAL'].append({
'title': panel_title,
'hostname': host,
'interface': interface_name
})
elif info['interface_type'] == 'PHYSICAL':
dashboard['PHYSICAL'].append({
'title': panel_title,
'hostname': host,
'interface': interface_name
})
result[dashboard_name] = dashboard
for customer in list(result.keys()):
lengths = [len(val) for val in result[customer].values()]
if sum(lengths) == 0:
# no services/interfaces, so remove it
del result[customer]
return result
def get_interface_data(interfaces):
"""
Helper for grouping interfaces into dashboards.
Extracts information from interfaces to be used in panels.
"""
result = {}
for interface in interfaces:
description = interface['description'].strip()
interface_name = interface['name']
host = interface['router']
router = host.replace('.geant.net', '')
panel_title = f"{router} - {{}} - {interface_name} - {description}"
dashboards_info = interface['dashboards_info']
for info in dashboards_info:
dashboard_name = info['name']
dashboard = result.get(dashboard_name, [])
dashboard.append({
'title': panel_title,
'interface': interface_name,
'hostname': host,
'has_v6': len(interface.get('ipv6', [])) > 0
})
result[dashboard_name] = dashboard
def get_aggregate_interface_data(interfaces, agg_type, group_field):
Helper for grouping interfaces into groups by fields, eg. remotes
(ISP/NREN/...) used for aggregate dashboards
Extracts information from interfaces to be used in panels.
Aggregate dashboards have aggregates at the top for all groups
as well as aggregate panels for specific groups.
This builds a dict with interfaces for each group
and one with all interfaces.
"""
def get_reduce_func_for_field(field):
def reduce_func(prev, curr):
groups = prev.get(curr[field], [])
groups.append(curr)
all_agg = prev.get('EVERYSINGLETARGET', [])
all_agg.append(curr)
prev[curr[field]] = groups
prev['EVERYSINGLETARGET'] = all_agg
return prev
return reduce_func
for interface in interfaces:
interface_name = interface.get('name')
host = interface.get('router', '')
for info in interface['dashboards_info']:
remote = info['name']
location = host.split('.')[1].upper()
result.append({
'type': agg_type,
'interface': interface_name,
'hostname': host,
'remote': remote,
'location': location,
'alias': f"{location} - {remote} ({interface_name})",
})
return reduce(get_reduce_func_for_field(group_field), result, {})
def get_aggregate_targets(targets):
"""
Helper used for generating panel fields for aggregate panels
with multiple target fields (ingress/egress)
"""
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
ingress = []
egress = []
# used to generate refIds
letters = letter_generator()
for target in targets:
ref_id = next(letters)
in_data = {
**target,
'alias': f"{target['alias']} - Ingress Traffic",
'refId': ref_id,
'select_field': 'ingress'
}
out_data = {
**target,
'alias': f"{target['alias']} - Egress Traffic",
'refId': ref_id,
'select_field': 'egress'
}
ingress_target = create_panel_target(in_data)
egress_target = create_panel_target(out_data)
ingress.append(ingress_target)
egress.append(egress_target)
return ingress, egress
def get_panel_fields(panel, panel_type, datasource):
"""
Helper for generating a single panel,
with ingress/egress and percentile targets
"""
letters = letter_generator()
def get_target_data(alias, field):
return {
# panel includes identifying information
# such as hostname, interface, etc.
**panel,
'alias': alias,
'refId': next(letters),
'select_field': field,
'percentile': 'percentile' in alias.lower(),
'errors': panel_type == 'errors' # used to remove *8 on value
}
error_fields = [('Ingress Errors', 'errorsIn'),
('Egress Errors', 'errorsOut'),
('Ingress Discards', 'discardsIn'),
('Egress Discards', 'discardsOut')]
ingress = ['Ingress Traffic', 'Ingress 95th Percentile']
egress = ['Egress Traffic', 'Egress 95th Percentile']
is_v6 = panel_type == 'IPv6'
is_error = panel_type == 'errors'
in_field = 'ingressv6' if is_v6 else 'ingress'
out_field = 'egressv6' if is_v6 else 'egress'
fields = [*product(ingress, [in_field]), *product(egress, [out_field])]
targets = error_fields if is_error else fields
return create_panel({
**panel,
'datasource': datasource,
'title': panel['title'].format(panel_type),
'panel_targets': [get_target_data(*target) for target in targets],
'y_axis_type': 'errors' if is_error else 'bits',
})
def default_interface_panel_generator(gridPos):
"""
Shared wrapper for shorter calls without
gridPos to generate panels.
Generates panels used in a normal dashboard
for all traffic + (conditionally) IPv6 + Errors
"""
def get_panel_definitions(panels, datasource, errors=False):
result.append(get_panel_fields({
**panel,
**next(gridPos)
}, 'traffic', datasource))
if panel.get('has_v6', False):
result.append(get_panel_fields({
**panel,
**next(gridPos)
}, 'IPv6', datasource))
result.append(get_panel_fields({
**panel,
**next(gridPos)
}, 'errors', datasource))
return get_panel_definitions
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
def get_nren_dashboard_data_single(data, datasource, tag):
nren, dash = data
id_gen = num_generator()
gridPos = gridPos_generator(id_gen, start=1)
if len(dash['AGGREGATES']) > 0:
agg_panels = create_aggregate_panel(
f'Aggregate - {nren}',
gridPos_generator(id_gen, agg=True),
dash['AGGREGATES'], datasource)
else:
# if there's no aggregate panel(s), start other stuff at y=0.
gridPos = gridPos_generator(id_gen, start=0)
agg_panels = []
panel_gen = default_interface_panel_generator(gridPos)
services_dropdown = create_dropdown_panel('Services', **next(gridPos))
service_panels = panel_gen(dash['SERVICES'], datasource)
iface_dropdown = create_dropdown_panel('Interfaces', **next(gridPos))
phys_panels = panel_gen(dash['PHYSICAL'], datasource, True)
result = {
'nren_name': nren,
'datasource': datasource,
'aggregate_panels': agg_panels,
'dropdown_groups': [
{
'dropdown': services_dropdown,
'panels': service_panels,
},
{
'dropdown': iface_dropdown,
'panels': phys_panels,
}
]
}
if isinstance(tag, list):
result['tags'] = tag
else:
result['tag'] = tag
return result
def get_nren_dashboard_data(data, datasource, tag):
"""
Generates all panels used in a NREN dashboard,
including dropdowns and aggregate panels.
"""
with ProcessPoolExecutor(max_workers=NUM_PROCESSES) as executor:
for dash in executor.map(
partial(
get_nren_dashboard_data_single,
datasource=datasource,
tag=tag),
data.items()
):
yield dash
def get_dashboard_data_single(
data, datasource, tag,
panel_generator=default_interface_panel_generator,
errors=False):
"""
Generates all panels used in a normal dashboard without aggregate panels
"""
id_gen = num_generator()
gridPos = gridPos_generator(id_gen)
panel_gen = panel_generator(gridPos)
name, panels = data
result = {
'title': name,
'datasource': datasource,
'panels': list(panel_gen(panels, datasource, errors)),
}
if isinstance(tag, list):
result['tags'] = tag
else:
result['tag'] = tag
return result
def get_dashboard_data(
data, datasource, tag,
panel_generator=default_interface_panel_generator,
errors=False):
"""
Generates all panels used in a normal dashboard without aggregate panels
"""
with ProcessPoolExecutor(max_workers=NUM_PROCESSES) as executor:
for dash in executor.map(
partial(
get_dashboard_data_single,
datasource=datasource,
tag=tag,
panel_generator=panel_generator,
errors=errors),
data.items()
):
yield dash
def create_aggregate_panel(title, gridpos, targets, datasource):
"""
Generates a single panel with multiple targets.
Each target is one interface / line on the graph
"""
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
ingress_targets, egress_targets = get_aggregate_targets(targets)
result = []
ingress_pos = next(gridpos)
egress_pos = next(gridpos)
is_total = 'totals' in title.lower()
def reduce_alias(prev, curr):
d = json.loads(curr)
alias = d['alias']
if 'egress' in alias.lower():
prev[alias] = '#0000FF'
else:
prev[alias] = '#00FF00'
return prev
ingress_colors = reduce(reduce_alias, ingress_targets, {})
egress_colors = reduce(reduce_alias, egress_targets, {})
result.append(create_panel({
**ingress_pos,
'stack': True,
'linewidth': 0 if is_total else 1,
'datasource': datasource,
'title': title + ' - ingress',
'targets': ingress_targets,
'y_axis_type': 'bits',
'alias_colors': json.dumps(ingress_colors) if is_total else {}
}))
result.append(create_panel({
**egress_pos,
'stack': True,
'linewidth': 0 if is_total else 1,
'datasource': datasource,
'title': title + ' - egress',
'targets': egress_targets,
'y_axis_type': 'bits',
'alias_colors': json.dumps(egress_colors) if is_total else {}
}))
return result
def get_aggregate_dashboard_data(title, targets, datasource, tag):
"""
Creates three types of aggregate panels:
Aggregate Ingress/Egress that contain
every target (interface) given as parameter
Totals Ingress/Egress which is the same as above,
but with a different line color.
Aggregates for each remote
(all interfaces for each remote (ISP/NREN/...)
on separate graphs
"""
id_gen = num_generator()
gridPos = gridPos_generator(id_gen, agg=True)
panels = []
all_targets = targets.get('EVERYSINGLETARGET', [])
ingress, egress = create_aggregate_panel(
title, gridPos, all_targets, datasource)
panels.extend([ingress, egress])
totals_title = title + ' - Totals'
t_in, t_eg = create_aggregate_panel(
totals_title, gridPos, all_targets, datasource)
panels.extend([t_in, t_eg])
if 'EVERYSINGLETARGET' in targets:
del targets['EVERYSINGLETARGET']
for target in targets:
_in, _out = create_aggregate_panel(
title + f' - {target}', gridPos, targets[target], datasource)
panels.extend([_in, _out])
'title': title,
'datasource': datasource,
'panels': panels,
if isinstance(tag, list):
result['tags'] = tag
else:
result['tag'] = tag
return result