From b18e7312f3bec103314cc70e5f64e936473fd1f3 Mon Sep 17 00:00:00 2001
From: Bjarke Madsen <bjarke.madsen@geant.org>
Date: Thu, 25 Aug 2022 13:38:36 +0200
Subject: [PATCH] Fix a bug with NREN Access panels being under the wrong
dropdowns. Add multiprocessing for generating dashboards to reduce
provisioning time
---
brian_dashboard_manager/templating/helpers.py | 149 +++++++++++-------
1 file changed, 94 insertions(+), 55 deletions(-)
diff --git a/brian_dashboard_manager/templating/helpers.py b/brian_dashboard_manager/templating/helpers.py
index 4dba5af..fc9930c 100644
--- a/brian_dashboard_manager/templating/helpers.py
+++ b/brian_dashboard_manager/templating/helpers.py
@@ -3,14 +3,17 @@ Helper functions used to group interfaces together and generate the
necessary data to generate the dashboards from templates.
"""
from collections import defaultdict
+from concurrent.futures import ProcessPoolExecutor
import logging
import json
from itertools import product
-from functools import reduce
+from functools import partial, reduce
from string import ascii_uppercase
from brian_dashboard_manager.templating.render import create_panel, \
create_panel_target, create_dropdown_panel
+NUM_PROCESSES = 4
+
PANEL_HEIGHT = 12
PANEL_WIDTH = 24
@@ -365,58 +368,72 @@ def default_interface_panel_generator(gridPos):
return get_panel_definitions
+def get_nren_dashboard_data_single(data, datasource, tag):
+
+ nren, dash = data
+ id_gen = num_generator()
+
+ gridPos = gridPos_generator(id_gen, start=1)
+
+ if len(dash['AGGREGATES']) > 0:
+ agg_panels = create_aggregate_panel(
+ f'Aggregate - {nren}',
+ gridPos_generator(id_gen, agg=True),
+ dash['AGGREGATES'], datasource)
+ else:
+ # if there's no aggregate panel(s), start other stuff at y=0.
+ gridPos = gridPos_generator(id_gen, start=0)
+ agg_panels = []
+
+ panel_gen = default_interface_panel_generator(gridPos)
+
+ services_dropdown = create_dropdown_panel('Services', **next(gridPos))
+ service_panels = panel_gen(dash['SERVICES'], datasource)
+ iface_dropdown = create_dropdown_panel('Interfaces', **next(gridPos))
+ phys_panels = panel_gen(dash['PHYSICAL'], datasource, True)
+
+ result = {
+ 'nren_name': nren,
+ 'datasource': datasource,
+ 'aggregate_panels': agg_panels,
+ 'dropdown_groups': [
+ {
+ 'dropdown': services_dropdown,
+ 'panels': service_panels,
+ },
+ {
+ 'dropdown': iface_dropdown,
+ 'panels': phys_panels,
+ }
+ ]
+ }
+ if isinstance(tag, list):
+ result['tags'] = tag
+ else:
+ result['tag'] = tag
+
+ return result
+
+
def get_nren_dashboard_data(data, datasource, tag):
"""
Generates all panels used in a NREN dashboard,
including dropdowns and aggregate panels.
"""
- for nren, dash in data.items():
- id_gen = num_generator()
-
- gridPos = gridPos_generator(id_gen, start=1)
-
- panel_gen = default_interface_panel_generator(gridPos)
-
- if len(dash['AGGREGATES']) > 0:
- agg_panels = create_aggregate_panel(
- f'Aggregate - {nren}',
- gridPos_generator(id_gen, agg=True),
- dash['AGGREGATES'], datasource)
- else:
- # if there's no aggregate panel(s), start other stuff at y=0.
- gridPos = gridPos_generator(id_gen, start=0)
- agg_panels = []
-
- services_dropdown = create_dropdown_panel('Services', **next(gridPos))
- service_panels = panel_gen(dash['SERVICES'], datasource)
- iface_dropdown = create_dropdown_panel('Interfaces', **next(gridPos))
- phys_panels = panel_gen(dash['PHYSICAL'], datasource, True)
-
- result = {
- 'nren_name': nren,
- 'datasource': datasource,
- 'aggregate_panels': agg_panels,
- 'dropdown_groups': [
- {
- 'dropdown': services_dropdown,
- 'panels': service_panels,
- },
- {
- 'dropdown': iface_dropdown,
- 'panels': phys_panels,
- }
- ]
- }
- if isinstance(tag, list):
- result['tags'] = tag
- else:
- result['tag'] = tag
+ with ProcessPoolExecutor(max_workers=NUM_PROCESSES) as executor:
+ for dash in executor.map(
+ partial(
+ get_nren_dashboard_data_single,
+ datasource=datasource,
+ tag=tag),
+ data.items()
+ ):
- yield result
+ yield dash
-def get_dashboard_data(
+def get_dashboard_data_single(
data, datasource, tag,
panel_generator=default_interface_panel_generator,
errors=False):
@@ -427,19 +444,41 @@ def get_dashboard_data(
gridPos = gridPos_generator(id_gen)
panel_gen = panel_generator(gridPos)
- for dashboard_name, panels in data.items():
- result = {
- 'title': dashboard_name,
- 'datasource': datasource,
- 'panels': list(panel_gen(panels, datasource, errors)),
- }
+ name, panels = data
+ result = {
+ 'title': name,
+ 'datasource': datasource,
+ 'panels': list(panel_gen(panels, datasource, errors)),
+ }
- if isinstance(tag, list):
- result['tags'] = tag
- else:
- result['tag'] = tag
+ if isinstance(tag, list):
+ result['tags'] = tag
+ else:
+ result['tag'] = tag
- yield result
+ return result
+
+
+def get_dashboard_data(
+ data, datasource, tag,
+ panel_generator=default_interface_panel_generator,
+ errors=False):
+ """
+ Generates all panels used in a normal dashboard without aggregate panels
+ """
+
+ with ProcessPoolExecutor(max_workers=NUM_PROCESSES) as executor:
+ for dash in executor.map(
+ partial(
+ get_dashboard_data_single,
+ datasource=datasource,
+ tag=tag,
+ panel_generator=panel_generator,
+ errors=errors),
+ data.items()
+ ):
+
+ yield dash
def create_aggregate_panel(title, gridpos, targets, datasource):
--
GitLab