Fix a bug with NREN Access panels being under the wrong dropdowns.

Add multiprocessing for generating dashboards to reduce provisioning time

brian_dashboard_manager/templating/helpers.py

@@ -3,14 +3,17 @@ Helper functions used to group interfaces together and generate the
 necessary data to generate the dashboards from templates.
 """
 from collections import defaultdict
+from concurrent.futures import ProcessPoolExecutor
 import logging
 import json
 from itertools import product
-from functools import reduce
+from functools import partial, reduce
 from string import ascii_uppercase
 from brian_dashboard_manager.templating.render import create_panel, \
     create_panel_target, create_dropdown_panel
 
+NUM_PROCESSES = 4
+
 PANEL_HEIGHT = 12
 PANEL_WIDTH = 24
 
@@ -365,58 +368,72 @@ def default_interface_panel_generator(gridPos):
     return get_panel_definitions
 
 
+def get_nren_dashboard_data_single(data, datasource, tag):
+
+    nren, dash = data
+    id_gen = num_generator()
+
+    gridPos = gridPos_generator(id_gen, start=1)
+
+    if len(dash['AGGREGATES']) > 0:
+        agg_panels = create_aggregate_panel(
+            f'Aggregate - {nren}',
+            gridPos_generator(id_gen, agg=True),
+            dash['AGGREGATES'], datasource)
+    else:
+        # if there's no aggregate panel(s), start other stuff at y=0.
+        gridPos = gridPos_generator(id_gen, start=0)
+        agg_panels = []
+
+    panel_gen = default_interface_panel_generator(gridPos)
+
+    services_dropdown = create_dropdown_panel('Services', **next(gridPos))
+    service_panels = panel_gen(dash['SERVICES'], datasource)
+    iface_dropdown = create_dropdown_panel('Interfaces', **next(gridPos))
+    phys_panels = panel_gen(dash['PHYSICAL'], datasource, True)
+
+    result = {
+        'nren_name': nren,
+        'datasource': datasource,
+        'aggregate_panels': agg_panels,
+        'dropdown_groups': [
+            {
+                'dropdown': services_dropdown,
+                'panels': service_panels,
+            },
+            {
+                'dropdown': iface_dropdown,
+                'panels': phys_panels,
+            }
+        ]
+    }
+    if isinstance(tag, list):
+        result['tags'] = tag
+    else:
+        result['tag'] = tag
+
+    return result
+
+
 def get_nren_dashboard_data(data, datasource, tag):
     """
     Generates all panels used in a NREN dashboard,
     including dropdowns and aggregate panels.
     """
 
-    for nren, dash in data.items():
-        id_gen = num_generator()
-
-        gridPos = gridPos_generator(id_gen, start=1)
-
-        panel_gen = default_interface_panel_generator(gridPos)
-
-        if len(dash['AGGREGATES']) > 0:
-            agg_panels = create_aggregate_panel(
-                f'Aggregate - {nren}',
-                gridPos_generator(id_gen, agg=True),
-                dash['AGGREGATES'], datasource)
-        else:
-            # if there's no aggregate panel(s), start other stuff at y=0.
-            gridPos = gridPos_generator(id_gen, start=0)
-            agg_panels = []
-
-        services_dropdown = create_dropdown_panel('Services', **next(gridPos))
-        service_panels = panel_gen(dash['SERVICES'], datasource)
-        iface_dropdown = create_dropdown_panel('Interfaces', **next(gridPos))
-        phys_panels = panel_gen(dash['PHYSICAL'], datasource, True)
-
-        result = {
-            'nren_name': nren,
-            'datasource': datasource,
-            'aggregate_panels': agg_panels,
-            'dropdown_groups': [
-                {
-                    'dropdown': services_dropdown,
-                    'panels': service_panels,
-                },
-                {
-                    'dropdown': iface_dropdown,
-                    'panels': phys_panels,
-                }
-            ]
-        }
-        if isinstance(tag, list):
-            result['tags'] = tag
-        else:
-            result['tag'] = tag
+    with ProcessPoolExecutor(max_workers=NUM_PROCESSES) as executor:
+        for dash in executor.map(
+            partial(
+                get_nren_dashboard_data_single,
+                datasource=datasource,
+                tag=tag),
+            data.items()
+        ):
 
-        yield result
+            yield dash
 
 
-def get_dashboard_data(
+def get_dashboard_data_single(
         data, datasource, tag,
         panel_generator=default_interface_panel_generator,
         errors=False):
@@ -427,19 +444,41 @@ def get_dashboard_data(
     gridPos = gridPos_generator(id_gen)
     panel_gen = panel_generator(gridPos)
 
-    for dashboard_name, panels in data.items():
-        result = {
-            'title': dashboard_name,
-            'datasource': datasource,
-            'panels': list(panel_gen(panels, datasource, errors)),
-        }
+    name, panels = data
+    result = {
+        'title': name,
+        'datasource': datasource,
+        'panels': list(panel_gen(panels, datasource, errors)),
+    }
 
-        if isinstance(tag, list):
-            result['tags'] = tag
-        else:
-            result['tag'] = tag
+    if isinstance(tag, list):
+        result['tags'] = tag
+    else:
+        result['tag'] = tag
 
-        yield result
+    return result
+
+
+def get_dashboard_data(
+        data, datasource, tag,
+        panel_generator=default_interface_panel_generator,
+        errors=False):
+    """
+    Generates all panels used in a normal dashboard without aggregate panels
+    """
+
+    with ProcessPoolExecutor(max_workers=NUM_PROCESSES) as executor:
+        for dash in executor.map(
+            partial(
+                get_dashboard_data_single,
+                datasource=datasource,
+                tag=tag,
+                panel_generator=panel_generator,
+                errors=errors),
+            data.items()
+        ):
+
+            yield dash
 
 
 def create_aggregate_panel(title, gridpos, targets, datasource):