Remove use of multiprocessing and simplify codebase a bit

Obtain significant speedups by looking up folder dashboards in advance

brian_dashboard_manager/grafana/dashboard.py

@@ -6,6 +6,7 @@ import os
 import json
 import time
 
+from typing import Dict, Any, Optional
 from requests.exceptions import HTTPError
 from brian_dashboard_manager.grafana.utils.request import TokenRequest
 
@@ -191,7 +192,8 @@ def _get_dashboard(request: TokenRequest, uid):
     return r.json()['dashboard']
 
 
-def create_dashboard(request: TokenRequest, dashboard: dict, folder_id=None):
+def create_dashboard(request: TokenRequest, dashboard: dict, folder_id=None,
+                     existing_folder_dashboards: Optional[Dict[str, Any]] = None):
     """
     Creates the given dashboard for the organization tied to the token.
     If the dashboard already exists, it will be updated.
@@ -205,22 +207,32 @@ def create_dashboard(request: TokenRequest, dashboard: dict, folder_id=None):
     title = dashboard['title']
     existing_dashboard = None
     has_uid = dashboard.get('uid') is not None
-    if has_uid:
-        existing_dashboard = _get_dashboard(request, uid=dashboard['uid'])
-    else:
-        existing_dashboard = _search_dashboard(request, dashboard, folder_id)
+
+    existing_dashboard = None
+
+    if existing_folder_dashboards:
+        existing_dashboard = existing_folder_dashboards.get(dashboard['title'].lower())
+        if has_uid:
+            assert existing_dashboard is None or existing_dashboard['uid'] == dashboard['uid'], \
+                f"UID mismatch for dashboard {title}: {existing_dashboard['uid']} != {dashboard['uid']}"
+
+    if not existing_dashboard:
+        if has_uid:
+            existing_dashboard = _get_dashboard(request, uid=dashboard['uid'])
+        else:
+            existing_dashboard = _search_dashboard(request, dashboard, folder_id)
 
     if existing_dashboard:
         dashboard['uid'] = existing_dashboard['uid']
         dashboard['id'] = existing_dashboard['id']
-        dashboard['version'] = existing_dashboard['version']
+        dashboard['version'] = 1
     else:
         # We are creating a new dashboard, delete ID if it exists.
         dashboard.pop('id', None)
 
     payload = {
         'dashboard': dashboard,
-        'overwrite': False
+        'overwrite': True
     }
     if folder_id:
         payload['folderId'] = folder_id

brian_dashboard_manager/grafana/folder.py

@@ -6,6 +6,24 @@ from brian_dashboard_manager.grafana.utils.request import TokenRequest
 logger = logging.getLogger(__name__)
 
 
+def list_folder_dashboards(request: TokenRequest, folder_uid):
+    """
+    Lists all dashboards in a folder.
+
+    :param request: TokenRequest object
+    :param folder_uid: folder UID
+    :return: list of dashboard definitions
+    """
+    try:
+        r = request.get(f'api/search?folderUIDs={folder_uid}')
+        dashboards = r.json()
+    except HTTPError:
+        logger.exception(f'Error when listing dashboards in folder: {folder_uid}')
+        return []
+
+    return {dash['title'].lower(): dash for dash in dashboards}
+
+
 def delete_folder(request: TokenRequest, title=None, uid=None):
     """
     Deletes a single folder for the organization

brian_dashboard_manager/templating/helpers.py

@@ -2,9 +2,11 @@
 Helper functions used to group interfaces together and generate the
 necessary data to generate the dashboards from templates.
 """
-from collections import defaultdict
-from concurrent.futures import ProcessPoolExecutor
 import logging
+
+from typing import Union
+from collections import defaultdict
+from concurrent.futures import ThreadPoolExecutor, ProcessPoolExecutor
 from itertools import product
 from functools import partial, reduce
 from string import ascii_uppercase
@@ -12,7 +14,9 @@ from string import ascii_uppercase
 from brian_dashboard_manager.templating.render import create_panel, \
     create_panel_target, create_dropdown_panel
 
-NUM_PROCESSES = 4
+AnyExecutor = Union[ProcessPoolExecutor, ThreadPoolExecutor]
+
+DEFAULT_TIMEOUT = 60
 
 PANEL_HEIGHT = 12
 PANEL_WIDTH = 24
@@ -686,29 +690,122 @@ def default_interface_panel_generator(gridPos, use_all_traffic=True, use_ipv6=Tr
     return get_panel_definitions
 
 
-def _get_dashboard_data(data, datasource, tag, single_data_func):
+def create_aggregate_panel(title, gridpos, targets, datasource):
     """
-    Helper for generating dashboard definitions.
-    Uses multiprocessing to speed up generation.
+    Helper for generating aggregate panels. Creates two panels, one for
+    ingress and one for egress.
 
-    :param data: the dashboard names and the panel data for each dashboard
+    :param title: title for the panel
+    :param gridpos: generator for grid position
+    :param targets: list of targets for the panels, used to build separate
+    targets for both ingress and egress.
+    :param datasource: datasource to use for the panels
+
+    :return: tuple of aggregate panels, one for ingress and one for egress
+    """
+
+    ingress_targets, egress_targets = get_aggregate_targets(targets)
+
+    ingress_pos = next(gridpos)
+    egress_pos = next(gridpos)
+
+    is_total = 'totals' in title.lower()
+
+    def reduce_alias(prev, curr):
+        alias = curr['alias']
+        if 'egress' in alias.lower():
+            prev[alias] = '#0000FF'
+        else:
+            prev[alias] = '#00FF00'
+        return prev
+
+    ingress_colors = reduce(reduce_alias, ingress_targets, {})
+    egress_colors = reduce(reduce_alias, egress_targets, {})
+
+    ingress = create_panel(
+        **ingress_pos,
+        stack=True,
+        linewidth=0 if is_total else 1,
+        datasource=datasource,
+        title=title + " - ingress",
+        targets=ingress_targets,
+        y_axis_type="bits",
+        alias_colors=ingress_colors if is_total else {},
+    )
+
+    egress = create_panel(
+        **egress_pos,
+        stack=True,
+        linewidth=0 if is_total else 1,
+        datasource=datasource,
+        title=title + " - egress",
+        targets=egress_targets,
+        y_axis_type="bits",
+        alias_colors=egress_colors if is_total else {},
+    )
+
+    return ingress, egress
+
+
+def get_aggregate_dashboard_data(title, remotes, datasource, tag):
+    """
+    Helper for generating aggregate dashboard definitions.
+    Aggregate dashboards consist only of aggregate panels that are
+    panels with data for multiple interfaces.
+
+    At the top of the dashboard are two aggregate panels showing
+    total ingress and egress data for all interfaces.
+
+    Below that are two aggregate panels for each target, one for
+    ingress and one for egress.
+
+    :param title: title for the dashboard
+    :param remotes: dictionary of targets for the panels, the key is the
+    remote (usually a customer) and the value is a list of targets
+    for that remote. A single target represents how to fetch
+    data for one interface.
     :param datasource: datasource to use for the panels
     :param tag: tag to use for the dashboard, used for dashboard dropdowns on
     the home dashboard.
-    :param single_data_func: function that gets data for one definition
 
-    :return: generator for dashboard definitions for each dashboard
+    :return: dashboard definition for the aggregate dashboard
     """
 
-    with ProcessPoolExecutor(max_workers=NUM_PROCESSES) as executor:
-        for dash in executor.map(
-                partial(
-                    single_data_func,
-                    datasource=datasource,
-                    tag=tag),
-                data.items()
-        ):
-            yield dash
+    id_gen = num_generator()
+    gridPos = gridPos_generator(id_gen, agg=True)
+
+    panels = []
+    all_targets = remotes.get('EVERYSINGLETARGET', [])
+
+    ingress, egress = create_aggregate_panel(
+        title, gridPos, all_targets, datasource)
+    panels.extend([ingress, egress])
+
+    totals_title = title + ' - Totals'
+    t_in, t_eg = create_aggregate_panel(
+        totals_title, gridPos, all_targets, datasource)
+    panels.extend([t_in, t_eg])
+
+    if 'EVERYSINGLETARGET' in remotes:
+        del remotes['EVERYSINGLETARGET']
+
+    for remote in remotes:
+        _in, _out = create_aggregate_panel(
+            title + f' - {remote}', gridPos, remotes[remote], datasource)
+        panels.extend([_in, _out])
+
+    result = {
+        'title': title,
+        'datasource': datasource,
+        'panels': panels,
+
+    }
+    if isinstance(tag, list):
+        result['tags'] = tag
+    else:
+        result['tag'] = tag
+
+    return result
 
 
 def get_nren_dashboard_data_single(data, datasource, tag):
@@ -790,10 +887,6 @@ def get_nren_dashboard_data_single(data, datasource, tag):
     return result
 
 
-def get_nren_dashboard_data(data, datasource, tag):
-    yield from _get_dashboard_data(data, datasource, tag, get_nren_dashboard_data_single)
-
-
 def get_re_peer_dashboard_data_single(data, datasource, tag):
     """
     Helper for generating dashboard definitions for a single R&E Peer.
@@ -863,10 +956,6 @@ def get_re_peer_dashboard_data_single(data, datasource, tag):
     return result
 
 
-def get_re_peer_dashboard_data(data, datasource, tag):
-    yield from _get_dashboard_data(data, datasource, tag, get_re_peer_dashboard_data_single)
-
-
 def get_service_dashboard_data_single(data, datasource, tag):
     """
     Helper for generating dashboard definitions for a single service.
@@ -917,10 +1006,6 @@ def get_service_dashboard_data_single(data, datasource, tag):
     return result
 
 
-def get_service_dashboard_data(data, datasource, tag):
-    yield from _get_dashboard_data(data, datasource, tag, get_service_dashboard_data_single)
-
-
 def get_dashboard_data_single(
         data, datasource, tag,
         panel_generator=default_interface_panel_generator,
@@ -963,8 +1048,7 @@ def get_dashboard_data(
         panel_generator=default_interface_panel_generator,
         errors=False):
     """
-    Helper for generating dashboard definitions for all non-NREN dashboards.
-    Uses multiprocessing to speed up generation.
+    Helper for generating dashboard definitions for interface-based non-NREN dashboards.
 
     :param data: the dashboard names and the panel data for each dashboard
     :param datasource: datasource to use for the panels
@@ -976,135 +1060,36 @@ def get_dashboard_data(
     :return: generator for dashboard definitions for each dashboard
     """
 
-    with ProcessPoolExecutor(max_workers=NUM_PROCESSES) as executor:
-        try:
-            for dash in executor.map(
-                    partial(
-                        get_dashboard_data_single,
-                        datasource=datasource,
-                        tag=tag,
-                        panel_generator=panel_generator,
-                        errors=errors),
-                    data.items()
-            ):
-                yield dash
-        finally:
-            executor.shutdown(wait=False)
-
-
-def create_aggregate_panel(title, gridpos, targets, datasource):
-    """
-    Helper for generating aggregate panels. Creates two panels, one for
-    ingress and one for egress.
-
-    :param title: title for the panel
-    :param gridpos: generator for grid position
-    :param targets: list of targets for the panels, used to build separate
-    targets for both ingress and egress.
-    :param datasource: datasource to use for the panels
-
-    :return: tuple of aggregate panels, one for ingress and one for egress
-    """
-
-    ingress_targets, egress_targets = get_aggregate_targets(targets)
+    func = partial(
+        get_dashboard_data_single,
+        datasource=datasource, tag=tag, panel_generator=panel_generator, errors=errors)
 
-    ingress_pos = next(gridpos)
-    egress_pos = next(gridpos)
+    yield from map(func, data.items())
 
-    is_total = 'totals' in title.lower()
 
-    def reduce_alias(prev, curr):
-        alias = curr['alias']
-        if 'egress' in alias.lower():
-            prev[alias] = '#0000FF'
-        else:
-            prev[alias] = '#00FF00'
-        return prev
-
-    ingress_colors = reduce(reduce_alias, ingress_targets, {})
-    egress_colors = reduce(reduce_alias, egress_targets, {})
-
-    ingress = create_panel(
-        **ingress_pos,
-        stack=True,
-        linewidth=0 if is_total else 1,
-        datasource=datasource,
-        title=title + " - ingress",
-        targets=ingress_targets,
-        y_axis_type="bits",
-        alias_colors=ingress_colors if is_total else {},
-    )
+def get_nren_dashboard_data(data, datasource, tag):
 
-    egress = create_panel(
-        **egress_pos,
-        stack=True,
-        linewidth=0 if is_total else 1,
+    func = partial(
+        get_nren_dashboard_data_single,
         datasource=datasource,
-        title=title + " - egress",
-        targets=egress_targets,
-        y_axis_type="bits",
-        alias_colors=egress_colors if is_total else {},
-    )
+        tag=tag)
 
-    return ingress, egress
+    yield from map(func, data.items())
 
 
-def get_aggregate_dashboard_data(title, remotes, datasource, tag):
-    """
-    Helper for generating aggregate dashboard definitions.
-    Aggregate dashboards consist only of aggregate panels that are
-    panels with data for multiple interfaces.
-
-    At the top of the dashboard are two aggregate panels showing
-    total ingress and egress data for all interfaces.
-
-    Below that are two aggregate panels for each target, one for
-    ingress and one for egress.
-
-    :param title: title for the dashboard
-    :param remotes: dictionary of targets for the panels, the key is the
-    remote (usually a customer) and the value is a list of targets
-    for that remote. A single target represents how to fetch
-    data for one interface.
-    :param datasource: datasource to use for the panels
-    :param tag: tag to use for the dashboard, used for dashboard dropdowns on
-    the home dashboard.
-
-    :return: dashboard definition for the aggregate dashboard
-    """
-
-    id_gen = num_generator()
-    gridPos = gridPos_generator(id_gen, agg=True)
-
-    panels = []
-    all_targets = remotes.get('EVERYSINGLETARGET', [])
-
-    ingress, egress = create_aggregate_panel(
-        title, gridPos, all_targets, datasource)
-    panels.extend([ingress, egress])
+def get_re_peer_dashboard_data(data, datasource, tag):
+    func = partial(
+        get_re_peer_dashboard_data_single,
+        datasource=datasource,
+        tag=tag)
 
-    totals_title = title + ' - Totals'
-    t_in, t_eg = create_aggregate_panel(
-        totals_title, gridPos, all_targets, datasource)
-    panels.extend([t_in, t_eg])
+    yield from map(func, data.items())
 
-    if 'EVERYSINGLETARGET' in remotes:
-        del remotes['EVERYSINGLETARGET']
 
-    for remote in remotes:
-        _in, _out = create_aggregate_panel(
-            title + f' - {remote}', gridPos, remotes[remote], datasource)
-        panels.extend([_in, _out])
-
-    result = {
-        'title': title,
-        'datasource': datasource,
-        'panels': panels,
-
-    }
-    if isinstance(tag, list):
-        result['tags'] = tag
-    else:
-        result['tag'] = tag
+def get_service_dashboard_data(data, datasource, tag):
+    func = partial(
+        get_service_dashboard_data_single,
+        datasource=datasource,
+        tag=tag)
 
-    return result
+    yield from map(func, data.items())