Finished release 0.80.

brian_dashboard_manager/config.py

@@ -60,6 +60,7 @@ DEFAULT_ORGANIZATIONS = [
             "EUMETSAT Multicast": True,
             "NREN Access LEGACY": True,
             "VLAN Interfaces": True,
+            "RE Peers": True,
         }
     },
     {
@@ -93,6 +94,7 @@ DEFAULT_ORGANIZATIONS = [
             "EUMETSAT Multicast": True,
             "NREN Access LEGACY": True,
             "VLAN Interfaces": True,
+            "RE Peers": True,
         }
     },
     {
@@ -115,6 +117,7 @@ DEFAULT_ORGANIZATIONS = [
             "EUMETSAT Multicast": True,
             "NREN Access LEGACY": True,
             "VLAN Interfaces": True,
+            "RE Peers": True,
         }
     },
     {
@@ -146,6 +149,7 @@ DEFAULT_ORGANIZATIONS = [
             "EUMETSAT Multicast": True,
             "NREN Access LEGACY": True,
             "VLAN Interfaces": True,
+            "RE Peers": True,
         }
     }
 ]

brian_dashboard_manager/grafana/folder.py

@@ -111,12 +111,9 @@ def create_folder(request: TokenRequest, title):
     :param title: folder title
     :return: folder definition
     """
-    try:
-        data = {'title': title, 'uid': title.replace(' ', '_')}
-        r = request.post('api/folders', json=data)
-    except HTTPError:
-        logger.exception(f'Error when creating folder {title}')
-        return None
+    data = {'title': title, 'uid': title.replace(' ', '_')}
+    r = request.post('api/folders', json=data)
+
     return r.json()
 
 

brian_dashboard_manager/grafana/provision.py

@@ -154,7 +154,14 @@ SERVICE_DASHBOARDS = {
         'folder_name': 'Managed Wavelength Service',
         'interfaces': [],
         'services': []
-    }
+    },
+    'RE_PEERS': {
+        'tag': ['RE_PEER'],
+        'service_type': 'IP PEERING - R&E',
+        'folder_name': 'RE Peers',
+        'interfaces': [],
+        'services': []
+    },
 }
 
 AGG_DASHBOARDS = {
@@ -835,96 +842,93 @@ def _add_service_data(org_config, services, regions):
 
 
 def _provision_org(config, org, org_config, interfaces, services, regions):
+    request = AdminRequest(**config)
+    org_id = org['id']
+    accounts = []
+
+    logger.info(f'--- Provisioning org {org["name"]} (ID #{org_id}) ---')
+
     try:
-        request = AdminRequest(**config)
-        org_id = org['id']
-        accounts = []
+        # create a service account for provisioning (>grafana 11.0)
+        account = get_or_create_service_account(request, org_id)
+        token = create_service_account_token(request, account['id'])
+        accounts.append((org_id, account))
+    except Exception:
+        # we're on a older version of grafana
+        token = create_api_token(request, org_id)
+        accounts.append((org_id, token))
+
+    token_request = TokenRequest(token=token['key'], **config)
+    logger.debug(accounts)
+
+    all_original_dashboards = list_dashboards(token_request)
+    all_original_dashboard_uids = {d['uid']: d.get('folderUrl', '') + d['url'] for d in all_original_dashboards}
+
+    datasource = _provision_datasource(config, token_request)
+    ds_name = datasource.get('name', 'PollerInfluxDB')
+
+    with ThreadPoolExecutor(max_workers=MAX_THREADS) as thread_executor:
+
+        args = (thread_executor, config, org_config, ds_name, token_request)
+
+        # initialise the aggregate dashboards with service data, to be used in the provisioning process
+        # it doesn't create the dashboards, just prepares the data
+        _add_service_data(org_config, services, regions)
+
+        # call to list is needed to queue up the futures
+        managed_dashboards = [f.result() if isinstance(f, Future) else f for f in list(itertools.chain(
+            _provision_interfaces(*args, interfaces, services, regions),
+            _provision_vlan_dashboards(*args, interfaces),
+            _provision_gws_indirect(*args),
+            _provision_gws_direct(*args),
+            _provision_eumetsat_multicast(*args),
+            _provision_aggregates(*args),
+            _provision_static_dashboards(*args),
+            _get_ignored_dashboards(*args)
+        ))]
+
+        managed_dashboard_uids = {}
+        for dashboard in managed_dashboards:
+            if isinstance(dashboard, Future):
+                dashboard = dashboard.result()
+            if dashboard is None:
+                continue
+            assert dashboard['uid'] not in managed_dashboard_uids, \
+                f'Dashboard with UID {dashboard["uid"]} already exists: {dashboard}'
+            managed_dashboard_uids[dashboard['uid']] = dashboard['url']
+
+    difference = set(all_original_dashboard_uids.keys()) - set(managed_dashboard_uids.keys())
+    for uid in difference:
+        info = all_original_dashboard_uids[uid]
+        # delete unmanaged dashboards
+        logger.info(f'Deleting stale dashboard {info} with UID {uid}')
+        delete_dashboard(token_request, {'uid': uid})
+
+    folders_to_keep = {
+        # General is a base folder present in Grafana
+        'General',
+        # other folders, created outside of the DASHBOARDS list
+        'GWS Indirect',
+        'GWS Direct',
+        'Aggregates',
+        'EUMETSAT Multicast',
+        'EAP Dashboard',
+        'VLAN Interfaces',
+    }
+    folders_to_keep.update({dash['folder_name']
+                            for dash in DASHBOARDS.values()})
+    folders_to_keep.update({dash['folder_name']
+                            for dash in SERVICE_DASHBOARDS.values()})
 
-        logger.info(f'--- Provisioning org {org["name"]} (ID #{org_id}) ---')
+    ignored_folders = config.get('ignored_folders', [])
+    folders_to_keep.update(ignored_folders)
 
-        try:
-            # create a service account for provisioning (>grafana 11.0)
-            account = get_or_create_service_account(request, org_id)
-            token = create_service_account_token(request, account['id'])
-            accounts.append((org_id, account))
-        except Exception:
-            # we're on a older version of grafana
-            token = create_api_token(request, org_id)
-            accounts.append((org_id, token))
-
-        token_request = TokenRequest(token=token['key'], **config)
-        logger.debug(accounts)
-
-        all_original_dashboards = list_dashboards(token_request)
-        all_original_dashboard_uids = {d['uid']: d.get('folderUrl', '') + d['url'] for d in all_original_dashboards}
-
-        datasource = _provision_datasource(config, token_request)
-        ds_name = datasource.get('name', 'PollerInfluxDB')
-
-        with ThreadPoolExecutor(max_workers=MAX_THREADS) as thread_executor:
-
-            args = (thread_executor, config, org_config, ds_name, token_request)
-
-            # initialise the aggregate dashboards with service data, to be used in the provisioning process
-            # it doesn't create the dashboards, just prepares the data
-            _add_service_data(org_config, services, regions)
-
-            # call to list is needed to queue up the futures
-            managed_dashboards = list(itertools.chain(
-                _provision_interfaces(*args, interfaces, services, regions),
-                _provision_vlan_dashboards(*args, interfaces),
-                _provision_gws_indirect(*args),
-                _provision_gws_direct(*args),
-                _provision_eumetsat_multicast(*args),
-                _provision_aggregates(*args),
-                _provision_static_dashboards(*args),
-                _get_ignored_dashboards(*args)
-            ))
-
-            managed_dashboard_uids = {}
-            for dashboard in managed_dashboards:
-                if isinstance(dashboard, Future):
-                    dashboard = dashboard.result()
-                if dashboard is None:
-                    continue
-                assert dashboard['uid'] not in managed_dashboard_uids, \
-                    f'Dashboard with UID {dashboard["uid"]} already exists: {dashboard}'
-                managed_dashboard_uids[dashboard['uid']] = dashboard['url']
-
-        difference = set(all_original_dashboard_uids.keys()) - set(managed_dashboard_uids.keys())
-        for uid in difference:
-            info = all_original_dashboard_uids[uid]
-            # delete unmanaged dashboards
-            logger.info(f'Deleting stale dashboard {info} with UID {uid}')
-            delete_dashboard(token_request, {'uid': uid})
-
-        folders_to_keep = {
-            # General is a base folder present in Grafana
-            'General',
-            # other folders, created outside of the DASHBOARDS list
-            'GWS Indirect',
-            'GWS Direct',
-            'Aggregates',
-            'EUMETSAT Multicast',
-            'EAP Dashboard',
-            'VLAN Interfaces',
-        }
-        folders_to_keep.update({dash['folder_name']
-                                for dash in DASHBOARDS.values()})
-        folders_to_keep.update({dash['folder_name']
-                                for dash in SERVICE_DASHBOARDS.values()})
-
-        ignored_folders = config.get('ignored_folders', [])
-        folders_to_keep.update(ignored_folders)
-
-        delete_unknown_folders(token_request, folders_to_keep)
-        try:
-            delete_service_account(request, account['id'])
-        except Exception:
-            # we're on a older version of grafana
-            delete_api_token(request, token['id'], org_id=org_id)
+    delete_unknown_folders(token_request, folders_to_keep)
+    try:
+        delete_service_account(request, account['id'])
     except Exception:
-        logger.exception(f'Error when provisioning org {org["name"]}')
+        # we're on a older version of grafana
+        delete_api_token(request, token['id'], org_id=org_id)
 
 
 def provision(config):

brian_dashboard_manager/routes/update.py

 import datetime
+import logging
 
 from flask import jsonify, Response
-from concurrent.futures import ThreadPoolExecutor
 from flask import Blueprint, current_app
 from brian_dashboard_manager.routes import common
 from brian_dashboard_manager.grafana.provision import provision
 from brian_dashboard_manager import CONFIG_KEY
 
+logger = logging.getLogger(__name__)
+
 provision_state = {
     'time': datetime.datetime.now(datetime.timezone.utc),
     'provisioning': False
@@ -29,13 +31,17 @@ def after_request(resp):
     return common.after_request(resp)
 
 
-def provision_maybe():
+@routes.route('/', methods=['GET'])
+def update():
     """
-    Check if we should provision in case of multiple requests hitting the endpoint.
-    We need to make sure we don't provision if another thread is still running.
+    This resource is used to trigger the dashboard provisioning to Grafana.
+
+    The response will be formatted according to the following schema:
+
+    .. asjson::
+       brian_dashboard_manager.routes.update.UPDATE_RESPONSE_SCHEMA
 
-    :return: tuple of (bool, datetime) representing if we can provision
-        and the timestamp of the last provisioning, respectively.
+    :return: json
     """
 
     global provision_state  # noqa: F824
@@ -43,9 +49,10 @@ def provision_maybe():
     now = datetime.datetime.now(datetime.timezone.utc)
     timestamp = provision_state['time']
     provisioning = provision_state['provisioning']
+    should = True
 
-    if provisioning and (now - timestamp).total_seconds() < 600:  # lockout for 10 minutes at most
-        return False, timestamp
+    if provisioning and (now - timestamp).total_seconds() < 300:  # lockout for 5 minutes at most
+        should = False
 
     def write_timestamp(timestamp, provisioning):
         provision_state['time'] = timestamp
@@ -55,32 +62,18 @@ def provision_maybe():
         now = datetime.datetime.now(datetime.timezone.utc)
         write_timestamp(now, False)
 
-    write_timestamp(now, True)
-
-    executor = ThreadPoolExecutor(max_workers=1)
-    f = executor.submit(provision, current_app.config[CONFIG_KEY])
-    f.add_done_callback(lambda _: _finish())
-    return True, now
-
-
-@routes.route('/', methods=['GET'])
-def update():
-    """
-    This resource is used to trigger the provisioning to Grafana.
-
-    It responds to the request immediately after starting
-    the provisioning process.
-
-    The response will be formatted according to the following schema:
+    if should:
+        write_timestamp(now, True)
 
-    .. asjson::
-       brian_dashboard_manager.routes.update.UPDATE_RESPONSE_SCHEMA
+        try:
+            provision(current_app.config[CONFIG_KEY])
+        except Exception:
+            logger.exception("Error during provisioning:")
+            return jsonify({'data': {'message': 'Provisioning failed, check logs for details.'}}), 500
+        finally:
+            _finish()
 
-    :return: json
-    """
-    should, timestamp = provision_maybe()
-    if should:
-        return jsonify({'data': {'message': 'Provisioning dashboards!'}})
+        return jsonify({'data': {'message': 'Provisioned dashboards!'}})
     else:
         seconds_ago = (datetime.datetime.now(datetime.timezone.utc) - timestamp).total_seconds()
         message = f'Provision already in progress since {timestamp} ({seconds_ago:.2f} seconds ago).'

brian_dashboard_manager/templating/helpers.py

@@ -458,7 +458,8 @@ def get_service_data(service_type, services, interfaces, excluded_dashboards):
 
     for customer, services in customers.items():
         dashboard = result.setdefault(customer, {
-            'SERVICES': []
+            'SERVICES': [],
+            'AGGREGATES': [],
         })
 
         for service in services:
@@ -469,6 +470,7 @@ def get_service_data(service_type, services, interfaces, excluded_dashboards):
 
             measurement = 'scid_rates'
             is_lambda = service_type.lower() in ['geant managed wavelength service', 'geant lambda']
+            is_re_peer = service_type.lower() in ['ip peering - r&e']
 
             if len(_interfaces) == 0:
                 continue
@@ -476,19 +478,31 @@ def get_service_data(service_type, services, interfaces, excluded_dashboards):
             if 'interface' in _interfaces[0]:
                 if_name = _interfaces[0].get('interface')
                 router = _interfaces[0].get('hostname')
+                addresses = _interfaces[0].get('addresses', [])
+                has_v6_interface = any(':' in addr for addr in addresses)
             else:
                 if_name = _interfaces[0].get('port')
                 router = _interfaces[0].get('equipment')
+                has_v6_interface = False
+
             router = router.replace('.geant.net', '')
             title = f'{router} - {{}} - {if_name} - {name} ({sid})'
 
-            dashboard['SERVICES'].append({
+            dash_info = {
                 'measurement': measurement,
                 'title': title,
                 'scid': scid,
                 'sort': (sid[:2], name),
-                'is_lambda': is_lambda
-            })
+                'is_lambda': is_lambda,
+                'has_v6': has_v6_interface
+            }
+
+            dashboard['SERVICES'].append(dash_info)
+
+            if is_re_peer:
+                agg_data = dash_info.copy()
+                agg_data['alias'] = title.replace(' - {}', '')
+                dashboard['AGGREGATES'].append(agg_data)
 
     for customer in list(result.keys()):
         lengths = [len(val) for val in result[customer].values()]

changelog.md

@@ -2,6 +2,11 @@
 
 All notable changes to this project will be documented in this file.
 
+## [0.80] - 2025-06-03
+- POL1-899: Add service-based RE Peers dashboard for GEANT staff validation
+- Make the /update endpoint synchronous instead of spinning off a thread to provision the dashboards.
+
+
 ## [0.79] - 2025-05-30
 - POL1-898: Unify logic for selecting interface between poller-udf and brian-dashboard-manager
 - Add EAP Nren dashboard to NREN Access dropdown

setup.py

@@ -2,7 +2,7 @@ from setuptools import setup, find_packages
 
 setup(
     name='brian-dashboard-manager',
-    version="0.79",
+    version="0.80",
     author='GEANT',
     author_email='swd@geant.org',
     description='',

test/conftest.py

@@ -377,7 +377,7 @@ def mock_grafana(data_config):
         query = request.params
         return (
             200,
-            grafana.list_dashboards(query.get("title"), query.get("folderIds")),
+            grafana.list_dashboards(query.get("title"), query.get("folderIds", query.get('folderUIDs'))),
         )
 
     responses.add_callback(