Merge branch 'feature/POL1-599-SERVICE-BASED-DASHBOARDS' into 'develop'

Feature/pol1 599 service based dashboards

See merge request live-projects/brian-dashboard-manager!14

brian_dashboard_manager/config.py

@@ -186,6 +186,7 @@ CONFIG_SCHEMA = {
         "hostname": {"type": "string"},
         "listen_port": {"type": "integer"},
         "inventory_provider": {"type": "string"},
+        "reporting_provider": {"type": "string"},
         "datasources": {
             "type": "object",
             "properties": {
@@ -203,6 +204,7 @@ CONFIG_SCHEMA = {
         "admin_password",
         "hostname",
         "inventory_provider",
+        "reporting_provider",
         "datasources",
         "ignored_folders"
     ]

brian_dashboard_manager/grafana/provision.py

@@ -14,6 +14,8 @@ from brian_dashboard_manager.config import DEFAULT_ORGANIZATIONS, STATE_PATH
 from brian_dashboard_manager.grafana.utils.request import AdminRequest, \
     TokenRequest
 
+from brian_dashboard_manager.services.api import fetch_services
+
 from brian_dashboard_manager.grafana.organization import \
     get_organizations, create_organization, create_api_token, \
     delete_api_token, delete_expired_api_tokens, set_home_dashboard
@@ -36,6 +38,7 @@ from brian_dashboard_manager.templating.gws import generate_gws, \
     generate_indirect
 from brian_dashboard_manager.templating.eumetsat \
     import generate_eumetsat_multicast
+from brian_dashboard_manager.templating.services import create_service_panels
 from brian_dashboard_manager.templating.render import render_dashboard
 
 logger = logging.getLogger(__name__)
@@ -356,6 +359,33 @@ def _provision_interfaces(config, org_config, ds_name, token):
             yield from folder
 
 
+def _provision_service_dashboards(config, org_config, ds_name, token):
+    """
+    Fetches service data from Reporting Provider
+    and creates dashboards for each customer with their services
+    """
+    logger.info('Provisioning Service dashboards')
+    folder_name = 'Service POC'
+
+    # hardcode the org for the POC
+    if org_config.get('name') != 'GÉANT Staff':
+        return []
+
+    if is_excluded_folder(org_config, folder_name):
+        # don't provision Services folder
+        delete_folder(token, title=folder_name)
+    else:
+        folder = find_folder(token, folder_name)
+        with ThreadPoolExecutor(max_workers=MAX_WORKERS) as executor:
+            services = fetch_services(config['reporting_provider'])
+            dashes = create_service_panels(services, ds_name)
+            for dashboard in dashes:
+                rendered = render_dashboard(dashboard)
+                yield executor.submit(create_dashboard,
+                                      token,
+                                      rendered, folder['id'])
+
+
 def _provision_gws_indirect(config, org_config, ds_name, token):
     # fetch GWS direct data and provision related dashboards
     logger.info('Provisioning GWS Indirect dashboards')
@@ -507,6 +537,7 @@ def _delete_unknown_folders(config, token):
         # General is a base folder present in Grafana
         'General',
         # other folders, created outside of the DASHBOARDS list
+        'Service POC',
         'GWS Indirect',
         'GWS Direct',
         'Aggregates',
@@ -521,6 +552,7 @@ def _delete_unknown_folders(config, token):
     for folder in all_folders:
         if folder['title'] in folders_to_keep:
             continue
+        logger.info(f'Deleting unknown folder: {folder.get("title")}')
         delete_folder(token, uid=folder['uid'])
 
 
@@ -598,6 +630,8 @@ def provision(config):
         ds_name = datasource.get('name', 'PollerInfluxDB')
 
         managed_dashboards = itertools.chain(
+            _provision_service_dashboards(
+                config, org_config, ds_name, token_request),
             _provision_interfaces(
                 config, org_config, ds_name, token_request),
             _provision_gws_indirect(

brian_dashboard_manager/services/api.py

+import requests
+
+
+def fetch_services(host):
+    """
+    Fetches the current service state from the Reporting Provider host
+    """
+    r = requests.get(f'{host}/scid/current')
+    r.raise_for_status()
+    services = r.json()
+    return services

brian_dashboard_manager/templating/services.py

+from typing import Dict, List
+from brian_dashboard_manager.templating.helpers import get_dashboard_data
+
+
+def get_panel_data(services):
+    result: Dict[str, List[Dict]] = {}
+
+    for service in services:
+
+        customers = service.get('customers')
+        name = service.get('name')
+        sid = service.get('sid')
+        scid = service.get('scid')
+
+        measurement = 'scid_rates'
+        title = f'{name} ({sid})'
+
+        for customer in customers:
+            result.setdefault(customer, []).append({
+                'measurement': measurement,
+                'title': title,
+                'scid': scid,
+                'has_v6': False
+            })
+    return result
+
+
+def create_service_panels(services, datasource):
+
+    panel_data = get_panel_data(services)
+    for dash in get_dashboard_data(
+            data=panel_data,
+            datasource=datasource,
+            tag='SERVICE'):
+        yield dash

brian_dashboard_manager/templating/templates/shared/panel_target.json.j2

@@ -79,6 +79,13 @@
             "operator": "=",
             "value": "{{ subscription }}"
         }
+        {% elif scid %}
+        {
+            "condition": null,
+            "key": "scid",
+            "operator": "=",
+            "value": "{{ scid }}"
+        }
         {% else %}
         {
             "condition": null,

test/conftest.py

@@ -12,6 +12,7 @@ def data_config():
         "admin_password": "fakeadmin",
         "hostname": "myfakehostname.org",
         "inventory_provider": "http://inventory-provider01.geant.org:8080",
+        "reporting_provider": "http://hgf-reporting-provider.geant.org:1337",
         "organizations": [
             {
                 "name": "Testorg1",
@@ -50,6 +51,15 @@ def data_config():
     }
 
 
+def _test_data(filename):
+    data_filename = os.path.join(
+        os.path.dirname(__file__),
+        'data',
+        filename)
+    with open(data_filename) as f:
+        return json.loads(f.read())
+
+
 @pytest.fixture
 def data_config_filename(data_config):
     with tempfile.NamedTemporaryFile() as f:

test/test_services.py

+import responses
+from brian_dashboard_manager.services.api import fetch_services
+from brian_dashboard_manager.templating.services import create_service_panels
+from conftest import _test_data
+
+
+@responses.activate
+def test_services(data_config, client):
+
+    responses.add(
+        method=responses.GET,
+        url=f"{data_config['reporting_provider']}/scid/current",
+        json=_test_data('services.json'))
+
+    services = fetch_services(data_config['reporting_provider'])
+
+    dashboards = list(create_service_panels(services, 'testdatasource'))
+
+    assert dashboards

test/test_update.py

@@ -3,6 +3,7 @@ import json
 
 from brian_dashboard_manager.grafana.provision import provision_folder, \
     provision
+from conftest import _test_data
 
 TEST_INTERFACES = [
     {
@@ -661,6 +662,11 @@ def test_provision_folder(data_config, mocker):
 @responses.activate
 def test_provision(data_config, mocker, client):
 
+    responses.add(
+        method=responses.GET,
+        url=f"{data_config['reporting_provider']}/scid/current",
+        json=_test_data('services.json'))
+
     responses.add(
         method=responses.GET,
         url=f"{data_config['inventory_provider']}/poller/interfaces",