diff --git a/Jenkinsfile b/Jenkinsfile
new file mode 100644
index 0000000000000000000000000000000000000000..bd5177efe9649b374c8215df5aa560d2a6e68f3d
--- /dev/null
+++ b/Jenkinsfile
@@ -0,0 +1,12 @@
+// https://gitlab.geant.net/live-projects/jenkins-pipeline/-/tree/master/vars
+library 'SWDPipeline'
+
+// Parameters:
+// project_name (must match the name of the project in GitLab/SWD release jenkins)
+// extra_recipients (optional, list of email addresses to always receive notifications, in addition to the default in jenkins)
+// python_test_versions (list of python versions, resolving to docker tags, to test against)
+String name = 'brian-dashboard-manager'
+List<String> extraRecipients = ['bjarke@nordu.net', 'erik.reid@geant.org', 'sam.roberts@geant.org', 'pelle.koster@geant.org']
+List<String> pythonTestVersions = ['3.6', '3.11']
+
+SimplePythonBuild(name, extraRecipients, pythonTestVersions)
diff --git a/brian_dashboard_manager/grafana/organization.py b/brian_dashboard_manager/grafana/organization.py
index a6408039a9c761d1c4a372a507ea17cc072ab6e7..3d6f30fac322637005c2e2b10de0a4a1c2b6f696 100644
--- a/brian_dashboard_manager/grafana/organization.py
+++ b/brian_dashboard_manager/grafana/organization.py
@@ -150,7 +150,7 @@ def get_or_create_service_account(request: AdminRequest, org_id):
# get provision service account, if it exists
try:
- service_accounts = request.get('api/serviceaccounts?perpage=10&page=1&query=provision').json()
+ service_accounts = request.get('api/serviceaccounts/search?query=provision').json()
if service_accounts and service_accounts.get('totalCount') > 0:
service_account = service_accounts.get('serviceAccounts')[0]
diff --git a/brian_dashboard_manager/grafana/provision.py b/brian_dashboard_manager/grafana/provision.py
index 69da36bf491509215fec9a4a55a6b3d7c281f86f..431d7c861c105c44b457f5624eb5fba084942dd8 100644
--- a/brian_dashboard_manager/grafana/provision.py
+++ b/brian_dashboard_manager/grafana/provision.py
@@ -390,7 +390,8 @@ def _interfaces_to_keep(interface, excluded_nrens):
return should_keep
-def _provision_interfaces(thread_executor: ThreadPoolExecutor, config, org_config, ds_name, token):
+def _provision_interfaces(thread_executor: ThreadPoolExecutor, config,
+ org_config, ds_name, token, interfaces, services, regions):
"""
This function is used to provision most dashboards,
overwriting existing ones.
@@ -403,10 +404,6 @@ def _provision_interfaces(thread_executor: ThreadPoolExecutor, config, org_confi
:return: generator of dashboards that were created
"""
- interfaces = get_interfaces(config['inventory_provider'])
- services = fetch_services(config['reporting_provider'])
- regions = get_nren_regions(config['inventory_provider'])
-
excluded_nrens = org_config['excluded_nrens']
excluded_folders = org_config.get('excluded_folders', {})
@@ -779,7 +776,7 @@ def _provision_orgs(config):
return all_orgs
-def _provision_org(config, org, org_config):
+def _provision_org(config, org, org_config, interfaces, services, regions):
try:
request = AdminRequest(**config)
org_id = org['id']
@@ -812,7 +809,7 @@ def _provision_org(config, org, org_config):
# call to list is needed to queue up the futures
managed_dashboards = list(itertools.chain(
- _provision_interfaces(*args),
+ _provision_interfaces(*args, interfaces, services, regions),
_provision_gws_indirect(*args),
_provision_gws_direct(*args),
_provision_eumetsat_multicast(*args),
@@ -901,9 +898,11 @@ def provision(config):
return None
orgs = list(filter(lambda t: t[1] is not None, [(org, _find_org_config(org)) for org in all_orgs]))
-
+ interfaces = get_interfaces(config['inventory_provider'])
+ services = fetch_services(config['reporting_provider'])
+ regions = get_nren_regions(config['inventory_provider'])
for org, org_config in orgs:
- _provision_org(config, org, org_config)
+ _provision_org(config, org, org_config, interfaces, services, regions)
logger.info(f'Time to complete: {time.time() - start}')
diff --git a/brian_dashboard_manager/grafana/utils/request.py b/brian_dashboard_manager/grafana/utils/request.py
index c126690c0bd83725df4c8bf3a96d52963fb1fc57..1f573ae24673631a2502c83794d4464765b44438 100644
--- a/brian_dashboard_manager/grafana/utils/request.py
+++ b/brian_dashboard_manager/grafana/utils/request.py
@@ -12,7 +12,7 @@ class Request(requests.Session):
super().__init__()
# allow using up to 16 connections
- adapter = HTTPAdapter(pool_maxsize=MAX_THREADS)
+ adapter = HTTPAdapter(pool_maxsize=MAX_THREADS * 2) # should prevent connection pool exhaustion
self.mount(url, adapter)
self.headers.update({
diff --git a/brian_dashboard_manager/templating/homedashboard.py b/brian_dashboard_manager/templating/homedashboard.py
index 0cffd76da0794692c2115c8373ab7dcd9b027e0d..1b7f8dd934ac1886beb9426fc499e59e1b0332d9 100644
--- a/brian_dashboard_manager/templating/homedashboard.py
+++ b/brian_dashboard_manager/templating/homedashboard.py
@@ -211,8 +211,8 @@ def _render_panels(staff):
{
"condition": "AND",
"key": "interface_name",
- "operator": "=~",
- "value": "/^$interface_name$/",
+ "operator": "=",
+ "value": "$interface_name",
},
],
},
@@ -243,8 +243,8 @@ def _render_panels(staff):
{
"condition": "AND",
"key": "interface_name",
- "operator": "=~",
- "value": "/^$interface_name$/",
+ "operator": "=",
+ "value": "$interface_name",
},
],
},
@@ -273,8 +273,8 @@ def _render_panels(staff):
{
"condition": "AND",
"key": "interface_name",
- "operator": "=~",
- "value": "/^$interface_name$/",
+ "operator": "=",
+ "value": "$interface_name",
},
],
},
@@ -303,8 +303,8 @@ def _render_panels(staff):
{
"condition": "AND",
"key": "interface_name",
- "operator": "=~",
- "value": "/^$interface_name$/",
+ "operator": "=",
+ "value": "$interface_name",
},
],
},
@@ -406,8 +406,8 @@ def _render_panels(staff):
{
"condition": "AND",
"key": "interface_name",
- "operator": "=~",
- "value": "/^$interface_name$/",
+ "operator": "=",
+ "value": "$interface_name",
},
],
},
@@ -438,8 +438,8 @@ def _render_panels(staff):
{
"condition": "AND",
"key": "interface_name",
- "operator": "=~",
- "value": "/^$interface_name$/",
+ "operator": "=",
+ "value": "$interface_name",
},
],
},
@@ -468,8 +468,8 @@ def _render_panels(staff):
{
"condition": "AND",
"key": "interface_name",
- "operator": "=~",
- "value": "/^$interface_name$/",
+ "operator": "=",
+ "value": "$interface_name",
},
],
},
@@ -498,8 +498,8 @@ def _render_panels(staff):
{
"condition": "AND",
"key": "interface_name",
- "operator": "=~",
- "value": "/^$interface_name$/",
+ "operator": "=",
+ "value": "$interface_name",
},
],
},
@@ -601,8 +601,8 @@ def _render_panels(staff):
{
"condition": "AND",
"key": "interface_name",
- "operator": "=~",
- "value": "/^$interface_name$/",
+ "operator": "=",
+ "value": "$interface_name",
},
],
},
@@ -632,8 +632,8 @@ def _render_panels(staff):
{
"condition": "AND",
"key": "interface_name",
- "operator": "=~",
- "value": "/^$interface_name$/",
+ "operator": "=",
+ "value": "$interface_name",
},
],
},
@@ -664,8 +664,8 @@ def _render_panels(staff):
{
"condition": "AND",
"key": "interface_name",
- "operator": "=~",
- "value": "/^$interface_name$/",
+ "operator": "=",
+ "value": "$interface_name",
},
],
},
@@ -697,8 +697,8 @@ def _render_panels(staff):
{
"condition": "AND",
"key": "interface_name",
- "operator": "=~",
- "value": "/^$interface_name$/",
+ "operator": "=",
+ "value": "$interface_name",
},
],
},
diff --git a/brian_dashboard_manager/templating/render.py b/brian_dashboard_manager/templating/render.py
index b706a24acdf3b64f4cf8436ed6286f4ec248916b..afb65e305a00404ce9cd3194bf316a6bef773062 100644
--- a/brian_dashboard_manager/templating/render.py
+++ b/brian_dashboard_manager/templating/render.py
@@ -1,3 +1,126 @@
+BASE_DROPDOWN_PANEL = {
+ "aliasColors": {},
+ "collapsed": False,
+ "datasource": None,
+ "fill": None,
+ "fillGradient": None,
+ "legend": None,
+ "lines": None,
+ "linewidth": None,
+ "search": None,
+ "stack": None,
+ "tags": None,
+ "targets": None,
+ "type": "row",
+ "xaxis": None,
+ "yaxes": None,
+ "yaxis": None,
+}
+
+BASE_YAXES = {
+ "logBase": 1,
+ "max": None,
+ "min": 0,
+ "show": True,
+}
+
+BASE_PANEL = {
+ "bars": False,
+ "collapsed": None,
+ "dashLength": 10,
+ "dashes": False,
+ "decimals": 2,
+ "fieldConfig": {"defaults": {"custom": {}}, "overrides": []},
+ "fill": 1,
+ "fillGradient": 10,
+ "hiddenSeries": False,
+ "lines": True,
+ "nullPointMode": "null",
+ "options": {"alertThreshold": True},
+ "percentage": False,
+ "pointradius": 2,
+ "points": False,
+ "renderer": "flot",
+ "search": None,
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "steppedLine": False,
+ "tags": None,
+ "thresholds": [],
+ "timeFrom": None,
+ "timeRegions": [],
+ "timeShift": None,
+ "tooltip": {"shared": True, "sort": 0, "value_type": "individual"},
+ "type": "graph",
+ "xaxis": {
+ "buckets": None,
+ "mode": "time",
+ "name": None,
+ "show": True,
+ "values": None,
+ },
+ "yaxis": {"align": False, "alignLevel": None},
+ "targets": [],
+}
+
+BASE_LEGEND = {
+ "alignAsTable": True,
+ "avg": True,
+ "current": True,
+ "max": True,
+ "min": False,
+ "rightSide": None,
+ "show": True,
+ "total": False,
+ "values": True,
+}
+
+BASE_DASHBOARD = {
+ "id": None,
+ "uid": None,
+ "annotations": {
+ "list": [
+ {
+ "builtIn": 1,
+ "datasource": "-- Grafana --",
+ "enable": True,
+ "hide": True,
+ "iconColor": "rgba(0, 211, 255, 1)",
+ "name": "Annotations & Alerts",
+ "type": "dashboard",
+ }
+ ]
+ },
+ "editable": False,
+ "gnetId": None,
+ "graphTooltip": 0,
+ "schemaVersion": 27,
+ "style": "dark",
+ "templating": {"list": []},
+ "time": {"from": "now-24h", "to": "now"},
+ "timepicker": {},
+ "timezone": "",
+ "version": 1,
+ "links": [],
+}
+
+INFOBOX = {
+ "datasource": None,
+ "gridPos": {"h": 2, "w": 24, "x": 0, "y": 0},
+ "id": 1,
+ "options": {
+ "content": """
+ <center style="margin-top:5px;">
+ INFO: The average values displayed are only mean values for timescales of 2 days or less
+ </center>""",
+ "mode": "html"
+ },
+ "pluginVersion": "8.2.5",
+ "title": "",
+ "type": "text",
+}
+
+
def create_dropdown_panel(title, id, y, **kwargs):
"""
Creates a dropdown panel from the given data.
@@ -10,25 +133,10 @@ def create_dropdown_panel(title, id, y, **kwargs):
:return: rendered dropdown panel JSON
"""
return {
- "aliasColors": {},
- "collapsed": False,
- "datasource": None,
- "fill": None,
- "fillGradient": None,
- "gridPos": {"h": 1, "w": 24, "x": 0, "y": y},
+ **BASE_DROPDOWN_PANEL,
"id": id,
- "legend": None,
- "lines": None,
- "linewidth": None,
- "search": None,
- "stack": None,
- "tags": None,
- "targets": None,
+ "gridPos": {"h": 1, "w": 24, "x": 0, "y": y},
"title": title,
- "type": "row",
- "xaxis": None,
- "yaxes": None,
- "yaxis": None,
}
@@ -43,41 +151,28 @@ def create_yaxes(type):
if type == "errors":
return [
{
+ **BASE_YAXES,
"format": "none",
"label": "errors and discards per second",
- "logBase": 1,
- "max": None,
- "min": 0,
- "show": True,
},
{
+ **BASE_YAXES,
"format": "none",
"label": "errors and discards per second",
- "logBase": 1,
- "max": None,
- "min": 0,
- "show": True,
- },
- ]
- else:
- return [
- {
- "format": "bps",
- "label": "bits per second",
- "logBase": 1,
- "max": None,
- "min": "",
- "show": True,
- },
- {
- "format": "bps",
- "label": "bits per second",
- "logBase": 1,
- "max": None,
- "min": "",
- "show": True,
},
]
+ return [
+ {
+ **BASE_YAXES,
+ "format": "bps",
+ "label": "bits per second",
+ },
+ {
+ **BASE_YAXES,
+ "format": "bps",
+ "label": "bits per second",
+ },
+ ]
def create_panel_target(
@@ -197,64 +292,19 @@ def create_panel(
interval = 5 if not is_lambda else 15
result = {
+ **BASE_PANEL,
"aliasColors": alias_colors or {},
- "bars": False,
- "collapsed": None,
- "dashLength": 10,
- "dashes": False,
"datasource": datasource,
- "decimals": 2,
- "fieldConfig": {"defaults": {"custom": {}}, "overrides": []},
- "fill": 1,
- "fillGradient": 10,
"gridPos": {"h": height, "w": width, "x": x, "y": y},
- "hiddenSeries": False,
"id": id,
"interval": f"{interval}m",
- "lines": True,
"linewidth": linewidth,
- "nullPointMode": "null",
- "options": {"alertThreshold": True},
- "percentage": False,
- "pointradius": 2,
- "points": False,
- "renderer": "flot",
- "search": None,
- "seriesOverrides": [],
- "spaceLength": 10,
"stack": stack,
- "steppedLine": False,
- "tags": None,
- "thresholds": [],
- "timeFrom": None,
- "timeRegions": [],
- "timeShift": None,
"title": title,
- "tooltip": {"shared": True, "sort": 0, "value_type": "individual"},
- "type": "graph",
- "xaxis": {
- "buckets": None,
- "mode": "time",
- "name": None,
- "show": True,
- "values": None,
- },
"yaxes": yaxes,
- "yaxis": {"align": False, "alignLevel": None},
- "targets": [],
}
if not disable_legend:
- result["legend"] = {
- "alignAsTable": True,
- "avg": True,
- "current": True,
- "max": True,
- "min": False,
- "rightSide": None,
- "show": True,
- "total": False,
- "values": True,
- }
+ result["legend"] = BASE_LEGEND
targets = targets or []
for target in panel_targets or []:
@@ -263,63 +313,19 @@ def create_panel(
return result
-def create_infobox():
- return {
- "datasource": None,
- "gridPos": {"h": 2, "w": 24, "x": 0, "y": 0},
- "id": 1,
- "options": {
- "content": """
- <center style="margin-top:5px;">
- INFO: The average values displayed are only mean values for timescales of 2 days or less
- </center>""",
- "mode": "html"
- },
- "pluginVersion": "8.2.5",
- "title": "",
- "type": "text",
- }
-
-
def render_with_aggregate_dashboard(
nren_name, aggregate_panels, dropdown_groups, tag=None, tags=None, **_
):
assert tag or tags
- panels = [create_infobox()]
- panels.extend(aggregate_panels)
+ panels = [INFOBOX, *aggregate_panels]
for group in dropdown_groups:
panels.append(group["dropdown"])
panels.extend(group["panels"])
return {
- "id": None,
- "uid": None,
- "annotations": {
- "list": [
- {
- "builtIn": 1,
- "datasource": "-- Grafana --",
- "enable": True,
- "hide": True,
- "iconColor": "rgba(0, 211, 255, 1)",
- "name": "Annotations & Alerts",
- "type": "dashboard",
- }
- ]
- },
- "editable": False,
- "gnetId": None,
- "graphTooltip": 0,
- "schemaVersion": 27,
- "style": "dark",
+ **BASE_DASHBOARD,
"tags": tags or [tag],
- "templating": {"list": []},
- "time": {"from": "now-24h", "to": "now"},
- "timepicker": {},
- "timezone": "",
"title": nren_name,
- "version": 1,
- "links": [],
"panels": panels,
}
@@ -327,36 +333,11 @@ def render_with_aggregate_dashboard(
def render_simple_dashboard(title, tag=None, tags=None, panels=None, **_):
assert tag or tags
return {
- "id": None,
- "uid": None,
- "annotations": {
- "list": [
- {
- "builtIn": 1,
- "datasource": "-- Grafana --",
- "enable": True,
- "hide": True,
- "iconColor": "rgba(0, 211, 255, 1)",
- "name": "Annotations & Alerts",
- "type": "dashboard",
- }
- ]
- },
- "editable": False,
- "gnetId": None,
- "graphTooltip": 0,
- "schemaVersion": 27,
- "style": "dark",
+ **BASE_DASHBOARD,
"tags": tags or [tag],
- "templating": {"list": []},
- "time": {"from": "now-24h", "to": "now"},
- "timepicker": {},
- "timezone": "",
"title": title,
- "version": 1,
- "links": [],
"panels": [
- create_infobox(),
+ INFOBOX,
*(panels or []),
],
}
diff --git a/changelog.md b/changelog.md
index 7c46ac02a9ce99371366db09bd81f21a401bfbdd..a41a06370af488a77e66e1ad0b1a99c8d9d96d26 100644
--- a/changelog.md
+++ b/changelog.md
@@ -2,6 +2,11 @@
All notable changes to this project will be documented in this file.
+## [0.75] - 2024-11-13
+- POL1-857: Fix issue with Grafana not properly escaping certain values for the home dashboard.
+- Fix connection pool thread exhaustion.
+- Reuse API call data, and fix service accounts API call
+
## [0.74] - 2024-11-05
- Ensure futures are all started at the same time, enabling proper multithreading
- Synchronize thread count and urllib pool size to avoid dropped connections
diff --git a/setup.py b/setup.py
index 98195d89f54f558ebaddf65595a6fc3e2d02327b..df7c84e74c3eab43f697fe351787ea0a953d2ff1 100644
--- a/setup.py
+++ b/setup.py
@@ -2,7 +2,7 @@ from setuptools import setup, find_packages
setup(
name='brian-dashboard-manager',
- version="0.74",
+ version="0.75",
author='GEANT',
author_email='swd@geant.org',
description='',