From 1466c212b3917cfa7c0af719edfcdc1b8099ffbc Mon Sep 17 00:00:00 2001
From: Bjarke Madsen <bjarke.madsen@geant.org>
Date: Wed, 27 Jan 2021 17:02:05 +0100
Subject: [PATCH] Only be dependent on grafana hostname Use datasource-specific
config instead of environment config Add influxdb datasource config Remove
unused config keys
---
brian_dashboard_manager/config.py | 46 +++++++++++++++----
brian_dashboard_manager/grafana/dashboard.py | 14 +++++-
brian_dashboard_manager/grafana/datasource.py | 14 ++++--
brian_dashboard_manager/grafana/provision.py | 12 +++--
.../grafana/utils/request.py | 8 ++--
test/conftest.py | 21 ++++++---
test/test_grafana_datasource.py | 13 +++---
test/test_grafana_request.py | 4 +-
8 files changed, 95 insertions(+), 37 deletions(-)
diff --git a/brian_dashboard_manager/config.py b/brian_dashboard_manager/config.py
index c9c3abd..ef05a1a 100644
--- a/brian_dashboard_manager/config.py
+++ b/brian_dashboard_manager/config.py
@@ -4,24 +4,55 @@ import jsonschema
CONFIG_SCHEMA = {
"$schema": "http://json-schema.org/draft-07/schema#",
+ "definitions": {
+ "influx-datasource": {
+ "type": "object",
+ "properties": {
+ "name": {"type": "string"},
+ "username": {"type": "string"},
+ "password": {"type": "string"},
+ "type": {"type": "string"},
+ "url": {"type": "string"},
+ "database": {"type": "string"},
+ "basicAuth": {"type": "boolean"},
+ "access": {"type": "string"},
+ "isDefault": {"type": "boolean"},
+ "readOnly": {"type": "boolean"}
+ },
+ "required": [
+ "name",
+ "type",
+ "url",
+ "database",
+ "basicAuth",
+ "access",
+ "isDefault",
+ "readOnly"
+ ]
+ }
+ },
+
"type": "object",
"properties": {
"admin_username": {"type": "string"},
"admin_password": {"type": "string"},
"hostname": {"type": "string"},
"listen_port": {"type": "integer"},
- "grafana_port": {"type": "integer"},
"organizations": {"type": "array", "items": {"type": "string"}},
- "BRIAN_ENVIRONMENT": {"type": "string"}
+ "datasources": {
+ "type": "object",
+ "properties": {
+ "influxdb": {"$ref": "#definitions/influx-datasource"}
+ },
+ "additionalProperties": False
+ },
},
"required": [
"admin_username",
"admin_password",
"hostname",
- "listen_port",
- "grafana_port",
"organizations",
- "BRIAN_ENVIRONMENT"
+ "datasources"
],
"additionalProperties": False
}
@@ -31,11 +62,10 @@ def defaults():
return {
"admin_username": "admin",
"admin_password": "admin",
- "hostname": "localhost",
+ "hostname": "localhost:3000",
"listen_port": 3001,
- "grafana_port": 3000,
"organizations": ["Main Org."],
- "BRIAN_ENVIRONMENT": "test"
+ "datasources": {}
}
diff --git a/brian_dashboard_manager/grafana/dashboard.py b/brian_dashboard_manager/grafana/dashboard.py
index ba107d5..e44502c 100644
--- a/brian_dashboard_manager/grafana/dashboard.py
+++ b/brian_dashboard_manager/grafana/dashboard.py
@@ -74,10 +74,21 @@ def _get_dashboard(request: TokenRequest, uid: int):
# supplied dashboards are JSON blobs exported from GUI with a UID.
def create_dashboard(request: TokenRequest, dashboard: Dict):
+ title = dashboard['title']
existing_dashboard = None
- if dashboard.get('uid') is not None:
+ has_uid = dashboard.get('uid') is not None
+ if has_uid:
existing_dashboard = _get_dashboard(request, uid=dashboard['uid'])
+
+ # The title might not match the one that's provisioned with that UID.
+ # Try to find it by searching for the title instead.
+ if existing_dashboard is not None:
+ grafana_title = existing_dashboard['dashboard']['title']
+ different = grafana_title != title
else:
+ different = False
+
+ if existing_dashboard is None or different:
existing_dashboard = _search_dashboard(request, dashboard)
if existing_dashboard:
@@ -92,7 +103,6 @@ def create_dashboard(request: TokenRequest, dashboard: Dict):
'dashboard': dashboard,
'overwrite': False
}
- title = dashboard["title"]
try:
action = "Updating" if existing_dashboard else "Provisioning"
logger.info(f'{action} dashboard: {title}')
diff --git a/brian_dashboard_manager/grafana/datasource.py b/brian_dashboard_manager/grafana/datasource.py
index 4313fbd..50ab365 100644
--- a/brian_dashboard_manager/grafana/datasource.py
+++ b/brian_dashboard_manager/grafana/datasource.py
@@ -1,5 +1,4 @@
import logging
-import re
import os
import json
from typing import Dict
@@ -44,10 +43,17 @@ def get_datasources(request: Request):
return request.get('api/datasources')
-def create_datasource(request: TokenRequest, datasource: Dict, environment):
+def create_datasource(request: TokenRequest, datasource: Dict, datasources):
try:
- datasource["url"] = re.sub(
- 'test|uat|prod', environment, datasource["url"])
+ ds_type = datasource["type"]
+ # find out which params
+ # we need to configure for this datasource type
+ config = datasources.get(ds_type, None)
+ if config is None:
+ logger.exception(
+ f'No datasource config could be found for {ds_type}')
+ return None
+ datasource.update(config)
r = request.post('api/datasources', json=datasource)
except HTTPError:
logger.exception('Error when provisioning datasource')
diff --git a/brian_dashboard_manager/grafana/provision.py b/brian_dashboard_manager/grafana/provision.py
index f9aea13..b258101 100644
--- a/brian_dashboard_manager/grafana/provision.py
+++ b/brian_dashboard_manager/grafana/provision.py
@@ -43,11 +43,13 @@ def provision(config):
# Provision missing data sources
for datasource in get_missing_datasource_definitions(token_request):
- ds = create_datasource(token_request,
- datasource,
- config.get('BRIAN_ENVIRONMENT', 'test'))
- if ds:
- logger.info(f'Provisioned datasource: {datasource["name"]}')
+ if datasource:
+ ds = create_datasource(token_request,
+ datasource,
+ config.get('datasources'))
+ if ds:
+ logger.info(
+ f'Provisioned datasource: {datasource["name"]}')
# Provision dashboards, overwriting existing ones.
for dashboard in get_dashboard_definitions():
diff --git a/brian_dashboard_manager/grafana/utils/request.py b/brian_dashboard_manager/grafana/utils/request.py
index cbb8556..c5f6987 100644
--- a/brian_dashboard_manager/grafana/utils/request.py
+++ b/brian_dashboard_manager/grafana/utils/request.py
@@ -53,10 +53,10 @@ class Request(object):
class AdminRequest(Request):
- def __init__(self, hostname, grafana_port, admin_username, admin_password,
+ def __init__(self, hostname, admin_username, admin_password,
**kwargs):
self.username = admin_username
- url = f'{admin_username}:{admin_password}@{hostname}:{grafana_port}/'
+ url = f'{admin_username}:{admin_password}@{hostname}/'
super().__init__('http://' + url)
def __str__(self):
@@ -64,10 +64,10 @@ class AdminRequest(Request):
class TokenRequest(Request):
- def __init__(self, hostname, grafana_port, token: str, **kwargs):
+ def __init__(self, hostname, token: str, **kwargs):
self.token = token
- super().__init__(f'http://{hostname}:{grafana_port}/', {
+ super().__init__(f'http://{hostname}/', {
'Authorization': 'Bearer ' + token
})
diff --git a/test/conftest.py b/test/conftest.py
index d1cf7ae..39e0477 100644
--- a/test/conftest.py
+++ b/test/conftest.py
@@ -8,18 +8,27 @@ import brian_dashboard_manager
@pytest.fixture
def data_config():
return {
- "admin_username": "admin",
- "admin_password": "admin",
- "hostname": "localhost",
- "listen_port": 65001,
- "grafana_port": 65005,
+ "admin_username": "fakeadmin",
+ "admin_password": "fakeadmin",
+ "hostname": "myfakehostname.org",
"organizations": [
'Testorg1',
'GÉANT Testorg2',
'NRENsTestorg3',
'General Public'
],
- "BRIAN_ENVIRONMENT": "test"
+ "datasources": {
+ "influxdb": {
+ "name": "PollerInfluxDB",
+ "type": "influxdb",
+ "access": "proxy",
+ "url": "http://prod-poller-ui01.geant.org:8086",
+ "database": "poller",
+ "basicAuth": False,
+ "isDefault": True,
+ "readOnly": False
+ }
+ }
}
diff --git a/test/test_grafana_datasource.py b/test/test_grafana_datasource.py
index 59c2606..af30deb 100644
--- a/test/test_grafana_datasource.py
+++ b/test/test_grafana_datasource.py
@@ -79,9 +79,6 @@ def test_create_prod_datasource(data_config):
def post_callback(request):
body = json.loads(request.body)
- # we are testing provisioning logic to prod, so the URL needs test ->
- # prod translation.
- assert body['url'] == 'http://prod-brian-datasource.geant.org:8086'
result = {
'datasource': {
'id': 1,
@@ -109,9 +106,12 @@ def test_create_prod_datasource(data_config):
url=request.BASE_URL + 'api/datasources',
callback=post_callback)
- data = provision.create_datasource(request, BODY, environment='prod')
+ data = provision.create_datasource(
+ request, BODY, datasources=data_config['datasources'])
- assert data is not None
+ datasource_type = data['datasource']['type']
+ datasource_config_url = data_config['datasources'][datasource_type]['url']
+ assert data['datasource']['url'] == datasource_config_url
@responses.activate
@@ -134,7 +134,8 @@ def test_create_prod_datasource_fails(data_config):
url=request.BASE_URL + 'api/datasources',
callback=lambda f: (400, {}, ''))
- data = provision.create_datasource(request, BODY, environment='prod')
+ data = provision.create_datasource(
+ request, BODY, datasources=data_config['datasources'])
# if an error occured when provisioning a datasource, we log the response
# but return None
diff --git a/test/test_grafana_request.py b/test/test_grafana_request.py
index c4f1139..d90b191 100644
--- a/test/test_grafana_request.py
+++ b/test/test_grafana_request.py
@@ -11,7 +11,7 @@ from brian_dashboard_manager.grafana.utils.request import \
def test_admin_request(data_config):
ENDPOINT = 'test/url/endpoint'
request = AdminRequest(**data_config)
- url = '{admin_username}:{admin_password}@{hostname}:{grafana_port}/'. \
+ url = '{admin_username}:{admin_password}@{hostname}/'. \
format(**data_config)
assert request.BASE_URL == 'http://' + url
assert request.username == data_config['admin_username']
@@ -33,7 +33,7 @@ def test_token_request(data_config):
TOKEN = '123'
ENDPOINT = 'test/url/endpoint'
request = TokenRequest(**data_config, token=TOKEN)
- assert request.BASE_URL == 'http://{hostname}:{grafana_port}/'.format(
+ assert request.BASE_URL == 'http://{hostname}/'.format(
**data_config)
assert request.token == TOKEN
--
GitLab