Add initial code for handling organizations.

(de)provisioning dashboards WIP.
Datasources still not implemented.
No tests implemented.

.gitignore

+# IDE related
+.idea
+.vscode
+
+# config
+config.json
+
+# dev / builds
+venv/
+*.egg-info
+__pycache__
+coverage.xml
+.coverage
+htmlcov
+.tox
+dist
+dashboards/dev/*.json
+
+
+# logs
+*.log

MANIFEST.in

+include api/logging_default_config.json
+include dashboards/*
\ No newline at end of file

README.md

+# Skeleton Web App
+
+## Overview
+
+This module implements a skeleton Flask-based webservice.
+
+The webservice is communicates with clients over HTTP.
+Responses to valid requests are returned as JSON messages.
+The server will therefore return an error unless
+`application/json` is in the `Accept` request header field.
+
+HTTP communication and JSON grammar details are
+beyond the scope of this document.
+Please refer to [RFC 2616](https://tools.ietf.org/html/rfc2616)
+and www.json.org for more details.
+
+
+## Configuration
+
+This app allows specification of a few
+example configuration parameters.  These
+parameters should stored in a file formatted
+similarly to `config.json.example`, and the name
+of this file should be stored in the environment
+variable `SETTINGS_FILENAME` when running the service.
+
+## Running this module
+
+This module has been tested in the following execution environments:
+
+- As an embedded Flask application.
+For example, the application could be launched as follows:
+
+```bash
+$ export FLASK_APP=app.py
+$ export SETTINGS_FILENAME=config.json
+$ flask run
+```
+
+- As an Apache/`mod_wsgi` service.
+  - Details of Apache and `mod_wsgi`
+    configuration are beyond the scope of this document.
+
+- As a `gunicorn` wsgi service.
+  - Details of `gunicorn` configuration are
+    beyond the scope of this document.
+
+
+## Protocol Specification
+
+The following resources can be requested from the webservice.
+
+### resources
+
+Any non-empty responses are JSON formatted messages.
+
+#### /data/version
+
+  * /version
+
+  The response will be an object
+  containing the module and protocol versions of the
+  running server and will be formatted as follows:
+
+  ```json
+  {
+        "$schema": "http://json-schema.org/draft-07/schema#",
+        "type": "object",
+        "properties": {
+            "api": {
+                "type": "string",
+                "pattern": r'\d+\.\d+'
+            },
+            "module": {
+                "type": "string",
+                "pattern": r'\d+\.\d+'
+            }
+        },
+        "required": ["api", "module"],
+        "additionalProperties": False
+    }
+  ```
+
+#### /test/test1
+
+The response will be some json data, as an example ...

brian_dashboard_manager/__init__.py

+"""
+automatically invoked app factory
+"""
+import logging
+import os
+
+from flask import Flask
+
+from brian_dashboard_manager import environment
+from brian_dashboard_manager import config
+
+CONFIG_KEY = 'CONFIG_PARAMS'
+
+
+def create_app():
+    """
+    overrides default settings with those found
+    in the file read from env var SETTINGS_FILENAME
+
+    :return: a new flask app instance
+    """
+
+    
+    app_config = config.defaults()
+    if 'SETTINGS_FILENAME' in os.environ:
+        with open(os.environ['SETTINGS_FILENAME']) as f:
+            app_config.update(config.load(f))
+
+    app = Flask(__name__)
+    app.secret_key = os.environ.get('SECRET_KEY') or 'super secret session key'
+    app.config[CONFIG_KEY] = app_config
+
+    from brian_dashboard_manager.routes import update
+    app.register_blueprint(update.routes, url_prefix='/update')
+
+    from brian_dashboard_manager.routes import test
+    app.register_blueprint(test.routes, url_prefix='/test')
+
+    logging.info('Flask app initialized')
+    environment.setup_logging()
+    return app

brian_dashboard_manager/app.py

+"""
+default app creation
+"""
+import brian_dashboard_manager
+from brian_dashboard_manager import environment, CONFIG_KEY
+
+environment.setup_logging()
+app = brian_dashboard_manager.create_app()
+
+if __name__ == "__main__":
+    app.run(host="::", port=f"{app.config[CONFIG_KEY]['listen_port']}")

brian_dashboard_manager/config.py

+import json
+import jsonschema
+
+CONFIG_SCHEMA = {
+    "$schema": "http://json-schema.org/draft-07/schema#",
+
+    "type": "object",
+    "properties": {
+        "admin_username": {"type": "string"},
+        "admin_password": {"type": "string"},
+        "hostname": {"type": "string"},
+        "listen_port": {"type": "integer"},
+        "grafana_port": {"type": "integer"},
+        "organizations": {"type": "array", "items": {"type": "string"}},
+        "dashboard_directory": {"type": "string"},
+    },
+    "required": ["admin_username", "admin_password", "hostname", "listen_port", "grafana_port", "organizations"],
+    "additionalProperties": False
+}
+
+
+def defaults():
+    return {
+        "admin_username": "admin",
+        "admin_password": "admin",
+        "hostname": "localhost",
+        "listen_port": 3001,
+        "grafana_port": 3000,
+        "organizations": ["Main Org."]
+    }
+
+
+def load(f):
+    """
+    loads, validates and returns configuration parameters
+
+    :param f: file-like object that produces the config file
+    :return:
+    """
+    config = json.loads(f.read())
+    jsonschema.validate(config, CONFIG_SCHEMA)
+    return config

brian_dashboard_manager/environment.py

+import json
+import logging.config
+import os
+
+
+def setup_logging():
+    """
+    set up logging using the configured filename
+
+    if LOGGING_CONFIG is defined in the environment, use this for
+    the filename, otherwise use logging_default_config.json
+    """
+    default_filename = os.path.join(
+        os.path.dirname(__file__), 'logging_default_config.json')
+    filename = os.getenv('LOGGING_CONFIG', default_filename)
+    with open(filename) as f:
+        # TODO: this mac workaround should be removed ...
+        d = json.loads(f.read())
+        import platform
+        if platform.system() == 'Darwin':
+            d['handlers']['syslog_handler']['address'] = '/var/run/syslog'
+        logging.config.dictConfig(d)
+        # logging.config.dictConfig(json.loads(f.read()))

brian_dashboard_manager/grafana/__init__.py

+from brian_dashboard_manager.grafana.provision import provision
+
+__all__ = ["provision"]
\ No newline at end of file

brian_dashboard_manager/grafana/dashboard.py

+import logging
+from typing import Dict
+from brian_dashboard_manager.grafana.utils.request import TokenRequest
+
+logger = logging.getLogger(__name__)
+
+def provision_dashboard(request: TokenRequest, dashboard: Dict):
+
+    del dashboard['uid']
+    del dashboard['id']
+    del dashboard['version']
+
+    payload = {
+        'dashboard': dashboard,
+        'overwrite': False
+    }
+    try:
+        r = request.post('api/dashboards/db', json=payload)
+    except Exception as e:
+        logger.error(f'Error when provisioning dashboard: ' + e.response.text)
+    return True

brian_dashboard_manager/grafana/organization.py

+import random
+import string
+import logging
+from typing import Dict, List, Union
+from datetime import datetime
+
+from brian_dashboard_manager.grafana.utils.request import Request, AdminRequest
+
+logger = logging.getLogger(__name__)
+
+
+def switch_active_organization(request: Request, org_id: int):
+    assert org_id
+
+    logger.debug(f'Switched {str(request)} active organization to #{org_id}')
+    return request.post(f'api/user/using/{org_id}', {})
+
+
+def get_organizations(request: AdminRequest) -> List:
+    return request.get('api/orgs')
+
+
+def create_organization(request: AdminRequest, name: str) -> Union[Dict, None]:
+    assert name
+
+    result = request.post('api/orgs', json={
+        'name': name
+    })
+
+    if result.get('message', '').lower() == 'organization created':
+        id = result.get('orgId')
+        logger.info(f'Created organization `{name}` with ID #{id}')
+        return {'id': id, 'name': name}
+    else:
+        return None
+
+
+def delete_organization(request: AdminRequest, id: int) -> bool:
+
+    result = request.delete(f'api/orgs/{id}')
+
+    return result.get('message', '').lower() == 'organization deleted'
+
+
+def create_api_token(request: AdminRequest, org_id: int, key_data=None):
+    data = {
+        'name': ''.join(random.choices(string.ascii_uppercase + string.digits, k=16)),
+        'role': 'Admin',
+        'secondsToLive': 3600  # 60 minutes
+    }
+    if key_data:
+        data.update(key_data)
+
+    switch_active_organization(request, org_id)
+    result = request.post(f'api/auth/keys', json=data)
+    token_id = result.get('id')
+
+    logger.debug(f'Created API token #{token_id} for organization #{org_id}')
+
+    return result
+
+
+def delete_api_token(request: AdminRequest, org_id: int, token_id: int):
+    assert token_id
+
+    switch_active_organization(request, org_id)
+    result = request.delete(f'api/auth/keys/{token_id}')
+    logger.debug(f'Deleted API token #{token_id} for organization #{org_id}')
+    return result
+
+
+def delete_expired_api_tokens(request: AdminRequest, org_id: int) -> bool:
+    assert org_id
+
+    tokens = request.get('api/auth/keys', params={'includeExpired': True})
+
+    now = datetime.now()
+    expired_tokens = [t for t in tokens if 'expiration' in t
+                      and datetime.strptime(t['expiration'], '%Y-%m-%dT%H:%M:%SZ') < now]
+
+    for token in expired_tokens:
+        delete_api_token(request, org_id, token['id'])
+    return True

brian_dashboard_manager/grafana/provision.py

+import logging
+import os
+import json
+from functools import reduce
+from typing import List
+from brian_dashboard_manager.grafana.utils.request import AdminRequest, TokenRequest
+from brian_dashboard_manager.grafana.organization import get_organizations, create_organization, delete_organization, create_api_token, delete_api_token, delete_expired_api_tokens
+from brian_dashboard_manager.grafana.dashboard import provision_dashboard
+
+
+logger = logging.getLogger(__name__)
+
+
+def provision(config):
+
+    hostname = config.get('hostname')
+    port = config.get('grafana_port')
+    username = config.get('admin_username')
+    password = config.get('admin_password')
+
+    request = AdminRequest(hostname, port, username, password)
+    all_orgs = get_organizations(request)
+
+    organizations_to_provision = config.get('organizations', [])
+
+    missing = [name for name in organizations_to_provision
+               if name not in [org['name'] for org in all_orgs]]
+
+    for org in missing:
+        org_data = create_organization(request, org)
+        all_orgs.append(org_data)
+
+    for org in all_orgs:
+        org_id = org['id']
+        delete_expired_api_tokens(request, org_id)
+        token = create_api_token(request, org_id)
+        token_request = TokenRequest(hostname, port, token['key'])
+        # TODO: (de)provision datasources and remove all existing dashboards in an organization before attempting to provision new dashboards.
+
+        for (dirpath, dirnames, filenames) in os.walk(config.get('dashboard_directory', 'dashboards')):
+            for file in filenames:
+                if file.endswith('.json'):
+                    filename = os.path.join(dirpath, file)
+                    logger.info(f'Provisioning dashboard: {file.strip(".json")}')
+                    provision_dashboard(token_request, json.load(open(filename, 'r')))
+
+        delete_api_token(request, org_id, token['id'])
+
+    return all_orgs

brian_dashboard_manager/grafana/utils/request.py

+import requests
+from typing import Dict, Any
+from flask import current_app
+
+
+class Request(object):
+    def __init__(self, url, headers=None):
+        self.headers = {
+            'Accept': 'application/json'
+        }
+        if headers:
+            self.headers.update(headers)
+
+        self.BASE_URL = url
+
+    def get(self, endpoint: str, headers=None, **kwargs):
+
+        r = requests.get(
+            self.BASE_URL + endpoint,
+            headers={**headers, **self.headers} if headers else self.headers,
+            **kwargs
+        )
+        r.raise_for_status()
+        return r.json()
+
+    def post(self, endpoint: str, headers=None, **kwargs):
+
+        r = requests.post(
+            self.BASE_URL + endpoint,
+            headers={**headers, **self.headers} if headers else self.headers,
+            **kwargs
+        )
+        r.raise_for_status()
+        return r.json()
+
+    def delete(self, endpoint: str, headers=None, **kwargs):
+
+        r = requests.delete(
+            self.BASE_URL + endpoint,
+            headers={**headers, **self.headers} if headers else self.headers,
+            **kwargs
+        )
+        r.raise_for_status()
+        return r.json()
+
+
+class AdminRequest(Request):
+    def __init__(self, hostname, port, username, password):
+        self.username = username
+        super().__init__(f'http://{username}:{password}@{hostname}:{port}/')
+
+    def __str__(self):
+        return f'admin user: {self.username}'
+
+
+class TokenRequest(Request):
+    def __init__(self, hostname, port, token: str):
+        self.token = token
+
+        super().__init__(f'http://{hostname}:{port}/', {
+            'Authorization': 'Bearer ' + token
+        })
+
+    def __str__(self):
+        return f'token {self.token}'

brian_dashboard_manager/logging_default_config.json

+{
+    "version": 1,
+    "disable_existing_loggers": false,
+    "formatters": {
+        "simple": {
+            "format": "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
+        }
+    },
+
+    "handlers": {
+        "console": {
+            "class": "logging.StreamHandler",
+            "level": "DEBUG",
+            "formatter": "simple",
+            "stream": "ext://sys.stdout"
+        },
+
+        "syslog_handler": {
+            "class": "logging.handlers.SysLogHandler",
+            "level": "DEBUG",
+            "address": "/dev/log",
+            "facility": "user",
+            "formatter": "simple"
+        },
+
+        "info_file_handler": {
+            "class": "logging.handlers.RotatingFileHandler",
+            "level": "INFO",
+            "formatter": "simple",
+            "filename": "info.log",
+            "maxBytes": 10485760,
+            "backupCount": 20,
+            "encoding": "utf8"
+        },
+
+        "error_file_handler": {
+            "class": "logging.handlers.RotatingFileHandler",
+            "level": "ERROR",
+            "formatter": "simple",
+            "filename": "errors.log",
+            "maxBytes": 10485760,
+            "backupCount": 20,
+            "encoding": "utf8"
+        }
+    },
+
+    "loggers": {
+        "api": {
+            "level": "DEBUG",
+            "handlers": ["console", "syslog_handler"],
+            "propagate": false
+        }
+    },
+
+    "root": {
+        "level": "INFO",
+        "handlers": ["console", "syslog_handler"]
+    }
+}
\ No newline at end of file

brian_dashboard_manager/routes/common.py

+"""
+Utilities used by multiple route blueprints.
+"""
+import functools
+import logging
+from flask import request, Response
+
+logger = logging.getLogger(__name__)
+
+def require_accepts_json(f):
+    """
+    used as a route handler decorator to return an error
+    unless the request allows responses with type "application/json"
+    :param f: the function to be decorated
+    :return: the decorated function
+    """
+    @functools.wraps(f)
+    def decorated_function(*args, **kwargs):
+        # TODO: use best_match to disallow */* ...?
+        if not request.accept_mimetypes.accept_json:
+            return Response(
+                response="response will be json",
+                status=406,
+                mimetype="text/html")
+        return f(*args, **kwargs)
+    return decorated_function
+
+
+def after_request(response):
+    """
+    Generic function to do additional logging of requests & responses.
+
+    :param response:
+    :return:
+    """
+    if response.status_code != 200:
+
+        try:
+            data = response.data.decode('utf-8')
+        except Exception:
+            # never expected to happen, but we don't want any failures here
+            logging.exception('INTERNAL DECODING ERROR')
+            data = 'decoding error (see logs)'
+
+        logger.warning(f'[{response.status_code}] {request.method} {request.path} {data}')
+    else:
+        logger.info(f'[{response.status_code}] {request.method} {request.path}')
+    return response

brian_dashboard_manager/routes/test.py

+from flask import Blueprint, jsonify, current_app
+from brian_dashboard_manager.routes import common
+from brian_dashboard_manager import CONFIG_KEY
+
+routes = Blueprint("api-test", __name__)
+
+
+@routes.after_request
+def after_request(resp):
+    return common.after_request(resp)
+
+
+@routes.route("/test1", methods=['GET', 'POST'])
+@common.require_accepts_json
+def test1():
+    return jsonify({
+        'config': current_app.config[CONFIG_KEY],
+        'success': True
+    })

brian_dashboard_manager/routes/update.py

+from flask import Blueprint, current_app, jsonify
+from brian_dashboard_manager.routes import common
+from brian_dashboard_manager.grafana import *
+from brian_dashboard_manager import CONFIG_KEY
+
+routes = Blueprint("update", __name__)
+
+
+@routes.after_request
+def after_request(resp):
+    return common.after_request(resp)
+
+
+@routes.route('/', methods=['GET'])
+def update():
+    success = provision(current_app.config[CONFIG_KEY])
+    return {'data': success}

readme.md

-Grafana Automation repo

requirements.txt

+requests
+jsonschema
+flask
+pytest