diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..52aaa0b837143e82d2cd75dff0d60b09cdc7b463 --- /dev/null +++ b/.gitignore @@ -0,0 +1,21 @@ +# IDE related +.idea +.vscode + +# config +config.json + +# dev / builds +venv/ +*.egg-info +__pycache__ +coverage.xml +.coverage +htmlcov +.tox +dist +dashboards/dev/*.json + + +# logs +*.log diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000000000000000000000000000000000000..197e812180e4bbedbfcb457daa7aa81fcdf1aad7 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,2 @@ +include api/logging_default_config.json +include dashboards/* \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000000000000000000000000000000000000..a548a947815d6e6f29801ca8b049dca1d00005d0 --- /dev/null +++ b/README.md @@ -0,0 +1,86 @@ +# Skeleton Web App + +## Overview + +This module implements a skeleton Flask-based webservice. + +The webservice is communicates with clients over HTTP. +Responses to valid requests are returned as JSON messages. +The server will therefore return an error unless +`application/json` is in the `Accept` request header field. + +HTTP communication and JSON grammar details are +beyond the scope of this document. +Please refer to [RFC 2616](https://tools.ietf.org/html/rfc2616) +and www.json.org for more details. + + +## Configuration + +This app allows specification of a few +example configuration parameters. These +parameters should stored in a file formatted +similarly to `config.json.example`, and the name +of this file should be stored in the environment +variable `SETTINGS_FILENAME` when running the service. + +## Running this module + +This module has been tested in the following execution environments: + +- As an embedded Flask application. +For example, the application could be launched as follows: + +```bash +$ export FLASK_APP=app.py +$ export SETTINGS_FILENAME=config.json +$ flask run +``` + +- As an Apache/`mod_wsgi` service. + - Details of Apache and `mod_wsgi` + configuration are beyond the scope of this document. + +- As a `gunicorn` wsgi service. + - Details of `gunicorn` configuration are + beyond the scope of this document. + + +## Protocol Specification + +The following resources can be requested from the webservice. + +### resources + +Any non-empty responses are JSON formatted messages. + +#### /data/version + + * /version + + The response will be an object + containing the module and protocol versions of the + running server and will be formatted as follows: + + ```json + { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "api": { + "type": "string", + "pattern": r'\d+\.\d+' + }, + "module": { + "type": "string", + "pattern": r'\d+\.\d+' + } + }, + "required": ["api", "module"], + "additionalProperties": False + } + ``` + +#### /test/test1 + +The response will be some json data, as an example ... diff --git a/brian_dashboard_manager/__init__.py b/brian_dashboard_manager/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..396ba78edf307868c9fd50f39c6c312172243c87 --- /dev/null +++ b/brian_dashboard_manager/__init__.py @@ -0,0 +1,41 @@ +""" +automatically invoked app factory +""" +import logging +import os + +from flask import Flask + +from brian_dashboard_manager import environment +from brian_dashboard_manager import config + +CONFIG_KEY = 'CONFIG_PARAMS' + + +def create_app(): + """ + overrides default settings with those found + in the file read from env var SETTINGS_FILENAME + + :return: a new flask app instance + """ + + + app_config = config.defaults() + if 'SETTINGS_FILENAME' in os.environ: + with open(os.environ['SETTINGS_FILENAME']) as f: + app_config.update(config.load(f)) + + app = Flask(__name__) + app.secret_key = os.environ.get('SECRET_KEY') or 'super secret session key' + app.config[CONFIG_KEY] = app_config + + from brian_dashboard_manager.routes import update + app.register_blueprint(update.routes, url_prefix='/update') + + from brian_dashboard_manager.routes import test + app.register_blueprint(test.routes, url_prefix='/test') + + logging.info('Flask app initialized') + environment.setup_logging() + return app diff --git a/brian_dashboard_manager/app.py b/brian_dashboard_manager/app.py new file mode 100644 index 0000000000000000000000000000000000000000..5bc14b921b5f95fb5c68b08e83b5491952de018e --- /dev/null +++ b/brian_dashboard_manager/app.py @@ -0,0 +1,11 @@ +""" +default app creation +""" +import brian_dashboard_manager +from brian_dashboard_manager import environment, CONFIG_KEY + +environment.setup_logging() +app = brian_dashboard_manager.create_app() + +if __name__ == "__main__": + app.run(host="::", port=f"{app.config[CONFIG_KEY]['listen_port']}") diff --git a/brian_dashboard_manager/config.py b/brian_dashboard_manager/config.py new file mode 100644 index 0000000000000000000000000000000000000000..b6c389a4840330dde3521690e875c35ab3f6176f --- /dev/null +++ b/brian_dashboard_manager/config.py @@ -0,0 +1,42 @@ +import json +import jsonschema + +CONFIG_SCHEMA = { + "$schema": "http://json-schema.org/draft-07/schema#", + + "type": "object", + "properties": { + "admin_username": {"type": "string"}, + "admin_password": {"type": "string"}, + "hostname": {"type": "string"}, + "listen_port": {"type": "integer"}, + "grafana_port": {"type": "integer"}, + "organizations": {"type": "array", "items": {"type": "string"}}, + "dashboard_directory": {"type": "string"}, + }, + "required": ["admin_username", "admin_password", "hostname", "listen_port", "grafana_port", "organizations"], + "additionalProperties": False +} + + +def defaults(): + return { + "admin_username": "admin", + "admin_password": "admin", + "hostname": "localhost", + "listen_port": 3001, + "grafana_port": 3000, + "organizations": ["Main Org."] + } + + +def load(f): + """ + loads, validates and returns configuration parameters + + :param f: file-like object that produces the config file + :return: + """ + config = json.loads(f.read()) + jsonschema.validate(config, CONFIG_SCHEMA) + return config diff --git a/brian_dashboard_manager/environment.py b/brian_dashboard_manager/environment.py new file mode 100644 index 0000000000000000000000000000000000000000..b5057098e019cdb840eaf1b3599ad3053a21c529 --- /dev/null +++ b/brian_dashboard_manager/environment.py @@ -0,0 +1,23 @@ +import json +import logging.config +import os + + +def setup_logging(): + """ + set up logging using the configured filename + + if LOGGING_CONFIG is defined in the environment, use this for + the filename, otherwise use logging_default_config.json + """ + default_filename = os.path.join( + os.path.dirname(__file__), 'logging_default_config.json') + filename = os.getenv('LOGGING_CONFIG', default_filename) + with open(filename) as f: + # TODO: this mac workaround should be removed ... + d = json.loads(f.read()) + import platform + if platform.system() == 'Darwin': + d['handlers']['syslog_handler']['address'] = '/var/run/syslog' + logging.config.dictConfig(d) + # logging.config.dictConfig(json.loads(f.read())) diff --git a/brian_dashboard_manager/grafana/__init__.py b/brian_dashboard_manager/grafana/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..a39afea8d2ae43facdf737af680cf5b9f911e052 --- /dev/null +++ b/brian_dashboard_manager/grafana/__init__.py @@ -0,0 +1,3 @@ +from brian_dashboard_manager.grafana.provision import provision + +__all__ = ["provision"] \ No newline at end of file diff --git a/brian_dashboard_manager/grafana/dashboard.py b/brian_dashboard_manager/grafana/dashboard.py new file mode 100644 index 0000000000000000000000000000000000000000..5b59a0bfa7e2cc301e84716c02bacb61bcc38b5f --- /dev/null +++ b/brian_dashboard_manager/grafana/dashboard.py @@ -0,0 +1,21 @@ +import logging +from typing import Dict +from brian_dashboard_manager.grafana.utils.request import TokenRequest + +logger = logging.getLogger(__name__) + +def provision_dashboard(request: TokenRequest, dashboard: Dict): + + del dashboard['uid'] + del dashboard['id'] + del dashboard['version'] + + payload = { + 'dashboard': dashboard, + 'overwrite': False + } + try: + r = request.post('api/dashboards/db', json=payload) + except Exception as e: + logger.error(f'Error when provisioning dashboard: ' + e.response.text) + return True diff --git a/brian_dashboard_manager/grafana/organization.py b/brian_dashboard_manager/grafana/organization.py new file mode 100644 index 0000000000000000000000000000000000000000..bd8e73595f4f7ded691328e7f1bb8ee48c2452e8 --- /dev/null +++ b/brian_dashboard_manager/grafana/organization.py @@ -0,0 +1,83 @@ +import random +import string +import logging +from typing import Dict, List, Union +from datetime import datetime + +from brian_dashboard_manager.grafana.utils.request import Request, AdminRequest + +logger = logging.getLogger(__name__) + + +def switch_active_organization(request: Request, org_id: int): + assert org_id + + logger.debug(f'Switched {str(request)} active organization to #{org_id}') + return request.post(f'api/user/using/{org_id}', {}) + + +def get_organizations(request: AdminRequest) -> List: + return request.get('api/orgs') + + +def create_organization(request: AdminRequest, name: str) -> Union[Dict, None]: + assert name + + result = request.post('api/orgs', json={ + 'name': name + }) + + if result.get('message', '').lower() == 'organization created': + id = result.get('orgId') + logger.info(f'Created organization `{name}` with ID #{id}') + return {'id': id, 'name': name} + else: + return None + + +def delete_organization(request: AdminRequest, id: int) -> bool: + + result = request.delete(f'api/orgs/{id}') + + return result.get('message', '').lower() == 'organization deleted' + + +def create_api_token(request: AdminRequest, org_id: int, key_data=None): + data = { + 'name': ''.join(random.choices(string.ascii_uppercase + string.digits, k=16)), + 'role': 'Admin', + 'secondsToLive': 3600 # 60 minutes + } + if key_data: + data.update(key_data) + + switch_active_organization(request, org_id) + result = request.post(f'api/auth/keys', json=data) + token_id = result.get('id') + + logger.debug(f'Created API token #{token_id} for organization #{org_id}') + + return result + + +def delete_api_token(request: AdminRequest, org_id: int, token_id: int): + assert token_id + + switch_active_organization(request, org_id) + result = request.delete(f'api/auth/keys/{token_id}') + logger.debug(f'Deleted API token #{token_id} for organization #{org_id}') + return result + + +def delete_expired_api_tokens(request: AdminRequest, org_id: int) -> bool: + assert org_id + + tokens = request.get('api/auth/keys', params={'includeExpired': True}) + + now = datetime.now() + expired_tokens = [t for t in tokens if 'expiration' in t + and datetime.strptime(t['expiration'], '%Y-%m-%dT%H:%M:%SZ') < now] + + for token in expired_tokens: + delete_api_token(request, org_id, token['id']) + return True diff --git a/brian_dashboard_manager/grafana/provision.py b/brian_dashboard_manager/grafana/provision.py new file mode 100644 index 0000000000000000000000000000000000000000..99da18b09599241c9ff628259d4bf99623341977 --- /dev/null +++ b/brian_dashboard_manager/grafana/provision.py @@ -0,0 +1,49 @@ +import logging +import os +import json +from functools import reduce +from typing import List +from brian_dashboard_manager.grafana.utils.request import AdminRequest, TokenRequest +from brian_dashboard_manager.grafana.organization import get_organizations, create_organization, delete_organization, create_api_token, delete_api_token, delete_expired_api_tokens +from brian_dashboard_manager.grafana.dashboard import provision_dashboard + + +logger = logging.getLogger(__name__) + + +def provision(config): + + hostname = config.get('hostname') + port = config.get('grafana_port') + username = config.get('admin_username') + password = config.get('admin_password') + + request = AdminRequest(hostname, port, username, password) + all_orgs = get_organizations(request) + + organizations_to_provision = config.get('organizations', []) + + missing = [name for name in organizations_to_provision + if name not in [org['name'] for org in all_orgs]] + + for org in missing: + org_data = create_organization(request, org) + all_orgs.append(org_data) + + for org in all_orgs: + org_id = org['id'] + delete_expired_api_tokens(request, org_id) + token = create_api_token(request, org_id) + token_request = TokenRequest(hostname, port, token['key']) + # TODO: (de)provision datasources and remove all existing dashboards in an organization before attempting to provision new dashboards. + + for (dirpath, dirnames, filenames) in os.walk(config.get('dashboard_directory', 'dashboards')): + for file in filenames: + if file.endswith('.json'): + filename = os.path.join(dirpath, file) + logger.info(f'Provisioning dashboard: {file.strip(".json")}') + provision_dashboard(token_request, json.load(open(filename, 'r'))) + + delete_api_token(request, org_id, token['id']) + + return all_orgs diff --git a/brian_dashboard_manager/grafana/utils/request.py b/brian_dashboard_manager/grafana/utils/request.py new file mode 100644 index 0000000000000000000000000000000000000000..a34d541c83dca9600bc1e958791ce8e2ab23472b --- /dev/null +++ b/brian_dashboard_manager/grafana/utils/request.py @@ -0,0 +1,65 @@ +import requests +from typing import Dict, Any +from flask import current_app + + +class Request(object): + def __init__(self, url, headers=None): + self.headers = { + 'Accept': 'application/json' + } + if headers: + self.headers.update(headers) + + self.BASE_URL = url + + def get(self, endpoint: str, headers=None, **kwargs): + + r = requests.get( + self.BASE_URL + endpoint, + headers={**headers, **self.headers} if headers else self.headers, + **kwargs + ) + r.raise_for_status() + return r.json() + + def post(self, endpoint: str, headers=None, **kwargs): + + r = requests.post( + self.BASE_URL + endpoint, + headers={**headers, **self.headers} if headers else self.headers, + **kwargs + ) + r.raise_for_status() + return r.json() + + def delete(self, endpoint: str, headers=None, **kwargs): + + r = requests.delete( + self.BASE_URL + endpoint, + headers={**headers, **self.headers} if headers else self.headers, + **kwargs + ) + r.raise_for_status() + return r.json() + + +class AdminRequest(Request): + def __init__(self, hostname, port, username, password): + self.username = username + super().__init__(f'http://{username}:{password}@{hostname}:{port}/') + + def __str__(self): + return f'admin user: {self.username}' + + +class TokenRequest(Request): + def __init__(self, hostname, port, token: str): + self.token = token + + super().__init__(f'http://{hostname}:{port}/', { + 'Authorization': 'Bearer ' + token + }) + + def __str__(self): + return f'token {self.token}' diff --git a/brian_dashboard_manager/logging_default_config.json b/brian_dashboard_manager/logging_default_config.json new file mode 100644 index 0000000000000000000000000000000000000000..c100d56cad34389f015313d906c69f1a85c6417e --- /dev/null +++ b/brian_dashboard_manager/logging_default_config.json @@ -0,0 +1,59 @@ +{ + "version": 1, + "disable_existing_loggers": false, + "formatters": { + "simple": { + "format": "%(asctime)s - %(name)s - %(levelname)s - %(message)s" + } + }, + + "handlers": { + "console": { + "class": "logging.StreamHandler", + "level": "DEBUG", + "formatter": "simple", + "stream": "ext://sys.stdout" + }, + + "syslog_handler": { + "class": "logging.handlers.SysLogHandler", + "level": "DEBUG", + "address": "/dev/log", + "facility": "user", + "formatter": "simple" + }, + + "info_file_handler": { + "class": "logging.handlers.RotatingFileHandler", + "level": "INFO", + "formatter": "simple", + "filename": "info.log", + "maxBytes": 10485760, + "backupCount": 20, + "encoding": "utf8" + }, + + "error_file_handler": { + "class": "logging.handlers.RotatingFileHandler", + "level": "ERROR", + "formatter": "simple", + "filename": "errors.log", + "maxBytes": 10485760, + "backupCount": 20, + "encoding": "utf8" + } + }, + + "loggers": { + "api": { + "level": "DEBUG", + "handlers": ["console", "syslog_handler"], + "propagate": false + } + }, + + "root": { + "level": "INFO", + "handlers": ["console", "syslog_handler"] + } +} \ No newline at end of file diff --git a/brian_dashboard_manager/routes/__init__.py b/brian_dashboard_manager/routes/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/brian_dashboard_manager/routes/common.py b/brian_dashboard_manager/routes/common.py new file mode 100644 index 0000000000000000000000000000000000000000..2714cfcec13980b20587fed2c60cf7b18873f37d --- /dev/null +++ b/brian_dashboard_manager/routes/common.py @@ -0,0 +1,48 @@ +""" +Utilities used by multiple route blueprints. +""" +import functools +import logging +from flask import request, Response + +logger = logging.getLogger(__name__) + +def require_accepts_json(f): + """ + used as a route handler decorator to return an error + unless the request allows responses with type "application/json" + :param f: the function to be decorated + :return: the decorated function + """ + @functools.wraps(f) + def decorated_function(*args, **kwargs): + # TODO: use best_match to disallow */* ...? + if not request.accept_mimetypes.accept_json: + return Response( + response="response will be json", + status=406, + mimetype="text/html") + return f(*args, **kwargs) + return decorated_function + + +def after_request(response): + """ + Generic function to do additional logging of requests & responses. + + :param response: + :return: + """ + if response.status_code != 200: + + try: + data = response.data.decode('utf-8') + except Exception: + # never expected to happen, but we don't want any failures here + logging.exception('INTERNAL DECODING ERROR') + data = 'decoding error (see logs)' + + logger.warning(f'[{response.status_code}] {request.method} {request.path} {data}') + else: + logger.info(f'[{response.status_code}] {request.method} {request.path}') + return response diff --git a/brian_dashboard_manager/routes/test.py b/brian_dashboard_manager/routes/test.py new file mode 100644 index 0000000000000000000000000000000000000000..e5ad83a849f06cccebbc83ff40438b63885dc099 --- /dev/null +++ b/brian_dashboard_manager/routes/test.py @@ -0,0 +1,19 @@ +from flask import Blueprint, jsonify, current_app +from brian_dashboard_manager.routes import common +from brian_dashboard_manager import CONFIG_KEY + +routes = Blueprint("api-test", __name__) + + +@routes.after_request +def after_request(resp): + return common.after_request(resp) + + +@routes.route("/test1", methods=['GET', 'POST']) +@common.require_accepts_json +def test1(): + return jsonify({ + 'config': current_app.config[CONFIG_KEY], + 'success': True + }) diff --git a/brian_dashboard_manager/routes/update.py b/brian_dashboard_manager/routes/update.py new file mode 100644 index 0000000000000000000000000000000000000000..1fe1f097364b5d33a3bbc9c2241cd6d2dba22ca8 --- /dev/null +++ b/brian_dashboard_manager/routes/update.py @@ -0,0 +1,17 @@ +from flask import Blueprint, current_app, jsonify +from brian_dashboard_manager.routes import common +from brian_dashboard_manager.grafana import * +from brian_dashboard_manager import CONFIG_KEY + +routes = Blueprint("update", __name__) + + +@routes.after_request +def after_request(resp): + return common.after_request(resp) + + +@routes.route('/', methods=['GET']) +def update(): + success = provision(current_app.config[CONFIG_KEY]) + return {'data': success} diff --git a/dashboards/.gitkeep b/dashboards/.gitkeep new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/readme.md b/readme.md deleted file mode 100644 index 0678d7fabc0f79ea6da31bb561b39ee63083cea3..0000000000000000000000000000000000000000 --- a/readme.md +++ /dev/null @@ -1 +0,0 @@ -Grafana Automation repo diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..48c54a2bceaadd63934dc0b821777057f31fe0a5 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,4 @@ +requests +jsonschema +flask +pytest diff --git a/setup.py b/setup.py new file mode 100644 index 0000000000000000000000000000000000000000..9aeb300d94ab2c5e6e148a20f61684ede963e1af --- /dev/null +++ b/setup.py @@ -0,0 +1,17 @@ +from setuptools import setup, find_packages + +setup( + name='brian-dashboard-manager', + version="0.1", + author='GEANT', + author_email='swd@geant.org', + description='', + url=('https://gitlab.geant.net/live-projects/brian-dashboard-manager/'), + packages=find_packages(), + install_requires=[ + 'requests', + 'jsonschema', + 'flask' + ], + include_package_data=True, +) diff --git a/test/conftest.py b/test/conftest.py new file mode 100644 index 0000000000000000000000000000000000000000..51f2c85dcb7558f1bfbd2558c188c3d6306641c1 --- /dev/null +++ b/test/conftest.py @@ -0,0 +1,26 @@ +import json +import os +import tempfile + +import pytest + +import brian_dashboard_manager + + +@pytest.fixture +def data_config_filename(): + test_config_data = { + 'str-param': 'test data string', + 'int-param': -47 + } + with tempfile.NamedTemporaryFile() as f: + f.write(json.dumps(test_config_data).encode('utf-8')) + f.flush() + yield f.name + + +@pytest.fixture +def client(data_config_filename): + os.environ['SETTINGS_FILENAME'] = data_config_filename + with brian_dashboard_manager.create_app().test_client() as c: + yield c diff --git a/test/test_routes.py b/test/test_routes.py new file mode 100644 index 0000000000000000000000000000000000000000..cfa3caa5beee28974432914f1b1701871f9067a0 --- /dev/null +++ b/test/test_routes.py @@ -0,0 +1,46 @@ +import json +import jsonschema + +DEFAULT_REQUEST_HEADERS = { + 'Content-type': 'application/json', + 'Accept': ['application/json'] +} + + +def test_version_request(client): + + version_schema = { + '$schema': 'http://json-schema.org/draft-07/schema#', + + 'type': 'object', + 'properties': { + 'api': { + 'type': 'string', + 'pattern': r'\d+\.\d+' + }, + 'module': { + 'type': 'string', + 'pattern': r'\d+\.\d+' + } + }, + 'required': ['api', 'module'], + 'additionalProperties': False + } + + rv = client.post( + 'version', + headers=DEFAULT_REQUEST_HEADERS) + assert rv.status_code == 200 + jsonschema.validate( + json.loads(rv.data.decode('utf-8')), + version_schema) + + +def test_test_proc1(client): + + rv = client.post( + 'test/test1', + headers=DEFAULT_REQUEST_HEADERS) + assert rv.status_code == 200 + result = json.loads(rv.data.decode('utf-8')) + assert result # boilerplate test ... diff --git a/tox.ini b/tox.ini new file mode 100644 index 0000000000000000000000000000000000000000..3a38339037ea9eb14e9276b92aa2932baa451690 --- /dev/null +++ b/tox.ini @@ -0,0 +1,17 @@ +[tox] +envlist = py36 + +[testenv] +deps = + coverage + flake8 + -r requirements.txt + +commands = + coverage erase + coverage run --source api -m py.test {posargs} + coverage xml + coverage html + coverage report --fail-under 85 + flake8 +