Finished release 0.7.

docker-setup/Dockerfile

+FROM alpine:3.8
+
+# Build arguments
+## The database user name
+ARG DBUSER
+## The user's password
+ARG DBPASS
+## The database name
+ARG DBNAME
+
+
+# Forward the args to the container
+ENV DBUSER=${DBUSER}
+ENV DBPASS=${DBPASS}
+ENV DBNAME=${DBNAME}
+
+ENV PGDATA "/var/lib/postgresql"
+
+RUN apk update && \
+    apk add postgresql postgresql-contrib
+
+RUN mkdir -p /run/postgresql && chmod a+w /run/postgresql
+
+ADD entrypoint.sh /entrypoint.sh
+RUN chmod +x /entrypoint.sh
+USER postgres
+
+VOLUME $PGDATA
+CMD ["/entrypoint.sh"]
+EXPOSE 5432

docker-setup/config/grafana.ini

+##################### Grafana Configuration Example #####################
+#
+# Everything has defaults so you only need to uncomment things you want to
+# change
+
+# possible values : production, development
+; app_mode = production
+
+#################################### Paths ####################################
+[paths]
+# Path to where grafana can store temp files, sessions, and the sqlite3 db (if that is used)
+#
+data = /home/git/grafana
+#
+# Directory where grafana can store logs
+#
+logs = /var/log/grafana
+
+#################################### Server ####################################
+[server]
+# Protocol (http or https)
+protocol = http
+
+# The ip address to bind to, empty will bind to all interfaces
+http_addr = 
+
+# The http port  to use
+http_port = 3000
+
+# The public facing domain name used to access grafana from a browser
+;domain = localhost
+
+# Redirect to correct domain if host header does not match domain
+# Prevents DNS rebinding attacks
+;enforce_domain = false
+
+# The full public facing url
+;root_url = %(protocol)s://%(domain)s:%(http_port)s/
+
+# Log web requests
+;router_logging = false
+
+# the path relative working path
+static_root_path = public
+
+# enable gzip
+;enable_gzip = false
+
+# https certs & key file
+;cert_file =
+;cert_key =
+
+#################################### Database ####################################
+[dataproxy]
+logging = true
+
+
+[database]
+# Either "mysql", "postgres" or "sqlite3", it's your choice
+type = postgres
+host = postgres
+name = grafana
+user = grafana
+password = grafana
+
+# For "postgres" only, either "disable", "require" or "verify-full"
+;ssl_mode = disable
+
+# For "sqlite3" only, path relative to data_path setting
+path = grafana.db
+
+
+#################################### Analytics ####################################
+[analytics]
+# Server reporting, sends usage counters to stats.grafana.org every 24 hours.
+# No ip addresses are being tracked, only simple counters to track
+# running instances, dashboard and error counts. It is very helpful to us.
+# Change this option to false to disable reporting.
+reporting_enabled = false
+
+# Google Analytics universal tracking code, only enabled if you specify an id here
+;google_analytics_ua_id =
+
+#################################### Security ####################################
+[security]
+# default admin user, created on startup
+admin_user = admin
+
+# default admin password, can be changed before first start of grafana,  or in profile settings
+admin_password = admin
+
+# used for signing
+;secret_key = SW2YcwTIb9zpOOhoPsMm
+
+# Auto-login remember days
+;login_remember_days = 7
+;cookie_username = grafana_user
+;cookie_remember_name = grafana_remember
+
+# disable gravatar profile images
+;disable_gravatar = false
+
+# data source proxy whitelist (ip_or_domain:port separated by spaces)
+;data_source_proxy_whitelist =
+
+#################################### Users ####################################
+[users]
+# disable user signup / registration
+allow_sign_up = false
+
+# Allow non admin users to create organizations
+;allow_org_create = true
+
+# Set to true to automatically assign new users to the default organization (id 1)
+; auto_assign_org = true
+
+# Default role new users will be automatically assigned (if disabled above is set to true)
+auto_assign_org_role = Viewer
+
+#################################### Anonymous Auth ##########################
+[auth.anonymous]
+# enable anonymous access
+enabled = true
+
+# specify organization name that should be used for unauthenticated users
+org_name = Public
+
+# specify role for unauthenticated users
+org_role = Viewer
+
+#################################### Github Auth ##########################
+[auth.github]
+;enabled = false
+;allow_sign_up = false
+;client_id = some_id
+;client_secret = some_secret
+;scopes = user:email,read:org
+;auth_url = https://github.com/login/oauth/authorize
+;token_url = https://github.com/login/oauth/access_token
+;api_url = https://api.github.com/user
+;team_ids =
+;allowed_organizations =
+
+#################################### Google Auth ##########################
+[auth.google]
+;enabled = false
+;allow_sign_up = false
+;client_id = some_client_id
+;client_secret = some_client_secret
+;scopes = https://www.googleapis.com/auth/userinfo.profile https://www.googleapis.com/auth/userinfo.email
+;auth_url = https://accounts.google.com/o/oauth2/auth
+;token_url = https://accounts.google.com/o/oauth2/token
+;api_url = https://www.googleapis.com/oauth2/v1/userinfo
+;allowed_domains =
+
+#################################### Auth Proxy ##########################
+[auth.proxy]
+;enabled = false
+;header_name = X-WEBAUTH-USER
+;header_property = username
+;auto_sign_up = true
+
+#################################### Basic Auth ##########################
+[auth.basic]
+enabled = true
+
+#################################### Auth LDAP ##########################
+[auth.ldap]
+;enabled = false
+;config_file = /etc/grafana/ldap.toml
+
+#################################### SMTP / Emailing ##########################
+[smtp]
+;enabled = false
+;host = localhost:25
+;user =
+;password =
+;cert_file =
+;key_file =
+;skip_verify = false
+;from_address = admin@grafana.localhost
+
+[emails]
+;welcome_email_on_sign_up = false
+
+#################################### Logging ##########################
+[log]
+# Either "console", "file", default is "console"
+# Use comma to separate multiple modes, e.g. "console, file"
+mode = console
+
+# Buffer length of channel, keep it as it is if you don't know what it is.
+;buffer_len = 10000
+
+# Either "Trace", "Debug", "Info", "Warn", "Error", "Critical", default is "Trace"
+;level = Info
+
+# For "console" mode only
+[log.console]
+;level =
+
+# For "file" mode only
+[log.file]
+;level =
+# This enables automated log rotate(switch of following options), default is true
+;log_rotate = true
+
+# Max line number of single file, default is 1000000
+;max_lines = 1000000
+
+# Max size shift of single file, default is 28 means 1 << 28, 256MB
+;max_lines_shift = 28
+
+# Segment log daily, default is true
+;daily_rotate = true
+
+# Expired days of log file(delete after max days), default is 7
+;max_days = 7
+
+#################################### AMPQ Event Publisher ##########################
+[event_publisher]
+;enabled = false
+;rabbitmq_url = amqp://localhost/
+;exchange = grafana_events
+
+;#################################### Dashboard JSON files ##########################
+[dashboards.json]
+enabled = false
+path = /home/git/grafana/grafana-dashboards/dashboards
+
+[date_formats]
+full_date = MMM Do, YYYY @ hh:mm:ss a
+interval_second = hh:mm:ss a
+interval_minute = hh:mm a
+interval_hour = DD/MM hh:mm a
+interval_day = DD/MM
+interval_month = YYYY-MM
+interval_year = YYYY
+

docker-setup/docker-compose.yaml

+version: '3'
+services:
+    grafana:
+      restart: always
+      image: grafana/grafana:7.2.1
+      ports:
+        - 3000:3000
+      user: "427"
+      volumes: 
+        - ./config:/etc/grafana/
+      environment: 
+        - GF_INSTALL_PLUGINS=grafana-simple-json-datasource
+    postgres:
+      restart: always
+      build:
+          context: .
+          args:
+              DBUSER: grafana
+              DBPASS: grafana
+              DBNAME: grafana
+      hostname: postgres
+      ports:
+          - "5432:5432"
+      volumes:
+          - ./postgresdata:/var/lib/postgresql:z
+

docker-setup/entrypoint.sh

+#!/bin/sh
+
+# Entrypoint for the containerized database
+# Ensures the required setup is performed on the share before
+# starting the database
+
+if [ "$PGDATA" = "" ]
+then
+    echo "Environment variable PGDATA is not set"
+    exit 1
+fi
+
+if [ ! -d "$PGDATA" ]
+then
+    echo "Directory $PGDATA does not exist - no volume mounted?"
+    exit 2
+fi
+
+# Postgres will refuse to work on a non-empty folder when initializing and the
+# mount point might have a dotfile
+PGDATA="$PGDATA/9.6"
+
+echo $PGDATA
+echo $DBNAME
+echo $DBUSER
+
+if [ ! -d "$PGDATA" ]
+then
+    echo "Database seems to be uninitialized - doing so"
+    mkdir $PGDATA
+    initdb --pgdata=$PGDATA && \
+        pg_ctl start && \
+        sleep 4 && \
+        createuser -d -l -s $DBUSER && \
+        createdb -O $DBUSER $DBNAME && \
+        psql -c "ALTER ROLE $DBUSER WITH PASSWORD '$DBPASS'" && \
+        psql -a $DBNAME -c 'CREATE EXTENSION "uuid-ossp" WITH SCHEMA pg_catalog' && \
+        echo "host all all all md5" >> $PGDATA/pg_hba.conf && \
+        echo "listen_addresses = '0.0.0.0'" >> $PGDATA/postgresql.conf && \
+        pg_ctl stop
+fi
+
+exec /usr/bin/postgres

docs/Makefile

+# Minimal makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line, and also
+# from the environment for the first two.
+SPHINXOPTS    ?=
+SPHINXBUILD   ?= sphinx-build
+SOURCEDIR     = source
+BUILDDIR      = build
+
+# Put it first so that "make" without argument is like "make help".
+help:
+	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+
+.PHONY: help Makefile
+
+# Catch-all target: route all unknown targets to Sphinx using the new
+# "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
+%: Makefile
+	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)

docs/source/conf.py

+# Configuration file for the Sphinx documentation builder.
+#
+# This file only contains a selection of the most common options. For a full
+# list see the documentation:
+# https://www.sphinx-doc.org/en/master/usage/configuration.html
+
+# -- Path setup --------------------------------------------------------------
+
+# If extensions (or modules to document with autodoc) are in another directory,
+# add these directories to sys.path here. If the directory is relative to the
+# documentation root, use os.path.abspath to make it absolute, like shown here.
+#
+# import os
+# import sys
+# sys.path.insert(0, os.path.abspath('.'))
+
+from importlib import import_module
+from docutils.parsers.rst import Directive
+from docutils import nodes
+from sphinx import addnodes
+import json
+import os
+import sys
+
+sys.path.insert(0, os.path.abspath(
+    os.path.join(
+        os.path.dirname(__file__),
+        '..', '..', 'brian_dashboard_manager')))
+
+
+class RenderAsJSON(Directive):
+    # cf. https://stackoverflow.com/a/59883833
+
+    required_arguments = 1
+
+    def run(self):
+        module_path, member_name = self.arguments[0].rsplit('.', 1)
+
+        member_data = getattr(import_module(module_path), member_name)
+        code = json.dumps(member_data, indent=2, ensure_ascii=False)
+
+        literal = nodes.literal_block(code, code)
+        literal['language'] = 'json'
+
+        return [
+            addnodes.desc_name(text=member_name),
+            addnodes.desc_content('', literal)
+        ]
+
+
+def setup(app):
+    app.add_directive('asjson', RenderAsJSON)
+
+
+# -- Project information -----------------------------------------------------
+
+project = 'BRIAN Dashboard Manager'
+copyright = '2021, swd@geant.org'
+author = 'swd@geant.org'
+
+# The full version, including alpha/beta/rc tags
+release = '0.0'
+
+
+# -- General configuration ---------------------------------------------------
+
+# Add any Sphinx extension module names here, as strings. They can be
+# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
+# ones.
+extensions = [
+    'sphinx_rtd_theme',
+    'sphinx.ext.autodoc',
+    'sphinx.ext.coverage'
+]
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ['_templates']
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+# This pattern also affects html_static_path and html_extra_path.
+exclude_patterns = []
+
+
+# -- Options for HTML output -------------------------------------------------
+
+# The theme to use for HTML and HTML Help pages.  See the documentation for
+# a list of builtin themes.
+#
+html_theme = 'sphinx_rtd_theme'
+# html_theme = 'alabaster'
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+html_static_path = ['_static']
+
+# Both the class’ and the __init__ method’s docstring
+# are concatenated and inserted.
+autoclass_content = "both"
+autodoc_typehints = "none"

docs/source/configuration.rst

+
+Configuration and Running
+=========================
+
+Configuration
+-------------
+
+This app allows specification of a few
+example configuration parameters. These
+parameters should stored in a file formatted
+similarly to `config.json.example`, and the name
+of this file should be stored in the environment
+variable `CONFIG_FILENAME` when running the service.
+
+Config
+---------
+.. automodule::  brian_dashboard_manager.config
+
+Running this module
+---------------------
+
+This module has been tested in the following execution environments:
+
+* As an embedded Flask application.
+  For example, the application could be launched as follows:
+
+.. code-block:: python
+
+   export FLASK_APP=/path/to/brian_dashboard_manager/app.py
+   export CONFIG_FILENAME=/path/to/config.json
+   flask run
+
+* As a `gunicorn` wsgi service.
+
+  * Details of `gunicorn` configuration can be found in the
+    brian_dashboard_manager Puppet repository.

docs/source/index.rst

+.. BRIAN Dashboard Manager documentation master file, created by
+   sphinx-quickstart on Tue Mar 16 14:42:57 2021.
+   You can adapt this file completely to your liking, but it should at least
+   contain the root `toctree` directive.
+
+
+BRIAN Dashboard Manager
+=======================
+
+The BRIAN Dashboard Manager is used
+provision Organizations and Dashboards in Grafana for BRIAN.
+
+
+.. toctree::
+   :maxdepth: 3
+   :caption: Contents:
+
+   configuration
+   overview
+   protocol

docs/source/overview.rst

+
+Overview
+=========================
+
+This module is used to provision Organizations and Dashboards inGrafana for BRIAN.
+
+The dashboards are generated from a list of interfaces obtained from Inventory Provider.
+
+Jinja templates are populated with data from these interfaces to render
+Dashboard JSON definitions sent to the Grafana API.
+
+.. automodule:: brian_dashboard_manager.grafana
+
+.. automodule:: brian_dashboard_manager.templating

docs/source/protocol.rst

+
+Protocol
+=========================
+
+
+This module implements a Flask-based webservice used only to
+trigger the provisioning process.
+
+The following resources can be requested from the webservice.
+
+resources
+-----------
+Any non-empty responses are JSON formatted messages.
+
+
+/update
+*********
+
+.. autofunction::  brian_dashboard_manager.routes.update.update

requirements.txt

 requests
 jsonschema
 flask
+jinja2
+
 pytest
 pytest-mock
 responses
-jinja2
+sphinx
\ No newline at end of file
+sphinx-rtd-theme
+

setup.py

@@ -2,7 +2,7 @@ from setuptools import setup, find_packages
 
 setup(
     name='brian-dashboard-manager',
-    version="0.6",
+    version="0.7",
     author='GEANT',
     author_email='swd@geant.org',
     description='',

test/test_grafana_dashboard.py

@@ -85,7 +85,38 @@ def test_delete_dashboards(data_config):
             ''))
 
     data = dashboard._delete_dashboard(request, UID + 1)
-    assert data is None
+    assert data is False
+
+
+@responses.activate
+def test_delete_dashboard(data_config):
+    UID = 1
+    ID = 1
+    VERSION = 1
+    FOLDER_ID = 1
+    TITLE = 'testdashboard'
+    dash = {'id': ID, 'uid': UID, 'title': TITLE, 'version': VERSION}
+    request = TokenRequest(**data_config, token='test')
+
+    def delete_callback(request):
+        return 200, {}, json.dumps({'message': 'deleted dashboard'})
+
+    responses.add_callback(method=responses.DELETE,
+                           url=request.BASE_URL + f'api/dashboards/uid/{UID}',
+                           callback=delete_callback)
+
+    def search_callback(request):
+        return 200, {}, json.dumps(dash)
+
+    responses.add_callback(method=responses.GET,
+                           url=request.BASE_URL + 'api/search',
+                           callback=search_callback)
+
+    deleted = dashboard.delete_dashboard(request, dash)
+    assert deleted
+    del dash['uid']
+    deleted = dashboard.delete_dashboard(request, dash, FOLDER_ID)
+    assert deleted
 
 
 @responses.activate

test/test_update.py

@@ -2,7 +2,7 @@ import responses
 import json
 from brian_dashboard_manager.templating.nren_access import get_nrens
 from brian_dashboard_manager.grafana.provision import provision_folder, \
-    generate_all_nrens
+    generate_all_nrens, provision
 from brian_dashboard_manager.grafana.provision import is_re_customer, \
     is_cls, is_ias_customer, is_ias_private, is_ias_public, is_ias_upstream, \
     is_lag_backbone, is_phy_upstream, is_re_peer, is_gcs, \
@@ -236,7 +236,8 @@ def test_provision_folder(data_config, mocker):
 
     for dashboard in dashboards:
         provision_folder(None, 'testfolder', dashboards[dashboard],
-                         TEST_INTERFACES, 'testdatasource')
+                         TEST_INTERFACES,
+                         'testdatasource', ['CLS TESTDASHBOARD'])
 
 
 def test_provision_nrens(data_config, mocker):
@@ -316,6 +317,11 @@ def test_provision(data_config, mocker, client):
         url=f"{data_config['inventory_provider']}/poller/interfaces",
         callback=get_callback)
 
+    responses.add_callback(
+        method=responses.GET,
+        url=f"{data_config['inventory_provider']}/data/interfaces",
+        callback=get_callback)
+
     def folder_get(request):
         return 200, {}, json.dumps([])
 
@@ -416,7 +422,4 @@ def test_provision(data_config, mocker, client):
         'brian_dashboard_manager.grafana.provision.delete_api_token')
     # we dont care about this, tested separately
     _mocked_delete_api_token.return_value = None
-    response = client.get('/update/', headers=DEFAULT_REQUEST_HEADERS)
-    assert response.status_code == 200
-    data = json.loads(response.data.decode('utf-8'))['data']
-    assert data is not None  # == EXISTING_ORGS + [PROVISIONED_ORGANIZATION]
+    provision(data_config)

tox.ini

@@ -16,4 +16,5 @@ commands =
     coverage xml
     coverage html
     coverage report --fail-under 75
-    flake8
+    flake8
\ No newline at end of file
+    sphinx-build -M html docs/source docs/build