From f163cf918ec2d206946119f3e25cdd57fbeaeab4 Mon Sep 17 00:00:00 2001 From: Bjarke Madsen <bjarke.madsen@geant.org> Date: Fri, 19 Feb 2021 11:24:41 +0100 Subject: [PATCH] Move dashboards and datasources into package --- MANIFEST.in | 3 +- .../dashboards}/cls_peers.json | 0 .../dashboards/home.json | 4 +- .../dashboards}/ias_peers.json | 0 .../dashboards}/infrastructure_backbone.json | 0 .../dashboards}/peers.json | 0 .../services_automated_l2_circuits.json | 0 .../dashboards}/services_cls.json | 0 .../dashboards}/services_geant_open.json | 0 .../dashboards}/services_ias.json | 0 .../dashboards}/services_l2_circuits.json | 0 .../dashboards}/services_lhcone.json | 0 .../dashboards}/services_mdvpn.json | 0 .../dashboards}/services_re.json | 0 .../datasources}/InfluxDB.json | 0 brian_dashboard_manager/grafana/dashboard.py | 18 +- brian_dashboard_manager/grafana/datasource.py | 2 +- dashboards/.gitkeep | 0 dashboards/routerdashboard.json | 785 ------------------ 19 files changed, 20 insertions(+), 792 deletions(-) rename {dashboards => brian_dashboard_manager/dashboards}/cls_peers.json (100%) rename dashboards/main.json => brian_dashboard_manager/dashboards/home.json (99%) rename {dashboards => brian_dashboard_manager/dashboards}/ias_peers.json (100%) rename {dashboards => brian_dashboard_manager/dashboards}/infrastructure_backbone.json (100%) rename {dashboards => brian_dashboard_manager/dashboards}/peers.json (100%) rename {dashboards => brian_dashboard_manager/dashboards}/services_automated_l2_circuits.json (100%) rename {dashboards => brian_dashboard_manager/dashboards}/services_cls.json (100%) rename {dashboards => brian_dashboard_manager/dashboards}/services_geant_open.json (100%) rename {dashboards => brian_dashboard_manager/dashboards}/services_ias.json (100%) rename {dashboards => brian_dashboard_manager/dashboards}/services_l2_circuits.json (100%) rename {dashboards => brian_dashboard_manager/dashboards}/services_lhcone.json (100%) rename {dashboards => brian_dashboard_manager/dashboards}/services_mdvpn.json (100%) rename {dashboards => brian_dashboard_manager/dashboards}/services_re.json (100%) rename {datasources => brian_dashboard_manager/datasources}/InfluxDB.json (100%) delete mode 100644 dashboards/.gitkeep delete mode 100644 dashboards/routerdashboard.json diff --git a/MANIFEST.in b/MANIFEST.in index 197e812..96a02e6 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,2 +1,3 @@ include api/logging_default_config.json -include dashboards/* \ No newline at end of file +include brian_dashboard_manager/dashboards/* +include brian_dashboard_manager/datasources/* \ No newline at end of file diff --git a/dashboards/cls_peers.json b/brian_dashboard_manager/dashboards/cls_peers.json similarity index 100% rename from dashboards/cls_peers.json rename to brian_dashboard_manager/dashboards/cls_peers.json diff --git a/dashboards/main.json b/brian_dashboard_manager/dashboards/home.json similarity index 99% rename from dashboards/main.json rename to brian_dashboard_manager/dashboards/home.json index edd9918..15fa12f 100755 --- a/dashboards/main.json +++ b/brian_dashboard_manager/dashboards/home.json @@ -29,7 +29,7 @@ "type": "dashboards" }, { - "asDropdown": false, + "asDropdown": true, "icon": "external link", "tags": [ "infrastructure" @@ -706,4 +706,4 @@ "timezone": "", "title": "Home", "version": 1 -} \ No newline at end of file +} diff --git a/dashboards/ias_peers.json b/brian_dashboard_manager/dashboards/ias_peers.json similarity index 100% rename from dashboards/ias_peers.json rename to brian_dashboard_manager/dashboards/ias_peers.json diff --git a/dashboards/infrastructure_backbone.json b/brian_dashboard_manager/dashboards/infrastructure_backbone.json similarity index 100% rename from dashboards/infrastructure_backbone.json rename to brian_dashboard_manager/dashboards/infrastructure_backbone.json diff --git a/dashboards/peers.json b/brian_dashboard_manager/dashboards/peers.json similarity index 100% rename from dashboards/peers.json rename to brian_dashboard_manager/dashboards/peers.json diff --git a/dashboards/services_automated_l2_circuits.json b/brian_dashboard_manager/dashboards/services_automated_l2_circuits.json similarity index 100% rename from dashboards/services_automated_l2_circuits.json rename to brian_dashboard_manager/dashboards/services_automated_l2_circuits.json diff --git a/dashboards/services_cls.json b/brian_dashboard_manager/dashboards/services_cls.json similarity index 100% rename from dashboards/services_cls.json rename to brian_dashboard_manager/dashboards/services_cls.json diff --git a/dashboards/services_geant_open.json b/brian_dashboard_manager/dashboards/services_geant_open.json similarity index 100% rename from dashboards/services_geant_open.json rename to brian_dashboard_manager/dashboards/services_geant_open.json diff --git a/dashboards/services_ias.json b/brian_dashboard_manager/dashboards/services_ias.json similarity index 100% rename from dashboards/services_ias.json rename to brian_dashboard_manager/dashboards/services_ias.json diff --git a/dashboards/services_l2_circuits.json b/brian_dashboard_manager/dashboards/services_l2_circuits.json similarity index 100% rename from dashboards/services_l2_circuits.json rename to brian_dashboard_manager/dashboards/services_l2_circuits.json diff --git a/dashboards/services_lhcone.json b/brian_dashboard_manager/dashboards/services_lhcone.json similarity index 100% rename from dashboards/services_lhcone.json rename to brian_dashboard_manager/dashboards/services_lhcone.json diff --git a/dashboards/services_mdvpn.json b/brian_dashboard_manager/dashboards/services_mdvpn.json similarity index 100% rename from dashboards/services_mdvpn.json rename to brian_dashboard_manager/dashboards/services_mdvpn.json diff --git a/dashboards/services_re.json b/brian_dashboard_manager/dashboards/services_re.json similarity index 100% rename from dashboards/services_re.json rename to brian_dashboard_manager/dashboards/services_re.json diff --git a/datasources/InfluxDB.json b/brian_dashboard_manager/datasources/InfluxDB.json similarity index 100% rename from datasources/InfluxDB.json rename to brian_dashboard_manager/datasources/InfluxDB.json diff --git a/brian_dashboard_manager/grafana/dashboard.py b/brian_dashboard_manager/grafana/dashboard.py index e44502c..525cab9 100644 --- a/brian_dashboard_manager/grafana/dashboard.py +++ b/brian_dashboard_manager/grafana/dashboard.py @@ -12,7 +12,7 @@ logger = logging.getLogger(__name__) # Returns dictionary for each dashboard JSON definition in supplied directory def get_dashboard_definitions(dir=None): # pragma: no cover dashboard_dir = dir or os.path.join( - os.path.dirname(__file__), '../../dashboards/') + os.path.dirname(__file__), '../dashboards/') for (dirpath, _, filenames) in os.walk(dashboard_dir): for file in filenames: if file.endswith('.json'): @@ -43,12 +43,21 @@ def delete_dashboards(request: TokenRequest): return True +# Searches for a dashboard with given title +def find_dashboard(request: TokenRequest, title): + r = request.get('api/search', params={ + 'query': title + }) + if r and len(r) > 0: + return r[0] + return None + # Searches Grafana for a dashboard # matching the title of the provided dashboard. def _search_dashboard(request: TokenRequest, dashboard: Dict): try: r = request.get('api/search', params={ - 'query': f'{dashboard["title"]}' + 'query': dashboard["title"] }) if r and isinstance(r, list): if len(r) >= 1: @@ -72,7 +81,7 @@ def _get_dashboard(request: TokenRequest, uid: int): # Creates or updates (if exists) given dashboard for the token's organization. # supplied dashboards are JSON blobs exported from GUI with a UID. -def create_dashboard(request: TokenRequest, dashboard: Dict): +def create_dashboard(request: TokenRequest, dashboard: Dict, folder_id=None): title = dashboard['title'] existing_dashboard = None @@ -103,6 +112,9 @@ def create_dashboard(request: TokenRequest, dashboard: Dict): 'dashboard': dashboard, 'overwrite': False } + if folder_id: + payload['folderId'] = folder_id + try: action = "Updating" if existing_dashboard else "Provisioning" logger.info(f'{action} dashboard: {title}') diff --git a/brian_dashboard_manager/grafana/datasource.py b/brian_dashboard_manager/grafana/datasource.py index 50ab365..bec5c16 100644 --- a/brian_dashboard_manager/grafana/datasource.py +++ b/brian_dashboard_manager/grafana/datasource.py @@ -23,7 +23,7 @@ def _datasource_provisioned(datasource_to_check, provisioned_datasources): def get_missing_datasource_definitions(request: Request, dir=None): datasource_dir = dir or os.path.join( - os.path.dirname(__file__), '../../datasources/') + os.path.dirname(__file__), '../datasources/') existing_datasources = get_datasources(request) def check_ds_not_provisioned(filename): diff --git a/dashboards/.gitkeep b/dashboards/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/dashboards/routerdashboard.json b/dashboards/routerdashboard.json deleted file mode 100644 index 0401d3c..0000000 --- a/dashboards/routerdashboard.json +++ /dev/null @@ -1,785 +0,0 @@ -{ - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": "-- Grafana --", - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": true, - "gnetId": null, - "graphTooltip": 0, - "id": 1, - "iteration": 1610439609801, - "links": [ - { - "asDropdown": true, - "icon": "external link", - "tags": [ - "services" - ], - "targetBlank": true, - "title": "Services", - "type": "dashboards" - }, - { - "asDropdown": false, - "icon": "external link", - "tags": [ - "infrastructure" - ], - "targetBlank": true, - "title": "Infrastructure", - "type": "dashboards" - }, - { - "asDropdown": true, - "icon": "external link", - "tags": [ - "customers" - ], - "targetBlank": true, - "title": "NREN Access", - "type": "dashboards" - }, - { - "asDropdown": true, - "icon": "external link", - "tags": [ - "peers" - ], - "targetBlank": true, - "title": "Peers", - "type": "dashboards" - } - ], - "panels": [ - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "PollerInfluxDB", - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 3, - "gridPos": { - "h": 14, - "w": 12, - "x": 0, - "y": 0 - }, - "hiddenSeries": false, - "id": 2, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "max": true, - "min": false, - "rightSide": false, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.2.1", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "Ingress Traffic", - "groupBy": [ - { - "params": [ - "5m" - ], - "type": "time" - }, - { - "params": [ - "linear" - ], - "type": "fill" - } - ], - "measurement": "interface_rates", - "orderByTime": "ASC", - "policy": "default", - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "ingress" - ], - "type": "field" - }, - { - "params": [], - "type": "mean" - }, - { - "params": [ - "*8" - ], - "type": "math" - } - ] - ], - "tags": [ - { - "key": "hostname", - "operator": "=~", - "value": "/^$hostname$/" - }, - { - "condition": "AND", - "key": "interface_name", - "operator": "=~", - "value": "/^$interface_name$/" - } - ] - }, - { - "alias": "Egress Traffic", - "groupBy": [ - { - "params": [ - "5m" - ], - "type": "time" - }, - { - "params": [ - "linear" - ], - "type": "fill" - } - ], - "measurement": "interface_rates", - "orderByTime": "ASC", - "policy": "default", - "refId": "B", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "egress" - ], - "type": "field" - }, - { - "params": [], - "type": "mean" - }, - { - "params": [ - "*8" - ], - "type": "math" - } - ] - ], - "tags": [ - { - "key": "hostname", - "operator": "=~", - "value": "/^$hostname$/" - }, - { - "condition": "AND", - "key": "interface_name", - "operator": "=~", - "value": "/^$interface_name$/" - } - ] - }, - { - "alias": "Ingress 95th Percentile", - "groupBy": [], - "measurement": "interface_rates", - "orderByTime": "ASC", - "policy": "default", - "refId": "C", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "ingress" - ], - "type": "field" - }, - { - "params": [ - 95 - ], - "type": "percentile" - }, - { - "params": [ - "*8" - ], - "type": "math" - } - ] - ], - "tags": [ - { - "condition": null, - "key": "hostname", - "operator": "=~", - "value": "/^$hostname$/" - }, - { - "condition": "AND", - "key": "interface_name", - "operator": "=~", - "value": "/^$interface_name$/" - } - ] - }, - { - "alias": "Egress 95th Percentile", - "groupBy": [], - "measurement": "interface_rates", - "orderByTime": "ASC", - "policy": "default", - "refId": "D", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "egress" - ], - "type": "field" - }, - { - "params": [ - 95 - ], - "type": "percentile" - }, - { - "params": [ - "*8" - ], - "type": "math" - } - ] - ], - "tags": [ - { - "condition": null, - "key": "hostname", - "operator": "=~", - "value": "/^$hostname$/" - }, - { - "condition": "AND", - "key": "interface_name", - "operator": "=~", - "value": "/^$interface_name$/" - } - ] - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "$hostname - $interface_name - Traffic", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "bps", - "label": "bits per second", - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "bps", - "label": "bits per second", - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "PollerInfluxDB", - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 3, - "gridPos": { - "h": 14, - "w": 12, - "x": 12, - "y": 0 - }, - "hiddenSeries": false, - "id": 3, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "max": true, - "min": false, - "rightSide": false, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.2.1", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "Inbound", - "groupBy": [ - { - "params": [ - "5m" - ], - "type": "time" - }, - { - "params": [ - "linear" - ], - "type": "fill" - } - ], - "measurement": "interface_rates", - "orderByTime": "ASC", - "policy": "default", - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "ingressv6" - ], - "type": "field" - }, - { - "params": [], - "type": "mean" - }, - { - "params": [ - "*8" - ], - "type": "math" - } - ] - ], - "tags": [ - { - "key": "hostname", - "operator": "=~", - "value": "/^$hostname$/" - }, - { - "condition": "AND", - "key": "interface_name", - "operator": "=~", - "value": "/^$interface_name$/" - } - ] - }, - { - "alias": "Outbound", - "groupBy": [ - { - "params": [ - "5m" - ], - "type": "time" - }, - { - "params": [ - "linear" - ], - "type": "fill" - } - ], - "measurement": "interface_rates", - "orderByTime": "ASC", - "policy": "default", - "refId": "B", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "egressv6" - ], - "type": "field" - }, - { - "params": [], - "type": "mean" - }, - { - "params": [ - "*8" - ], - "type": "math" - } - ] - ], - "tags": [ - { - "key": "hostname", - "operator": "=~", - "value": "/^$hostname$/" - }, - { - "condition": "AND", - "key": "interface_name", - "operator": "=~", - "value": "/^$interface_name$/" - } - ] - }, - { - "alias": "Ingress 95th Percentile", - "groupBy": [], - "measurement": "interface_rates", - "orderByTime": "ASC", - "policy": "default", - "refId": "C", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "ingressv6" - ], - "type": "field" - }, - { - "params": [ - 95 - ], - "type": "percentile" - }, - { - "params": [ - "*8" - ], - "type": "math" - } - ] - ], - "tags": [ - { - "condition": null, - "key": "hostname", - "operator": "=~", - "value": "/^$hostname$/" - }, - { - "condition": "AND", - "key": "interface_name", - "operator": "=~", - "value": "/^$interface_name$/" - } - ] - }, - { - "alias": "Egress 95th Percentile", - "groupBy": [], - "measurement": "interface_rates", - "orderByTime": "ASC", - "policy": "default", - "refId": "D", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "egressv6" - ], - "type": "field" - }, - { - "params": [ - 95 - ], - "type": "percentile" - }, - { - "params": [ - "*8" - ], - "type": "math" - } - ] - ], - "tags": [ - { - "condition": null, - "key": "hostname", - "operator": "=~", - "value": "/^$hostname$/" - }, - { - "condition": "AND", - "key": "interface_name", - "operator": "=~", - "value": "/^$interface_name$/" - } - ] - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "$hostname - $interface_name - IPv6 Traffic", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "bps", - "label": "bits per second", - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "bps", - "label": "bits per second", - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - } - ], - "schemaVersion": 26, - "style": "dark", - "tags": [], - "templating": { - "list": [ - { - "allValue": null, - "current": { - "selected": false, - "text": "mx1.ams.nl.geant.net", - "value": "mx1.ams.nl.geant.net" - }, - "datasource": "PollerInfluxDB", - "definition": "SHOW TAG VALUES ON poller WITH KEY=hostname", - "hide": 0, - "includeAll": false, - "label": "Router:", - "multi": false, - "name": "hostname", - "options": [], - "query": "SHOW TAG VALUES ON poller WITH KEY=hostname", - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "sort": 0, - "tagValuesQuery": "", - "tags": [], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": null, - "current": { - "selected": false, - "text": "ae1", - "value": "ae1" - }, - "datasource": "PollerInfluxDB", - "definition": "SHOW TAG VALUES ON poller WITH KEY IN (interface_name) WHERE hostname =~ /$hostname/ ", - "hide": 0, - "includeAll": false, - "label": "Interface :", - "multi": false, - "name": "interface_name", - "options": [], - "query": "SHOW TAG VALUES ON poller WITH KEY IN (interface_name) WHERE hostname =~ /$hostname/ ", - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "sort": 0, - "tagValuesQuery": "", - "tags": [], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": null, - "current": { - "selected": false, - "text": "mx1.ams.nl.geant.net", - "value": "mx1.ams.nl.geant.net" - }, - "datasource": "PollerInfluxDB", - "definition": "SHOW TAG VALUES ON poller WITH KEY=hostname", - "hide": 0, - "includeAll": false, - "label": "Router:", - "multi": false, - "name": "copy_of_hostname", - "options": [], - "query": "SHOW TAG VALUES ON poller WITH KEY=hostname", - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "sort": 0, - "tagValuesQuery": "", - "tags": [], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": null, - "current": { - "selected": false, - "text": "JISC", - "value": "JISC" - }, - "hide": 0, - "includeAll": false, - "label": "NREN", - "multi": false, - "name": "NREN", - "options": [ - { - "selected": true, - "text": "JISC", - "value": "JISC" - }, - { - "selected": false, - "text": "NORDUnet", - "value": "NORDUnet" - }, - { - "selected": false, - "text": "PSNC", - "value": "PSNC" - } - ], - "query": "JISC, NORDUnet, PSNC", - "skipUrlSync": false, - "type": "custom" - } - ] - }, - "time": { - "from": "now-6h", - "to": "now" - }, - "timepicker": { - "refresh_intervals": [ - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ] - }, - "timezone": "", - "title": "Poller", - "uid": "2xdIhv-Gz", - "version": 4 -} \ No newline at end of file -- GitLab