provision.py

"""
This module is responsible for the
entire provisioning lifecycle.
"""
import itertools
import os
import logging
import time
import json
import datetime
from concurrent.futures import Future
from concurrent.futures import ThreadPoolExecutor
from brian_dashboard_manager.config import DEFAULT_ORGANIZATIONS, STATE_PATH
from brian_dashboard_manager.grafana.utils.request import AdminRequest, \
    TokenRequest

from brian_dashboard_manager.services.api import fetch_services

from brian_dashboard_manager.grafana.organization import \
    get_organizations, create_organization, create_api_token, \
    delete_api_token, delete_expired_api_tokens, set_home_dashboard
from brian_dashboard_manager.grafana.dashboard import list_dashboards, \
    get_dashboard_definitions, create_dashboard, delete_dashboard
from brian_dashboard_manager.grafana.datasource import \
    datasource_exists, create_datasource
from brian_dashboard_manager.grafana.folder import find_folder, \
    delete_folder, delete_unknown_folders
from brian_dashboard_manager.inventory_provider.interfaces import \
    get_gws_direct, get_gws_indirect, get_interfaces, \
    get_eumetsat_multicast_subscriptions

from brian_dashboard_manager.templating.helpers import \
    get_aggregate_dashboard_data, get_interface_data, \
    get_nren_interface_data, get_dashboard_data, \
    get_nren_dashboard_data, get_aggregate_interface_data, \
    get_nren_interface_data_old, get_re_peer_dashboard_data, get_re_peer_interface_data, get_service_data, \
    get_service_dashboard_data

from brian_dashboard_manager.templating.gws import generate_gws, generate_indirect
from brian_dashboard_manager.templating.eumetsat import generate_eumetsat_multicast
from brian_dashboard_manager.templating.render import (
    render_complex_dashboard,
    render_simple_dashboard,
)

logger = logging.getLogger(__name__)

MAX_WORKERS = 4
DASHBOARDS = {
    'NRENBETA': {  # needed for POL1-642 BETA
        'tag': ['customersbeta'],
        'folder_name': 'NREN Access BETA',
        'interfaces': []
    },
    'NREN': {
        'tag': ['customers'],
        'folder_name': 'NREN Access',
        'interfaces': []
    },
    'RE_PEER': {
        'tag': 'RE_PEER',
        'folder_name': 'RE Peer',
        'interfaces': []
    },
    'RE_CUST': {
        'tag': 'RE_CUST',
        'folder_name': 'RE Customer',
        'interfaces': []
    },
    'GEANTOPEN': {
        'tag': 'GEANTOPEN',
        'folder_name': 'GEANTOPEN',
        'interfaces': []
    },
    'GCS': {
        'tag': 'AUTOMATED_L2_CIRCUITS',
        'folder_name': 'GCS',
        'interfaces': []
    },
    'L2_CIRCUIT': {
        'tag': 'L2_CIRCUITS',
        'folder_name': 'L2 Circuit',
        'interfaces': []
    },
    'LHCONE_PEER': {
        'tag': 'LHCONE_PEER',
        'folder_name': 'LHCONE Peer',
        'interfaces': []
    },
    'LHCONE_CUST': {
        'tag': 'LHCONE_CUST',
        'folder_name': 'LHCONE Customer',
        'interfaces': []
    },
    'MDVPN_CUSTOMERS': {
        'tag': 'MDVPN',
        'folder_name': 'MDVPN Customers',
        'interfaces': []
    },
    'INFRASTRUCTURE_BACKBONE': {
        'tag': 'BACKBONE',
        'errors': True,
        'folder_name': 'Infrastructure Backbone',
        'interfaces': []
    },
    'IAS_PRIVATE': {
        'tag': 'IAS_PRIVATE',
        'folder_name': 'IAS Private',
        'interfaces': []
    },
    'IAS_PUBLIC': {
        'tag': 'IAS_PUBLIC',
        'folder_name': 'IAS Public',
        'interfaces': []
    },
    'IAS_CUSTOMER': {
        'tag': 'IAS_CUSTOMER',
        'folder_name': 'IAS Customer',
        'interfaces': []
    },
    'IAS_UPSTREAM': {
        'tag': ['IAS_UPSTREAM', 'UPSTREAM'],
        'folder_name': 'IAS Upstream',
        'interfaces': []
    },
    'GWS_PHY_UPSTREAM': {
        'tag': ['GWS_UPSTREAM', 'UPSTREAM'],
        'errors': True,
        'folder_name': 'GWS PHY Upstream',
        'interfaces': []
    },
    'GBS_10G': {
        'tag': 'GBS_10G',
        'errors': True,
        'folder_name': '10G Guaranteed Bandwidth Service',
        'interfaces': []
    }
}

SERVICE_DASHBOARDS = {
    'MWS': {
        'tag': ['mws'],
        'service_type': 'GEANT MANAGED WAVELENGTH SERVICE',
        'folder_name': 'Managed Wavelength Service',
        'interfaces': [],
        'services': []
    }
}

AGG_DASHBOARDS = {
    'CLS_PEERS': {
        'tag': 'cls_peers',
        'dashboard_name': 'CLS Peers',
        'interfaces': []
    },
    'IAS_PEERS': {
        'tag': 'ias_peers',
        'dashboard_name': 'IAS Peers',
        'interfaces': []
    },
    'IAS_UPSTREAM': {
        'tag': 'gws_upstreams',
        'dashboard_name': 'GWS Upstreams',
        'interfaces': []
    },
    'LHCONE': {
        'tag': 'lhcone',
        'dashboard_name': 'LHCONE',
        'interfaces': []
    },
    'CAE1': {
        'tag': 'cae',
        'dashboard_name': 'CAE1',
        'interfaces': []
    },
    'IC1': {
        'tag': ['ic1', 'peer-aggregate'],
        'dashboard_name': 'IC-1',
        'interfaces': []
    },
    'COPERNICUS': {
        'tag': ['copernicus', 'services', 'peer-aggregate'],
        'dashboard_name': 'COPERNICUS',
        'group_by': 'location',
        'interfaces': []
    },
    'ANA': {
        'tag': ['ana', 'peer-aggregate'],
        'dashboard_name': 'ANA',
        'interfaces': []
    },
}


def provision_folder(token_request, folder_name, dash, services,
                     ds_name, excluded_dashboards):
    """
    Function to provision dashboards within a folder.

    :param token_request: TokenRequest object
    :param folder_name: Name of the folder to provision dashboards in
    :param dash: the dashboards to provision, with interface data to generate
    the dashboards from
    :param services: service data from reporting provider for service-based dashboards
    :param ds_name: the name of the datasource to query in the dashboard panels
    :param excluded_dashboards: list of dashboards to exclude from provisioning
    for the organisation

    :return: list of dashboard definitions for the created dashboards
    """

    if not isinstance(excluded_dashboards, (list, set)):
        excluded_dashboards = set()
    else:
        excluded_dashboards = set([s.lower() for s in excluded_dashboards])

    folder = find_folder(token_request, folder_name)
    tag = dash['tag']
    interfaces = list(
        filter(
            lambda x: x['dashboards_info'],
            dash['interfaces']
        )
    )

    # dashboard should include error panels
    errors = dash.get('errors', False)

    is_nren_beta = folder_name == "NREN Access BETA"  # needed for POL1-642 BETA
    is_nren = folder_name == "NREN Access"
    is_re_peer = folder_name == "RE Peer"
    is_service = 'service_type' in dash

    if is_nren:
        data = get_nren_interface_data_old(interfaces)
        dash_data = get_nren_dashboard_data(data, ds_name, tag)
    elif is_nren_beta:
        # needed for POL1-642 BETA
        data = get_nren_interface_data(
            services, interfaces, excluded_dashboards)
        dash_data = get_nren_dashboard_data(data, ds_name, tag)
    elif is_re_peer:
        data = get_re_peer_interface_data(interfaces)
        dash_data = get_re_peer_dashboard_data(data, ds_name, tag)
    elif is_service:
        data = get_service_data(dash['service_type'], services, interfaces, excluded_dashboards)
        dash_data = get_service_dashboard_data(data, ds_name, tag)
    else:
        data = get_interface_data(interfaces)
        dash_data = get_dashboard_data(
            data=data,
            datasource=ds_name,
            tag=tag,
            errors=errors)

    provisioned = []

    with ThreadPoolExecutor(max_workers=MAX_WORKERS) as executor:
        for dashboard in dash_data:
            if is_nren or is_nren_beta or is_re_peer or is_service:
                rendered = render_complex_dashboard(**dashboard)
            else:
                rendered = render_simple_dashboard(**dashboard)

            if rendered.get("title").lower() in excluded_dashboards:
                executor.submit(delete_dashboard, token_request, rendered, folder["id"])
                continue
            provisioned.append(executor.submit(create_dashboard, token_request,
                                               rendered, folder['id']))
    return [r.result() for r in provisioned]


def provision_aggregate(token_request, folder,
                        dash, ds_name):
    """
    Function to provision an aggregate dashboard within a folder.

    :param token_request: TokenRequest object
    :param folder: the folder to provision dashboards in
    :param dash: the dashboards to provision, with interface data to generate
    the dashboards from
    :param ds_name: the name of the datasource to query in the dashboard panels

    :return: dashboard definition for the created dashboard
    """

    name = dash['dashboard_name']
    tag = dash['tag']
    interfaces = dash['interfaces']
    group_field = dash.get('group_by', 'remote')
    data = get_aggregate_interface_data(interfaces, name, group_field)

    dashboard = get_aggregate_dashboard_data(
        f'Aggregate - {name}', data, ds_name, tag)

    rendered = render_simple_dashboard(**dashboard)
    return create_dashboard(token_request, rendered, folder['id'])


def is_excluded_folder(excluded_folders, folder_name):
    """
    Function to determine if a folder should be excluded from provisioning.

    :param excluded_folders: dict of excluded folders and dashboards
    within them, it looks like this:
    {
        "Aggregates": ["GWS UPSTREAMS", "IAS PEERS"],
        "IAS CUSTOMER": True,
        "IAS PRIVATE": True,
        "IAS PUBLIC": True,
        "IAS UPSTREAM": True,
        "GWS PHY Upstream": True,
        "EUMETSAT Multicast": True,
        "NREN Access BETA": True
    }

    If the value is True, the entire folder is excluded.

    If the value is a list, the list contains the names of the dashboards
    within the folder that should be excluded.

    The case of a boolean `True` value is handled by this function.

    The case of a list is handled at provision time by the
    excluded_folder_dashboards and provision_folder functions.

    :param folder_name: the name of the folder to check against the
    excluded_folders

    :return: True if the folder should be excluded, False otherwise
    """

    excluded = excluded_folders.get(folder_name, False)
    return isinstance(excluded, bool) and excluded


def excluded_folder_dashboards(org_config, folder_name):
    """
    Function to get the list of dashboards to exclude from provisioning
    for a given folder.

    If the folder is the NREN Access folder, the list of excluded NRENs
    is also added to the list of excluded dashboards.

    :param org_config: the organisation config
    :param folder_name: the name of the folder to check against the
    excluded_folders

    :return: list of dashboard names to exclude from provisioning for the
    organisation
    """

    excluded_folders = org_config.get('excluded_folders', {})
    excluded = excluded_folders.get(folder_name, [])
    # in is needed for POL1-642 BETA ('NREN Access BETA' folder)
    # revert to == 'NREN Access' when beta is over
    if 'NREN Access' in folder_name:
        excluded_nrens = org_config.get('excluded_nrens', [])
        excluded = list(set(excluded).union(set(excluded_nrens)))
    return excluded if isinstance(excluded, list) else []


def _interfaces_to_keep(interface, excluded_nrens):
    dash_info = interface.get('dashboards_info')
    if dash_info is None:
        logger.info(f'No "dashboards_info" for '
                    f'{interface["router"]}:{interface["name"]}')
        # throw it away
        return False
    dashboards = {nren['name'].lower() for nren in dash_info}
    is_lab_router = 'lab.office' in interface['router'].lower()
    should_keep = not (is_lab_router or any(
        nren.lower() in dashboards for nren in excluded_nrens))
    return should_keep


def _provision_interfaces(config, org_config, ds_name, token):
    """
    This function is used to provision most dashboards,
    overwriting existing ones.

    :param config: the application config
    :param org_config: the organisation config
    :param ds_name: the name of the datasource to query in the dashboards
    :param token: a token_request object
    :return: generator of dashboards that were created
    """

    interfaces = get_interfaces(config['inventory_provider'])
    services = fetch_services(config['reporting_provider'])

    excluded_nrens = org_config['excluded_nrens']
    excluded_folders = org_config.get('excluded_folders', {})

    relevant_interfaces = list(filter(lambda x: _interfaces_to_keep(x, excluded_nrens), interfaces))
    for interface in relevant_interfaces:
        interface['dashboards_info'] = list(filter(
            lambda x: x['name'] != '',
            interface['dashboards_info']
        ))

    # loop over interfaces and add them to the dashboard_name
    # -> folder mapping structure `dashboards` above, for convenience.

    for dash in DASHBOARDS:
        DASHBOARDS[dash]['interfaces'] = []

    for dash in AGG_DASHBOARDS:
        AGG_DASHBOARDS[dash]['interfaces'] = []

    for iface in relevant_interfaces:
        for dash_name in iface['dashboards']:

            # add interface to matched dashboard
            if dash_name in DASHBOARDS:
                ifaces = DASHBOARDS[dash_name]['interfaces']
                ifaces.append(iface)

                # TODO: remove all references to NRENBETA
                # when NREN service BETA is over (homedashboard/helpers)
                if dash_name == 'NREN':
                    ifaces = DASHBOARDS['NRENBETA']['interfaces']
                    ifaces.append(iface)

            # add to matched aggregate dashboard
            if dash_name in AGG_DASHBOARDS:
                ifaces = AGG_DASHBOARDS[dash_name]['interfaces']
                ifaces.append(iface)

    # provision dashboards and their folders
    with ThreadPoolExecutor(max_workers=MAX_WORKERS) as executor:
        provisioned = []
        for folder in DASHBOARDS.values():
            folder_name = folder['folder_name']

            # boolean True means entire folder excluded
            # if list, it is specific dashboard names not to provision
            # so is handled at provision time.
            if is_excluded_folder(excluded_folders, folder_name):
                executor.submit(
                    delete_folder, token, title=folder_name)
                continue

            logger.info(
                f'Provisioning {org_config["name"]}/{folder_name} dashboards')
            res = executor.submit(
                provision_folder, token,
                folder_name, folder, services, ds_name,
                excluded_folder_dashboards(org_config, folder_name))
            provisioned.append(res)

        for result in provisioned:
            folder = result.result()
            if folder is None:
                continue
            yield from folder


def _provision_gws_indirect(config, org_config, ds_name, token):
    """
    This function is used to provision GWS Indirect dashboards,
    overwriting existing ones.

    :param config: the application config
    :param org_config: the organisation config
    :param ds_name: the name of the datasource to query in the dashboards
    :param token: a token_request object
    :return: generator of futures of dashboards that were created
    """

    logger.info('Provisioning GWS Indirect dashboards')
    folder_name = 'GWS Indirect'
    excluded_folders = org_config.get('excluded_folders', {})
    if is_excluded_folder(excluded_folders, folder_name):
        # don't provision GWS Direct folder
        delete_folder(token, title=folder_name)
    else:
        folder = find_folder(token, folder_name)
        with ThreadPoolExecutor(max_workers=MAX_WORKERS) as executor:
            gws_indirect_data = get_gws_indirect(
                config['inventory_provider'])
            provisioned = []
            dashes = generate_indirect(gws_indirect_data, ds_name)
            for dashboard in dashes:
                rendered = render_simple_dashboard(**dashboard)
                provisioned.append(executor.submit(create_dashboard,
                                                   token,
                                                   rendered, folder['id']))

            yield from provisioned


def _provision_gws_direct(config, org_config, ds_name, token):
    """
    This function is used to provision GWS Direct dashboards,
    overwriting existing ones.

    :param config: the application config
    :param org_config: the organisation config
    :param ds_name: the name of the datasource to query in the dashboards
    :param token: a token_request object
    :return: generator of futures of dashboards that were created
    """

    logger.info('Provisioning GWS Direct dashboards')
    folder_name = 'GWS Direct'
    excluded_folders = org_config.get('excluded_folders', {})
    if is_excluded_folder(excluded_folders, folder_name):
        # don't provision GWS Direct folder
        delete_folder(token, title=folder_name)
    else:
        folder = find_folder(token, folder_name)
        if not folder:
            logger.error(f'Folder {folder_name} not found')
        with ThreadPoolExecutor(max_workers=MAX_WORKERS) as executor:
            gws_data = get_gws_direct(config['inventory_provider'])
            provisioned = []

            for dashboard in generate_gws(gws_data, ds_name):
                rendered = render_simple_dashboard(**dashboard)
                provisioned.append(executor.submit(create_dashboard,
                                                   token,
                                                   rendered, folder['id']))

            yield from provisioned


def _provision_eumetsat_multicast(config, org_config, ds_name, token):
    """
    This function is used to provision EUMETSAT Multicast dashboards,
    overwriting existing ones.

    :param config: the application config
    :param org_config: the organisation config
    :param ds_name: the name of the datasource to query in the dashboards
    :param token: a token_request object
    :return: generator of futures of dashboards that were created
    """

    logger.info('Provisioning EUMETSAT Multicast dashboards')
    folder_name = 'EUMETSAT Multicast'
    excluded_folders = org_config.get('excluded_folders', {})
    if is_excluded_folder(excluded_folders, folder_name):
        # don't provision EUMETSAT Multicast folder
        delete_folder(token, title=folder_name)
    else:
        folder = find_folder(token, folder_name)
        with ThreadPoolExecutor(max_workers=MAX_WORKERS) as executor:
            subscriptions = get_eumetsat_multicast_subscriptions(
                config['inventory_provider'])
            provisioned = []

            for dashboard in generate_eumetsat_multicast(
                    subscriptions, ds_name):
                rendered = render_simple_dashboard(**dashboard)
                provisioned.append(
                    executor.submit(
                        create_dashboard,
                        token,
                        rendered,
                        folder['id']))

            yield from provisioned


def _provision_aggregates(config, org_config, ds_name, token):
    """
    This function is used to provision Aggregate dashboards,
    overwriting existing ones.

    :param config: the application config
    :param org_config: the organisation config
    :param ds_name: the name of the datasource to query in the dashboards
    :param token: a token_request object
    :return: generator of futures of dashboards that were created
    """

    excluded_folders = org_config.get('excluded_folders', {})
    folder_name = 'Aggregates'
    if is_excluded_folder(excluded_folders, folder_name):
        # don't provision aggregate folder
        delete_folder(token, title=folder_name)
    else:
        with ThreadPoolExecutor(max_workers=MAX_WORKERS) as executor:
            provisioned = []
            agg_folder = find_folder(token, folder_name)
            for dash in AGG_DASHBOARDS.values():
                excluded_dashboards = excluded_folder_dashboards(
                    org_config, folder_name)
                if dash['dashboard_name'] in excluded_dashboards:
                    dash_name = {
                        'title': f'Aggregate - {dash["dashboard_name"]}'}
                    executor.submit(delete_dashboard,
                                    token, dash_name,
                                    agg_folder['id'])
                    continue
                logger.info(f'Provisioning {org_config["name"]}' +
                            f'/Aggregate {dash["dashboard_name"]} dashboards')
                res = executor.submit(
                    provision_aggregate, token,
                    agg_folder, dash, ds_name)
                provisioned.append(res)

            yield from provisioned


def _provision_service_dashboards(config, org_config, ds_name, token):
    """
    This function is used to provision service-specific dashboards,
    overwriting existing ones.

    :param config: the application config
    :param org_config: the organisation config
    :param ds_name: the name of the datasource to query in the dashboards
    :param token: a token_request object
    :return: generator of UIDs of dashboards that were created
    """
    services = fetch_services(config['reporting_provider'])

    excluded_folders = org_config.get('excluded_folders', {})

    logger.info('Provisioning service-specific dashboards')

    # loop over service dashboards and get service types we care about
    dash_service_types = {SERVICE_DASHBOARDS[dash]['service_type']: dash for dash in SERVICE_DASHBOARDS}
    # loop over services and append to dashboards
    for service in services:
        if service['service_type'] in dash_service_types:
            dash = dash_service_types[service['service_type']]
            svcs = SERVICE_DASHBOARDS[dash]['services']
            svcs.append(service)

    # provision dashboards and their folders
    with ThreadPoolExecutor(max_workers=MAX_WORKERS) as executor:
        provisioned = []
        for folder in SERVICE_DASHBOARDS.values():
            folder_name = folder['folder_name']

            # boolean True means entire folder excluded
            # if list, it is specific dashboard names not to provision
            # so is handled at provision time.
            if is_excluded_folder(excluded_folders, folder_name):
                executor.submit(
                    delete_folder, token, title=folder_name)
                continue

            logger.info(
                f'Provisioning {org_config["name"]}/{folder_name} dashboards')
            res = executor.submit(
                provision_folder, token,
                folder_name, folder, services, ds_name,
                excluded_folder_dashboards(org_config, folder_name))
            provisioned.append(res)

        for result in provisioned:
            folder = result.result()
            if folder is None:
                continue
            yield from folder


def _provision_static_dashboards(config, org_config, ds_name, token):
    """
    This function is used to provision static dashboards from json files,
    overwriting existing ones.

    :param config: unused
    :param org_config: the organisation config
    :param ds_name: unused
    :param token: a token_request object
    :return: generator of UIDs of dashboards that were created
    """

    # Statically defined dashboards from json files
    excluded_dashboards = org_config.get('excluded_dashboards', [])
    logger.info('Provisioning static dashboards')
    for dashboard in get_dashboard_definitions():
        if dashboard['title'] not in excluded_dashboards:
            res = create_dashboard(token, dashboard)
            if res:
                # yield a fake dashboard dict
                # ... only the 'uid' element is referenced
                yield {'uid': res.get('uid')}
        else:
            delete_dashboard(token, dashboard)

    # Home dashboard is always called "Home"
    # Make sure it's set for the organization
    logger.info('Configuring Home dashboard')
    set_home_dashboard(token, is_staff=org_config['name'] == 'GÉANT Staff')

    yield {'uid': 'home'}


def _get_ignored_dashboards(config, org_config, token):
    """
    This function is used to get a list of dashboards that should not be
    touched by the provisioning process.

    :param config: the application config
    :param org_config: the organisation config
    :param token: a token_request object

    :return: generator of UIDs of dashboards that should not be touched
    """

    ignored_folders = config.get('ignored_folders', [])
    for name in ignored_folders:
        logger.info(
            'Ignoring dashboards under '
            f'the folder {org_config["name"]}/{name}')
        folder = find_folder(token, name, create=False)
        if folder is None:
            continue
        to_ignore = list_dashboards(token, folder_id=folder['id'])

        if not to_ignore:
            continue

        for dash in to_ignore:
            # return a hard-coded fake dashboard dict
            # ... only the 'uid' element is referenced
            yield {'uid': dash['uid']}  # could just yield dash


def _provision_datasource(config, token):
    """
    This function is used to provision the datasource from the config.

    :param config: the application config
    :param token: a token_request object

    :return: the datasource config
    """

    datasource = config.get('datasources').get('influxdb')

    # Provision missing data sources
    if not datasource_exists(token, datasource):
        create_datasource(token, datasource)

    return datasource


def _provision_orgs(config):
    """
    This function is used to provision the organisations from the config.

    :param config: the application config

    :return: a list of all organisations
    """

    request = AdminRequest(**config)
    all_orgs = get_organizations(request)

    orgs_to_provision = config.get('organizations', DEFAULT_ORGANIZATIONS)

    missing = (org['name'] for org in orgs_to_provision
               if org['name'] not in [org['name'] for org in all_orgs])

    for org_name in missing:
        org_data = create_organization(request, org_name)
        all_orgs.append(org_data)

    return all_orgs


def provision_maybe(config):
    """
    This function writes a timestamp and whether the provisioning process
    is running to a state file, and then runs the provisioning process.

    The boolean is used to determine if the provisioning process
    should run from other worker processes using the shared state file.

    The timestamp is written as a safety measure to ensure that the
    provisioning process is not stuck in case a worker process crashes
    mid-provisioning.

    This behaviour is disabled in development mode.

    :param config: the application config

    :return:
    """
    with open(STATE_PATH, 'r+') as f:
        def write_timestamp(timestamp, provisioning):
            f.seek(0)
            f.write(json.dumps(
                {'timestamp': timestamp, 'provisioning': provisioning}))
            f.truncate()

        try:
            # don't conditionally provision in dev
            provisioning = os.environ.get('FLASK_ENV') != 'development'
            now = datetime.datetime.now()
            write_timestamp(now.timestamp(), provisioning)
            provision(config)
        finally:
            now = datetime.datetime.now()
            write_timestamp(now.timestamp(), False)


def provision(config, raise_exceptions=False):
    """
    The entrypoint for the provisioning process.

    Provisions organisations, datasources, and dashboards within Grafana.

    Removes dashboards and folders not controlled by the provisioning process.


    :param config: the application config

    :return:
    """

    start = time.time()
    tokens = []
    all_orgs = _provision_orgs(config)
    request = AdminRequest(**config)
    delete_expired_api_tokens(request)

    def _find_org_config(org):
        orgs_to_provision = config.get('organizations', DEFAULT_ORGANIZATIONS)
        try:
            return next(
                o for o in orgs_to_provision if o['name'] == org['name'])
        except StopIteration:
            logger.info(
                f'Org {org["name"]} does not have valid configuration.')
            return None

    for org in all_orgs:
        try:
            org_id = org['id']

            logger.info(
                f'--- Provisioning org {org["name"]} (ID #{org_id}) ---')
            org_config = _find_org_config(org)
            if not org_config:
                # message logged from _find_org_config
                continue

            token = create_api_token(request, org_id)
            token_request = TokenRequest(token=token['key'], **config)
            tokens.append((org_id, token['id']))
            logger.debug(tokens)

            all_original_dashboards = list_dashboards(token_request)
            all_original_dashboard_uids = {
                d['uid'] for d in all_original_dashboards}

            datasource = _provision_datasource(config, token_request)
            ds_name = datasource.get('name', 'PollerInfluxDB')

            managed_dashboards = itertools.chain(
                _provision_interfaces(
                    config, org_config, ds_name, token_request),
                _provision_gws_indirect(
                    config, org_config, ds_name, token_request),
                _provision_gws_direct(
                    config, org_config, ds_name, token_request),
                _provision_eumetsat_multicast(
                    config, org_config, ds_name, token_request),
                _provision_aggregates(
                    config, org_config, ds_name, token_request),
                _provision_service_dashboards(
                    config, org_config, ds_name, token_request),
                _provision_static_dashboards(
                    config, org_config, ds_name, token_request),
                _get_ignored_dashboards(
                    config, org_config, token_request)
            )

            managed_dashboard_uids = set()
            for dashboard in managed_dashboards:
                if isinstance(dashboard, Future):
                    dashboard = dashboard.result()
                if dashboard is None:
                    continue
                managed_dashboard_uids.add(dashboard['uid'])

            for uid in all_original_dashboard_uids - managed_dashboard_uids:
                # delete unmanaged dashboards
                logger.info(f'Deleting stale dashboard with UID {uid}')
                delete_dashboard(token_request, {'uid': uid})

            folders_to_keep = {
                # General is a base folder present in Grafana
                'General',
                # other folders, created outside of the DASHBOARDS list
                'GWS Indirect',
                'GWS Direct',
                'Aggregates',
                'EUMETSAT Multicast'
            }
            folders_to_keep.update({dash['folder_name']
                                    for dash in DASHBOARDS.values()})
            folders_to_keep.update({dash['folder_name']
                                    for dash in SERVICE_DASHBOARDS.values()})

            ignored_folders = config.get('ignored_folders', [])
            folders_to_keep.update(ignored_folders)

            delete_unknown_folders(token_request, folders_to_keep)
            delete_api_token(request, token['id'], org_id=org_id)
        except Exception:
            logger.exception(f'Error when provisioning org {org["name"]}')
            if raise_exceptions:
                raise
            break

    logger.info(f'Time to complete: {time.time() - start}')

    return all_orgs