Skip to content
Snippets Groups Projects
poller.py 53.93 KiB
"""
BRIAN support Endpoints
=========================

These endpoints are intended for use by BRIAN.

.. contents:: :local:

/poller/interfaces</hostname>
---------------------------------

.. autofunction:: inventory_provider.routes.poller.interfaces


/poller/speeds</hostname>
---------------------------------

.. autofunction:: inventory_provider.routes.poller.interface_speeds


/poller/eumetsat-multicast
---------------------------------

.. autofunction:: inventory_provider.routes.poller.eumetsat_multicast


/poller/gws/direct
---------------------------------

.. autofunction:: inventory_provider.routes.poller.gws_direct


/poller/gws/direct-config
---------------------------------

.. autofunction:: inventory_provider.routes.poller.gws_direct_config


/poller/gws/indirect
---------------------------------

.. autofunction:: inventory_provider.routes.poller.gws_indirect


/poller/services</service-type>
---------------------------------

.. autofunction:: inventory_provider.routes.poller.get_services


/poller/service-types
---------------------------------

.. autofunction:: inventory_provider.routes.poller.get_service_types


/poller/error-report-interfaces</hostname>
------------------------------------------

.. autofunction:: inventory_provider.routes.poller.error_report_interfaces


support method: _get_dashboards
---------------------------------

.. autofunction:: inventory_provider.routes.poller._get_dashboards


"""
from collections import defaultdict
from enum import Enum, auto
import itertools
import json
import logging
import re
from functools import partial
from typing import Dict

from flask import Blueprint, Response, current_app, request, jsonify

from inventory_provider import juniper
from inventory_provider.routes import common
from inventory_provider.tasks.common import ims_sorted_service_type_key
from inventory_provider.tasks import common as tasks_common
from inventory_provider.routes.classifier import get_ims_equipment_name, \
    get_ims_interface
from inventory_provider.routes.common import _ignore_cache_or_retrieve

logger = logging.getLogger(__name__)
routes = Blueprint('poller-support-routes', __name__)

Mb = 1 << 20
Gb = 1 << 30
OC = Mb * 51.84


class INTERFACE_TYPES(Enum):
    UNKNOWN = auto()
    LOGICAL = auto()
    PHYSICAL = auto()
    AGGREGATE = auto()


class BRIAN_DASHBOARDS(Enum):
    CLS = auto()
    RE_PEER = auto()
    RE_CUST = auto()
    GEANTOPEN = auto()
    GCS = auto()
    L2_CIRCUIT = auto()
    LHCONE_PEER = auto()
    LHCONE_CUST = auto()
    MDVPN_CUSTOMERS = auto()
    INFRASTRUCTURE_BACKBONE = auto()
    IAS_PRIVATE = auto()
    IAS_PUBLIC = auto()
    IAS_CUSTOMER = auto()
    IAS_UPSTREAM = auto()
    GWS_PHY_UPSTREAM = auto()
    GBS_10G = auto()

    # aggregate dashboards
    CLS_PEERS = auto()
    IAS_PEERS = auto()
    GWS_UPSTREAMS = auto()
    LHCONE = auto()
    CAE1 = auto()
    IC1 = auto()
    COPERNICUS = auto()
    ANA = auto()

    # NREN customer
    NREN = auto()


class PORT_TYPES(Enum):
    ACCESS = auto()
    SERVICE = auto()
    UNKNOWN = auto()


# only used in INTERFACE_LIST_SCHEMA and sphinx docs
_DASHBOARD_IDS = [d.name for d in list(BRIAN_DASHBOARDS)]

_PORT_TYPES = [t.name for t in list(PORT_TYPES)]

_INTERFACE_TYPES = [i.name for i in list(INTERFACE_TYPES)]

INTERFACE_LIST_SCHEMA = {
    '$schema': 'https://json-schema.org/draft-07/schema#',

    'definitions': {
        'service': {
            'type': 'object',
            'properties': {
                'id': {'type': 'integer'},
                'name': {'type': 'string'},
                'type': {'type': 'string'},
                'status': {'type': 'string'},
            },
            'required': ['id', 'name', 'type', 'status'],
            'additionalProperties': False
        },
        'db_info': {
            'type': 'object',
            'properties': {
                'name': {'type': 'string'},
                'interface_type': {'enum': _INTERFACE_TYPES}
            },
            'required': ['name', 'interface_type'],
            'additionalProperties': False
        },
        'interface': {
            'type': 'object',
            'properties': {
                'router': {'type': 'string'},
                'name': {'type': 'string'},
                'description': {'type': 'string'},
                'snmp-index': {
                    'type': 'integer',
                    'minimum': 1
                },
                'bundle': {
                    'type': 'array',
                    'items': {'type': 'string'}
                },
                'bundle-parents': {
                    'type': 'array',
                    'items': {'type': 'string'}
                },
                'circuits': {
                    'type': 'array',
                    'items': {'$ref': '#/definitions/service'}
                },
                'dashboards': {
                    'type': 'array',
                    'items': {'enum': _DASHBOARD_IDS}
                },
                'dashboard_info': {
                    '$ref': '#/definitions/db_info',
                },
                'dashboards_info': {
                    'type': 'array',
                    'items': {'$ref': '#/definitions/db_info'}
                },
                'port_type': {'enum': _PORT_TYPES}
            },
            'required': [
                'router', 'name', 'description',
                'snmp-index', 'bundle', 'bundle-parents',
                'circuits', 'dashboards', 'port_type'],
            'additionalProperties': False
        },
    },

    'type': 'array',
    'items': {'$ref': '#/definitions/interface'}
}

ERROR_REPORT_INTERFACE_LIST_SCHEMA = {
    '$schema': 'https://json-schema.org/draft-07/schema#',
    'definitions': {
        'interface': {
            'type': 'object',
            'properties': {
                'router': {'type': 'string'},
                'name': {'type': 'string'},
                'description': {'type': 'string'},
                'vendor': {'type': 'string', 'enum': ['juniper', 'nokia']}
            },
            'required': ['router', 'name', 'description', 'vendor'],
            'additionalProperties': False
        },
    },
    'type': 'array',
    'items': {'$ref': '#/definitions/interface'}
}

INTERFACE_SPEED_LIST_SCHEMA = {
    '$schema': 'https://json-schema.org/draft-07/schema#',

    'definitions': {
        'interface': {
            'type': 'object',
            'properties': {
                'router': {'type': 'string'},
                'name': {'type': 'string'},
                'speed': {'type': 'integer'}
            },
            'required': ['router', 'name', 'speed'],
            'additionalProperties': False
        },
    },

    'type': 'array',
    'items': {'$ref': '#/definitions/interface'}
}

MULTICAST_SUBSCRIPTION_LIST_SCHEMA = {
    '$schema': 'https://json-schema.org/draft-07/schema#',

    'definitions': {
        'ipv4-address': {
            'type': 'string',
            'pattern': r'^(\d+\.){3}\d+$'
        },
        'subscription': {
            'type': 'object',
            'properties': {
                'router': {'type': 'string'},
                'subscription': {'$ref': '#/definitions/ipv4-address'},
                'endpoint': {'$ref': '#/definitions/ipv4-address'},
                'oid': {
                    'type': 'string',
                    'pattern': r'^(\d+\.)*\d+$'
                },
                'community': {'type': 'string'}
            },
            'required': [
                'router', 'subscription', 'endpoint', 'oid', 'community'],
            'additionalProperties': False
        },
    },

    'type': 'array',
    'items': {'$ref': '#/definitions/subscription'}
}

GWS_DIRECT_DATA_SCHEMA = {
    '$schema': 'https://json-schema.org/draft-07/schema#',

    'definitions': {
        'oid': {
            'type': 'string',
            'pattern': r'^(\d+\.)*\d+$'
        },
        'snmp-v2': {
            'type': 'object',
            'properties': {
                'community': {'type': 'string'}
            },
            'required': ['community'],
            'additionalProperties': False
        },
        'snmp-v3-cred': {
            'type': 'object',
            'properties': {
                'protocol': {'enum': ['MD5', 'DES']},
                'password': {'type': 'string'}
            },
            'required': ['protocol', 'password'],
            'additionalProperties': False
        },
        'snmp-v3': {
            'type': 'object',
            'properties': {
                'sec-name': {'type': 'string'},
                'auth': {'$ref': '#/definitions/snmp-v3-cred'},
                'priv': {'$ref': '#/definitions/snmp-v3-cred'}
            },
            'required': ['sec-name'],
            'additionalProperties': False
        },
        'counter': {
            'type': 'object',
            'properties': {
                'field': {
                    'enum': [
                        'discards_in',
                        'discards_out',
                        'errors_in',
                        'errors_out',
                        'traffic_in',
                        'traffic_out'
                    ]
                },
                'oid': {'$ref': '#/definitions/oid'},
                'snmp': {
                    'oneOf': [
                        {'$ref': '#/definitions/snmp-v2'},
                        {'$ref': '#/definitions/snmp-v3'}
                    ]
                }
            },
            'required': ['field', 'oid', 'snmp'],
            'additionalProperties': False
        },
        'interface-counters': {
            'type': 'object',
            'properties': {
                'nren': {'type': 'string'},
                'isp': {
                    'type': 'string',
                    'enum': ['Cogent', 'Telia', 'CenturyLink']
                },
                'hostname': {'type': 'string'},
                'tag': {'type': 'string'},
                'counters': {
                    'type': 'array',
                    'items': {'$ref': '#/definitions/counter'},
                    'minItems': 1
                },
                'info': {'type': 'string'}
            },
            'required': [
                'nren', 'isp', 'hostname', 'tag', 'counters'],
            'additionalProperties': False
        }
    },

    'type': 'array',
    'items': {'$ref': '#/definitions/interface-counters'}
}

SERVICES_LIST_SCHEMA = {
    '$schema': 'https://json-schema.org/draft-07/schema#',

    'definitions': {
        'oid': {
            'type': 'string',
            'pattern': r'^(\d+\.)*\d+$'
        },
        'counter-field': {
            'type': 'object',
            'properties': {
                'field': {
                    'type': 'string',
                    'enum': ['egressOctets', 'ingressOctets']
                },
                'oid': {'$ref': '#/definitions/oid'}
            },
            'required': ['field', 'oid'],
            'additionalProperties': False
        },
        'snmp-info': {
            'type': 'object',
            'properties': {
                'ifIndex': {'type': 'integer'},
                'community': {'type': 'string'},
                'counters': {
                    'type': 'array',
                    'items': {'$ref': '#/definitions/counter-field'},
                    'minItems': 1
                }
            },
            'required': ['ifIndex', 'community'],
            'additionalProperties': False
        },
        'service': {
            'type': 'object',
            'properties': {
                'id': {'type': 'integer'},
                'name': {'type': 'string'},
                'customer': {'type': 'string'},
                'speed': {'type': 'integer'},
                'pop': {'type': 'string'},
                'hostname': {'type': 'string'},
                'interface': {'type': 'string'},
                'type': {'type': 'string'},
                'status': {'type': 'string'},
                'snmp': {'$ref': '#/definitions/snmp-info'}
            },
            'required': [
                'id', 'name', 'customer',
                'speed', 'pop', 'hostname',
                'interface', 'type', 'status'],
            'additionalProperties': False
        }
    },

    'type': 'array',
    'items': {'$ref': '#/definitions/service'}
}

STRING_LIST_SCHEMA = {
    '$schema': 'https://json-schema.org/draft-07/schema#',
    'type': 'array',
    'items': {'type': 'string'}
}


@routes.after_request
def after_request(resp):
    return common.after_request(resp)


def _get_dashboards(interface):
    """
    Yield enums from BRIAN_DASHBOARDS to indicate which dashboards
    this interface should be included in.

    cf. POL1-482

    Possible dashboard id's are:

    .. asjson::
       inventory_provider.routes.poller._DASHBOARD_IDS

    :param interface: a dict with keys like router, name, description
    :return: generator that yields enums from BRIAN_DASHBOARDS
    """

    router = interface.get('router', '').lower()
    ifc_name = interface.get('name', '')
    description = interface.get('description', '').strip()
    if 'SRV_L3VPN' in description and 'COPERNICUS' in description:
        yield BRIAN_DASHBOARDS.COPERNICUS
    if re.match(r'SRV_CLS\s', description):
        yield BRIAN_DASHBOARDS.CLS
    if re.match(r'SRV_CLS PRIVATE\s', description):
        yield BRIAN_DASHBOARDS.CLS_PEERS
    if re.match(r'SRV_IAS PUBLIC\s', description):
        yield BRIAN_DASHBOARDS.IAS_PUBLIC
        yield BRIAN_DASHBOARDS.IAS_PEERS
    if re.match(r'SRV_IAS PRIVATE\s', description):
        yield BRIAN_DASHBOARDS.IAS_PRIVATE
        yield BRIAN_DASHBOARDS.IAS_PEERS
    if re.match(r'SRV_IAS CUSTOMER\s', description):
        yield BRIAN_DASHBOARDS.IAS_CUSTOMER
    if re.match(r'SRV_IAS UPSTREAM\s', description):
        yield BRIAN_DASHBOARDS.IAS_UPSTREAM
    if re.match(r'SRV_10GGBS CUSTOMER\s', description):
        yield BRIAN_DASHBOARDS.GBS_10G
    if re.match(r'(SRV_GLOBAL|SRV_L3VPN|LAG|PHY) RE_INTERCONNECT\s', description):
        yield BRIAN_DASHBOARDS.RE_PEER
    if re.match(r'(PHY|LAG|SRV_GLOBAL) CUSTOMER\s', description):
        yield BRIAN_DASHBOARDS.RE_CUST
    if re.match('^SRV_GCS', description):
        yield BRIAN_DASHBOARDS.GCS
    if re.match(r'PHY CUSTOMER_GEO\s', description):
        yield BRIAN_DASHBOARDS.GEANTOPEN
    if 'SRV_L2CIRCUIT' in description:
        yield BRIAN_DASHBOARDS.L2_CIRCUIT
    if 'LHCONE' in description:
        if 'SRV_L3VPN RE' in description:
            yield BRIAN_DASHBOARDS.LHCONE_PEER
        if 'SRV_L3VPN CUSTOMER' in description:
            yield BRIAN_DASHBOARDS.LHCONE_CUST
    if re.match(r'SRV_L3VPN (CUSTOMER|RE_INTERCONNECT)\s', description):
        if '-LHCONE' in description:
            # `-LHCONE` can be preceded by tons of different stuff, so it's
            # simpler to check for this rather than a full regex
            yield BRIAN_DASHBOARDS.LHCONE
    if re.match(r'SRV_MDVPN CUSTOMER\s', description):
        yield BRIAN_DASHBOARDS.MDVPN_CUSTOMERS
    if re.match(r'(SRV_GLOBAL|LAG|PHY) INFRASTRUCTURE BACKBONE', description):
        yield BRIAN_DASHBOARDS.INFRASTRUCTURE_BACKBONE
    if router == 'mx1.lon.uk.geant.net' \
            and re.match(r'^ae12\.\d+$', ifc_name):
        yield BRIAN_DASHBOARDS.CAE1
    if router == 'rt1.mar.fr.geant.net' \
            and re.match(r'^ae12\.\d+$', ifc_name):
        yield BRIAN_DASHBOARDS.IC1
    if re.match(r'PHY UPSTREAM\s', description):
        yield BRIAN_DASHBOARDS.GWS_PHY_UPSTREAM
    regex = r'(PHY|LAG|(SRV_(GLOBAL|LHCONE|MDVPN|IAS|CLS|L3VPN))) CUSTOMER\s'
    if re.match(regex, description):
        yield BRIAN_DASHBOARDS.NREN
    if "GA-" in description and "ANA-" in description:
        yield BRIAN_DASHBOARDS.ANA


def _get_dashboard_data(ifc, customers):
    def _get_interface_type(description):
        if re.match(r'^PHY', description):
            return INTERFACE_TYPES.PHYSICAL
        if re.match(r'^SRV_', description):
            return INTERFACE_TYPES.LOGICAL
        if re.match(r'^LAG', description):
            return INTERFACE_TYPES.AGGREGATE

        return INTERFACE_TYPES.UNKNOWN

    description = ifc.get('description', '').strip()
    dashboards = ifc.get('dashboards', [])

    interface_type = _get_interface_type(description)

    if len(dashboards) == 0:
        return ifc

    def _get_customer_name(description):
        name = description.split(' ')
        if len(name) >= 3:
            return name[2].strip().upper()
        else:
            # if the description isn't properly formatted
            # use it as the name to make it obvious something is wrong
            return description

    def _get_backbone_name(description):
        name = description.split('|')
        if len(name) >= 2:
            name = name[1].strip()
            return name.replace('( ', '(')
        else:
            # if the description isn't properly formatted
            # use it as the name to make it obvious something is wrong
            return description

    if BRIAN_DASHBOARDS.INFRASTRUCTURE_BACKBONE.name in dashboards:
        names = [_get_backbone_name(description)]
    elif BRIAN_DASHBOARDS.GWS_PHY_UPSTREAM.name in dashboards:
        host = ifc['router']
        location = host.split('.')[1].upper()
        names = [f'{_get_customer_name(description)} - {location}']
    elif BRIAN_DASHBOARDS.L2_CIRCUIT.name in dashboards:
        # This will give names derived from the Interface Description
        # names = list(_get_l2_customer_names(description))
        # This will give first 2 names IMS
        names = [c['name'] for c in customers][:2]

    else:
        names = [_get_customer_name(description)]

    # if no customers found just return the original data
    if not names:
        return ifc

    # to maintain compatability with current brian dashboard manager we will
    # continue to return dashboard_info with the first customer name. We will
    # also return dashboards_info (note the plural of dashboards) with up to
    # two customers
    return {
        **ifc,
        'dashboard_info': {
            'name': names[0],
            'interface_type': interface_type.name
        },
        'dashboards_info': [{
            'name': name,
            'interface_type': interface_type.name
        } for name in set(names)]
    }


def _load_interface_bundles(config, hostname=None, use_next_redis=False):
    result = dict()

    def _load_docs(key_pattern):
        for doc in common.load_json_docs(
                config_params=config,
                key_pattern=key_pattern,
                num_threads=20,
                use_next_redis=use_next_redis):
            m = re.match(
                r'.*netconf-interface-bundles:([^:]+):(.+)', doc['key'])
            assert m

            router = m.group(1)
            interface = m.group(2)
            result.setdefault(router, dict())
            result[router][interface] = doc['value']

    base_key = 'netconf-interface-bundles'
    base_key_pattern = f'{base_key}:{hostname}:*' \
        if hostname else f'{base_key}:*'

    _load_docs(base_key_pattern)
    _load_docs(f'lab:{base_key_pattern}')

    return result


def _get_services_and_customers(config, hostname=None, use_next_redis=False):
    if hostname:
        hostname = get_ims_equipment_name(hostname)

    result = defaultdict(dict)
    key_pattern = f'ims:interface_services:{hostname}:*' \
        if hostname else 'ims:interface_services:*'

    for doc in common.load_json_docs(
            config_params=config,
            key_pattern=key_pattern,
            num_threads=20,
            use_next_redis=use_next_redis):
        cs = {
            'services': [],
            'customers': []
        }
        included_service_ids = set()
        for s in doc['value']:
            if s['id'] in included_service_ids:
                continue
            if s.get('port_type', '') == 'ab':
                continue
            included_service_ids.add(s['id'])

            cs['customers'].append({
                'name': s['customer'],
                'type': 'UNKNOWN'
            })
            for c in s.get('additional_customers', []):
                cs['customers'].append({
                    'name': c['name'],
                    'type': c.get('type', 'UNKNOWN')
                })
            if s['circuit_type'] == 'service':
                cs['services'].append({
                    'id': s['id'],
                    'name': s['name'],
                    'type': s['service_type'],
                    'status': s['status'],
                })
            result[s['equipment']][s['port']] = cs
    return result


def _load_netconf_docs(
        config, filter_pattern, use_next_redis=False):
    """
    yields dicts like:
        {
            'router': router hostname
            'netconf': loaded netconf xml doc
        }

    :param config: app config
    :param filter_pattern: search filter, including 'netconf:'
    :param use_next_redis: use next instead of current redis, if true
    :return: yields netconf docs, formatted as above
    """

    m = re.match(r'^(.*netconf:).+', filter_pattern)
    # TODO: probably better to not required netconf: to be passed in
    assert m  # sanity
    key_prefix_len = len(m.group(1))
    assert key_prefix_len >= len('netconf:')  # sanity

    for doc in common.load_xml_docs(
            config_params=config,
            key_pattern=filter_pattern,
            num_threads=10,
            use_next_redis=use_next_redis):
        yield {
            'router': doc['key'][key_prefix_len:],
            'netconf': doc['value']
        }


def _load_netconf_parsed_cache(config, cache_name, hostname=None, is_lab=False, use_next_redis=False):
    hostname_suffix = hostname if hostname else ''
    lab_prefix = 'lab:' if is_lab else ''
    filter_pattern = f'{lab_prefix}{cache_name}:{hostname_suffix}*'
    for doc in common.load_json_docs(
            config_params=config,
            key_pattern=filter_pattern,
            num_threads=20,
            use_next_redis=use_next_redis):
        m = re.match(fr'(lab:)?{cache_name}:(.+:.+)', doc['key'])
        if m:
            # preparse the key as interfaces can include : in the name
            key_parts = m.group(2).split(':')
            key = f'{key_parts[0]}-----{":".join(key_parts[1:])}'
            yield key, doc['value']


def _load_interfaces(
        config, hostname=None, no_lab=False, use_next_redis=False):
    """
    loads basic interface data for production & lab routers

    :param config:
    :param hostname:
    :param use_next_redis:
    :return:
    """

    def _load_netconf_caches(is_lab=False):

        interfaces = dict(_load_netconf_parsed_cache(config, 'netconf-interfaces',
                                                     hostname, is_lab, use_next_redis))
        interface_bundles = dict(_load_netconf_parsed_cache(config, 'netconf-interface-bundles',
                                                            hostname, is_lab, use_next_redis))

        for key, ifc in interfaces.items():
            if not ifc['description']:
                continue
            router, interface_name = key.split('-----')
            bundle = interface_bundles.get(key, [])

            yield {
                'router': router,
                'name': interface_name,
                'bundle': bundle,
                'bundle-parents': [],
                'description': ifc['description'],
                'circuits': []
            }

    yield from _load_netconf_caches()
    if not no_lab:
        logger.debug('lab')
        yield from _load_netconf_caches(is_lab=True)


def _add_speeds(interfaces):
    r = common.get_current_redis()
    all_netconf_interfaces = json.loads(r.get('netconf-interfaces:all')) or []

    netconf_interface_index = {}
    for nc_ifc in all_netconf_interfaces:
        if 'hostname' in nc_ifc and 'name' in nc_ifc:
            netconf_interface_index[f"{nc_ifc['hostname']}---{nc_ifc['name']}"] = nc_ifc

    for ifc in interfaces:
        nc_ifc = netconf_interface_index.get(f"{ifc['router']}---{ifc['name']}", {})
        if 'speed' in nc_ifc:
            ifc['speed'] = nc_ifc['speed']
        yield ifc


def _add_bundle_parents(interfaces, hostname=None):
    """
    generator that adds bundle-parents info to each interface.

    :param interfaces: result of _load_interfaces
    :param hostname: hostname or None for all
    :return: generator with bundle-parents populated in each element
    """

    def _get_base_name(name):
        return name.split('.')[0]

    bundles = _load_interface_bundles(
        current_app.config['INVENTORY_PROVIDER_CONFIG'], hostname)
    # create a quick look-up for interface details
    interface_index = {f"{ifc['router']}---{ifc['name']}": ifc for ifc in interfaces}

    for ifc in interfaces:
        router_bundle = bundles.get(ifc['router'], None)
        if router_bundle:
            base_ifc = _get_base_name(ifc['name'])
            bundle_parents = [interface_index.get(f"{ifc['router']}---{_get_base_name(bundle_ifc)}")
                              for bundle_ifc in router_bundle.get(base_ifc, [])]
            ifc['bundle-parents'] = bundle_parents
        yield ifc


def _get_port_type(description):
    rex = re.search(r'\$([a-zA-Z]+\-\d+)', description)
    if rex:
        sid = rex.group(1)
        if 'GA' in sid:
            return PORT_TYPES.ACCESS.name
        elif 'GS' in sid:
            return PORT_TYPES.SERVICE.name
    return PORT_TYPES.UNKNOWN.name


def load_interfaces_to_poll(config, hostname=None, no_lab=False, use_next_redis=False):
    def is_relevant(ifc):
        return not re.match(r"^(lt-|so-|dsc\.|fxp\d|lo\d).*", ifc["name"])

    basic_interfaces = list(
        filter(is_relevant, _load_interfaces(config, hostname, no_lab, use_next_redis))
    )
    bundles = _load_interface_bundles(config, hostname, use_next_redis)
    services_and_customers = \
        _get_services_and_customers(config, hostname, use_next_redis)
    snmp_indexes = common.load_snmp_indexes(config, hostname, use_next_redis)

    def _get_populated_interfaces(all_interfaces):
        if use_next_redis:
            r = tasks_common.get_next_redis(config)
        else:
            r = tasks_common.get_current_redis(config)
        for ifc in all_interfaces:
            router_snmp = snmp_indexes.get(ifc['router'], None)
            if router_snmp and ifc['name'] in router_snmp:
                ifc['snmp-index'] = router_snmp[ifc['name']]['index']

                router_bundle = bundles.get(ifc['router'], None)
                if router_bundle:
                    base_ifc = ifc['name'].split('.')[0]
                    ifc['bundle-parents'] = router_bundle.get(base_ifc, [])

                router_services_and_customers = services_and_customers.get(
                    get_ims_equipment_name(ifc['router'], r), {})
                ifc_services_and_customers = \
                    router_services_and_customers.get(
                        get_ims_interface(ifc['name']), {}
                    )

                if 'services' in ifc_services_and_customers \
                        and ifc_services_and_customers['services']:
                    ifc['circuits'] = ifc_services_and_customers['services']

                dashboards = _get_dashboards(ifc)
                ifc['dashboards'] = sorted([d.name for d in dashboards])

                ifc = _get_dashboard_data(
                    ifc, ifc_services_and_customers.get('customers', []))
                port_type = _get_port_type(ifc['description'])
                ifc['port_type'] = port_type
                yield ifc
            else:
                continue

    return _get_populated_interfaces(basic_interfaces)


@routes.route("/interfaces", methods=['GET'])
@routes.route('/interfaces/<hostname>', methods=['GET'])
@common.require_accepts_json
def interfaces(hostname=None):
    """
    Handler for `/poller/interfaces` and
    `/poller/interfaces/<hostname>`
    which returns information for either all interfaces
    or those on the requested hostname.

    The optional `no-lab` parameter omits lab routers
    if it's truthiness evaluates to True.

    The response is a list of information for all
    interfaces that should be polled, including service
    information and snmp information.

    .. asjson::
       inventory_provider.routes.poller.INTERFACE_LIST_SCHEMA

    :meth:`inventory_provider.routes.poller._get_services`
    is where dashboard mappings is handled.

    :param hostname: optional, if present should be a router hostname
    :return:
    """

    cache_key = f'classifier-cache:poller-interfaces:{hostname}' \
        if hostname else 'classifier-cache:poller-interfaces:all'

    no_lab = common.get_bool_request_arg('no-lab', False)
    if no_lab:
        cache_key = f'{cache_key}:no_lab'

    r = common.get_current_redis()

    result = _ignore_cache_or_retrieve(request, cache_key, r)

    if not result:
        result = list(load_interfaces_to_poll(
            current_app.config['INVENTORY_PROVIDER_CONFIG'], hostname, no_lab))
        if not result:
            return Response(
                response='no interfaces found',
                status=404,
                mimetype='text/html')

        result = json.dumps(result)
        # cache this data for the next call
        r.set(cache_key, result.encode('utf-8'))

    return Response(result, mimetype="application/json")


def get_netdash_equipment(config, use_next_redis=False) -> Dict[str, str]:
    """Get the netdash equipment mapping from redis."""
    if use_next_redis:
        r = tasks_common.get_next_redis(config)
    else:
        r = tasks_common.get_current_redis(config)
    return json.loads(r.get('netdash').decode('utf-8'))


def load_error_report_interfaces(
    config, hostname=None, use_next_redis=False
):
    interfaces = _load_interfaces(config, hostname, use_next_redis=use_next_redis)
    netdash_equipment = get_netdash_equipment(config, use_next_redis)

    def filter_interface(interface: dict):
        return all(
                (
                    "phy" in interface["description"].lower(),
                    "spare" not in interface["description"].lower(),
                    "non-operational" not in interface["description"].lower(),
                    "reserved" not in interface["description"].lower(),
                    "test" not in interface["description"].lower(),
                    "dsc." not in interface["name"].lower(),
                    "fxp" not in interface["name"].lower(),
                )
            )

    def transform_interface(interface: dict) -> Dict:
        return {
            "router": interface["router"],
            "name": interface["name"],
            "description": interface["description"],
            "vendor": netdash_equipment.get(interface["router"]),
        }

    return sorted(
        map(transform_interface, filter(filter_interface, interfaces)),
        key=lambda i: (i["router"], i["name"]),
    )


@routes.route("/error-report-interfaces", methods=['GET'])
@routes.route('/error-report-interfaces/<hostname>', methods=['GET'])
@common.require_accepts_json
def error_report_interfaces(hostname=None):
    """
    Handler for `/poller/error-report-interfaces` and
    `/poller/error-report-interfaces/<hostname>`
    which returns information for either all interfaces
    or those on the requested hostname.

    The optional `no-lab` parameter omits lab routers
    if it's truthiness evaluates to True.

    The response is a list of information for all
    interfaces that should be included in the neteng error report
    and includes vendor information (either juniper or nokia)

    .. asjson::
       inventory_provider.routes.poller.ERROR_REPORT_INTERFACE_LIST_SCHEMA

    :param hostname: optional, if present should be a router hostname
    :return:
    """

    suffix = hostname or "all"
    cache_key = f'classifier-cache:error-report-interfaces:{suffix}'

    r = common.get_current_redis()

    result = _ignore_cache_or_retrieve(request, cache_key, r)

    if not result:
        interfaces = load_error_report_interfaces(
            current_app.config['INVENTORY_PROVIDER_CONFIG'], hostname
        )
        result = json.dumps(interfaces).encode('utf-8')
        # cache this data for the next call
        r.set(cache_key, result)

    if not result or result == b'[]':
        return Response(
            response='no interfaces found',
            status=404,
            mimetype='text/plain'
        )

    return Response(result, mimetype="application/json")


def interface_speed(ifc):
    """
    Return the maximum bits per second expected for the given interface.

    cf. https://www.juniper.net/documentation/us/en/software/
               vmx/vmx-getting-started/topics/task/
               vmx-chassis-interface-type-configuring.html

    :param ifc:
    :return: an integer bits per second
    """

    def _name_to_speed(ifc_name):
        if ifc_name.startswith('ge'):
            return Gb
        if ifc_name.startswith('xe'):
            return 10 * Gb
        if ifc_name.startswith('et'):
            return 400 * Gb
        logger.warning(f'unrecognized interface name: {ifc_name}')
        return -1

    def _get_speed(ifc):

        rate_conversions = {
            "mbps": Mb,
            "gbps": Gb

        }

        if "speed" in ifc:
            speed = ifc["speed"]
            rate_match = re.match(r"(\d+)(.+)", speed)
            if rate_match:
                value = int(rate_match.group(1))
                rate = rate_match.group(2).strip().lower()
                if rate in rate_conversions:
                    return int(value * rate_conversions[rate])
                else:
                    logger.warning(f'unrecognised rate: {rate}, using _name_to_speed fallback')
                    return _name_to_speed(ifc['name'])
            else:
                oc_match = re.match(r"OC(\d+)", speed)
                if oc_match:
                    value = int(oc_match.group(1))
                    return int(value * OC)
                else:
                    logger.warning(f'unrecognised speed: {speed}, using _name_to_speed fallback')
                    return _name_to_speed(ifc['name'])
        else:
            logger.warning('no speed data for interface, using _name_to_speed fallback')
            return _name_to_speed(ifc['name'])

    if ifc['bundle-parents']:
        if not ifc['name'].startswith('ae'):
            logger.warning(
                f'ifc has bundle-parents, but name is {ifc["name"]}')
        return int(sum(_get_speed(parent_ifc) for parent_ifc in ifc['bundle-parents']))

    return int(_get_speed(ifc))


def _load_interfaces_and_speeds(hostname=None):
    """
    prepares the result of a call to /speeds

    :param hostname: hostname or None for all
    :return: generator yielding interface elements
    """
    no_lab = common.get_bool_request_arg('no-lab', False)
    basic_interfaces = _load_interfaces(
        current_app.config['INVENTORY_PROVIDER_CONFIG'],
        hostname,
        no_lab=no_lab)
    basic_interfaces_with_speeds = _add_speeds(basic_interfaces)
    with_bundles = _add_bundle_parents(list(basic_interfaces_with_speeds), hostname)

    def _result_ifc(ifc):
        return {
            'router': ifc['router'],
            'name': ifc['name'],
            'speed': interface_speed(ifc)
        }

    return map(_result_ifc, with_bundles)


@routes.route("/speeds", methods=['GET'])
@routes.route('/speeds/<hostname>', methods=['GET'])
@common.require_accepts_json
def interface_speeds(hostname=None):
    """
    Handler for `/poller/speeds` and
    `/poller/speeds/<hostname>`
    which returns information for either all interfaces
    or those on the requested hostname.

    The response is a list of maximum speed information (in bits
    per second) for all known interfaces.

    *speed <= 0 means the max interface speed can't be determined*

    .. asjson::
       inventory_provider.routes.poller.INTERFACE_SPEED_LIST_SCHEMA

    :param hostname: optional, if present should be a router hostname
    :return:
    """

    cache_key = f'classifier-cache:poller-interface-speeds:{hostname}' \
        if hostname else 'classifier-cache:poller-interface-speeds:all'

    r = common.get_current_redis()

    result = _ignore_cache_or_retrieve(request, cache_key, r)

    if not result:
        result = list(_load_interfaces_and_speeds(hostname))
        if not result:
            return Response(
                response='no interfaces found',
                status=404,
                mimetype='text/html')

        result = json.dumps(result)
        # cache this data for the next call
        r.set(cache_key, result.encode('utf-8'))

    return Response(result, mimetype="application/json")


def _load_community_strings(base_key_pattern):
    for doc in _load_netconf_docs(
            config=current_app.config['INVENTORY_PROVIDER_CONFIG'],
            filter_pattern=base_key_pattern):
        community = juniper.snmp_community_string(doc['netconf'])
        if not community:
            yield {
                'router': doc['router'],
                'error':
                    f'error extracting community string for {doc["router"]}'
            }
        else:
            yield {
                'router': doc['router'],
                'community': community
            }


@routes.route('/eumetsat-multicast', methods=['GET'])
@routes.route('/eumetsat-multicast/<hostname>', methods=['GET'])
@common.require_accepts_json
def eumetsat_multicast(hostname=None):
    """
    Handler for `/poller/eumetsat-multicast</hostname>` which returns
    information about multicast subscriptions.

    The hostname parameter is optional.  If it is present, only hostnames
    matching `hostname*` are returned.  If not present, data for all
    `mx*` routers is returned.

    The response is a list of oid/router/community structures that all
    all subscription octet counters to be polled.

    .. asjson::
       inventory_provider.routes.poller.MULTICAST_SUBSCRIPTION_LIST_SCHEMA

    This method returns essentially hard-coded data,
    based on the information in POL1-395.

    :return:
    """

    SUBSCRIPTIONS = [{
        'subscription': f'232.223.222.{idx}',
        'endpoint': '193.17.9.3',
    } for idx in range(1, 73)]

    SUBSCRIPTIONS.append(
        {'subscription': '232.223.223.1', 'endpoint': '193.17.9.7'})
    SUBSCRIPTIONS.append(
        {'subscription': '232.223.223.22', 'endpoint': '193.17.9.7'})

    def _oid(sub):
        return ('1.3.6.1.2.1.83.1.1.2.1.16'
                f'.{sub["subscription"]}.{sub["endpoint"]}'
                '.255.255.255.255')

    r = common.get_current_redis()

    cache_key = 'classifier-cache:poller-eumetsat-multicast'
    if hostname:
        cache_key = f'{cache_key}:{hostname}'

    result = r.get(cache_key)
    if result:
        result = result.decode('utf-8')
    else:

        def _multicast_oids(router_info):

            def _rsp_element(sub):
                result = {
                    'router': router_info['router'],
                    'oid': _oid(sub),
                    'community': router_info['community']
                }
                result.update(sub)
                return result

            yield from map(_rsp_element, SUBSCRIPTIONS)

        routers = list(_load_community_strings(
            base_key_pattern=f'netconf:{hostname}*'
            if hostname else 'netconf:mx*'))
        errors = list(filter(lambda x: 'error' in x, routers))
        if errors:
            errors = [e['error'] for e in errors]
            return Response(
                response=', '.join(errors),
                status=403,  # forbidden
                mimetype='text/html')

        assert all('community' in r for r in routers)  # sanity

        result = list(map(_multicast_oids, routers))
        result = itertools.chain(*result)
        result = list(result)
        if not result:
            target = hostname or 'any routers!'
            return Response(
                response=f'no multicast config for {target}',
                status=404,
                mimetype='text/html')

        result = json.dumps(result)
        # cache this data for the next call
        r.set(cache_key, result.encode('utf-8'))

    return Response(result, mimetype="application/json")


@routes.route("/gws/direct", methods=['GET'])
@common.require_accepts_json
def gws_direct():
    """
    Handler for `/poller/gws/direct` which returns required for polling
     customer equipment counters for ISP connetions.

    The response is a list of nren/isp/counter structures that must be
    polled.

    .. asjson::
       inventory_provider.routes.poller.GWS_DIRECT_DATA_SCHEMA

    WARNING: interface tags in the `gws-direct` section of the config data
    should be unique for each nren/isp combination.  i.e. if there
    are multiple community strings in use for a particular host, then please
    keep the interface tags unique.

    :return:
    """

    cache_key = 'classifier-cache:gws-direct'
    r = common.get_current_redis()

    result = r.get(cache_key)
    if result:
        result = result.decode('utf-8')
    else:

        def _interfaces():
            config_params = current_app.config['INVENTORY_PROVIDER_CONFIG']
            for nren_isp in config_params['gws-direct']:
                for host in nren_isp['hosts']:

                    snmp_params = {}
                    if 'community' in host:
                        # (snmp v2)
                        # sanity (already guaranteed by schema check)
                        assert 'sec-name' not in host
                        snmp_params['community'] = host['community']
                    else:
                        # (snmp v3)
                        # sanity (already guaranteed by schema check)
                        assert 'sec-name' in host
                        snmp_params['sec-name'] = host['sec-name']
                        if 'auth' in host:
                            snmp_params['auth'] = host['auth']
                            if 'priv' in host:
                                snmp_params['priv'] = host['priv']
                    for ifc in host['interfaces']:

                        ifc_data = {
                            'nren': nren_isp['nren'],
                            'isp': nren_isp['isp'],
                            'hostname': host['hostname'],
                            'tag': ifc['tag'],
                            'counters': [
                                {
                                    'field': k,
                                    'oid': v,
                                    'snmp': snmp_params
                                }
                                for k, v in ifc['counters'].items()]
                        }
                        if 'info' in ifc:
                            ifc_data['info'] = ifc['info']
                        yield ifc_data

        result = json.dumps(list(_interfaces()))

        # cache this data for the next call
        r.set(cache_key, result.encode('utf-8'))

    return Response(result, mimetype="application/json")


# cf. https://gitlab.geant.net/puppet-apps/cacti/-/blob/production/files/scripts/juniper-firewall-dws.pl  # noqa: E501

JNX_DCU_STATS_BYTES_OID = '1.3.6.1.4.1.2636.3.6.2.1.5'
JNX_FW_COUNTER_BYTES_OID = '1.3.6.1.4.1.2636.3.5.2.1.5'

JNX_ADDRESS_FAMILY = {
    'ipv4': 1,
    'ipv6': 2
}

JNX_FW_COUNTER_TYPE = {
    'other': 1,
    'counter': 2,
    'policer': 3
}


def _str2oid(s):
    chars = '.'.join(str(ord(c)) for c in s)
    return f'{len(s)}.{chars}'


def _jnx_dcu_byte_count_oid(
        ifIndex,
        class_name='dws-in',
        address_family=JNX_ADDRESS_FAMILY['ipv4']):
    # sanity checks (in case of programming errors)
    assert isinstance(ifIndex, int)
    assert isinstance(class_name, str)
    assert isinstance(address_family, int)
    return '.'.join([
        JNX_DCU_STATS_BYTES_OID,
        str(ifIndex),
        str(address_family),
        _str2oid(class_name)
    ])


def _jnx_fw_counter_bytes_oid(
        customer,
        interface_name,
        filter_name=None,
        counter_name=None):
    # sanity checks (in case of programming errors)
    assert isinstance(customer, str)
    assert isinstance(interface_name, str)
    assert filter_name is None or isinstance(filter_name, str)
    assert counter_name is None or isinstance(counter_name, str)

    if filter_name is None:
        filter_name = f'nren_IAS_{customer}_OUT-{interface_name}-o'
    if counter_name is None:
        counter_name = f'DWS-out-{interface_name}-o'

    return '.'.join([
        JNX_FW_COUNTER_BYTES_OID,
        _str2oid(filter_name),
        _str2oid(counter_name),
        str(JNX_FW_COUNTER_TYPE['counter'])
    ])


def _get_services_internal(service_type=None):
    """
    Performs the lookup and caching done for calls to
    `/poller/services</service-type>`

    This is a separate private utility so that it can be called by
    :meth:`inventory_provider.routes.poller.get_services`
    and :meth:`inventory_provider.routes.poller.get_service_types`

    The response will be formatted according to the following schema:

    .. asjson::
       inventory_provider.routes.poller.SERVICES_LIST_SCHEMA

    :param service_type: a service type, or None to return all
    :return: service list, json-serialized to a string
    """

    return_all = common.get_bool_request_arg('all', True)
    include_snmp = common.get_bool_request_arg('snmp', False)

    def _services():
        key_pattern = f'ims:services:{service_type}:*' \
            if service_type else 'ims:services:*'

        for doc in common.load_json_docs(
                config_params=current_app.config['INVENTORY_PROVIDER_CONFIG'],
                key_pattern=key_pattern,
                num_threads=20):
            yield doc['value']

    def _add_snmp(s, all_snmp_info):
        snmp_interfaces = all_snmp_info.get(s['hostname'], {})
        interface_info = snmp_interfaces.get(s['interface'], None)
        if interface_info:
            s['snmp'] = {
                'ifIndex': interface_info['index'],
                'community': interface_info['community'],
            }
            if s['type'] == 'GWS - INDIRECT':
                s['snmp']['counters'] = [
                    {
                        'field': 'ingressOctets',
                        'oid': _jnx_dcu_byte_count_oid(
                            interface_info['index']),
                    },
                    {
                        'field': 'egressOctets',
                        'oid': _jnx_fw_counter_bytes_oid(
                            s['customer'], s['interface'])
                    }
                ]
        return s

    def _wanted_in_output(s):
        return return_all or (s['status'].lower() == 'operational')

    def _format_services(s):
        return {
            'id': s['id'],
            'name': s['name'],
            'customer': s['project'],
            'speed': s['speed_value'],
            'pop': s['here']['pop']['name'],
            'hostname': common.ims_equipment_to_hostname(
                s['here']['equipment']),
            'interface': s['here']['port'].lower(),
            'type': s['type'],
            'status': s['status']
        }

    cache_key = f'classifier-cache:poller:services:{service_type}' \
        if service_type else 'classifier-cache:poller:services:all-types'

    if return_all:
        cache_key = f'{cache_key}:all'
    if include_snmp:
        cache_key = f'{cache_key}:snmp'

    redis = common.get_current_redis()
    result = _ignore_cache_or_retrieve(request, cache_key, redis)

    if not result:
        result = _services()
        result = filter(_wanted_in_output, result)
        result = map(_format_services, result)
        if include_snmp:
            all_snmp_info = common.load_snmp_indexes(current_app.config['INVENTORY_PROVIDER_CONFIG'])
            result = map(partial(_add_snmp, all_snmp_info=all_snmp_info), result)
        result = list(result)

        if not result:
            return None

        # cache this data for the next call
        result = json.dumps(result)
        redis.set(cache_key, result)

    return result


@routes.route('/services', methods=['GET'])
@routes.route('/services/<service_type>', methods=['GET'])
@common.require_accepts_json
def get_services(service_type=None):
    """
    Handler for `/poller/services</service-type>`

    Use `/poller/service-types` for possible values of `service-type`.

    If the endpoint is called with no `service-type`,
    all services are returned.

    Supported url parameters:

    :all: Default is False to return only operational services.
          If present and evaluates to True, then return all services.
    :snmp: If present and evalutes to True, add snmp interface polling
           params to the response.

    (sphinx bug: the param names aren't capitalized)

    The response will be formatted according to the following schema:

    .. asjson::
       inventory_provider.routes.poller.SERVICES_LIST_SCHEMA

    A few possible values for `service_type` are (currently): gws_internal,
    geant_ip, geant_lambda, l3_vpn, md_vpn_proxy (there are more)

    If the endpoint is called with no `service_type`,
    all services are returned

    :return:
    """

    services_json_str = _get_services_internal(service_type)

    if not services_json_str:
        message = f'no {service_type} services found' \
            if service_type else 'no services found'
        return Response(
            response=message,
            status=404,
            mimetype='text/html')

    return Response(services_json_str, mimetype='application/json')


@routes.route("/gws/indirect", methods=['GET'])
@common.require_accepts_json
def gws_indirect():
    """
    Handler for `/poller/gws/indirect`

    Same as `/poller/services/gws_indirect`

    cf. :meth:`inventory_provider.routes.poller.get_services`
    :return:
    """
    return get_services(service_type='gws_indirect')


@routes.route('/service-types', methods=['GET'])
@common.require_accepts_json
def get_service_types():
    """
    Handler for `/poller/service-types`

    This method returns a list of all values of `service_type` that
    can be used with `/poller/services/service-type` to return
    a non-empty list of services.

    Adding a truthy `all` request parameter (e.g. `?all=1`) will
    return also return valid service types for which none of the
    defined services are operational.

    The response will be formatted according to the following schema:

    .. asjson::
       inventory_provider.routes.poller.STRING_LIST_SCHEMA

    """
    cache_key = 'classifier-cache:poller:service-types'

    return_all = common.get_bool_request_arg('all', False)
    if return_all:
        cache_key = f'{cache_key}-all'

    redis = common.get_current_redis()
    service_types = _ignore_cache_or_retrieve(request, cache_key, redis)

    if not service_types:
        all_services = json.loads(_get_services_internal())
        service_types = {
            ims_sorted_service_type_key(s['type'])
            for s in all_services
        }

        if not service_types:
            return Response(
                response='no service types found',
                status=404,
                mimetype='text/html')

        # cache this data for the next call
        service_types = sorted(list(service_types))
        service_types = json.dumps(service_types)
        redis.set(cache_key, service_types)

    return Response(service_types, mimetype='application/json')


@routes.route('/gws/direct-config', methods=['GET'])
def gws_direct_config():
    """
    Handler for `/poller/gws/direct-config` which returns
    the basic gws-direct config.

    This api is only intended for config validation.
    :return:
    """

    wanted = request.args.get('format', default='json', type=str)
    wanted = wanted.lower()
    if wanted not in ('html', 'json'):
        return Response(
            response='format must be one of: html, json',
            status=400,
            mimetype="text/html")

    def _counters():
        config_params = current_app.config['INVENTORY_PROVIDER_CONFIG']
        for nren_isp in config_params['gws-direct']:
            for host in nren_isp['hosts']:
                snmp_version = '2' if 'community' in host.keys() else '3'
                for ifc in host['interfaces']:
                    for field, oid in ifc['counters'].items():
                        yield {
                            'nren': nren_isp['nren'],
                            'isp': nren_isp['isp'],
                            'hostname': host['hostname'],
                            'snmp': snmp_version,
                            'interface': ifc['tag'],
                            'field': field,
                            'oid': oid,
                            'info': ifc.get('info', '')
                        }

    if wanted == 'json':
        if not request.accept_mimetypes.accept_json:
            return Response(
                response="response will be json",
                status=406,
                mimetype="text/html")
        else:
            return jsonify(list(_counters()))

    if not request.accept_mimetypes.accept_html:
        return Response(
            response="response will be html",
            status=406,
            mimetype="text/html")

    page = '''<!doctype html>
<html lang="en">
  <head>
    <!-- Required meta tags -->
    <meta charset="utf-8">
    <meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no">

    <!-- Bootstrap CSS -->
    <link rel="stylesheet" href="https://stackpath.bootstrapcdn.com/bootstrap/4.3.1/css/bootstrap.min.css" integrity="sha384-ggOyR0iXCbMQv3Xipma34MD+dH/1fQ784/j6cY/iJTQUOhcWr7x9JvoRxT2MZw1T" crossorigin="anonymous">

    <title>GWS Direct config</title>
  </head>
  <body>
  <table class="table table-striped">
      <thead>
{header_row}
      </thead>
      <tbody>
{data_rows}
      </tbody>
  </table>
  </body>
</html>'''  # noqa: E501

    def _to_row(counter, header=False):
        _columns = (
            'nren', 'isp', 'hostname',
            'snmp', 'interface', 'field', 'oid', 'info')
        elems = ['<tr>']
        for name in _columns:
            if header:
                elems.append(f'<th scope="col">{name}</th>')
            else:
                elems.append(f'<td scope="row">{counter[name]}</td>')
        elems.append('</tr>')
        return ''.join(elems)

    header_row = _to_row(None, header=True)
    data_rows = map(_to_row, _counters())
    page = page.format(
        header_row=header_row,
        data_rows='\n'.join(data_rows))

    return Response(
        response=page,
        status=200,
        mimetype="text/html")