Skip to content
Snippets Groups Projects
poller.py 49.82 KiB
"""
BRIAN support Endpoints
=========================

These endpoints are intended for use by BRIAN.

.. contents:: :local:

/poller/interfaces</hostname>
---------------------------------

.. autofunction:: inventory_provider.routes.poller.interfaces


/poller/speeds</hostname>
---------------------------------

.. autofunction:: inventory_provider.routes.poller.interface_speeds


/poller/eumetsat-multicast
---------------------------------

.. autofunction:: inventory_provider.routes.poller.eumetsat_multicast


/poller/gws/direct
---------------------------------

.. autofunction:: inventory_provider.routes.poller.gws_direct


/poller/gws/direct-config
---------------------------------

.. autofunction:: inventory_provider.routes.poller.gws_direct_config


/poller/gws/indirect
---------------------------------

.. autofunction:: inventory_provider.routes.poller.gws_indirect


/poller/services</service-type>
---------------------------------

.. autofunction:: inventory_provider.routes.poller.get_services


/poller/service-types
---------------------------------

.. autofunction:: inventory_provider.routes.poller.get_service_types


support method: _get_dashboards
---------------------------------

.. autofunction:: inventory_provider.routes.poller._get_dashboards


"""
from collections import defaultdict
from enum import Enum, auto
import itertools
import json
import logging
import re
from functools import partial

from flask import Blueprint, Response, current_app, request, jsonify

from inventory_provider import juniper
from inventory_provider.routes import common
from inventory_provider.tasks.common import ims_sorted_service_type_key
from inventory_provider.tasks import common as tasks_common
from inventory_provider.routes.classifier import get_ims_equipment_name, \
    get_ims_interface
from inventory_provider.routes.common import _ignore_cache_or_retrieve

logger = logging.getLogger(__name__)
routes = Blueprint('poller-support-routes', __name__)

Mb = 1 << 20
Gb = 1 << 30
OC = Mb * 51.84


class INTERFACE_TYPES(Enum):
    UNKNOWN = auto()
    LOGICAL = auto()
    PHYSICAL = auto()
    AGGREGATE = auto()


class BRIAN_DASHBOARDS(Enum):
    CLS = auto()
    RE_PEER = auto()
    RE_CUST = auto()
    GEANTOPEN = auto()
    GCS = auto()
    L2_CIRCUIT = auto()
    LHCONE_PEER = auto()
    LHCONE_CUST = auto()
    MDVPN_CUSTOMERS = auto()
    INFRASTRUCTURE_BACKBONE = auto()
    IAS_PRIVATE = auto()
    IAS_PUBLIC = auto()
    IAS_CUSTOMER = auto()
    IAS_UPSTREAM = auto()
    GWS_PHY_UPSTREAM = auto()
    GBS_10G = auto()

    # aggregate dashboards
    CLS_PEERS = auto()
    IAS_PEERS = auto()
    GWS_UPSTREAMS = auto()
    LHCONE = auto()
    CAE1 = auto()
    IC1 = auto()
    COPERNICUS = auto()

    # NREN customer
    NREN = auto()


class PORT_TYPES(Enum):
    ACCESS = auto()
    SERVICE = auto()
    UNKNOWN = auto()


# only used in INTERFACE_LIST_SCHEMA and sphinx docs
_DASHBOARD_IDS = [d.name for d in list(BRIAN_DASHBOARDS)]

_PORT_TYPES = [t.name for t in list(PORT_TYPES)]

_INTERFACE_TYPES = [i.name for i in list(INTERFACE_TYPES)]

INTERFACE_LIST_SCHEMA = {
    '$schema': 'https://json-schema.org/draft-07/schema#',

    'definitions': {
        'service': {
            'type': 'object',
            'properties': {
                'id': {'type': 'integer'},
                'name': {'type': 'string'},
                'type': {'type': 'string'},
                'status': {'type': 'string'},
            },
            'required': ['id', 'name', 'type', 'status'],
            'additionalProperties': False
        },
        'db_info': {
            'type': 'object',
            'properties': {
                'name': {'type': 'string'},
                'interface_type': {'enum': _INTERFACE_TYPES}
            },
            'required': ['name', 'interface_type'],
            'additionalProperties': False
        },
        'interface': {
            'type': 'object',
            'properties': {
                'router': {'type': 'string'},
                'name': {'type': 'string'},
                'description': {'type': 'string'},
                'snmp-index': {
                    'type': 'integer',
                    'minimum': 1
                },
                'bundle': {
                    'type': 'array',
                    'items': {'type': 'string'}
                },
                'bundle-parents': {
                    'type': 'array',
                    'items': {'type': 'string'}
                },
                'circuits': {
                    'type': 'array',
                    'items': {'$ref': '#/definitions/service'}
                },
                'dashboards': {
                    'type': 'array',
                    'items': {'enum': _DASHBOARD_IDS}
                },
                'dashboard_info': {
                    '$ref': '#/definitions/db_info',
                },
                'dashboards_info': {
                    'type': 'array',
                    'items': {'$ref': '#/definitions/db_info'}
                },
                'port_type': {'enum': _PORT_TYPES}
            },
            'required': [
                'router', 'name', 'description',
                'snmp-index', 'bundle', 'bundle-parents',
                'circuits', 'dashboards', 'port_type'],
            'additionalProperties': False
        },
    },

    'type': 'array',
    'items': {'$ref': '#/definitions/interface'}
}

INTERFACE_SPEED_LIST_SCHEMA = {
    '$schema': 'https://json-schema.org/draft-07/schema#',

    'definitions': {
        'interface': {
            'type': 'object',
            'properties': {
                'router': {'type': 'string'},
                'name': {'type': 'string'},
                'speed': {'type': 'integer'}
            },
            'required': ['router', 'name', 'speed'],
            'additionalProperties': False
        },
    },

    'type': 'array',
    'items': {'$ref': '#/definitions/interface'}
}

MULTICAST_SUBSCRIPTION_LIST_SCHEMA = {
    '$schema': 'https://json-schema.org/draft-07/schema#',

    'definitions': {
        'ipv4-address': {
            'type': 'string',
            'pattern': r'^(\d+\.){3}\d+$'
        },
        'subscription': {
            'type': 'object',
            'properties': {
                'router': {'type': 'string'},
                'subscription': {'$ref': '#/definitions/ipv4-address'},
                'endpoint': {'$ref': '#/definitions/ipv4-address'},
                'oid': {
                    'type': 'string',
                    'pattern': r'^(\d+\.)*\d+$'
                },
                'community': {'type': 'string'}
            },
            'required': [
                'router', 'subscription', 'endpoint', 'oid', 'community'],
            'additionalProperties': False
        },
    },

    'type': 'array',
    'items': {'$ref': '#/definitions/subscription'}
}

GWS_DIRECT_DATA_SCHEMA = {
    '$schema': 'https://json-schema.org/draft-07/schema#',

    'definitions': {
        'oid': {
            'type': 'string',
            'pattern': r'^(\d+\.)*\d+$'
        },
        'snmp-v2': {
            'type': 'object',
            'properties': {
                'community': {'type': 'string'}
            },
            'required': ['community'],
            'additionalProperties': False
        },
        'snmp-v3-cred': {
            'type': 'object',
            'properties': {
                'protocol': {'enum': ['MD5', 'DES']},
                'password': {'type': 'string'}
            },
            'required': ['protocol', 'password'],
            'additionalProperties': False
        },
        'snmp-v3': {
            'type': 'object',
            'properties': {
                'sec-name': {'type': 'string'},
                'auth': {'$ref': '#/definitions/snmp-v3-cred'},
                'priv': {'$ref': '#/definitions/snmp-v3-cred'}
            },
            'required': ['sec-name'],
            'additionalProperties': False
        },
        'counter': {
            'type': 'object',
            'properties': {
                'field': {
                    'enum': [
                        'discards_in',
                        'discards_out',
                        'errors_in',
                        'errors_out',
                        'traffic_in',
                        'traffic_out'
                    ]
                },
                'oid': {'$ref': '#/definitions/oid'},
                'snmp': {
                    'oneOf': [
                        {'$ref': '#/definitions/snmp-v2'},
                        {'$ref': '#/definitions/snmp-v3'}
                    ]
                }
            },
            'required': ['field', 'oid', 'snmp'],
            'additionalProperties': False
        },
        'interface-counters': {
            'type': 'object',
            'properties': {
                'nren': {'type': 'string'},
                'isp': {
                    'type': 'string',
                    'enum': ['Cogent', 'Telia', 'CenturyLink']
                },
                'hostname': {'type': 'string'},
                'tag': {'type': 'string'},
                'counters': {
                    'type': 'array',
                    'items': {'$ref': '#/definitions/counter'},
                    'minItems': 1
                },
                'info': {'type': 'string'}
            },
            'required': [
                'nren', 'isp', 'hostname', 'tag', 'counters'],
            'additionalProperties': False
        }
    },

    'type': 'array',
    'items': {'$ref': '#/definitions/interface-counters'}
}

SERVICES_LIST_SCHEMA = {
    '$schema': 'https://json-schema.org/draft-07/schema#',

    'definitions': {
        'oid': {
            'type': 'string',
            'pattern': r'^(\d+\.)*\d+$'
        },
        'counter-field': {
            'type': 'object',
            'properties': {
                'field': {
                    'type': 'string',
                    'enum': ['egressOctets', 'ingressOctets']
                },
                'oid': {'$ref': '#/definitions/oid'}
            },
            'required': ['field', 'oid'],
            'additionalProperties': False
        },
        'snmp-info': {
            'type': 'object',
            'properties': {
                'ifIndex': {'type': 'integer'},
                'community': {'type': 'string'},
                'counters': {
                    'type': 'array',
                    'items': {'$ref': '#/definitions/counter-field'},
                    'minItems': 1
                }
            },
            'required': ['ifIndex', 'community'],
            'additionalProperties': False
        },
        'service': {
            'type': 'object',
            'properties': {
                'id': {'type': 'integer'},
                'name': {'type': 'string'},
                'customer': {'type': 'string'},
                'speed': {'type': 'integer'},
                'pop': {'type': 'string'},
                'hostname': {'type': 'string'},
                'interface': {'type': 'string'},
                'type': {'type': 'string'},
                'status': {'type': 'string'},
                'snmp': {'$ref': '#/definitions/snmp-info'}
            },
            'required': [
                'id', 'name', 'customer',
                'speed', 'pop', 'hostname',
                'interface', 'type', 'status'],
            'additionalProperties': False
        }
    },

    'type': 'array',
    'items': {'$ref': '#/definitions/service'}
}

STRING_LIST_SCHEMA = {
    '$schema': 'https://json-schema.org/draft-07/schema#',
    'type': 'array',
    'items': {'type': 'string'}
}


@routes.after_request
def after_request(resp):
    return common.after_request(resp)


def _get_dashboards(interface):
    """
    Yield enums from BRIAN_DASHBOARDS to indicate which dashboards
    this interface should be included in.

    cf. POL1-482

    Possible dashboard id's are:

    .. asjson::
       inventory_provider.routes.poller._DASHBOARD_IDS

    :param interface: a dict with keys like router, name, description
    :return: generator that yields enums from BRIAN_DASHBOARDS
    """

    router = interface.get('router', '').lower()
    ifc_name = interface.get('name', '')
    description = interface.get('description', '').strip()
    if 'SRV_L3VPN' in description and 'COPERNICUS' in description:
        yield BRIAN_DASHBOARDS.COPERNICUS
    if re.match(r'SRV_CLS\s', description):
        yield BRIAN_DASHBOARDS.CLS
    if re.match(r'SRV_CLS PRIVATE\s', description):
        yield BRIAN_DASHBOARDS.CLS_PEERS
    if re.match(r'SRV_IAS PUBLIC\s', description):
        yield BRIAN_DASHBOARDS.IAS_PUBLIC
        yield BRIAN_DASHBOARDS.IAS_PEERS
    if re.match(r'SRV_IAS PRIVATE\s', description):
        yield BRIAN_DASHBOARDS.IAS_PRIVATE
        yield BRIAN_DASHBOARDS.IAS_PEERS
    if re.match(r'SRV_IAS CUSTOMER\s', description):
        yield BRIAN_DASHBOARDS.IAS_CUSTOMER
    if re.match(r'SRV_IAS UPSTREAM\s', description):
        yield BRIAN_DASHBOARDS.IAS_UPSTREAM
    if re.match(r'SRV_10GGBS CUSTOMER\s', description):
        yield BRIAN_DASHBOARDS.GBS_10G
    if re.match(r'(SRV_GLOBAL|SRV_L3VPN|LAG) RE_INTERCONNECT\s', description):
        yield BRIAN_DASHBOARDS.RE_PEER
    if re.match(r'(PHY|LAG|SRV_GLOBAL) CUSTOMER\s', description):
        yield BRIAN_DASHBOARDS.RE_CUST
    if re.match('^SRV_GCS', description):
        yield BRIAN_DASHBOARDS.GCS
    if re.match(r'PHY CUSTOMER_GEO\s', description):
        yield BRIAN_DASHBOARDS.GEANTOPEN
    if 'SRV_L2CIRCUIT' in description:
        yield BRIAN_DASHBOARDS.L2_CIRCUIT
    if 'LHCONE' in description:
        if 'SRV_L3VPN RE' in description:
            yield BRIAN_DASHBOARDS.LHCONE_PEER
        if 'SRV_L3VPN CUSTOMER' in description:
            yield BRIAN_DASHBOARDS.LHCONE_CUST
    if re.match(r'SRV_L3VPN (CUSTOMER|RE_INTERCONNECT)\s', description):
        if '-LHCONE' in description:
            # `-LHCONE` can be preceded by tons of different stuff, so it's
            # simpler to check for this rather than a full regex
            yield BRIAN_DASHBOARDS.LHCONE
    if re.match(r'SRV_MDVPN CUSTOMER\s', description):
        yield BRIAN_DASHBOARDS.MDVPN_CUSTOMERS
    if re.match(r'(SRV_GLOBAL|LAG|PHY) INFRASTRUCTURE BACKBONE', description):
        yield BRIAN_DASHBOARDS.INFRASTRUCTURE_BACKBONE
    if router == 'mx1.lon.uk.geant.net' \
            and re.match(r'^ae12(\.\d+|$)$', ifc_name):
        yield BRIAN_DASHBOARDS.CAE1
    if router == 'rt1.mar.fr.geant.net' \
            and re.match(r'^ae12(\.\d+|$)$', ifc_name):
        yield BRIAN_DASHBOARDS.IC1
    if re.match(r'PHY UPSTREAM\s', description):
        yield BRIAN_DASHBOARDS.GWS_PHY_UPSTREAM
    regex = r'(PHY|LAG|(SRV_(GLOBAL|LHCONE|MDVPN|IAS|CLS|L3VPN))) CUSTOMER\s'
    if re.match(regex, description):
        yield BRIAN_DASHBOARDS.NREN


def _get_dashboard_data(ifc, customers):
    def _get_interface_type(description):
        if re.match(r'^PHY', description):
            return INTERFACE_TYPES.PHYSICAL
        if re.match(r'^SRV_', description):
            return INTERFACE_TYPES.LOGICAL
        if re.match(r'^LAG', description):
            return INTERFACE_TYPES.AGGREGATE

        return INTERFACE_TYPES.UNKNOWN

    description = ifc.get('description', '').strip()
    dashboards = ifc.get('dashboards', [])

    interface_type = _get_interface_type(description)

    if len(dashboards) == 0:
        return ifc

    def _get_customer_name(description):
        name = description.split(' ')
        if len(name) >= 3:
            return name[2].strip().upper()
        else:
            # if the description isn't properly formatted
            # use it as the name to make it obvious something is wrong
            return description

    def _get_backbone_name(description):
        name = description.split('|')
        if len(name) >= 2:
            name = name[1].strip()
            return name.replace('( ', '(')
        else:
            # if the description isn't properly formatted
            # use it as the name to make it obvious something is wrong
            return description

    if BRIAN_DASHBOARDS.INFRASTRUCTURE_BACKBONE.name in dashboards:
        names = [_get_backbone_name(description)]
    elif BRIAN_DASHBOARDS.GWS_PHY_UPSTREAM.name in dashboards:
        host = ifc['router']
        location = host.split('.')[1].upper()
        names = [f'{_get_customer_name(description)} - {location}']
    elif BRIAN_DASHBOARDS.L2_CIRCUIT.name in dashboards:
        # This will give names derived from the Interface Description
        # names = list(_get_l2_customer_names(description))
        # This will give first 2 names IMS
        names = [c['name'] for c in customers][:2]

    else:
        names = [_get_customer_name(description)]

    # if no customers found just return the original data
    if not names:
        return ifc

    # to maintain compatability with current brian dashboard manager we will
    # continue to return dashboard_info with the first customer name. We will
    # also return dashboards_info (note the plural of dashboards) with up to
    # two customers
    return {
        **ifc,
        'dashboard_info': {
            'name': names[0],
            'interface_type': interface_type.name
        },
        'dashboards_info': [{
            'name': name,
            'interface_type': interface_type.name
        } for name in set(names)]
    }


def _load_interface_bundles(config, hostname=None, use_next_redis=False):
    result = dict()

    def _load_docs(key_pattern):
        for doc in common.load_json_docs(
                config_params=config,
                key_pattern=key_pattern,
                num_threads=20,
                use_next_redis=use_next_redis):
            m = re.match(
                r'.*netconf-interface-bundles:([^:]+):(.+)', doc['key'])
            assert m

            router = m.group(1)
            interface = m.group(2)
            result.setdefault(router, dict())
            result[router][interface] = doc['value']

    base_key = 'netconf-interface-bundles'
    base_key_pattern = f'{base_key}:{hostname}:*' \
        if hostname else f'{base_key}:*'

    _load_docs(base_key_pattern)
    _load_docs(f'lab:{base_key_pattern}')

    return result


def _get_services_and_customers(config, hostname=None, use_next_redis=False):
    if hostname:
        hostname = get_ims_equipment_name(hostname)

    result = defaultdict(dict)
    key_pattern = f'ims:interface_services:{hostname}:*' \
        if hostname else 'ims:interface_services:*'

    for doc in common.load_json_docs(
            config_params=config,
            key_pattern=key_pattern,
            num_threads=20,
            use_next_redis=use_next_redis):
        cs = {
            'services': [],
            'customers': []
        }
        included_service_ids = set()
        for s in doc['value']:
            if s['id'] in included_service_ids:
                continue
            if s.get('port_type', '') == 'ab':
                continue
            included_service_ids.add(s['id'])

            cs['customers'].append({
                'name': s['customer'],
                'type': 'UNKNOWN'
            })
            for c in s.get('additional_customers', []):
                cs['customers'].append({
                    'name': c['name'],
                    'type': c.get('type', 'UNKNOWN')
                })
            if s['circuit_type'] == 'service':
                cs['services'].append({
                    'id': s['id'],
                    'name': s['name'],
                    'type': s['service_type'],
                    'status': s['status'],
                })
            result[s['equipment']][s['port']] = cs
    return result


def _load_netconf_docs(
        config, filter_pattern, use_next_redis=False):
    """
    yields dicts like:
        {
            'router': router hostname
            'netconf': loaded netconf xml doc
        }

    :param config: app config
    :param filter_pattern: search filter, including 'netconf:'
    :param use_next_redis: use next instead of current redis, if true
    :return: yields netconf docs, formatted as above
    """

    m = re.match(r'^(.*netconf:).+', filter_pattern)
    # TODO: probably better to not required netconf: to be passed in
    assert m  # sanity
    key_prefix_len = len(m.group(1))
    assert key_prefix_len >= len('netconf:')  # sanity

    for doc in common.load_xml_docs(
            config_params=config,
            key_pattern=filter_pattern,
            num_threads=10,
            use_next_redis=use_next_redis):
        yield {
            'router': doc['key'][key_prefix_len:],
            'netconf': doc['value']
        }


def _load_netconf_parsed_cache(config, cache_name, hostname=None, is_lab=False, use_next_redis=False):
    hostname_suffix = hostname if hostname else ''
    lab_prefix = 'lab:' if is_lab else ''
    filter_pattern = f'{lab_prefix}{cache_name}:{hostname_suffix}*'
    for doc in common.load_json_docs(
            config_params=config,
            key_pattern=filter_pattern,
            num_threads=20,
            use_next_redis=use_next_redis):
        m = re.match(fr'(lab:)?{cache_name}:(.+:.+)', doc['key'])
        if m:
            # preparse the key as interfaces can include : in the name
            key_parts = m.group(2).split(':')
            key = f'{key_parts[0]}-----{":".join(key_parts[1:])}'
            yield key, doc['value']


def _load_interfaces(
        config, hostname=None, no_lab=False, use_next_redis=False):
    """
    loads basic interface data for production & lab routers

    :param config:
    :param hostname:
    :param use_next_redis:
    :return:
    """

    def _load_netconf_caches(is_lab=False):

        interfaces = dict(_load_netconf_parsed_cache(config, 'netconf-interfaces',
                                                     hostname, is_lab, use_next_redis))
        interface_bundles = dict(_load_netconf_parsed_cache(config, 'netconf-interface-bundles',
                                                            hostname, is_lab, use_next_redis))

        for key, ifc in interfaces.items():
            if not ifc['description']:
                continue
            router, interface_name = key.split('-----')
            bundle = interface_bundles.get(key, [])

            yield {
                'router': router,
                'name': interface_name,
                'bundle': bundle,
                'bundle-parents': [],
                'description': ifc['description'],
                'circuits': []
            }

    yield from _load_netconf_caches()
    if not no_lab:
        logger.debug('lab')
        yield from _load_netconf_caches(is_lab=True)


def _add_speeds(interfaces):
    r = common.get_current_redis()
    all_netconf_interfaces = json.loads(r.get('netconf-interfaces:all')) or []

    netconf_interface_index = {}
    for nc_ifc in all_netconf_interfaces:
        if 'hostname' in nc_ifc and 'name' in nc_ifc:
            netconf_interface_index[f"{nc_ifc['hostname']}---{nc_ifc['name']}"] = nc_ifc

    for ifc in interfaces:
        nc_ifc = netconf_interface_index.get(f"{ifc['router']}---{ifc['name']}", {})
        if 'speed' in nc_ifc:
            ifc['speed'] = nc_ifc['speed']
        yield ifc


def _add_bundle_parents(interfaces, hostname=None):
    """
    generator that adds bundle-parents info to each interface.

    :param interfaces: result of _load_interfaces
    :param hostname: hostname or None for all
    :return: generator with bundle-parents populated in each element
    """

    def _get_base_name(name):
        return name.split('.')[0]

    bundles = _load_interface_bundles(
        current_app.config['INVENTORY_PROVIDER_CONFIG'], hostname)
    # create a quick look-up for interface details
    interface_index = {f"{ifc['router']}---{ifc['name']}": ifc for ifc in interfaces}

    for ifc in interfaces:
        router_bundle = bundles.get(ifc['router'], None)
        if router_bundle:
            base_ifc = _get_base_name(ifc['name'])
            bundle_parents = [interface_index.get(f"{ifc['router']}---{_get_base_name(bundle_ifc)}")
                              for bundle_ifc in router_bundle.get(base_ifc, [])]
            ifc['bundle-parents'] = bundle_parents
        yield ifc


def _get_port_type(description):
    rex = re.search(r'\$([a-zA-Z]+\-\d+)', description)
    if rex:
        sid = rex.group(1)
        if 'GA' in sid:
            return PORT_TYPES.ACCESS.name
        elif 'GS' in sid:
            return PORT_TYPES.SERVICE.name
    return PORT_TYPES.UNKNOWN.name


def load_interfaces_to_poll(
        config, hostname=None, no_lab=False, use_next_redis=False):
    basic_interfaces = \
        list(_load_interfaces(config, hostname, no_lab, use_next_redis))
    bundles = _load_interface_bundles(config, hostname, use_next_redis)
    services_and_customers = \
        _get_services_and_customers(config, hostname, use_next_redis)
    snmp_indexes = common.load_snmp_indexes(config, hostname, use_next_redis)

    def _get_populated_interfaces(all_interfaces):
        if use_next_redis:
            r = tasks_common.get_next_redis(config)
        else:
            r = tasks_common.get_current_redis(config)
        for ifc in all_interfaces:
            router_snmp = snmp_indexes.get(ifc['router'], None)
            if router_snmp and ifc['name'] in router_snmp:
                ifc['snmp-index'] = router_snmp[ifc['name']]['index']

                router_bundle = bundles.get(ifc['router'], None)
                if router_bundle:
                    base_ifc = ifc['name'].split('.')[0]
                    ifc['bundle-parents'] = router_bundle.get(base_ifc, [])

                router_services_and_customers = services_and_customers.get(
                    get_ims_equipment_name(ifc['router'], r), {})
                ifc_services_and_customers = \
                    router_services_and_customers.get(
                        get_ims_interface(ifc['name']), {}
                    )

                if 'services' in ifc_services_and_customers \
                        and ifc_services_and_customers['services']:
                    ifc['circuits'] = ifc_services_and_customers['services']

                dashboards = _get_dashboards(ifc)
                ifc['dashboards'] = sorted([d.name for d in dashboards])

                ifc = _get_dashboard_data(
                    ifc, ifc_services_and_customers.get('customers', []))
                port_type = _get_port_type(ifc['description'])
                ifc['port_type'] = port_type
                yield ifc
            else:
                continue

    return _get_populated_interfaces(basic_interfaces)


@routes.route("/interfaces", methods=['GET'])
@routes.route('/interfaces/<hostname>', methods=['GET'])
@common.require_accepts_json
def interfaces(hostname=None):
    """
    Handler for `/poller/interfaces` and
    `/poller/interfaces/<hostname>`
    which returns information for either all interfaces
    or those on the requested hostname.

    The optional `no-lab` parameter omits lab routers
    if it's truthiness evaluates to True.

    The response is a list of information for all
    interfaces that should be polled, including service
    information and snmp information.

    .. asjson::
       inventory_provider.routes.poller.INTERFACE_LIST_SCHEMA

    :meth:`inventory_provider.routes.poller._get_services`
    is where dashboard mappings is handled.

    :param hostname: optional, if present should be a router hostname
    :return:
    """

    cache_key = f'classifier-cache:poller-interfaces:{hostname}' \
        if hostname else 'classifier-cache:poller-interfaces:all'

    no_lab = common.get_bool_request_arg('no-lab', False)
    if no_lab:
        cache_key = f'{cache_key}:no_lab'

    r = common.get_current_redis()

    result = _ignore_cache_or_retrieve(request, cache_key, r)

    if not result:
        result = list(load_interfaces_to_poll(
            current_app.config['INVENTORY_PROVIDER_CONFIG'], hostname, no_lab))
        if not result:
            return Response(
                response='no interfaces found',
                status=404,
                mimetype='text/html')

        result = json.dumps(result)
        # cache this data for the next call
        r.set(cache_key, result.encode('utf-8'))

    return Response(result, mimetype="application/json")


def interface_speed(ifc):
    """
    Return the maximum bits per second expected for the given interface.

    cf. https://www.juniper.net/documentation/us/en/software/
               vmx/vmx-getting-started/topics/task/
               vmx-chassis-interface-type-configuring.html

    :param ifc:
    :return: an integer bits per second
    """

    def _name_to_speed(ifc_name):
        if ifc_name.startswith('ge'):
            return Gb
        if ifc_name.startswith('xe'):
            return 10 * Gb
        if ifc_name.startswith('et'):
            return 400 * Gb
        logger.warning(f'unrecognized interface name: {ifc_name}')
        return -1

    def _get_speed(ifc):

        rate_conversions = {
            "mbps": Mb,
            "gbps": Gb

        }

        if "speed" in ifc:
            speed = ifc["speed"]
            rate_match = re.match(r"(\d+)(.+)", speed)
            if rate_match:
                value = int(rate_match.group(1))
                rate = rate_match.group(2).strip().lower()
                if rate in rate_conversions:
                    return int(value * rate_conversions[rate])
                else:
                    logger.warning(f'unrecognised rate: {rate}, using _name_to_speed fallback')
                    return _name_to_speed(ifc['name'])
            else:
                oc_match = re.match(r"OC(\d+)", speed)
                if oc_match:
                    value = int(oc_match.group(1))
                    return int(value * OC)
                else:
                    logger.warning(f'unrecognised speed: {speed}, using _name_to_speed fallback')
                    return _name_to_speed(ifc['name'])
        else:
            logger.warning('no speed data for interface, using _name_to_speed fallback')
            return _name_to_speed(ifc['name'])

    if ifc['bundle-parents']:
        if not ifc['name'].startswith('ae'):
            logger.warning(
                f'ifc has bundle-parents, but name is {ifc["name"]}')
        return int(sum(_get_speed(parent_ifc) for parent_ifc in ifc['bundle-parents']))

    return int(_get_speed(ifc))


def _load_interfaces_and_speeds(hostname=None):
    """
    prepares the result of a call to /speeds

    :param hostname: hostname or None for all
    :return: generator yielding interface elements
    """
    no_lab = common.get_bool_request_arg('no-lab', False)
    basic_interfaces = _load_interfaces(
        current_app.config['INVENTORY_PROVIDER_CONFIG'],
        hostname,
        no_lab=no_lab)
    basic_interfaces_with_speeds = _add_speeds(basic_interfaces)
    with_bundles = _add_bundle_parents(list(basic_interfaces_with_speeds), hostname)

    def _result_ifc(ifc):
        return {
            'router': ifc['router'],
            'name': ifc['name'],
            'speed': interface_speed(ifc)
        }

    return map(_result_ifc, with_bundles)


@routes.route("/speeds", methods=['GET'])
@routes.route('/speeds/<hostname>', methods=['GET'])
@common.require_accepts_json
def interface_speeds(hostname=None):
    """
    Handler for `/poller/speeds` and
    `/poller/speeds/<hostname>`
    which returns information for either all interfaces
    or those on the requested hostname.

    The response is a list of maximum speed information (in bits
    per second) for all known interfaces.

    *speed <= 0 means the max interface speed can't be determined*

    .. asjson::
       inventory_provider.routes.poller.INTERFACE_SPEED_LIST_SCHEMA

    :param hostname: optional, if present should be a router hostname
    :return:
    """

    cache_key = f'classifier-cache:poller-interface-speeds:{hostname}' \
        if hostname else 'classifier-cache:poller-interface-speeds:all'

    r = common.get_current_redis()

    result = _ignore_cache_or_retrieve(request, cache_key, r)

    if not result:
        result = list(_load_interfaces_and_speeds(hostname))
        if not result:
            return Response(
                response='no interfaces found',
                status=404,
                mimetype='text/html')

        result = json.dumps(result)
        # cache this data for the next call
        r.set(cache_key, result.encode('utf-8'))

    return Response(result, mimetype="application/json")


def _load_community_strings(base_key_pattern):
    for doc in _load_netconf_docs(
            config=current_app.config['INVENTORY_PROVIDER_CONFIG'],
            filter_pattern=base_key_pattern):
        community = juniper.snmp_community_string(doc['netconf'])
        if not community:
            yield {
                'router': doc['router'],
                'error':
                    f'error extracting community string for {doc["router"]}'
            }
        else:
            yield {
                'router': doc['router'],
                'community': community
            }


@routes.route('/eumetsat-multicast', methods=['GET'])
@routes.route('/eumetsat-multicast/<hostname>', methods=['GET'])
@common.require_accepts_json
def eumetsat_multicast(hostname=None):
    """
    Handler for `/poller/eumetsat-multicast</hostname>` which returns
    information about multicast subscriptions.

    The hostname parameter is optional.  If it is present, only hostnames
    matching `hostname*` are returned.  If not present, data for all
    `mx*` routers is returned.

    The response is a list of oid/router/community structures that all
    all subscription octet counters to be polled.

    .. asjson::
       inventory_provider.routes.poller.MULTICAST_SUBSCRIPTION_LIST_SCHEMA

    This method returns essentially hard-coded data,
    based on the information in POL1-395.

    :return:
    """

    SUBSCRIPTIONS = [{
        'subscription': f'232.223.222.{idx}',
        'endpoint': '193.17.9.3',
    } for idx in range(1, 73)]

    SUBSCRIPTIONS.append(
        {'subscription': '232.223.223.1', 'endpoint': '193.17.9.7'})
    SUBSCRIPTIONS.append(
        {'subscription': '232.223.223.22', 'endpoint': '193.17.9.7'})

    def _oid(sub):
        return ('1.3.6.1.2.1.83.1.1.2.1.16'
                f'.{sub["subscription"]}.{sub["endpoint"]}'
                '.255.255.255.255')

    r = common.get_current_redis()

    cache_key = 'classifier-cache:poller-eumetsat-multicast'
    if hostname:
        cache_key = f'{cache_key}:{hostname}'

    result = r.get(cache_key)
    if result:
        result = result.decode('utf-8')
    else:

        def _multicast_oids(router_info):

            def _rsp_element(sub):
                result = {
                    'router': router_info['router'],
                    'oid': _oid(sub),
                    'community': router_info['community']
                }
                result.update(sub)
                return result

            yield from map(_rsp_element, SUBSCRIPTIONS)

        routers = list(_load_community_strings(
            base_key_pattern=f'netconf:{hostname}*'
            if hostname else 'netconf:mx*'))
        errors = list(filter(lambda x: 'error' in x, routers))
        if errors:
            errors = [e['error'] for e in errors]
            return Response(
                response=', '.join(errors),
                status=403,  # forbidden
                mimetype='text/html')

        assert all('community' in r for r in routers)  # sanity

        result = list(map(_multicast_oids, routers))
        result = itertools.chain(*result)
        result = list(result)
        if not result:
            target = hostname or 'any routers!'
            return Response(
                response=f'no multicast config for {target}',
                status=404,
                mimetype='text/html')

        result = json.dumps(result)
        # cache this data for the next call
        r.set(cache_key, result.encode('utf-8'))

    return Response(result, mimetype="application/json")


@routes.route("/gws/direct", methods=['GET'])
@common.require_accepts_json
def gws_direct():
    """
    Handler for `/poller/gws/direct` which returns required for polling
     customer equipment counters for ISP connetions.

    The response is a list of nren/isp/counter structures that must be
    polled.

    .. asjson::
       inventory_provider.routes.poller.GWS_DIRECT_DATA_SCHEMA

    WARNING: interface tags in the `gws-direct` section of the config data
    should be unique for each nren/isp combination.  i.e. if there
    are multiple community strings in use for a particular host, then please
    keep the interface tags unique.

    :return:
    """

    cache_key = 'classifier-cache:gws-direct'
    r = common.get_current_redis()

    result = r.get(cache_key)
    if result:
        result = result.decode('utf-8')
    else:

        def _interfaces():
            config_params = current_app.config['INVENTORY_PROVIDER_CONFIG']
            for nren_isp in config_params['gws-direct']:
                for host in nren_isp['hosts']:

                    snmp_params = {}
                    if 'community' in host:
                        # (snmp v2)
                        # sanity (already guaranteed by schema check)
                        assert 'sec-name' not in host
                        snmp_params['community'] = host['community']
                    else:
                        # (snmp v3)
                        # sanity (already guaranteed by schema check)
                        assert 'sec-name' in host
                        snmp_params['sec-name'] = host['sec-name']
                        if 'auth' in host:
                            snmp_params['auth'] = host['auth']
                            if 'priv' in host:
                                snmp_params['priv'] = host['priv']
                    for ifc in host['interfaces']:

                        ifc_data = {
                            'nren': nren_isp['nren'],
                            'isp': nren_isp['isp'],
                            'hostname': host['hostname'],
                            'tag': ifc['tag'],
                            'counters': [
                                {
                                    'field': k,
                                    'oid': v,
                                    'snmp': snmp_params
                                }
                                for k, v in ifc['counters'].items()]
                        }
                        if 'info' in ifc:
                            ifc_data['info'] = ifc['info']
                        yield ifc_data

        result = json.dumps(list(_interfaces()))

        # cache this data for the next call
        r.set(cache_key, result.encode('utf-8'))

    return Response(result, mimetype="application/json")


# cf. https://gitlab.geant.net/puppet-apps/cacti/-/blob/production/files/scripts/juniper-firewall-dws.pl  # noqa: E501

JNX_DCU_STATS_BYTES_OID = '1.3.6.1.4.1.2636.3.6.2.1.5'
JNX_FW_COUNTER_BYTES_OID = '1.3.6.1.4.1.2636.3.5.2.1.5'

JNX_ADDRESS_FAMILY = {
    'ipv4': 1,
    'ipv6': 2
}

JNX_FW_COUNTER_TYPE = {
    'other': 1,
    'counter': 2,
    'policer': 3
}


def _str2oid(s):
    chars = '.'.join(str(ord(c)) for c in s)
    return f'{len(s)}.{chars}'


def _jnx_dcu_byte_count_oid(
        ifIndex,
        class_name='dws-in',
        address_family=JNX_ADDRESS_FAMILY['ipv4']):
    # sanity checks (in case of programming errors)
    assert isinstance(ifIndex, int)
    assert isinstance(class_name, str)
    assert isinstance(address_family, int)
    return '.'.join([
        JNX_DCU_STATS_BYTES_OID,
        str(ifIndex),
        str(address_family),
        _str2oid(class_name)
    ])


def _jnx_fw_counter_bytes_oid(
        customer,
        interface_name,
        filter_name=None,
        counter_name=None):
    # sanity checks (in case of programming errors)
    assert isinstance(customer, str)
    assert isinstance(interface_name, str)
    assert filter_name is None or isinstance(filter_name, str)
    assert counter_name is None or isinstance(counter_name, str)

    if filter_name is None:
        filter_name = f'nren_IAS_{customer}_OUT-{interface_name}-o'
    if counter_name is None:
        counter_name = f'DWS-out-{interface_name}-o'

    return '.'.join([
        JNX_FW_COUNTER_BYTES_OID,
        _str2oid(filter_name),
        _str2oid(counter_name),
        str(JNX_FW_COUNTER_TYPE['counter'])
    ])


def _get_services_internal(service_type=None):
    """
    Performs the lookup and caching done for calls to
    `/poller/services</service-type>`

    This is a separate private utility so that it can be called by
    :meth:`inventory_provider.routes.poller.get_services`
    and :meth:`inventory_provider.routes.poller.get_service_types`

    The response will be formatted according to the following schema:

    .. asjson::
       inventory_provider.routes.poller.SERVICES_LIST_SCHEMA

    :param service_type: a service type, or None to return all
    :return: service list, json-serialized to a string
    """

    return_all = common.get_bool_request_arg('all', True)
    include_snmp = common.get_bool_request_arg('snmp', False)

    def _services():
        key_pattern = f'ims:services:{service_type}:*' \
            if service_type else 'ims:services:*'

        for doc in common.load_json_docs(
                config_params=current_app.config['INVENTORY_PROVIDER_CONFIG'],
                key_pattern=key_pattern,
                num_threads=20):
            yield doc['value']

    def _add_snmp(s, all_snmp_info):
        snmp_interfaces = all_snmp_info.get(s['hostname'], {})
        interface_info = snmp_interfaces.get(s['interface'], None)
        if interface_info:
            s['snmp'] = {
                'ifIndex': interface_info['index'],
                'community': interface_info['community'],
            }
            if s['type'] == 'GWS - INDIRECT':
                s['snmp']['counters'] = [
                    {
                        'field': 'ingressOctets',
                        'oid': _jnx_dcu_byte_count_oid(
                            interface_info['index']),
                    },
                    {
                        'field': 'egressOctets',
                        'oid': _jnx_fw_counter_bytes_oid(
                            s['customer'], s['interface'])
                    }
                ]
        return s

    def _wanted_in_output(s):
        return return_all or (s['status'].lower() == 'operational')

    def _format_services(s):
        return {
            'id': s['id'],
            'name': s['name'],
            'customer': s['project'],
            'speed': s['speed_value'],
            'pop': s['here']['pop']['name'],
            'hostname': common.ims_equipment_to_hostname(
                s['here']['equipment']),
            'interface': s['here']['port'].lower(),
            'type': s['type'],
            'status': s['status']
        }

    cache_key = f'classifier-cache:poller:services:{service_type}' \
        if service_type else 'classifier-cache:poller:services:all-types'

    if return_all:
        cache_key = f'{cache_key}:all'
    if include_snmp:
        cache_key = f'{cache_key}:snmp'

    redis = common.get_current_redis()
    result = _ignore_cache_or_retrieve(request, cache_key, redis)

    if not result:
        result = _services()
        result = filter(_wanted_in_output, result)
        result = map(_format_services, result)
        if include_snmp:
            all_snmp_info = common.load_snmp_indexes(current_app.config['INVENTORY_PROVIDER_CONFIG'])
            result = map(partial(_add_snmp, all_snmp_info=all_snmp_info), result)
        result = list(result)

        if not result:
            return None

        # cache this data for the next call
        result = json.dumps(result)
        redis.set(cache_key, result)

    return result


@routes.route('/services', methods=['GET'])
@routes.route('/services/<service_type>', methods=['GET'])
@common.require_accepts_json
def get_services(service_type=None):
    """
    Handler for `/poller/services</service-type>`

    Use `/poller/service-types` for possible values of `service-type`.

    If the endpoint is called with no `service-type`,
    all services are returned.

    Supported url parameters:

    :all: Default is False to return only operational services.
          If present and evaluates to True, then return all services.
    :snmp: If present and evalutes to True, add snmp interface polling
           params to the response.

    (sphinx bug: the param names aren't capitalized)

    The response will be formatted according to the following schema:

    .. asjson::
       inventory_provider.routes.poller.SERVICES_LIST_SCHEMA

    A few possible values for `service_type` are (currently): gws_internal,
    geant_ip, geant_lambda, l3_vpn, md_vpn_proxy (there are more)

    If the endpoint is called with no `service_type`,
    all services are returned

    :return:
    """

    services_json_str = _get_services_internal(service_type)

    if not services_json_str:
        message = f'no {service_type} services found' \
            if service_type else 'no services found'
        return Response(
            response=message,
            status=404,
            mimetype='text/html')

    return Response(services_json_str, mimetype='application/json')


@routes.route("/gws/indirect", methods=['GET'])
@common.require_accepts_json
def gws_indirect():
    """
    Handler for `/poller/gws/indirect`

    Same as `/poller/services/gws_indirect`

    cf. :meth:`inventory_provider.routes.poller.get_services`
    :return:
    """
    return get_services(service_type='gws_indirect')


@routes.route('/service-types', methods=['GET'])
@common.require_accepts_json
def get_service_types():
    """
    Handler for `/poller/service-types`

    This method returns a list of all values of `service_type` that
    can be used with `/poller/services/service-type` to return
    a non-empty list of services.

    Adding a truthy `all` request parameter (e.g. `?all=1`) will
    return also return valid service types for which none of the
    defined services are operational.

    The response will be formatted according to the following schema:

    .. asjson::
       inventory_provider.routes.poller.STRING_LIST_SCHEMA

    """
    cache_key = 'classifier-cache:poller:service-types'

    return_all = common.get_bool_request_arg('all', False)
    if return_all:
        cache_key = f'{cache_key}-all'

    redis = common.get_current_redis()
    service_types = _ignore_cache_or_retrieve(request, cache_key, redis)

    if not service_types:
        all_services = json.loads(_get_services_internal())
        service_types = {
            ims_sorted_service_type_key(s['type'])
            for s in all_services
        }

        if not service_types:
            return Response(
                response='no service types found',
                status=404,
                mimetype='text/html')

        # cache this data for the next call
        service_types = sorted(list(service_types))
        service_types = json.dumps(service_types)
        redis.set(cache_key, service_types)

    return Response(service_types, mimetype='application/json')


@routes.route('/gws/direct-config', methods=['GET'])
def gws_direct_config():
    """
    Handler for `/poller/gws/direct-config` which returns
    the basic gws-direct config.

    This api is only intended for config validation.
    :return:
    """

    wanted = request.args.get('format', default='json', type=str)
    wanted = wanted.lower()
    if wanted not in ('html', 'json'):
        return Response(
            response='format must be one of: html, json',
            status=400,
            mimetype="text/html")

    def _counters():
        config_params = current_app.config['INVENTORY_PROVIDER_CONFIG']
        for nren_isp in config_params['gws-direct']:
            for host in nren_isp['hosts']:
                snmp_version = '2' if 'community' in host.keys() else '3'
                for ifc in host['interfaces']:
                    for field, oid in ifc['counters'].items():
                        yield {
                            'nren': nren_isp['nren'],
                            'isp': nren_isp['isp'],
                            'hostname': host['hostname'],
                            'snmp': snmp_version,
                            'interface': ifc['tag'],
                            'field': field,
                            'oid': oid,
                            'info': ifc.get('info', '')
                        }

    if wanted == 'json':
        if not request.accept_mimetypes.accept_json:
            return Response(
                response="response will be json",
                status=406,
                mimetype="text/html")
        else:
            return jsonify(list(_counters()))

    if not request.accept_mimetypes.accept_html:
        return Response(
            response="response will be html",
            status=406,
            mimetype="text/html")

    page = '''<!doctype html>
<html lang="en">
  <head>
    <!-- Required meta tags -->
    <meta charset="utf-8">
    <meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no">

    <!-- Bootstrap CSS -->
    <link rel="stylesheet" href="https://stackpath.bootstrapcdn.com/bootstrap/4.3.1/css/bootstrap.min.css" integrity="sha384-ggOyR0iXCbMQv3Xipma34MD+dH/1fQ784/j6cY/iJTQUOhcWr7x9JvoRxT2MZw1T" crossorigin="anonymous">

    <title>GWS Direct config</title>
  </head>
  <body>
  <table class="table table-striped">
      <thead>
{header_row}
      </thead>
      <tbody>
{data_rows}
      </tbody>
  </table>
  </body>
</html>'''  # noqa: E501

    def _to_row(counter, header=False):
        _columns = (
            'nren', 'isp', 'hostname',
            'snmp', 'interface', 'field', 'oid', 'info')
        elems = ['<tr>']
        for name in _columns:
            if header:
                elems.append(f'<th scope="col">{name}</th>')
            else:
                elems.append(f'<td scope="row">{counter[name]}</td>')
        elems.append('</tr>')
        return ''.join(elems)

    header_row = _to_row(None, header=True)
    data_rows = map(_to_row, _counters())
    page = page.format(
        header_row=header_row,
        data_rows='\n'.join(data_rows))

    return Response(
        response=page,
        status=200,
        mimetype="text/html")