poller.py

import json
import logging
import re

from flask import Blueprint, Response, current_app
from inventory_provider import juniper
from inventory_provider.routes import common

logger = logging.getLogger(__name__)
routes = Blueprint('poller-support-routes', __name__)

INTERFACE_LIST_SCHEMA = {
    '$schema': 'http://json-schema.org/draft-07/schema#',

    'definitions': {
        'service': {
            'type': 'object',
            'properties': {
                'id': {'type': 'integer'},
                'name': {'type': 'string'},
                'type': {'type': 'string'},
                'status': {'type': 'string'},
            },
            'required': ['id', 'name', 'type', 'status'],
            'additionalProperties': False
        },
        'interface': {
            'type': 'object',
            'properties': {
                'router': {'type': 'string'},
                'name': {'type': 'string'},
                'description': {'type': 'string'},
                'snmp-index': {
                    'type': 'integer',
                    'minimum': 1
                },
                'bundle': {
                    'type': 'array',
                    'items': {'type': 'string'}
                },
                'bundle-parents': {
                    'type': 'array',
                    'items': {'type': 'string'}
                },
                'circuits': {
                    'type': 'array',
                    'items': {'$ref': '#/definitions/service'}
                }
            },
            'required': [
                'router', 'name', 'description',
                'snmp-index', 'bundle', 'bundle-parents',
                'circuits'],
            'additionalProperties': False
        },
    },

    'type': 'array',
    'items': {'$ref': '#/definitions/interface'}
}


@routes.after_request
def after_request(resp):
    return common.after_request(resp)


def _load_snmp_indexes(hostname=None):
    result = dict()
    key_pattern = f'snmp-interfaces:{hostname}*' \
        if hostname else 'snmp-interfaces:*'

    for doc in common.load_json_docs(
            config_params=current_app.config['INVENTORY_PROVIDER_CONFIG'],
            key_pattern=key_pattern):
        router = doc['key'][len('snmp-interfaces:'):]
        result[router] = {e['name']: e for e in doc['value']}

    return result


def _load_interface_bundles(hostname=None):
    result = dict()
    key_pattern = f'netconf-interface-bundles:{hostname}:*' \
        if hostname else 'netconf-interface-bundles:*'

    for doc in common.load_json_docs(
            config_params=current_app.config['INVENTORY_PROVIDER_CONFIG'],
            key_pattern=key_pattern,
            num_threads=20):

        m = re.match(r'^netconf-interface-bundles:([^:]+):(.+)', doc['key'])
        assert m

        router = m.group(1)
        interface = m.group(2)
        result.setdefault(router, dict())
        result[router][interface] = doc['value']

    return result


def _load_services(hostname=None):
    result = dict()
    key_pattern = f'opsdb:interface_services:{hostname}:*' \
        if hostname else 'opsdb:interface_services:*'

    def _service_params(full_service_info):
        return {
            'id': full_service_info['id'],
            'name': full_service_info['name'],
            'type': full_service_info['service_type'],
            'status': full_service_info['status']
        }

    for doc in common.load_json_docs(
            config_params=current_app.config['INVENTORY_PROVIDER_CONFIG'],
            key_pattern=key_pattern,
            num_threads=20):

        m = re.match(r'^opsdb:interface_services:([^:]+):(.+)', doc['key'])
        if not m:
            logger.warning(f'can\'t parse redis service key {doc["key"]}')
            # there are some weird records (dtn*, dp1*)
            continue

        router = m.group(1)
        interface = m.group(2)
        result.setdefault(router, dict())
        result[router][interface] = [_service_params(s) for s in doc['value']]

    return result


def _load_interfaces(hostname):
    key_pattern = f'netconf:{hostname}*' if hostname else 'netconf:*'
    for doc in common.load_xml_docs(
            config_params=current_app.config['INVENTORY_PROVIDER_CONFIG'],
            key_pattern=key_pattern,
            num_threads=10):

        router = doc['key'][len('netconf:'):]

        for ifc in juniper.list_interfaces(doc['value']):
            if not ifc['description']:
                continue

            yield {
                'router': router,
                'name': ifc['name'],
                'bundle': ifc['bundle'],
                'bundle-parents': [],
                'snmp-index': -1,
                'description': ifc['description'],
                'circuits': []
            }


def _load_poller_interfaces(hostname=None):

    snmp_indexes = _load_snmp_indexes(hostname)
    bundles = _load_interface_bundles(hostname)
    services = _load_services(hostname)

    for ifc in _load_interfaces(hostname):

        router_snmp = snmp_indexes.get(ifc['router'], None)
        if not router_snmp or ifc['name'] not in router_snmp:
            # there's no way to poll this interface
            continue
        ifc['snmp-index'] = router_snmp[ifc['name']]['index']
        # TODO: uncomment this when it won't break poller-admin-service
        # not urgent ... it looks empirically like all logical-system
        #                interfaces are repeated for both communities
        # ifc['snmp-community'] = router_snmp[ifc['name']]['community']

        router_bundle = bundles.get(ifc['router'], None)
        if router_bundle:
            base_ifc = ifc['name'].split('.')[0]
            ifc['bundle-parents'] = router_bundle.get(base_ifc, [])

        router_services = services.get(ifc['router'], None)
        if router_services:
            ifc['circuits'] = router_services.get(ifc['name'], [])

        yield ifc


@routes.route("/interfaces", methods=['GET', 'POST'])
@routes.route('/interfaces/<hostname>', methods=['GET', 'POST'])
@common.require_accepts_json
def interfaces(hostname=None):
    """
    Handler for `/poller/interfaces` and
    `/poller/interfaces/<hostname>`
    which returns information for either all interfaces
    or those on the requested hostname.

    The response is a list of information for all
    interfaces that should be polled, including service
    information and snmp information.

    .. asjson::
       inventory_provider.routes.poller.INTERFACE_LIST_SCHEMA

    :param hostname: optional, if present should be a router hostname
    :return:
    """

    cache_key = f'classifier-cache:poller-interfaces:{hostname}' \
        if hostname else 'classifier-cache:poller-interfaces:all'

    r = common.get_current_redis()

    result = r.get(cache_key)
    if result:
        result = result.decode('utf-8')
    else:
        result = list(_load_poller_interfaces(hostname))
        if not result:
            return Response(
                response='no interfaces found',
                status=404,
                mimetype='text/html')

        result = json.dumps(result)
        # cache this data for the next call
        r.set(cache_key, result.encode('utf-8'))

    return Response(result, mimetype="application/json")