data.py

import json
import logging
import re

from inventory_provider.db import opsdb, db
from inventory_provider.tasks.common import get_next_redis

logger = logging.getLogger(__name__)


def build_service_interface_user_list(config):

    def _interfaces():
        """
        yields interface info from netconf
        :return:
        """
        r = get_next_redis(config)
        for k in r.scan_iter('netconf-interfaces:*'):
            k = k.decode('utf-8')
            (_, router_name, ifc_name) = k.split(':')

            info = r.get(k).decode('utf-8')
            info = json.loads(info)

            assert ifc_name == info['name']
            yield {
                'router': router_name,
                'interface': info['name'],
                'description': info['description']
            }

    def _lookup_interface_services(wanted_interfaces):
        """
        yields interface info from opsdb (with service id)
        ... only interfaces in wanted_interfaces
        :param wanted_interfaces:
        :return:
        """
        r = get_next_redis(config)
        for k in r.scan_iter('opsdb:interface_services:*'):
            k = k.decode('utf-8')
            fields = k.split(':')
            if len(fields) < 4:
                # there are some strange records
                # e.g. TS1.*, ts1.*, dp1.*, dtn*, ...
                continue
            router = fields[2]
            ifc_name = fields[3]

            router_interface_key = f'{router}:{ifc_name}'
            if router_interface_key not in wanted_interfaces:
                continue

            info = r.get(k).decode('utf-8')
            info = json.loads(info)

            yield {
                'router': router,
                'interface': ifc_name,
                'service_ids': set([service['id'] for service in info])
            }

    # dict: 'router:interface' -> {'router', 'interface', 'description'}
    netconf_interface_map = dict([
        (f'{i["router"]}:{i["interface"]}', i) for i in _interfaces()])

    # dict: 'router:interface' -> {'router', 'interface', set([service_ids])}
    opsdb_interface_map = dict([
        (f'{i["router"]}:{i["interface"]}', i)
        for i in _lookup_interface_services(netconf_interface_map.keys())])

    all_service_ids = set()
    for r in opsdb_interface_map.values():
        all_service_ids |= r['service_ids']
    all_service_ids = list(all_service_ids)

    # dict: service_id[int] -> [list of users]
    service_user_map = dict()
    with db.connection(config["ops-db"]) as cx:
        # for user in opsdb.get_service_users(cx, list(all_service_ids)):
        service_users = list(opsdb.get_service_users(cx, all_service_ids))
        for user in service_users:
            service_user_map.setdefault(
                user['service_id'], []).append(user['user'])

    def _users(ifc_key):
        """
        ifc = 'router:ifc_name'
        :param ifc:
        :return: list of users
        """
        users = set()
        if ifc_key not in opsdb_interface_map:
            return []
        service_id_list = opsdb_interface_map[ifc_key].get('service_ids', [])
        for service_id in service_id_list:
            users |= set(service_user_map.get(service_id, []))
        return list(users)

    for k, v in netconf_interface_map.items():
        v['users'] = _users(k)
        yield v


def derive_router_hostnames(config):
    r = get_next_redis(config)

    routers = r.get('netdash')
    assert routers
    netdash_equipment = json.loads(routers.decode('utf-8'))
    assert isinstance(netdash_equipment, (list, tuple))
    netdash_equipment = set(netdash_equipment)

    opsdb_equipment = set()
    for k in r.scan_iter('opsdb:interface_services:*'):
        m = re.match(
            'opsdb:interface_services:([^:]+):.*$',
            k.decode('utf-8'))
        if m:
            opsdb_equipment.add(m.group(1))
        else:
            logger.info("Unable to derive router name from %s" %
                        k.decode('utf-8'))
    return netdash_equipment & opsdb_equipment