import json import logging import re from inventory_provider.db import opsdb, db from inventory_provider.tasks.common import get_next_redis logger = logging.getLogger(__name__) def build_service_interface_user_list(config): def _interfaces(): """ yields interface info from netconf :return: """ r = get_next_redis(config) for k in r.scan_iter('netconf-interfaces:*'): k = k.decode('utf-8') (_, router_name, ifc_name) = k.split(':') info = r.get(k).decode('utf-8') info = json.loads(info) assert ifc_name == info['name'] yield { 'router': router_name, 'interface': info['name'], 'description': info['description'] } def _lookup_interface_services(wanted_interfaces): """ yields interface info from opsdb (with service id) ... only interfaces in wanted_interfaces :param wanted_interfaces: :return: """ r = get_next_redis(config) for k in r.scan_iter('opsdb:interface_services:*'): k = k.decode('utf-8') fields = k.split(':') if len(fields) < 4: # there are some strange records # e.g. TS1.*, ts1.*, dp1.*, dtn*, ... continue router = fields[2] ifc_name = fields[3] router_interface_key = f'{router}:{ifc_name}' if router_interface_key not in wanted_interfaces: continue info = r.get(k).decode('utf-8') info = json.loads(info) yield { 'router': router, 'interface': ifc_name, 'service_ids': set([service['id'] for service in info]) } # dict: 'router:interface' -> {'router', 'interface', 'description'} netconf_interface_map = dict([ (f'{i["router"]}:{i["interface"]}', i) for i in _interfaces()]) # dict: 'router:interface' -> {'router', 'interface', set([service_ids])} opsdb_interface_map = dict([ (f'{i["router"]}:{i["interface"]}', i) for i in _lookup_interface_services(netconf_interface_map.keys())]) all_service_ids = set() for r in opsdb_interface_map.values(): all_service_ids |= r['service_ids'] all_service_ids = list(all_service_ids) # dict: service_id[int] -> [list of users] service_user_map = dict() with db.connection(config["ops-db"]) as cx: # for user in opsdb.get_service_users(cx, list(all_service_ids)): service_users = list(opsdb.get_service_users(cx, all_service_ids)) for user in service_users: service_user_map.setdefault( user['service_id'], []).append(user['user']) def _users(ifc_key): """ ifc = 'router:ifc_name' :param ifc: :return: list of users """ users = set() if ifc_key not in opsdb_interface_map: return [] service_id_list = opsdb_interface_map[ifc_key].get('service_ids', []) for service_id in service_id_list: users |= set(service_user_map.get(service_id, [])) return list(users) for k, v in netconf_interface_map.items(): v['users'] = _users(k) yield v def derive_router_hostnames(config): r = get_next_redis(config) routers = r.get('netdash') assert routers netdash_equipment = json.loads(routers.decode('utf-8')) assert isinstance(netdash_equipment, (list, tuple)) netdash_equipment = set(netdash_equipment) opsdb_equipment = set() for k in r.scan_iter('opsdb:interface_services:*'): m = re.match( 'opsdb:interface_services:([^:]+):.*$', k.decode('utf-8')) if m: opsdb_equipment.add(m.group(1)) else: logger.info("Unable to derive router name from %s" % k.decode('utf-8')) return netdash_equipment & opsdb_equipment