diff --git a/inventory_provider/tasks/data.py b/inventory_provider/tasks/data.py index 4c8cd82c0507db4c7eba13c524b4ddb0960e602e..cc1c8adee5fc2275bd4340785d115404171dbbde 100644 --- a/inventory_provider/tasks/data.py +++ b/inventory_provider/tasks/data.py @@ -2,112 +2,11 @@ import json import logging import re -from inventory_provider.db import opsdb, db from inventory_provider.tasks.common import get_next_redis logger = logging.getLogger(__name__) -def build_service_interface_user_list(config): - - def _interfaces(): - """ - yields interface info from netconf - :return: - """ - r = get_next_redis(config) - for k in r.scan_iter('netconf-interfaces:*'): - k = k.decode('utf-8') - m = re.match('^netconf-interfaces:([^:]+):(.+)$', k) - if not m: - logger.error(f'unexpected redis key: "{k}"') - continue # skip, rather than fail the entire update - router_name = m.group(1) - ifc_name = m.group(2) - - info = r.get(k).decode('utf-8') - info = json.loads(info) - - assert ifc_name == info['name'] # sanity - yield { - 'router': router_name, - 'interface': info['name'], - 'description': info['description'] - } - - def _lookup_interface_services(wanted_interfaces): - """ - yields interface info from opsdb (with service id) - ... only interfaces in wanted_interfaces - :param wanted_interfaces: - :return: - """ - r = get_next_redis(config) - for k in r.scan_iter('opsdb:interface_services:*'): - k = k.decode('utf-8') - fields = k.split(':') - if len(fields) < 4: - # there are some strange records - # e.g. TS1.*, ts1.*, dp1.*, dtn*, ... - continue - router = fields[2] - ifc_name = fields[3] - - router_interface_key = f'{router}:{ifc_name}' - if router_interface_key not in wanted_interfaces: - continue - - info = r.get(k).decode('utf-8') - info = json.loads(info) - - yield { - 'router': router, - 'interface': ifc_name, - 'service_ids': set([service['id'] for service in info]) - } - - # dict: 'router:interface' -> {'router', 'interface', 'description'} - netconf_interface_map = dict([ - (f'{i["router"]}:{i["interface"]}', i) for i in _interfaces()]) - - # dict: 'router:interface' -> {'router', 'interface', set([service_ids])} - opsdb_interface_map = dict([ - (f'{i["router"]}:{i["interface"]}', i) - for i in _lookup_interface_services(netconf_interface_map.keys())]) - - all_service_ids = set() - for r in opsdb_interface_map.values(): - all_service_ids |= r['service_ids'] - all_service_ids = list(all_service_ids) - - # dict: service_id[int] -> [list of users] - service_user_map = dict() - with db.connection(config["ops-db"]) as cx: - # for user in opsdb.get_service_users(cx, list(all_service_ids)): - service_users = list(opsdb.get_service_users(cx, all_service_ids)) - for user in service_users: - service_user_map.setdefault( - user['service_id'], []).append(user['user']) - - def _users(ifc_key): - """ - ifc = 'router:ifc_name' - :param ifc: - :return: list of users - """ - users = set() - if ifc_key not in opsdb_interface_map: - return [] - service_id_list = opsdb_interface_map[ifc_key].get('service_ids', []) - for service_id in service_id_list: - users |= set(service_user_map.get(service_id, [])) - return list(users) - - for k, v in netconf_interface_map.items(): - v['users'] = _users(k) - yield v - - def derive_router_hostnames(config): r = get_next_redis(config)