diff --git a/inventory_provider/routes/poller.py b/inventory_provider/routes/poller.py index ad87d6866e6f017371e2efcbca720cb52543af5b..52fcd5dc5065ae919629fe6d732542c5dd8624ce 100644 --- a/inventory_provider/routes/poller.py +++ b/inventory_provider/routes/poller.py @@ -88,12 +88,23 @@ def poller_interface_oids(hostname): @common.require_accepts_json def service_category_interfaces(category): - result = [] - - r = common.get_current_redis() - for k in r.scan_iter(f'interface-services:{category.lower()}:*'): - ifc = r.get(k.decode('utf-8')) - result.append(json.loads(ifc.decode('utf-8'))) + def _interfaces(): + r = common.get_current_redis() + for k in r.scan_iter(f'interface-services:{category.lower()}:*'): + cached_ifc = r.get(k.decode('utf-8')).decode('utf-8') + cached_ifc = json.loads(cached_ifc) + basic_ifc_info = dict() + for k in ['description', 'interface', 'router']: + basic_ifc_info[k] = cached_ifc[k] + if not cached_ifc['users']: + yield basic_ifc_info + else: + for user in cached_ifc['users']: + ifc = {'user': user} + ifc.update(basic_ifc_info) + yield ifc + + result = list(_interfaces()) if not result: return Response( diff --git a/inventory_provider/tasks/worker.py b/inventory_provider/tasks/worker.py index b321d5957f4ea66e08a45c1e9f9777581f163699..ef7f351e11e2d67f994dd016b4c769a374597365 100644 --- a/inventory_provider/tasks/worker.py +++ b/inventory_provider/tasks/worker.py @@ -621,31 +621,32 @@ def _build_service_interface_user_list(): info = r.get(k).decode('utf-8') info = json.loads(info) - for service in info: - yield { - 'router': router, - 'interface': ifc_name, - 'service_id': service['id'] - } + yield { + 'router': router, + 'interface': ifc_name, + 'service_ids': set([service['id'] for service in info]) + } # dict: 'router:interface' -> {'router', 'interface', 'description'} netconf_interface_map = dict([ (f'{i["router"]}:{i["interface"]}', i) for i in _interfaces()]) - # dict: 'router:interface' -> [list of service_ids] - opsdb_interface_map = {} - for i in _lookup_interface_services(netconf_interface_map.keys()): - key = f'{i["router"]}:{i["interface"]}' - opsdb_interface_map.setdefault(key, []).append(i['service_id']) + # dict: 'router:interface' -> {'router', 'interface', set([service_ids])} + opsdb_interface_map = dict([ + (f'{i["router"]}:{i["interface"]}', i) + for i in _lookup_interface_services(netconf_interface_map.keys())]) + + all_service_ids = set() + for r in opsdb_interface_map.values(): + all_service_ids |= r['service_ids'] + all_service_ids = list(all_service_ids) - # dict service_id[int] -> [list of users] - service_user_map = {} + # dict: service_id[int] -> [list of users] + service_user_map = dict() with db.connection(InventoryTask.config["ops-db"]) as cx: - service_ids = set() - for l in opsdb_interface_map.values(): - for id in l: - service_ids.add(id) - for user in opsdb.get_service_users(cx, list(service_ids)): + # for user in opsdb.get_service_users(cx, list(all_service_ids)): + service_users = list(opsdb.get_service_users(cx, all_service_ids)) + for user in service_users: service_user_map.setdefault( user['service_id'], []).append(user['user']) @@ -655,19 +656,17 @@ def _build_service_interface_user_list(): :param ifc: :return: list of users """ - for service_id in opsdb_interface_map.get(ifc_key, []): - for user in service_user_map.get(service_id, []): - yield user + users = set() + if ifc_key not in opsdb_interface_map: + return [] + service_id_list = opsdb_interface_map[ifc_key].get('service_ids', []) + for service_id in service_id_list: + users |= set(service_user_map.get(service_id, [])) + return list(users) for k, v in netconf_interface_map.items(): - users = _users(k) - if not users: - yield v - else: - for u in users: - info = {'user': u} - info.update(v) - yield info + v['users'] = _users(k) + yield v def _build_service_category_interface_list(update_callback=lambda s: None): @@ -680,14 +679,14 @@ def _build_service_category_interface_list(update_callback=lambda s: None): return 'lhcone' return None - r = get_next_redis(InventoryTask.config) - rp = r.pipeline() - update_callback('loading all known interfaces') interfaces = list(_build_service_interface_user_list()) update_callback(f'loaded {len(interfaces)} interfaces, ' 'saving by service category') + r = get_next_redis(InventoryTask.config) + rp = r.pipeline() + for ifc in interfaces: service_type = _classify(ifc) if not service_type: diff --git a/test/test_worker_utils.py b/test/test_worker_utils.py index ad51b87e88cdbc1237995c3eafdd7a7c67f69fa7..4637dedb7a4560cd415a97bac7053e7950d18192 100644 --- a/test/test_worker_utils.py +++ b/test/test_worker_utils.py @@ -33,7 +33,10 @@ def test_build_interface_services(mocked_worker_module): 'description': {'type': 'string'}, 'router': {'type': 'string'}, 'interface': {'type': 'string'}, - 'user': {'type': 'string'} + 'users': { + 'type': 'array', + 'items': {'type': 'string'} + } }, 'required': ['router', 'interface', 'description'], 'additionalProperties': False @@ -47,6 +50,7 @@ def test_build_interface_services(mocked_worker_module): if not k.startswith('interface-services:'): continue + print(v) (_, type, router, ifc_name) = k.split(':') ifc_info = json.loads(v)