diff --git a/inventory_provider/tasks/worker.py b/inventory_provider/tasks/worker.py index b321d5957f4ea66e08a45c1e9f9777581f163699..1baad7906511d54c8333b19426485e1a8a8bc1d1 100644 --- a/inventory_provider/tasks/worker.py +++ b/inventory_provider/tasks/worker.py @@ -621,32 +621,32 @@ def _build_service_interface_user_list(): info = r.get(k).decode('utf-8') info = json.loads(info) - for service in info: - yield { - 'router': router, - 'interface': ifc_name, - 'service_id': service['id'] - } + yield { + 'router': router, + 'interface': ifc_name, + 'service_ids': set([service['id'] for service in info]) + } # dict: 'router:interface' -> {'router', 'interface', 'description'} netconf_interface_map = dict([ (f'{i["router"]}:{i["interface"]}', i) for i in _interfaces()]) - # dict: 'router:interface' -> [list of service_ids] - opsdb_interface_map = {} - for i in _lookup_interface_services(netconf_interface_map.keys()): - key = f'{i["router"]}:{i["interface"]}' - opsdb_interface_map.setdefault(key, []).append(i['service_id']) + # dict: 'router:interface' -> {'router', 'interface', set([service_ids])} + opsdb_interface_map = dict([ + (f'{i["router"]}:{i["interface"]}', i) + for i in _lookup_interface_services(netconf_interface_map.keys())]) + + all_service_ids = set() + for r in opsdb_interface_map.values(): + all_service_ids |= r['service_ids'] - # dict service_id[int] -> [list of users] - service_user_map = {} + # dict: service_id[int] -> [list of users] + service_user_map = dict() with db.connection(InventoryTask.config["ops-db"]) as cx: - service_ids = set() - for l in opsdb_interface_map.values(): - for id in l: - service_ids.add(id) - for user in opsdb.get_service_users(cx, list(service_ids)): - service_user_map.setdefault( + # for user in opsdb.get_service_users(cx, list(all_service_ids)): + service_users = list(opsdb.get_service_users(cx, list(all_service_ids))) + for user in service_users: + service_user_map.setdefault( user['service_id'], []).append(user['user']) def _users(ifc_key): @@ -655,19 +655,17 @@ def _build_service_interface_user_list(): :param ifc: :return: list of users """ - for service_id in opsdb_interface_map.get(ifc_key, []): - for user in service_user_map.get(service_id, []): - yield user + users = set() + if ifc_key not in opsdb_interface_map: + return [] + service_id_list = opsdb_interface_map[ifc_key].get('service_ids', []) + for service_id in service_id_list: + users |= set(service_user_map.get(service_id, [])) + return list(users) for k, v in netconf_interface_map.items(): - users = _users(k) - if not users: - yield v - else: - for u in users: - info = {'user': u} - info.update(v) - yield info + v['users'] = _users(k) + yield v def _build_service_category_interface_list(update_callback=lambda s: None): @@ -680,14 +678,15 @@ def _build_service_category_interface_list(update_callback=lambda s: None): return 'lhcone' return None - r = get_next_redis(InventoryTask.config) - rp = r.pipeline() update_callback('loading all known interfaces') interfaces = list(_build_service_interface_user_list()) update_callback(f'loaded {len(interfaces)} interfaces, ' 'saving by service category') + r = get_next_redis(InventoryTask.config) + rp = r.pipeline() + for ifc in interfaces: service_type = _classify(ifc) if not service_type: diff --git a/test/test_worker_utils.py b/test/test_worker_utils.py index ad51b87e88cdbc1237995c3eafdd7a7c67f69fa7..4637dedb7a4560cd415a97bac7053e7950d18192 100644 --- a/test/test_worker_utils.py +++ b/test/test_worker_utils.py @@ -33,7 +33,10 @@ def test_build_interface_services(mocked_worker_module): 'description': {'type': 'string'}, 'router': {'type': 'string'}, 'interface': {'type': 'string'}, - 'user': {'type': 'string'} + 'users': { + 'type': 'array', + 'items': {'type': 'string'} + } }, 'required': ['router', 'interface', 'description'], 'additionalProperties': False @@ -47,6 +50,7 @@ def test_build_interface_services(mocked_worker_module): if not k.startswith('interface-services:'): continue + print(v) (_, type, router, ifc_name) = k.split(':') ifc_info = json.loads(v)