Skip to content
Snippets Groups Projects
Commit 9522c3da authored by Erik Reid's avatar Erik Reid
Browse files

less flat service category cache schema

parent d8d81c80
No related branches found
No related tags found
No related merge requests found
...@@ -621,32 +621,32 @@ def _build_service_interface_user_list(): ...@@ -621,32 +621,32 @@ def _build_service_interface_user_list():
info = r.get(k).decode('utf-8') info = r.get(k).decode('utf-8')
info = json.loads(info) info = json.loads(info)
for service in info: yield {
yield { 'router': router,
'router': router, 'interface': ifc_name,
'interface': ifc_name, 'service_ids': set([service['id'] for service in info])
'service_id': service['id'] }
}
# dict: 'router:interface' -> {'router', 'interface', 'description'} # dict: 'router:interface' -> {'router', 'interface', 'description'}
netconf_interface_map = dict([ netconf_interface_map = dict([
(f'{i["router"]}:{i["interface"]}', i) for i in _interfaces()]) (f'{i["router"]}:{i["interface"]}', i) for i in _interfaces()])
# dict: 'router:interface' -> [list of service_ids] # dict: 'router:interface' -> {'router', 'interface', set([service_ids])}
opsdb_interface_map = {} opsdb_interface_map = dict([
for i in _lookup_interface_services(netconf_interface_map.keys()): (f'{i["router"]}:{i["interface"]}', i)
key = f'{i["router"]}:{i["interface"]}' for i in _lookup_interface_services(netconf_interface_map.keys())])
opsdb_interface_map.setdefault(key, []).append(i['service_id'])
all_service_ids = set()
for r in opsdb_interface_map.values():
all_service_ids |= r['service_ids']
# dict service_id[int] -> [list of users] # dict: service_id[int] -> [list of users]
service_user_map = {} service_user_map = dict()
with db.connection(InventoryTask.config["ops-db"]) as cx: with db.connection(InventoryTask.config["ops-db"]) as cx:
service_ids = set() # for user in opsdb.get_service_users(cx, list(all_service_ids)):
for l in opsdb_interface_map.values(): service_users = list(opsdb.get_service_users(cx, list(all_service_ids)))
for id in l: for user in service_users:
service_ids.add(id) service_user_map.setdefault(
for user in opsdb.get_service_users(cx, list(service_ids)):
service_user_map.setdefault(
user['service_id'], []).append(user['user']) user['service_id'], []).append(user['user'])
def _users(ifc_key): def _users(ifc_key):
...@@ -655,19 +655,17 @@ def _build_service_interface_user_list(): ...@@ -655,19 +655,17 @@ def _build_service_interface_user_list():
:param ifc: :param ifc:
:return: list of users :return: list of users
""" """
for service_id in opsdb_interface_map.get(ifc_key, []): users = set()
for user in service_user_map.get(service_id, []): if ifc_key not in opsdb_interface_map:
yield user return []
service_id_list = opsdb_interface_map[ifc_key].get('service_ids', [])
for service_id in service_id_list:
users |= set(service_user_map.get(service_id, []))
return list(users)
for k, v in netconf_interface_map.items(): for k, v in netconf_interface_map.items():
users = _users(k) v['users'] = _users(k)
if not users: yield v
yield v
else:
for u in users:
info = {'user': u}
info.update(v)
yield info
def _build_service_category_interface_list(update_callback=lambda s: None): def _build_service_category_interface_list(update_callback=lambda s: None):
...@@ -680,14 +678,15 @@ def _build_service_category_interface_list(update_callback=lambda s: None): ...@@ -680,14 +678,15 @@ def _build_service_category_interface_list(update_callback=lambda s: None):
return 'lhcone' return 'lhcone'
return None return None
r = get_next_redis(InventoryTask.config)
rp = r.pipeline()
update_callback('loading all known interfaces') update_callback('loading all known interfaces')
interfaces = list(_build_service_interface_user_list()) interfaces = list(_build_service_interface_user_list())
update_callback(f'loaded {len(interfaces)} interfaces, ' update_callback(f'loaded {len(interfaces)} interfaces, '
'saving by service category') 'saving by service category')
r = get_next_redis(InventoryTask.config)
rp = r.pipeline()
for ifc in interfaces: for ifc in interfaces:
service_type = _classify(ifc) service_type = _classify(ifc)
if not service_type: if not service_type:
......
...@@ -33,7 +33,10 @@ def test_build_interface_services(mocked_worker_module): ...@@ -33,7 +33,10 @@ def test_build_interface_services(mocked_worker_module):
'description': {'type': 'string'}, 'description': {'type': 'string'},
'router': {'type': 'string'}, 'router': {'type': 'string'},
'interface': {'type': 'string'}, 'interface': {'type': 'string'},
'user': {'type': 'string'} 'users': {
'type': 'array',
'items': {'type': 'string'}
}
}, },
'required': ['router', 'interface', 'description'], 'required': ['router', 'interface', 'description'],
'additionalProperties': False 'additionalProperties': False
...@@ -47,6 +50,7 @@ def test_build_interface_services(mocked_worker_module): ...@@ -47,6 +50,7 @@ def test_build_interface_services(mocked_worker_module):
if not k.startswith('interface-services:'): if not k.startswith('interface-services:'):
continue continue
print(v)
(_, type, router, ifc_name) = k.split(':') (_, type, router, ifc_name) = k.split(':')
ifc_info = json.loads(v) ifc_info = json.loads(v)
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment