less flat service category cache schema

inventory_provider/tasks/worker.py

@@ -621,32 +621,32 @@ def _build_service_interface_user_list():
             info = r.get(k).decode('utf-8')
             info = json.loads(info)
 
-            for service in info:
-                yield {
-                    'router': router,
-                    'interface': ifc_name,
-                    'service_id': service['id']
-                }
+            yield {
+                'router': router,
+                'interface': ifc_name,
+                'service_ids': set([service['id'] for service in info])
+            }
 
     # dict: 'router:interface' -> {'router', 'interface', 'description'}
     netconf_interface_map = dict([
         (f'{i["router"]}:{i["interface"]}', i) for i in _interfaces()])
 
-    # dict: 'router:interface' -> [list of service_ids]
-    opsdb_interface_map = {}
-    for i in _lookup_interface_services(netconf_interface_map.keys()):
-        key = f'{i["router"]}:{i["interface"]}'
-        opsdb_interface_map.setdefault(key, []).append(i['service_id'])
+    # dict: 'router:interface' -> {'router', 'interface', set([service_ids])}
+    opsdb_interface_map = dict([
+        (f'{i["router"]}:{i["interface"]}', i)
+        for i in _lookup_interface_services(netconf_interface_map.keys())])
+
+    all_service_ids = set()
+    for r in opsdb_interface_map.values():
+        all_service_ids |= r['service_ids']
 
-    # dict service_id[int] -> [list of users]
-    service_user_map = {}
+    # dict: service_id[int] -> [list of users]
+    service_user_map = dict()
     with db.connection(InventoryTask.config["ops-db"]) as cx:
-        service_ids = set()
-        for l in opsdb_interface_map.values():
-            for id in l:
-                service_ids.add(id)
-        for user in opsdb.get_service_users(cx, list(service_ids)):
-            service_user_map.setdefault(
+        # for user in opsdb.get_service_users(cx, list(all_service_ids)):
+        service_users = list(opsdb.get_service_users(cx, list(all_service_ids)))
+        for user in service_users:
+                service_user_map.setdefault(
                 user['service_id'], []).append(user['user'])
 
     def _users(ifc_key):
@@ -655,19 +655,17 @@ def _build_service_interface_user_list():
         :param ifc:
         :return: list of users
         """
-        for service_id in opsdb_interface_map.get(ifc_key, []):
-            for user in service_user_map.get(service_id, []):
-                yield user
+        users = set()
+        if ifc_key not in opsdb_interface_map:
+            return []
+        service_id_list = opsdb_interface_map[ifc_key].get('service_ids', [])
+        for service_id in service_id_list:
+            users |= set(service_user_map.get(service_id, []))
+        return list(users)
 
     for k, v in netconf_interface_map.items():
-        users = _users(k)
-        if not users:
-            yield v
-        else:
-            for u in users:
-                info = {'user': u}
-                info.update(v)
-                yield info
+        v['users'] = _users(k)
+        yield v
 
 
 def _build_service_category_interface_list(update_callback=lambda s: None):
@@ -680,14 +678,15 @@ def _build_service_category_interface_list(update_callback=lambda s: None):
             return 'lhcone'
         return None
 
-    r = get_next_redis(InventoryTask.config)
-    rp = r.pipeline()
 
     update_callback('loading all known interfaces')
     interfaces = list(_build_service_interface_user_list())
     update_callback(f'loaded {len(interfaces)} interfaces, '
                     'saving by service category')
 
+    r = get_next_redis(InventoryTask.config)
+    rp = r.pipeline()
+
     for ifc in interfaces:
         service_type = _classify(ifc)
         if not service_type:

test/test_worker_utils.py

@@ -33,7 +33,10 @@ def test_build_interface_services(mocked_worker_module):
             'description': {'type': 'string'},
             'router': {'type': 'string'},
             'interface': {'type': 'string'},
-            'user': {'type': 'string'}
+            'users': {
+                'type': 'array',
+                'items': {'type': 'string'}
+            }
         },
         'required': ['router', 'interface', 'description'],
         'additionalProperties': False
@@ -47,6 +50,7 @@ def test_build_interface_services(mocked_worker_module):
         if not k.startswith('interface-services:'):
             continue
 
+        print(v)
         (_, type, router, ifc_name) = k.split(':')
 
         ifc_info = json.loads(v)