Finished feature POL1-135-refactor-storage-model.

inventory_provider/routes/poller.py

@@ -88,12 +88,23 @@ def poller_interface_oids(hostname):
 @common.require_accepts_json
 def service_category_interfaces(category):
 
-    result = []
-
-    r = common.get_current_redis()
-    for k in r.scan_iter(f'interface-services:{category.lower()}:*'):
-        ifc = r.get(k.decode('utf-8'))
-        result.append(json.loads(ifc.decode('utf-8')))
+    def _interfaces():
+        r = common.get_current_redis()
+        for k in r.scan_iter(f'interface-services:{category.lower()}:*'):
+            cached_ifc = r.get(k.decode('utf-8')).decode('utf-8')
+            cached_ifc = json.loads(cached_ifc)
+            basic_ifc_info = dict()
+            for k in ['description', 'interface', 'router']:
+                basic_ifc_info[k] = cached_ifc[k]
+            if not cached_ifc['users']:
+                yield basic_ifc_info
+            else:
+                for user in cached_ifc['users']:
+                    ifc = {'user': user}
+                    ifc.update(basic_ifc_info)
+                    yield ifc
+
+    result = list(_interfaces())
 
     if not result:
         return Response(

inventory_provider/tasks/worker.py

@@ -621,31 +621,32 @@ def _build_service_interface_user_list():
             info = r.get(k).decode('utf-8')
             info = json.loads(info)
 
-            for service in info:
-                yield {
-                    'router': router,
-                    'interface': ifc_name,
-                    'service_id': service['id']
-                }
+            yield {
+                'router': router,
+                'interface': ifc_name,
+                'service_ids': set([service['id'] for service in info])
+            }
 
     # dict: 'router:interface' -> {'router', 'interface', 'description'}
     netconf_interface_map = dict([
         (f'{i["router"]}:{i["interface"]}', i) for i in _interfaces()])
 
-    # dict: 'router:interface' -> [list of service_ids]
-    opsdb_interface_map = {}
-    for i in _lookup_interface_services(netconf_interface_map.keys()):
-        key = f'{i["router"]}:{i["interface"]}'
-        opsdb_interface_map.setdefault(key, []).append(i['service_id'])
+    # dict: 'router:interface' -> {'router', 'interface', set([service_ids])}
+    opsdb_interface_map = dict([
+        (f'{i["router"]}:{i["interface"]}', i)
+        for i in _lookup_interface_services(netconf_interface_map.keys())])
+
+    all_service_ids = set()
+    for r in opsdb_interface_map.values():
+        all_service_ids |= r['service_ids']
+    all_service_ids = list(all_service_ids)
 
-    # dict service_id[int] -> [list of users]
-    service_user_map = {}
+    # dict: service_id[int] -> [list of users]
+    service_user_map = dict()
     with db.connection(InventoryTask.config["ops-db"]) as cx:
-        service_ids = set()
-        for l in opsdb_interface_map.values():
-            for id in l:
-                service_ids.add(id)
-        for user in opsdb.get_service_users(cx, list(service_ids)):
+        # for user in opsdb.get_service_users(cx, list(all_service_ids)):
+        service_users = list(opsdb.get_service_users(cx, all_service_ids))
+        for user in service_users:
             service_user_map.setdefault(
                 user['service_id'], []).append(user['user'])
 
@@ -655,19 +656,17 @@ def _build_service_interface_user_list():
         :param ifc:
         :return: list of users
         """
-        for service_id in opsdb_interface_map.get(ifc_key, []):
-            for user in service_user_map.get(service_id, []):
-                yield user
+        users = set()
+        if ifc_key not in opsdb_interface_map:
+            return []
+        service_id_list = opsdb_interface_map[ifc_key].get('service_ids', [])
+        for service_id in service_id_list:
+            users |= set(service_user_map.get(service_id, []))
+        return list(users)
 
     for k, v in netconf_interface_map.items():
-        users = _users(k)
-        if not users:
-            yield v
-        else:
-            for u in users:
-                info = {'user': u}
-                info.update(v)
-                yield info
+        v['users'] = _users(k)
+        yield v
 
 
 def _build_service_category_interface_list(update_callback=lambda s: None):
@@ -680,14 +679,14 @@ def _build_service_category_interface_list(update_callback=lambda s: None):
             return 'lhcone'
         return None
 
-    r = get_next_redis(InventoryTask.config)
-    rp = r.pipeline()
-
     update_callback('loading all known interfaces')
     interfaces = list(_build_service_interface_user_list())
     update_callback(f'loaded {len(interfaces)} interfaces, '
                     'saving by service category')
 
+    r = get_next_redis(InventoryTask.config)
+    rp = r.pipeline()
+
     for ifc in interfaces:
         service_type = _classify(ifc)
         if not service_type:

test/test_worker_utils.py

@@ -33,7 +33,10 @@ def test_build_interface_services(mocked_worker_module):
             'description': {'type': 'string'},
             'router': {'type': 'string'},
             'interface': {'type': 'string'},
-            'user': {'type': 'string'}
+            'users': {
+                'type': 'array',
+                'items': {'type': 'string'}
+            }
         },
         'required': ['router', 'interface', 'description'],
         'additionalProperties': False
@@ -47,6 +50,7 @@ def test_build_interface_services(mocked_worker_module):
         if not k.startswith('interface-services:'):
             continue
 
+        print(v)
         (_, type, router, ifc_name) = k.split(':')
 
         ifc_info = json.loads(v)