Finished release 0.43.

Changelog.md

@@ -2,6 +2,9 @@
 
 All notable changes to this project will be documented in this file.
 
+## [0.43] - 2020-05-12
+- added poller api response caching
+
 ## [0.42] - 2020-05-11
 - DBOARD3-277: stop using Junos Space
 - DBOARD3-244: add visible status details for update inventory processing

inventory_provider/db/opsdb.py

@@ -523,9 +523,9 @@ AND circuit_type = 'Path'
 
 def get_service_users(connection, service_ids):
 
-    def _sublists(l, n):
-        for x in range(0, len(l), n):
-            yield l[x:x + n]
+    def _sublists(all, n):
+        for x in range(0, len(all), n):
+            yield all[x:x + n]
 
     # not sure how to use a tuple in a prepared statement,
     # so this is just doing a dumb string replacement ...

inventory_provider/juniper.py

@@ -313,7 +313,7 @@ def load_routers_from_netdash(url):
     r = requests.get(url=url)
     r.raise_for_status()
     return [
-        l.strip() for l in r.text.splitlines() if l.strip()
+        ln.strip() for ln in r.text.splitlines() if ln.strip()
     ]
 
 

inventory_provider/routes/classifier.py

@@ -21,14 +21,14 @@ def _LOCATION(equipment, name, abbreviation):
     }
 
 
-def _remove_duplicates_from_list(l):
+def _remove_duplicates_from_list(all):
     """
     removes duplicates from the input list
     the list items must be encodable as json
     :param l:
     :return: a new list with unique elements
     """
-    tmp_dict = dict([(json.dumps(item, sort_keys=True), item) for item in l])
+    tmp_dict = dict([(json.dumps(item, sort_keys=True), item) for item in all])
     return list(tmp_dict.values())
 
 

inventory_provider/routes/ims_lg.py

@@ -34,7 +34,7 @@ def routers(access):
 
     def _routers():
         i = 0
-        for k in redis.scan_iter(f'ims:lg:*'):
+        for k in redis.scan_iter('ims:lg:*'):
             rtr = redis.get(k.decode('utf-8')).decode('utf-8')
             rtr = json.loads(rtr)
             i += 1

inventory_provider/routes/lg.py

@@ -34,7 +34,7 @@ def routers(access):
 
     def _routers():
         i = 0
-        for k in redis.scan_iter(f'opsdb:lg:*'):
+        for k in redis.scan_iter('opsdb:lg:*'):
             rtr = redis.get(k.decode('utf-8')).decode('utf-8')
             rtr = json.loads(rtr)
             i += 1

inventory_provider/routes/msr.py

@@ -23,7 +23,7 @@ def access_services():
             service = redis.get(k.decode('utf-8')).decode('utf-8')
             yield json.loads(service)
 
-    cache_key = f'classifier-cache:msr:access-services'
+    cache_key = 'classifier-cache:msr:access-services'
     result = redis.get(cache_key)
 
     if result:
@@ -35,7 +35,7 @@ def access_services():
 
     if not result:
         return Response(
-            response=f'no access services found',
+            response='no access services found',
             status=404,
             mimetype="text/html")
 

inventory_provider/routes/poller.py

@@ -88,10 +88,9 @@ def poller_interface_oids(hostname):
 @common.require_accepts_json
 def service_category_interfaces(category):
 
-    def _interfaces():
-        r = common.get_current_redis()
-        for k in r.scan_iter(f'interface-services:{category.lower()}:*'):
-            cached_ifc = r.get(k.decode('utf-8')).decode('utf-8')
+    def _interfaces(rc, cat):
+        for k in rc.scan_iter(f'interface-services:{cat}:*'):
+            cached_ifc = rc.get(k.decode('utf-8')).decode('utf-8')
             cached_ifc = json.loads(cached_ifc)
             basic_ifc_info = dict()
             for k in ['description', 'interface', 'router']:
@@ -104,12 +103,22 @@ def service_category_interfaces(category):
                     ifc.update(basic_ifc_info)
                     yield ifc
 
-    result = list(_interfaces())
+    category = category.lower()
 
-    if not result:
-        return Response(
-            response=f'no info available for service category {category}',
-            status=404,
-            mimetype="text/html")
+    r = common.get_current_redis()
+    cache_key = f'classifier-cache:poller:service:{category}'
+    result = r.get(cache_key)
+
+    if result:
+        result = json.loads(result.decode('utf-8'))
+    else:
+        result = list(_interfaces(r, category))
+        if result:
+            r.set(cache_key, json.dumps(result).encode('utf-8'))
+        else:
+            return Response(
+                response=f'no info available for service category {category}',
+                status=404,
+                mimetype="text/html")
 
     return jsonify(result)

inventory_provider/tasks/config.py

@@ -39,7 +39,7 @@ _broker_db_index = params.get('celery-db-index', DEFAULT_CELERY_DB_INDEX)
 
 if ':' in _broker_hostname:
     # assume this means hostname is an ipv6 address
-    _broker_hostname = f'[_broker_hostname]'
+    _broker_hostname = f'[{_broker_hostname}]'
 
 broker_url = result_backend = (f'{_broker_scheme}://{_broker_hostname}'
                                f':{_broker_port}/{_broker_db_index}')

setup.py

@@ -2,7 +2,7 @@ from setuptools import setup, find_packages
 
 setup(
     name='inventory-provider',
-    version="0.42",
+    version="0.43",
     author='GEANT',
     author_email='swd@geant.org',
     description='Dashboard inventory provider',