added ignore-cache to all routes that cache data

inventory_provider/routes/classifier.py

@@ -2,13 +2,13 @@ import ipaddress
 import json
 import logging
 import re
-from distutils.util import strtobool
 from typing import Optional
 
 from flask import Blueprint, Response, request
 from redis import Redis
 
 from inventory_provider.routes import common
+from inventory_provider.routes.common import _ignore_cache_or_retrieve
 
 routes = Blueprint("inventory-data-classifier-support-routes", __name__)
 
@@ -261,19 +261,9 @@ def get_juniper_link_info(source_equipment: str, interface: str) -> Response:
     cache_key = \
         f'classifier-cache:juniper:{ims_source_equipment}:{ims_interface}'
 
-    ignore_cache = request.args.get('ignore-cache', default='false', type=str)
-    try:
-        ignore_cache = strtobool(ignore_cache)
-    except ValueError:
-        ignore_cache = False
-    if ignore_cache:
-        result = False
-    else:
-        result = r.get(cache_key)
+    result = _ignore_cache_or_retrieve(request, cache_key, r)
 
-    if result:
-        result = result.decode('utf-8')
-    else:
+    if not result:
         result = {
             'interface': _link_interface_info(r, source_equipment, interface)
         }
@@ -522,21 +512,9 @@ def peer_info(address_str: str) -> Response:
     r = common.get_current_redis()
 
     cache_key = f'classifier-cache:peer:{address_str}'
+    result = _ignore_cache_or_retrieve(request, cache_key, r)
 
-    ignore_cache = request.args.get('ignore-cache', default='false', type=str)
-    try:
-        ignore_cache = strtobool(ignore_cache)
-    except ValueError:
-        ignore_cache = False
-    if ignore_cache:
-        result = False
-    else:
-        result = r.get(cache_key)
-
-    if result:
-        result = result.decode('utf-8')
-    else:
-
+    if not result:
         result = {
             'interfaces': [],
             'locations': [],
@@ -626,19 +604,9 @@ def get_trap_metadata(source_equipment: str, interface: str, circuit_id: str) \
 
     r = common.get_current_redis()
 
-    ignore_cache = request.args.get('ignore-cache', default='false', type=str)
-    try:
-        ignore_cache = strtobool(ignore_cache)
-    except ValueError:
-        ignore_cache = False
-    if ignore_cache:
-        result = False
-    else:
-        result = r.get(cache_key)
+    result = _ignore_cache_or_retrieve(request, cache_key, r)
 
-    if result:
-        result = result.decode('utf-8')
-    else:
+    if not result:
         result = {
             'locations': []
         }
@@ -693,19 +661,9 @@ def get_fiberlink_trap_metadata(ne_name_str: str, object_name_str: str) \
     cache_key = \
         f'classifier-cache:fiberlink:{ne_name_str}:{object_name_str}'
 
-    ignore_cache = request.args.get('ignore-cache', default='false', type=str)
-    try:
-        ignore_cache = strtobool(ignore_cache)
-    except ValueError:
-        ignore_cache = False
-    if ignore_cache:
-        result = False
-    else:
-        result = r.get(cache_key)
+    result = _ignore_cache_or_retrieve(request, cache_key, r)
 
-    if result:
-        result = result.decode('utf-8')
-    else:
+    if not result:
         equipment_a = matches[0][0]
         equipment_b = matches[1][0]
         interface_a = interfaces[0]
@@ -828,19 +786,9 @@ def get_coriant_info(equipment_name: str, entity_string: str) -> Response:
     cache_key = 'classifier-cache:coriant:' \
                 f'{ims_source_equipment}:{ims_interface}'
 
-    ignore_cache = request.args.get('ignore-cache', default='false', type=str)
-    try:
-        ignore_cache = strtobool(ignore_cache)
-    except ValueError:
-        ignore_cache = False
-    if ignore_cache:
-        result = False
-    else:
-        result = r.get(cache_key)
+    result = _ignore_cache_or_retrieve(request, cache_key, r)
 
-    if result:
-        result = result.decode('utf-8')
-    else:
+    if not result:
 
         m = re.match(r'^(\d+\-\d+)\.(\d+)', ims_interface)
         if not m:

inventory_provider/routes/common.py

@@ -6,6 +6,7 @@ import queue
 import random
 import threading
 
+from distutils.util import strtobool
 from lxml import etree
 import requests
 from flask import request, Response, current_app, g
@@ -16,6 +17,24 @@ _DECODE_TYPE_XML = 'xml'
 _DECODE_TYPE_JSON = 'json'
 
 
+def _ignore_cache_or_retrieve(request_, cache_key, r):
+
+    ignore_cache = request_.args.get('ignore-cache', default='false', type=str)
+    try:
+        ignore_cache = strtobool(ignore_cache)
+    except ValueError:
+        ignore_cache = False
+    if ignore_cache:
+        result = False
+        logger.debug('ignoring cache')
+    else:
+        result = r.get(cache_key)
+
+    if result:
+        result = result.decode('utf-8')
+    return result
+
+
 def ims_hostname_decorator(field):
     """
     Decorator to convert host names to various formats to try to match what is

inventory_provider/routes/data.py

@@ -2,9 +2,10 @@ import json
 import logging
 import re
 
-from flask import Blueprint, jsonify, Response, current_app
+from flask import Blueprint, jsonify, Response, current_app, request
 
 from inventory_provider.routes import common
+from inventory_provider.routes.common import _ignore_cache_or_retrieve
 
 logger = logging.getLogger(__name__)
 
@@ -139,11 +140,10 @@ def router_interfaces(hostname=None):
         if hostname else 'classifier-cache:netconf-interfaces:all'
 
     r = common.get_current_redis()
-    result = r.get(cache_key)
-    if result:
-        result = result.decode('utf-8')
 
-    else:
+    result = _ignore_cache_or_retrieve(request, cache_key, r)
+
+    if not result:
         key_pattern = f'netconf-interfaces:{hostname}:*' \
             if hostname else 'netconf-interfaces:*'
         config = current_app.config['INVENTORY_PROVIDER_CONFIG']

inventory_provider/routes/lg.py

 import json
 import logging
 
-from flask import Blueprint, jsonify, Response
+from flask import Blueprint, jsonify, Response, request
 
 from inventory_provider.routes import common
+from inventory_provider.routes.common import _ignore_cache_or_retrieve
 
 logger = logging.getLogger(__name__)
 
@@ -107,11 +108,10 @@ def routers(access):
                 yield rtr
 
     cache_key = f'classifier-cache:ims-lg:{access}'
-    result = redis.get(cache_key)
 
-    if result:
-        result = json.loads(result.decode('utf-8'))
-    else:
+    result = _ignore_cache_or_retrieve(request, cache_key, redis)
+
+    if not result:
         result = list(_routers())
         # cache this data for the next call
         redis.set(cache_key, json.dumps(result).encode('utf-8'))

inventory_provider/routes/msr.py

 import itertools
 import json
 
-from flask import Blueprint, jsonify, Response
+from flask import Blueprint, jsonify, Response, request
 
 from inventory_provider.routes import common
+from inventory_provider.routes.common import _ignore_cache_or_retrieve
 
 routes = Blueprint("msr-query-routes", __name__)
 
@@ -104,11 +105,10 @@ def access_services():
             yield json.loads(service)
 
     cache_key = 'classifier-cache:msr:access-services'
-    result = redis.get(cache_key)
 
-    if result:
-        result = json.loads(result.decode('utf-8'))
-    else:
+    result = _ignore_cache_or_retrieve(request, cache_key, redis)
+
+    if not result:
         result = list(_services())
         # cache this data for the next call
         redis.set(cache_key, json.dumps(result).encode('utf-8'))
@@ -159,11 +159,9 @@ def _handle_peering_group_request(name, cache_key, group_key_base):
     if name:
         cache_key = f'{cache_key}:{name}'
 
-    items = r.get(cache_key)
+    items = _ignore_cache_or_retrieve(request, cache_key, r)
 
-    if items:
-        items = json.loads(items.decode('utf-8'))
-    else:
+    if not items:
         if name:
             items = _load_list_items(f'{group_key_base}:{name}')
         else:
@@ -265,11 +263,9 @@ def _handle_peering_group_list_request(cache_key, group_key_base):
             k = k.decode('utf-8')
             yield k[len(group_key_base) + 1:]
 
-    names = r.get(cache_key)
+    names = _ignore_cache_or_retrieve(request, cache_key, r)
 
-    if names:
-        names = json.loads(names.decode('utf-8'))
-    else:
+    if not names:
         names = list(_get_all_subkeys())
         if not names:
             return Response(

inventory_provider/routes/poller.py

@@ -2,11 +2,12 @@ import json
 import logging
 import re
 
-from flask import Blueprint, Response, current_app
+from flask import Blueprint, Response, current_app, request
 from inventory_provider import juniper
 from inventory_provider.routes import common
 from inventory_provider.routes.classifier import get_ims_equipment_name, \
     get_ims_interface
+from inventory_provider.routes.common import _ignore_cache_or_retrieve
 
 logger = logging.getLogger(__name__)
 routes = Blueprint('poller-support-routes', __name__)
@@ -291,10 +292,9 @@ def interfaces(hostname=None):
 
     r = common.get_current_redis()
 
-    result = r.get(cache_key)
-    if result:
-        result = result.decode('utf-8')
-    else:
+    result = _ignore_cache_or_retrieve(request, cache_key, r)
+
+    if not result:
         result = list(_load_interfaces_to_poll(hostname))
         if not result:
             return Response(
@@ -387,10 +387,9 @@ def interface_speeds(hostname=None):
 
     r = common.get_current_redis()
 
-    result = r.get(cache_key)
-    if result:
-        result = result.decode('utf-8')
-    else:
+    result = _ignore_cache_or_retrieve(request, cache_key, r)
+
+    if not result:
         result = list(_load_interfaces_and_speeds(hostname))
         if not result:
             return Response(