From 1708c1b3a2d08c4ecd7e30e83b2b70b64db38e8e Mon Sep 17 00:00:00 2001
From: Robert Latta <robert.latta@geant.org>
Date: Fri, 26 Mar 2021 17:09:59 +0000
Subject: [PATCH] added ignore-cache to all routes that cache data
---
inventory_provider/routes/classifier.py | 74 ++++---------------------
inventory_provider/routes/common.py | 19 +++++++
inventory_provider/routes/data.py | 10 ++--
inventory_provider/routes/lg.py | 10 ++--
inventory_provider/routes/msr.py | 22 +++-----
inventory_provider/routes/poller.py | 17 +++---
6 files changed, 57 insertions(+), 95 deletions(-)
diff --git a/inventory_provider/routes/classifier.py b/inventory_provider/routes/classifier.py
index 6dcb9e42..c2615758 100644
--- a/inventory_provider/routes/classifier.py
+++ b/inventory_provider/routes/classifier.py
@@ -2,13 +2,13 @@ import ipaddress
import json
import logging
import re
-from distutils.util import strtobool
from typing import Optional
from flask import Blueprint, Response, request
from redis import Redis
from inventory_provider.routes import common
+from inventory_provider.routes.common import _ignore_cache_or_retrieve
routes = Blueprint("inventory-data-classifier-support-routes", __name__)
@@ -261,19 +261,9 @@ def get_juniper_link_info(source_equipment: str, interface: str) -> Response:
cache_key = \
f'classifier-cache:juniper:{ims_source_equipment}:{ims_interface}'
- ignore_cache = request.args.get('ignore-cache', default='false', type=str)
- try:
- ignore_cache = strtobool(ignore_cache)
- except ValueError:
- ignore_cache = False
- if ignore_cache:
- result = False
- else:
- result = r.get(cache_key)
+ result = _ignore_cache_or_retrieve(request, cache_key, r)
- if result:
- result = result.decode('utf-8')
- else:
+ if not result:
result = {
'interface': _link_interface_info(r, source_equipment, interface)
}
@@ -522,21 +512,9 @@ def peer_info(address_str: str) -> Response:
r = common.get_current_redis()
cache_key = f'classifier-cache:peer:{address_str}'
+ result = _ignore_cache_or_retrieve(request, cache_key, r)
- ignore_cache = request.args.get('ignore-cache', default='false', type=str)
- try:
- ignore_cache = strtobool(ignore_cache)
- except ValueError:
- ignore_cache = False
- if ignore_cache:
- result = False
- else:
- result = r.get(cache_key)
-
- if result:
- result = result.decode('utf-8')
- else:
-
+ if not result:
result = {
'interfaces': [],
'locations': [],
@@ -626,19 +604,9 @@ def get_trap_metadata(source_equipment: str, interface: str, circuit_id: str) \
r = common.get_current_redis()
- ignore_cache = request.args.get('ignore-cache', default='false', type=str)
- try:
- ignore_cache = strtobool(ignore_cache)
- except ValueError:
- ignore_cache = False
- if ignore_cache:
- result = False
- else:
- result = r.get(cache_key)
+ result = _ignore_cache_or_retrieve(request, cache_key, r)
- if result:
- result = result.decode('utf-8')
- else:
+ if not result:
result = {
'locations': []
}
@@ -693,19 +661,9 @@ def get_fiberlink_trap_metadata(ne_name_str: str, object_name_str: str) \
cache_key = \
f'classifier-cache:fiberlink:{ne_name_str}:{object_name_str}'
- ignore_cache = request.args.get('ignore-cache', default='false', type=str)
- try:
- ignore_cache = strtobool(ignore_cache)
- except ValueError:
- ignore_cache = False
- if ignore_cache:
- result = False
- else:
- result = r.get(cache_key)
+ result = _ignore_cache_or_retrieve(request, cache_key, r)
- if result:
- result = result.decode('utf-8')
- else:
+ if not result:
equipment_a = matches[0][0]
equipment_b = matches[1][0]
interface_a = interfaces[0]
@@ -828,19 +786,9 @@ def get_coriant_info(equipment_name: str, entity_string: str) -> Response:
cache_key = 'classifier-cache:coriant:' \
f'{ims_source_equipment}:{ims_interface}'
- ignore_cache = request.args.get('ignore-cache', default='false', type=str)
- try:
- ignore_cache = strtobool(ignore_cache)
- except ValueError:
- ignore_cache = False
- if ignore_cache:
- result = False
- else:
- result = r.get(cache_key)
+ result = _ignore_cache_or_retrieve(request, cache_key, r)
- if result:
- result = result.decode('utf-8')
- else:
+ if not result:
m = re.match(r'^(\d+\-\d+)\.(\d+)', ims_interface)
if not m:
diff --git a/inventory_provider/routes/common.py b/inventory_provider/routes/common.py
index b9801d64..af20bf9a 100644
--- a/inventory_provider/routes/common.py
+++ b/inventory_provider/routes/common.py
@@ -6,6 +6,7 @@ import queue
import random
import threading
+from distutils.util import strtobool
from lxml import etree
import requests
from flask import request, Response, current_app, g
@@ -16,6 +17,24 @@ _DECODE_TYPE_XML = 'xml'
_DECODE_TYPE_JSON = 'json'
+def _ignore_cache_or_retrieve(request_, cache_key, r):
+
+ ignore_cache = request_.args.get('ignore-cache', default='false', type=str)
+ try:
+ ignore_cache = strtobool(ignore_cache)
+ except ValueError:
+ ignore_cache = False
+ if ignore_cache:
+ result = False
+ logger.debug('ignoring cache')
+ else:
+ result = r.get(cache_key)
+
+ if result:
+ result = result.decode('utf-8')
+ return result
+
+
def ims_hostname_decorator(field):
"""
Decorator to convert host names to various formats to try to match what is
diff --git a/inventory_provider/routes/data.py b/inventory_provider/routes/data.py
index b8d9c59c..275d7a1a 100644
--- a/inventory_provider/routes/data.py
+++ b/inventory_provider/routes/data.py
@@ -2,9 +2,10 @@ import json
import logging
import re
-from flask import Blueprint, jsonify, Response, current_app
+from flask import Blueprint, jsonify, Response, current_app, request
from inventory_provider.routes import common
+from inventory_provider.routes.common import _ignore_cache_or_retrieve
logger = logging.getLogger(__name__)
@@ -139,11 +140,10 @@ def router_interfaces(hostname=None):
if hostname else 'classifier-cache:netconf-interfaces:all'
r = common.get_current_redis()
- result = r.get(cache_key)
- if result:
- result = result.decode('utf-8')
- else:
+ result = _ignore_cache_or_retrieve(request, cache_key, r)
+
+ if not result:
key_pattern = f'netconf-interfaces:{hostname}:*' \
if hostname else 'netconf-interfaces:*'
config = current_app.config['INVENTORY_PROVIDER_CONFIG']
diff --git a/inventory_provider/routes/lg.py b/inventory_provider/routes/lg.py
index b30ddc03..c615aa7d 100644
--- a/inventory_provider/routes/lg.py
+++ b/inventory_provider/routes/lg.py
@@ -1,9 +1,10 @@
import json
import logging
-from flask import Blueprint, jsonify, Response
+from flask import Blueprint, jsonify, Response, request
from inventory_provider.routes import common
+from inventory_provider.routes.common import _ignore_cache_or_retrieve
logger = logging.getLogger(__name__)
@@ -107,11 +108,10 @@ def routers(access):
yield rtr
cache_key = f'classifier-cache:ims-lg:{access}'
- result = redis.get(cache_key)
- if result:
- result = json.loads(result.decode('utf-8'))
- else:
+ result = _ignore_cache_or_retrieve(request, cache_key, redis)
+
+ if not result:
result = list(_routers())
# cache this data for the next call
redis.set(cache_key, json.dumps(result).encode('utf-8'))
diff --git a/inventory_provider/routes/msr.py b/inventory_provider/routes/msr.py
index cead3b83..f168845d 100644
--- a/inventory_provider/routes/msr.py
+++ b/inventory_provider/routes/msr.py
@@ -1,9 +1,10 @@
import itertools
import json
-from flask import Blueprint, jsonify, Response
+from flask import Blueprint, jsonify, Response, request
from inventory_provider.routes import common
+from inventory_provider.routes.common import _ignore_cache_or_retrieve
routes = Blueprint("msr-query-routes", __name__)
@@ -104,11 +105,10 @@ def access_services():
yield json.loads(service)
cache_key = 'classifier-cache:msr:access-services'
- result = redis.get(cache_key)
- if result:
- result = json.loads(result.decode('utf-8'))
- else:
+ result = _ignore_cache_or_retrieve(request, cache_key, redis)
+
+ if not result:
result = list(_services())
# cache this data for the next call
redis.set(cache_key, json.dumps(result).encode('utf-8'))
@@ -159,11 +159,9 @@ def _handle_peering_group_request(name, cache_key, group_key_base):
if name:
cache_key = f'{cache_key}:{name}'
- items = r.get(cache_key)
+ items = _ignore_cache_or_retrieve(request, cache_key, r)
- if items:
- items = json.loads(items.decode('utf-8'))
- else:
+ if not items:
if name:
items = _load_list_items(f'{group_key_base}:{name}')
else:
@@ -265,11 +263,9 @@ def _handle_peering_group_list_request(cache_key, group_key_base):
k = k.decode('utf-8')
yield k[len(group_key_base) + 1:]
- names = r.get(cache_key)
+ names = _ignore_cache_or_retrieve(request, cache_key, r)
- if names:
- names = json.loads(names.decode('utf-8'))
- else:
+ if not names:
names = list(_get_all_subkeys())
if not names:
return Response(
diff --git a/inventory_provider/routes/poller.py b/inventory_provider/routes/poller.py
index 6d516d8b..ae6fd03b 100644
--- a/inventory_provider/routes/poller.py
+++ b/inventory_provider/routes/poller.py
@@ -2,11 +2,12 @@ import json
import logging
import re
-from flask import Blueprint, Response, current_app
+from flask import Blueprint, Response, current_app, request
from inventory_provider import juniper
from inventory_provider.routes import common
from inventory_provider.routes.classifier import get_ims_equipment_name, \
get_ims_interface
+from inventory_provider.routes.common import _ignore_cache_or_retrieve
logger = logging.getLogger(__name__)
routes = Blueprint('poller-support-routes', __name__)
@@ -291,10 +292,9 @@ def interfaces(hostname=None):
r = common.get_current_redis()
- result = r.get(cache_key)
- if result:
- result = result.decode('utf-8')
- else:
+ result = _ignore_cache_or_retrieve(request, cache_key, r)
+
+ if not result:
result = list(_load_interfaces_to_poll(hostname))
if not result:
return Response(
@@ -387,10 +387,9 @@ def interface_speeds(hostname=None):
r = common.get_current_redis()
- result = r.get(cache_key)
- if result:
- result = result.decode('utf-8')
- else:
+ result = _ignore_cache_or_retrieve(request, cache_key, r)
+
+ if not result:
result = list(_load_interfaces_and_speeds(hostname))
if not result:
return Response(
--
GitLab