From a22c98c240eba056efeb4e02e5215e7984989018 Mon Sep 17 00:00:00 2001
From: Robert Latta <robert.latta@geant.org>
Date: Thu, 19 Aug 2021 15:19:08 +0000
Subject: [PATCH] context corrections
---
inventory_provider/routes/common.py | 5 +-
inventory_provider/routes/lnetd.py | 3 +-
inventory_provider/routes/poller.py | 21 +++--
inventory_provider/routes/testing.py | 6 ++
inventory_provider/tasks/worker.py | 122 ++++++++++++++++++---------
test/test_worker.py | 12 +--
6 files changed, 112 insertions(+), 57 deletions(-)
diff --git a/inventory_provider/routes/common.py b/inventory_provider/routes/common.py
index c2eda21e..79d7c481 100644
--- a/inventory_provider/routes/common.py
+++ b/inventory_provider/routes/common.py
@@ -275,14 +275,13 @@ def load_xml_docs(
use_next_redis=use_next_redis)
-@functools.lru_cache(maxsize=None)
-def load_snmp_indexes(hostname=None, use_next_redis=False):
+def load_snmp_indexes(config, hostname=None, use_next_redis=False):
result = dict()
key_pattern = f'snmp-interfaces:{hostname}*' \
if hostname else 'snmp-interfaces:*'
for doc in load_json_docs(
- config_params=current_app.config['INVENTORY_PROVIDER_CONFIG'],
+ config_params=config,
key_pattern=key_pattern,
use_next_redis=use_next_redis):
router = doc['key'][len('snmp-interfaces:'):]
diff --git a/inventory_provider/routes/lnetd.py b/inventory_provider/routes/lnetd.py
index 346f857e..8402a0e0 100644
--- a/inventory_provider/routes/lnetd.py
+++ b/inventory_provider/routes/lnetd.py
@@ -74,7 +74,8 @@ def _add_snmp_indexes(interfaces, hostname=None):
:param hostname: hostname or None for all
:return: generator that yields interfaces with 'ifIndex' added
"""
- snmp_indexes = common.load_snmp_indexes(hostname)
+ snmp_indexes = common.load_snmp_indexes(
+ current_app.config['INVENTORY_PROVIDER_CONFIG'], hostname)
for ifc in interfaces:
hostname = ifc['hostname']
diff --git a/inventory_provider/routes/poller.py b/inventory_provider/routes/poller.py
index c6cf843b..2ec37100 100644
--- a/inventory_provider/routes/poller.py
+++ b/inventory_provider/routes/poller.py
@@ -584,7 +584,8 @@ def _load_services(config, hostname=None, use_next_redis=False):
return result
-def _load_interfaces(config, hostname, use_next_redis=False):
+def _load_interfaces(
+ config, hostname=None, no_lab=False, use_next_redis=False):
"""
loads basic interface data for production & lab routers
@@ -623,7 +624,6 @@ def _load_interfaces(config, hostname, use_next_redis=False):
base_key_pattern = f'netconf:{hostname}*' if hostname else 'netconf:*'
yield from _load_docs(base_key_pattern)
- no_lab = common.get_bool_request_arg('no-lab', False)
if not no_lab:
yield from _load_docs(f'lab:{base_key_pattern}')
@@ -679,7 +679,8 @@ def _add_snmp_indexes(interfaces, hostname=None):
:param hostname: hostname or None for all
:return: generator with 'snmp-index' optionally added to each element
"""
- snmp_indexes = common.load_snmp_indexes(hostname)
+ snmp_indexes = common.load_snmp_indexes(
+ current_app.config['INVENTORY_PROVIDER_CONFIG'], hostname)
for ifc in interfaces:
router_snmp = snmp_indexes.get(ifc['router'], None)
if router_snmp and ifc['name'] in router_snmp:
@@ -698,8 +699,12 @@ def _load_interfaces_to_poll(hostname=None):
:param hostname: hostname or None for all
:return: generator yielding interface elements
"""
+
+ no_lab = common.get_bool_request_arg('no-lab', False)
basic_interfaces = _load_interfaces(
- current_app.config['INVENTORY_PROVIDER_CONFIG'], hostname)
+ current_app.config['INVENTORY_PROVIDER_CONFIG'],
+ hostname,
+ no_lab=no_lab)
# basic_interfaces = list(basic_interfaces)
with_bundles = _add_bundle_parents(basic_interfaces, hostname)
with_circuits = _add_circuits(with_bundles, hostname)
@@ -807,8 +812,11 @@ def _load_interfaces_and_speeds(hostname=None):
:param hostname: hostname or None for all
:return: generator yielding interface elements
"""
+ no_lab = common.get_bool_request_arg('no-lab', False)
basic_interfaces = _load_interfaces(
- current_app.config['INVENTORY_PROVIDER_CONFIG'], hostname)
+ current_app.config['INVENTORY_PROVIDER_CONFIG'],
+ hostname,
+ no_lab=no_lab)
with_bundles = _add_bundle_parents(basic_interfaces, hostname)
def _result_ifc(ifc):
@@ -1106,7 +1114,8 @@ def _get_services_internal(service_type=None):
yield doc['value']
def _add_snmp(s):
- all_snmp_info = common.load_snmp_indexes()
+ all_snmp_info = common.load_snmp_indexes(
+ current_app.config['INVENTORY_PROVIDER_CONFIG'], )
snmp_interfaces = all_snmp_info.get(s['hostname'], {})
interface_info = snmp_interfaces.get(s['interface'], None)
if interface_info:
diff --git a/inventory_provider/routes/testing.py b/inventory_provider/routes/testing.py
index 8240efe5..ac74703e 100644
--- a/inventory_provider/routes/testing.py
+++ b/inventory_provider/routes/testing.py
@@ -18,6 +18,12 @@ routes = Blueprint("inventory-data-testing-support-routes", __name__)
logger = logging.getLogger(__name__)
+@routes.route("just-poller-cache", methods=['GET', 'POST'])
+def just_poller_cache_update():
+ r = worker.just_poller_cache.delay().get()
+ return jsonify(r)
+
+
@routes.route("chord-update", methods=['GET', 'POST'])
def chord_update():
r = worker.update_entry_point.delay().get()
diff --git a/inventory_provider/tasks/worker.py b/inventory_provider/tasks/worker.py
index 1d41d539..743f0fc3 100644
--- a/inventory_provider/tasks/worker.py
+++ b/inventory_provider/tasks/worker.py
@@ -940,7 +940,7 @@ def refresh_finalizer(self, pending_task_ids_json):
_build_subnet_db(update_callback=self.log_info)
_build_snmp_peering_db(update_callback=self.log_info)
_build_juniper_peering_db(update_callback=self.log_info)
- populate_poller_interfaces_cache()
+ populate_poller_interfaces_cache(warning_callback=self.log_warning)
except (jsonschema.ValidationError,
json.JSONDecodeError,
@@ -1836,55 +1836,93 @@ def final_task(self):
self.log_info('latched current/next dbs')
-def populate_poller_interfaces_cache():
+def populate_poller_interfaces_cache(warning_callback=lambda s: None):
+ no_lab_cache_key = 'classifier-cache:poller-interfaces:no-lab'
+ all_cache_key = 'classifier-cache:poller-interfaces:all'
+ non_lab_populated_interfaces = None
+ all_populated_interfaces = None
- base_key_pattern = 'netconf:*'
- standard_interfaces = _load_interfaces(
- InventoryTask.config, base_key_pattern, use_next_redis=True)
- lab_interfaces = _load_interfaces(
- InventoryTask.config, f'lab:{base_key_pattern}')
+ r = get_next_redis(InventoryTask.config)
- bundles = _load_interface_bundles(
- InventoryTask.config,
- use_next_redis=True
- )
- snmp_indexes = load_snmp_indexes(use_next_redis=True)
+ try:
+ lab_keys_pattern = 'lab:netconf-interfaces-hosts:*'
+ lab_equipment = [h.decode('utf-8')[len(lab_keys_pattern) - 1:]
+ for h in r.keys(lab_keys_pattern)]
+ logger.debug(lab_equipment)
+ standard_interfaces = _load_interfaces(
+ InventoryTask.config,
+ no_lab=False,
+ use_next_redis=True)
+
+ bundles = _load_interface_bundles(
+ InventoryTask.config,
+ use_next_redis=True
+ )
+ snmp_indexes = load_snmp_indexes(
+ InventoryTask.config, use_next_redis=True)
- services = _load_services(InventoryTask.config, use_next_redis=True)
+ services = _load_services(InventoryTask.config, use_next_redis=True)
- r = get_next_redis(InventoryTask.config)
+ def _get_populated_interfaces(interfaces):
- def _get_populated_interfaces(interfaces):
- for ifc in interfaces:
+ for ifc in interfaces:
+ router_snmp = snmp_indexes.get(ifc['router'], None)
+ if router_snmp and ifc['name'] in router_snmp:
+ ifc['snmp-index'] = router_snmp[ifc['name']]['index']
- router_snmp = snmp_indexes.get(ifc['router'], None)
- if router_snmp and ifc['name'] in router_snmp:
- ifc['snmp-index'] = router_snmp[ifc['name']]['index']
+ router_bundle = bundles.get(ifc['router'], None)
+ if router_bundle:
+ base_ifc = ifc['name'].split('.')[0]
+ ifc['bundle-parents'] = router_bundle.get(base_ifc, [])
- router_bundle = bundles.get(ifc['router'], None)
- if router_bundle:
- base_ifc = ifc['name'].split('.')[0]
- ifc['bundle-parents'] = router_bundle.get(base_ifc, [])
+ router_services = services.get(
+ get_ims_equipment_name(ifc['router'], r), None)
+ if router_services:
+ ifc['circuits'] = router_services.get(
+ get_ims_interface(ifc['name']), []
+ )
- router_services = services.get(
- get_ims_equipment_name(ifc['router'], r), None)
- if router_services:
- ifc['circuits'] = router_services.get(
- get_ims_interface(ifc['name']), []
- )
+ dashboards = _get_dashboards(ifc)
+ ifc['dashboards'] = sorted([d.name for d in dashboards])
+ yield _get_dashboard_data(ifc)
+ else:
+ continue
- dashboards = _get_dashboards(ifc)
- ifc['dashboards'] = sorted([d.name for d in dashboards])
- yield _get_dashboard_data(ifc)
- else:
- continue
+ all_populated_interfaces = \
+ list(_get_populated_interfaces(standard_interfaces))
+ non_lab_populated_interfaces = [x for x in all_populated_interfaces
+ if x['router'] not in lab_equipment]
- non_lab_populated_interfaces = \
- list(_get_populated_interfaces(standard_interfaces))
- cache_key = 'classifier-cache:poller-interfaces:no-lab'
- r.set(cache_key, json.dumps(non_lab_populated_interfaces))
+ except Exception as e:
+ warning_callback(f"Failed to retrieve all required data {e}")
- all_populated_interfaces = non_lab_populated_interfaces + \
- list(_get_populated_interfaces(lab_interfaces))
- cache_key = 'classifier-cache:poller-interfaces:all'
- r.set(cache_key, json.dumps(all_populated_interfaces))
+ if not non_lab_populated_interfaces or not all_populated_interfaces:
+ previous_r = get_current_redis(InventoryTask.config)
+
+ def _load_previous(key):
+ try:
+ warning_callback(f"populating {key} "
+ "from previously cached data")
+ return json.loads(previous_r.get(key))
+ except Exception as e:
+ warning_callback(f"Failed to load {key} "
+ f"from previously cached data: {e}")
+
+ if not non_lab_populated_interfaces:
+ non_lab_populated_interfaces = _load_previous(no_lab_cache_key)
+
+ if not all_populated_interfaces:
+ all_populated_interfaces = _load_previous(all_cache_key)
+
+ r.set(no_lab_cache_key, json.dumps(non_lab_populated_interfaces))
+ r.set(all_cache_key, json.dumps(all_populated_interfaces))
+
+
+@app.task(base=InventoryTask, bind=True, name='just_poller_cache')
+@log_task_entry_and_exit
+def just_poller_cache(self):
+
+ populate_poller_interfaces_cache(warning_callback=self.log_warning)
+
+ latch_db(InventoryTask.config)
+ self.log_info('latched current/next dbs')
diff --git a/test/test_worker.py b/test/test_worker.py
index 34dfd68c..fa254f67 100644
--- a/test/test_worker.py
+++ b/test/test_worker.py
@@ -330,7 +330,7 @@ def test_retrieve_and_persist_neteng_managed_device_list(
def test_populate_poller_interfaces_cache(
mocker, data_config, mocked_redis):
r = common._get_redis(data_config)
- standard_interfaces = [
+ all_interfaces = [
{
"router": "router_a.geant.net",
"name": "interface_a",
@@ -355,9 +355,6 @@ def test_populate_poller_interfaces_cache(
"description": "DESCRIPTION C",
"circuits": []
},
- ]
-
- lab_interfaces = [
{
"router": "lab_router_a.geant.net",
"name": "lab_interface_a",
@@ -482,8 +479,13 @@ def test_populate_poller_interfaces_cache(
},
]
+ for k in r.keys("lab:netconf-interfaces-hosts:*"):
+ r.delete(k)
+ r.set("lab:netconf-interfaces-hosts:lab_router_a.geant.net", "dummy")
+ r.set("lab:netconf-interfaces-hosts:lab_router_b.geant.net", "dummy")
+
mocker.patch('inventory_provider.tasks.worker._load_interfaces',
- side_effect=[standard_interfaces, lab_interfaces])
+ side_effect=[all_interfaces, ])
mocker.patch('inventory_provider.tasks.worker._load_interface_bundles',
return_value=bundles)
mocker.patch('inventory_provider.tasks.worker.load_snmp_indexes',
--
GitLab