diff --git a/inventory_provider/routes/poller.py b/inventory_provider/routes/poller.py
index 1c789f62f36d18153b204fede4f4be4a7b938759..22b699ed4b43c99904b129daa9576f888a1ecbd9 100644
--- a/inventory_provider/routes/poller.py
+++ b/inventory_provider/routes/poller.py
@@ -72,6 +72,7 @@ from flask import Blueprint, Response, current_app, request, jsonify
from inventory_provider import juniper
from inventory_provider.routes import common
from inventory_provider.tasks.common import ims_sorted_service_type_key
+from inventory_provider.tasks import common as tasks_common
from inventory_provider.routes.classifier import get_ims_equipment_name, \
get_ims_interface
from inventory_provider.routes.common import _ignore_cache_or_retrieve
@@ -654,6 +655,9 @@ def _load_interfaces(
:return:
"""
def _load_docs(key_pattern):
+ # print('')
+ # logger.debug(f'docs Key: {key_pattern}')
+ # print('')
for doc in _load_netconf_docs(config, key_pattern, use_next_redis):
@@ -673,6 +677,7 @@ def _load_interfaces(
base_key_pattern = f'netconf:{hostname}*' if hostname else 'netconf:*'
yield from _load_docs(base_key_pattern)
if not no_lab:
+ logger.debug('lab')
yield from _load_docs(f'lab:{base_key_pattern}')
@@ -695,78 +700,42 @@ def _add_bundle_parents(interfaces, hostname=None):
yield ifc
-def _add_circuits(interfaces, hostname=None):
- """
- generator that adds service info to each interface.
-
- :param interfaces: result of _load_interfaces
- :param hostname: hostname or None for all
- :return: generator with 'circuits' populated in each element, if present
- """
-
- if hostname:
- hostname = get_ims_equipment_name(hostname)
- services = _load_services(
- current_app.config['INVENTORY_PROVIDER_CONFIG'], hostname=hostname)
- for ifc in interfaces:
- router_services = services.get(
- get_ims_equipment_name(ifc['router']), None)
- if router_services:
- ifc['circuits'] = router_services.get(
- get_ims_interface(ifc['name']), []
- )
-
- yield ifc
-
-
-def _add_snmp_indexes(interfaces, hostname=None):
- """
- generator that adds snmp info to each interface, if available
-
- :param interfaces: result of _load_interfaces
- :param hostname: hostname or None for all
- :return: generator with 'snmp-index' optionally added to each element
- """
- snmp_indexes = common.load_snmp_indexes(
- current_app.config['INVENTORY_PROVIDER_CONFIG'], hostname)
- for ifc in interfaces:
- router_snmp = snmp_indexes.get(ifc['router'], None)
- if router_snmp and ifc['name'] in router_snmp:
- ifc['snmp-index'] = router_snmp[ifc['name']]['index']
- # TODO: uncomment this when it won't break poller-admin-service
- # not urgent ... it looks empirically like all logical-system
- # interfaces are repeated for both communities
- # ifc['snmp-community'] = router_snmp[ifc['name']]['community']
- yield ifc
-
-
-def _load_interfaces_to_poll(hostname=None):
- """
- prepares the result of a call to /interfaces
-
- :param hostname: hostname or None for all
- :return: generator yielding interface elements
- """
-
- no_lab = common.get_bool_request_arg('no-lab', False)
- basic_interfaces = _load_interfaces(
- current_app.config['INVENTORY_PROVIDER_CONFIG'],
- hostname,
- no_lab=no_lab)
- # basic_interfaces = list(basic_interfaces)
- with_bundles = _add_bundle_parents(basic_interfaces, hostname)
- with_circuits = _add_circuits(with_bundles, hostname)
- # with_circuits = list(with_circuits)
- with_snmp = _add_snmp_indexes(with_circuits, hostname)
- # with_snmp = list(with_snmp)
-
- # only return interfaces that can be polled
- def _has_snmp_index(ifc):
- return 'snmp-index' in ifc
-
- to_poll = filter(_has_snmp_index, with_snmp)
-
- return _add_dashboards(to_poll)
+def load_interfaces_to_poll(
+ config, hostname=None, no_lab=False, use_next_redis=False):
+ basic_interfaces = \
+ list(_load_interfaces(config, hostname, no_lab, use_next_redis))
+ bundles = _load_interface_bundles(config, hostname, use_next_redis)
+ services = _load_services(config, hostname, use_next_redis)
+ snmp_indexes = common.load_snmp_indexes(config, hostname)
+
+ def _get_populated_interfaces(all_interfaces):
+ if use_next_redis:
+ r = tasks_common.get_next_redis(config)
+ else:
+ r = tasks_common.get_current_redis(config)
+ for ifc in all_interfaces:
+ router_snmp = snmp_indexes.get(ifc['router'], None)
+ if router_snmp and ifc['name'] in router_snmp:
+ ifc['snmp-index'] = router_snmp[ifc['name']]['index']
+
+ router_bundle = bundles.get(ifc['router'], None)
+ if router_bundle:
+ base_ifc = ifc['name'].split('.')[0]
+ ifc['bundle-parents'] = router_bundle.get(base_ifc, [])
+
+ router_services = services.get(
+ get_ims_equipment_name(ifc['router'], r), None)
+ if router_services:
+ ifc['circuits'] = router_services.get(
+ get_ims_interface(ifc['name']), []
+ )
+
+ dashboards = _get_dashboards(ifc)
+ ifc['dashboards'] = sorted([d.name for d in dashboards])
+ yield _get_dashboard_data(ifc)
+ else:
+ continue
+ return _get_populated_interfaces(basic_interfaces)
@routes.route("/interfaces", methods=['GET', 'POST'])
@@ -808,7 +777,8 @@ def interfaces(hostname=None):
result = _ignore_cache_or_retrieve(request, cache_key, r)
if not result:
- result = list(_load_interfaces_to_poll(hostname))
+ result = list(load_interfaces_to_poll(
+ current_app.config['INVENTORY_PROVIDER_CONFIG'], hostname, no_lab))
if not result:
return Response(
response='no interfaces found',
diff --git a/inventory_provider/tasks/worker.py b/inventory_provider/tasks/worker.py
index 07e4a2edd623392ec3afbbc20f96d18c0d38ad85..c6057747dfb24b55f1f938d92ed680a68762768f 100644
--- a/inventory_provider/tasks/worker.py
+++ b/inventory_provider/tasks/worker.py
@@ -17,12 +17,7 @@ from lxml import etree
from inventory_provider.db import ims_data
from inventory_provider.db.ims import IMS
-from inventory_provider.routes.classifier import get_ims_interface, \
- get_ims_equipment_name
-from inventory_provider.routes.common import load_snmp_indexes
-from inventory_provider.routes.poller import _load_interfaces, \
- _load_interface_bundles, _get_dashboard_data, _get_dashboards, \
- _load_services
+from inventory_provider.routes.poller import load_interfaces_to_poll
from inventory_provider.tasks.app import app
from inventory_provider.tasks.common \
import get_next_redis, get_current_redis, \
@@ -1203,47 +1198,9 @@ def populate_poller_interfaces_cache(warning_callback=lambda s: None):
lab_keys_pattern = 'lab:netconf-interfaces-hosts:*'
lab_equipment = [h.decode('utf-8')[len(lab_keys_pattern) - 1:]
for h in r.keys(lab_keys_pattern)]
- standard_interfaces = _load_interfaces(
- InventoryTask.config,
- no_lab=False,
- use_next_redis=True)
-
- bundles = _load_interface_bundles(
- InventoryTask.config,
- use_next_redis=True
- )
- snmp_indexes = load_snmp_indexes(
- InventoryTask.config, use_next_redis=True)
-
- services = _load_services(InventoryTask.config, use_next_redis=True)
-
- def _get_populated_interfaces(interfaces):
-
- for ifc in interfaces:
- router_snmp = snmp_indexes.get(ifc['router'], None)
- if router_snmp and ifc['name'] in router_snmp:
- ifc['snmp-index'] = router_snmp[ifc['name']]['index']
-
- router_bundle = bundles.get(ifc['router'], None)
- if router_bundle:
- base_ifc = ifc['name'].split('.')[0]
- ifc['bundle-parents'] = router_bundle.get(base_ifc, [])
-
- router_services = services.get(
- get_ims_equipment_name(ifc['router'], r), None)
- if router_services:
- ifc['circuits'] = router_services.get(
- get_ims_interface(ifc['name']), []
- )
-
- dashboards = _get_dashboards(ifc)
- ifc['dashboards'] = sorted([d.name for d in dashboards])
- yield _get_dashboard_data(ifc)
- else:
- continue
- all_populated_interfaces = \
- list(_get_populated_interfaces(standard_interfaces))
+ all_populated_interfaces = list(
+ load_interfaces_to_poll(InventoryTask.config, use_next_redis=True))
non_lab_populated_interfaces = [x for x in all_populated_interfaces
if x['router'] not in lab_equipment]
diff --git a/test/test_worker.py b/test/test_worker.py
index 7a6a30231f5460a52e62d627bcdef736625a8cec..2ff0ce69d32d2bc8da47ea5eb728489565c3a9f5 100644
--- a/test/test_worker.py
+++ b/test/test_worker.py
@@ -487,6 +487,11 @@ def test_retrieve_and_persist_neteng_managed_device_list(
def test_populate_poller_interfaces_cache(
mocker, data_config, mocked_redis):
r = common._get_redis(data_config)
+
+ mocker.patch('inventory_provider.tasks.common.get_next_redis',
+ return_value=r)
+ mocker.patch('inventory_provider.tasks.worker.get_next_redis',
+ return_value=r)
all_interfaces = [
{
"router": "router_a.geant.net",
@@ -641,19 +646,17 @@ def test_populate_poller_interfaces_cache(
r.set("lab:netconf-interfaces-hosts:lab_router_a.geant.net", "dummy")
r.set("lab:netconf-interfaces-hosts:lab_router_b.geant.net", "dummy")
- mocker.patch('inventory_provider.tasks.worker._load_interfaces',
+ mocker.patch('inventory_provider.routes.poller._load_interfaces',
side_effect=[all_interfaces, ])
- mocker.patch('inventory_provider.tasks.worker._load_interface_bundles',
+ mocker.patch('inventory_provider.routes.poller._load_interface_bundles',
return_value=bundles)
- mocker.patch('inventory_provider.tasks.worker.load_snmp_indexes',
+ mocker.patch('inventory_provider.routes.common.load_snmp_indexes',
return_value=snmp_indexes)
- mocker.patch('inventory_provider.tasks.worker._load_services',
+ mocker.patch('inventory_provider.routes.poller._load_services',
return_value=services)
mocker.patch(
'inventory_provider.tasks.worker.InventoryTask.config'
)
- mocker.patch('inventory_provider.tasks.worker.get_next_redis',
- return_value=r)
populate_poller_interfaces_cache()
assert r.exists("classifier-cache:poller-interfaces:all:no_lab")