diff --git a/inventory_provider/tasks/worker.py b/inventory_provider/tasks/worker.py index aad5ece9ce51bebfdc7b433e8ce043cec2676e68..e11bf16e3f42708a492c15f390e32125c3c2fa6d 100644 --- a/inventory_provider/tasks/worker.py +++ b/inventory_provider/tasks/worker.py @@ -32,6 +32,7 @@ from inventory_provider import environment from inventory_provider import snmp from inventory_provider import juniper from redis import RedisError +from requests import HTTPError FINALIZER_POLLING_FREQUENCY_S = 2.5 FINALIZER_TIMEOUT_S = 300 @@ -1062,7 +1063,7 @@ def cache_extracted_ims_data(extracted_data, use_current=False): @log_task_entry_and_exit def ims_task(self, use_current=False): try: - extracted_data = extract_ims_data() + extracted_data = extract_ims_data(log_warning=self.log_warning) cache_extracted_ims_data(extracted_data) transformed_data = transform_ims_data(extracted_data) @@ -1074,17 +1075,20 @@ def ims_task(self, use_current=False): update_latch_status(InventoryTask.config, pending=True, failure=True) -def extract_ims_data(): +def extract_ims_data(log_warning=lambda s: None): c = InventoryTask.config["ims"] return _extract_ims_data( ims_api_url=c['api'], ims_username=c['username'], ims_password=c['password'], - verify_ssl=c.get('verify-ssl', False) + verify_ssl=c.get('verify-ssl', False), + log_warning=log_warning ) -def _extract_ims_data(ims_api_url, ims_username, ims_password, verify_ssl): +def _extract_ims_data( + ims_api_url, ims_username, ims_password, verify_ssl, log_warning +): """ convenient entry point for testing ... @@ -1212,7 +1216,22 @@ def _extract_ims_data(ims_api_url, ims_username, ims_password, verify_ssl): @log_task_entry_and_exit def _populate_flexils_data(): nonlocal flexils_data - flexils_data = ims_data.get_flexils_by_circuitid(ds=_ds()) + + def _transform_key(key): + if key == "null": + return None + try: + return int(key) + except ValueError: + return key + + try: + flexils_data = ims_data.get_flexils_by_circuitid(ds=_ds()) + except HTTPError: + log_warning("Failure reading FlexILS data from IMS. Using cache instead") + redis = get_current_redis(InventoryTask.config) + raw_data = json.loads(redis.get("ims:cache:flexils_data")) + flexils_data = {_transform_key(k): v for k, v in raw_data.items()} @log_task_entry_and_exit def _populate_hierarchy(): diff --git a/test/test_worker.py b/test/test_worker.py index 9118ee1e74044f3ec1c62aa7c913cfe00da5baf1..1d61282f43e7fe028fe11ad2ed130256374af4a1 100644 --- a/test/test_worker.py +++ b/test/test_worker.py @@ -13,9 +13,12 @@ from inventory_provider.tasks.worker import populate_error_report_interfaces_cac retrieve_and_persist_neteng_managed_device_list, \ populate_poller_interfaces_cache, refresh_nokia_interface_list, \ retrieve_and_persist_config_nokia +import pytest +from requests import HTTPError -def test_extract_ims_data(mocker): +@pytest.fixture +def mock_ims_data(mocker): mocker.patch( 'inventory_provider.tasks.worker.InventoryTask.config' ) @@ -111,7 +114,9 @@ def test_extract_ims_data(mocker): ) mocker.patch( 'inventory_provider.tasks.worker.ims_data.get_flexils_by_circuitid', - return_value={} + return_value={ + 1: [{"node_name": "NODE1", "full_port_name": "1-A", "key": "NODE1:1-A"}] + } ) mocker.patch( 'inventory_provider.tasks.worker.IMS.get_all_entities', @@ -132,6 +137,9 @@ def test_extract_ims_data(mocker): } ] ) + + +def test_extract_ims_data(mock_ims_data): res = extract_ims_data() assert res['locations'] == {'loc_a': 'LOC A', 'loc_b': 'LOC B'} assert res['site_locations'] == { @@ -181,6 +189,21 @@ def test_extract_ims_data(mocker): 111112: 'TPID-02', 111113: 'TPID-03' } + assert res['flexils_data'] == { + 1: [{"node_name": "NODE1", "full_port_name": "1-A", "key": "NODE1:1-A"}] + } + + +def test_use_cached_flexils_data_on_ims_error(mocker, data_config, mock_ims_data, mocked_redis): + r = common._get_redis(data_config) + mocker.patch('inventory_provider.tasks.worker.get_current_redis', return_value=r) + mocker.patch( + 'inventory_provider.tasks.worker.ims_data.get_flexils_by_circuitid', + side_effect=HTTPError + ) + res = extract_ims_data()['flexils_data'] + assert len(res) > 1 + assert all(isinstance(k, (int, type(None))) for k in res) def test_transform_ims_data():