import json import pathlib import jsonschema from lxml import etree from ncclient.transport import TransportError from inventory_provider.nokia import remove_xml_namespaces from inventory_provider.tasks import common from inventory_provider.tasks.worker import populate_error_report_interfaces_cache, \ transform_ims_data, \ extract_ims_data, persist_ims_data, \ retrieve_and_persist_neteng_managed_device_list, \ populate_poller_interfaces_cache, refresh_nokia_interface_list, \ retrieve_and_persist_config_nokia import pytest from requests import HTTPError @pytest.fixture def mock_ims_data(mocker): mocker.patch( 'inventory_provider.tasks.worker.InventoryTask.config' ) mocker.patch( 'inventory_provider.tasks.worker.ims_data.get_node_locations', return_value=[('loc_a', 'LOC A'), ('loc_b', 'LOC B')] ) mocker.patch( 'inventory_provider.tasks.worker.ims_data.get_site_locations', return_value=[ ('JEN-SPL', { 'abbreviation': 'JEN', 'city': 'MILAN', 'country': 'ITALY', 'latitude': 12.12, 'longitude': 12.45, 'name': 'JEN-SPL' }), ] ) mocker.patch( 'inventory_provider.tasks.worker.IMS.clear_dynamic_context_cache' ) mocker.patch( 'inventory_provider.tasks.worker.ims_data.lookup_lg_routers', return_value=['lg router 1', 'lg router 2'] ) mocker.patch( 'inventory_provider.tasks.worker.ims_data.get_customer_tts_contacts', return_value=[('123', 'CON A'), ('456', 'CON B')] ) mocker.patch( 'inventory_provider.tasks.worker.ims_data.' 'get_customer_planned_work_contacts', return_value=[('223', 'CON PW A'), ('556', 'CON PW B')] ) mocker.patch( 'inventory_provider.tasks.worker.ims_data.get_monitored_circuit_ids', return_value=[123, 456, 789] ) mocker.patch( 'inventory_provider.tasks.worker.ims_data.' 'get_circuit_related_customers', return_value=[ { 'id a': [ {'id': 'A', 'name': 'customer a'}, {'A2'} ] }, {'id b': [{'id': 'B', 'name': 'customer b'}]} ] ) mocker.patch( 'inventory_provider.tasks.worker.ims_data.get_circuit_hierarchy', return_value=[ {'id': '1', 'value': 'A'}, {'id': '2', 'value': 'B'} ] ) mocker.patch( 'inventory_provider.tasks.worker.ims_data.get_port_details', return_value=[ {'port_id': 'A', 'value': 'a'}, {'port_id': 'B', 'value': 'b'}, {'port_id': 'B', 'value': 'c'} ] ) mocker.patch( 'inventory_provider.tasks.worker.ims_data.get_port_id_services', return_value=[ {'port_a_id': '1', 'value': '1A'}, {'port_a_id': '1', 'value': '1B'}, {'port_a_id': '2', 'value': '2A'} ] ) mocker.patch( 'inventory_provider.tasks.worker.ims_data.get_ids_and_sids', return_value=(x for x in [ (111111, 'SID-01'), (111112, 'SID-02'), (111113, 'SID-03') ]) ) mocker.patch( 'inventory_provider.tasks.worker.ims_data.get_ids_and_third_party_ids', return_value=(x for x in [ (111111, 'TPID-01'), (111112, 'TPID-02'), (111113, 'TPID-03') ]) ) mocker.patch( 'inventory_provider.tasks.worker.ims_data.lookup_geant_nodes', return_value=[] ) mocker.patch( 'inventory_provider.tasks.worker.ims_data.get_flexils_by_circuitid', return_value={ 1: [{"node_name": "NODE1", "full_port_name": "1-A", "key": "NODE1:1-A"}] } ) mocker.patch( 'inventory_provider.tasks.worker.IMS.get_all_entities', return_value=[ { 'id': 1, 'name': 'Cust 1' } ] ) mocker.patch( 'inventory_provider.tasks.worker.ims_data.get_customer_regions', return_value=[ { 'id': 1, 'name': 'Cust 1', 'region': 'REGION' } ] ) def test_extract_ims_data(mock_ims_data): res = extract_ims_data() assert res['locations'] == {'loc_a': 'LOC A', 'loc_b': 'LOC B'} assert res['site_locations'] == { 'JEN-SPL': {'abbreviation': 'JEN', 'city': 'MILAN', 'country': 'ITALY', 'latitude': 12.12, 'longitude': 12.45, 'name': 'JEN-SPL'}} assert res['lg_routers'] == ['lg router 1', 'lg router 2'] assert res['customer_contacts'] == {'123': 'CON A', '456': 'CON B'} assert res['customer_regions'] == {1: {'id': 1, 'name': 'Cust 1', 'region': 'REGION'}} assert res['planned_work_contacts'] == \ {'223': 'CON PW A', '556': 'CON PW B'} assert res['circuit_ids_to_monitor'] == [123, 456, 789] assert res['additional_circuit_customers'] == \ [ { 'id a': [ {'id': 'A', 'name': 'customer a'}, {'A2'} ] }, {'id b': [{'id': 'B', 'name': 'customer b'}]} ] assert res['hierarchy'] == { '1': {'id': '1', 'value': 'A'}, '2': {'id': '2', 'value': 'B'} } assert res['port_id_details'] == { 'A': [{'port_id': 'A', 'value': 'a'}], 'B': [ {'port_id': 'B', 'value': 'b'}, {'port_id': 'B', 'value': 'c'} ] } assert res['port_id_services'] == { '1': [ {'port_a_id': '1', 'value': '1A'}, {'port_a_id': '1', 'value': '1B'} ], '2': [{'port_a_id': '2', 'value': '2A'}] } assert res['circuit_ids_sids'] == { 111111: 'SID-01', 111112: 'SID-02', 111113: 'SID-03' } assert res['circuit_ids_third_party_ids'] == { 111111: 'TPID-01', 111112: 'TPID-02', 111113: 'TPID-03' } assert res['flexils_data'] == { 1: [{"node_name": "NODE1", "full_port_name": "1-A", "key": "NODE1:1-A"}] } def test_use_cached_flexils_data_on_ims_error(mocker, data_config, mock_ims_data, mocked_redis): r = common._get_redis(data_config) mocker.patch('inventory_provider.tasks.worker.get_current_redis', return_value=r) mocker.patch( 'inventory_provider.tasks.worker.ims_data.get_flexils_by_circuitid', side_effect=HTTPError ) res = extract_ims_data()['flexils_data'] assert len(res) > 1 assert all(isinstance(k, (int, type(None))) for k in res) def test_transform_ims_data(): locations = { "eq_a": { "equipment-name": "eq_a", "pop": { "name": "pop_loc_a", "abbreviation": "pla", } }, "eq_b": { "equipment-name": "eq_b", "pop": { "name": "pop_loc_b", "abbreviation": "plb", } }, "UNKNOWN_LOC": { "equipment-name": "UNKNOWN_LOC", "pop": { "name": "UNKNOWN", "abbreviation": "UNKNOWN", } } } site_locations = { 'JEN-SPL': {'abbreviation': 'JEN', 'city': 'MILAN', 'country': 'ITALY', 'latitude': 12.12323, 'longitude': 4.90123, 'name': 'JEN-SPL'} } lg_routers = [ {"equipment name": "lg_eq1"}, {"equipment name": "lg_eq2"} ] additional_circuit_customer_ids = { "circ_id_1": [ {"id": "cu_1_1", "name": "customer_1"} ] } customer_contacts = { "cu_1": ["customer_1@a.org"], "cu_1_1": ["customer_1_1@a.org"], "cu_sub_1": ["customer_sub_1@a.org"], "cu_sub_2": ["customer_sub_2@a.org"] } planned_work_contacts = { "cu_1": ["customer_1_PW@a.org"], "cu_1_1": ["customer_1_1_PW@a.org"], "cu_sub_1": ["customer_sub_1_PW@a.org"], "cu_sub_2": ["customer_sub_2_PW@a.org"] } port_id_details = { "port_id_1": [{ "equipment_name": "eq_a", "interface_name": "if_a", "port_id": "port_id_1" }], "port_id_2": [{ "equipment_name": "eq_b", "interface_name": "if_b", "port_id": "port_id_2" }], "port_id_3": [{ "equipment_name": "eq_a", "interface_name": "if_c", "port_id": "port_id_3" }], "port_id_4": [{ "equipment_name": "eq_b", "interface_name": "if_c", "port_id": "port_id_4" }] } port_id_services = { "port_id_1": [ { "id": "circ_id_1", "name": "circ_name_1", "customerid": "cu_1", "customer": "customer_1", "project": "customer_1", "circuit_type": "circuit", "service_type": "ETHERNET", "status": "operational", "port_a_id": "port_id_1", "port_b_id": "port_id_2", } ], "port_id_2": [ { "id": "circ_id_1", "name": "circ_name_1", "customerid": "cu_1", "customer": "customer_1", "project": "customer_1", "circuit_type": "circuit", "service_type": "ETHERNET", "status": "operational", "port_a_id": "port_id_2", "port_b_id": "port_id_1", } ], "port_id_3": [ { "id": "sub_circuit_2", "name": "sub_circuit_2", "customerid": "cu_1", "customer": "customer_1", "project": "customer_1", "circuit_type": "service", "service_type": "PEERING R & E", "status": "operational", "port_a_id": "port_id_3", "port_b_id": "port_id_4", } ], "port_id_4": [ { "id": "sub_circuit_2", "name": "sub_circuit_2", "customerid": "cu_1", "customer": "customer_1", "project": "customer_1", "circuit_type": "service", "service_type": "PEERING R & E", "status": "operational", "port_a_id": "port_id_4", "port_b_id": "port_id_3", }, ] } hierarchy = { "circ_id_1": { "id": "circ_id_1", "name": "circ_name_1", "status": "operational", "circuit-type": "circuit", "service_type": "ETHERNET", "product": "ethernet", "speed": "not fibre_route", "project": "customer_1", "carrier-circuits": ["carrier_id_1"], "sub-circuits": ["sub_circuit_1"], "customerid": "cu_1" }, "carrier_id_1": { "id": "carrier_id_1", "name": "circ_carrier_name_1", "status": "operational", "circuit-type": "circuit", "product": "ethernet", "speed": "10G", "project": "customer_1", "carrier-circuits": ["carrier_id_2"], "sub-circuits": ["circ_id_1"], "customerid": "cu_1", }, "carrier_id_2": { "id": "carrier_id_2", "name": "circ_carrier_name_3", "status": "operational", "circuit-type": "circuit", "product": "ethernet", "speed": "not fibre_route", "project": "customer_1", "carrier-circuits": ["carrier_id_3"], "sub-circuits": ["carrier_id_1"], "customerid": "cu_1", }, "carrier_id_3": { "id": "carrier_id_3", "name": "Fiber Route Circuit", "status": "operational", "circuit-type": "circuit", "product": "OCG4", "speed": "fibre_route", "project": "customer_1", "carrier-circuits": [], "sub-circuits": ["carrier_id_2"], "customerid": "cu_1", }, "sub_circuit_1": { "id": "sub_circuit_1", "name": "sub_circuit_name_1", "status": "operational", "circuit-type": "circuit", "product": "ethernet", "speed": "not fibre_route", "project": "customer_1", "carrier-circuits": ["circ_id_1"], "sub-circuits": ["sub_circuit_2"], "customerid": "cu_Sub_1", }, "sub_circuit_2": { "id": "sub_circuit_2", "name": "sub_circuit_name_2", "status": "operational", "circuit-type": "service", "product": "PEERING R & E", "speed": "not fiber route", "project": "customer_1", "carrier-circuits": ["sub_circuit_1"], "sub-circuits": [], "customerid": "cu_sub_2", }, 702203: { "id": 702203, "name": "FLEX ILS CIRCUIT", "status": "operational", "circuit-type": "circuit", "product": "ethernet", "speed": "not fiber route", "project": "customer_1", "carrier-circuits": ["circ_id_1"], "sub-circuits": ["sub_circuit_2"], "customerid": "cu_1", } } circuit_ids_and_sids = { "sub_circuit_2": 'SID-01', "circ_id_2": 'SID-02', "circ_id_3": 'SID-03' } circuit_ids_third_party_ids = { "sub_circuit_2": 'TPID-01', "circ_id_2": 'TPID-02', "circ_id_3": 'TPID-03' } flexils_data = { 702203: [ { 'node_name': 'CAM01-MTC6-3', 'full_port_name': '1-A-1-S1-1', 'key': 'CAM01-MTC6-3:1-A-1-S1-1' }, { 'node_name': 'CAM01-MTC6-3', 'full_port_name': '1-A-1-L1-1', 'key': 'CAM01-MTC6-3:1-A-1-L1-1' } ] } customers = { 99: { 'id': 99, 'name': 'flake' } } data = { "locations": locations, "site_locations": site_locations, "customer_contacts": customer_contacts, "planned_work_contacts": planned_work_contacts, "circuit_ids_to_monitor": ["sub_circuit_2"], "additional_circuit_customers": additional_circuit_customer_ids, "hierarchy": hierarchy, "port_id_details": port_id_details, "port_id_services": port_id_services, "circuit_ids_sids": circuit_ids_and_sids, "circuit_ids_third_party_ids": circuit_ids_third_party_ids, "geant_nodes": ["eq_b"], "flexils_data": flexils_data, "customers": customers, "lg_routers": lg_routers, } orig_port_id_services_len = len(port_id_services.keys()) res = transform_ims_data(data) assert len(port_id_services) == orig_port_id_services_len + 2 pop_nodes_res = { "pop_loc_a": ["eq_a"], "pop_loc_b": ["eq_b"], "UNKNOWN": ["UNKNOWN_LOC"] } assert sorted(res["pop_nodes"].keys()) == sorted(pop_nodes_res.keys()) for k, v in pop_nodes_res.items(): assert v == res["pop_nodes"][k] ifs = res["interface_services"] assert list(ifs.keys()) == [ "eq_a:if_a", "eq_b:if_b", "eq_a:if_c", "eq_b:if_c", "CAM01-MTC6-3:1-A-1-S1-1", "CAM01-MTC6-3:1-A-1-L1-1" ] assert hierarchy["circ_id_1"]["contacts"] == ["customer_1@a.org", "customer_1_1@a.org"] for v in ifs.values(): assert len(v) == 1 assert len(v[0]["related-services"]) == 1 assert v[0]["related-services"][0]["id"] == "sub_circuit_2" assert len(v[0]["fibre-routes"]) == 1 assert v[0]["fibre-routes"][0]["id"] == "carrier_id_3" assert v[0]["contacts"] == ["customer_sub_2@a.org"] assert v[0]["planned_work_contacts"] == ["customer_sub_2_PW@a.org"] nps = res["node_pair_services"] assert list(nps.keys()) == ["eq_a/eq_b", "eq_b/eq_a"] v1 = nps["eq_a/eq_b"]["circ_id_1"] v2 = nps["eq_a/eq_b"]["circ_id_1"] assert v1 == v2 assert json.dumps(v1, sort_keys=True) == json.dumps(v2, sort_keys=True) assert len(v) == 1 assert len(v[0]["related-services"]) == 1 assert v[0]["related-services"][0]["id"] == "sub_circuit_2" assert len(v[0]["fibre-routes"]) == 1 assert v[0]["fibre-routes"][0]["id"] == "carrier_id_3" assert len(res["sid_services"]['SID-01']) == 2 for x in [ { 'circuit_id': "sub_circuit_2", 'sid': "SID-01", 'status': 'operational', 'monitored': True, 'name': "sub_circuit_2", 'speed': 10 << 30, 'service_type': "PEERING R & E", 'project': "customer_1", 'customer': "customer_1", 'equipment': "eq_a", 'port': "if_c", 'geant_equipment': False }, { 'circuit_id': "sub_circuit_2", 'sid': "SID-01", 'status': 'operational', 'monitored': True, 'name': "sub_circuit_2", 'speed': 10 << 30, 'service_type': "PEERING R & E", 'project': "customer_1", 'customer': "customer_1", 'equipment': "eq_b", 'port': "if_c", 'geant_equipment': True } ]: assert json.dumps(x, sort_keys=True) in [ json.dumps( y, sort_keys=True) for y in res["sid_services"]['SID-01']] def test_persist_ims_data(mocker, data_config, mocked_redis): r = common._get_redis(data_config) mocker.patch('inventory_provider.tasks.worker.get_next_redis', return_value=r) data = { "pop_nodes": { "LOC A": ["eq_a"], "LOC B": ["eq_b"] }, "locations": { "eq_a": { 'equipment-name': 'eq_a', 'pop': {'name': "LOC A", 'abbreviation': 'aaa'} }, "eq_b": { 'equipment-name': 'eq_b', 'pop': {'name': "LOC B", 'abbreviation': 'bbb'} }, }, "site_locations": { 'JEN-SPL': {'abbreviation': 'JEN', 'city': 'MILAN', 'country': 'ITALY', 'latitude': 12.1, 'longitude': -1.2, 'name': 'JEN-SPL'}}, "lg_routers": [ {"equipment name": "lg_eq1"}, {"equipment name": "lg_eq2"} ], "hierarchy": {"c1": {"id": "123"}, "c2": {"id": "456"}}, "interface_services": { "if1": [ { "equipment": "eq1", "port": "port1", "id": "id1", "name": "name1", "service_type": "type1", "status": "operational", "pop_name": "LOC A", "pop_abbreviation": "aaa", "contacts": [], "planned_work_contacts": [], "third_party_id": "I2-S13168 - RTSW.NEWY32AOA.NET.INTERNET2.EDU", "related-services": [ { "id": "sub_circuit_1", "name": "sub_circuit_1", "status": "operational", "sid": "SID-01", "service_type": "type a", "contacts": ["c1", "c2"], "planned_work_contacts": ["c1"], "third_party_id": "I2-S13168 - RTSW.NEWY32AOA.NET.INTERNET2.EDU" } ], }, { "equipment": "eq1", "port": "port2", "id": "id3", "name": "name2", "service_type": "type2", "status": "operational" } ], "if2": [ { "equipment": "eq2", "port": "port1", "id": "id3", "name": "name3", "service_type": "type1", "status": "operational" } ] }, "node_pair_services": { "np1": {"id_1": "data for np1"}, "np2": {"id_2": "data for np2"}, }, "sid_services": {"SID-001": [{"k1": "data"}, {"k1": "data"}]}, "services_by_type": {}, "geant_nodes": [], "circuit_ids_to_monitor": ["123", "456", "id1", "sub_circuit_1"] } for k in r.keys("ims:*"): r.delete(k) persist_ims_data(data) assert {k.decode("utf-8") for k in r.keys("ims:pop_nodes:*")} == \ {"ims:pop_nodes:LOC A", "ims:pop_nodes:LOC B"} assert {k.decode("utf-8") for k in r.keys("ims:location:*")} == \ {"ims:location:eq_a", "ims:location:eq_b"} assert {k.decode("utf-8") for k in r.keys("ims:lg:*")} == \ {"ims:lg:lg_eq1", "ims:lg:lg_eq2"} assert {k.decode("utf-8") for k in r.keys("ims:circuit_hierarchy:*")} == \ {"ims:circuit_hierarchy:123", "ims:circuit_hierarchy:456"} assert {k.decode("utf-8") for k in r.keys("ims:interface_services:*")} == \ {"ims:interface_services:if1", "ims:interface_services:if2"} assert {k.decode("utf-8") for k in r.keys("ims:node_pair_services:*")} == \ {"ims:node_pair_services:np1", "ims:node_pair_services:np2"} assert {k.decode("utf-8") for k in r.keys("poller_cache:*")} == \ {"poller_cache:eq1", "poller_cache:eq2"} assert json.loads(r.get("ims:sid_services").decode("utf-8")) == \ data["sid_services"] def test_retrieve_and_persist_neteng_managed_device_list( mocker, data_config, mocked_redis): device_list = [{'abc': 'juniper'}, {'def': 'nokia'}] r = common._get_redis(data_config) mocker.patch( 'inventory_provider.tasks.worker.InventoryTask.config' ) mocker.patch('inventory_provider.tasks.worker.get_next_redis', return_value=r) r.delete('netdash') mocker.patch('inventory_provider.tasks.worker.get_current_redis', return_value=r) mocker.patch('inventory_provider.gap.load_routers_from_orchestrator', return_value=device_list) result = retrieve_and_persist_neteng_managed_device_list() assert result == device_list assert json.loads(r.get('netdash')) == device_list def test_populate_poller_interfaces_cache( mocker, data_config, mocked_redis): r = common._get_redis(data_config) mocker.patch('inventory_provider.tasks.common.get_next_redis') mocker.patch('inventory_provider.tasks.worker.get_next_redis', return_value=r) all_interfaces = [ { "router": "router_a.geant.net", "name": "et/1", "bundle": ["ae_a"], "bundle-parents": [], "description": "DESCRIPTION A", "circuits": [] }, { "router": "router_a.geant.net", "name": "ae_a", "bundle": [], "bundle-parents": [], "description": "DESCRIPTION B $GA-0001", "circuits": [] }, { "router": "router_a.geant.net", "name": "ae_a.123", "bundle": [], "bundle-parents": [], "description": "DESCRIPTION C", "circuits": [] }, { "router": "lab_router_a.geant.net", "name": "et/2/lab", "bundle": ["ae_c"], "bundle-parents": [], "description": "DESCRIPTION C $GA-0001", "circuits": [] }, { "router": "lab_router_a.geant.net", "name": "ae_c", "bundle": [], "bundle-parents": [], "description": "DESCRIPTION D $GS-0001", "circuits": [] }, ] bundles = { "router_z.geant.net": {"ae_1": ["interface_z"]}, "lab_router_a.geant.net": {"ae_c": ["et/2/lab"]}, "router_a.geant.net": {"ae_a": ["et/1"]}, } snmp_indexes = { "router_a.geant.net": { "ae_a": { "name": "ae_a", "index": 1, "community": "COMMUNITY_A" }, "ae_a.123": { "name": "ae_a.123", "index": 1231, "community": "COMMUNITY_A" }, "et/1": { "name": "et/1", "index": 12, "community": "COMMUNITY_A" } }, "router_b.geant.net": { "ae_a": { "name": "ae_a", "index": 2, "community": "COMMUNITY_A" } }, "lab_router_a.geant.net": { "ae_c": { "name": "ae_c", "index": 3, "community": "COMMUNITY_A" } }, } services_and_customers = { "ROUTER_A": { "AE_A.123": { 'services': [{ "id": 321, "name": "SERVICE A", "type": "SERVICE TYPE", "status": "operational" }], 'customers': [{'name': 'GEANT', 'type': 'UNKNOWN'}] }, "AE_A.456": { 'services': [{ "id": 654, "name": "SERVICE B", "type": "SERVICE TYPE", "status": "operational" }], 'customers': [{'name': 'GEANT', 'type': 'UNKNOWN'}] } } } no_lab_res = [ { 'router': 'router_a.geant.net', 'name': 'et/1', 'bundle': ['ae_a'], 'bundle-parents': [], 'description': 'DESCRIPTION A', 'circuits': [], 'port_type': 'UNKNOWN', 'snmp-index': 12, 'dashboards': [], 'vlan_type': 'ACCESS' }, { 'router': 'router_a.geant.net', 'name': 'ae_a', 'bundle': [], 'bundle-parents': ['et/1'], 'description': 'DESCRIPTION B $GA-0001', 'circuits': [], 'port_type': 'ACCESS', 'snmp-index': 1, 'dashboards': [], 'vlan_type': 'TRUNK' }, { 'router': 'router_a.geant.net', 'name': 'ae_a.123', 'bundle': [], 'bundle-parents': ['et/1'], 'description': 'DESCRIPTION C', 'circuits': [{ 'id': 321, 'name': 'SERVICE A', 'type': 'SERVICE TYPE', 'status': 'operational' }], 'port_type': 'UNKNOWN', 'snmp-index': 1231, 'dashboards': [], 'vlan_type': 'VLAN' } ] lab_res = [ { "router": "lab_router_a.geant.net", "name": "ae_c", "bundle": [], "bundle-parents": ["et/2/lab"], "description": "DESCRIPTION D $GS-0001", "circuits": [], "snmp-index": 3, "dashboards": [], "port_type": "SERVICE", 'vlan_type': 'ACCESS' } ] nren_regions = { 'NREN A': 'REGION A' } for k in r.keys("lab:netconf-interfaces-hosts:*"): r.delete(k) r.set("lab:netconf-interfaces-hosts:lab_router_a.geant.net", "dummy") r.set("lab:netconf-interfaces-hosts:lab_router_b.geant.net", "dummy") mocker.patch('inventory_provider.routes.poller._load_interfaces', side_effect=[all_interfaces, ]) mocker.patch('inventory_provider.routes.poller._load_interface_bundles', return_value=bundles) mocker.patch('inventory_provider.routes.common.load_snmp_indexes', return_value=snmp_indexes) mocker.patch( 'inventory_provider.routes.poller._get_services_and_customers', return_value=services_and_customers) mocker.patch( 'inventory_provider.routes.poller._load_nren_regions', return_value=nren_regions) mocker.patch( 'inventory_provider.tasks.worker.InventoryTask.config' ) populate_poller_interfaces_cache() assert r.exists("classifier-cache:poller-interfaces:all:no_lab") assert r.exists("classifier-cache:poller-interfaces:all") no_lab = \ r.get("classifier-cache:poller-interfaces:all:no_lab").decode("utf-8") all = r.get("classifier-cache:poller-interfaces:all").decode("utf-8") assert json.loads(no_lab) == no_lab_res all_res = no_lab_res + lab_res assert json.loads(all) == all_res def test_refresh_nokia_interface_list(mocked_redis, data_config): netconf_config = remove_xml_namespaces(etree.parse(pathlib.Path(__file__).parent.joinpath( 'data/nokia/rt0.lon.uk.lab.office.geant.net-netconf-nokia.xml'))) r = common._get_redis(data_config) refresh_nokia_interface_list('rt0.lon.uk.lab.office.geant.net', netconf_config, r, True) keybase = 'lab:netconf-interface-bundles:rt0.lon.uk.lab.office.geant.net:' keys = r.keys(f'{keybase}*') bundles = {} for k in keys: k = k.decode('utf-8') bundles[k] = json.loads(r.get(k).decode('utf-8')) expected_bundles = { f'{keybase}lag-1': ['1/1/c8/1', '1/1/c9/1'], f'{keybase}lag-2': ['2/1/c8/1'], f'{keybase}lag-3': ['1/1/c2/2'], f'{keybase}lag-31': ['1/1/c2/1'], } assert bundles == expected_bundles interface_hosts = json.loads(r.get( 'lab:netconf-interfaces-hosts:rt0.lon.uk.lab.office.geant.net' ).decode('utf-8')) assert len(interface_hosts) == 15 interface_hosts_schema = { '$schema': 'http://json-schema.org/draft-07/schema#', 'type': 'array', 'items': { 'type': 'object', 'properties': { 'name': { 'type': 'string' }, 'interface address': { 'type': 'string' }, 'interface name': { 'type': 'string' } }, 'required': ['name', 'interface address', 'interface name'], 'additionalProperties': False } } jsonschema.validate(interface_hosts, interface_hosts_schema) interfaces_schema = { '$schema': 'http://json-schema.org/draft-07/schema#', 'type': 'object', 'properties': { 'name': { 'type': 'string' }, 'description': { 'type': 'string' }, 'bundle': { 'type': 'array', 'items': { 'type': 'string' } }, 'speed': { 'anyOf': [ { 'type': 'string', 'pattern': r'^\d+\wbps$' }, { 'type': 'string', 'enum': [''] } ] }, 'ipv4': { 'type': 'array', 'items': { 'type': 'string', 'format': 'ipv6' } }, 'ipv6': { 'type': 'array', 'items': { 'type': 'string', 'format': 'ipv6' } } }, 'required': ['name', 'description', 'bundle', 'speed', 'ipv4', 'ipv6'] } keybase = 'lab:netconf-interfaces:rt0.lon.uk.lab.office.geant.net:' keys = r.keys(f'{keybase}*') interfaces = {} for k in keys: k = k.decode('utf-8') interface = json.loads(r.get(k).decode('utf-8')) interfaces[k] = interface jsonschema.validate(interface, interfaces_schema) lag_to_check = interfaces[f'{keybase}lag-1'] components_to_check = [ interfaces[f'{keybase}1/1/c8/1'], interfaces[f'{keybase}1/1/c9/1'] ] assert lag_to_check['bundle'] == [] for component in components_to_check: assert component['bundle'] == ['lag-1'] def test_populate_error_report_interfaces_cache(mocker, data_config, mocked_redis): r = common._get_redis(data_config) mocker.patch('inventory_provider.tasks.worker.get_next_redis', return_value=r) all_interfaces = [ { "router": "router_a.geant.net", "name": "interface_a", "bundle": ["ae_a"], "bundle-parents": [], "description": "PHY DESCRIPTION A", "circuits": [] }, { "router": "router_a.geant.net", "name": "ae_a", "bundle": [], "bundle-parents": [], "description": "PHY DESCRIPTION B", "circuits": [] }, { "router": "router_a.geant.net", "name": "ae_a.123", "bundle": [], "bundle-parents": [], "description": "PHY DESCRIPTION C", "circuits": [] }, { "router": "rt0.geant.net", "name": "lag-1", "bundle": ["ae_c"], "bundle-parents": [], "description": "PHY DESCRIPTION D", "circuits": [] }, { "router": "router_a.geant.net", "name": "ae_a.456", "bundle": [], "bundle-parents": [], "description": "Spare Not included", "circuits": [] }, { "router": "router_a.geant.net", "name": "dsc.1", "bundle": [], "bundle-parents": [], "description": "PHY discard", "circuits": [] }, ] mocker.patch('inventory_provider.routes.poller._load_interfaces', return_value=all_interfaces) mocker.patch( 'inventory_provider.tasks.worker.InventoryTask.config' ) netdash_equipment = { "router_a.geant.net": "juniper", "rt0.geant.net": "nokia" } mocker.patch('inventory_provider.routes.poller.get_netdash_equipment', return_value=netdash_equipment) exp_router_a_interfaces = [ { "router": "router_a.geant.net", "name": "ae_a", "description": "PHY DESCRIPTION B", "vendor": "juniper" }, { "router": "router_a.geant.net", "name": "interface_a", "description": "PHY DESCRIPTION A", "vendor": "juniper" }, ] exp_nokia_router_interfaces = [ { "router": "rt0.geant.net", "name": "lag-1", "description": "PHY DESCRIPTION D", "vendor": "nokia" } ] populate_error_report_interfaces_cache() all = r.get("classifier-cache:error-report-interfaces:all").decode("utf-8") assert json.loads(all) == exp_router_a_interfaces + exp_nokia_router_interfaces router_a = r.get("classifier-cache:error-report-interfaces:router_a.geant.net") assert json.loads(router_a) == exp_router_a_interfaces nokia_router = r.get("classifier-cache:error-report-interfaces:rt0.geant.net") assert json.loads(nokia_router) == exp_nokia_router_interfaces def test_nokia_retrieval_failure(mocker, data_config, mocked_redis): mocker.patch('inventory_provider.tasks.worker.InventoryTask.config') mocker.patch("inventory_provider.tasks.worker.nokia.load_docs", side_effect=TransportError("Mocked TransportError")) r = common._get_redis(data_config) mocker.patch('inventory_provider.tasks.worker.get_current_redis', return_value=r) config_doc, state_doc = retrieve_and_persist_config_nokia("unknown.router.geant.net") assert config_doc is None assert state_doc is None