import json
from inventory_provider.tasks import common
from inventory_provider.tasks.worker import transform_ims_data, \
extract_ims_data, persist_ims_data, \
retrieve_and_persist_neteng_managed_device_list, \
populate_poller_interfaces_cache
def test_extract_ims_data(mocker):
mocker.patch(
'inventory_provider.tasks.worker.InventoryTask.config'
)
mocker.patch(
'inventory_provider.tasks.worker.ims_data.get_node_locations',
return_value=[('loc_a', 'LOC A'), ('loc_b', 'LOC B')]
)
mocker.patch(
'inventory_provider.tasks.worker.ims_data.lookup_lg_routers',
return_value=['lg router 1', 'lg router 2']
)
mocker.patch(
'inventory_provider.tasks.worker.ims_data.get_customer_service_emails',
return_value=[('123', 'CON A'), ('456', 'CON B')]
)
mocker.patch(
'inventory_provider.tasks.worker.ims_data.get_monitored_circuit_ids',
return_value=[123, 456, 789]
)
mocker.patch(
'inventory_provider.tasks.worker.ims_data.'
'get_circuit_related_customer_ids',
return_value=[{'id a': ['A', 'A2']}, {'id b': ['B']}]
)
mocker.patch(
'inventory_provider.tasks.worker.ims_data.get_circuit_hierarchy',
return_value=[
{'id': '1', 'value': 'A'},
{'id': '2', 'value': 'B'}
]
)
mocker.patch(
'inventory_provider.tasks.worker.ims_data.get_port_details',
return_value=[
{'port_id': 'A', 'value': 'a'},
{'port_id': 'B', 'value': 'b'},
{'port_id': 'B', 'value': 'c'}
]
)
mocker.patch(
'inventory_provider.tasks.worker.ims_data.get_port_id_services',
return_value=[
{'port_a_id': '1', 'value': '1A'},
{'port_a_id': '1', 'value': '1B'},
{'port_a_id': '2', 'value': '2A'}
]
)
mocker.patch(
'inventory_provider.tasks.worker.ims_data.get_ids_and_sids',
return_value=(x for x in [
(111111, 'SID-01'),
(111112, 'SID-02'),
(111113, 'SID-03')
])
)
mocker.patch(
'inventory_provider.tasks.worker.ims_data.lookup_geant_nodes',
return_value=[]
)
res = extract_ims_data()
assert res['locations'] == {'loc_a': 'LOC A', 'loc_b': 'LOC B'}
assert res['lg_routers'] == ['lg router 1', 'lg router 2']
assert res['customer_contacts'] == {'123': 'CON A', '456': 'CON B'}
assert res['circuit_ids_to_monitor'] == [123, 456, 789]
assert res['additional_circuit_customer_ids'] == \
[{'id a': ['A', 'A2']}, {'id b': ['B']}]
assert res['hierarchy'] == {
'1': {'id': '1', 'value': 'A'},
'2': {'id': '2', 'value': 'B'}
}
assert res['port_id_details'] == {
'A': [{'port_id': 'A', 'value': 'a'}],
'B': [
{'port_id': 'B', 'value': 'b'},
{'port_id': 'B', 'value': 'c'}
]
}
assert res['port_id_services'] == {
'1': [
{'port_a_id': '1', 'value': '1A'},
{'port_a_id': '1', 'value': '1B'}
],
'2': [{'port_a_id': '2', 'value': '2A'}]
}
assert res['circuit_ids_sids'] == {
111111: 'SID-01',
111112: 'SID-02',
111113: 'SID-03'
}
def test_transform_ims_data():
locations = {
"eq_a": {
"pop": {
"name": "pop_loc_a",
"abbreviation": "pla",
}
},
"eq_b": {
"pop": {
"name": "pop_loc_b",
"abbreviation": "plb",
}
},
"UNKNOWN_LOC": {
"pop": {
"name": "UNKNOWN",
"abbreviation": "UNKNOWN",
}
}
}
additional_circuit_customer_ids = {
"circ_id_1": "cu_1_1"
}
customer_contacts = {
"cu_1": ["customer_1@a.org"],
"cu_1_1": ["customer_1_1@a.org"]
}
port_id_details = {
"port_id_1": [{
"equipment_name": "eq_a",
"interface_name": "if_a",
"port_id": "port_id_1"
}],
"port_id_2": [{
"equipment_name": "eq_b",
"interface_name": "if_b",
"port_id": "port_id_2"
}],
"port_id_3": [{
"equipment_name": "eq_a",
"interface_name": "if_c",
"port_id": "port_id_3"
}],
"port_id_4": [{
"equipment_name": "eq_b",
"interface_name": "if_c",
"port_id": "port_id_4"
}]
}
port_id_services = {
"port_id_1": [
{
"id": "circ_id_1",
"name": "circ_name_1",
"customerid": "cu_1",
"customer": "customer_1",
"project": "customer_1",
"circuit_type": "circuit",
"service_type": "ETHERNET",
"status": "operational",
"port_a_id": "port_id_1",
"port_b_id": "port_id_2",
}
],
"port_id_2": [
{
"id": "circ_id_1",
"name": "circ_name_1",
"customerid": "cu_1",
"customer": "customer_1",
"project": "customer_1",
"circuit_type": "circuit",
"service_type": "ETHERNET",
"status": "operational",
"port_a_id": "port_id_2",
"port_b_id": "port_id_1",
}
],
"port_id_3": [
{
"id": "sub_circuit_2",
"name": "sub_circuit_2",
"customerid": "cu_1",
"customer": "customer_1",
"project": "customer_1",
"circuit_type": "service",
"service_type": "PEERING R & E",
"status": "operational",
"port_a_id": "port_id_3",
"port_b_id": "port_id_4",
}
],
"port_id_4": [
{
"id": "sub_circuit_2",
"name": "sub_circuit_2",
"customerid": "cu_1",
"customer": "customer_1",
"project": "customer_1",
"circuit_type": "service",
"service_type": "PEERING R & E",
"status": "operational",
"port_a_id": "port_id_4",
"port_b_id": "port_id_3",
},
]
}
hierarchy = {
"circ_id_1": {
"id": "circ_id_1",
"name": "circ_name_1",
"status": "operational",
"circuit-type": "circuit",
"service_type": "ETHERNET",
"product": "ethernet",
"speed": "not fibre_route",
"project": "customer_1",
"carrier-circuits": ["carrier_id_1"],
"sub-circuits": ["sub_circuit_1"],
"customerid": "cu_1",
},
"carrier_id_1": {
"id": "carrier_id_1",
"name": "circ_carrier_name_1",
"status": "operational",
"circuit-type": "circuit",
"product": "ethernet",
"speed": "10G",
"project": "customer_1",
"carrier-circuits": ["carrier_id_2"],
"sub-circuits": ["circ_id_1"],
"customerid": "cu_1",
},
"carrier_id_2": {
"id": "carrier_id_2",
"name": "circ_carrier_name_3",
"status": "operational",
"circuit-type": "circuit",
"product": "ethernet",
"speed": "not fibre_route",
"project": "customer_1",
"carrier-circuits": ["carrier_id_3"],
"sub-circuits": ["carrier_id_1"],
"customerid": "cu_1",
},
"carrier_id_3": {
"id": "carrier_id_3",
"name": "Fiber Route Circuit",
"status": "operational",
"circuit-type": "circuit",
"product": "OCG4",
"speed": "fibre_route",
"project": "customer_1",
"carrier-circuits": [],
"sub-circuits": ["carrier_id_2"],
"customerid": "cu_1",
},
"sub_circuit_1": {
"id": "sub_circuit_1",
"name": "sub_circuit_name_1",
"status": "operational",
"circuit-type": "circuit",
"product": "ethernet",
"speed": "not fibre_route",
"project": "customer_1",
"carrier-circuits": ["circ_id_1"],
"sub-circuits": ["sub_circuit_2"],
"customerid": "cu_1",
},
"sub_circuit_2": {
"id": "sub_circuit_2",
"name": "sub_circuit_name_2",
"status": "operational",
"circuit-type": "service",
"product": "PEERING R & E",
"speed": "not fiber route",
"project": "customer_1",
"carrier-circuits": ["sub_circuit_1"],
"sub-circuits": [],
"customerid": "cu_1",
}
}
circuit_ids_and_sids = {
"sub_circuit_2": 'SID-01',
"circ_id_2": 'SID-02',
"circ_id_3": 'SID-03'
}
data = {
"locations": locations,
"customer_contacts": customer_contacts,
"circuit_ids_to_monitor": ["sub_circuit_2"],
"additional_circuit_customer_ids": additional_circuit_customer_ids,
"hierarchy": hierarchy,
"port_id_details": port_id_details,
"port_id_services": port_id_services,
"circuit_ids_sids": circuit_ids_and_sids,
"geant_nodes": ["eq_b"]
}
res = transform_ims_data(data)
ifs = res["interface_services"]
assert list(ifs.keys()) == [
"eq_a:if_a", "eq_b:if_b", "eq_a:if_c", "eq_b:if_c"]
for v in ifs.values():
assert len(v) == 1
assert len(v[0]["related-services"]) == 1
assert v[0]["related-services"][0]["id"] == "sub_circuit_2"
assert len(v[0]["fibre-routes"]) == 1
assert v[0]["fibre-routes"][0]["id"] == "carrier_id_3"
nps = res["node_pair_services"]
assert list(nps.keys()) == ["eq_a/eq_b", "eq_b/eq_a"]
v1 = nps["eq_a/eq_b"]["circ_id_1"]
v2 = nps["eq_a/eq_b"]["circ_id_1"]
assert v1 == v2
assert json.dumps(v1, sort_keys=True) == json.dumps(v2, sort_keys=True)
assert len(v) == 1
assert len(v[0]["related-services"]) == 1
assert v[0]["related-services"][0]["id"] == "sub_circuit_2"
assert len(v[0]["fibre-routes"]) == 1
assert v[0]["fibre-routes"][0]["id"] == "carrier_id_3"
assert len(res["sid_services"]['SID-01']) == 2
for x in [
{
'circuit_id': "sub_circuit_2",
'sid': "SID-01",
'name': "sub_circuit_2",
'speed': 10 << 30,
'service_type': "PEERING R & E",
'project': "customer_1",
'customer': "customer_1",
'equipment': "eq_a",
'port': "if_c",
'geant_equipment': False
},
{
'circuit_id': "sub_circuit_2",
'sid': "SID-01",
'name': "sub_circuit_2",
'speed': 10 << 30,
'service_type': "PEERING R & E",
'project': "customer_1",
'customer': "customer_1",
'equipment': "eq_b",
'port': "if_c",
'geant_equipment': True
}
]:
assert json.dumps(x, sort_keys=True) in [
json.dumps(
y, sort_keys=True) for y in res["sid_services"]['SID-01']]
def test_persist_ims_data(mocker, data_config, mocked_redis):
r = common._get_redis(data_config)
mocker.patch('inventory_provider.tasks.worker.get_next_redis',
return_value=r)
data = {
"locations": {"loc_a": "LOC A", "loc_b": "LOC B"},
"lg_routers": [
{"equipment name": "lg_eq1"}, {"equipment name": "lg_eq2"}
],
"hierarchy": {"c1": {"id": "123"}, "c2": {"id": "456"}},
"interface_services": {
"if1": [
{
"equipment": "eq1",
"port": "port1",
"id": "id1",
"name": "name1",
"service_type": "type1",
"status": "operational"
},
{
"equipment": "eq1",
"port": "port2",
"id": "id3",
"name": "name2",
"service_type": "type2",
"status": "operational"
}
],
"if2": [
{
"equipment": "eq2",
"port": "port1",
"id": "id3",
"name": "name3",
"service_type": "type1",
"status": "operational"
}
]
},
"node_pair_services": {
"np1": {"id_1": "data for np1"},
"np2": {"id_2": "data for np2"},
},
"sid_services": {"SID-001": [{"k1": "data"}, {"k1": "data"}]},
"services_by_type": {},
"geant_nodes": []
}
for k in r.keys("ims:*"):
r.delete(k)
persist_ims_data(data)
assert [k.decode("utf-8") for k in r.keys("ims:location:*")] == \
["ims:location:loc_a", "ims:location:loc_b"]
assert [k.decode("utf-8") for k in r.keys("ims:lg:*")] == \
["ims:lg:lg_eq1", "ims:lg:lg_eq2"]
assert [k.decode("utf-8") for k in r.keys("ims:circuit_hierarchy:*")] == \
["ims:circuit_hierarchy:123", "ims:circuit_hierarchy:456"]
assert [k.decode("utf-8") for k in r.keys("ims:interface_services:*")] == \
["ims:interface_services:if1", "ims:interface_services:if2"]
assert [k.decode("utf-8") for k in r.keys("ims:node_pair_services:*")] == \
["ims:node_pair_services:np1", "ims:node_pair_services:np2"]
assert [k.decode("utf-8") for k in r.keys("poller_cache:*")] == \
["poller_cache:eq1", "poller_cache:eq2"]
assert json.loads(r.get("ims:sid_services").decode("utf-8")) == \
data["sid_services"]
def test_retrieve_and_persist_neteng_managed_device_list(
mocker, data_config, mocked_redis):
device_list = ['abc', 'def']
r = common._get_redis(data_config)
mocker.patch(
'inventory_provider.tasks.worker.InventoryTask.config'
)
mocker.patch('inventory_provider.tasks.worker.get_next_redis',
return_value=r)
r.delete('netdash')
mocked_j = mocker.patch(
'inventory_provider.tasks.worker.juniper.load_routers_from_netdash'
)
mocked_j.return_value = device_list
result = retrieve_and_persist_neteng_managed_device_list()
assert result == device_list
assert json.loads(r.get('netdash')) == device_list
def test_populate_poller_interfaces_cache(
mocker, data_config, mocked_redis):
r = common._get_redis(data_config)
all_interfaces = [
{
"router": "router_a.geant.net",
"name": "interface_a",
"bundle": ["ae_a"],
"bundle-parents": [],
"description": "DESCRIPTION A",
"circuits": []
},
{
"router": "router_a.geant.net",
"name": "ae_a",
"bundle": [],
"bundle-parents": [],
"description": "DESCRIPTION B",
"circuits": []
},
{
"router": "router_a.geant.net",
"name": "ae_a.123",
"bundle": [],
"bundle-parents": [],
"description": "DESCRIPTION C",
"circuits": []
},
{
"router": "lab_router_a.geant.net",
"name": "lab_interface_a",
"bundle": ["ae_c"],
"bundle-parents": [],
"description": "DESCRIPTION C",
"circuits": []
},
{
"router": "lab_router_a.geant.net",
"name": "ae_c",
"bundle": [],
"bundle-parents": [],
"description": "DESCRIPTION D",
"circuits": []
},
]
bundles = {
"router_z.geant.net": {"ae_1": ["interface_z"]},
"lab_router_a.geant.net": {"ae_c": ["lab_interface_a"]},
"router_a.geant.net": {"ae_a": ["interface_a"]},
}
snmp_indexes = {
"router_a.geant.net": {
"ae_a": {
"name": "ae_a",
"index": 1,
"community": "COMMUNITY_A"
},
"ae_a.123": {
"name": "ae_a.123",
"index": 1231,
"community": "COMMUNITY_A"
},
"interface_a": {
"name": "interface_a",
"index": 12,
"community": "COMMUNITY_A"
}
},
"router_b.geant.net": {
"ae_a": {
"name": "ae_a",
"index": 2,
"community": "COMMUNITY_A"
}
},
"lab_router_a.geant.net": {
"ae_c": {
"name": "ae_c",
"index": 3,
"community": "COMMUNITY_A"
}
},
}
services = {
"ROUTER_A": {
"AE_A.123": [{
"id": 321,
"name": "SERVICE A",
"type": "SERVICE TYPE",
"status": "operational"
}],
"AE_A.456": [{
"id": 654,
"name": "SERVICE B",
"type": "SERVICE TYPE",
"status": "operational"
}]
}
}
no_lab_res = [
{
"router": "router_a.geant.net",
"name": "interface_a",
"bundle": ["ae_a"],
"bundle-parents": [],
"description": "DESCRIPTION A",
"circuits": [],
"snmp-index": 12,
"dashboards": []
},
{
"router": "router_a.geant.net",
"name": "ae_a",
"bundle": [],
"bundle-parents": ["interface_a"],
"description": "DESCRIPTION B",
"circuits": [],
"snmp-index": 1,
"dashboards": []
},
{
"router": "router_a.geant.net",
"name": "ae_a.123",
"bundle": [],
"bundle-parents": ["interface_a"],
"description": "DESCRIPTION C",
"circuits": [{
"id": 321,
"name": "SERVICE A",
"type": "SERVICE TYPE",
"status": "operational"
}],
"snmp-index": 1231,
"dashboards": []
},
]
lab_res = [
{
"router": "lab_router_a.geant.net",
"name": "ae_c",
"bundle": [],
"bundle-parents": ["lab_interface_a"],
"description": "DESCRIPTION D",
"circuits": [],
"snmp-index": 3,
"dashboards": []
},
]
for k in r.keys("lab:netconf-interfaces-hosts:*"):
r.delete(k)
r.set("lab:netconf-interfaces-hosts:lab_router_a.geant.net", "dummy")
r.set("lab:netconf-interfaces-hosts:lab_router_b.geant.net", "dummy")
mocker.patch('inventory_provider.tasks.worker._load_interfaces',
side_effect=[all_interfaces, ])
mocker.patch('inventory_provider.tasks.worker._load_interface_bundles',
return_value=bundles)
mocker.patch('inventory_provider.tasks.worker.load_snmp_indexes',
return_value=snmp_indexes)
mocker.patch('inventory_provider.tasks.worker._load_services',
return_value=services)
mocker.patch(
'inventory_provider.tasks.worker.InventoryTask.config'
)
mocker.patch('inventory_provider.tasks.worker.get_next_redis',
return_value=r)
populate_poller_interfaces_cache()
assert r.exists("classifier-cache:poller-interfaces:all:no_lab")
assert r.exists("classifier-cache:poller-interfaces:all")
no_lab = \
r.get("classifier-cache:poller-interfaces:all:no_lab").decode("utf-8")
all = r.get("classifier-cache:poller-interfaces:all").decode("utf-8")
assert json.loads(no_lab) == no_lab_res
all_res = no_lab_res + lab_res
assert json.loads(all) == all_res