diff --git a/inventory_provider/db/ims_data.py b/inventory_provider/db/ims_data.py
index 271faaf0b290bfd75da3006e5be321c3f9c1a547..e918835c2ee01fb560d92e84a2f9fa89d97ca174 100644
--- a/inventory_provider/db/ims_data.py
+++ b/inventory_provider/db/ims_data.py
@@ -203,6 +203,7 @@ def get_port_id_services(ds: IMS):
'circuit_type': circuit['circuit_type'],
'service_type': products[circuit['productid']],
'project': customers[circuit['customerid']],
+ 'customer': customers[circuit['customerid']],
'customerid': circuit['customerid']
}
ports = []
@@ -248,6 +249,7 @@ def get_port_id_services(ds: IMS):
'circuit_type': _get_circuit_type(circuit),
'service_type': products[circuit['productid']],
'project': customers[circuit['customerid']],
+ 'customer': customers[circuit['customerid']],
'customerid': circuit['customerid'],
'port_a_id': portrelate.get(
'portid',
@@ -474,3 +476,11 @@ def lookup_lg_routers(ds: IMS):
}
}
yield eq
+
+
+def lookup_geant_nodes(ds: IMS):
+
+ return (n["name"]for n in ds.get_filtered_entities(
+ 'Node',
+ 'customer.Name == "GEANT"',
+ ims.EQUIP_DEF_PROPERTIES['Nodes']))
diff --git a/inventory_provider/tasks/worker.py b/inventory_provider/tasks/worker.py
index f841b942c1c5ddceeae2a49ccb87b203687a005d..47aeb0dadd881f9d8eec7f77d3e9ffd0a4b52bae 100644
--- a/inventory_provider/tasks/worker.py
+++ b/inventory_provider/tasks/worker.py
@@ -717,6 +717,7 @@ def extract_ims_data():
locations = {}
lg_routers = []
+ geant_nodes = []
customer_contacts = {}
circuit_ids_to_monitor = []
circuit_ids_and_sids = {}
@@ -734,6 +735,10 @@ def extract_ims_data():
nonlocal lg_routers
lg_routers = list(ims_data.lookup_lg_routers(ds=_ds()))
+ def _populate_geant_nodes():
+ nonlocal geant_nodes
+ geant_nodes = list(ims_data.lookup_geant_nodes(ds=_ds()))
+
def _populate_customer_contacts():
nonlocal customer_contacts
customer_contacts = \
@@ -758,6 +763,7 @@ def extract_ims_data():
with concurrent.futures.ThreadPoolExecutor() as executor:
futures = {
executor.submit(_populate_locations): 'locations',
+ executor.submit(_populate_geant_nodes): 'geant_nodes',
executor.submit(_populate_lg_routers): 'lg_routers',
executor.submit(_populate_customer_contacts): 'customer_contacts',
executor.submit(_populate_circuit_ids_to_monitor):
@@ -815,7 +821,8 @@ def extract_ims_data():
'additional_circuit_customer_ids': additional_circuit_customer_ids,
'hierarchy': hierarchy,
'port_id_details': port_id_details,
- 'port_id_services': port_id_services
+ 'port_id_services': port_id_services,
+ 'geant_nodes': geant_nodes
}
@@ -827,6 +834,10 @@ def transform_ims_data(data):
hierarchy = data['hierarchy']
port_id_details = data['port_id_details']
port_id_services = data['port_id_services']
+ circuit_ids_and_sids = data['circuit_ids_sids']
+ geant_nodes = data['geant_nodes']
+
+ sid_services = defaultdict(list)
def _get_circuit_contacts(c):
customer_ids = {c['customerid']}
@@ -982,20 +993,43 @@ def transform_ims_data(data):
_format_service(circ)
type_services = services_by_type.setdefault(
- ims_sorted_service_type_key(circ['service_type']), dict())
+ ims_sorted_service_type_key(circ['service_type']), dict()) # comment
type_services[circ['id']] = circ
if circ['other_end_equipment']:
node_pair_services[
f"{circ['equipment']}/{circ['other_end_equipment']}"
][circ['id']] = circ
+ if circ['id'] in circuit_ids_and_sids \
+ and circ['status'] == 'operational':
+ sid = circuit_ids_and_sids[circ['id']]
+ if circ['circuit_type'] == 'circuit':
+ logger.info(f'SID ({sid}) Circuit ({circ["id"]})'
+ f' Name ({circ["name"]}) not a service')
+ else:
+ sid_info = {
+ 'circuit_id': circ['id'],
+ 'sid': sid,
+ 'name': circ['name'],
+ 'speed': circ['calculated-speed'],
+ 'service_type': circ['service_type'],
+ 'project': circ['project'],
+ 'customer': circ['customer'],
+ 'equipment': circ['equipment'],
+ 'port': circ['port'],
+ 'geant_equipment': circ['equipment'] in geant_nodes
+ }
+ if sid_info not in sid_services[sid]:
+ sid_services[sid].append(sid_info)
+
interface_services[k].extend(circuits)
return {
'hierarchy': hierarchy,
'interface_services': interface_services,
'services_by_type': services_by_type,
- 'node_pair_services': node_pair_services
+ 'node_pair_services': node_pair_services,
+ 'sid_services': sid_services
}
@@ -1006,6 +1040,7 @@ def persist_ims_data(data, use_current=False):
interface_services = data['interface_services']
services_by_type = data['services_by_type']
node_pair_services = data['node_pair_services']
+ sid_services = data['sid_services']
if use_current:
r = get_current_redis(InventoryTask.config)
@@ -1021,12 +1056,15 @@ def persist_ims_data(data, use_current=False):
'ims:gws_indirect:*',
'ims:node_pair_services:*'
]:
+
+ r.delete('ims:sid_services')
rp = r.pipeline()
for k in r.scan_iter(key_pattern, count=1000):
rp.delete(k)
else:
r = get_next_redis(InventoryTask.config)
+ r.set('ims:sid_services', json.dumps(sid_services))
rp = r.pipeline()
for h, d in locations.items():
rp.set(f'ims:location:{h}', json.dumps([d]))
diff --git a/test/test_ims_data.py b/test/test_ims_data.py
index eb0adb431d00c1498774d15df89f0425de1e999c..ae2a2e8cbd713adc806e3bd0bfaf177ee8bdc850 100644
--- a/test/test_ims_data.py
+++ b/test/test_ims_data.py
@@ -162,6 +162,7 @@ def test_get_port_id_services(mocker):
'circuit_type': 'service',
'service_type': 'GEANT IP',
'project': 'ORG A',
+ 'customer': 'ORG A',
'port_a_id': 224507,
'customerid': 57658
},
@@ -172,6 +173,7 @@ def test_get_port_id_services(mocker):
'circuit_type': 'service',
'service_type': 'GEANT PEERING',
'project': 'ORG B',
+ 'customer': 'ORG B',
'port_a_id': 224464,
'customerid': 57664
},
@@ -182,6 +184,7 @@ def test_get_port_id_services(mocker):
'circuit_type': 'circuit',
'service_type': 'ETHERNET',
'project': 'ETH',
+ 'customer': 'ETH',
'port_a_id': 6423107,
'port_b_id': 6419340,
'customerid': 57744
@@ -193,6 +196,7 @@ def test_get_port_id_services(mocker):
'circuit_type': 'circuit',
'service_type': 'ETHERNET',
'project': 'ETH',
+ 'customer': 'ETH',
'port_a_id': 6419340,
'port_b_id': 6423107,
'customerid': 57744
@@ -204,6 +208,7 @@ def test_get_port_id_services(mocker):
'circuit_type': 'circuit',
'service_type': 'ETHERNET',
'project': 'ETH',
+ 'customer': 'ETH',
'port_a_id': 6423111,
'customerid': 57744
},
@@ -214,6 +219,7 @@ def test_get_port_id_services(mocker):
'circuit_type': 'service',
'service_type': 'PRODUCT A',
'project': 'ORG C',
+ 'customer': 'ORG C',
'port_a_id': 6419453,
'customerid': 57640
}
diff --git a/test/test_worker.py b/test/test_worker.py
index 2dbce4732031de4decae67b95d0db64d0a524994..7b75b400c18418ae8d0d65f729ba5db3f9132cd9 100644
--- a/test/test_worker.py
+++ b/test/test_worker.py
@@ -64,6 +64,10 @@ def test_extract_ims_data(mocker):
(111113, 'SID-03')
])
)
+ mocker.patch(
+ 'inventory_provider.tasks.worker.ims_data.lookup_geant_nodes',
+ return_value=[]
+ )
res = extract_ims_data()
assert res['locations'] == {'loc_a': 'LOC A', 'loc_b': 'LOC B'}
assert res['lg_routers'] == ['lg router 1', 'lg router 2']
@@ -137,6 +141,16 @@ def test_transform_ims_data():
"equipment_name": "eq_b",
"interface_name": "if_b",
"port_id": "port_id_2"
+ }],
+ "port_id_3": [{
+ "equipment_name": "eq_a",
+ "interface_name": "if_c",
+ "port_id": "port_id_3"
+ }],
+ "port_id_4": [{
+ "equipment_name": "eq_b",
+ "interface_name": "if_c",
+ "port_id": "port_id_4"
}]
}
@@ -144,7 +158,10 @@ def test_transform_ims_data():
"port_id_1": [
{
"id": "circ_id_1",
+ "name": "circ_name_1",
"customerid": "cu_1",
+ "customer": "customer_1",
+ "project": "customer_1",
"circuit_type": "circuit",
"service_type": "ETHERNET",
"status": "operational",
@@ -156,7 +173,10 @@ def test_transform_ims_data():
"port_id_2": [
{
"id": "circ_id_1",
+ "name": "circ_name_1",
"customerid": "cu_1",
+ "customer": "customer_1",
+ "project": "customer_1",
"circuit_type": "circuit",
"service_type": "ETHERNET",
"status": "operational",
@@ -164,6 +184,34 @@ def test_transform_ims_data():
"port_b_id": "port_id_1",
}
+ ],
+ "port_id_3": [
+ {
+ "id": "sub_circuit_2",
+ "name": "sub_circuit_2",
+ "customerid": "cu_1",
+ "customer": "customer_1",
+ "project": "customer_1",
+ "circuit_type": "service",
+ "service_type": "PEERING R & E",
+ "status": "operational",
+ "port_a_id": "port_id_3",
+ "port_b_id": "port_id_4",
+ }
+ ],
+ "port_id_4": [
+ {
+ "id": "sub_circuit_2",
+ "name": "sub_circuit_2",
+ "customerid": "cu_1",
+ "customer": "customer_1",
+ "project": "customer_1",
+ "circuit_type": "service",
+ "service_type": "PEERING R & E",
+ "status": "operational",
+ "port_a_id": "port_id_4",
+ "port_b_id": "port_id_3",
+ },
]
}
@@ -173,8 +221,10 @@ def test_transform_ims_data():
"name": "circ_name_1",
"status": "operational",
"circuit-type": "circuit",
+ "service_type": "ETHERNET",
"product": "ethernet",
"speed": "not fibre_route",
+ "project": "customer_1",
"carrier-circuits": ["carrier_id_1"],
"sub-circuits": ["sub_circuit_1"],
"customerid": "cu_1",
@@ -186,6 +236,7 @@ def test_transform_ims_data():
"circuit-type": "circuit",
"product": "ethernet",
"speed": "10G",
+ "project": "customer_1",
"carrier-circuits": ["carrier_id_2"],
"sub-circuits": ["circ_id_1"],
"customerid": "cu_1",
@@ -197,6 +248,7 @@ def test_transform_ims_data():
"circuit-type": "circuit",
"product": "ethernet",
"speed": "not fibre_route",
+ "project": "customer_1",
"carrier-circuits": ["carrier_id_3"],
"sub-circuits": ["carrier_id_1"],
"customerid": "cu_1",
@@ -208,6 +260,7 @@ def test_transform_ims_data():
"circuit-type": "circuit",
"product": "OCG4",
"speed": "fibre_route",
+ "project": "customer_1",
"carrier-circuits": [],
"sub-circuits": ["carrier_id_2"],
"customerid": "cu_1",
@@ -219,6 +272,7 @@ def test_transform_ims_data():
"circuit-type": "circuit",
"product": "ethernet",
"speed": "not fibre_route",
+ "project": "customer_1",
"carrier-circuits": ["circ_id_1"],
"sub-circuits": ["sub_circuit_2"],
"customerid": "cu_1",
@@ -230,24 +284,33 @@ def test_transform_ims_data():
"circuit-type": "service",
"product": "PEERING R & E",
"speed": "not fiber route",
- "project": "Project A",
+ "project": "customer_1",
"carrier-circuits": ["sub_circuit_1"],
"sub-circuits": [],
"customerid": "cu_1",
}
}
+
+ circuit_ids_and_sids = {
+ "sub_circuit_2": 'SID-01',
+ "circ_id_2": 'SID-02',
+ "circ_id_3": 'SID-03'
+ }
data = {
"locations": locations,
"customer_contacts": customer_contacts,
- "circuit_ids_to_monitor": [],
+ "circuit_ids_to_monitor": ["sub_circuit_2"],
"additional_circuit_customer_ids": additional_circuit_customer_ids,
"hierarchy": hierarchy,
"port_id_details": port_id_details,
- "port_id_services": port_id_services
+ "port_id_services": port_id_services,
+ "circuit_ids_sids": circuit_ids_and_sids,
+ "geant_nodes": ["eq_b"]
}
res = transform_ims_data(data)
ifs = res["interface_services"]
- assert list(ifs.keys()) == ["eq_a:if_a", "eq_b:if_b"]
+ assert list(ifs.keys()) == [
+ "eq_a:if_a", "eq_b:if_b", "eq_a:if_c", "eq_b:if_c"]
for v in ifs.values():
assert len(v) == 1
assert len(v[0]["related-services"]) == 1
@@ -267,6 +330,38 @@ def test_transform_ims_data():
assert len(v[0]["fibre-routes"]) == 1
assert v[0]["fibre-routes"][0]["id"] == "carrier_id_3"
+ assert len(res["sid_services"]['SID-01']) == 2
+
+ for x in [
+ {
+ 'circuit_id': "sub_circuit_2",
+ 'sid': "SID-01",
+ 'name': "sub_circuit_2",
+ 'speed': 10 << 30,
+ 'service_type': "PEERING R & E",
+ 'project': "customer_1",
+ 'customer': "customer_1",
+ 'equipment': "eq_a",
+ 'port': "if_c",
+ 'geant_equipment': False
+ },
+ {
+ 'circuit_id': "sub_circuit_2",
+ 'sid': "SID-01",
+ 'name': "sub_circuit_2",
+ 'speed': 10 << 30,
+ 'service_type': "PEERING R & E",
+ 'project': "customer_1",
+ 'customer': "customer_1",
+ 'equipment': "eq_b",
+ 'port': "if_c",
+ 'geant_equipment': True
+ }
+ ]:
+ assert json.dumps(x, sort_keys=True) in [
+ json.dumps(
+ y, sort_keys=True) for y in res["sid_services"]['SID-01']]
+
def test_persist_ims_data(mocker, data_config, mocked_redis):
@@ -314,7 +409,9 @@ def test_persist_ims_data(mocker, data_config, mocked_redis):
"np1": {"id_1": "data for np1"},
"np2": {"id_2": "data for np2"},
},
+ "sid_services": {"SID-001": [{"k1": "data"}, {"k1": "data"}]},
"services_by_type": {},
+ "geant_nodes": []
}
for k in r.keys("ims:*"):
r.delete(k)
@@ -338,6 +435,9 @@ def test_persist_ims_data(mocker, data_config, mocked_redis):
assert [k.decode("utf-8") for k in r.keys("poller_cache:*")] == \
["poller_cache:eq1", "poller_cache:eq2"]
+ assert json.loads(r.get("ims:sid_services").decode("utf-8")) == \
+ data["sid_services"]
+
def test_retrieve_and_persist_neteng_managed_device_list(
mocker, data_config, mocked_redis):