diff --git a/inventory_provider/db/ims.py b/inventory_provider/db/ims.py
index ef5210af75f39ebd2f87c8fef4342ab9303c60b9..d7b4eebd7297576c0936b5b2461bfca3117702f8 100644
--- a/inventory_provider/db/ims.py
+++ b/inventory_provider/db/ims.py
@@ -221,6 +221,20 @@ class IMS(object):
cls.bearer_token_init_time = re_init_time
cls.bearer_token = response.text
+ def clear_dynamic_context_cache(self):
+ if not IMS.bearer_token:
+ IMS._init_bearer_token(self.username, self.password)
+ while True:
+ logger.info('Clearing Dynamic Context Cache')
+ response = requests.put(
+ f'{self.base_url + IMS.IMS_PATH}/ClearDynamicContextCache',
+ headers={'Authorization': f'Bearer {self.bearer_token}'})
+ if response.status_code == 401:
+ IMS._init_bearer_token(self.username, self.password)
+ continue
+ response.raise_for_status()
+ break
+
def _get_entity(
self,
url,
diff --git a/inventory_provider/db/ims_data.py b/inventory_provider/db/ims_data.py
index 36167cb3292b14606b63ac03ac9fc6e62512cca8..d4ef64ecf25928e8231642060993eef2e640418c 100644
--- a/inventory_provider/db/ims_data.py
+++ b/inventory_provider/db/ims_data.py
@@ -67,6 +67,22 @@ NODE_LOCATION_SCHEMA = {
}
+def get_flexils_by_circuitid(ds: IMS):
+ by_circuit = defaultdict(list)
+ found_keys = set()
+ for entity in ds.get_all_entities('FLEXILS_SCHF_SUBINTERFACES'):
+ k = f'{entity["nodename"]}:{entity["port_ref"]}'
+ if k in found_keys:
+ continue
+ found_keys.add(k)
+ by_circuit[entity['circuitid']].append({
+ 'node_name': entity['nodename'],
+ 'full_port_name': entity['port_ref'],
+ 'key': k
+ })
+ return dict(by_circuit)
+
+
def get_non_monitored_circuit_ids(ds: IMS):
# note the id for the relevant field is hard-coded. I didn't want to use
# the name of the field as this can be changed by users
diff --git a/inventory_provider/routes/classifier.py b/inventory_provider/routes/classifier.py
index 18e1553fb2920326c1b4253d64a5bc1a2b95252a..b9fc8c4fbbf2e73c0334fcb5baa72861ba4efba8 100644
--- a/inventory_provider/routes/classifier.py
+++ b/inventory_provider/routes/classifier.py
@@ -617,10 +617,6 @@ def get_mtc_interface_info(node, interface):
"""
r = common.get_current_redis()
- interface = interface.rsplit('-', 1)[0]
-
- # ims_source_equipment = get_ims_equipment_name(equipment_name, r)
- # ims_interface = get_ims_interface(entity_string)
cache_key = f'classifier-cache:mtc:{node}:{interface}'
diff --git a/inventory_provider/tasks/worker.py b/inventory_provider/tasks/worker.py
index d5b22a0fe7af71e597a6e5a4ba1ba82d1317238f..2919c18a3bbb0b9192384f4cd02cd54d3df19023 100644
--- a/inventory_provider/tasks/worker.py
+++ b/inventory_provider/tasks/worker.py
@@ -741,6 +741,8 @@ def _extract_ims_data(ims_api_url, ims_username, ims_password):
def _ds() -> IMS:
return IMS(ims_api_url, ims_username, ims_password)
+ _ds().clear_dynamic_context_cache()
+
locations = {}
lg_routers = []
geant_nodes = []
@@ -749,6 +751,7 @@ def _extract_ims_data(ims_api_url, ims_username, ims_password):
circuit_ids_to_monitor = []
circuit_ids_and_sids = {}
additional_circuit_customers = {}
+ flexils_data = {}
hierarchy = {}
port_id_details = defaultdict(list)
@@ -823,6 +826,11 @@ def _extract_ims_data(ims_api_url, ims_username, ims_password):
if exceptions:
raise InventoryTaskError(json.dumps(exceptions, indent=2))
+ @log_task_entry_and_exit
+ def _populate_flexils_data():
+ nonlocal flexils_data
+ flexils_data = ims_data.get_flexils_by_circuitid(ds=_ds())
+
@log_task_entry_and_exit
def _populate_hierarchy():
nonlocal hierarchy
@@ -848,7 +856,8 @@ def _extract_ims_data(ims_api_url, ims_username, ims_password):
futures = {
executor.submit(_populate_hierarchy): 'hierarchy',
executor.submit(_populate_port_id_details): 'port_id_details',
- executor.submit(_populate_circuit_info): 'circuit_info'
+ executor.submit(_populate_circuit_info): 'circuit_info',
+ executor.submit(_populate_flexils_data): 'flexils_data'
}
for future in concurrent.futures.as_completed(futures):
@@ -869,7 +878,8 @@ def _extract_ims_data(ims_api_url, ims_username, ims_password):
'hierarchy': hierarchy,
'port_id_details': port_id_details,
'port_id_services': port_id_services,
- 'geant_nodes': geant_nodes
+ 'geant_nodes': geant_nodes,
+ 'flexils_data': flexils_data
}
@@ -884,6 +894,7 @@ def transform_ims_data(data):
port_id_services = data['port_id_services']
circuit_ids_and_sids = data['circuit_ids_sids']
geant_nodes = data['geant_nodes']
+ flexils_data = data['flexils_data']
sid_services = defaultdict(list)
@@ -903,6 +914,32 @@ def transform_ims_data(data):
d['contacts'] = sorted(list(c))
d['planned_work_contacts'] = sorted(list(ttc))
+ # add flexils data to port_id_details and port_id_services
+ all_ils_details = flexils_data.get(d['id'])
+ if all_ils_details:
+ for ils_details in all_ils_details:
+ pid = port_id_details.get(ils_details['key'], [])
+ pid.append({
+ 'port_id': ils_details['key'],
+ 'equipment_name': ils_details['node_name'],
+ 'interface_name': ils_details['full_port_name']
+ })
+ port_id_details[ils_details['key']] = pid
+
+ pis = port_id_services.get(ils_details['key'], [])
+ pis.append({
+ 'id': d['id'],
+ 'name': d['name'],
+ 'project': d['project'],
+ 'port_a_id': ils_details['key'],
+ 'circuit_type': d['circuit-type'],
+ 'status': d['status'],
+ 'service_type': d['product'],
+ 'contacts': d['contacts'],
+ 'planned_work_contacts': d['planned_work_contacts']
+ })
+ port_id_services[ils_details['key']] = pis
+
def _convert_to_bits(value, unit):
unit = unit.lower()
conversions = {
diff --git a/test/test_ims_data.py b/test/test_ims_data.py
index a84590d621bcf6272da510c979cc8634aead6c8a..1c89f5a9b18254a253c4897282dd378f59566f98 100644
--- a/test/test_ims_data.py
+++ b/test/test_ims_data.py
@@ -8,7 +8,8 @@ from inventory_provider.db.ims import InventoryStatus
from inventory_provider.db.ims_data import lookup_lg_routers, \
get_node_locations, IMS_OPSDB_STATUS_MAP, \
get_port_id_services, get_port_details, \
- get_circuit_hierarchy, get_ids_and_sids, NODE_LOCATION_SCHEMA
+ get_circuit_hierarchy, get_ids_and_sids, NODE_LOCATION_SCHEMA, \
+ get_flexils_by_circuitid
def _json_test_data(filename):
@@ -385,3 +386,63 @@ def test_get_circuit_ids_and_sids(mocker):
'dummy_base', 'dummy_username', 'dummy_password')
res = list(get_ids_and_sids(ds))
assert res == expected_response
+
+
+def test_get_flexils_by_circuit_id(mocker):
+ ims = mocker.patch('inventory_provider.db.ims.IMS')
+ ims.return_value.get_all_entities.return_value = \
+ [
+ {
+ "$id": "1630",
+ "circuitid": 702203,
+ "circuitname": "GEANT LAB-GEANT LAB-OCH-003",
+ "id": 1,
+ "nodename": "CAM01-MTC6-3",
+ "node_id": 130498,
+ "port": "A-1-S1",
+ "port_ref": "1-A-1-S1-1",
+ "shelfname": "1-MTC-6",
+ "rowVersion": "0001-01-01T00:00:00",
+ "circuit": None,
+ "node_": None,
+ "errors": None,
+ "haserrors": False,
+ "customproperties": {}
+ },
+ {
+ "$id": "1631",
+ "circuitid": 702203,
+ "circuitname": "GEANT LAB-GEANT LAB-OCH-003",
+ "id": 2,
+ "nodename": "CAM01-MTC6-3",
+ "node_id": 130498,
+ "port": "A-1-L1",
+ "port_ref": "1-A-1-L1-1",
+ "shelfname": "1-MTC-6",
+ "rowversion": "0001-01-01T00:00:00",
+ "circuit": None,
+ "node_": None,
+ "errors": None,
+ "haserrors": False,
+ "customproperties": {}
+ }
+ ]
+ expected_result = {
+ 702203: [
+ {
+ 'node_name': 'CAM01-MTC6-3',
+ 'full_port_name': '1-A-1-S1-1',
+ 'key': 'CAM01-MTC6-3:1-A-1-S1-1'
+ },
+ {
+ 'node_name': 'CAM01-MTC6-3',
+ 'full_port_name': '1-A-1-L1-1',
+ 'key': 'CAM01-MTC6-3:1-A-1-L1-1'
+ }
+ ]
+ }
+
+ ds = inventory_provider.db.ims.IMS(
+ 'dummy_base', 'dummy_username', 'dummy_password')
+ res = get_flexils_by_circuitid(ds)
+ assert res == expected_result
diff --git a/test/test_worker.py b/test/test_worker.py
index 7018c2c608d62afe9ea2e0a43777b704c8212537..970e9ed1f2f462f9367495b45136f8fe79fe9ce0 100644
--- a/test/test_worker.py
+++ b/test/test_worker.py
@@ -16,6 +16,9 @@ def test_extract_ims_data(mocker):
'inventory_provider.tasks.worker.ims_data.get_node_locations',
return_value=[('loc_a', 'LOC A'), ('loc_b', 'LOC B')]
)
+ mocker.patch(
+ 'inventory_provider.tasks.worker.IMS.clear_dynamic_context_cache'
+ )
mocker.patch(
'inventory_provider.tasks.worker.ims_data.lookup_lg_routers',
return_value=['lg router 1', 'lg router 2']
@@ -81,6 +84,10 @@ def test_extract_ims_data(mocker):
'inventory_provider.tasks.worker.ims_data.lookup_geant_nodes',
return_value=[]
)
+ mocker.patch(
+ 'inventory_provider.tasks.worker.ims_data.get_flexils_by_circuitid',
+ return_value={}
+ )
res = extract_ims_data()
assert res['locations'] == {'loc_a': 'LOC A', 'loc_b': 'LOC B'}
assert res['lg_routers'] == ['lg router 1', 'lg router 2']
@@ -321,6 +328,18 @@ def test_transform_ims_data():
"carrier-circuits": ["sub_circuit_1"],
"sub-circuits": [],
"customerid": "cu_1",
+ },
+ 702203: {
+ "id": 702203,
+ "name": "FLEX ILS CIRCUIT",
+ "status": "operational",
+ "circuit-type": "circuit",
+ "product": "ethernet",
+ "speed": "not fiber route",
+ "project": "customer_1",
+ "carrier-circuits": ["circ_id_1"],
+ "sub-circuits": ["sub_circuit_2"],
+ "customerid": "cu_1",
}
}
@@ -329,6 +348,20 @@ def test_transform_ims_data():
"circ_id_2": 'SID-02',
"circ_id_3": 'SID-03'
}
+ flexils_data = {
+ 702203: [
+ {
+ 'node_name': 'CAM01-MTC6-3',
+ 'full_port_name': '1-A-1-S1-1',
+ 'key': 'CAM01-MTC6-3:1-A-1-S1-1'
+ },
+ {
+ 'node_name': 'CAM01-MTC6-3',
+ 'full_port_name': '1-A-1-L1-1',
+ 'key': 'CAM01-MTC6-3:1-A-1-L1-1'
+ }
+ ]
+ }
data = {
"locations": locations,
"customer_contacts": customer_contacts,
@@ -339,9 +372,12 @@ def test_transform_ims_data():
"port_id_details": port_id_details,
"port_id_services": port_id_services,
"circuit_ids_sids": circuit_ids_and_sids,
- "geant_nodes": ["eq_b"]
+ "geant_nodes": ["eq_b"],
+ "flexils_data": flexils_data
}
+ orig_port_id_services_len = len(port_id_services.keys())
res = transform_ims_data(data)
+ assert len(port_id_services) == orig_port_id_services_len + 2
pop_nodes_res = {
"pop_loc_a": ["eq_a"],
"pop_loc_b": ["eq_b"],
@@ -352,7 +388,13 @@ def test_transform_ims_data():
assert v == res["pop_nodes"][k]
ifs = res["interface_services"]
assert list(ifs.keys()) == [
- "eq_a:if_a", "eq_b:if_b", "eq_a:if_c", "eq_b:if_c"]
+ "eq_a:if_a",
+ "eq_b:if_b",
+ "eq_a:if_c",
+ "eq_b:if_c",
+ "CAM01-MTC6-3:1-A-1-S1-1",
+ "CAM01-MTC6-3:1-A-1-L1-1"
+ ]
for v in ifs.values():
assert len(v) == 1
assert len(v[0]["related-services"]) == 1