Merge branch 'feature/POL1-539' into develop

inventory_provider/db/ims_data.py

@@ -203,6 +203,7 @@ def get_port_id_services(ds: IMS):
             'circuit_type': circuit['circuit_type'],
             'service_type': products[circuit['productid']],
             'project': customers[circuit['customerid']],
+            'customer': customers[circuit['customerid']],
             'customerid': circuit['customerid']
         }
         ports = []
@@ -248,6 +249,7 @@ def get_port_id_services(ds: IMS):
                 'circuit_type': _get_circuit_type(circuit),
                 'service_type': products[circuit['productid']],
                 'project': customers[circuit['customerid']],
+                'customer': customers[circuit['customerid']],
                 'customerid': circuit['customerid'],
                 'port_a_id': portrelate.get(
                     'portid',
@@ -474,3 +476,11 @@ def lookup_lg_routers(ds: IMS):
                     }
                 }
             yield eq
+
+
+def lookup_geant_nodes(ds: IMS):
+
+    return (n["name"]for n in ds.get_filtered_entities(
+        'Node',
+        'customer.Name == "GEANT"',
+        ims.EQUIP_DEF_PROPERTIES['Nodes']))

inventory_provider/tasks/worker.py

@@ -717,6 +717,7 @@ def extract_ims_data():
 
     locations = {}
     lg_routers = []
+    geant_nodes = []
     customer_contacts = {}
     circuit_ids_to_monitor = []
     circuit_ids_and_sids = {}
@@ -734,6 +735,10 @@ def extract_ims_data():
         nonlocal lg_routers
         lg_routers = list(ims_data.lookup_lg_routers(ds=_ds()))
 
+    def _populate_geant_nodes():
+        nonlocal geant_nodes
+        geant_nodes = list(ims_data.lookup_geant_nodes(ds=_ds()))
+
     def _populate_customer_contacts():
         nonlocal customer_contacts
         customer_contacts = \
@@ -758,6 +763,7 @@ def extract_ims_data():
     with concurrent.futures.ThreadPoolExecutor() as executor:
         futures = {
             executor.submit(_populate_locations): 'locations',
+            executor.submit(_populate_geant_nodes): 'geant_nodes',
             executor.submit(_populate_lg_routers): 'lg_routers',
             executor.submit(_populate_customer_contacts): 'customer_contacts',
             executor.submit(_populate_circuit_ids_to_monitor):
@@ -815,7 +821,8 @@ def extract_ims_data():
         'additional_circuit_customer_ids': additional_circuit_customer_ids,
         'hierarchy': hierarchy,
         'port_id_details': port_id_details,
-        'port_id_services': port_id_services
+        'port_id_services': port_id_services,
+        'geant_nodes': geant_nodes
     }
 
 
@@ -827,6 +834,10 @@ def transform_ims_data(data):
     hierarchy = data['hierarchy']
     port_id_details = data['port_id_details']
     port_id_services = data['port_id_services']
+    circuit_ids_and_sids = data['circuit_ids_sids']
+    geant_nodes = data['geant_nodes']
+
+    sid_services = defaultdict(list)
 
     def _get_circuit_contacts(c):
         customer_ids = {c['customerid']}
@@ -982,20 +993,43 @@ def transform_ims_data(data):
                 _format_service(circ)
 
                 type_services = services_by_type.setdefault(
-                    ims_sorted_service_type_key(circ['service_type']), dict())
+                    ims_sorted_service_type_key(circ['service_type']), dict())  # comment
                 type_services[circ['id']] = circ
                 if circ['other_end_equipment']:
                     node_pair_services[
                         f"{circ['equipment']}/{circ['other_end_equipment']}"
                     ][circ['id']] = circ
 
+                if circ['id'] in circuit_ids_and_sids \
+                        and circ['status'] == 'operational':
+                    sid = circuit_ids_and_sids[circ['id']]
+                    if circ['circuit_type'] == 'circuit':
+                        logger.info(f'SID ({sid}) Circuit ({circ["id"]})'
+                                    f' Name ({circ["name"]}) not a service')
+                    else:
+                        sid_info = {
+                            'circuit_id': circ['id'],
+                            'sid': sid,
+                            'name': circ['name'],
+                            'speed': circ['calculated-speed'],
+                            'service_type': circ['service_type'],
+                            'project': circ['project'],
+                            'customer': circ['customer'],
+                            'equipment': circ['equipment'],
+                            'port': circ['port'],
+                            'geant_equipment': circ['equipment'] in geant_nodes
+                        }
+                        if sid_info not in sid_services[sid]:
+                            sid_services[sid].append(sid_info)
+
         interface_services[k].extend(circuits)
 
     return {
         'hierarchy': hierarchy,
         'interface_services': interface_services,
         'services_by_type': services_by_type,
-        'node_pair_services': node_pair_services
+        'node_pair_services': node_pair_services,
+        'sid_services': sid_services
     }
 
 
@@ -1006,6 +1040,7 @@ def persist_ims_data(data, use_current=False):
     interface_services = data['interface_services']
     services_by_type = data['services_by_type']
     node_pair_services = data['node_pair_services']
+    sid_services = data['sid_services']
 
     if use_current:
         r = get_current_redis(InventoryTask.config)
@@ -1021,12 +1056,15 @@ def persist_ims_data(data, use_current=False):
             'ims:gws_indirect:*',
             'ims:node_pair_services:*'
         ]:
+
+            r.delete('ims:sid_services')
             rp = r.pipeline()
             for k in r.scan_iter(key_pattern, count=1000):
                 rp.delete(k)
     else:
         r = get_next_redis(InventoryTask.config)
 
+    r.set('ims:sid_services', json.dumps(sid_services))
     rp = r.pipeline()
     for h, d in locations.items():
         rp.set(f'ims:location:{h}', json.dumps([d]))

test/test_ims_data.py

@@ -162,6 +162,7 @@ def test_get_port_id_services(mocker):
             'circuit_type': 'service',
             'service_type': 'GEANT IP',
             'project': 'ORG A',
+            'customer': 'ORG A',
             'port_a_id': 224507,
             'customerid': 57658
         },
@@ -172,6 +173,7 @@ def test_get_port_id_services(mocker):
             'circuit_type': 'service',
             'service_type': 'GEANT PEERING',
             'project': 'ORG B',
+            'customer': 'ORG B',
             'port_a_id': 224464,
             'customerid': 57664
         },
@@ -182,6 +184,7 @@ def test_get_port_id_services(mocker):
             'circuit_type': 'circuit',
             'service_type': 'ETHERNET',
             'project': 'ETH',
+            'customer': 'ETH',
             'port_a_id': 6423107,
             'port_b_id': 6419340,
             'customerid': 57744
@@ -193,6 +196,7 @@ def test_get_port_id_services(mocker):
             'circuit_type': 'circuit',
             'service_type': 'ETHERNET',
             'project': 'ETH',
+            'customer': 'ETH',
             'port_a_id': 6419340,
             'port_b_id': 6423107,
             'customerid': 57744
@@ -204,6 +208,7 @@ def test_get_port_id_services(mocker):
             'circuit_type': 'circuit',
             'service_type': 'ETHERNET',
             'project': 'ETH',
+            'customer': 'ETH',
             'port_a_id': 6423111,
             'customerid': 57744
         },
@@ -214,6 +219,7 @@ def test_get_port_id_services(mocker):
             'circuit_type': 'service',
             'service_type': 'PRODUCT A',
             'project': 'ORG C',
+            'customer': 'ORG C',
             'port_a_id': 6419453,
             'customerid': 57640
         }

test/test_worker.py

@@ -64,6 +64,10 @@ def test_extract_ims_data(mocker):
             (111113, 'SID-03')
         ])
     )
+    mocker.patch(
+        'inventory_provider.tasks.worker.ims_data.lookup_geant_nodes',
+        return_value=[]
+    )
     res = extract_ims_data()
     assert res['locations'] == {'loc_a': 'LOC A', 'loc_b': 'LOC B'}
     assert res['lg_routers'] == ['lg router 1', 'lg router 2']
@@ -137,6 +141,16 @@ def test_transform_ims_data():
             "equipment_name": "eq_b",
             "interface_name": "if_b",
             "port_id": "port_id_2"
+        }],
+        "port_id_3": [{
+            "equipment_name": "eq_a",
+            "interface_name": "if_c",
+            "port_id": "port_id_3"
+        }],
+        "port_id_4": [{
+            "equipment_name": "eq_b",
+            "interface_name": "if_c",
+            "port_id": "port_id_4"
         }]
     }
 
@@ -144,7 +158,10 @@ def test_transform_ims_data():
         "port_id_1": [
             {
                 "id": "circ_id_1",
+                "name": "circ_name_1",
                 "customerid": "cu_1",
+                "customer": "customer_1",
+                "project": "customer_1",
                 "circuit_type": "circuit",
                 "service_type": "ETHERNET",
                 "status": "operational",
@@ -156,7 +173,10 @@ def test_transform_ims_data():
         "port_id_2": [
             {
                 "id": "circ_id_1",
+                "name": "circ_name_1",
                 "customerid": "cu_1",
+                "customer": "customer_1",
+                "project": "customer_1",
                 "circuit_type": "circuit",
                 "service_type": "ETHERNET",
                 "status": "operational",
@@ -164,6 +184,34 @@ def test_transform_ims_data():
                 "port_b_id": "port_id_1",
 
             }
+        ],
+        "port_id_3": [
+            {
+                "id": "sub_circuit_2",
+                "name": "sub_circuit_2",
+                "customerid": "cu_1",
+                "customer": "customer_1",
+                "project": "customer_1",
+                "circuit_type": "service",
+                "service_type": "PEERING R & E",
+                "status": "operational",
+                "port_a_id": "port_id_3",
+                "port_b_id": "port_id_4",
+                }
+        ],
+        "port_id_4": [
+            {
+                "id": "sub_circuit_2",
+                "name": "sub_circuit_2",
+                "customerid": "cu_1",
+                "customer": "customer_1",
+                "project": "customer_1",
+                "circuit_type": "service",
+                "service_type": "PEERING R & E",
+                "status": "operational",
+                "port_a_id": "port_id_4",
+                "port_b_id": "port_id_3",
+            },
         ]
     }
 
@@ -173,8 +221,10 @@ def test_transform_ims_data():
             "name": "circ_name_1",
             "status": "operational",
             "circuit-type": "circuit",
+            "service_type": "ETHERNET",
             "product": "ethernet",
             "speed": "not fibre_route",
+            "project": "customer_1",
             "carrier-circuits": ["carrier_id_1"],
             "sub-circuits": ["sub_circuit_1"],
             "customerid": "cu_1",
@@ -186,6 +236,7 @@ def test_transform_ims_data():
             "circuit-type": "circuit",
             "product": "ethernet",
             "speed": "10G",
+            "project": "customer_1",
             "carrier-circuits": ["carrier_id_2"],
             "sub-circuits": ["circ_id_1"],
             "customerid": "cu_1",
@@ -197,6 +248,7 @@ def test_transform_ims_data():
             "circuit-type": "circuit",
             "product": "ethernet",
             "speed": "not fibre_route",
+            "project": "customer_1",
             "carrier-circuits": ["carrier_id_3"],
             "sub-circuits": ["carrier_id_1"],
             "customerid": "cu_1",
@@ -208,6 +260,7 @@ def test_transform_ims_data():
             "circuit-type": "circuit",
             "product": "OCG4",
             "speed": "fibre_route",
+            "project": "customer_1",
             "carrier-circuits": [],
             "sub-circuits": ["carrier_id_2"],
             "customerid": "cu_1",
@@ -219,6 +272,7 @@ def test_transform_ims_data():
             "circuit-type": "circuit",
             "product": "ethernet",
             "speed": "not fibre_route",
+            "project": "customer_1",
             "carrier-circuits": ["circ_id_1"],
             "sub-circuits": ["sub_circuit_2"],
             "customerid": "cu_1",
@@ -230,24 +284,33 @@ def test_transform_ims_data():
             "circuit-type": "service",
             "product": "PEERING R & E",
             "speed": "not fiber route",
-            "project": "Project A",
+            "project": "customer_1",
             "carrier-circuits": ["sub_circuit_1"],
             "sub-circuits": [],
             "customerid": "cu_1",
         }
     }
+
+    circuit_ids_and_sids = {
+        "sub_circuit_2": 'SID-01',
+        "circ_id_2": 'SID-02',
+        "circ_id_3": 'SID-03'
+    }
     data = {
         "locations": locations,
         "customer_contacts": customer_contacts,
-        "circuit_ids_to_monitor": [],
+        "circuit_ids_to_monitor": ["sub_circuit_2"],
         "additional_circuit_customer_ids": additional_circuit_customer_ids,
         "hierarchy": hierarchy,
         "port_id_details": port_id_details,
-        "port_id_services": port_id_services
+        "port_id_services": port_id_services,
+        "circuit_ids_sids": circuit_ids_and_sids,
+        "geant_nodes": ["eq_b"]
     }
     res = transform_ims_data(data)
     ifs = res["interface_services"]
-    assert list(ifs.keys()) == ["eq_a:if_a", "eq_b:if_b"]
+    assert list(ifs.keys()) == [
+        "eq_a:if_a", "eq_b:if_b", "eq_a:if_c", "eq_b:if_c"]
     for v in ifs.values():
         assert len(v) == 1
         assert len(v[0]["related-services"]) == 1
@@ -267,6 +330,38 @@ def test_transform_ims_data():
     assert len(v[0]["fibre-routes"]) == 1
     assert v[0]["fibre-routes"][0]["id"] == "carrier_id_3"
 
+    assert len(res["sid_services"]['SID-01']) == 2
+
+    for x in [
+        {
+            'circuit_id': "sub_circuit_2",
+            'sid': "SID-01",
+            'name': "sub_circuit_2",
+            'speed': 10 << 30,
+            'service_type': "PEERING R & E",
+            'project': "customer_1",
+            'customer': "customer_1",
+            'equipment': "eq_a",
+            'port': "if_c",
+            'geant_equipment': False
+        },
+        {
+            'circuit_id': "sub_circuit_2",
+            'sid': "SID-01",
+            'name': "sub_circuit_2",
+            'speed': 10 << 30,
+            'service_type': "PEERING R & E",
+            'project': "customer_1",
+            'customer': "customer_1",
+            'equipment': "eq_b",
+            'port': "if_c",
+            'geant_equipment': True
+        }
+    ]:
+        assert json.dumps(x, sort_keys=True) in [
+            json.dumps(
+                y, sort_keys=True) for y in res["sid_services"]['SID-01']]
+
 
 def test_persist_ims_data(mocker, data_config, mocked_redis):
 
@@ -314,7 +409,9 @@ def test_persist_ims_data(mocker, data_config, mocked_redis):
             "np1": {"id_1": "data for np1"},
             "np2": {"id_2": "data for np2"},
         },
+        "sid_services": {"SID-001": [{"k1": "data"}, {"k1": "data"}]},
         "services_by_type": {},
+        "geant_nodes": []
     }
     for k in r.keys("ims:*"):
         r.delete(k)
@@ -338,6 +435,9 @@ def test_persist_ims_data(mocker, data_config, mocked_redis):
     assert [k.decode("utf-8") for k in r.keys("poller_cache:*")] == \
            ["poller_cache:eq1", "poller_cache:eq2"]
 
+    assert json.loads(r.get("ims:sid_services").decode("utf-8")) == \
+           data["sid_services"]
+
 
 def test_retrieve_and_persist_neteng_managed_device_list(
         mocker, data_config, mocked_redis):