persisted sid data

inventory_provider/tasks/worker.py

@@ -829,7 +829,7 @@ def transform_ims_data(data):
     port_id_services = data['port_id_services']
     circuit_ids_and_sids = data['circuit_ids_sids']
 
-    sid_circuits = defaultdict(list)
+    sid_services = defaultdict(list)
 
     def _get_circuit_contacts(c):
         customer_ids = {c['customerid']}
@@ -998,18 +998,19 @@ def transform_ims_data(data):
                         logger.info(f'SID ({sid}) Circuit ({circ["id"]})'
                                     f' not a service. IMS')
                     else:
-                        sid_circuits[sid].append(
-                            {
-                                'circuit_id': circ['id'],
-                                'sid': sid,
-                                'name': circ['name'],
-                                'speed': circ['calculated-speed'],
-                                'service_type': circ['service_type'],
-                                'project': circ['project'],
-                                'customer': circ['customer'],
-                                'equipment': circ['equipment'],
-                                'port': circ['port']
-                            })
+                        sid_info = {
+                            'circuit_id': circ['id'],
+                            'sid': sid,
+                            'name': circ['name'],
+                            'speed': circ['calculated-speed'],
+                            'service_type': circ['service_type'],
+                            'project': circ['project'],
+                            'customer': circ['customer'],
+                            'equipment': circ['equipment'],
+                            'port': circ['port']
+                        }
+                        if sid_info not in sid_services[sid]:
+                            sid_services[sid].append(sid_info)
 
         interface_services[k].extend(circuits)
 
@@ -1018,7 +1019,7 @@ def transform_ims_data(data):
         'interface_services': interface_services,
         'services_by_type': services_by_type,
         'node_pair_services': node_pair_services,
-        'sid_circuits': sid_circuits
+        'sid_services': sid_services
     }
 
 
@@ -1029,6 +1030,7 @@ def persist_ims_data(data, use_current=False):
     interface_services = data['interface_services']
     services_by_type = data['services_by_type']
     node_pair_services = data['node_pair_services']
+    sid_services = data['sid_services']
 
     if use_current:
         r = get_current_redis(InventoryTask.config)
@@ -1044,12 +1046,15 @@ def persist_ims_data(data, use_current=False):
             'ims:gws_indirect:*',
             'ims:node_pair_services:*'
         ]:
+
+            r.delete('ims:sid_services')
             rp = r.pipeline()
             for k in r.scan_iter(key_pattern, count=1000):
                 rp.delete(k)
     else:
         r = get_next_redis(InventoryTask.config)
 
+    r.set('ims:sid_services', json.dumps(sid_services))
     rp = r.pipeline()
     for h, d in locations.items():
         rp.set(f'ims:location:{h}', json.dumps([d]))

test/test_worker.py

@@ -325,7 +325,7 @@ def test_transform_ims_data():
     assert len(v[0]["fibre-routes"]) == 1
     assert v[0]["fibre-routes"][0]["id"] == "carrier_id_3"
 
-    assert len(res["sid_circuits"]['SID-01']) == 2
+    assert len(res["sid_services"]['SID-01']) == 2
 
     for x in [
         {
@@ -353,7 +353,7 @@ def test_transform_ims_data():
     ]:
         assert json.dumps(x, sort_keys=True) in [
             json.dumps(
-                y, sort_keys=True) for y in res["sid_circuits"]['SID-01']]
+                y, sort_keys=True) for y in res["sid_services"]['SID-01']]
 
 
 def test_persist_ims_data(mocker, data_config, mocked_redis):
@@ -402,6 +402,7 @@ def test_persist_ims_data(mocker, data_config, mocked_redis):
             "np1": {"id_1": "data for np1"},
             "np2": {"id_2": "data for np2"},
         },
+        "sid_services": {"SID-001": [{"k1": "data"}, {"k1": "data"}]},
         "services_by_type": {},
     }
     for k in r.keys("ims:*"):
@@ -426,6 +427,9 @@ def test_persist_ims_data(mocker, data_config, mocked_redis):
     assert [k.decode("utf-8") for k in r.keys("poller_cache:*")] == \
            ["poller_cache:eq1", "poller_cache:eq2"]
 
+    assert json.loads(r.get("ims:sid_services").decode("utf-8")) == \
+           data["sid_services"]
+
 
 def test_retrieve_and_persist_neteng_managed_device_list(
         mocker, data_config, mocked_redis):