updated fiberlink data, route, test and test data

inventory_provider/routes/classifier.py

@@ -717,11 +717,10 @@ def get_fiberlink_trap_metadata(ne_name_str: str, object_name_str: str) \
     :return:
     """
 
-    objects = object_name_str.split('_')
-    shelves = [x.split('-')[0] for x in objects]
+    interfaces = object_name_str.split('_')
     p = r'([a-zA-Z\d]+?-(OLA|DTNX)\d+(-\d)?)'
     matches = re.findall(p, ne_name_str)
-    if len(matches) != 2 or len(shelves) != 2:
+    if len(matches) != 2 or len(interfaces) != 2:
         raise ClassifierProcessingError(
             f'unable to parse {ne_name_str} {object_name_str} '
             'into two elements')
@@ -747,64 +746,83 @@ def get_fiberlink_trap_metadata(ne_name_str: str, object_name_str: str) \
     else:
         equipment_a = matches[0][0]
         equipment_b = matches[1][0]
-        nes_a = f'{equipment_a}-{shelves[0]}'
-        nes_b = f'{equipment_b}-{shelves[1]}'
-        result = []
-        df_a = r.get(f'ims:ne_fibre_spans:{nes_a}')
-        df_b = r.get(f'ims:ne_fibre_spans:{nes_b}')
-
-        if df_a and df_b:
-            a = json.loads(df_a.decode('utf-8'))
-            b = json.loads(df_b.decode('utf-8'))
-
-            matches = [x for x in itertools.product(a, b) if
-                       x[0]['df_route_id'] == x[1]['df_route_id']]
-            if matches:
-                match = matches[0]
-                location_a = _location_from_equipment(equipment_a, r)
-                location_b = _location_from_equipment(equipment_b, r)
-                if location_a:
-                    loc_a = location_a
-                else:
-                    loc_a = _LOCATION(equipment_a, '', '')
-                if location_b:
-                    loc_b = location_b
-                else:
-                    loc_b = _LOCATION(equipment_b, '', '')
-
-                # added locations in preparation for refactoring to be in-line
-                # with other location data. Once Dashboard has been altered to
-                # use this for fiberlink alarms the 'ends' attribute can be
-                # removed
-                result = {
-                    'locations': [
-                        build_locations(loc_a, loc_b)
-                    ],
-                    'ends': {
-                        'a': {
-                            'pop': loc_a['name'],
-                            'pop_abbreviation': loc_a['abbreviation'],
-                            'equipment': loc_a['equipment']
-                        },
-                        'b': {
-                            'pop': loc_b['name'],
-                            'pop_abbreviation': loc_b['abbreviation'],
-                            'equipment': loc_b['equipment']
-                        },
+        interface_a = interfaces[0]
+        interface_b = interfaces[1]
+        circuits_a = \
+            r.get(f'ims:interface_services:{equipment_a}:{interface_a}')
+        logger.debug(f'ims:interface_services:{equipment_a}:{interface_a}')
+        circuits_b = \
+            r.get(f'ims:interface_services:{equipment_b}:{interface_b}')
+        logger.debug(f'ims:interface_services:{equipment_b}:{interface_b}')
+
+        def _get_fr(circs):
+            for c in circs:
+                h = r.get(f'ims:circuit_hierarchy:{c["id"]}')
+                if h:
+                    h = json.loads(h.decode('utf-8'))
+                    for sc in h:
+                        yield from sc['fibre-routes']
+
+        fr_a_ids = set()
+        fr_b_ids = set()
+        all_frs = {}
+        if circuits_a:
+            circuits_a = json.loads(circuits_a.decode('utf-8'))
+            for fr in _get_fr(circuits_a):
+                fr_a_ids.add(fr['id'])
+                all_frs[fr['id']] = fr
+        if circuits_b:
+            circuits_b = json.loads(circuits_b.decode('utf-8'))
+            for fr in _get_fr(circuits_b):
+                fr_b_ids.add(fr['id'])
+                all_frs[fr['id']] = fr
+
+        fr_ids = fr_a_ids & fr_b_ids
+        if not fr_ids:
+            fr_ids = fr_a_ids | fr_b_ids
+        fibre_routes = [all_frs[x] for x in fr_ids]
+
+        if fibre_routes:
+            location_a = _location_from_equipment(equipment_a, r)
+            location_b = _location_from_equipment(equipment_b, r)
+            if location_a:
+                loc_a = location_a
+            else:
+                loc_a = _LOCATION(equipment_a, '', '')
+            if location_b:
+                loc_b = location_b
+            else:
+                loc_b = _LOCATION(equipment_b, '', '')
+
+            # added locations in preparation for refactoring to be in-line
+            # with other location data. Once Dashboard has been altered to
+            # use this for fiberlink alarms the 'ends' attribute can be
+            # removed
+            result = {
+                'locations': [
+                    build_locations(loc_a, loc_b)
+                ],
+                'ends': {
+                    'a': {
+                        'pop': loc_a['name'],
+                        'pop_abbreviation': loc_a['abbreviation'],
+                        'equipment': loc_a['equipment']
                     },
-                    'df_route': {
-                        'id': match[0]['df_route_id'],
-                        'name': match[0]['df_route'],
-                        'status': match[0]['df_status'],
+                    'b': {
+                        'pop': loc_b['name'],
+                        'pop_abbreviation': loc_b['abbreviation'],
+                        'equipment': loc_b['equipment']
                     },
-                    'related-services':
-                        get_top_level_services(match[0]['df_route_id'], r)
-                }
-                for rs in result['related-services']:
-                    rs.pop('id', None)
-
-                result = json.dumps(result)
-                r.set(cache_key, result)
+                },
+                'df_route': fibre_routes[0],
+                'related-services':
+                    get_top_level_services(fibre_routes[0]['id'], r)
+            }
+            for rs in result['related-services']:
+                rs.pop('id', None)
+
+            result = json.dumps(result)
+            r.set(cache_key, result)
         if not result:
             return Response(
                 response="no available info for "

inventory_provider/tasks/worker.py

@@ -8,6 +8,7 @@ import tempfile
 import time
 from datetime import datetime
 from enum import IntFlag
+from functools import lru_cache
 from pathlib import Path
 from uuid import uuid4
 
@@ -422,7 +423,6 @@ def launch_refresh_cache_all(config):
             update_neteng_managed_device_list.apply_async(),
             update_equipment_locations.apply_async(),
             update_lg_routers.apply_async(),
-            update_fibre_spans.apply_async(),
         ]
         [x.get() for x in subtasks]
 
@@ -471,30 +471,6 @@ def internal_refresh_phase_2(self):
         raise
 
 
-@app.task(base=InventoryTask, bind=True, name='update_fibre_spans')
-@log_task_entry_and_exit
-def update_fibre_spans(self, use_current=False):
-    if use_current:
-        r = get_current_redis(InventoryTask.config)
-    else:
-        r = get_next_redis(InventoryTask.config)
-    rp = r.pipeline()
-    # scan with bigger batches, to mitigate network latency effects
-    for key in r.scan_iter('ims:ne_fibre_spans:*', count=1000):
-        rp.delete(key)
-    rp.execute()
-
-    c = InventoryTask.config["ims"]
-    ds = IMS(c['api'], c['username'], c['password'])
-
-    rp = r.pipeline()
-    for ne, fs in ims_data.get_fibre_info(ds):
-        rp.set(
-            f'ims:ne_fibre_spans:{ne}',
-            json.dumps(fs))
-    rp.execute()
-
-
 @app.task(
     base=InventoryTask, bind=True, name='update_interfaces_to_services')
 @log_task_entry_and_exit
@@ -546,6 +522,8 @@ def update_interfaces_to_services(self, use_current=False):
         s.pop('port_a_id', None)
         s.pop('port_b_id', None)
 
+    interface_services = defaultdict(list)
+
     for port_info in port_id_details.values():
         service_ids = set()
         services = []
@@ -555,16 +533,20 @@ def update_interfaces_to_services(self, use_current=False):
                 services.append(service)
                 service_ids.add(service['id'])
 
+        if_key = f'{port_info["equipment_name"]}:{port_info["interface_name"]}'
+        interface_services[if_key].extend(services)
+
+    for k, v in interface_services.items():
         rp.set(
-            f'ims:interface_services:{port_info["equipment_name"]}:'
-            f'{port_info["interface_name"]}',
-            json.dumps(services))
+            f'ims:interface_services:{k}',
+            json.dumps(v))
     rp.execute()
 
 
 @app.task(base=InventoryTask, bind=True, name='update_circuit_hierarchy')
 @log_task_entry_and_exit
 def update_circuit_hierarchy(self, use_current=False):
+    hierarchy = {}
     if use_current:
         r = get_current_redis(InventoryTask.config)
     else:
@@ -577,9 +559,28 @@ def update_circuit_hierarchy(self, use_current=False):
     c = InventoryTask.config["ims"]
     ds = IMS(c['api'], c['username'], c['password'])
 
+    hierarchy = {d['id']: d for d in ims_data.get_circuit_hierarchy(ds)}
+
+    def _get_fibre_routes(c_id):
+        _circ = hierarchy.get(c_id, None)
+        if _circ is None:
+            return
+        if _circ['speed'].lower() == 'fibre_route':
+            yield _circ['id']
+        else:
+            for cc in _circ['carrier-circuits']:
+                yield from _get_fibre_routes(cc)
+
     rp = r.pipeline()
-    for d in ims_data.get_circuit_hierarchy(ds):
-        rp.set(f'ims:circuit_hierarchy:{d["id"]}', json.dumps([d]))
+    for circ in hierarchy.values():
+        circ['fibre-routes'] = \
+            [{
+                'id': hierarchy[x]['id'],
+                'name': hierarchy[x]['name'],
+                'status': hierarchy[x]['status']
+            } for x in set(_get_fibre_routes(circ['id']))]
+        rp.set(f'ims:circuit_hierarchy:{circ["id"]}', json.dumps([circ]))
+
     rp.execute()
 
 

test/test_classifier_routes.py

@@ -4,7 +4,8 @@ import pytest
 
 from inventory_provider.routes.classifier_schema \
     import JUNIPER_LINK_RESPONSE_SCHEMA, PEER_INFO_RESPONSE_SCHEMA, \
-    INFINERA_LAMBDA_INFO_RESPONSE_SCHEMA, CORIANT_INFO_RESPONSE_SCHEMA
+    INFINERA_LAMBDA_INFO_RESPONSE_SCHEMA, CORIANT_INFO_RESPONSE_SCHEMA, \
+    INFINERA_FIBERLINK_INFO_RESPONSE_SCHEMA
 
 DEFAULT_REQUEST_HEADERS = {
     "Content-type": "application/json",
@@ -165,17 +166,17 @@ def test_infinera_fiberlink_not_found(client):
 
 
 # Temporarily removed for merge
-# def test_infinera_fiberlink(client):
-#
-#     rv = client.get(
-#         '/classifier/infinera-fiberlink-info/'
-#         'OOS-OLA1-GHE-OLA1/1-A-3-L1_1-A-2-L1',
-#         headers=DEFAULT_REQUEST_HEADERS)
-#     assert rv.status_code == 200
-#     assert rv.is_json
-#     response_data = json.loads(rv.data.decode('utf-8'))
-#     jsonschema.validate(
-#     response_data, INFINERA_FIBERLINK_INFO_RESPONSE_SCHEMA)
+def test_infinera_fiberlink(client):
+
+    rv = client.get(
+        '/classifier/infinera-fiberlink-info/'
+        'OOS-OLA1-GHE-OLA1/1-A-3-L1_1-A-2-L1',
+        headers=DEFAULT_REQUEST_HEADERS)
+    assert rv.status_code == 200
+    assert rv.is_json
+    response_data = json.loads(rv.data.decode('utf-8'))
+    jsonschema.validate(
+    response_data, INFINERA_FIBERLINK_INFO_RESPONSE_SCHEMA)
 
 
 def test_infinera_lambda(client):