From ecaeddcfd77980ad7acc66a33744c65005df950d Mon Sep 17 00:00:00 2001
From: Robert Latta <robert.latta@geant.org>
Date: Thu, 11 Mar 2021 16:14:51 +0000
Subject: [PATCH] switched to using interface_services and removed redundant
code
---
inventory_provider/routes/classifier.py | 125 +++++-------------------
inventory_provider/routes/testing.py | 12 ---
inventory_provider/tasks/worker.py | 66 -------------
3 files changed, 24 insertions(+), 179 deletions(-)
diff --git a/inventory_provider/routes/classifier.py b/inventory_provider/routes/classifier.py
index 052930df..33209c3c 100644
--- a/inventory_provider/routes/classifier.py
+++ b/inventory_provider/routes/classifier.py
@@ -5,8 +5,7 @@ import logging
import re
from copy import copy
from distutils.util import strtobool
-from functools import lru_cache
-from typing import Optional, List, Iterator
+from typing import Optional, List
from flask import Blueprint, Response, request
from redis import Redis
@@ -41,7 +40,7 @@ def _remove_duplicates_from_list(all_: list) -> list:
"""
removes duplicates from the input list
the list items must be encodable as json
- :param l:
+ :param all_:
:return: a new list with unique elements
"""
tmp_dict = dict(
@@ -75,18 +74,6 @@ def _location_from_services(services, r: Redis):
yield build_locations(loc_a, loc_b)
-@lru_cache(256, typed=False)
-def _location_from_port_id(port_id: Optional[str], r: Redis) -> Optional[dict]:
- if not port_id:
- return None
- port_info = r.get(f'ims:port_id_interface:{port_id}')
- if port_info:
- port_info = json.loads(port_info.decode('utf-8'))
- equipment_name = port_info['equipment_name']
- return _location_from_equipment(equipment_name, r)
- return None
-
-
class ClassifierRequestError(Exception):
status_code = 500
@@ -213,89 +200,26 @@ def get_related_services(source_equipment: str, interface: str, r) -> dict:
yield from get_top_level_services(s['id'], r)
-def get_top_level_services_by_port_id(port_id: str, r: Redis) -> Iterator:
- services = r.get(f'ims:port_id_services:{port_id}')
- if services:
- for s in json.loads(services.decode('utf-8')):
- yield from get_top_level_services(s['id'], r)
-
-
-def get_full_service_info(s: dict, r: Redis) -> dict:
- port_a_id = s.get('port_a_id', None)
- port_b_id = s.get('port_b_id', None)
- service = s.copy()
- service.pop('port_a_id', None)
- service.pop('port_b_id', None)
- loc_a = _location_from_port_id(port_a_id, r)
- loc_b = _location_from_port_id(port_b_id, r)
- if loc_a:
- service['pop_name'] = loc_a['name']
- service['pop_abbreviation'] = loc_a['abbreviation']
- service['equipment'] = loc_a['equipment']
- else:
- service['pop_name'] = ''
- service['pop_abbreviation'] = ''
- service['equipment'] = ''
- if loc_b:
- service['other_end_pop_name'] = loc_b['name']
- service['other_end_pop_abbreviation'] = loc_b['abbreviation']
- service['other_end_equipment'] = loc_b['equipment']
- else:
- service['other_end_pop_name'] = ''
- service['other_end_pop_abbreviation'] = ''
- service['other_end_equipment'] = ''
- return service
-
-
def get_interface_services_and_locs(ims_source_equipment, ims_interface, r):
result = {
- 'locations': [],
- 'services': [],
- 'related-services': []
+ 'locations': []
}
- services_dict = {}
- rs_dict = {}
- port_info = r.get(
- f'ims:interface_port_ids:{ims_source_equipment}:{ims_interface}')
- if port_info:
- port_info = json.loads(port_info.decode('utf-8'))
- port_id_services = r.get(
- f'ims:port_id_services:{port_info["port_id"]}')
-
- if port_id_services:
- port_id_services = json.loads(port_id_services.decode('utf-8'))
- for ps in port_id_services:
- services_dict[ps['id']] = get_full_service_info(ps, r)
- port_a_id = ps.get('port_a_id', None)
- port_b_id = ps.get('port_b_id', None)
- rs_dict.update(
- {x['id']: x for x in get_top_level_services_by_port_id(
- port_a_id, r
- )})
- rs_dict.update(
- {x['id']: x for x in get_top_level_services_by_port_id(
- port_b_id, r
- )})
- loc = build_locations(
- _location_from_port_id(port_a_id, r),
- _location_from_port_id(port_b_id, r)
- )
- if loc:
- result['locations'].append(loc)
- result['services'] = list(services_dict.values())
- result['related-services'] = list(rs_dict.values())
- for r in result['related-services']:
- r.pop('id', None)
-
- if not result.get('locations'):
+ raw_services = r.get(
+ f'ims:interface_services:{ims_source_equipment}:{ims_interface}')
+ if raw_services:
+ result['services'] = json.loads(raw_services.decode('utf-8'))
+ result['related-services'] = \
+ list(get_related_services(ims_source_equipment, ims_interface, r))
+ result['locations'] = _location_from_services(result['services'], r)
+ if not result['services']:
+ result.pop('services', None)
+ if not result['related-services']:
+ result.pop('related-services', None)
+
+ if not result.get('locations', None):
locations = build_locations(
_location_from_equipment(ims_source_equipment, r))
result['locations'] = [locations] if locations else []
- if not result['services']:
- result.pop('services', None)
- if not result['related-services']:
- result.pop('related-services', None)
-
result['locations'] = _remove_duplicates_from_list(result['locations'])
return result
@@ -386,12 +310,9 @@ def get_juniper_link_info(source_equipment: str, interface: str) -> Response:
result = result.decode('utf-8')
else:
result = {
- 'locations': []
+ 'interface': _link_interface_info(r, source_equipment, interface)
}
- result['interface'] = _link_interface_info(
- r, source_equipment, interface)
-
bundle_members = r.get(
f'netconf-interface-bundles:{source_equipment}:{interface}')
if bundle_members:
@@ -400,11 +321,13 @@ def get_juniper_link_info(source_equipment: str, interface: str) -> Response:
else:
result['interface']['bundle_members'] = []
- result.update(get_interface_services_and_locs(
- ims_source_equipment,
- ims_interface,
- r
- ))
+ result.update(
+ get_interface_services_and_locs(
+ ims_source_equipment,
+ ims_interface,
+ r
+ )
+ )
result = json.dumps(result)
# cache this data for the next call
diff --git a/inventory_provider/routes/testing.py b/inventory_provider/routes/testing.py
index 1ceefe00..d6142eab 100644
--- a/inventory_provider/routes/testing.py
+++ b/inventory_provider/routes/testing.py
@@ -33,18 +33,6 @@ def update_fibre_spans_ims():
return Response('OK')
-@routes.route("update-interfaces-to-port-ids", methods=['GET', 'POST'])
-def update_interfaces_to_port_id_ims():
- worker.update_interfaces_to_port_ids.delay(use_current=True)
- return Response('OK')
-
-
-@routes.route("update-port-ids-to-services", methods=['GET', 'POST'])
-def update_port_ids_to_services_ims():
- worker.update_port_ids_to_services.delay(use_current=True)
- return Response('OK')
-
-
@routes.route("update-circuit-hierarchy", methods=['GET', 'POST'])
def update_circuit_hierarchy_ims():
worker.update_circuit_hierarchy.delay(use_current=True)
diff --git a/inventory_provider/tasks/worker.py b/inventory_provider/tasks/worker.py
index 84e39ae4..bf02682e 100644
--- a/inventory_provider/tasks/worker.py
+++ b/inventory_provider/tasks/worker.py
@@ -451,8 +451,6 @@ def internal_refresh_phase_2(self):
subtasks = [
update_circuit_hierarchy.apply_async(),
update_interfaces_to_services.apply_asynch(),
- update_interfaces_to_port_ids.apply_async(),
- update_port_ids_to_services.apply_async(),
import_unmanaged_interfaces.apply_async()
]
@@ -501,39 +499,6 @@ def update_fibre_spans(self, use_current=False):
rp.execute()
-@app.task(
- base=InventoryTask, bind=True, name='update_interfaces_to_port_ids')
-@log_task_entry_and_exit
-def update_interfaces_to_port_ids(self, use_current=False):
- if use_current:
- r = get_current_redis(InventoryTask.config)
- else:
- r = get_next_redis(InventoryTask.config)
-
- rp = r.pipeline()
- # scan with bigger batches, to mitigate network latency effects
- for key in r.scan_iter('ims:interface_port_ids:*', count=2000):
- rp.delete(key)
- for key in r.scan_iter('ims:port_id_interface:*', count=2000):
- rp.delete(key)
- rp.execute()
-
- c = InventoryTask.config["ims"]
- ds = IMS(c['api'], c['username'], c['password'])
-
- rp = r.pipeline()
- for pd in ims_data.get_port_details(ds):
- d = json.dumps(pd)
- rp.set(
- f'ims:interface_port_ids:{pd["equipment_name"]}'
- f':{pd["interface_name"]}',
- d)
- rp.set(
- f'ims:port_id_interface:{pd["port_id"]}',
- d)
- rp.execute()
-
-
@app.task(
base=InventoryTask, bind=True, name='update_interfaces_to_services')
@log_task_entry_and_exit
@@ -601,37 +566,6 @@ def update_interfaces_to_services(self, use_current=False):
rp.execute()
-@app.task(
- base=InventoryTask, bind=True, name='update_port_ids_to_services')
-@log_task_entry_and_exit
-def update_port_ids_to_services(self, use_current=False):
- interface_services = defaultdict(list)
-
- c = InventoryTask.config["ims"]
- ds = IMS(c['api'], c['username'], c['password'])
-
- for service in ims_data.get_port_id_services(ds):
- interface_services[service["port_a_id"]].append(service)
-
- if use_current:
- r = get_current_redis(InventoryTask.config)
- else:
- r = get_next_redis(InventoryTask.config)
-
- rp = r.pipeline()
- # scan with bigger batches, to mitigate network latency effects
- for key in r.scan_iter('ims:port_id_services:*', count=2000):
- rp.delete(key)
- rp.execute()
-
- rp = r.pipeline()
- for port_id, services in interface_services.items():
- rp.set(
- f'ims:port_id_services:{port_id}',
- json.dumps(services))
- rp.execute()
-
-
@app.task(base=InventoryTask, bind=True, name='update_circuit_hierarchy')
@log_task_entry_and_exit
def update_circuit_hierarchy(self, use_current=False):
--
GitLab