Skip to content
Snippets Groups Projects
Commit 84482533 authored by Robert Latta's avatar Robert Latta
Browse files

refactored service resolution

parent e937cc4a
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
inventory_provider/db/ims.py
+ 21
− 0
View file @ 84482533
......@@ -93,6 +93,27 @@ VENDOR_RELATED_CONTACT_PROPERTIES = {
NO_FILTERED_RESULTS_MESSAGE = 'no records found for entity:'
# this is here as it seems a logical place to store it
IMS_SERVICE_NAMES = {
'ETHERNET',
'GEANT LAMBDA',
'GEANT PLUS',
'GEANT IP',
'L3-VPN',
'IP PEERING - NON R&E (PUBLIC)',
'IP PEERING - NON R&E (PRIVATE)',
'IP PEERING - R&E',
'IP TRUNK',
'IP ACCESS',
'GEANT CLOUD PEERING',
'GEANT OPEN PORT',
'GEANT OPEN CROSS CONNECT',
'POP LAN LINK',
'SERVER LINK',
'GEANT - GBS',
'GEANT PEERING',
}
class InventoryStatus(Enum):
PLANNED = 1
......
inventory_provider/db/ims_data.py
+ 70
− 58
View file @ 84482533
import logging
import re
from collections import OrderedDict, defaultdict
from itertools import chain
from inventory_provider import environment
from inventory_provider.db import ims
from inventory_provider.db.ims import InventoryStatus, IMS
from inventory_provider.db.ims import InventoryStatus, IMS, IMS_SERVICE_NAMES
environment.setup_logging()
logger = logging.getLogger(__name__)
......@@ -74,48 +75,31 @@ def get_fibre_info(ds: IMS):
yield from by_ne.items()
def get_interface_services(ds: IMS):
def get_port_id_services(ds: IMS):
# get subset of circuits -
# start with just the ams fra lag, then expand to
# include the relevant carrier circuits
# service types come from the 'product' information
service_types = {
'ETHERNET',
'GEANT LAMBDA',
'GEANT PLUS',
'GEANT IP',
'L3-VPN',
'IP PEERING - NON R&E (PUBLIC)',
'IP PEERING - NON R&E (PRIVATE)',
'IP PEERING - R&E',
'IP TRUNK',
'IP ACCESS',
'GEANT CLOUD PEERING',
'GEANT OPEN PORT',
'GEANT OPEN CROSS CONNECT',
'POP LAN LINK',
'SERVER LINK',
'GEANT - GBS',
}
speeds = {
'L3VPN',
'LAG',
'FIBRE_ROUTE'
'FIBRE_ROUTE',
# 'BGP',
}
circuit_nav_props = [
ims.CIRCUIT_PROPERTIES['Speed'],
ims.CIRCUIT_PROPERTIES['Product'],
ims.CIRCUIT_PROPERTIES['PortsFullDetails'],
ims.CIRCUIT_PROPERTIES['InternalPortsFullDetails'],
ims.CIRCUIT_PROPERTIES['Ports'],
ims.CIRCUIT_PROPERTIES['InternalPorts'],
]
def _get_circuits():
for st in service_types:
for st in IMS_SERVICE_NAMES:
for c in ds.get_filtered_entities(
'Circuit',
f'product.name == "{st}"',
circuit_nav_props,
step_count=1000):
step_count=2000):
c['circuit_type'] = 'service'
yield c
......@@ -124,42 +108,31 @@ def get_interface_services(ds: IMS):
'Circuit',
f'speed.name == "{spd}"',
circuit_nav_props,
step_count=1000):
step_count=2000):
c['circuit_type'] = 'circuit'
yield c
circuits = _get_circuits()
def _populate_end_info(_circuit, _port_info, vendor):
port_count = len(_port_info)
if _port_info:
port = _port_info[0]
_circuit['equipment'] = port['node']['name']
_circuit['interface_name'] = interface_generators.get(
vendor.lower(), lambda x: x['name'])(port)
_circuit['other_end_equipment'] = ''
_circuit['other_end_interface_name'] = ''
if port_count > 1:
port = _port_info[1]
_circuit['other_end_equipment'] = port['node']['name']
_circuit['other_end_interface_name'] = \
interface_generators.get(
vendor.lower(),
lambda x: x['name'])(port)
_circuit['other_end_location'] = port['site']['name']
if port_count > 2:
logger.error(
'More that two (internal)ports found for'
f' {_circuit["name"]} ({_circuit["id"]})')
# order of preference
# internalports (first and last sequencenumbers)
# ports (first and last sequencenumbers)
# internal port a / internal port b
# port a / port b
def _populate_end_info(_circuit, _port_ids):
port_ids = [p for p in _port_ids if p]
if not port_ids:
return []
port_a_id = port_ids[0]
port_b_id = port_ids[-1]
_circuit['port_a_id'] = port_a_id
if port_a_id != port_b_id:
_circuit['port_b_id'] = port_b_id
yield _circuit
if port_count > 1:
_circuit['equipment'], _circuit['other_end_equipment'] = \
_circuit['other_end_equipment'], _circuit['equipment']
_circuit['interface_name'], \
_circuit['other_end_interface_name'] = \
_circuit['other_end_interface_name'], \
_circuit['interface_name']
yield _circuit
_circuit['port_a_id'], _circuit['port_b_id'] = \
_circuit['port_b_id'], _circuit['port_a_id']
yield _circuit
for circuit in circuits:
cd = {
......@@ -171,9 +144,48 @@ def get_interface_services(ds: IMS):
'service_type': circuit['product']['name'],
'project': circuit['product']['name']
}
yield from _populate_end_info(cd, circuit['ports'], circuit['vendor'])
yield from _populate_end_info(
cd, circuit['internalports'], circuit['vendor'])
ports = []
if circuit['internalports']:
tmp_ports = sorted(
circuit['internalports'], key=lambda x: x['sequencenumber'])
ports = [tmp_ports[0]['id'], tmp_ports[-1]['id']]
elif circuit['ports']:
tmp_ports = sorted(
circuit['ports'], key=lambda x: x['sequencenumber'])
ports = [tmp_ports[0]['id'], tmp_ports[-1]['id']]
elif circuit['internalportaid'] or circuit['internalportbid']:
ports = [circuit['internalportaid'], circuit['internalportbid']]
elif circuit['portaid'] or circuit['portbid']:
ports = [circuit['portaid'], circuit['portbid']]
yield from _populate_end_info(cd, ports)
def get_port_details(ds: IMS):
port_nav_props = [
ims.PORT_PROPERTIES['Node'],
ims.PORT_PROPERTIES['Shelf']
]
internal_port_nav_props = {
ims.INTERNAL_PORT_PROPERTIES['Node'],
ims.PORT_PROPERTIES['Shelf']
}
for p in chain(
ds.get_all_entities('port', port_nav_props, step_count=2000),
ds.get_all_entities(
'internalport', internal_port_nav_props, step_count=2000)
):
vendor = p['node']['equipmentdefinition']['name']
interface_name = \
interface_generators.get(
vendor.lower(),
lambda x: x['name']
)(p)
data = {
'port_id': p['id'],
'equipment_name': p['node']['name'],
'interface_name': interface_name
}
yield data
def get_circuit_hierarchy(ds: IMS):
......
inventory_provider/routes/ims_classifier.py
+ 99
− 19
View file @ 84482533
......@@ -3,9 +3,11 @@ import itertools
import json
import logging
import re
from functools import lru_cache
from flask import Blueprint, Response, current_app
from inventory_provider.db.ims import IMS_SERVICE_NAMES
from inventory_provider.routes import common
routes = Blueprint("ims-inventory-data-classifier-support-routes", __name__)
......@@ -41,10 +43,7 @@ def _remove_duplicates_from_list(all):
return list(tmp_dict.values())
# once the switchover is done then will refactor to get rid of
# _locations_from_router
def _location_from_equipment(equipment_name, r):
# return _locations_from_router(equipment_name, r)
result = r.get(f'ims:location:{equipment_name}')
if not result:
logger.error(f'error looking up location for {equipment_name}')
......@@ -69,6 +68,18 @@ def _location_from_services(services, r):
yield build_locations(loc_a, loc_b)
@lru_cache(256, typed=False)
def _location_from_port_id(port_id, r):
if not port_id:
return None
port_info = r.get(f'ims:port_id_interface:{port_id}')
if port_info:
port_info = json.loads(port_info.decode('utf-8'))
equipment_name = port_info['equipment_name']
return _location_from_equipment(equipment_name, r)
return None
class ClassifierRequestError(Exception):
status_code = 500
......@@ -122,34 +133,47 @@ def related_interfaces(hostname, interface):
def get_top_level_services(circuit_id, r):
tls = {}
key = "ims:circuit_hierarchy:{}".format(circuit_id)
key = f'ims:circuit_hierarchy:{circuit_id}'
results = r.get(key)
if results:
results = json.loads(results.decode('utf-8'))
# should only ever be one, may refactor this
for c in results:
if c['sub-circuits']:
if c['product'] in IMS_SERVICE_NAMES:
circuit_type = 'service'
else:
circuit_type = 'circuit'
if c['sub-circuits'] and circuit_type == 'circuit':
for sub in c['sub-circuits']:
temp_parents = \
get_top_level_services(sub, r)
tls.update({t['id']: t for t in temp_parents})
else:
elif circuit_type == 'service':
tls[c['id']] = {
'id': c['id'],
'name': c['name'],
'status': c['status'],
'circuit_type': c['product'].lower(),
'circuit_type': circuit_type,
'project': c['project']
}
return list(tls.values())
def get_related_services(source_equipment, interface, r):
"""
This not only finds the top-level-services for the given interface
but also gets the top-level-services for the related interfaces
e.g. ae20 will also find services on all logical units of ae20 (ae20.1 ...)
:param source_equipment:
:param interface:
:param r:
:return:
"""
ims_source_equipment = get_ims_equipment_name(source_equipment)
ims_interface = get_ims_interface(interface)
if_services = r.get(f'ims:interface_services:{ims_source_equipment}:'
f'{ims_interface}')
f'{ims_interface}')
if if_services:
for s in json.loads(if_services.decode('utf-8')):
yield from get_top_level_services(s['id'], r)
......@@ -163,6 +187,40 @@ def get_related_services(source_equipment, interface, r):
yield from get_top_level_services(s['id'], r)
def get_top_level_services_by_port_id(port_id, r):
services = r.get(f'ims:port_id_services:{port_id}')
if services:
for s in json.loads(services.decode('utf-8')):
yield from get_top_level_services(s['id'], r)
def get_full_service_info(s, r):
port_a_id = s.get('port_a_id', None)
port_b_id = s.get('port_b_id', None)
service = s.copy()
service.pop('port_a_id', None)
service.pop('port_b_id', None)
loc_a = _location_from_port_id(port_a_id, r)
loc_b = _location_from_port_id(port_b_id, r)
if loc_a:
service['pop_name'] = loc_a['name']
service['pop_abbreviation'] = loc_a['abbreviation']
service['equipment'] = loc_a['equipment']
else:
service['pop_name'] = ''
service['pop_abbreviation'] = ''
service['equipment'] = ''
if loc_b:
service['other_end_pop_name'] = loc_b['name']
service['other_end_pop_abbreviation'] = loc_b['abbreviation']
service['other_end_equipment'] = loc_b['equipment']
else:
service['other_end_pop_name'] = ''
service['other_end_pop_abbreviation'] = ''
service['other_end_equipment'] = ''
return service
@routes.route("/juniper-link-info/<source_equipment>/<path:interface>",
methods=['GET', 'POST'])
@common.require_accepts_json
......@@ -207,16 +265,38 @@ def get_juniper_link_info(source_equipment: str, interface: str):
else:
result['interface']['bundle_members'] = []
# use a dict to get rid of duplicates
rs_dict = {r['id']: r for r in get_related_services(
source_equipment, interface, r)}
result['related-services'] = list(rs_dict.values())
if_services = r.get(f'ims:interface_services:{ims_source_equipment}:'
f'{ims_interface}')
if if_services:
if_services = json.loads(if_services.decode('utf-8'))
result['locations'] = list(_location_from_services(if_services, r))
result['locations'] = []
services_dict = {}
rs_dict = {}
port_info = r.get(
f'ims:interface_port_ids:{ims_source_equipment}:{ims_interface}')
if port_info:
port_info = json.loads(port_info.decode('utf-8'))
port_id_services = r.get(
f'ims:port_id_services:{port_info["port_id"]}')
if port_id_services:
port_id_services = json.loads(port_id_services.decode('utf-8'))
for ps in port_id_services:
services_dict[ps['id']] = get_full_service_info(ps, r)
port_a_id = ps.get('port_a_id', None)
port_b_id = ps.get('port_b_id', None)
rs_dict.update(
{x['id']: x for x in get_top_level_services_by_port_id(
port_a_id, r
)})
rs_dict.update(
{x['id']: x for x in get_top_level_services_by_port_id(
port_b_id, r
)})
loc = build_locations(
_location_from_port_id(port_a_id, r),
_location_from_port_id(port_b_id, r)
)
if loc:
result['locations'].append(loc)
result['services'] = list(services_dict.values())
result['related-services'] = list(rs_dict.values())
if not result.get('locations'):
result['locations'] = [
......@@ -251,7 +331,7 @@ def ix_peering_info(peer_info):
address = ipaddress.ip_address(peer_info['name'])
except ValueError:
raise ClassifierProcessingError(
f'unable to parse {address} as an ip address')
f'unable to parse {peer_info["name"]} as an ip address')
description = peer_info['description']
assert description is not None # sanity
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment