Skip to content
Snippets Groups Projects

Compare revisions

Changes are shown as if the source revision was being merged into the target revision. Learn more about comparing revisions.

Source

Select target project
No results found
Select Git revision

Target

Select target project
  • geant-swd/dashboardv3/inventory-provider
1 result
Select Git revision
Show changes
Commits on Source (45)
...@@ -2,6 +2,29 @@ ...@@ -2,6 +2,29 @@
All notable changes to this project will be documented in this file. All notable changes to this project will be documented in this file.
## [0.131] - 2024-09-23
- DBOARD3-1006: Added endpoint for handling additional Coriant traps
## [0.130] - 2024-09-04
- POL1-430: Add NREN region tags to dashboard info based on NREN region
- POL1-430: Add /poller/regions endpoint to get NREN regions where specified in IMS
## [0.129] - 2024-08-29
- Revert DBOARD3-1005
## [0.128] - 2024-08-22
- Small fix to DBOARD3-1005
## [0.127] - 2024-08-21
- DBOARD3-1005: Temporarily use a local snapshot of flexils data
## [0.126] - 2024-08-13
- DBOARD3-998: Skipped router during update if no data retrieved and no cached data found
- DBOARD3-997: excluded pxc ports and related lags for Nokia routers
- DBOARD3-996: added TimeoutExpiredError to handled exceptions
- DBOARD3-987: added interfaces from other stanzas for Nokia routers
- DBOARD3-958: added BGP peer extraction for Nokia routers
## [0.125] - 2024-07-18 ## [0.125] - 2024-07-18
- DBOARD3-971: Added handling for GRV 10G interfaces - DBOARD3-971: Added handling for GRV 10G interfaces
- POL1-836: Included PHY RE_INTERCONNECT in R&E Peer dashboards - POL1-836: Included PHY RE_INTERCONNECT in R&E Peer dashboards
......
...@@ -647,3 +647,15 @@ def get_router_vendors(ds: IMS): ...@@ -647,3 +647,15 @@ def get_router_vendors(ds: IMS):
ed_nav_properties): ed_nav_properties):
for r in ed.get('nodes', []): for r in ed.get('nodes', []):
yield r['name'], ed.get('vendor', {}).get('name', None) yield r['name'], ed.get('vendor', {}).get('name', None)
@log_entry_and_exit
def get_customer_regions(ds: IMS):
customer_id_map = {_c['id']: _c['name'] for _c in ds.get_all_entities('Customer', step_count=500)}
for _d in ds.get_filtered_entities(
'ExtraFieldValue',
'extrafield.id == 3329', # 3329: region field, will not change
EXTRA_FIELD_VALUE_PROPERTIES['ExtraFieldValueObjectInfo']):
_obj_id = _d['objectid']
if _obj_id in customer_id_map:
yield {'id': _obj_id, 'name': customer_id_map[_obj_id], 'region': _d['value']}
import ipaddress import ipaddress
import logging import logging
import re import re
from functools import lru_cache
from lxml import etree from lxml import etree
from ncclient import manager, xml_ from ncclient import manager, xml_
...@@ -173,6 +174,19 @@ def get_interfaces_state(state_doc): ...@@ -173,6 +174,19 @@ def get_interfaces_state(state_doc):
yield details yield details
@lru_cache
def get_pxc_ports(netconf_config):
# these ports will be ignored for the purposes of the update
pxc_ports = set()
for port in netconf_config.findall('./configure/port-xc/pxc'):
pxc_ports.add(port.find('port-id').text)
for port in netconf_config.findall('./configure/port'):
port_id = port.find('port-id').text
if port_id.startswith('pxc'):
pxc_ports.add(port_id)
return pxc_ports
def get_ports_config(netconf_config): def get_ports_config(netconf_config):
def _port_info(e): def _port_info(e):
pi = { pi = {
...@@ -193,10 +207,14 @@ def get_ports_config(netconf_config): ...@@ -193,10 +207,14 @@ def get_ports_config(netconf_config):
breakout_match.group('unit'), 'Unknown') breakout_match.group('unit'), 'Unknown')
return pi return pi
pxc_ports = get_pxc_ports(netconf_config)
# making the assumption that the breakout ports are listed directly before their # making the assumption that the breakout ports are listed directly before their
current_parent_port = None current_parent_port = None
# child ports # child ports
for port in netconf_config.findall('./configure/port'): for port in netconf_config.findall('./configure/port'):
port_id = port.find('port-id').text
if port_id in pxc_ports:
continue
port_info = _port_info(port) port_info = _port_info(port)
if 'breakout' in port_info: if 'breakout' in port_info:
current_parent_port = port_info current_parent_port = port_info
...@@ -212,6 +230,7 @@ def get_lags_config(netconf_config): ...@@ -212,6 +230,7 @@ def get_lags_config(netconf_config):
enabled_ports = { enabled_ports = {
p['port-id'] for p in get_ports_config(netconf_config) if p['admin-state'] == 'enable' p['port-id'] for p in get_ports_config(netconf_config) if p['admin-state'] == 'enable'
} }
pxc_ports = get_pxc_ports(netconf_config)
def _lag_info(e): def _lag_info(e):
_name = e.find('./lag-name').text _name = e.find('./lag-name').text
...@@ -228,6 +247,9 @@ def get_lags_config(netconf_config): ...@@ -228,6 +247,9 @@ def get_lags_config(netconf_config):
return ifc return ifc
for lag in netconf_config.findall('./configure/lag'): for lag in netconf_config.findall('./configure/lag'):
ports = {p.find('./port-id').text for p in lag.findall('./port')}
if pxc_ports > ports:
continue
yield _lag_info(lag) yield _lag_info(lag)
......
...@@ -23,10 +23,15 @@ These endpoints are intended for use by Dashboard V3. ...@@ -23,10 +23,15 @@ These endpoints are intended for use by Dashboard V3.
.. autofunction:: .. autofunction::
inventory_provider.routes.classifier.get_fiberlink_trap_metadata inventory_provider.routes.classifier.get_fiberlink_trap_metadata
/classifier/coriant-info /classifier/coriant-port-info
------------------------ -----------------------------
.. autofunction:: inventory_provider.routes.classifier.get_coriant_port_info
/classifier/coriant-tp-info
---------------------------
.. autofunction:: inventory_provider.routes.classifier.get_coriant_info .. autofunction:: inventory_provider.routes.classifier.get_coriant_tp_info
/classifier/mtc-interface-info /classifier/mtc-interface-info
...@@ -985,8 +990,31 @@ def get_tnms_fibre_trap_metadata(enms_pc_name: str) -> Response: ...@@ -985,8 +990,31 @@ def get_tnms_fibre_trap_metadata(enms_pc_name: str) -> Response:
@routes.route('/coriant-info/<equipment_name>/<path:entity_string>', @routes.route('/coriant-info/<equipment_name>/<path:entity_string>',
methods=['GET']) methods=['GET'])
@routes.route('/coriant-port-info/<equipment_name>/<path:entity_string>',
methods=['GET'])
@common.require_accepts_json
def get_coriant_port_info(equipment_name: str, entity_string: str) -> Response:
"""
Handler for /classifier/coriant-info that
returns metadata for a coriant path.
The response will be formatted according to the following schema:
.. asjson::
inventory_provider.routes.classifier_schema.CORIANT_INFO_RESPONSE_SCHEMA
:param equipment_name: grv hostname
:param entity_string: path name
:return:
"""
card_port_regex = r'^(\d+\-\d+)\.(\d+(\.\d+)*)'
return _get_coriant_info(equipment_name, entity_string, card_port_regex)
@routes.route('/coriant-tp-info/<equipment_name>/<path:entity_string>',
methods=['GET'])
@common.require_accepts_json @common.require_accepts_json
def get_coriant_info(equipment_name: str, entity_string: str) -> Response: def get_coriant_tp_info(equipment_name: str, entity_string: str) -> Response:
""" """
Handler for /classifier/coriant-info that Handler for /classifier/coriant-info that
returns metadata for a coriant path. returns metadata for a coriant path.
...@@ -996,10 +1024,16 @@ def get_coriant_info(equipment_name: str, entity_string: str) -> Response: ...@@ -996,10 +1024,16 @@ def get_coriant_info(equipment_name: str, entity_string: str) -> Response:
.. asjson:: .. asjson::
inventory_provider.routes.classifier_schema.CORIANT_INFO_RESPONSE_SCHEMA inventory_provider.routes.classifier_schema.CORIANT_INFO_RESPONSE_SCHEMA
:param source_equipment: grv hostname :param equipment_name: grv hostname
:param entity_string: path name :param entity_string: path name
:return: :return:
""" """
card_port_regex = r'^(\d+\-\d+)\.(\d+)'
return _get_coriant_info(equipment_name, entity_string, card_port_regex)
def _get_coriant_info(
equipment_name: str, entity_string: str, card_port_regex: str) -> Response:
r = common.get_current_redis() r = common.get_current_redis()
...@@ -1013,7 +1047,7 @@ def get_coriant_info(equipment_name: str, entity_string: str) -> Response: ...@@ -1013,7 +1047,7 @@ def get_coriant_info(equipment_name: str, entity_string: str) -> Response:
if not result: if not result:
m = re.match(r'^(\d+\-\d+)\.((\d+)(\.\d+)*)', ims_interface) m = re.match(card_port_regex, ims_interface)
if not m: if not m:
logger.error( logger.error(
f'invalid coriant entity string format: {ims_interface}') f'invalid coriant entity string format: {ims_interface}')
......
...@@ -118,6 +118,7 @@ def update(): ...@@ -118,6 +118,7 @@ def update():
response='an update is already in progress', response='an update is already in progress',
status=503, status=503,
mimetype="text/html") mimetype="text/html")
monitor.clear_joblog(r)
update_task_id = worker.update_entry_point.delay().get() update_task_id = worker.update_entry_point.delay().get()
r.set('classifier-cache:update-task-id', update_task_id.encode('utf-8')) r.set('classifier-cache:update-task-id', update_task_id.encode('utf-8'))
return jsonify({'task id': update_task_id}) return jsonify({'task id': update_task_id})
......
...@@ -60,6 +60,12 @@ These endpoints are intended for use by BRIAN. ...@@ -60,6 +60,12 @@ These endpoints are intended for use by BRIAN.
.. autofunction:: inventory_provider.routes.poller.error_report_interfaces .. autofunction:: inventory_provider.routes.poller.error_report_interfaces
/poller/regions
---------------
.. autofunction:: inventory_provider.routes.poller.get_nren_regions
support method: _get_dashboards support method: _get_dashboards
--------------------------------- ---------------------------------
...@@ -132,6 +138,9 @@ class BRIAN_DASHBOARDS(Enum): ...@@ -132,6 +138,9 @@ class BRIAN_DASHBOARDS(Enum):
# NREN customer # NREN customer
NREN = auto() NREN = auto()
# EAP NRENs
EAP = auto()
class PORT_TYPES(Enum): class PORT_TYPES(Enum):
ACCESS = auto() ACCESS = auto()
...@@ -438,6 +447,22 @@ STRING_LIST_SCHEMA = { ...@@ -438,6 +447,22 @@ STRING_LIST_SCHEMA = {
'items': {'type': 'string'} 'items': {'type': 'string'}
} }
NREN_REGION_LIST_SCHEMA = {
'$schema': 'https://json-schema.org/draft-07/schema#',
'definitions': {
'nren_region': {
'type': 'object',
'properties': {
'name': {'type': 'string'},
'region': {'type': 'string'}
}
}
},
'type': 'array',
'items': {'$ref': '#/definitions/nren_region'}
}
@routes.after_request @routes.after_request
def after_request(resp): def after_request(resp):
...@@ -520,7 +545,10 @@ def _get_dashboards(interface): ...@@ -520,7 +545,10 @@ def _get_dashboards(interface):
yield BRIAN_DASHBOARDS.ANA yield BRIAN_DASHBOARDS.ANA
def _get_dashboard_data(ifc, customers): def _get_dashboard_data(ifc, customers, regions=None):
if regions is None:
regions = []
def _get_interface_type(description): def _get_interface_type(description):
if re.match(r'^PHY', description): if re.match(r'^PHY', description):
return INTERFACE_TYPES.PHYSICAL return INTERFACE_TYPES.PHYSICAL
...@@ -577,6 +605,15 @@ def _get_dashboard_data(ifc, customers): ...@@ -577,6 +605,15 @@ def _get_dashboard_data(ifc, customers):
if not names: if not names:
return ifc return ifc
# add region-based dashboard names
region_names = []
for name in names:
if name in regions:
region = regions[name]
if region == 'EAP':
region_names.append(BRIAN_DASHBOARDS.EAP.name)
names.extend(region_names)
# to maintain compatability with current brian dashboard manager we will # to maintain compatability with current brian dashboard manager we will
# continue to return dashboard_info with the first customer name. We will # continue to return dashboard_info with the first customer name. We will
# also return dashboards_info (note the plural of dashboards) with up to # also return dashboards_info (note the plural of dashboards) with up to
...@@ -809,6 +846,21 @@ def _get_port_type(description): ...@@ -809,6 +846,21 @@ def _get_port_type(description):
return PORT_TYPES.UNKNOWN.name return PORT_TYPES.UNKNOWN.name
def _load_nren_regions(config, use_next_redis=False):
result = defaultdict(dict)
key_pattern = 'ims:cache:customer_regions'
if use_next_redis:
r = tasks_common.get_next_redis(config)
else:
r = tasks_common.get_current_redis(config)
cache = r.get(key_pattern)
if cache:
for id, nren in json.loads(cache.decode('utf-8')).items():
result[nren['name']] = nren['region']
return result
def load_interfaces_to_poll(config, hostname=None, no_lab=False, use_next_redis=False): def load_interfaces_to_poll(config, hostname=None, no_lab=False, use_next_redis=False):
def is_relevant(ifc): def is_relevant(ifc):
return not re.match(r"^(lt-|so-|dsc\.|fxp\d|lo\d).*", ifc["name"]) return not re.match(r"^(lt-|so-|dsc\.|fxp\d|lo\d).*", ifc["name"])
...@@ -820,6 +872,7 @@ def load_interfaces_to_poll(config, hostname=None, no_lab=False, use_next_redis= ...@@ -820,6 +872,7 @@ def load_interfaces_to_poll(config, hostname=None, no_lab=False, use_next_redis=
services_and_customers = \ services_and_customers = \
_get_services_and_customers(config, hostname, use_next_redis) _get_services_and_customers(config, hostname, use_next_redis)
snmp_indexes = common.load_snmp_indexes(config, hostname, use_next_redis) snmp_indexes = common.load_snmp_indexes(config, hostname, use_next_redis)
nren_regions = _load_nren_regions(config, use_next_redis)
def _get_populated_interfaces(all_interfaces): def _get_populated_interfaces(all_interfaces):
if use_next_redis: if use_next_redis:
...@@ -851,7 +904,7 @@ def load_interfaces_to_poll(config, hostname=None, no_lab=False, use_next_redis= ...@@ -851,7 +904,7 @@ def load_interfaces_to_poll(config, hostname=None, no_lab=False, use_next_redis=
ifc['dashboards'] = sorted([d.name for d in dashboards]) ifc['dashboards'] = sorted([d.name for d in dashboards])
ifc = _get_dashboard_data( ifc = _get_dashboard_data(
ifc, ifc_services_and_customers.get('customers', [])) ifc, ifc_services_and_customers.get('customers', []), nren_regions)
port_type = _get_port_type(ifc['description']) port_type = _get_port_type(ifc['description'])
ifc['port_type'] = port_type ifc['port_type'] = port_type
yield ifc yield ifc
...@@ -931,24 +984,30 @@ def load_error_report_interfaces( ...@@ -931,24 +984,30 @@ def load_error_report_interfaces(
netdash_equipment = get_netdash_equipment(config, use_next_redis) netdash_equipment = get_netdash_equipment(config, use_next_redis)
def filter_interface(interface: dict): def filter_interface(interface: dict):
return all( description = interface["description"].lower()
( name = interface["name"].lower()
"phy" in interface["description"].lower(), return (
"spare" not in interface["description"].lower(), "phy" in description
"non-operational" not in interface["description"].lower(), and "spare" not in description
"reserved" not in interface["description"].lower(), and "non-operational" not in description
"test" not in interface["description"].lower(), and "reserved" not in description
"dsc." not in interface["name"].lower(), and "test" not in description
"fxp" not in interface["name"].lower(), and "dsc." not in name
) and "fxp" not in name
) and not re.match(r".*\.\d+$", name)
)
def get_vendor(router: str):
return netdash_equipment.get(
router, "nokia" if router.startswith("rt0") else "juniper"
)
def transform_interface(interface: dict) -> Dict: def transform_interface(interface: dict) -> Dict:
return { return {
"router": interface["router"], "router": interface["router"],
"name": interface["name"], "name": interface["name"],
"description": interface["description"], "description": interface["description"],
"vendor": netdash_equipment.get(interface["router"]), "vendor": get_vendor(interface["router"]),
} }
return sorted( return sorted(
...@@ -1697,3 +1756,19 @@ def gws_direct_config(): ...@@ -1697,3 +1756,19 @@ def gws_direct_config():
response=page, response=page,
status=200, status=200,
mimetype="text/html") mimetype="text/html")
@routes.route('/regions', methods=['GET'])
@common.require_accepts_json
def get_nren_regions():
"""
Handler for fetching regions of NRENs, as cached by the Inventory Provider update process.
:return:
"""
config_params = current_app.config['INVENTORY_PROVIDER_CONFIG']
nren_region_dict = _load_nren_regions(config_params)
nren_regions = [{
'nren': nren,
'region': region
} for nren, region in nren_region_dict.items()]
return jsonify(nren_regions)
...@@ -7,6 +7,7 @@ import logging ...@@ -7,6 +7,7 @@ import logging
import os import os
import re import re
import ncclient.operations
import ncclient.transport.errors import ncclient.transport.errors
from celery import Task, states, chord from celery import Task, states, chord
from celery.result import AsyncResult from celery.result import AsyncResult
...@@ -26,7 +27,6 @@ from inventory_provider.tasks.common \ ...@@ -26,7 +27,6 @@ from inventory_provider.tasks.common \
import get_next_redis, get_current_redis, \ import get_next_redis, get_current_redis, \
latch_db, get_latch, set_latch, update_latch_status, \ latch_db, get_latch, set_latch, update_latch_status, \
ims_sorted_service_type_key, set_single_latch ims_sorted_service_type_key, set_single_latch
from inventory_provider.tasks import monitor
from inventory_provider import config, nokia, gap from inventory_provider import config, nokia, gap
from inventory_provider import environment from inventory_provider import environment
from inventory_provider import snmp from inventory_provider import snmp
...@@ -461,7 +461,6 @@ def update_entry_point(self): ...@@ -461,7 +461,6 @@ def update_entry_point(self):
try: try:
_erase_next_db(InventoryTask.config) _erase_next_db(InventoryTask.config)
update_latch_status(InventoryTask.config, pending=True) update_latch_status(InventoryTask.config, pending=True)
monitor.clear_joblog(get_current_redis(InventoryTask.config))
self.log_info("Starting update") self.log_info("Starting update")
routers = retrieve_and_persist_neteng_managed_device_list( routers = retrieve_and_persist_neteng_managed_device_list(
...@@ -609,6 +608,8 @@ def _reload_router_config_nokia( ...@@ -609,6 +608,8 @@ def _reload_router_config_nokia(
f'loading netconf data for {"lab " if lab else ""} {hostname}') f'loading netconf data for {"lab " if lab else ""} {hostname}')
netconf_doc, state_doc = retrieve_and_persist_config_nokia( netconf_doc, state_doc = retrieve_and_persist_config_nokia(
hostname, lab, warning_callback) hostname, lab, warning_callback)
if netconf_doc is None or state_doc is None:
return
r = get_next_redis(InventoryTask.config) r = get_next_redis(InventoryTask.config)
refresh_nokia_interface_list(hostname, netconf_doc, r, lab) refresh_nokia_interface_list(hostname, netconf_doc, r, lab)
communities = _nokia_community_strings(InventoryTask.config) communities = _nokia_community_strings(InventoryTask.config)
...@@ -644,12 +645,13 @@ def retrieve_and_persist_config_nokia( ...@@ -644,12 +645,13 @@ def retrieve_and_persist_config_nokia(
if not state_str: if not state_str:
failed_docs.append('port state') failed_docs.append('port state')
failed_keys.append(redis_state_key) failed_keys.append(redis_state_key)
update_callback(f'no cached info for {failed_keys}')
if failed_docs: if failed_docs:
raise InventoryTaskError( update_callback(f'no cached info for {failed_keys}. Ignoring this host')
logger.warning(
f'Nokia doc error with {hostname}' f'Nokia doc error with {hostname}'
f' and no cached data found for {failed_docs}' f' and no cached data found for {failed_docs}. Ignoring this host'
) )
return None, None
netconf_config = nokia.remove_xml_namespaces(etree.fromstring(netconf_str)) netconf_config = nokia.remove_xml_namespaces(etree.fromstring(netconf_str))
state = nokia.remove_xml_namespaces(etree.fromstring(state_str)) state = nokia.remove_xml_namespaces(etree.fromstring(state_str))
update_callback(f'Returning cached nokia data for {hostname}') update_callback(f'Returning cached nokia data for {hostname}')
...@@ -946,7 +948,8 @@ def retrieve_and_persist_interface_info_juniper( ...@@ -946,7 +948,8 @@ def retrieve_and_persist_interface_info_juniper(
try: try:
interface_info_str = juniper.get_interface_info_for_router(hostname, InventoryTask.config["ssh"]) interface_info_str = juniper.get_interface_info_for_router(hostname, InventoryTask.config["ssh"])
logger.info(f'interface-info rpc success from {hostname}') logger.info(f'interface-info rpc success from {hostname}')
except (ConnectionError, juniper.TimeoutError, InventoryTaskError, ncclient.transport.errors.SSHError): except (ConnectionError, juniper.TimeoutError, InventoryTaskError,
ncclient.transport.errors.SSHError, ncclient.operations.errors.TimeoutExpiredError):
msg = f'error loading interface-info data from {hostname}' msg = f'error loading interface-info data from {hostname}'
logger.exception(msg) logger.exception(msg)
update_callback(msg) update_callback(msg)
...@@ -1052,8 +1055,10 @@ def ims_task(self, use_current=False): ...@@ -1052,8 +1055,10 @@ def ims_task(self, use_current=False):
transformed_data = transform_ims_data(extracted_data) transformed_data = transform_ims_data(extracted_data)
persist_ims_data(transformed_data, use_current) persist_ims_data(transformed_data, use_current)
except Exception: except Exception as e:
errmsg = f'Error in IMS task {e}'
logger.exception('Error in IMS task:') logger.exception('Error in IMS task:')
self.log_error(errmsg)
update_latch_status(InventoryTask.config, pending=True, failure=True) update_latch_status(InventoryTask.config, pending=True, failure=True)
...@@ -1094,6 +1099,7 @@ def _extract_ims_data(ims_api_url, ims_username, ims_password, verify_ssl): ...@@ -1094,6 +1099,7 @@ def _extract_ims_data(ims_api_url, ims_username, ims_password, verify_ssl):
additional_circuit_customers = {} additional_circuit_customers = {}
flexils_data = {} flexils_data = {}
customers = {} customers = {}
customer_regions = {}
hierarchy = {} hierarchy = {}
port_id_details = defaultdict(list) port_id_details = defaultdict(list)
...@@ -1186,6 +1192,11 @@ def _extract_ims_data(ims_api_url, ims_username, ims_password, verify_ssl): ...@@ -1186,6 +1192,11 @@ def _extract_ims_data(ims_api_url, ims_username, ims_password, verify_ssl):
nonlocal customers nonlocal customers
customers = {c['id']: c for c in _ds().get_all_entities('customer')} customers = {c['id']: c for c in _ds().get_all_entities('customer')}
@log_task_entry_and_exit
def _populate_customer_regions():
nonlocal customer_regions
customer_regions = {c['id']: c for c in ims_data.get_customer_regions(ds=_ds())}
@log_task_entry_and_exit @log_task_entry_and_exit
def _populate_flexils_data(): def _populate_flexils_data():
nonlocal flexils_data nonlocal flexils_data
...@@ -1218,7 +1229,8 @@ def _extract_ims_data(ims_api_url, ims_username, ims_password, verify_ssl): ...@@ -1218,7 +1229,8 @@ def _extract_ims_data(ims_api_url, ims_username, ims_password, verify_ssl):
executor.submit(_populate_port_id_details): 'port_id_details', executor.submit(_populate_port_id_details): 'port_id_details',
executor.submit(_populate_circuit_info): 'circuit_info', executor.submit(_populate_circuit_info): 'circuit_info',
executor.submit(_populate_flexils_data): 'flexils_data', executor.submit(_populate_flexils_data): 'flexils_data',
executor.submit(_populate_customers): 'customers' executor.submit(_populate_customers): 'customers',
executor.submit(_populate_customer_regions): 'customer_regions'
} }
for future in concurrent.futures.as_completed(futures): for future in concurrent.futures.as_completed(futures):
...@@ -1243,7 +1255,8 @@ def _extract_ims_data(ims_api_url, ims_username, ims_password, verify_ssl): ...@@ -1243,7 +1255,8 @@ def _extract_ims_data(ims_api_url, ims_username, ims_password, verify_ssl):
'port_id_services': port_id_services, 'port_id_services': port_id_services,
'geant_nodes': geant_nodes, 'geant_nodes': geant_nodes,
'flexils_data': flexils_data, 'flexils_data': flexils_data,
'customers': customers 'customers': customers,
'customer_regions': customer_regions
} }
......
...@@ -2,7 +2,7 @@ from setuptools import setup, find_packages ...@@ -2,7 +2,7 @@ from setuptools import setup, find_packages
setup( setup(
name='inventory-provider', name='inventory-provider',
version="0.126", version="0.132",
author='GEANT', author='GEANT',
author_email='swd@geant.org', author_email='swd@geant.org',
description='Dashboard inventory provider', description='Dashboard inventory provider',
......
File suppressed by a .gitattributes entry or the file's encoding is unsupported.
...@@ -121,14 +121,20 @@ def test_peer_not_found(client): ...@@ -121,14 +121,20 @@ def test_peer_not_found(client):
assert response_data == {'locations': [], "contacts": []} assert response_data == {'locations': [], "contacts": []}
@pytest.mark.parametrize('equipment,entity_name,expected_card_id,expected_port', [ @pytest.mark.parametrize('equipment,entity_name,expected_card_id,expected_port,endpoint', [
('grv3.ams.nl.geant.net', '1-3.3.1-Optical-10GbE-TTP', '1/3', '3.1'), ('grv3.ams.nl.geant.net', '1-3.3.1-Optical-10GbE-TTP',
('bogus-hostname.with&special.char', '1/3', '3.1', 'coriant-port-info'),
'234-2345234.7878i234crazynamewithslash/1-2.3', '234/2345234', '7878') ('grv3.ams.nl.geant.net', '1-3.3.1-Optical-10GbE-TTP',
'1/3', '3.1', 'coriant-info'),
('grv3.ams.nl.geant.net', '1-1.3.1-100GbE-ODU4-TTP',
'1/1', '3', 'coriant-tp-info'),
('bogus-hostname.with&special.char', '234-2345234.7878i234crazynamewithslash/1-2.3',
'234/2345234', '7878', 'coriant-port-info')
]) ])
def test_coriant_info(client, equipment, entity_name, expected_card_id, expected_port): def test_coriant_info(
client, equipment, entity_name, expected_card_id, expected_port, endpoint):
rv = client.get( rv = client.get(
f'/classifier/coriant-info/{equipment}/{entity_name}', f'/classifier/{endpoint}/{equipment}/{entity_name}',
headers=DEFAULT_REQUEST_HEADERS) headers=DEFAULT_REQUEST_HEADERS)
assert rv.status_code == 200 assert rv.status_code == 200
assert rv.is_json assert rv.is_json
......
...@@ -512,6 +512,11 @@ def test_get_all_error_report_interfaces(client): ...@@ -512,6 +512,11 @@ def test_get_all_error_report_interfaces(client):
assert len(response_routers) > 1, 'there should data from be lots of routers' assert len(response_routers) > 1, 'there should data from be lots of routers'
def test_error_report_interfaces_has_no_logical_interfaces(client):
rv = client.get("/poller/error-report-interfaces", headers=DEFAULT_REQUEST_HEADERS)
assert not [i["name"] for i in rv.json if i["name"].endswith(".1")]
def test_get_single_router_error_report_interfaces(client): def test_get_single_router_error_report_interfaces(client):
rv = client.get( rv = client.get(
'/poller/error-report-interfaces/mx1.ams.nl.geant.net', '/poller/error-report-interfaces/mx1.ams.nl.geant.net',
...@@ -522,3 +527,12 @@ def test_get_single_router_error_report_interfaces(client): ...@@ -522,3 +527,12 @@ def test_get_single_router_error_report_interfaces(client):
response_data = json.loads(rv.data) response_data = json.loads(rv.data)
jsonschema.validate(response_data, poller.ERROR_REPORT_INTERFACE_LIST_SCHEMA) jsonschema.validate(response_data, poller.ERROR_REPORT_INTERFACE_LIST_SCHEMA)
assert {ifc['router'] for ifc in response_data} == {"mx1.ams.nl.geant.net"} assert {ifc['router'] for ifc in response_data} == {"mx1.ams.nl.geant.net"}
def test_get_nren_regions(client):
rv = client.get("/poller/regions", headers=DEFAULT_REQUEST_HEADERS)
assert rv.status_code == 200
assert rv.is_json
response_data = json.loads(rv.data)
jsonschema.validate(response_data, poller.NREN_REGION_LIST_SCHEMA)
assert {nren_region['region'] for nren_region in response_data} # no values should be empty strings
...@@ -3,13 +3,16 @@ import pathlib ...@@ -3,13 +3,16 @@ import pathlib
import jsonschema import jsonschema
from lxml import etree from lxml import etree
from ncclient.transport import TransportError
from inventory_provider.nokia import remove_xml_namespaces from inventory_provider.nokia import remove_xml_namespaces
from inventory_provider.tasks import common from inventory_provider.tasks import common
from inventory_provider.tasks.worker import populate_error_report_interfaces_cache, transform_ims_data, \ from inventory_provider.tasks.worker import populate_error_report_interfaces_cache, \
transform_ims_data, \
extract_ims_data, persist_ims_data, \ extract_ims_data, persist_ims_data, \
retrieve_and_persist_neteng_managed_device_list, \ retrieve_and_persist_neteng_managed_device_list, \
populate_poller_interfaces_cache, refresh_nokia_interface_list populate_poller_interfaces_cache, refresh_nokia_interface_list, \
retrieve_and_persist_config_nokia
def test_extract_ims_data(mocker): def test_extract_ims_data(mocker):
...@@ -119,6 +122,16 @@ def test_extract_ims_data(mocker): ...@@ -119,6 +122,16 @@ def test_extract_ims_data(mocker):
} }
] ]
) )
mocker.patch(
'inventory_provider.tasks.worker.ims_data.get_customer_regions',
return_value=[
{
'id': 1,
'name': 'Cust 1',
'region': 'REGION'
}
]
)
res = extract_ims_data() res = extract_ims_data()
assert res['locations'] == {'loc_a': 'LOC A', 'loc_b': 'LOC B'} assert res['locations'] == {'loc_a': 'LOC A', 'loc_b': 'LOC B'}
assert res['site_locations'] == { assert res['site_locations'] == {
...@@ -126,6 +139,7 @@ def test_extract_ims_data(mocker): ...@@ -126,6 +139,7 @@ def test_extract_ims_data(mocker):
'name': 'JEN-SPL'}} 'name': 'JEN-SPL'}}
assert res['lg_routers'] == ['lg router 1', 'lg router 2'] assert res['lg_routers'] == ['lg router 1', 'lg router 2']
assert res['customer_contacts'] == {'123': 'CON A', '456': 'CON B'} assert res['customer_contacts'] == {'123': 'CON A', '456': 'CON B'}
assert res['customer_regions'] == {1: {'id': 1, 'name': 'Cust 1', 'region': 'REGION'}}
assert res['planned_work_contacts'] == \ assert res['planned_work_contacts'] == \
{'223': 'CON PW A', '556': 'CON PW B'} {'223': 'CON PW A', '556': 'CON PW B'}
assert res['circuit_ids_to_monitor'] == [123, 456, 789] assert res['circuit_ids_to_monitor'] == [123, 456, 789]
...@@ -817,6 +831,9 @@ def test_populate_poller_interfaces_cache( ...@@ -817,6 +831,9 @@ def test_populate_poller_interfaces_cache(
"port_type": "SERVICE" "port_type": "SERVICE"
} }
] ]
nren_regions = {
'NREN A': 'REGION A'
}
for k in r.keys("lab:netconf-interfaces-hosts:*"): for k in r.keys("lab:netconf-interfaces-hosts:*"):
r.delete(k) r.delete(k)
...@@ -832,6 +849,9 @@ def test_populate_poller_interfaces_cache( ...@@ -832,6 +849,9 @@ def test_populate_poller_interfaces_cache(
mocker.patch( mocker.patch(
'inventory_provider.routes.poller._get_services_and_customers', 'inventory_provider.routes.poller._get_services_and_customers',
return_value=services_and_customers) return_value=services_and_customers)
mocker.patch(
'inventory_provider.routes.poller._load_nren_regions',
return_value=nren_regions)
mocker.patch( mocker.patch(
'inventory_provider.tasks.worker.InventoryTask.config' 'inventory_provider.tasks.worker.InventoryTask.config'
) )
...@@ -989,7 +1009,7 @@ def test_populate_error_report_interfaces_cache(mocker, data_config, mocked_redi ...@@ -989,7 +1009,7 @@ def test_populate_error_report_interfaces_cache(mocker, data_config, mocked_redi
}, },
{ {
"router": "rt0.geant.net", "router": "rt0.geant.net",
"name": "lag-1.0", "name": "lag-1",
"bundle": ["ae_c"], "bundle": ["ae_c"],
"bundle-parents": [], "bundle-parents": [],
"description": "PHY DESCRIPTION D", "description": "PHY DESCRIPTION D",
...@@ -1032,12 +1052,6 @@ def test_populate_error_report_interfaces_cache(mocker, data_config, mocked_redi ...@@ -1032,12 +1052,6 @@ def test_populate_error_report_interfaces_cache(mocker, data_config, mocked_redi
"description": "PHY DESCRIPTION B", "description": "PHY DESCRIPTION B",
"vendor": "juniper" "vendor": "juniper"
}, },
{
"router": "router_a.geant.net",
"name": "ae_a.123",
"description": "PHY DESCRIPTION C",
"vendor": "juniper"
},
{ {
"router": "router_a.geant.net", "router": "router_a.geant.net",
"name": "interface_a", "name": "interface_a",
...@@ -1048,7 +1062,7 @@ def test_populate_error_report_interfaces_cache(mocker, data_config, mocked_redi ...@@ -1048,7 +1062,7 @@ def test_populate_error_report_interfaces_cache(mocker, data_config, mocked_redi
exp_nokia_router_interfaces = [ exp_nokia_router_interfaces = [
{ {
"router": "rt0.geant.net", "router": "rt0.geant.net",
"name": "lag-1.0", "name": "lag-1",
"description": "PHY DESCRIPTION D", "description": "PHY DESCRIPTION D",
"vendor": "nokia" "vendor": "nokia"
} }
...@@ -1064,3 +1078,13 @@ def test_populate_error_report_interfaces_cache(mocker, data_config, mocked_redi ...@@ -1064,3 +1078,13 @@ def test_populate_error_report_interfaces_cache(mocker, data_config, mocked_redi
nokia_router = r.get("classifier-cache:error-report-interfaces:rt0.geant.net") nokia_router = r.get("classifier-cache:error-report-interfaces:rt0.geant.net")
assert json.loads(nokia_router) == exp_nokia_router_interfaces assert json.loads(nokia_router) == exp_nokia_router_interfaces
def test_nokia_retrieval_failure(mocker, data_config, mocked_redis):
mocker.patch('inventory_provider.tasks.worker.InventoryTask.config')
mocker.patch("inventory_provider.tasks.worker.nokia.load_docs", side_effect=TransportError("Mocked TransportError"))
r = common._get_redis(data_config)
mocker.patch('inventory_provider.tasks.worker.get_current_redis', return_value=r)
config_doc, state_doc = retrieve_and_persist_config_nokia("unknown.router.geant.net")
assert config_doc is None
assert state_doc is None