From 8cf6b933d498eee73452d3175d43085634f66490 Mon Sep 17 00:00:00 2001
From: Robert Latta <robert.latta@geant.org>
Date: Tue, 23 Apr 2024 07:31:26 +0000
Subject: [PATCH] Feature/dboard3 920 populate snmp interface info
---
inventory_provider/config.py | 17 +--
inventory_provider/nokia.py | 144 ++++++++++++++++++++++--
inventory_provider/routes/classifier.py | 5 +-
inventory_provider/tasks/worker.py | 116 +++++++++++++++----
test/test_nokia.py | 52 +++++++--
5 files changed, 279 insertions(+), 55 deletions(-)
diff --git a/inventory_provider/config.py b/inventory_provider/config.py
index 8e11f6e5..e1007de5 100644
--- a/inventory_provider/config.py
+++ b/inventory_provider/config.py
@@ -10,17 +10,6 @@ CONFIG_SCHEMA = {
'maximum': 60, # sanity
'exclusiveMinimum': 0
},
- 'database-credentials': {
- 'type': 'object',
- 'properties': {
- 'hostname': {'type': 'string'},
- 'dbname': {'type': 'string'},
- 'username': {'type': 'string'},
- 'password': {'type': 'string'}
- },
- 'required': ['hostname', 'dbname', 'username', 'password'],
- 'additionalProperties': False
- },
'ssh-credentials': {
'type': 'object',
'properties': {
@@ -212,7 +201,6 @@ CONFIG_SCHEMA = {
'type': 'object',
'properties': {
- 'ops-db': {'$ref': '#/definitions/database-credentials'},
'ssh': {'$ref': '#/definitions/ssh-credentials'},
'nokia-ssh': {'$ref': '#/definitions/nokia-ssh-credentials'},
'redis': {'$ref': '#/definitions/redis-credentials'},
@@ -234,7 +222,10 @@ CONFIG_SCHEMA = {
'items': {'$ref': '#/definitions/interface-address'}
},
'gws-direct': {'$ref': '#/definitions/gws-direct'},
- 'nren-asn-map': {'$ref': '#/definitions/nren-asn-map'}
+ 'nren-asn-map': {'$ref': '#/definitions/nren-asn-map'},
+ 'nokia-community-inventory-provider': {'type': 'string'},
+ 'nokia-community-dashboard': {'type': 'string'},
+ 'nokia-community-brian': {'type': 'string'},
},
'oneOf': [
diff --git a/inventory_provider/nokia.py b/inventory_provider/nokia.py
index a19a47c7..640adaba 100644
--- a/inventory_provider/nokia.py
+++ b/inventory_provider/nokia.py
@@ -1,7 +1,7 @@
import logging
import re
-from ncclient import manager
+from ncclient import manager, xml_
logger = logging.getLogger(__name__)
@@ -9,9 +9,11 @@ logger = logging.getLogger(__name__)
ROOT_NS = 'urn:ietf:params:xml:ns:netconf:base:1.0'
NOKIA_NS = 'urn:nokia.com:sros:ns:yang:sr:conf'
+NOKIA_STATE_NS = 'urn:nokia.com:sros:ns:yang:sr:state'
NS = {
'r': ROOT_NS,
'n': NOKIA_NS,
+ 's': NOKIA_STATE_NS,
}
BREAKOUT_PATTERN = re.compile(
@@ -34,19 +36,137 @@ SPEED_UNITS = {
'g': 'Gbps',
'G': 'Gbps',
}
+STATE_FILTER = '''<filter>
+ <nokia-state:state xmlns:nokia-state="urn:nokia.com:sros:ns:yang:sr:state">
+ <nokia-state:port>
+ <nokia-state:port-id/>
+ <nokia-state:down-reason/>
+ <nokia-state:oper-state/>
+ <nokia-state:port-state/>
+ <nokia-state:type/>
+ <nokia-state:if-index/>
+ <nokia-state:ethernet>
+ <nokia-state:oper-speed/>
+ </nokia-state:ethernet>
+ </nokia-state:port>
+ <nokia-state:lag>
+ <nokia-state:lag-name/>
+ <nokia-state:oper-state/>
+ <nokia-state:if-index/>
+ <nokia-state:number-port-up/>
+ <nokia-state:port>
+ <nokia-state:port-id/>
+ </nokia-state:port>
+ </nokia-state:lag>
+ <nokia-state:router>
+ <nokia-state:interface>
+ <nokia-state:interface-name/>
+ <nokia-state:if-index/>
+ <nokia-state:oper-state/>
+ <nokia-state:protocol/>
+ <nokia-state:ipv4>
+ <nokia-state:oper-state/>
+ <nokia-state:down-reason/>
+ <nokia-state:primary>
+ <nokia-state:oper-address/>
+ </nokia-state:primary>
+ <nokia-state:secondary>
+ <nokia-state:address/>
+ <nokia-state:oper-address/>
+ </nokia-state:secondary>
+ </nokia-state:ipv4>
+ <nokia-state:ipv6>
+ <nokia-state:oper-state/>
+ <nokia-state:down-reason/>
+ <nokia-state:address>
+ <nokia-state:ipv6-address/>
+ <nokia-state:address-state/>
+ <nokia-state:oper-address/>
+ <nokia-state:primary-preferred/>
+ </nokia-state:address>
+ </nokia-state:ipv6>
+ </nokia-state:interface>
+ </nokia-state:router>
+ </nokia-state:state>
+</filter>'''
-def load_config(hostname, ssh_params, hostkey_verify=False):
+def load_docs(hostname, ssh_params, hostkey_verify=False):
logger.info(f'capturing netconf data for "{hostname}"')
- with manager.connect(
- host=hostname,
- hostkey_verify=hostkey_verify,
- **ssh_params
- ) as m:
- return m.get_config(source='running').data
+ params = {
+ 'host': hostname,
+ 'hostkey_verify': hostkey_verify,
+ 'device_params': {'name': 'sros'},
+ 'nc_params': {'capabilities': ['urn:nokia.com:nc:pysros:pc']},
+ 'timeout': 60
+ }
+ params.update(ssh_params)
+ with manager.connect(**params) as m:
+ # when adding the device_params to the connect call, the object returned from
+ # manager.get() and manager.get_config is a ncclient.xml_.NCElement object,
+ # which does not provide the data property this is a workaround to get the data
+ # property. The alternative would be to access the private _NCElement__result
+ # which is of type ncclient.operations.retrieve.GetReply and has a data property
+ running = xml_.to_ele(m.get_config(source='running').data_xml).getchildren()[0]
+ state = xml_.to_ele(m.get(filter=STATE_FILTER).data_xml).getchildren()[0]
+ return running, state
-def get_ports(netconf_config):
+def get_ports_state(state_doc):
+ def _port_info(e):
+ pi = {
+ 'port-id': e.find('s:port-id', namespaces=NS).text,
+ 'oper-state': e.find('s:oper-state', namespaces=NS).text,
+ 'port-state': e.find('s:port-state', namespaces=NS).text,
+ 'type': e.find('s:type', namespaces=NS).text,
+ 'if-index': e.find('s:if-index', namespaces=NS).text,
+ }
+ down_reason = e.find('s:down-reason', namespaces=NS)
+ if down_reason is not None:
+ pi['down-reason'] = down_reason.text
+ return pi
+ ports = state_doc.findall('./s:state/s:port', namespaces=NS)
+ for port in ports:
+ yield _port_info(port)
+
+
+def get_lags_state(state_doc):
+ def _lag_info(e):
+ _name = e.find('s:lag-name', namespaces=NS).text
+ _state = e.find('s:oper-state', namespaces=NS).text
+ _if_index = e.find('s:if-index', namespaces=NS).text
+ _number_port_up = e.find('s:number-port-up', namespaces=NS).text
+ ports = [p.find('s:port-id', namespaces=NS).text for p in e.findall('s:port', namespaces=NS)]
+ return {
+ 'name': _name,
+ 'oper-state': _state,
+ 'if-index': _if_index,
+ 'number-port-up': _number_port_up,
+ 'ports': ports,
+ }
+ lags = state_doc.findall('./s:state/s:lag', namespaces=NS)
+ for lag in lags:
+ yield _lag_info(lag)
+
+
+def get_interfaces_state(state_doc):
+ interfaces = state_doc.findall('./s:state/s:router/s:interface', namespaces=NS)
+ for interface_ in interfaces:
+ details = {
+ "interface-name": interface_.find('s:interface-name', namespaces=NS).text,
+ "if-index": interface_.find('s:if-index', namespaces=NS).text,
+ "oper-state": interface_.find('s:oper-state', namespaces=NS).text,
+ }
+ ipv4 = interface_.find('s:ipv4', namespaces=NS)
+ if ipv4 is not None:
+ details['ipv4-state'] = ipv4.find('s:oper-state', namespaces=NS).text
+ ipv6 = interface_.find('s:ipv6', namespaces=NS)
+ if ipv6 is not None:
+ details['ipv6-state'] = ipv6.find('s:oper-state', namespaces=NS).text
+ yield details
+
+
+def get_ports_config(netconf_config):
def _port_info(e):
pi = {
'port-id': e.find('n:port-id', namespaces=NS).text,
@@ -81,7 +201,7 @@ def get_ports(netconf_config):
yield port_info
-def get_lags(netconf_config):
+def get_lags_config(netconf_config):
def _lag_info(e):
_name = e.find('./n:lag-name', namespaces=NS).text
@@ -97,7 +217,7 @@ def get_lags(netconf_config):
'description': (
description_e.text if description_e is not None else ''
),
- 'admin-status': admin_state,
+ 'admin-state': admin_state,
'ports': ports,
}
return ifc
@@ -107,7 +227,7 @@ def get_lags(netconf_config):
yield _lag_info(lag)
-def interface_info(netconf_config):
+def get_interfaces_config(netconf_config):
def _get_ip_address(e):
for details_parent in e:
diff --git a/inventory_provider/routes/classifier.py b/inventory_provider/routes/classifier.py
index bb051c5f..5266bf84 100644
--- a/inventory_provider/routes/classifier.py
+++ b/inventory_provider/routes/classifier.py
@@ -248,9 +248,12 @@ def _link_interface_info(r, hostname, interface):
if snmp_info:
snmp_info = json.loads(snmp_info.decode('utf-8'))
ifc_info['snmp'] = {
- 'community': snmp_info['community'],
'index': snmp_info['index']
}
+ if 'communities' in snmp_info:
+ ifc_info['snmp']['community'] = snmp_info['communities']['dashboard']
+ else:
+ ifc_info['snmp']['community'] = snmp_info['community']
return ifc_info
diff --git a/inventory_provider/tasks/worker.py b/inventory_provider/tasks/worker.py
index 627985b1..c1800b4f 100644
--- a/inventory_provider/tasks/worker.py
+++ b/inventory_provider/tasks/worker.py
@@ -103,6 +103,22 @@ def _unmanaged_interfaces():
InventoryTask.config.get('unmanaged-interfaces', []))
+def _nokia_community_strings(config_):
+ return {
+ 'inventory-provider': config_['nokia-community-inventory-provider'],
+ 'dashboard': config_['nokia-community-dashboard'],
+ 'brian': config_['nokia-community-brian']
+ }
+
+
+def _general_community_strings(community):
+ return {
+ 'inventory-provider': community,
+ 'dashboard': community,
+ 'brian': community
+ }
+
+
@log_task_entry_and_exit
def refresh_juniper_bgp_peers(hostname, netconf):
host_peerings = list(juniper.all_bgp_peers(netconf))
@@ -615,42 +631,96 @@ def _reload_router_config_nokia(
warning_callback=lambda s: None):
info_callback(
f'loading netconf data for {"lab " if lab else ""} {hostname}')
- netconf_doc = retrieve_and_persist_netconf_config_nokia(
+ netconf_doc, state_doc = retrieve_and_persist_config_nokia(
hostname, lab, warning_callback)
r = get_next_redis(InventoryTask.config)
refresh_nokia_interface_list(hostname, netconf_doc, r, lab)
+ communities = _nokia_community_strings(InventoryTask.config)
+ snmp_refresh_interfaces_nokia(hostname, state_doc, communities, r, info_callback)
-def retrieve_and_persist_netconf_config_nokia(
- hostname, lab=False, update_callback=lambda s: None):
- redis_key = f'netconf-nokia:{hostname}'
+def retrieve_and_persist_config_nokia(
+ hostname, lab=False, update_callback=lambda s: None
+):
+ redis_netconf_key = f'netconf-nokia:{hostname}'
+ redis_state_key = f'nokia-state:{hostname}'
if lab:
- redis_key = f'lab:{redis_key}'
+ redis_netconf_key = f'lab:{redis_netconf_key}'
+ redis_state_key = f'lab:{redis_state_key}'
try:
- netconf_config = nokia.load_config(
- hostname, InventoryTask.config["nokia-ssh"])
- netconf_str = etree.tostring(netconf_config)
+ netconf_config, state = nokia.load_docs(
+ hostname, InventoryTask.config['nokia-ssh']
+ )
except (ConnectionError, TransportError) as e:
- msg = f'error loading netconf data from {hostname}'
+ msg = f'error loading nokia data from {hostname}'
logger.exception(e)
logger.exception(msg)
update_callback(msg)
r = get_current_redis(InventoryTask.config)
-
- netconf_str = r.get(redis_key)
+ netconf_str = r.get(redis_netconf_key)
+ state_str = r.get(redis_state_key)
+ failed_docs = []
+ failed_keys = []
if not netconf_str:
- update_callback(f'no cached netconf for {redis_key}')
+ failed_docs.append('netconf')
+ failed_keys.append(redis_netconf_key)
+ if not state_str:
+ failed_docs.append('port state')
+ failed_keys.append(redis_state_key)
+ update_callback(f'no cached info for {failed_keys}')
+ if failed_docs:
raise InventoryTaskError(
- f'netconf error with {hostname}'
- f' and no cached netconf data found')
+ f'Nokia doc error with {hostname}'
+ f' and no cached data found for {failed_docs}'
+ )
netconf_config = etree.fromstring(netconf_str)
- update_callback(f'Returning cached netconf data for {hostname}')
+ state = etree.fromstring(state_str)
+ update_callback(f'Returning cached nokia data for {hostname}')
+ else:
+ netconf_str = etree.tostring(netconf_config)
+ state_str = etree.tostring(state)
- r = get_next_redis(InventoryTask.config)
- r.set(redis_key, netconf_str)
- logger.info(f'netconf info loaded from {hostname}')
- return netconf_config
+ rw = get_next_redis(InventoryTask.config)
+ rwp = rw.pipeline()
+ rwp.set(redis_netconf_key, netconf_str)
+ rwp.set(redis_state_key, state_str)
+ rwp.execute()
+ logger.info(f'Nokia docs info loaded from {hostname}')
+ return netconf_config, state
+
+
+@log_task_entry_and_exit
+def snmp_refresh_interfaces_nokia(
+ hostname, state_doc, communities, redis, update_callback=lambda s: None):
+
+ def _interface_info(interface_, name_field):
+ return {
+ 'name': interface_[name_field],
+ 'index': interface_['if-index'],
+ 'communities': communities
+ }
+
+ interfaces = (_interface_info(ifc, 'interface-name') for ifc in
+ nokia.get_interfaces_state(state_doc))
+ ports = (_interface_info(port, 'port-id') for port in
+ nokia.get_ports_state(state_doc))
+ lags = (_interface_info(lag, 'name') for lag in
+ nokia.get_lags_state(state_doc))
+ all_interfaces = list(itertools.chain(interfaces, ports, lags))
+
+ rp = redis.pipeline()
+ rp.set(f'snmp-interfaces:{hostname}', json.dumps(all_interfaces))
+
+ for ifc in all_interfaces:
+ ifc['hostname'] = hostname
+ rp.set(
+ f'snmp-interfaces-single:{hostname}:{ifc["name"]}',
+ json.dumps(ifc))
+
+ rp.execute()
+
+ update_callback(f'snmp interface info loaded from {hostname}')
def refresh_nokia_interface_list(hostname, netconf_config, redis, lab=False):
@@ -671,10 +741,10 @@ def refresh_nokia_interface_list(hostname, netconf_config, redis, lab=False):
rp.delete(k)
rp.execute()
- ports_by_port_id = {p['port-id']: p for p in nokia.get_ports(netconf_config)}
- lags_by_name = {lag['name']: lag for lag in nokia.get_lags(netconf_config)}
+ ports_by_port_id = {p['port-id']: p for p in nokia.get_ports_config(netconf_config)}
+ lags_by_name = {lag['name']: lag for lag in nokia.get_lags_config(netconf_config)}
interfaces_by_name = \
- {ifc['interface-name']: ifc for ifc in nokia.interface_info(netconf_config)}
+ {ifc['interface-name']: ifc for ifc in nokia.get_interfaces_config(netconf_config)}
def _save_interfaces_details(_details, _rp):
_rp.set(
@@ -934,6 +1004,8 @@ def snmp_refresh_interfaces_juniper(
interfaces = json.loads(interfaces.decode('utf-8'))
update_callback(f'using cached snmp interface data for {hostname}')
+ for ifc in interfaces:
+ ifc['communities'] = _general_community_strings(community)
r = get_next_redis(InventoryTask.config)
rp = r.pipeline()
diff --git a/test/test_nokia.py b/test/test_nokia.py
index 2c4eb0a1..a55f331d 100644
--- a/test/test_nokia.py
+++ b/test/test_nokia.py
@@ -2,23 +2,24 @@ import pathlib
from lxml import etree
-from inventory_provider.nokia import get_lags, interface_info, get_ports
+from inventory_provider.nokia import get_lags_config, get_interfaces_config, \
+ get_ports_config, get_ports_state, get_lags_state, get_interfaces_state
netconf_doc = etree.parse(pathlib.Path(__file__).parent.joinpath(
'data/rt0.lon.uk.lab.office.geant.net-netconf-nokia.xml'))
+state_doc = etree.parse(pathlib.Path(__file__).parent.joinpath(
+ 'data/rt0.lon.uk.lab.office.geant.net-netconf-nokia-state.xml'))
def test_get_lags():
- lags = list(get_lags(netconf_doc))
- assert len(lags) == 4
+ lags = list(get_lags_config(netconf_doc))
found_names = {lag['name'] for lag in lags}
expected_names = {'lag-1', 'lag-2', 'lag-3', 'lag-31'}
assert found_names == expected_names
def test_interface_info():
- if_addresses = list(interface_info(netconf_doc))
- assert len(if_addresses) == 5
+ if_addresses = list(get_interfaces_config(netconf_doc))
found_names = {_interface['interface-name'] for _interface in if_addresses}
expected_names = {'lag-1.0', 'lag-2.0', 'system', 'to_rt0_ams_ZR-INFINERA',
'to_rt0_ams_ZR-NOKIA'}
@@ -26,8 +27,7 @@ def test_interface_info():
def test_get_ports():
- ports = list(get_ports(netconf_doc))
- assert len(ports) == 18
+ ports = list(get_ports_config(netconf_doc))
found_port_ids = {port['port-id'] for port in ports}
expected_ports = {
'1/1/c2', '1/1/c2/1', '1/1/c2/2', '1/1/c2/3', '1/1/c7', '1/1/c7/1',
@@ -35,3 +35,41 @@ def test_get_ports():
'2/1/c7', '2/1/c7/1', '2/1/c8', '2/1/c8/1', '2/1/c13', '2/1/c13/1',
}
assert found_port_ids == expected_ports
+
+
+def test_get_port_state():
+ ports = {p['port-id']: p for p in get_ports_state(state_doc)}
+ assert len(ports) == 197
+ found_port_ids = {p_id for p_id in ports}
+ ports_sample = {
+ '1/1/c2', '1/1/c2/1', '1/1/c2/2', '1/1/c2/3', '1/1/c7', '1/1/c7/1',
+ '1/1/c8', '1/1/c8/1', '1/1/c9', '1/1/c9/1', '1/1/c13', '1/1/c13/1',
+ '2/1/c7', '2/1/c7/1', '2/1/c8', '2/1/c8/1', '2/1/c13', '2/1/c13/1',
+ }
+ assert found_port_ids > ports_sample
+ expected_up_ports = {
+ '1/1/c2', '1/1/c5', '1/1/c5/1', '1/1/c7', '1/1/c8', '1/1/c9', '1/1/c13',
+ '1/1/c13/1', '1/1/c19', '1/1/c26', '1/1/c26/1', '1/1/c27', '1/1/c27/1',
+ '1/1/c30', '2/1/c7', '2/1/c8', '2/1/c13', '2/1/c13/1', 'A/1', 'B/1'
+ }
+ assert {k for k, v in ports.items() if v['oper-state'] == 'up'} == expected_up_ports
+
+
+def test_get_lag_state():
+ lags = {x['name']: x for x in get_lags_state(state_doc)}
+ found_names = {lag_name for lag_name in lags}
+ expected_names = {'lag-1', 'lag-2', 'lag-3'}
+ assert found_names == expected_names
+ expected_up_lags = {'lag-2'}
+ assert {k for k, v in lags.items() if v['oper-state'] == 'up'} == expected_up_lags
+
+
+def test_get_interface_state():
+ interfaces = {x['interface-name']: x for x in get_interfaces_state(state_doc)}
+ found_names = {interface_name for interface_name in interfaces}
+ expected_names = {'system', 'lag-1.0', 'lag-2.0', 'lag-3.0', 'exfo400', 'guy',
+ 'exfo400-100', 'management'}
+ assert found_names == expected_names
+ expected_up_interfaces = {'system', 'lag-2.0', 'guy', 'exfo400-100', 'management'}
+ assert {k for k, v in interfaces.items() if
+ v['oper-state'] == 'up'} == expected_up_interfaces
--
GitLab