poller.py 11.92 KiB
import json
import logging
import re
from flask import Blueprint, Response, current_app
from inventory_provider import juniper
from inventory_provider.routes import common
logger = logging.getLogger(__name__)
routes = Blueprint('poller-support-routes', __name__)
Gb = 1 << 30
INTERFACE_LIST_SCHEMA = {
'$schema': 'http://json-schema.org/draft-07/schema#',
'definitions': {
'service': {
'type': 'object',
'properties': {
'id': {'type': 'integer'},
'name': {'type': 'string'},
'type': {'type': 'string'},
'status': {'type': 'string'},
},
'required': ['id', 'name', 'type', 'status'],
'additionalProperties': False
},
'interface': {
'type': 'object',
'properties': {
'router': {'type': 'string'},
'name': {'type': 'string'},
'description': {'type': 'string'},
'snmp-index': {
'type': 'integer',
'minimum': 1
},
'bundle': {
'type': 'array',
'items': {'type': 'string'}
},
'bundle-parents': {
'type': 'array',
'items': {'type': 'string'}
},
'circuits': {
'type': 'array',
'items': {'$ref': '#/definitions/service'}
}
},
'required': [
'router', 'name', 'description',
'snmp-index', 'bundle', 'bundle-parents',
'circuits'],
'additionalProperties': False
},
},
'type': 'array',
'items': {'$ref': '#/definitions/interface'}
}
INTERFACE_SPEED_LIST_SCHEMA = {
'$schema': 'http://json-schema.org/draft-07/schema#',
'definitions': {
'interface': {
'type': 'object',
'properties': {
'router': {'type': 'string'},
'name': {'type': 'string'},
'speed': {'type': 'integer'}
},
'required': ['router', 'name', 'speed'],
'additionalProperties': False
},
},
'type': 'array',
'items': {'$ref': '#/definitions/interface'}
}
@routes.after_request
def after_request(resp):
return common.after_request(resp)
def _load_snmp_indexes(hostname=None):
result = dict()
key_pattern = f'snmp-interfaces:{hostname}*' \
if hostname else 'snmp-interfaces:*'
for doc in common.load_json_docs(
config_params=current_app.config['INVENTORY_PROVIDER_CONFIG'],
key_pattern=key_pattern):
router = doc['key'][len('snmp-interfaces:'):]
result[router] = {e['name']: e for e in doc['value']}
return result
def _load_interface_bundles(hostname=None):
result = dict()
key_pattern = f'netconf-interface-bundles:{hostname}:*' \
if hostname else 'netconf-interface-bundles:*'
for doc in common.load_json_docs(
config_params=current_app.config['INVENTORY_PROVIDER_CONFIG'],
key_pattern=key_pattern,
num_threads=20):
m = re.match(r'^netconf-interface-bundles:([^:]+):(.+)', doc['key'])
assert m
router = m.group(1)
interface = m.group(2)
result.setdefault(router, dict())
result[router][interface] = doc['value']
return result
def _load_services(hostname=None):
result = dict()
key_pattern = f'opsdb:interface_services:{hostname}:*' \
if hostname else 'opsdb:interface_services:*'
def _service_params(full_service_info):
return {
'id': full_service_info['id'],
'name': full_service_info['name'],
'type': full_service_info['service_type'],
'status': full_service_info['status']
}
for doc in common.load_json_docs(
config_params=current_app.config['INVENTORY_PROVIDER_CONFIG'],
key_pattern=key_pattern,
num_threads=20):
m = re.match(r'^opsdb:interface_services:([^:]+):(.+)', doc['key'])
if not m:
logger.warning(f'can\'t parse redis service key {doc["key"]}')
# there are some weird records (dtn*, dp1*)
continue
router = m.group(1)
interface = m.group(2)
result.setdefault(router, dict())
result[router][interface] = [_service_params(s) for s in doc['value']]
return result
def _load_interfaces(hostname):
key_pattern = f'netconf:{hostname}*' if hostname else 'netconf:*'
for doc in common.load_xml_docs(
config_params=current_app.config['INVENTORY_PROVIDER_CONFIG'],
key_pattern=key_pattern,
num_threads=10):
router = doc['key'][len('netconf:'):]
for ifc in juniper.list_interfaces(doc['value']):
if not ifc['description']:
continue
yield {
'router': router,
'name': ifc['name'],
'bundle': ifc['bundle'],
'bundle-parents': [],
'description': ifc['description'],
'circuits': []
}
def _add_bundle_parents(interfaces, hostname=None):
"""
generator that adds bundle-parents info to each interface.
:param interfaces: result of _load_interfaces
:param hostname: hostname or None for all
:return: generator with bundle-parents populated in each element
"""
bundles = _load_interface_bundles(hostname)
for ifc in interfaces:
router_bundle = bundles.get(ifc['router'], None)
if router_bundle:
base_ifc = ifc['name'].split('.')[0]
ifc['bundle-parents'] = router_bundle.get(base_ifc, [])
yield ifc
def _add_circuits(interfaces, hostname=None):
"""
generator that adds service info to each interface.
:param interfaces: result of _load_interfaces
:param hostname: hostname or None for all
:return: generator with 'circuits' populated in each element, if present
"""
services = _load_services(hostname)
for ifc in interfaces:
router_services = services.get(ifc['router'], None)
if router_services:
ifc['circuits'] = router_services.get(ifc['name'], [])
yield ifc
def _add_snmp_indexes(interfaces, hostname=None):
"""
generator that adds snmp info to each interface, if available
:param interfaces: result of _load_interfaces
:param hostname: hostname or None for all
:return: generator with 'snmp-index' optionally added to each element
"""
snmp_indexes = _load_snmp_indexes(hostname)
for ifc in interfaces:
router_snmp = snmp_indexes.get(ifc['router'], None)
if router_snmp and ifc['name'] in router_snmp:
ifc['snmp-index'] = router_snmp[ifc['name']]['index']
# TODO: uncomment this when it won't break poller-admin-service
# not urgent ... it looks empirically like all logical-system
# interfaces are repeated for both communities
# ifc['snmp-community'] = router_snmp[ifc['name']]['community']
yield ifc
def _load_interfaces_to_poll(hostname=None):
"""
prepares the result of a call to /interfaces
:param hostname: hostname or None for all
:return: generator yielding interface elements
"""
basic_interfaces = _load_interfaces(hostname)
with_bundles = _add_bundle_parents(basic_interfaces, hostname)
with_circuits = _add_circuits(with_bundles, hostname)
with_snmp = _add_snmp_indexes(with_circuits, hostname)
def _has_snmp_index(ifc):
return 'snmp-index' in ifc
# only return interfaces that can be polled
return filter(_has_snmp_index, with_snmp)
@routes.route("/interfaces", methods=['GET', 'POST'])
@routes.route('/interfaces/<hostname>', methods=['GET', 'POST'])
@common.require_accepts_json
def interfaces(hostname=None):
"""
Handler for `/poller/interfaces` and
`/poller/interfaces/<hostname>`
which returns information for either all interfaces
or those on the requested hostname.
The response is a list of information for all
interfaces that should be polled, including service
information and snmp information.
.. asjson::
inventory_provider.routes.poller.INTERFACE_LIST_SCHEMA
:param hostname: optional, if present should be a router hostname
:return:
"""
cache_key = f'classifier-cache:poller-interfaces:{hostname}' \
if hostname else 'classifier-cache:poller-interfaces:all'
r = common.get_current_redis()
result = r.get(cache_key)
if result:
result = result.decode('utf-8')
else:
result = list(_load_interfaces_to_poll(hostname))
if not result:
return Response(
response='no interfaces found',
status=404,
mimetype='text/html')
result = json.dumps(result)
# cache this data for the next call
r.set(cache_key, result.encode('utf-8'))
return Response(result, mimetype="application/json")
def interface_speed(ifc):
"""
Return the maximum bits per second expected for the given interface.
cf. https://www.juniper.net/documentation/us/en/software/
vmx/vmx-getting-started/topics/task/
vmx-chassis-interface-type-configuring.html
:param ifc:
:return: an integer bits per second
"""
def _name_to_speed(ifc_name):
if ifc_name.startswith('ge'):
return Gb
if ifc_name.startswith('xe'):
return 10 * Gb
if ifc_name.startswith('et'):
return 100 * Gb
logger.warning(f'unrecognized interface name: {ifc_name}')
return -1
if ifc['bundle-parents']:
if not ifc['name'].startswith('ae'):
logger.warning(
f'ifc has bundle-parents, but name is {ifc["name"]}')
return sum(_name_to_speed(name) for name in ifc['bundle-parents'])
return _name_to_speed(ifc['name'])
def _load_interfaces_and_speeds(hostname=None):
"""
prepares the result of a call to /speeds
:param hostname: hostname or None for all
:return: generator yielding interface elements
"""
basic_interfaces = _load_interfaces(hostname)
with_bundles = _add_bundle_parents(basic_interfaces, hostname)
with_bundles = list(with_bundles)
def _result_ifc(ifc):
return {
'router': ifc['router'],
'name': ifc['name'],
'speed': interface_speed(ifc)
}
return map(_result_ifc, with_bundles)
@routes.route("/speeds", methods=['GET', 'POST'])
@routes.route('/speeds/<hostname>', methods=['GET', 'POST'])
@common.require_accepts_json
def interface_speeds(hostname=None):
"""
Handler for `/poller/speeds` and
`/poller/speeds/<hostname>`
which returns information for either all interfaces
or those on the requested hostname.
The response is a list of maximum speed information (in bits
per second) for all known interfaces.
[speed <= 0 means the max interface speed can't be determined]
.. asjson::
inventory_provider.routes.poller.INTERFACE_SPEED_LIST_SCHEMA
:param hostname: optional, if present should be a router hostname
:return:
"""
cache_key = f'classifier-cache:poller-interface-speeds:{hostname}' \
if hostname else 'classifier-cache:poller-interface-speeds:all'
r = common.get_current_redis()
result = r.get(cache_key)
if result:
result = result.decode('utf-8')
else:
result = list(_load_interfaces_and_speeds(hostname))
if not result:
return Response(
response='no interfaces found',
status=404,
mimetype='text/html')
result = json.dumps(result)
# cache this data for the next call
r.set(cache_key, result.encode('utf-8'))
return Response(result, mimetype="application/json")