-
Pelle Koster authoredPelle Koster authored
poller.py 53.93 KiB
"""
BRIAN support Endpoints
=========================
These endpoints are intended for use by BRIAN.
.. contents:: :local:
/poller/interfaces</hostname>
---------------------------------
.. autofunction:: inventory_provider.routes.poller.interfaces
/poller/speeds</hostname>
---------------------------------
.. autofunction:: inventory_provider.routes.poller.interface_speeds
/poller/eumetsat-multicast
---------------------------------
.. autofunction:: inventory_provider.routes.poller.eumetsat_multicast
/poller/gws/direct
---------------------------------
.. autofunction:: inventory_provider.routes.poller.gws_direct
/poller/gws/direct-config
---------------------------------
.. autofunction:: inventory_provider.routes.poller.gws_direct_config
/poller/gws/indirect
---------------------------------
.. autofunction:: inventory_provider.routes.poller.gws_indirect
/poller/services</service-type>
---------------------------------
.. autofunction:: inventory_provider.routes.poller.get_services
/poller/service-types
---------------------------------
.. autofunction:: inventory_provider.routes.poller.get_service_types
/poller/error-report-interfaces</hostname>
------------------------------------------
.. autofunction:: inventory_provider.routes.poller.error_report_interfaces
support method: _get_dashboards
---------------------------------
.. autofunction:: inventory_provider.routes.poller._get_dashboards
"""
from collections import defaultdict
from enum import Enum, auto
import itertools
import json
import logging
import re
from functools import partial
from typing import Dict
from flask import Blueprint, Response, current_app, request, jsonify
from inventory_provider import juniper
from inventory_provider.routes import common
from inventory_provider.tasks.common import ims_sorted_service_type_key
from inventory_provider.tasks import common as tasks_common
from inventory_provider.routes.classifier import get_ims_equipment_name, \
get_ims_interface
from inventory_provider.routes.common import _ignore_cache_or_retrieve
logger = logging.getLogger(__name__)
routes = Blueprint('poller-support-routes', __name__)
Mb = 1 << 20
Gb = 1 << 30
OC = Mb * 51.84
class INTERFACE_TYPES(Enum):
UNKNOWN = auto()
LOGICAL = auto()
PHYSICAL = auto()
AGGREGATE = auto()
class BRIAN_DASHBOARDS(Enum):
CLS = auto()
RE_PEER = auto()
RE_CUST = auto()
GEANTOPEN = auto()
GCS = auto()
L2_CIRCUIT = auto()
LHCONE_PEER = auto()
LHCONE_CUST = auto()
MDVPN_CUSTOMERS = auto()
INFRASTRUCTURE_BACKBONE = auto()
IAS_PRIVATE = auto()
IAS_PUBLIC = auto()
IAS_CUSTOMER = auto()
IAS_UPSTREAM = auto()
GWS_PHY_UPSTREAM = auto()
GBS_10G = auto()
# aggregate dashboards
CLS_PEERS = auto()
IAS_PEERS = auto()
GWS_UPSTREAMS = auto()
LHCONE = auto()
CAE1 = auto()
IC1 = auto()
COPERNICUS = auto()
ANA = auto()
# NREN customer
NREN = auto()
class PORT_TYPES(Enum):
ACCESS = auto()
SERVICE = auto()
UNKNOWN = auto()
# only used in INTERFACE_LIST_SCHEMA and sphinx docs
_DASHBOARD_IDS = [d.name for d in list(BRIAN_DASHBOARDS)]
_PORT_TYPES = [t.name for t in list(PORT_TYPES)]
_INTERFACE_TYPES = [i.name for i in list(INTERFACE_TYPES)]
INTERFACE_LIST_SCHEMA = {
'$schema': 'https://json-schema.org/draft-07/schema#',
'definitions': {
'service': {
'type': 'object',
'properties': {
'id': {'type': 'integer'},
'name': {'type': 'string'},
'type': {'type': 'string'},
'status': {'type': 'string'},
},
'required': ['id', 'name', 'type', 'status'],
'additionalProperties': False
},
'db_info': {
'type': 'object',
'properties': {
'name': {'type': 'string'},
'interface_type': {'enum': _INTERFACE_TYPES}
},
'required': ['name', 'interface_type'],
'additionalProperties': False
},
'interface': {
'type': 'object',
'properties': {
'router': {'type': 'string'},
'name': {'type': 'string'},
'description': {'type': 'string'},
'snmp-index': {
'type': 'integer',
'minimum': 1
},
'bundle': {
'type': 'array',
'items': {'type': 'string'}
},
'bundle-parents': {
'type': 'array',
'items': {'type': 'string'}
},
'circuits': {
'type': 'array',
'items': {'$ref': '#/definitions/service'}
},
'dashboards': {
'type': 'array',
'items': {'enum': _DASHBOARD_IDS}
},
'dashboard_info': {
'$ref': '#/definitions/db_info',
},
'dashboards_info': {
'type': 'array',
'items': {'$ref': '#/definitions/db_info'}
},
'port_type': {'enum': _PORT_TYPES}
},
'required': [
'router', 'name', 'description',
'snmp-index', 'bundle', 'bundle-parents',
'circuits', 'dashboards', 'port_type'],
'additionalProperties': False
},
},
'type': 'array',
'items': {'$ref': '#/definitions/interface'}
}
ERROR_REPORT_INTERFACE_LIST_SCHEMA = {
'$schema': 'https://json-schema.org/draft-07/schema#',
'definitions': {
'interface': {
'type': 'object',
'properties': {
'router': {'type': 'string'},
'name': {'type': 'string'},
'description': {'type': 'string'},
'vendor': {'type': 'string', 'enum': ['juniper', 'nokia']}
},
'required': ['router', 'name', 'description', 'vendor'],
'additionalProperties': False
},
},
'type': 'array',
'items': {'$ref': '#/definitions/interface'}
}
INTERFACE_SPEED_LIST_SCHEMA = {
'$schema': 'https://json-schema.org/draft-07/schema#',
'definitions': {
'interface': {
'type': 'object',
'properties': {
'router': {'type': 'string'},
'name': {'type': 'string'},
'speed': {'type': 'integer'}
},
'required': ['router', 'name', 'speed'],
'additionalProperties': False
},
},
'type': 'array',
'items': {'$ref': '#/definitions/interface'}
}
MULTICAST_SUBSCRIPTION_LIST_SCHEMA = {
'$schema': 'https://json-schema.org/draft-07/schema#',
'definitions': {
'ipv4-address': {
'type': 'string',
'pattern': r'^(\d+\.){3}\d+$'
},
'subscription': {
'type': 'object',
'properties': {
'router': {'type': 'string'},
'subscription': {'$ref': '#/definitions/ipv4-address'},
'endpoint': {'$ref': '#/definitions/ipv4-address'},
'oid': {
'type': 'string',
'pattern': r'^(\d+\.)*\d+$'
},
'community': {'type': 'string'}
},
'required': [
'router', 'subscription', 'endpoint', 'oid', 'community'],
'additionalProperties': False
},
},
'type': 'array',
'items': {'$ref': '#/definitions/subscription'}
}
GWS_DIRECT_DATA_SCHEMA = {
'$schema': 'https://json-schema.org/draft-07/schema#',
'definitions': {
'oid': {
'type': 'string',
'pattern': r'^(\d+\.)*\d+$'
},
'snmp-v2': {
'type': 'object',
'properties': {
'community': {'type': 'string'}
},
'required': ['community'],
'additionalProperties': False
},
'snmp-v3-cred': {
'type': 'object',
'properties': {
'protocol': {'enum': ['MD5', 'DES']},
'password': {'type': 'string'}
},
'required': ['protocol', 'password'],
'additionalProperties': False
},
'snmp-v3': {
'type': 'object',
'properties': {
'sec-name': {'type': 'string'},
'auth': {'$ref': '#/definitions/snmp-v3-cred'},
'priv': {'$ref': '#/definitions/snmp-v3-cred'}
},
'required': ['sec-name'],
'additionalProperties': False
},
'counter': {
'type': 'object',
'properties': {
'field': {
'enum': [
'discards_in',
'discards_out',
'errors_in',
'errors_out',
'traffic_in',
'traffic_out'
]
},
'oid': {'$ref': '#/definitions/oid'},
'snmp': {
'oneOf': [
{'$ref': '#/definitions/snmp-v2'},
{'$ref': '#/definitions/snmp-v3'}
]
}
},
'required': ['field', 'oid', 'snmp'],
'additionalProperties': False
},
'interface-counters': {
'type': 'object',
'properties': {
'nren': {'type': 'string'},
'isp': {
'type': 'string',
'enum': ['Cogent', 'Telia', 'CenturyLink']
},
'hostname': {'type': 'string'},
'tag': {'type': 'string'},
'counters': {
'type': 'array',
'items': {'$ref': '#/definitions/counter'},
'minItems': 1
},
'info': {'type': 'string'}
},
'required': [
'nren', 'isp', 'hostname', 'tag', 'counters'],
'additionalProperties': False
}
},
'type': 'array',
'items': {'$ref': '#/definitions/interface-counters'}
}
SERVICES_LIST_SCHEMA = {
'$schema': 'https://json-schema.org/draft-07/schema#',
'definitions': {
'oid': {
'type': 'string',
'pattern': r'^(\d+\.)*\d+$'
},
'counter-field': {
'type': 'object',
'properties': {
'field': {
'type': 'string',
'enum': ['egressOctets', 'ingressOctets']
},
'oid': {'$ref': '#/definitions/oid'}
},
'required': ['field', 'oid'],
'additionalProperties': False
},
'snmp-info': {
'type': 'object',
'properties': {
'ifIndex': {'type': 'integer'},
'community': {'type': 'string'},
'counters': {
'type': 'array',
'items': {'$ref': '#/definitions/counter-field'},
'minItems': 1
}
},
'required': ['ifIndex', 'community'],
'additionalProperties': False
},
'service': {
'type': 'object',
'properties': {
'id': {'type': 'integer'},
'name': {'type': 'string'},
'customer': {'type': 'string'},
'speed': {'type': 'integer'},
'pop': {'type': 'string'},
'hostname': {'type': 'string'},
'interface': {'type': 'string'},
'type': {'type': 'string'},
'status': {'type': 'string'},
'snmp': {'$ref': '#/definitions/snmp-info'}
},
'required': [
'id', 'name', 'customer',
'speed', 'pop', 'hostname',
'interface', 'type', 'status'],
'additionalProperties': False
}
},
'type': 'array',
'items': {'$ref': '#/definitions/service'}
}
STRING_LIST_SCHEMA = {
'$schema': 'https://json-schema.org/draft-07/schema#',
'type': 'array',
'items': {'type': 'string'}
}
@routes.after_request
def after_request(resp):
return common.after_request(resp)
def _get_dashboards(interface):
"""
Yield enums from BRIAN_DASHBOARDS to indicate which dashboards
this interface should be included in.
cf. POL1-482
Possible dashboard id's are:
.. asjson::
inventory_provider.routes.poller._DASHBOARD_IDS
:param interface: a dict with keys like router, name, description
:return: generator that yields enums from BRIAN_DASHBOARDS
"""
router = interface.get('router', '').lower()
ifc_name = interface.get('name', '')
description = interface.get('description', '').strip()
if 'SRV_L3VPN' in description and 'COPERNICUS' in description:
yield BRIAN_DASHBOARDS.COPERNICUS
if re.match(r'SRV_CLS\s', description):
yield BRIAN_DASHBOARDS.CLS
if re.match(r'SRV_CLS PRIVATE\s', description):
yield BRIAN_DASHBOARDS.CLS_PEERS
if re.match(r'SRV_IAS PUBLIC\s', description):
yield BRIAN_DASHBOARDS.IAS_PUBLIC
yield BRIAN_DASHBOARDS.IAS_PEERS
if re.match(r'SRV_IAS PRIVATE\s', description):
yield BRIAN_DASHBOARDS.IAS_PRIVATE
yield BRIAN_DASHBOARDS.IAS_PEERS
if re.match(r'SRV_IAS CUSTOMER\s', description):
yield BRIAN_DASHBOARDS.IAS_CUSTOMER
if re.match(r'SRV_IAS UPSTREAM\s', description):
yield BRIAN_DASHBOARDS.IAS_UPSTREAM
if re.match(r'SRV_10GGBS CUSTOMER\s', description):
yield BRIAN_DASHBOARDS.GBS_10G
if re.match(r'(SRV_GLOBAL|SRV_L3VPN|LAG|PHY) RE_INTERCONNECT\s', description):
yield BRIAN_DASHBOARDS.RE_PEER
if re.match(r'(PHY|LAG|SRV_GLOBAL) CUSTOMER\s', description):
yield BRIAN_DASHBOARDS.RE_CUST
if re.match('^SRV_GCS', description):
yield BRIAN_DASHBOARDS.GCS
if re.match(r'PHY CUSTOMER_GEO\s', description):
yield BRIAN_DASHBOARDS.GEANTOPEN
if 'SRV_L2CIRCUIT' in description:
yield BRIAN_DASHBOARDS.L2_CIRCUIT
if 'LHCONE' in description:
if 'SRV_L3VPN RE' in description:
yield BRIAN_DASHBOARDS.LHCONE_PEER
if 'SRV_L3VPN CUSTOMER' in description:
yield BRIAN_DASHBOARDS.LHCONE_CUST
if re.match(r'SRV_L3VPN (CUSTOMER|RE_INTERCONNECT)\s', description):
if '-LHCONE' in description:
# `-LHCONE` can be preceded by tons of different stuff, so it's
# simpler to check for this rather than a full regex
yield BRIAN_DASHBOARDS.LHCONE
if re.match(r'SRV_MDVPN CUSTOMER\s', description):
yield BRIAN_DASHBOARDS.MDVPN_CUSTOMERS
if re.match(r'(SRV_GLOBAL|LAG|PHY) INFRASTRUCTURE BACKBONE', description):
yield BRIAN_DASHBOARDS.INFRASTRUCTURE_BACKBONE
if router == 'mx1.lon.uk.geant.net' \
and re.match(r'^ae12\.\d+$', ifc_name):
yield BRIAN_DASHBOARDS.CAE1
if router == 'rt1.mar.fr.geant.net' \
and re.match(r'^ae12\.\d+$', ifc_name):
yield BRIAN_DASHBOARDS.IC1
if re.match(r'PHY UPSTREAM\s', description):
yield BRIAN_DASHBOARDS.GWS_PHY_UPSTREAM
regex = r'(PHY|LAG|(SRV_(GLOBAL|LHCONE|MDVPN|IAS|CLS|L3VPN))) CUSTOMER\s'
if re.match(regex, description):
yield BRIAN_DASHBOARDS.NREN
if "GA-" in description and "ANA-" in description:
yield BRIAN_DASHBOARDS.ANA
def _get_dashboard_data(ifc, customers):
def _get_interface_type(description):
if re.match(r'^PHY', description):
return INTERFACE_TYPES.PHYSICAL
if re.match(r'^SRV_', description):
return INTERFACE_TYPES.LOGICAL
if re.match(r'^LAG', description):
return INTERFACE_TYPES.AGGREGATE
return INTERFACE_TYPES.UNKNOWN
description = ifc.get('description', '').strip()
dashboards = ifc.get('dashboards', [])
interface_type = _get_interface_type(description)
if len(dashboards) == 0:
return ifc
def _get_customer_name(description):
name = description.split(' ')
if len(name) >= 3:
return name[2].strip().upper()
else:
# if the description isn't properly formatted
# use it as the name to make it obvious something is wrong
return description
def _get_backbone_name(description):
name = description.split('|')
if len(name) >= 2:
name = name[1].strip()
return name.replace('( ', '(')
else:
# if the description isn't properly formatted
# use it as the name to make it obvious something is wrong
return description
if BRIAN_DASHBOARDS.INFRASTRUCTURE_BACKBONE.name in dashboards:
names = [_get_backbone_name(description)]
elif BRIAN_DASHBOARDS.GWS_PHY_UPSTREAM.name in dashboards:
host = ifc['router']
location = host.split('.')[1].upper()
names = [f'{_get_customer_name(description)} - {location}']
elif BRIAN_DASHBOARDS.L2_CIRCUIT.name in dashboards:
# This will give names derived from the Interface Description
# names = list(_get_l2_customer_names(description))
# This will give first 2 names IMS
names = [c['name'] for c in customers][:2]
else:
names = [_get_customer_name(description)]
# if no customers found just return the original data
if not names:
return ifc
# to maintain compatability with current brian dashboard manager we will
# continue to return dashboard_info with the first customer name. We will
# also return dashboards_info (note the plural of dashboards) with up to
# two customers
return {
**ifc,
'dashboard_info': {
'name': names[0],
'interface_type': interface_type.name
},
'dashboards_info': [{
'name': name,
'interface_type': interface_type.name
} for name in set(names)]
}
def _load_interface_bundles(config, hostname=None, use_next_redis=False):
result = dict()
def _load_docs(key_pattern):
for doc in common.load_json_docs(
config_params=config,
key_pattern=key_pattern,
num_threads=20,
use_next_redis=use_next_redis):
m = re.match(
r'.*netconf-interface-bundles:([^:]+):(.+)', doc['key'])
assert m
router = m.group(1)
interface = m.group(2)
result.setdefault(router, dict())
result[router][interface] = doc['value']
base_key = 'netconf-interface-bundles'
base_key_pattern = f'{base_key}:{hostname}:*' \
if hostname else f'{base_key}:*'
_load_docs(base_key_pattern)
_load_docs(f'lab:{base_key_pattern}')
return result
def _get_services_and_customers(config, hostname=None, use_next_redis=False):
if hostname:
hostname = get_ims_equipment_name(hostname)
result = defaultdict(dict)
key_pattern = f'ims:interface_services:{hostname}:*' \
if hostname else 'ims:interface_services:*'
for doc in common.load_json_docs(
config_params=config,
key_pattern=key_pattern,
num_threads=20,
use_next_redis=use_next_redis):
cs = {
'services': [],
'customers': []
}
included_service_ids = set()
for s in doc['value']:
if s['id'] in included_service_ids:
continue
if s.get('port_type', '') == 'ab':
continue
included_service_ids.add(s['id'])
cs['customers'].append({
'name': s['customer'],
'type': 'UNKNOWN'
})
for c in s.get('additional_customers', []):
cs['customers'].append({
'name': c['name'],
'type': c.get('type', 'UNKNOWN')
})
if s['circuit_type'] == 'service':
cs['services'].append({
'id': s['id'],
'name': s['name'],
'type': s['service_type'],
'status': s['status'],
})
result[s['equipment']][s['port']] = cs
return result
def _load_netconf_docs(
config, filter_pattern, use_next_redis=False):
"""
yields dicts like:
{
'router': router hostname
'netconf': loaded netconf xml doc
}
:param config: app config
:param filter_pattern: search filter, including 'netconf:'
:param use_next_redis: use next instead of current redis, if true
:return: yields netconf docs, formatted as above
"""
m = re.match(r'^(.*netconf:).+', filter_pattern)
# TODO: probably better to not required netconf: to be passed in
assert m # sanity
key_prefix_len = len(m.group(1))
assert key_prefix_len >= len('netconf:') # sanity
for doc in common.load_xml_docs(
config_params=config,
key_pattern=filter_pattern,
num_threads=10,
use_next_redis=use_next_redis):
yield {
'router': doc['key'][key_prefix_len:],
'netconf': doc['value']
}
def _load_netconf_parsed_cache(config, cache_name, hostname=None, is_lab=False, use_next_redis=False):
hostname_suffix = hostname if hostname else ''
lab_prefix = 'lab:' if is_lab else ''
filter_pattern = f'{lab_prefix}{cache_name}:{hostname_suffix}*'
for doc in common.load_json_docs(
config_params=config,
key_pattern=filter_pattern,
num_threads=20,
use_next_redis=use_next_redis):
m = re.match(fr'(lab:)?{cache_name}:(.+:.+)', doc['key'])
if m:
# preparse the key as interfaces can include : in the name
key_parts = m.group(2).split(':')
key = f'{key_parts[0]}-----{":".join(key_parts[1:])}'
yield key, doc['value']
def _load_interfaces(
config, hostname=None, no_lab=False, use_next_redis=False):
"""
loads basic interface data for production & lab routers
:param config:
:param hostname:
:param use_next_redis:
:return:
"""
def _load_netconf_caches(is_lab=False):
interfaces = dict(_load_netconf_parsed_cache(config, 'netconf-interfaces',
hostname, is_lab, use_next_redis))
interface_bundles = dict(_load_netconf_parsed_cache(config, 'netconf-interface-bundles',
hostname, is_lab, use_next_redis))
for key, ifc in interfaces.items():
if not ifc['description']:
continue
router, interface_name = key.split('-----')
bundle = interface_bundles.get(key, [])
yield {
'router': router,
'name': interface_name,
'bundle': bundle,
'bundle-parents': [],
'description': ifc['description'],
'circuits': []
}
yield from _load_netconf_caches()
if not no_lab:
logger.debug('lab')
yield from _load_netconf_caches(is_lab=True)
def _add_speeds(interfaces):
r = common.get_current_redis()
all_netconf_interfaces = json.loads(r.get('netconf-interfaces:all')) or []
netconf_interface_index = {}
for nc_ifc in all_netconf_interfaces:
if 'hostname' in nc_ifc and 'name' in nc_ifc:
netconf_interface_index[f"{nc_ifc['hostname']}---{nc_ifc['name']}"] = nc_ifc
for ifc in interfaces:
nc_ifc = netconf_interface_index.get(f"{ifc['router']}---{ifc['name']}", {})
if 'speed' in nc_ifc:
ifc['speed'] = nc_ifc['speed']
yield ifc
def _add_bundle_parents(interfaces, hostname=None):
"""
generator that adds bundle-parents info to each interface.
:param interfaces: result of _load_interfaces
:param hostname: hostname or None for all
:return: generator with bundle-parents populated in each element
"""
def _get_base_name(name):
return name.split('.')[0]
bundles = _load_interface_bundles(
current_app.config['INVENTORY_PROVIDER_CONFIG'], hostname)
# create a quick look-up for interface details
interface_index = {f"{ifc['router']}---{ifc['name']}": ifc for ifc in interfaces}
for ifc in interfaces:
router_bundle = bundles.get(ifc['router'], None)
if router_bundle:
base_ifc = _get_base_name(ifc['name'])
bundle_parents = [interface_index.get(f"{ifc['router']}---{_get_base_name(bundle_ifc)}")
for bundle_ifc in router_bundle.get(base_ifc, [])]
ifc['bundle-parents'] = bundle_parents
yield ifc
def _get_port_type(description):
rex = re.search(r'\$([a-zA-Z]+\-\d+)', description)
if rex:
sid = rex.group(1)
if 'GA' in sid:
return PORT_TYPES.ACCESS.name
elif 'GS' in sid:
return PORT_TYPES.SERVICE.name
return PORT_TYPES.UNKNOWN.name
def load_interfaces_to_poll(config, hostname=None, no_lab=False, use_next_redis=False):
def is_relevant(ifc):
return not re.match(r"^(lt-|so-|dsc\.|fxp\d|lo\d).*", ifc["name"])
basic_interfaces = list(
filter(is_relevant, _load_interfaces(config, hostname, no_lab, use_next_redis))
)
bundles = _load_interface_bundles(config, hostname, use_next_redis)
services_and_customers = \
_get_services_and_customers(config, hostname, use_next_redis)
snmp_indexes = common.load_snmp_indexes(config, hostname, use_next_redis)
def _get_populated_interfaces(all_interfaces):
if use_next_redis:
r = tasks_common.get_next_redis(config)
else:
r = tasks_common.get_current_redis(config)
for ifc in all_interfaces:
router_snmp = snmp_indexes.get(ifc['router'], None)
if router_snmp and ifc['name'] in router_snmp:
ifc['snmp-index'] = router_snmp[ifc['name']]['index']
router_bundle = bundles.get(ifc['router'], None)
if router_bundle:
base_ifc = ifc['name'].split('.')[0]
ifc['bundle-parents'] = router_bundle.get(base_ifc, [])
router_services_and_customers = services_and_customers.get(
get_ims_equipment_name(ifc['router'], r), {})
ifc_services_and_customers = \
router_services_and_customers.get(
get_ims_interface(ifc['name']), {}
)
if 'services' in ifc_services_and_customers \
and ifc_services_and_customers['services']:
ifc['circuits'] = ifc_services_and_customers['services']
dashboards = _get_dashboards(ifc)
ifc['dashboards'] = sorted([d.name for d in dashboards])
ifc = _get_dashboard_data(
ifc, ifc_services_and_customers.get('customers', []))
port_type = _get_port_type(ifc['description'])
ifc['port_type'] = port_type
yield ifc
else:
continue
return _get_populated_interfaces(basic_interfaces)
@routes.route("/interfaces", methods=['GET'])
@routes.route('/interfaces/<hostname>', methods=['GET'])
@common.require_accepts_json
def interfaces(hostname=None):
"""
Handler for `/poller/interfaces` and
`/poller/interfaces/<hostname>`
which returns information for either all interfaces
or those on the requested hostname.
The optional `no-lab` parameter omits lab routers
if it's truthiness evaluates to True.
The response is a list of information for all
interfaces that should be polled, including service
information and snmp information.
.. asjson::
inventory_provider.routes.poller.INTERFACE_LIST_SCHEMA
:meth:`inventory_provider.routes.poller._get_services`
is where dashboard mappings is handled.
:param hostname: optional, if present should be a router hostname
:return:
"""
cache_key = f'classifier-cache:poller-interfaces:{hostname}' \
if hostname else 'classifier-cache:poller-interfaces:all'
no_lab = common.get_bool_request_arg('no-lab', False)
if no_lab:
cache_key = f'{cache_key}:no_lab'
r = common.get_current_redis()
result = _ignore_cache_or_retrieve(request, cache_key, r)
if not result:
result = list(load_interfaces_to_poll(
current_app.config['INVENTORY_PROVIDER_CONFIG'], hostname, no_lab))
if not result:
return Response(
response='no interfaces found',
status=404,
mimetype='text/html')
result = json.dumps(result)
# cache this data for the next call
r.set(cache_key, result.encode('utf-8'))
return Response(result, mimetype="application/json")
def get_netdash_equipment(config, use_next_redis=False) -> Dict[str, str]:
"""Get the netdash equipment mapping from redis."""
if use_next_redis:
r = tasks_common.get_next_redis(config)
else:
r = tasks_common.get_current_redis(config)
return json.loads(r.get('netdash').decode('utf-8'))
def load_error_report_interfaces(
config, hostname=None, use_next_redis=False
):
interfaces = _load_interfaces(config, hostname, use_next_redis=use_next_redis)
netdash_equipment = get_netdash_equipment(config, use_next_redis)
def filter_interface(interface: dict):
return all(
(
"phy" in interface["description"].lower(),
"spare" not in interface["description"].lower(),
"non-operational" not in interface["description"].lower(),
"reserved" not in interface["description"].lower(),
"test" not in interface["description"].lower(),
"dsc." not in interface["name"].lower(),
"fxp" not in interface["name"].lower(),
)
)
def transform_interface(interface: dict) -> Dict:
return {
"router": interface["router"],
"name": interface["name"],
"description": interface["description"],
"vendor": netdash_equipment.get(interface["router"]),
}
return sorted(
map(transform_interface, filter(filter_interface, interfaces)),
key=lambda i: (i["router"], i["name"]),
)
@routes.route("/error-report-interfaces", methods=['GET'])
@routes.route('/error-report-interfaces/<hostname>', methods=['GET'])
@common.require_accepts_json
def error_report_interfaces(hostname=None):
"""
Handler for `/poller/error-report-interfaces` and
`/poller/error-report-interfaces/<hostname>`
which returns information for either all interfaces
or those on the requested hostname.
The optional `no-lab` parameter omits lab routers
if it's truthiness evaluates to True.
The response is a list of information for all
interfaces that should be included in the neteng error report
and includes vendor information (either juniper or nokia)
.. asjson::
inventory_provider.routes.poller.ERROR_REPORT_INTERFACE_LIST_SCHEMA
:param hostname: optional, if present should be a router hostname
:return:
"""
suffix = hostname or "all"
cache_key = f'classifier-cache:error-report-interfaces:{suffix}'
r = common.get_current_redis()
result = _ignore_cache_or_retrieve(request, cache_key, r)
if not result:
interfaces = load_error_report_interfaces(
current_app.config['INVENTORY_PROVIDER_CONFIG'], hostname
)
result = json.dumps(interfaces).encode('utf-8')
# cache this data for the next call
r.set(cache_key, result)
if not result or result == b'[]':
return Response(
response='no interfaces found',
status=404,
mimetype='text/plain'
)
return Response(result, mimetype="application/json")
def interface_speed(ifc):
"""
Return the maximum bits per second expected for the given interface.
cf. https://www.juniper.net/documentation/us/en/software/
vmx/vmx-getting-started/topics/task/
vmx-chassis-interface-type-configuring.html
:param ifc:
:return: an integer bits per second
"""
def _name_to_speed(ifc_name):
if ifc_name.startswith('ge'):
return Gb
if ifc_name.startswith('xe'):
return 10 * Gb
if ifc_name.startswith('et'):
return 400 * Gb
logger.warning(f'unrecognized interface name: {ifc_name}')
return -1
def _get_speed(ifc):
rate_conversions = {
"mbps": Mb,
"gbps": Gb
}
if "speed" in ifc:
speed = ifc["speed"]
rate_match = re.match(r"(\d+)(.+)", speed)
if rate_match:
value = int(rate_match.group(1))
rate = rate_match.group(2).strip().lower()
if rate in rate_conversions:
return int(value * rate_conversions[rate])
else:
logger.warning(f'unrecognised rate: {rate}, using _name_to_speed fallback')
return _name_to_speed(ifc['name'])
else:
oc_match = re.match(r"OC(\d+)", speed)
if oc_match:
value = int(oc_match.group(1))
return int(value * OC)
else:
logger.warning(f'unrecognised speed: {speed}, using _name_to_speed fallback')
return _name_to_speed(ifc['name'])
else:
logger.warning('no speed data for interface, using _name_to_speed fallback')
return _name_to_speed(ifc['name'])
if ifc['bundle-parents']:
if not ifc['name'].startswith('ae'):
logger.warning(
f'ifc has bundle-parents, but name is {ifc["name"]}')
return int(sum(_get_speed(parent_ifc) for parent_ifc in ifc['bundle-parents']))
return int(_get_speed(ifc))
def _load_interfaces_and_speeds(hostname=None):
"""
prepares the result of a call to /speeds
:param hostname: hostname or None for all
:return: generator yielding interface elements
"""
no_lab = common.get_bool_request_arg('no-lab', False)
basic_interfaces = _load_interfaces(
current_app.config['INVENTORY_PROVIDER_CONFIG'],
hostname,
no_lab=no_lab)
basic_interfaces_with_speeds = _add_speeds(basic_interfaces)
with_bundles = _add_bundle_parents(list(basic_interfaces_with_speeds), hostname)
def _result_ifc(ifc):
return {
'router': ifc['router'],
'name': ifc['name'],
'speed': interface_speed(ifc)
}
return map(_result_ifc, with_bundles)
@routes.route("/speeds", methods=['GET'])
@routes.route('/speeds/<hostname>', methods=['GET'])
@common.require_accepts_json
def interface_speeds(hostname=None):
"""
Handler for `/poller/speeds` and
`/poller/speeds/<hostname>`
which returns information for either all interfaces
or those on the requested hostname.
The response is a list of maximum speed information (in bits
per second) for all known interfaces.
*speed <= 0 means the max interface speed can't be determined*
.. asjson::
inventory_provider.routes.poller.INTERFACE_SPEED_LIST_SCHEMA
:param hostname: optional, if present should be a router hostname
:return:
"""
cache_key = f'classifier-cache:poller-interface-speeds:{hostname}' \
if hostname else 'classifier-cache:poller-interface-speeds:all'
r = common.get_current_redis()
result = _ignore_cache_or_retrieve(request, cache_key, r)
if not result:
result = list(_load_interfaces_and_speeds(hostname))
if not result:
return Response(
response='no interfaces found',
status=404,
mimetype='text/html')
result = json.dumps(result)
# cache this data for the next call
r.set(cache_key, result.encode('utf-8'))
return Response(result, mimetype="application/json")
def _load_community_strings(base_key_pattern):
for doc in _load_netconf_docs(
config=current_app.config['INVENTORY_PROVIDER_CONFIG'],
filter_pattern=base_key_pattern):
community = juniper.snmp_community_string(doc['netconf'])
if not community:
yield {
'router': doc['router'],
'error':
f'error extracting community string for {doc["router"]}'
}
else:
yield {
'router': doc['router'],
'community': community
}
@routes.route('/eumetsat-multicast', methods=['GET'])
@routes.route('/eumetsat-multicast/<hostname>', methods=['GET'])
@common.require_accepts_json
def eumetsat_multicast(hostname=None):
"""
Handler for `/poller/eumetsat-multicast</hostname>` which returns
information about multicast subscriptions.
The hostname parameter is optional. If it is present, only hostnames
matching `hostname*` are returned. If not present, data for all
`mx*` routers is returned.
The response is a list of oid/router/community structures that all
all subscription octet counters to be polled.
.. asjson::
inventory_provider.routes.poller.MULTICAST_SUBSCRIPTION_LIST_SCHEMA
This method returns essentially hard-coded data,
based on the information in POL1-395.
:return:
"""
SUBSCRIPTIONS = [{
'subscription': f'232.223.222.{idx}',
'endpoint': '193.17.9.3',
} for idx in range(1, 73)]
SUBSCRIPTIONS.append(
{'subscription': '232.223.223.1', 'endpoint': '193.17.9.7'})
SUBSCRIPTIONS.append(
{'subscription': '232.223.223.22', 'endpoint': '193.17.9.7'})
def _oid(sub):
return ('1.3.6.1.2.1.83.1.1.2.1.16'
f'.{sub["subscription"]}.{sub["endpoint"]}'
'.255.255.255.255')
r = common.get_current_redis()
cache_key = 'classifier-cache:poller-eumetsat-multicast'
if hostname:
cache_key = f'{cache_key}:{hostname}'
result = r.get(cache_key)
if result:
result = result.decode('utf-8')
else:
def _multicast_oids(router_info):
def _rsp_element(sub):
result = {
'router': router_info['router'],
'oid': _oid(sub),
'community': router_info['community']
}
result.update(sub)
return result
yield from map(_rsp_element, SUBSCRIPTIONS)
routers = list(_load_community_strings(
base_key_pattern=f'netconf:{hostname}*'
if hostname else 'netconf:mx*'))
errors = list(filter(lambda x: 'error' in x, routers))
if errors:
errors = [e['error'] for e in errors]
return Response(
response=', '.join(errors),
status=403, # forbidden
mimetype='text/html')
assert all('community' in r for r in routers) # sanity
result = list(map(_multicast_oids, routers))
result = itertools.chain(*result)
result = list(result)
if not result:
target = hostname or 'any routers!'
return Response(
response=f'no multicast config for {target}',
status=404,
mimetype='text/html')
result = json.dumps(result)
# cache this data for the next call
r.set(cache_key, result.encode('utf-8'))
return Response(result, mimetype="application/json")
@routes.route("/gws/direct", methods=['GET'])
@common.require_accepts_json
def gws_direct():
"""
Handler for `/poller/gws/direct` which returns required for polling
customer equipment counters for ISP connetions.
The response is a list of nren/isp/counter structures that must be
polled.
.. asjson::
inventory_provider.routes.poller.GWS_DIRECT_DATA_SCHEMA
WARNING: interface tags in the `gws-direct` section of the config data
should be unique for each nren/isp combination. i.e. if there
are multiple community strings in use for a particular host, then please
keep the interface tags unique.
:return:
"""
cache_key = 'classifier-cache:gws-direct'
r = common.get_current_redis()
result = r.get(cache_key)
if result:
result = result.decode('utf-8')
else:
def _interfaces():
config_params = current_app.config['INVENTORY_PROVIDER_CONFIG']
for nren_isp in config_params['gws-direct']:
for host in nren_isp['hosts']:
snmp_params = {}
if 'community' in host:
# (snmp v2)
# sanity (already guaranteed by schema check)
assert 'sec-name' not in host
snmp_params['community'] = host['community']
else:
# (snmp v3)
# sanity (already guaranteed by schema check)
assert 'sec-name' in host
snmp_params['sec-name'] = host['sec-name']
if 'auth' in host:
snmp_params['auth'] = host['auth']
if 'priv' in host:
snmp_params['priv'] = host['priv']
for ifc in host['interfaces']:
ifc_data = {
'nren': nren_isp['nren'],
'isp': nren_isp['isp'],
'hostname': host['hostname'],
'tag': ifc['tag'],
'counters': [
{
'field': k,
'oid': v,
'snmp': snmp_params
}
for k, v in ifc['counters'].items()]
}
if 'info' in ifc:
ifc_data['info'] = ifc['info']
yield ifc_data
result = json.dumps(list(_interfaces()))
# cache this data for the next call
r.set(cache_key, result.encode('utf-8'))
return Response(result, mimetype="application/json")
# cf. https://gitlab.geant.net/puppet-apps/cacti/-/blob/production/files/scripts/juniper-firewall-dws.pl # noqa: E501
JNX_DCU_STATS_BYTES_OID = '1.3.6.1.4.1.2636.3.6.2.1.5'
JNX_FW_COUNTER_BYTES_OID = '1.3.6.1.4.1.2636.3.5.2.1.5'
JNX_ADDRESS_FAMILY = {
'ipv4': 1,
'ipv6': 2
}
JNX_FW_COUNTER_TYPE = {
'other': 1,
'counter': 2,
'policer': 3
}
def _str2oid(s):
chars = '.'.join(str(ord(c)) for c in s)
return f'{len(s)}.{chars}'
def _jnx_dcu_byte_count_oid(
ifIndex,
class_name='dws-in',
address_family=JNX_ADDRESS_FAMILY['ipv4']):
# sanity checks (in case of programming errors)
assert isinstance(ifIndex, int)
assert isinstance(class_name, str)
assert isinstance(address_family, int)
return '.'.join([
JNX_DCU_STATS_BYTES_OID,
str(ifIndex),
str(address_family),
_str2oid(class_name)
])
def _jnx_fw_counter_bytes_oid(
customer,
interface_name,
filter_name=None,
counter_name=None):
# sanity checks (in case of programming errors)
assert isinstance(customer, str)
assert isinstance(interface_name, str)
assert filter_name is None or isinstance(filter_name, str)
assert counter_name is None or isinstance(counter_name, str)
if filter_name is None:
filter_name = f'nren_IAS_{customer}_OUT-{interface_name}-o'
if counter_name is None:
counter_name = f'DWS-out-{interface_name}-o'
return '.'.join([
JNX_FW_COUNTER_BYTES_OID,
_str2oid(filter_name),
_str2oid(counter_name),
str(JNX_FW_COUNTER_TYPE['counter'])
])
def _get_services_internal(service_type=None):
"""
Performs the lookup and caching done for calls to
`/poller/services</service-type>`
This is a separate private utility so that it can be called by
:meth:`inventory_provider.routes.poller.get_services`
and :meth:`inventory_provider.routes.poller.get_service_types`
The response will be formatted according to the following schema:
.. asjson::
inventory_provider.routes.poller.SERVICES_LIST_SCHEMA
:param service_type: a service type, or None to return all
:return: service list, json-serialized to a string
"""
return_all = common.get_bool_request_arg('all', True)
include_snmp = common.get_bool_request_arg('snmp', False)
def _services():
key_pattern = f'ims:services:{service_type}:*' \
if service_type else 'ims:services:*'
for doc in common.load_json_docs(
config_params=current_app.config['INVENTORY_PROVIDER_CONFIG'],
key_pattern=key_pattern,
num_threads=20):
yield doc['value']
def _add_snmp(s, all_snmp_info):
snmp_interfaces = all_snmp_info.get(s['hostname'], {})
interface_info = snmp_interfaces.get(s['interface'], None)
if interface_info:
s['snmp'] = {
'ifIndex': interface_info['index'],
'community': interface_info['community'],
}
if s['type'] == 'GWS - INDIRECT':
s['snmp']['counters'] = [
{
'field': 'ingressOctets',
'oid': _jnx_dcu_byte_count_oid(
interface_info['index']),
},
{
'field': 'egressOctets',
'oid': _jnx_fw_counter_bytes_oid(
s['customer'], s['interface'])
}
]
return s
def _wanted_in_output(s):
return return_all or (s['status'].lower() == 'operational')
def _format_services(s):
return {
'id': s['id'],
'name': s['name'],
'customer': s['project'],
'speed': s['speed_value'],
'pop': s['here']['pop']['name'],
'hostname': common.ims_equipment_to_hostname(
s['here']['equipment']),
'interface': s['here']['port'].lower(),
'type': s['type'],
'status': s['status']
}
cache_key = f'classifier-cache:poller:services:{service_type}' \
if service_type else 'classifier-cache:poller:services:all-types'
if return_all:
cache_key = f'{cache_key}:all'
if include_snmp:
cache_key = f'{cache_key}:snmp'
redis = common.get_current_redis()
result = _ignore_cache_or_retrieve(request, cache_key, redis)
if not result:
result = _services()
result = filter(_wanted_in_output, result)
result = map(_format_services, result)
if include_snmp:
all_snmp_info = common.load_snmp_indexes(current_app.config['INVENTORY_PROVIDER_CONFIG'])
result = map(partial(_add_snmp, all_snmp_info=all_snmp_info), result)
result = list(result)
if not result:
return None
# cache this data for the next call
result = json.dumps(result)
redis.set(cache_key, result)
return result
@routes.route('/services', methods=['GET'])
@routes.route('/services/<service_type>', methods=['GET'])
@common.require_accepts_json
def get_services(service_type=None):
"""
Handler for `/poller/services</service-type>`
Use `/poller/service-types` for possible values of `service-type`.
If the endpoint is called with no `service-type`,
all services are returned.
Supported url parameters:
:all: Default is False to return only operational services.
If present and evaluates to True, then return all services.
:snmp: If present and evalutes to True, add snmp interface polling
params to the response.
(sphinx bug: the param names aren't capitalized)
The response will be formatted according to the following schema:
.. asjson::
inventory_provider.routes.poller.SERVICES_LIST_SCHEMA
A few possible values for `service_type` are (currently): gws_internal,
geant_ip, geant_lambda, l3_vpn, md_vpn_proxy (there are more)
If the endpoint is called with no `service_type`,
all services are returned
:return:
"""
services_json_str = _get_services_internal(service_type)
if not services_json_str:
message = f'no {service_type} services found' \
if service_type else 'no services found'
return Response(
response=message,
status=404,
mimetype='text/html')
return Response(services_json_str, mimetype='application/json')
@routes.route("/gws/indirect", methods=['GET'])
@common.require_accepts_json
def gws_indirect():
"""
Handler for `/poller/gws/indirect`
Same as `/poller/services/gws_indirect`
cf. :meth:`inventory_provider.routes.poller.get_services`
:return:
"""
return get_services(service_type='gws_indirect')
@routes.route('/service-types', methods=['GET'])
@common.require_accepts_json
def get_service_types():
"""
Handler for `/poller/service-types`
This method returns a list of all values of `service_type` that
can be used with `/poller/services/service-type` to return
a non-empty list of services.
Adding a truthy `all` request parameter (e.g. `?all=1`) will
return also return valid service types for which none of the
defined services are operational.
The response will be formatted according to the following schema:
.. asjson::
inventory_provider.routes.poller.STRING_LIST_SCHEMA
"""
cache_key = 'classifier-cache:poller:service-types'
return_all = common.get_bool_request_arg('all', False)
if return_all:
cache_key = f'{cache_key}-all'
redis = common.get_current_redis()
service_types = _ignore_cache_or_retrieve(request, cache_key, redis)
if not service_types:
all_services = json.loads(_get_services_internal())
service_types = {
ims_sorted_service_type_key(s['type'])
for s in all_services
}
if not service_types:
return Response(
response='no service types found',
status=404,
mimetype='text/html')
# cache this data for the next call
service_types = sorted(list(service_types))
service_types = json.dumps(service_types)
redis.set(cache_key, service_types)
return Response(service_types, mimetype='application/json')
@routes.route('/gws/direct-config', methods=['GET'])
def gws_direct_config():
"""
Handler for `/poller/gws/direct-config` which returns
the basic gws-direct config.
This api is only intended for config validation.
:return:
"""
wanted = request.args.get('format', default='json', type=str)
wanted = wanted.lower()
if wanted not in ('html', 'json'):
return Response(
response='format must be one of: html, json',
status=400,
mimetype="text/html")
def _counters():
config_params = current_app.config['INVENTORY_PROVIDER_CONFIG']
for nren_isp in config_params['gws-direct']:
for host in nren_isp['hosts']:
snmp_version = '2' if 'community' in host.keys() else '3'
for ifc in host['interfaces']:
for field, oid in ifc['counters'].items():
yield {
'nren': nren_isp['nren'],
'isp': nren_isp['isp'],
'hostname': host['hostname'],
'snmp': snmp_version,
'interface': ifc['tag'],
'field': field,
'oid': oid,
'info': ifc.get('info', '')
}
if wanted == 'json':
if not request.accept_mimetypes.accept_json:
return Response(
response="response will be json",
status=406,
mimetype="text/html")
else:
return jsonify(list(_counters()))
if not request.accept_mimetypes.accept_html:
return Response(
response="response will be html",
status=406,
mimetype="text/html")
page = '''<!doctype html>
<html lang="en">
<head>
<!-- Required meta tags -->
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no">
<!-- Bootstrap CSS -->
<link rel="stylesheet" href="https://stackpath.bootstrapcdn.com/bootstrap/4.3.1/css/bootstrap.min.css" integrity="sha384-ggOyR0iXCbMQv3Xipma34MD+dH/1fQ784/j6cY/iJTQUOhcWr7x9JvoRxT2MZw1T" crossorigin="anonymous">
<title>GWS Direct config</title>
</head>
<body>
<table class="table table-striped">
<thead>
{header_row}
</thead>
<tbody>
{data_rows}
</tbody>
</table>
</body>
</html>''' # noqa: E501
def _to_row(counter, header=False):
_columns = (
'nren', 'isp', 'hostname',
'snmp', 'interface', 'field', 'oid', 'info')
elems = ['<tr>']
for name in _columns:
if header:
elems.append(f'<th scope="col">{name}</th>')
else:
elems.append(f'<td scope="row">{counter[name]}</td>')
elems.append('</tr>')
return ''.join(elems)
header_row = _to_row(None, header=True)
data_rows = map(_to_row, _counters())
page = page.format(
header_row=header_row,
data_rows='\n'.join(data_rows))
return Response(
response=page,
status=200,
mimetype="text/html")