"""
BRIAN support Endpoints
=========================
These endpoints are intended for use by BRIAN.
.. contents:: :local:
/poller/interfaces</hostname>
---------------------------------
.. autofunction:: inventory_provider.routes.poller.interfaces
/poller/speeds</hostname>
---------------------------------
.. autofunction:: inventory_provider.routes.poller.interface_speeds
/poller/eumetsat-multicast
---------------------------------
.. autofunction:: inventory_provider.routes.poller.eumetsat_multicast
/poller/gws/direct
---------------------------------
.. autofunction:: inventory_provider.routes.poller.gws_direct
/poller/gws/direct-config
---------------------------------
.. autofunction:: inventory_provider.routes.poller.gws_direct_config
/poller/gws/indirect
---------------------------------
.. autofunction:: inventory_provider.routes.poller.gws_indirect
/poller/services</service-type>
---------------------------------
.. autofunction:: inventory_provider.routes.poller.get_services
/poller/service-types
---------------------------------
.. autofunction:: inventory_provider.routes.poller.get_service_types
support method: _get_dashboards
---------------------------------
.. autofunction:: inventory_provider.routes.poller._get_dashboards
"""
from collections import defaultdict
from enum import Enum, auto
import itertools
import json
import logging
import re
from functools import partial
from flask import Blueprint, Response, current_app, request, jsonify
from inventory_provider import juniper
from inventory_provider.routes import common
from inventory_provider.tasks.common import ims_sorted_service_type_key
from inventory_provider.tasks import common as tasks_common
from inventory_provider.routes.classifier import get_ims_equipment_name, \
get_ims_interface
from inventory_provider.routes.common import _ignore_cache_or_retrieve
logger = logging.getLogger(__name__)
routes = Blueprint('poller-support-routes', __name__)
Mb = 1 << 20
Gb = 1 << 30
class INTERFACE_TYPES(Enum):
UNKNOWN = auto()
LOGICAL = auto()
PHYSICAL = auto()
AGGREGATE = auto()
class BRIAN_DASHBOARDS(Enum):
CLS = auto()
RE_PEER = auto()
RE_CUST = auto()
GEANTOPEN = auto()
GCS = auto()
L2_CIRCUIT = auto()
LHCONE_PEER = auto()
LHCONE_CUST = auto()
MDVPN_CUSTOMERS = auto()
INFRASTRUCTURE_BACKBONE = auto()
IAS_PRIVATE = auto()
IAS_PUBLIC = auto()
IAS_CUSTOMER = auto()
IAS_UPSTREAM = auto()
GWS_PHY_UPSTREAM = auto()
GBS_10G = auto()
# aggregate dashboards
CLS_PEERS = auto()
IAS_PEERS = auto()
GWS_UPSTREAMS = auto()
LHCONE = auto()
CAE1 = auto()
COPERNICUS = auto()
# NREN customer
NREN = auto()
class PORT_TYPES(Enum):
ACCESS = auto()
SERVICE = auto()
UNKNOWN = auto()
# only used in INTERFACE_LIST_SCHEMA and sphinx docs
_DASHBOARD_IDS = [d.name for d in list(BRIAN_DASHBOARDS)]
_PORT_TYPES = [t.name for t in list(PORT_TYPES)]
_INTERFACE_TYPES = [i.name for i in list(INTERFACE_TYPES)]
INTERFACE_LIST_SCHEMA = {
'$schema': 'https://json-schema.org/draft-07/schema#',
'definitions': {
'service': {
'type': 'object',
'properties': {
'id': {'type': 'integer'},
'name': {'type': 'string'},
'type': {'type': 'string'},
'status': {'type': 'string'},
},
'required': ['id', 'name', 'type', 'status'],
'additionalProperties': False
},
'db_info': {
'type': 'object',
'properties': {
'name': {'type': 'string'},
'interface_type': {'enum': _INTERFACE_TYPES}
},
'required': ['name', 'interface_type'],
'additionalProperties': False
},
'interface': {
'type': 'object',
'properties': {
'router': {'type': 'string'},
'name': {'type': 'string'},
'description': {'type': 'string'},
'snmp-index': {
'type': 'integer',
'minimum': 1
},
'bundle': {
'type': 'array',
'items': {'type': 'string'}
},
'bundle-parents': {
'type': 'array',
'items': {'type': 'string'}
},
'circuits': {
'type': 'array',
'items': {'$ref': '#/definitions/service'}
},
'dashboards': {
'type': 'array',
'items': {'enum': _DASHBOARD_IDS}
},
'dashboard_info': {
'$ref': '#/definitions/db_info',
},
'dashboards_info': {
'type': 'array',
'items': {'$ref': '#/definitions/db_info'}
},
'port_type': {'enum': _PORT_TYPES}
},
'required': [
'router', 'name', 'description',
'snmp-index', 'bundle', 'bundle-parents',
'circuits', 'dashboards', 'port_type'],
'additionalProperties': False
},
},
'type': 'array',
'items': {'$ref': '#/definitions/interface'}
}
INTERFACE_SPEED_LIST_SCHEMA = {
'$schema': 'https://json-schema.org/draft-07/schema#',
'definitions': {
'interface': {
'type': 'object',
'properties': {
'router': {'type': 'string'},
'name': {'type': 'string'},
'speed': {'type': 'integer'}
},
'required': ['router', 'name', 'speed'],
'additionalProperties': False
},
},
'type': 'array',
'items': {'$ref': '#/definitions/interface'}
}
MULTICAST_SUBSCRIPTION_LIST_SCHEMA = {
'$schema': 'https://json-schema.org/draft-07/schema#',
'definitions': {
'ipv4-address': {
'type': 'string',
'pattern': r'^(\d+\.){3}\d+$'
},
'subscription': {
'type': 'object',
'properties': {
'router': {'type': 'string'},
'subscription': {'$ref': '#/definitions/ipv4-address'},
'endpoint': {'$ref': '#/definitions/ipv4-address'},
'oid': {
'type': 'string',
'pattern': r'^(\d+\.)*\d+$'
},
'community': {'type': 'string'}
},
'required': [
'router', 'subscription', 'endpoint', 'oid', 'community'],
'additionalProperties': False
},
},
'type': 'array',
'items': {'$ref': '#/definitions/subscription'}
}
GWS_DIRECT_DATA_SCHEMA = {
'$schema': 'https://json-schema.org/draft-07/schema#',
'definitions': {
'oid': {
'type': 'string',
'pattern': r'^(\d+\.)*\d+$'
},
'snmp-v2': {
'type': 'object',
'properties': {
'community': {'type': 'string'}
},
'required': ['community'],
'additionalProperties': False
},
'snmp-v3-cred': {
'type': 'object',
'properties': {
'protocol': {'enum': ['MD5', 'DES']},
'password': {'type': 'string'}
},
'required': ['protocol', 'password'],
'additionalProperties': False
},
'snmp-v3': {
'type': 'object',
'properties': {
'sec-name': {'type': 'string'},
'auth': {'$ref': '#/definitions/snmp-v3-cred'},
'priv': {'$ref': '#/definitions/snmp-v3-cred'}
},
'required': ['sec-name'],
'additionalProperties': False
},
'counter': {
'type': 'object',
'properties': {
'field': {
'enum': [
'discards_in',
'discards_out',
'errors_in',
'errors_out',
'traffic_in',
'traffic_out'
]
},
'oid': {'$ref': '#/definitions/oid'},
'snmp': {
'oneOf': [
{'$ref': '#/definitions/snmp-v2'},
{'$ref': '#/definitions/snmp-v3'}
]
}
},
'required': ['field', 'oid', 'snmp'],
'additionalProperties': False
},
'interface-counters': {
'type': 'object',
'properties': {
'nren': {'type': 'string'},
'isp': {
'type': 'string',
'enum': ['Cogent', 'Telia', 'CenturyLink']
},
'hostname': {'type': 'string'},
'tag': {'type': 'string'},
'counters': {
'type': 'array',
'items': {'$ref': '#/definitions/counter'},
'minItems': 1
},
'info': {'type': 'string'}
},
'required': [
'nren', 'isp', 'hostname', 'tag', 'counters'],
'additionalProperties': False
}
},
'type': 'array',
'items': {'$ref': '#/definitions/interface-counters'}
}
SERVICES_LIST_SCHEMA = {
'$schema': 'https://json-schema.org/draft-07/schema#',
'definitions': {
'oid': {
'type': 'string',
'pattern': r'^(\d+\.)*\d+$'
},
'counter-field': {
'type': 'object',
'properties': {
'field': {
'type': 'string',
'enum': ['egressOctets', 'ingressOctets']
},
'oid': {'$ref': '#/definitions/oid'}
},
'required': ['field', 'oid'],
'additionalProperties': False
},
'snmp-info': {
'type': 'object',
'properties': {
'ifIndex': {'type': 'integer'},
'community': {'type': 'string'},
'counters': {
'type': 'array',
'items': {'$ref': '#/definitions/counter-field'},
'minItems': 1
}
},
'required': ['ifIndex', 'community'],
'additionalProperties': False
},
'service': {
'type': 'object',
'properties': {
'id': {'type': 'integer'},
'name': {'type': 'string'},
'customer': {'type': 'string'},
'speed': {'type': 'integer'},
'pop': {'type': 'string'},
'hostname': {'type': 'string'},
'interface': {'type': 'string'},
'type': {'type': 'string'},
'status': {'type': 'string'},
'snmp': {'$ref': '#/definitions/snmp-info'}
},
'required': [
'id', 'name', 'customer',
'speed', 'pop', 'hostname',
'interface', 'type', 'status'],
'additionalProperties': False
}
},
'type': 'array',
'items': {'$ref': '#/definitions/service'}
}
STRING_LIST_SCHEMA = {
'$schema': 'https://json-schema.org/draft-07/schema#',
'type': 'array',
'items': {'type': 'string'}
}
@routes.after_request
def after_request(resp):
return common.after_request(resp)
def _get_dashboards(interface):
"""
Yield enums from BRIAN_DASHBOARDS to indicate which dashboards
this interface should be included in.
cf. POL1-482
Possible dashboard id's are:
.. asjson::
inventory_provider.routes.poller._DASHBOARD_IDS
:param interface: a dict with keys like router, name, description
:return: generator that yields enums from BRIAN_DASHBOARDS
"""
router = interface.get('router', '').lower()
ifc_name = interface.get('name', '')
description = interface.get('description', '').strip()
if 'SRV_L3VPN' in description and 'COPERNICUS' in description:
yield BRIAN_DASHBOARDS.COPERNICUS
if re.match(r'SRV_CLS\s', description):
yield BRIAN_DASHBOARDS.CLS
if re.match(r'SRV_CLS PRIVATE\s', description):
yield BRIAN_DASHBOARDS.CLS_PEERS
if re.match(r'SRV_IAS PUBLIC\s', description):
yield BRIAN_DASHBOARDS.IAS_PUBLIC
yield BRIAN_DASHBOARDS.IAS_PEERS
if re.match(r'SRV_IAS PRIVATE\s', description):
yield BRIAN_DASHBOARDS.IAS_PRIVATE
yield BRIAN_DASHBOARDS.IAS_PEERS
if re.match(r'SRV_IAS CUSTOMER\s', description):
yield BRIAN_DASHBOARDS.IAS_CUSTOMER
if re.match(r'SRV_IAS UPSTREAM\s', description):
yield BRIAN_DASHBOARDS.IAS_UPSTREAM
if re.match(r'SRV_10GGBS CUSTOMER\s', description):
yield BRIAN_DASHBOARDS.GBS_10G
if re.match(r'(SRV_GLOBAL|SRV_L3VPN|LAG) RE_INTERCONNECT\s', description):
yield BRIAN_DASHBOARDS.RE_PEER
if re.match(r'(PHY|LAG|SRV_GLOBAL) CUSTOMER\s', description):
yield BRIAN_DASHBOARDS.RE_CUST
if re.match('^SRV_GCS', description):
yield BRIAN_DASHBOARDS.GCS
if re.match(r'PHY CUSTOMER_GEO\s', description):
yield BRIAN_DASHBOARDS.GEANTOPEN
if 'SRV_L2CIRCUIT' in description:
yield BRIAN_DASHBOARDS.L2_CIRCUIT
if 'LHCONE' in description:
if 'SRV_L3VPN RE' in description:
yield BRIAN_DASHBOARDS.LHCONE_PEER
if 'SRV_L3VPN CUSTOMER' in description:
yield BRIAN_DASHBOARDS.LHCONE_CUST
if re.match(r'SRV_L3VPN (CUSTOMER|RE_INTERCONNECT)\s', description):
if '-LHCONE' in description:
# `-LHCONE` can be preceded by tons of different stuff, so it's
# simpler to check for this rather than a full regex
yield BRIAN_DASHBOARDS.LHCONE
if re.match(r'SRV_MDVPN CUSTOMER\s', description):
yield BRIAN_DASHBOARDS.MDVPN_CUSTOMERS
if re.match(r'(SRV_GLOBAL|LAG|PHY) INFRASTRUCTURE BACKBONE', description):
yield BRIAN_DASHBOARDS.INFRASTRUCTURE_BACKBONE
if router == 'mx1.lon.uk.geant.net' \
and re.match(r'^ae12(\.\d+|$)$', ifc_name):
yield BRIAN_DASHBOARDS.CAE1
if re.match(r'PHY UPSTREAM\s', description):
yield BRIAN_DASHBOARDS.GWS_PHY_UPSTREAM
regex = r'(PHY|LAG|(SRV_(GLOBAL|LHCONE|MDVPN|IAS|CLS|L3VPN))) CUSTOMER\s'
if re.match(regex, description):
yield BRIAN_DASHBOARDS.NREN
def _get_dashboard_data(ifc, customers):
def _get_interface_type(description):
if re.match(r'^PHY', description):
return INTERFACE_TYPES.PHYSICAL
if re.match(r'^SRV_', description):
return INTERFACE_TYPES.LOGICAL
if re.match(r'^LAG', description):
return INTERFACE_TYPES.AGGREGATE
return INTERFACE_TYPES.UNKNOWN
description = ifc.get('description', '').strip()
dashboards = ifc.get('dashboards', [])
interface_type = _get_interface_type(description)
if len(dashboards) == 0:
return ifc
def _get_customer_name(description):
name = description.split(' ')
if len(name) >= 3:
return name[2].strip().upper()
else:
# if the description isn't properly formatted
# use it as the name to make it obvious something is wrong
return description
def _get_backbone_name(description):
name = description.split('|')
if len(name) >= 2:
name = name[1].strip()
return name.replace('( ', '(')
else:
# if the description isn't properly formatted
# use it as the name to make it obvious something is wrong
return description
if BRIAN_DASHBOARDS.INFRASTRUCTURE_BACKBONE.name in dashboards:
names = [_get_backbone_name(description)]
elif BRIAN_DASHBOARDS.GWS_PHY_UPSTREAM.name in dashboards:
host = ifc['router']
location = host.split('.')[1].upper()
names = [f'{_get_customer_name(description)} - {location}']
elif BRIAN_DASHBOARDS.L2_CIRCUIT.name in dashboards:
# This will give names derived from the Interface Description
# names = list(_get_l2_customer_names(description))
# This will give first 2 names IMS
names = [c['name'] for c in customers][:2]
else:
names = [_get_customer_name(description)]
# if no customers found just return the original data
if not names:
return ifc
# to maintain compatability with current brian dashboard manager we will
# continue to return dashboard_info with the first customer name. We will
# also return dashboards_info (note the plural of dashboards) with up to
# two customers
return {
**ifc,
'dashboard_info': {
'name': names[0],
'interface_type': interface_type.name
},
'dashboards_info': [{
'name': name,
'interface_type': interface_type.name
} for name in set(names)]
}
def _load_interface_bundles(config, hostname=None, use_next_redis=False):
result = dict()
def _load_docs(key_pattern):
for doc in common.load_json_docs(
config_params=config,
key_pattern=key_pattern,
num_threads=20,
use_next_redis=use_next_redis):
m = re.match(
r'.*netconf-interface-bundles:([^:]+):(.+)', doc['key'])
assert m
router = m.group(1)
interface = m.group(2)
result.setdefault(router, dict())
result[router][interface] = doc['value']
base_key = 'netconf-interface-bundles'
base_key_pattern = f'{base_key}:{hostname}:*' \
if hostname else f'{base_key}:*'
_load_docs(base_key_pattern)
_load_docs(f'lab:{base_key_pattern}')
return result
def _get_services_and_customers(config, hostname=None, use_next_redis=False):
if hostname:
hostname = get_ims_equipment_name(hostname)
result = defaultdict(dict)
key_pattern = f'ims:interface_services:{hostname}:*' \
if hostname else 'ims:interface_services:*'
for doc in common.load_json_docs(
config_params=config,
key_pattern=key_pattern,
num_threads=20,
use_next_redis=use_next_redis):
cs = {
'services': [],
'customers': []
}
included_service_ids = set()
for s in doc['value']:
if s['id'] in included_service_ids:
continue
if s.get('port_type', '') == 'ab':
continue
included_service_ids.add(s['id'])
cs['customers'].append({
'name': s['customer'],
'type': 'UNKNOWN'
})
for c in s.get('additional_customers', []):
cs['customers'].append({
'name': c['name'],
'type': c.get('type', 'UNKNOWN')
})
if s['circuit_type'] == 'service':
cs['services'].append({
'id': s['id'],
'name': s['name'],
'type': s['service_type'],
'status': s['status'],
})
result[s['equipment']][s['port']] = cs
return result
def _load_netconf_docs(
config, filter_pattern, use_next_redis=False):
"""
yields dicts like:
{
'router': router hostname
'netconf': loaded netconf xml doc
}
:param config: app config
:param filter_pattern: search filter, including 'netconf:'
:param use_next_redis: use next instead of current redis, if true
:return: yields netconf docs, formatted as above
"""
m = re.match(r'^(.*netconf:).+', filter_pattern)
# TODO: probably better to not required netconf: to be passed in
assert m # sanity
key_prefix_len = len(m.group(1))
assert key_prefix_len >= len('netconf:') # sanity
for doc in common.load_xml_docs(
config_params=config,
key_pattern=filter_pattern,
num_threads=10,
use_next_redis=use_next_redis):
yield {
'router': doc['key'][key_prefix_len:],
'netconf': doc['value']
}
def _load_interfaces(
config, hostname=None, no_lab=False, use_next_redis=False):
"""
loads basic interface data for production & lab routers
:param config:
:param hostname:
:param use_next_redis:
:return:
"""
def _load_docs(key_pattern):
for doc in _load_netconf_docs(config, key_pattern, use_next_redis):
for ifc in juniper.list_interfaces(doc['netconf']):
if not ifc['description']:
continue
yield {
'router': doc['router'],
'name': ifc['name'],
'bundle': ifc['bundle'],
'bundle-parents': [],
'description': ifc['description'],
'circuits': []
}
base_key_pattern = f'netconf:{hostname}*' if hostname else 'netconf:*'
yield from _load_docs(base_key_pattern)
if not no_lab:
logger.debug('lab')
yield from _load_docs(f'lab:{base_key_pattern}')
def _add_speeds(config, interfaces):
all_netconf_interfaces = list(common.load_json_docs(config, 'netconf-interfaces:*'))
netconf_interface_index = {}
for netconf_interface_doc in all_netconf_interfaces:
nc_ifc = netconf_interface_doc['value']
if 'router' in nc_ifc and 'name' in nc_ifc:
netconf_interface_index[f"{nc_ifc['router']}---{nc_ifc['name']}"] = nc_ifc
for ifc in interfaces:
nc_ifc = netconf_interface_index.get(f"{ifc['router']}---{ifc['name']}", {})
if 'speed' in nc_ifc:
ifc['speed'] = nc_ifc['speed']
else:
ifc['speed'] = ''
yield ifc
def _add_bundle_parents(interfaces, hostname=None):
"""
generator that adds bundle-parents info to each interface.
:param interfaces: result of _load_interfaces
:param hostname: hostname or None for all
:return: generator with bundle-parents populated in each element
"""
bundles = _load_interface_bundles(
current_app.config['INVENTORY_PROVIDER_CONFIG'], hostname)
# create a quick look-up for interface details
interface_index = {f"{ifc['router']}---{ifc['name']}": ifc for ifc in interfaces}
for ifc in interfaces:
router_bundle = bundles.get(ifc['router'], None)
if router_bundle:
base_ifc = ifc['name'].split('.')[0]
bundle_parents = [interface_index.get(f"{ifc['router']}---{bundle_ifc}")
for bundle_ifc in router_bundle.get(base_ifc, [])]
ifc['bundle-parents'] = bundle_parents
yield ifc
def _get_port_type(description):
rex = re.search(r'\$([a-zA-Z]+\-\d+)', description)
if rex:
sid = rex.group(1)
if 'GA' in sid:
return PORT_TYPES.ACCESS.name
elif 'GS' in sid:
return PORT_TYPES.SERVICE.name
return PORT_TYPES.UNKNOWN.name
def load_interfaces_to_poll(
config, hostname=None, no_lab=False, use_next_redis=False):
basic_interfaces = \
list(_load_interfaces(config, hostname, no_lab, use_next_redis))
bundles = _load_interface_bundles(config, hostname, use_next_redis)
services_and_customers = \
_get_services_and_customers(config, hostname, use_next_redis)
snmp_indexes = common.load_snmp_indexes(config, hostname, use_next_redis)
def _get_populated_interfaces(all_interfaces):
if use_next_redis:
r = tasks_common.get_next_redis(config)
else:
r = tasks_common.get_current_redis(config)
for ifc in all_interfaces:
router_snmp = snmp_indexes.get(ifc['router'], None)
if router_snmp and ifc['name'] in router_snmp:
ifc['snmp-index'] = router_snmp[ifc['name']]['index']
router_bundle = bundles.get(ifc['router'], None)
if router_bundle:
base_ifc = ifc['name'].split('.')[0]
ifc['bundle-parents'] = router_bundle.get(base_ifc, [])
router_services_and_customers = services_and_customers.get(
get_ims_equipment_name(ifc['router'], r), {})
ifc_services_and_customers = \
router_services_and_customers.get(
get_ims_interface(ifc['name']), {}
)
if 'services' in ifc_services_and_customers \
and ifc_services_and_customers['services']:
ifc['circuits'] = ifc_services_and_customers['services']
dashboards = _get_dashboards(ifc)
ifc['dashboards'] = sorted([d.name for d in dashboards])
ifc = _get_dashboard_data(
ifc, ifc_services_and_customers.get('customers', []))
port_type = _get_port_type(ifc['description'])
ifc['port_type'] = port_type
yield ifc
else:
continue
return _get_populated_interfaces(basic_interfaces)
@routes.route("/interfaces", methods=['GET'])
@routes.route('/interfaces/<hostname>', methods=['GET'])
@common.require_accepts_json
def interfaces(hostname=None):
"""
Handler for `/poller/interfaces` and
`/poller/interfaces/<hostname>`
which returns information for either all interfaces
or those on the requested hostname.
The optional `no-lab` parameter omits lab routers
if it's truthiness evaluates to True.
The response is a list of information for all
interfaces that should be polled, including service
information and snmp information.
.. asjson::
inventory_provider.routes.poller.INTERFACE_LIST_SCHEMA
:meth:`inventory_provider.routes.poller._get_services`
is where dashboard mappings is handled.
:param hostname: optional, if present should be a router hostname
:return:
"""
cache_key = f'classifier-cache:poller-interfaces:{hostname}' \
if hostname else 'classifier-cache:poller-interfaces:all'
no_lab = common.get_bool_request_arg('no-lab', False)
if no_lab:
cache_key = f'{cache_key}:no_lab'
r = common.get_current_redis()
result = _ignore_cache_or_retrieve(request, cache_key, r)
if not result:
result = list(load_interfaces_to_poll(
current_app.config['INVENTORY_PROVIDER_CONFIG'], hostname, no_lab))
if not result:
return Response(
response='no interfaces found',
status=404,
mimetype='text/html')
result = json.dumps(result)
# cache this data for the next call
r.set(cache_key, result.encode('utf-8'))
return Response(result, mimetype="application/json")
def interface_speed(ifc):
"""
Return the maximum bits per second expected for the given interface.
cf. https://www.juniper.net/documentation/us/en/software/
vmx/vmx-getting-started/topics/task/
vmx-chassis-interface-type-configuring.html
:param ifc:
:return: an integer bits per second
"""
def _name_to_speed(ifc_name):
if ifc_name.startswith('ge'):
return Gb
if ifc_name.startswith('xe'):
return 10 * Gb
if ifc_name.startswith('et'):
return 400 * Gb
logger.warning(f'unrecognized interface name: {ifc_name}')
return -1
def _get_speed(ifc):
rate_conversions = {
"mbps": Mb,
"gbps": Gb
}
if "speed" in ifc:
speed = ifc["speed"]
match = re.match(r"(\d+)(.+)", speed)
if match:
value = int(match.group(1))
rate = match.group(2).strip().lower()
if rate in rate_conversions:
return value * rate_conversions[rate]
else:
logger.warning(f'unrecognised rate: {rate}, using _name_to_speed fallback')
return _name_to_speed(ifc['name'])
else:
logger.warning(f'unrecognised speed: {speed}, using _name_to_speed fallback')
return _name_to_speed(ifc['name'])
else:
logger.warning('no speed data for interface, using _name_to_speed fallback')
return _name_to_speed(ifc['name'])
if ifc['bundle-parents']:
if not ifc['name'].startswith('ae'):
logger.warning(
f'ifc has bundle-parents, but name is {ifc["name"]}')
return sum(_get_speed(parent_ifc) for parent_ifc in ifc['bundle-parents'])
return _get_speed(ifc)
def _load_interfaces_and_speeds(hostname=None):
"""
prepares the result of a call to /speeds
:param hostname: hostname or None for all
:return: generator yielding interface elements
"""
no_lab = common.get_bool_request_arg('no-lab', False)
basic_interfaces = _load_interfaces(
current_app.config['INVENTORY_PROVIDER_CONFIG'],
hostname,
no_lab=no_lab)
basic_interfaces_with_speeds = _add_speeds(
current_app.config['INVENTORY_PROVIDER_CONFIG'],
basic_interfaces)
with_bundles = _add_bundle_parents(list(basic_interfaces_with_speeds), hostname)
def _result_ifc(ifc):
return {
'router': ifc['router'],
'name': ifc['name'],
'speed': interface_speed(ifc)
}
return map(_result_ifc, with_bundles)
@routes.route("/speeds", methods=['GET'])
@routes.route('/speeds/<hostname>', methods=['GET'])
@common.require_accepts_json
def interface_speeds(hostname=None):
"""
Handler for `/poller/speeds` and
`/poller/speeds/<hostname>`
which returns information for either all interfaces
or those on the requested hostname.
The response is a list of maximum speed information (in bits
per second) for all known interfaces.
*speed <= 0 means the max interface speed can't be determined*
.. asjson::
inventory_provider.routes.poller.INTERFACE_SPEED_LIST_SCHEMA
:param hostname: optional, if present should be a router hostname
:return:
"""
cache_key = f'classifier-cache:poller-interface-speeds:{hostname}' \
if hostname else 'classifier-cache:poller-interface-speeds:all'
r = common.get_current_redis()
result = _ignore_cache_or_retrieve(request, cache_key, r)
if not result:
result = list(_load_interfaces_and_speeds(hostname))
if not result:
return Response(
response='no interfaces found',
status=404,
mimetype='text/html')
result = json.dumps(result)
# cache this data for the next call
r.set(cache_key, result.encode('utf-8'))
return Response(result, mimetype="application/json")
def _load_community_strings(base_key_pattern):
for doc in _load_netconf_docs(
config=current_app.config['INVENTORY_PROVIDER_CONFIG'],
filter_pattern=base_key_pattern):
community = juniper.snmp_community_string(doc['netconf'])
if not community:
yield {
'router': doc['router'],
'error':
f'error extracting community string for {doc["router"]}'
}
else:
yield {
'router': doc['router'],
'community': community
}
@routes.route('/eumetsat-multicast', methods=['GET'])
@routes.route('/eumetsat-multicast/<hostname>', methods=['GET'])
@common.require_accepts_json
def eumetsat_multicast(hostname=None):
"""
Handler for `/poller/eumetsat-multicast</hostname>` which returns
information about multicast subscriptions.
The hostname parameter is optional. If it is present, only hostnames
matching `hostname*` are returned. If not present, data for all
`mx*` routers is returned.
The response is a list of oid/router/community structures that all
all subscription octet counters to be polled.
.. asjson::
inventory_provider.routes.poller.MULTICAST_SUBSCRIPTION_LIST_SCHEMA
This method returns essentially hard-coded data,
based on the information in POL1-395.
:return:
"""
SUBSCRIPTIONS = [{
'subscription': f'232.223.222.{idx}',
'endpoint': '193.17.9.3',
} for idx in range(1, 73)]
SUBSCRIPTIONS.append(
{'subscription': '232.223.223.1', 'endpoint': '193.17.9.7'})
SUBSCRIPTIONS.append(
{'subscription': '232.223.223.22', 'endpoint': '193.17.9.7'})
def _oid(sub):
return ('1.3.6.1.2.1.83.1.1.2.1.16'
f'.{sub["subscription"]}.{sub["endpoint"]}'
'.255.255.255.255')
r = common.get_current_redis()
cache_key = 'classifier-cache:poller-eumetsat-multicast'
if hostname:
cache_key = f'{cache_key}:{hostname}'
result = r.get(cache_key)
if result:
result = result.decode('utf-8')
else:
def _multicast_oids(router_info):
def _rsp_element(sub):
result = {
'router': router_info['router'],
'oid': _oid(sub),
'community': router_info['community']
}
result.update(sub)
return result
yield from map(_rsp_element, SUBSCRIPTIONS)
routers = list(_load_community_strings(
base_key_pattern=f'netconf:{hostname}*'
if hostname else 'netconf:mx*'))
errors = list(filter(lambda x: 'error' in x, routers))
if errors:
errors = [e['error'] for e in errors]
return Response(
response=', '.join(errors),
status=403, # forbidden
mimetype='text/html')
assert all('community' in r for r in routers) # sanity
result = list(map(_multicast_oids, routers))
result = itertools.chain(*result)
result = list(result)
if not result:
target = hostname or 'any routers!'
return Response(
response=f'no multicast config for {target}',
status=404,
mimetype='text/html')
result = json.dumps(result)
# cache this data for the next call
r.set(cache_key, result.encode('utf-8'))
return Response(result, mimetype="application/json")
@routes.route("/gws/direct", methods=['GET'])
@common.require_accepts_json
def gws_direct():
"""
Handler for `/poller/gws/direct` which returns required for polling
customer equipment counters for ISP connetions.
The response is a list of nren/isp/counter structures that must be
polled.
.. asjson::
inventory_provider.routes.poller.GWS_DIRECT_DATA_SCHEMA
WARNING: interface tags in the `gws-direct` section of the config data
should be unique for each nren/isp combination. i.e. if there
are multiple community strings in use for a particular host, then please
keep the interface tags unique.
:return:
"""
cache_key = 'classifier-cache:gws-direct'
r = common.get_current_redis()
result = r.get(cache_key)
if result:
result = result.decode('utf-8')
else:
def _interfaces():
config_params = current_app.config['INVENTORY_PROVIDER_CONFIG']
for nren_isp in config_params['gws-direct']:
for host in nren_isp['hosts']:
snmp_params = {}
if 'community' in host:
# (snmp v2)
# sanity (already guaranteed by schema check)
assert 'sec-name' not in host
snmp_params['community'] = host['community']
else:
# (snmp v3)
# sanity (already guaranteed by schema check)
assert 'sec-name' in host
snmp_params['sec-name'] = host['sec-name']
if 'auth' in host:
snmp_params['auth'] = host['auth']
if 'priv' in host:
snmp_params['priv'] = host['priv']
for ifc in host['interfaces']:
ifc_data = {
'nren': nren_isp['nren'],
'isp': nren_isp['isp'],
'hostname': host['hostname'],
'tag': ifc['tag'],
'counters': [
{
'field': k,
'oid': v,
'snmp': snmp_params
}
for k, v in ifc['counters'].items()]
}
if 'info' in ifc:
ifc_data['info'] = ifc['info']
yield ifc_data
result = json.dumps(list(_interfaces()))
# cache this data for the next call
r.set(cache_key, result.encode('utf-8'))
return Response(result, mimetype="application/json")
# cf. https://gitlab.geant.net/puppet-apps/cacti/-/blob/production/files/scripts/juniper-firewall-dws.pl # noqa: E501
JNX_DCU_STATS_BYTES_OID = '1.3.6.1.4.1.2636.3.6.2.1.5'
JNX_FW_COUNTER_BYTES_OID = '1.3.6.1.4.1.2636.3.5.2.1.5'
JNX_ADDRESS_FAMILY = {
'ipv4': 1,
'ipv6': 2
}
JNX_FW_COUNTER_TYPE = {
'other': 1,
'counter': 2,
'policer': 3
}
def _str2oid(s):
chars = '.'.join(str(ord(c)) for c in s)
return f'{len(s)}.{chars}'
def _jnx_dcu_byte_count_oid(
ifIndex,
class_name='dws-in',
address_family=JNX_ADDRESS_FAMILY['ipv4']):
# sanity checks (in case of programming errors)
assert isinstance(ifIndex, int)
assert isinstance(class_name, str)
assert isinstance(address_family, int)
return '.'.join([
JNX_DCU_STATS_BYTES_OID,
str(ifIndex),
str(address_family),
_str2oid(class_name)
])
def _jnx_fw_counter_bytes_oid(
customer,
interface_name,
filter_name=None,
counter_name=None):
# sanity checks (in case of programming errors)
assert isinstance(customer, str)
assert isinstance(interface_name, str)
assert filter_name is None or isinstance(filter_name, str)
assert counter_name is None or isinstance(counter_name, str)
if filter_name is None:
filter_name = f'nren_IAS_{customer}_OUT-{interface_name}-o'
if counter_name is None:
counter_name = f'DWS-out-{interface_name}-o'
return '.'.join([
JNX_FW_COUNTER_BYTES_OID,
_str2oid(filter_name),
_str2oid(counter_name),
str(JNX_FW_COUNTER_TYPE['counter'])
])
def _get_services_internal(service_type=None):
"""
Performs the lookup and caching done for calls to
`/poller/services</service-type>`
This is a separate private utility so that it can be called by
:meth:`inventory_provider.routes.poller.get_services`
and :meth:`inventory_provider.routes.poller.get_service_types`
The response will be formatted according to the following schema:
.. asjson::
inventory_provider.routes.poller.SERVICES_LIST_SCHEMA
:param service_type: a service type, or None to return all
:return: service list, json-serialized to a string
"""
return_all = common.get_bool_request_arg('all', True)
include_snmp = common.get_bool_request_arg('snmp', False)
def _services():
key_pattern = f'ims:services:{service_type}:*' \
if service_type else 'ims:services:*'
for doc in common.load_json_docs(
config_params=current_app.config['INVENTORY_PROVIDER_CONFIG'],
key_pattern=key_pattern,
num_threads=20):
yield doc['value']
def _add_snmp(s, all_snmp_info):
snmp_interfaces = all_snmp_info.get(s['hostname'], {})
interface_info = snmp_interfaces.get(s['interface'], None)
if interface_info:
s['snmp'] = {
'ifIndex': interface_info['index'],
'community': interface_info['community'],
}
if s['type'] == 'GWS - INDIRECT':
s['snmp']['counters'] = [
{
'field': 'ingressOctets',
'oid': _jnx_dcu_byte_count_oid(
interface_info['index']),
},
{
'field': 'egressOctets',
'oid': _jnx_fw_counter_bytes_oid(
s['customer'], s['interface'])
}
]
return s
def _wanted_in_output(s):
return return_all or (s['status'].lower() == 'operational')
def _format_services(s):
return {
'id': s['id'],
'name': s['name'],
'customer': s['project'],
'speed': s['speed_value'],
'pop': s['here']['pop']['name'],
'hostname': common.ims_equipment_to_hostname(
s['here']['equipment']),
'interface': s['here']['port'].lower(),
'type': s['type'],
'status': s['status']
}
cache_key = f'classifier-cache:poller:services:{service_type}' \
if service_type else 'classifier-cache:poller:services:all-types'
if return_all:
cache_key = f'{cache_key}:all'
if include_snmp:
cache_key = f'{cache_key}:snmp'
redis = common.get_current_redis()
result = _ignore_cache_or_retrieve(request, cache_key, redis)
if not result:
result = _services()
result = filter(_wanted_in_output, result)
result = map(_format_services, result)
if include_snmp:
all_snmp_info = common.load_snmp_indexes(current_app.config['INVENTORY_PROVIDER_CONFIG'])
result = map(partial(_add_snmp, all_snmp_info=all_snmp_info), result)
result = list(result)
if not result:
return None
# cache this data for the next call
result = json.dumps(result)
redis.set(cache_key, result)
return result
@routes.route('/services', methods=['GET'])
@routes.route('/services/<service_type>', methods=['GET'])
@common.require_accepts_json
def get_services(service_type=None):
"""
Handler for `/poller/services</service-type>`
Use `/poller/service-types` for possible values of `service-type`.
If the endpoint is called with no `service-type`,
all services are returned.
Supported url parameters:
:all: Default is False to return only operational services.
If present and evaluates to True, then return all services.
:snmp: If present and evalutes to True, add snmp interface polling
params to the response.
(sphinx bug: the param names aren't capitalized)
The response will be formatted according to the following schema:
.. asjson::
inventory_provider.routes.poller.SERVICES_LIST_SCHEMA
A few possible values for `service_type` are (currently): gws_internal,
geant_ip, geant_lambda, l3_vpn, md_vpn_proxy (there are more)
If the endpoint is called with no `service_type`,
all services are returned
:return:
"""
services_json_str = _get_services_internal(service_type)
if not services_json_str:
message = f'no {service_type} services found' \
if service_type else 'no services found'
return Response(
response=message,
status=404,
mimetype='text/html')
return Response(services_json_str, mimetype='application/json')
@routes.route("/gws/indirect", methods=['GET'])
@common.require_accepts_json
def gws_indirect():
"""
Handler for `/poller/gws/indirect`
Same as `/poller/services/gws_indirect`
cf. :meth:`inventory_provider.routes.poller.get_services`
:return:
"""
return get_services(service_type='gws_indirect')
@routes.route('/service-types', methods=['GET'])
@common.require_accepts_json
def get_service_types():
"""
Handler for `/poller/service-types`
This method returns a list of all values of `service_type` that
can be used with `/poller/services/service-type` to return
a non-empty list of services.
Adding a truthy `all` request parameter (e.g. `?all=1`) will
return also return valid service types for which none of the
defined services are operational.
The response will be formatted according to the following schema:
.. asjson::
inventory_provider.routes.poller.STRING_LIST_SCHEMA
"""
cache_key = 'classifier-cache:poller:service-types'
return_all = common.get_bool_request_arg('all', False)
if return_all:
cache_key = f'{cache_key}-all'
redis = common.get_current_redis()
service_types = _ignore_cache_or_retrieve(request, cache_key, redis)
if not service_types:
all_services = json.loads(_get_services_internal())
service_types = {
ims_sorted_service_type_key(s['type'])
for s in all_services
}
if not service_types:
return Response(
response='no service types found',
status=404,
mimetype='text/html')
# cache this data for the next call
service_types = sorted(list(service_types))
service_types = json.dumps(service_types)
redis.set(cache_key, service_types)
return Response(service_types, mimetype='application/json')
@routes.route('/gws/direct-config', methods=['GET'])
def gws_direct_config():
"""
Handler for `/poller/gws/direct-config` which returns
the basic gws-direct config.
This api is only intended for config validation.
:return:
"""
wanted = request.args.get('format', default='json', type=str)
wanted = wanted.lower()
if wanted not in ('html', 'json'):
return Response(
response='format must be one of: html, json',
status=400,
mimetype="text/html")
def _counters():
config_params = current_app.config['INVENTORY_PROVIDER_CONFIG']
for nren_isp in config_params['gws-direct']:
for host in nren_isp['hosts']:
snmp_version = '2' if 'community' in host.keys() else '3'
for ifc in host['interfaces']:
for field, oid in ifc['counters'].items():
yield {
'nren': nren_isp['nren'],
'isp': nren_isp['isp'],
'hostname': host['hostname'],
'snmp': snmp_version,
'interface': ifc['tag'],
'field': field,
'oid': oid,
'info': ifc.get('info', '')
}
if wanted == 'json':
if not request.accept_mimetypes.accept_json:
return Response(
response="response will be json",
status=406,
mimetype="text/html")
else:
return jsonify(list(_counters()))
if not request.accept_mimetypes.accept_html:
return Response(
response="response will be html",
status=406,
mimetype="text/html")
page = '''<!doctype html>
<html lang="en">
<head>
<!-- Required meta tags -->
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no">
<!-- Bootstrap CSS -->
<link rel="stylesheet" href="https://stackpath.bootstrapcdn.com/bootstrap/4.3.1/css/bootstrap.min.css" integrity="sha384-ggOyR0iXCbMQv3Xipma34MD+dH/1fQ784/j6cY/iJTQUOhcWr7x9JvoRxT2MZw1T" crossorigin="anonymous">
<title>GWS Direct config</title>
</head>
<body>
<table class="table table-striped">
<thead>
{header_row}
</thead>
<tbody>
{data_rows}
</tbody>
</table>
</body>
</html>''' # noqa: E501
def _to_row(counter, header=False):
_columns = (
'nren', 'isp', 'hostname',
'snmp', 'interface', 'field', 'oid', 'info')
elems = ['<tr>']
for name in _columns:
if header:
elems.append(f'<th scope="col">{name}</th>')
else:
elems.append(f'<td scope="row">{counter[name]}</td>')
elems.append('</tr>')
return ''.join(elems)
header_row = _to_row(None, header=True)
data_rows = map(_to_row, _counters())
page = page.format(
header_row=header_row,
data_rows='\n'.join(data_rows))
return Response(
response=page,
status=200,
mimetype="text/html")