diff --git a/inventory_provider/routes/common.py b/inventory_provider/routes/common.py
index a025f3a39c23abe26ad28c7d20cd7cc54663889a..8f286962b766ff2be1e5eb830af15a90d744a407 100644
--- a/inventory_provider/routes/common.py
+++ b/inventory_provider/routes/common.py
@@ -18,13 +18,18 @@ _DECODE_TYPE_XML = 'xml'
_DECODE_TYPE_JSON = 'json'
-def _ignore_cache_or_retrieve(request_, cache_key, r):
-
- ignore_cache = request_.args.get('ignore-cache', default='false', type=str)
+def get_bool_request_arg(name, default=False):
+ assert isinstance(default, bool) # sanity, otherwise caller error
+ value = request.args.get(name, default=str(default), type=str)
try:
- ignore_cache = strtobool(ignore_cache)
+ value = strtobool(value)
except ValueError:
- ignore_cache = False
+ value = default
+ return value
+
+
+def _ignore_cache_or_retrieve(request_, cache_key, r):
+ ignore_cache = get_bool_request_arg('ignore-cache', default=False)
if ignore_cache:
result = False
logger.debug('ignoring cache')
@@ -255,6 +260,7 @@ def load_xml_docs(config_params, key_pattern, num_threads=10):
config_params, key_pattern, num_threads, doc_type=_DECODE_TYPE_XML)
+@functools.lru_cache
def load_snmp_indexes(hostname=None):
result = dict()
key_pattern = f'snmp-interfaces:{hostname}*' \
diff --git a/inventory_provider/routes/jobs.py b/inventory_provider/routes/jobs.py
index 078e8a56392bd5bb4bb4076d83b6ced585f89e97..b33476fb1d031b4133585eb204c2ca587b466b59 100644
--- a/inventory_provider/routes/jobs.py
+++ b/inventory_provider/routes/jobs.py
@@ -28,8 +28,7 @@ These endpoints are used for monitoring running jobs.
import json
import logging
-from distutils.util import strtobool
-from flask import Blueprint, current_app, jsonify, Response, request
+from flask import Blueprint, current_app, jsonify, Response
from inventory_provider.tasks import monitor
from inventory_provider.tasks import worker
@@ -125,15 +124,10 @@ def update():
:return:
"""
- force = request.args.get('force', default='false', type=str)
- try:
- force = strtobool(force)
- except ValueError:
- force = False
-
config = current_app.config['INVENTORY_PROVIDER_CONFIG']
r = get_current_redis(config)
+ force = common.get_bool_request_arg('force', default=False)
if not force:
latch = get_latch(r)
if latch and latch['pending']:
diff --git a/inventory_provider/routes/poller.py b/inventory_provider/routes/poller.py
index d812b0a8b1ebe34254a50752ddf615a6115db25f..a8942266980c6314be6928d2216ae90f5bdeb928 100644
--- a/inventory_provider/routes/poller.py
+++ b/inventory_provider/routes/poller.py
@@ -36,12 +36,18 @@ These endpoints are intended for use by BRIAN.
.. autofunction:: inventory_provider.routes.poller.gws_indirect
-/poller/services</type>
+/poller/services</service-type>
---------------------------------
.. autofunction:: inventory_provider.routes.poller.get_services
+/poller/service-types
+---------------------------------
+
+.. autofunction:: inventory_provider.routes.poller.get_service_types
+
+
"""
import json
import logging
@@ -52,6 +58,7 @@ from lxml import etree
from inventory_provider import juniper
from inventory_provider.routes import common
+from inventory_provider.tasks.common import ims_sorted_service_type_key
from inventory_provider.routes.classifier import get_ims_equipment_name, \
get_ims_interface
from inventory_provider.routes.common import _ignore_cache_or_retrieve
@@ -220,6 +227,15 @@ SERVICES_LIST_SCHEMA = {
'$schema': 'http://json-schema.org/draft-07/schema#',
'definitions': {
+ 'snmp-info': {
+ 'type': 'object',
+ 'properties': {
+ 'ifIndex': {'type': 'integer'},
+ 'community': {'type': 'string'}
+ },
+ 'required': ['ifIndex', 'community'],
+ 'additionalProperties': False
+ },
'service': {
'type': 'object',
'properties': {
@@ -230,12 +246,14 @@ SERVICES_LIST_SCHEMA = {
'pop': {'type': 'string'},
'hostname': {'type': 'string'},
'interface': {'type': 'string'},
- 'type': {'type': 'string'}
+ 'type': {'type': 'string'},
+ 'status': {'type': 'string'},
+ 'snmp': {'$ref': '#/definitions/snmp-info'}
},
'required': [
'id', 'name', 'customer',
'speed', 'pop', 'hostname',
- 'interface', 'type'],
+ 'interface', 'type', 'status'],
'additionalProperties': False
}
},
@@ -245,6 +263,13 @@ SERVICES_LIST_SCHEMA = {
}
+STRING_LIST_SCHEMA = {
+ '$schema': 'http://json-schema.org/draft-07/schema#',
+ 'type': 'array',
+ 'items': {'type': 'string'}
+}
+
+
@routes.after_request
def after_request(resp):
return common.after_request(resp)
@@ -706,26 +731,27 @@ def gws_direct():
return Response(result, mimetype="application/json")
-@routes.route('/services', methods=['GET', 'POST'])
-@routes.route('/services/<service_type>', methods=['GET', 'POST'])
-@common.require_accepts_json
-def get_services(service_type=None):
+def _get_services_internal(service_type=None):
"""
- Handler for `/poller/services/service_type`.
+ Performs the lookup and caching done for calls to
+ `/poller/services</service-type>`
+
+ This is a separate private utility so that it can be called by
+ :meth:`inventory_provider.routes.poller.get_services`
+ and :meth:`inventory_provider.routes.poller.get_service_types`
The response will be formatted according to the following schema:
.. asjson::
inventory_provider.routes.poller.SERVICES_LIST_SCHEMA
- A few possible values for `service_type` are (currently): gws_internal,
- geant_ip, geant_lambda, l3_vpn, md_vpn_proxy (there are more)
+ :param service_type: a service type, or None to return all
+ :return: service list, json-serialized to a string
+ """
- If the endpoint is called with no `service_type`,
- all services are returned
+ return_all = common.get_bool_request_arg('all', False)
+ include_snmp = common.get_bool_request_arg('snmp', False)
- :return:
- """
def _services():
key_pattern = f'ims:services:{service_type}:*' \
if service_type else 'ims:services:*'
@@ -736,8 +762,19 @@ def get_services(service_type=None):
num_threads=20):
yield doc['value']
- def _operational(s):
- return s['status'].lower() == 'operational'
+ def _add_snmp(s):
+ all_snmp_info = common.load_snmp_indexes()
+ snmp_interfaces = all_snmp_info.get(s['hostname'], {})
+ interface_info = snmp_interfaces.get(s['interface'], None)
+ if interface_info:
+ s['snmp'] = {
+ 'ifIndex': interface_info['index'],
+ 'community': interface_info['community']
+ }
+ return s
+
+ def _wanted_in_output(s):
+ return return_all or (s['status'].lower() == 'operational')
def _format_services(s):
return {
@@ -748,35 +785,86 @@ def get_services(service_type=None):
'pop': s['here']['pop']['name'],
'hostname': common.ims_equipment_to_hostname(
s['here']['equipment']),
- 'interface': s['here']['port'],
- 'type': s['type']
+ 'interface': s['here']['port'].lower(),
+ 'type': s['type'],
+ 'status': s['status']
}
cache_key = f'classifier-cache:poller:services:{service_type}' \
- if service_type else 'classifier-cache:poller:services:all'
+ if service_type else 'classifier-cache:poller:services:all-types'
+
+ if return_all:
+ cache_key = f'{cache_key}:all'
+ if include_snmp:
+ cache_key = f'{cache_key}:snmp'
redis = common.get_current_redis()
result = _ignore_cache_or_retrieve(request, cache_key, redis)
if not result:
result = _services()
- result = filter(_operational, result)
+ result = filter(_wanted_in_output, result)
result = map(_format_services, result)
+ if include_snmp:
+ result = map(_add_snmp, result)
result = list(result)
if not result:
- message = f'no {service_type} services found' \
- if service_type else 'no services found'
- return Response(
- response=message,
- status=404,
- mimetype='text/html')
+ return None
# cache this data for the next call
result = json.dumps(result)
- redis.set(cache_key, result.encode('utf-8'))
+ redis.set(cache_key, result)
+
+ return result
+
+
+@routes.route('/services', methods=['GET', 'POST'])
+@routes.route('/services/<service_type>', methods=['GET', 'POST'])
+@common.require_accepts_json
+def get_services(service_type=None):
+ """
+ Handler for `/poller/services</service-type>`
+
+ Use `/poller/service-types` for possible values of `service-type`.
- return Response(result, mimetype='application/json')
+ If the endpoint is called with no `service-type`,
+ all services are returned.
+
+ Supported url parameters:
+
+ :all: Default is False to return only operational services.
+ If present and evaluates to True, then return all services.
+ :snmp: If present and evalutes to True, add snmp interface polling
+ params to the response.
+
+ (sphinx bug: the param names aren't capitalized)
+
+ The response will be formatted according to the following schema:
+
+ .. asjson::
+ inventory_provider.routes.poller.SERVICES_LIST_SCHEMA
+
+ A few possible values for `service_type` are (currently): gws_internal,
+ geant_ip, geant_lambda, l3_vpn, md_vpn_proxy (there are more)
+
+ If the endpoint is called with no `service_type`,
+ all services are returned
+
+ :return:
+ """
+
+ services_json_str = _get_services_internal(service_type)
+
+ if not services_json_str:
+ message = f'no {service_type} services found' \
+ if service_type else 'no services found'
+ return Response(
+ response=message,
+ status=404,
+ mimetype='text/html')
+
+ return Response(services_json_str, mimetype='application/json')
@routes.route("/gws/indirect", methods=['GET', 'POST'])
@@ -791,3 +879,53 @@ def gws_indirect():
:return:
"""
return get_services(service_type='gws_indirect')
+
+
+@routes.route('/service-types', methods=['GET', 'POST'])
+@common.require_accepts_json
+def get_service_types():
+ """
+ Handler for `/poller/service-types`
+
+ This method returns a list of all values of `service_type` that
+ can be used with `/poller/services/service-type` to return
+ a non-empty list of services.
+
+ Adding a truthy `all` request parameter (e.g. `?all=1`) will
+ return also return valid service types for which none of the
+ defined services are operational.
+
+ The response will be formatted according to the following schema:
+
+ .. asjson::
+ inventory_provider.routes.poller.STRING_LIST_SCHEMA
+
+ """
+ cache_key = 'classifier-cache:poller:service-types'
+
+ return_all = common.get_bool_request_arg('all', False)
+ if return_all:
+ cache_key = f'{cache_key}-all'
+
+ redis = common.get_current_redis()
+ service_types = _ignore_cache_or_retrieve(request, cache_key, redis)
+
+ if not service_types:
+ all_services = json.loads(_get_services_internal())
+ service_types = {
+ ims_sorted_service_type_key(s['type'])
+ for s in all_services
+ }
+
+ if not service_types:
+ return Response(
+ response='no service types found',
+ status=404,
+ mimetype='text/html')
+
+ # cache this data for the next call
+ service_types = sorted(list(service_types))
+ service_types = json.dumps(service_types)
+ redis.set(cache_key, service_types)
+
+ return Response(service_types, mimetype='application/json')
diff --git a/inventory_provider/tasks/common.py b/inventory_provider/tasks/common.py
index 427d51d8a18e4128f3dd3b174274e65fa17f5f8f..ad929ed4677cee9750f0b9b52c2846ecc26584d1 100644
--- a/inventory_provider/tasks/common.py
+++ b/inventory_provider/tasks/common.py
@@ -1,5 +1,6 @@
import json
import logging
+import re
import time
import jsonschema
@@ -218,3 +219,23 @@ def get_next_redis(config):
'derived next id: {}'.format(next_id))
return _get_redis(config, next_id)
+
+
+def ims_sorted_service_type_key(service_type):
+ """
+ special-purpose function used for mapping IMS service type strings
+ to the key used in redis ('ims:services:*')
+
+ this method is only used by
+ :meth:`inventory_provider:tasks:worker.update_circuit_hierarchy_and_port_id_services
+ and :meth:`inventory_provider:routes.poller.get_service_types
+
+ :param service_type: IMS-formatted service_type string
+ :return: ims sorted service type redis key
+ """ # noqa: E501
+
+ service_type = re.sub(
+ r'[^a-zA-Z0-9]+', '_', service_type.lower())
+ # prettification ... e.g. no trailing _ for 'MD-VPN (NATIVE)'
+ service_type = re.sub('_+$', '', service_type)
+ return re.sub('^_+', '', service_type)
diff --git a/inventory_provider/tasks/worker.py b/inventory_provider/tasks/worker.py
index 58bccc8252ffda083a0243fa4def71477e6f7200..4d7e33c506390db0a5b76b129dabe06d02156ffd 100644
--- a/inventory_provider/tasks/worker.py
+++ b/inventory_provider/tasks/worker.py
@@ -29,7 +29,8 @@ from inventory_provider.db.ims import IMS
from inventory_provider.tasks.app import app
from inventory_provider.tasks.common \
import get_next_redis, get_current_redis, \
- latch_db, get_latch, set_latch, update_latch_status
+ latch_db, get_latch, set_latch, update_latch_status, \
+ ims_sorted_service_type_key
from inventory_provider.tasks import monitor
from inventory_provider import config
from inventory_provider import environment
@@ -762,14 +763,8 @@ def update_circuit_hierarchy_and_port_id_services(self, use_current=False):
circ['calculated-speed'] = _get_speed(circ['id'])
_format_service(circ)
- service_type_key = re.sub(
- r'[^a-zA-Z0-9]+', '_', circ['service_type'].lower())
- # prettification ... e.g. no trailing _ for 'MD-VPN (NATIVE)'
- service_type_key = re.sub('_+$', '', service_type_key)
- service_type_key = re.sub('^_+', '', service_type_key)
-
type_services = services_by_type.setdefault(
- service_type_key, dict())
+ ims_sorted_service_type_key(circ['service_type']), dict())
type_services[circ['id']] = circ
interface_services[k].extend(circuits)
diff --git a/test/test_general_poller_routes.py b/test/test_general_poller_routes.py
index b2179230221b5d5b2cc2c4fbeec8196512016b12..b00dc988719a5319581d15abb9a2416b3dfc3cd0 100644
--- a/test/test_general_poller_routes.py
+++ b/test/test_general_poller_routes.py
@@ -78,7 +78,7 @@ def test_gws_indirect(client):
assert all(s['type'] == 'GWS - INDIRECT' for s in response_data)
-def test_all_services(client):
+def test_all_services_default(client):
rv = client.get(
'/poller/services',
headers=DEFAULT_REQUEST_HEADERS)
@@ -88,6 +88,39 @@ def test_all_services(client):
jsonschema.validate(response_data, poller.SERVICES_LIST_SCHEMA)
assert response_data # test data is non-empty
+ assert all(s['status'] == 'operational' for s in response_data)
+
+
+def test_all_services_all(client):
+ rv = client.get(
+ '/poller/services?all=1',
+ headers=DEFAULT_REQUEST_HEADERS)
+ assert rv.status_code == 200
+ assert rv.is_json
+ response_data = json.loads(rv.data.decode('utf-8'))
+ jsonschema.validate(response_data, poller.SERVICES_LIST_SCHEMA)
+
+ assert response_data # test data is non-empty
+ assert any(s['status'] != 'operational' for s in response_data)
+ assert all('snmp' not in s for s in response_data)
+
+
+@pytest.mark.parametrize('service_type', [
+ # services with these types should all have snmp configs
+ 'gws_indirect', 'geant_ip', 'l3_vpn'])
+def test_services_snmp(client, service_type):
+
+ rv = client.get(
+ f'/poller/services/{service_type}?snmp=1',
+ headers=DEFAULT_REQUEST_HEADERS)
+ assert rv.status_code == 200
+ assert rv.is_json
+ response_data = json.loads(rv.data.decode('utf-8'))
+ jsonschema.validate(response_data, poller.SERVICES_LIST_SCHEMA)
+
+ assert response_data # test data is non-empty
+ # all access services should have snmp info
+ assert all('snmp' in s for s in response_data)
@pytest.mark.parametrize('uri_type,expected_type', [
@@ -112,3 +145,18 @@ def test_all_services_by_type(client, uri_type, expected_type):
assert response_data # test data is non-empty
assert all(s['type'] == expected_type for s in response_data)
+
+
+def test_get_service_types(client):
+ rv = client.get(
+ '/poller/service-types',
+ headers=DEFAULT_REQUEST_HEADERS)
+ assert rv.status_code == 200
+ assert rv.is_json
+ response_data = json.loads(rv.data.decode('utf-8'))
+ jsonschema.validate(response_data, poller.STRING_LIST_SCHEMA)
+
+ assert response_data # test data is non-empty
+ # some hard-coded values we expect to see from the test data ...
+ assert 'gws_indirect' in response_data
+ assert 'geant_ip' in response_data