diff --git a/inventory_provider/routes/poller.py b/inventory_provider/routes/poller.py
index 03aea603f9cd88fcb095d98f5289762659db0b13..3cc88a4478f10c3eac9f49a6cf77fd923671a395 100644
--- a/inventory_provider/routes/poller.py
+++ b/inventory_provider/routes/poller.py
@@ -62,12 +62,12 @@ support method: _get_dashboards
"""
from enum import Enum, auto
+import itertools
import json
import logging
import re
from flask import Blueprint, Response, current_app, request, jsonify
-from lxml import etree
from inventory_provider import juniper
from inventory_provider.routes import common
@@ -605,6 +605,39 @@ def _load_services(config, hostname=None, use_next_redis=False):
return result
+def _load_netconf_docs(
+ config, filter_pattern, use_next_redis=False):
+ """
+ yields dicts like:
+ {
+ 'router': router hostname
+ 'netconf': loaded netconf xml doc
+ }
+
+ :param config: app config
+ :param filter_pattern: search filter, including 'netconf:'
+ :param use_next_redis: use next instead of current redis, if true
+ :return: yields netconf docs, formatted as above
+ """
+
+ m = re.match(r'^(.*netconf:).+', filter_pattern)
+ # TODO: probably better to not required netconf: to be passed in
+ assert m # sanity
+ key_prefix_len = len(m.group(1))
+ assert key_prefix_len >= len('netconf:') # sanity
+
+ for doc in common.load_xml_docs(
+ config_params=config,
+ key_pattern=filter_pattern,
+ num_threads=10,
+ use_next_redis=use_next_redis):
+
+ yield {
+ 'router': doc['key'][key_prefix_len:],
+ 'netconf': doc['value']
+ }
+
+
def _load_interfaces(
config, hostname=None, no_lab=False, use_next_redis=False):
"""
@@ -617,25 +650,14 @@ def _load_interfaces(
"""
def _load_docs(key_pattern):
- m = re.match(r'^(.*netconf:).+', key_pattern)
- assert m # sanity
- key_prefix_len = len(m.group(1))
- assert key_prefix_len >= len('netconf:') # sanity
-
- for doc in common.load_xml_docs(
- config_params=config,
- key_pattern=key_pattern,
- num_threads=10,
- use_next_redis=use_next_redis):
-
- router = doc['key'][key_prefix_len:]
+ for doc in _load_netconf_docs(config, key_pattern, use_next_redis):
- for ifc in juniper.list_interfaces(doc['value']):
+ for ifc in juniper.list_interfaces(doc['netconf']):
if not ifc['description']:
continue
yield {
- 'router': router,
+ 'router': doc['router'],
'name': ifc['name'],
'bundle': ifc['bundle'],
'bundle-parents': [],
@@ -894,18 +916,35 @@ def interface_speeds(hostname=None):
return Response(result, mimetype="application/json")
-MX1_FRA = 'mx1.fra.de.geant.net'
+def _load_community_strings(base_key_pattern):
+ for doc in _load_netconf_docs(
+ config=current_app.config['INVENTORY_PROVIDER_CONFIG'],
+ filter_pattern=base_key_pattern):
+ community = juniper.snmp_community_string(doc['netconf'])
+ if not community:
+ yield {
+ 'router': doc['router'],
+ 'error':
+ f'error extracting community string for {doc["router"]}'
+ }
+ else:
+ yield {
+ 'router': doc['router'],
+ 'community': community
+ }
@routes.route('/eumetsat-multicast', methods=['GET', 'POST'])
@routes.route('/eumetsat-multicast/<hostname>', methods=['GET', 'POST'])
@common.require_accepts_json
-def eumetsat_multicast(hostname=MX1_FRA):
+def eumetsat_multicast(hostname=None):
"""
Handler for `/poller/eumetsat-multicast` which returns information about
multicast subscriptions on mx1.fra.de.geant.net.
- The hostname is optional, with default value mx1.fra.de.geant.net
+ The hostname parameter is optional. If it is present, only hostnames
+ matching `hostname*` are returned. If not present, data for all
+ `mx*` routers is returned.
The response is a list of oid/router/community structures that all
all subscription octet counters to be polled.
@@ -934,41 +973,52 @@ def eumetsat_multicast(hostname=MX1_FRA):
f'.{sub["subscription"]}.{sub["endpoint"]}'
'.255.255.255.255')
- cache_key = f'classifier-cache:poller-eumetsat-multicast:{hostname}'
-
r = common.get_current_redis()
+ cache_key = 'classifier-cache:poller-eumetsat-multicast'
+ if hostname:
+ cache_key = f'{cache_key}:{hostname}'
+
result = r.get(cache_key)
if result:
result = result.decode('utf-8')
else:
- netconf = r.get(f'netconf:{hostname}')
- if not netconf:
- return Response(
- status=503,
- response=f'error loading netconf for {hostname}')
- netconf_doc = etree.fromstring(netconf.decode('utf-8'))
- community = juniper.snmp_community_string(netconf_doc)
- if not community:
- # HACKHACK: vpn source ip isn't in acl
- # TODO: remove this when done testing!
- community = '0pBiFbD'
- if not community:
+ def _multicast_oids(router_info):
+
+ def _rsp_element(sub):
+ result = {
+ 'router': router_info['router'],
+ 'oid': _oid(sub),
+ 'community': router_info['community']
+ }
+ result.update(sub)
+ return result
+
+ yield from map(_rsp_element, SUBSCRIPTIONS)
+
+ routers = list(_load_community_strings(
+ base_key_pattern=f'netconf:{hostname}*'
+ if hostname else 'netconf:mx*'))
+ errors = list(filter(lambda x: 'error' in x, routers))
+ if errors:
+ errors = [e['error'] for e in errors]
return Response(
- status=503,
- response=f'error extracting community string for {hostname}')
+ response=', '.join(errors),
+ status=403, # forbidden
+ mimetype='text/html')
- def _rsp_element(sub):
- result = {
- 'router': hostname,
- 'oid': _oid(sub),
- 'community': community
- }
- result.update(sub)
- return result
+ assert all('community' in r for r in routers) # sanity
- result = [_rsp_element(sub) for sub in SUBSCRIPTIONS]
+ result = list(map(_multicast_oids, routers))
+ result = itertools.chain(*result)
+ result = list(result)
+ if not result:
+ target = hostname or 'any routers!'
+ return Response(
+ response=f'no multicast config for {target}',
+ status=404,
+ mimetype='text/html')
result = json.dumps(result)
# cache this data for the next call
diff --git a/test/test_general_poller_routes.py b/test/test_general_poller_routes.py
index 869d7e9472c7ad1ebafc7d64c308e81e0228ee2f..66408cc064c1936f00523a4f85389c58bf28011a 100644
--- a/test/test_general_poller_routes.py
+++ b/test/test_general_poller_routes.py
@@ -50,7 +50,7 @@ def test_all_router_interface_speeds(client):
'there should data from be lots of routers'
-def test_eumetsat_multicast(mocker, client):
+def test_eumetsat_multicast_all(mocker, client):
# routers don't have snmp acl's for us
mocker.patch('inventory_provider.juniper.snmp_community_string') \
@@ -66,6 +66,58 @@ def test_eumetsat_multicast(mocker, client):
response_data, poller.MULTICAST_SUBSCRIPTION_LIST_SCHEMA)
assert response_data, "the subscription list shouldn't be empty"
+ # only 'mx*' routers are returned, by default
+ assert all([s['router'].startswith('mx') for s in response_data])
+
+
+@pytest.mark.parametrize('hostname', [
+ 'mx1.ams.nl.geant.net',
+ 'mx1.ams',
+ 'qfx.fra.de.geant.net' # expect to be able to explicitly select others
+])
+def test_eumetsat_multicast_hostname(mocker, client, hostname):
+
+ # routers don't have snmp acl's for us
+ mocker.patch('inventory_provider.juniper.snmp_community_string') \
+ .return_value = 'blah'
+
+ rv = client.get(
+ f'/poller/eumetsat-multicast/{hostname}',
+ headers=DEFAULT_REQUEST_HEADERS)
+ assert rv.status_code == 200
+ assert rv.is_json
+ response_data = json.loads(rv.data.decode('utf-8'))
+ jsonschema.validate(
+ response_data, poller.MULTICAST_SUBSCRIPTION_LIST_SCHEMA)
+ assert response_data, "the subscription list shouldn't be empty"
+
+ # only 'mx*' routers are returned, by default
+ assert all([s['router'].startswith(hostname) for s in response_data])
+
+
+def test_eumetsat_multicast_404(mocker, client):
+
+ # routers don't have snmp acl's for us
+ mocker.patch('inventory_provider.juniper.snmp_community_string') \
+ .return_value = 'blah'
+
+ rv = client.get(
+ '/poller/eumetsat-multicast/XYZ123',
+ headers=DEFAULT_REQUEST_HEADERS)
+ assert rv.status_code == 404
+
+
+def test_eumetsat_multicast_forbidden(mocker, client):
+
+ # routers don't have snmp acl's for us
+ mocker.patch('inventory_provider.juniper.snmp_community_string') \
+ .return_value = ''
+
+ rv = client.get(
+ '/poller/eumetsat-multicast',
+ headers=DEFAULT_REQUEST_HEADERS)
+ assert rv.status_code == 403
+
def test_gws_direct(client):