Finished feature POL1-463-multicast-subscriptions-return-all.

inventory_provider/routes/poller.py

@@ -62,12 +62,12 @@ support method: _get_dashboards
 
 """
 from enum import Enum, auto
+import itertools
 import json
 import logging
 import re
 
 from flask import Blueprint, Response, current_app, request, jsonify
-from lxml import etree
 
 from inventory_provider import juniper
 from inventory_provider.routes import common
@@ -605,6 +605,39 @@ def _load_services(config, hostname=None, use_next_redis=False):
     return result
 
 
+def _load_netconf_docs(
+        config, filter_pattern, use_next_redis=False):
+    """
+    yields dicts like:
+        {
+            'router': router hostname
+            'netconf': loaded netconf xml doc
+        }
+
+    :param config: app config
+    :param filter_pattern: search filter, including 'netconf:'
+    :param use_next_redis: use next instead of current redis, if true
+    :return: yields netconf docs, formatted as above
+    """
+
+    m = re.match(r'^(.*netconf:).+', filter_pattern)
+    # TODO: probably better to not required netconf: to be passed in
+    assert m  # sanity
+    key_prefix_len = len(m.group(1))
+    assert key_prefix_len >= len('netconf:')  # sanity
+
+    for doc in common.load_xml_docs(
+            config_params=config,
+            key_pattern=filter_pattern,
+            num_threads=10,
+            use_next_redis=use_next_redis):
+
+        yield {
+            'router': doc['key'][key_prefix_len:],
+            'netconf': doc['value']
+        }
+
+
 def _load_interfaces(
         config, hostname=None, no_lab=False, use_next_redis=False):
     """
@@ -617,25 +650,14 @@ def _load_interfaces(
     """
     def _load_docs(key_pattern):
 
-        m = re.match(r'^(.*netconf:).+', key_pattern)
-        assert m  # sanity
-        key_prefix_len = len(m.group(1))
-        assert key_prefix_len >= len('netconf:')  # sanity
-
-        for doc in common.load_xml_docs(
-                config_params=config,
-                key_pattern=key_pattern,
-                num_threads=10,
-                use_next_redis=use_next_redis):
-
-            router = doc['key'][key_prefix_len:]
+        for doc in _load_netconf_docs(config, key_pattern, use_next_redis):
 
-            for ifc in juniper.list_interfaces(doc['value']):
+            for ifc in juniper.list_interfaces(doc['netconf']):
                 if not ifc['description']:
                     continue
 
                 yield {
-                    'router': router,
+                    'router': doc['router'],
                     'name': ifc['name'],
                     'bundle': ifc['bundle'],
                     'bundle-parents': [],
@@ -894,18 +916,35 @@ def interface_speeds(hostname=None):
     return Response(result, mimetype="application/json")
 
 
-MX1_FRA = 'mx1.fra.de.geant.net'
+def _load_community_strings(base_key_pattern):
+    for doc in _load_netconf_docs(
+            config=current_app.config['INVENTORY_PROVIDER_CONFIG'],
+            filter_pattern=base_key_pattern):
+        community = juniper.snmp_community_string(doc['netconf'])
+        if not community:
+            yield {
+                'router': doc['router'],
+                'error':
+                    f'error extracting community string for {doc["router"]}'
+            }
+        else:
+            yield {
+                'router': doc['router'],
+                'community': community
+            }
 
 
 @routes.route('/eumetsat-multicast', methods=['GET', 'POST'])
 @routes.route('/eumetsat-multicast/<hostname>', methods=['GET', 'POST'])
 @common.require_accepts_json
-def eumetsat_multicast(hostname=MX1_FRA):
+def eumetsat_multicast(hostname=None):
     """
     Handler for `/poller/eumetsat-multicast` which returns information about
     multicast subscriptions on mx1.fra.de.geant.net.
 
-    The hostname is optional, with default value mx1.fra.de.geant.net
+    The hostname parameter is optional.  If it is present, only hostnames
+    matching `hostname*` are returned.  If not present, data for all
+    `mx*` routers is returned.
 
     The response is a list of oid/router/community structures that all
     all subscription octet counters to be polled.
@@ -934,41 +973,52 @@ def eumetsat_multicast(hostname=MX1_FRA):
                 f'.{sub["subscription"]}.{sub["endpoint"]}'
                 '.255.255.255.255')
 
-    cache_key = f'classifier-cache:poller-eumetsat-multicast:{hostname}'
-
     r = common.get_current_redis()
 
+    cache_key = 'classifier-cache:poller-eumetsat-multicast'
+    if hostname:
+        cache_key = f'{cache_key}:{hostname}'
+
     result = r.get(cache_key)
     if result:
         result = result.decode('utf-8')
     else:
-        netconf = r.get(f'netconf:{hostname}')
-        if not netconf:
-            return Response(
-                status=503,
-                response=f'error loading netconf for {hostname}')
 
-        netconf_doc = etree.fromstring(netconf.decode('utf-8'))
-        community = juniper.snmp_community_string(netconf_doc)
-        if not community:
-            # HACKHACK: vpn source ip isn't in acl
-            # TODO: remove this when done testing!
-            community = '0pBiFbD'
-        if not community:
+        def _multicast_oids(router_info):
+
+            def _rsp_element(sub):
+                result = {
+                    'router': router_info['router'],
+                    'oid': _oid(sub),
+                    'community': router_info['community']
+                }
+                result.update(sub)
+                return result
+
+            yield from map(_rsp_element, SUBSCRIPTIONS)
+
+        routers = list(_load_community_strings(
+            base_key_pattern=f'netconf:{hostname}*'
+            if hostname else 'netconf:mx*'))
+        errors = list(filter(lambda x: 'error' in x, routers))
+        if errors:
+            errors = [e['error'] for e in errors]
             return Response(
-                status=503,
-                response=f'error extracting community string for {hostname}')
+                response=', '.join(errors),
+                status=403,  # forbidden
+                mimetype='text/html')
 
-        def _rsp_element(sub):
-            result = {
-                'router': hostname,
-                'oid': _oid(sub),
-                'community': community
-            }
-            result.update(sub)
-            return result
+        assert all('community' in r for r in routers)  # sanity
 
-        result = [_rsp_element(sub) for sub in SUBSCRIPTIONS]
+        result = list(map(_multicast_oids, routers))
+        result = itertools.chain(*result)
+        result = list(result)
+        if not result:
+            target = hostname or 'any routers!'
+            return Response(
+                response=f'no multicast config for {target}',
+                status=404,
+                mimetype='text/html')
 
         result = json.dumps(result)
         # cache this data for the next call

test/test_general_poller_routes.py

@@ -50,7 +50,7 @@ def test_all_router_interface_speeds(client):
         'there should data from be lots of routers'
 
 
-def test_eumetsat_multicast(mocker, client):
+def test_eumetsat_multicast_all(mocker, client):
 
     # routers don't have snmp acl's for us
     mocker.patch('inventory_provider.juniper.snmp_community_string') \
@@ -66,6 +66,58 @@ def test_eumetsat_multicast(mocker, client):
         response_data, poller.MULTICAST_SUBSCRIPTION_LIST_SCHEMA)
     assert response_data, "the subscription list shouldn't be empty"
 
+    # only 'mx*' routers are returned, by default
+    assert all([s['router'].startswith('mx') for s in response_data])
+
+
+@pytest.mark.parametrize('hostname', [
+    'mx1.ams.nl.geant.net',
+    'mx1.ams',
+    'qfx.fra.de.geant.net'  # expect to be able to explicitly select others
+])
+def test_eumetsat_multicast_hostname(mocker, client, hostname):
+
+    # routers don't have snmp acl's for us
+    mocker.patch('inventory_provider.juniper.snmp_community_string') \
+        .return_value = 'blah'
+
+    rv = client.get(
+        f'/poller/eumetsat-multicast/{hostname}',
+        headers=DEFAULT_REQUEST_HEADERS)
+    assert rv.status_code == 200
+    assert rv.is_json
+    response_data = json.loads(rv.data.decode('utf-8'))
+    jsonschema.validate(
+        response_data, poller.MULTICAST_SUBSCRIPTION_LIST_SCHEMA)
+    assert response_data, "the subscription list shouldn't be empty"
+
+    # only 'mx*' routers are returned, by default
+    assert all([s['router'].startswith(hostname) for s in response_data])
+
+
+def test_eumetsat_multicast_404(mocker, client):
+
+    # routers don't have snmp acl's for us
+    mocker.patch('inventory_provider.juniper.snmp_community_string') \
+        .return_value = 'blah'
+
+    rv = client.get(
+        '/poller/eumetsat-multicast/XYZ123',
+        headers=DEFAULT_REQUEST_HEADERS)
+    assert rv.status_code == 404
+
+
+def test_eumetsat_multicast_forbidden(mocker, client):
+
+    # routers don't have snmp acl's for us
+    mocker.patch('inventory_provider.juniper.snmp_community_string') \
+        .return_value = ''
+
+    rv = client.get(
+        '/poller/eumetsat-multicast',
+        headers=DEFAULT_REQUEST_HEADERS)
+    assert rv.status_code == 403
+
 
 def test_gws_direct(client):