return all mx* routers if hostname param isn't provided

inventory_provider/routes/poller.py

@@ -62,6 +62,7 @@ support method: _get_dashboards
 
 """
 from enum import Enum, auto
+import itertools
 import json
 import logging
 import re
@@ -605,6 +606,39 @@ def _load_services(config, hostname=None, use_next_redis=False):
     return result
 
 
+def _load_netconf_docs(
+        config, filter_pattern, use_next_redis=False):
+    """
+    yields dicts like:
+        {
+            'router': router hostname
+            'netconf': loaded netconf xml doc
+        }
+
+    :param config: app config
+    :param filter_pattern: search filter, including 'netconf:'
+    :param use_next_redis: use next instead of current redis, if true
+    :return: yields netconf docs, formatted as above
+    """
+
+    m = re.match(r'^(.*netconf:).+', filter_pattern)
+    # TODO: probably better to not required netconf: to be passed in
+    assert m  # sanity
+    key_prefix_len = len(m.group(1))
+    assert key_prefix_len >= len('netconf:')  # sanity
+
+    for doc in common.load_xml_docs(
+            config_params=config,
+            key_pattern=filter_pattern,
+            num_threads=10,
+            use_next_redis=use_next_redis):
+
+        yield {
+            'router': doc['key'][key_prefix_len:],
+            'netconf': doc['value']
+        }
+
+
 def _load_interfaces(
         config, hostname=None, no_lab=False, use_next_redis=False):
     """
@@ -617,25 +651,14 @@ def _load_interfaces(
     """
     def _load_docs(key_pattern):
 
-        m = re.match(r'^(.*netconf:).+', key_pattern)
-        assert m  # sanity
-        key_prefix_len = len(m.group(1))
-        assert key_prefix_len >= len('netconf:')  # sanity
+        for doc in _load_netconf_docs(config, key_pattern, use_next_redis):
 
-        for doc in common.load_xml_docs(
-                config_params=config,
-                key_pattern=key_pattern,
-                num_threads=10,
-                use_next_redis=use_next_redis):
-
-            router = doc['key'][key_prefix_len:]
-
-            for ifc in juniper.list_interfaces(doc['value']):
+            for ifc in juniper.list_interfaces(doc['netconf']):
                 if not ifc['description']:
                     continue
 
                 yield {
-                    'router': router,
+                    'router': doc['router'],
                     'name': ifc['name'],
                     'bundle': ifc['bundle'],
                     'bundle-parents': [],
@@ -894,18 +917,38 @@ def interface_speeds(hostname=None):
     return Response(result, mimetype="application/json")
 
 
-MX1_FRA = 'mx1.fra.de.geant.net'
+def _load_community_strings(base_key_pattern):
+    for doc in _load_netconf_docs(
+            config=current_app.config['INVENTORY_PROVIDER_CONFIG'],
+            filter_pattern=base_key_pattern):
+        community = juniper.snmp_community_string(doc['netconf'])
+        if not community:
+            # HACKHACK: vpn source ip isn't in acl
+            # TODO: remove this when done testing!
+            community = '0pBiFbD'
+        if not community:
+            yield {
+                'router': doc['router'],
+                'error': f'error extracting community string for {hostname}'
+            }
+        else:
+            yield {
+                'router': doc['router'],
+                'community': community
+            }
 
 
 @routes.route('/eumetsat-multicast', methods=['GET', 'POST'])
 @routes.route('/eumetsat-multicast/<hostname>', methods=['GET', 'POST'])
 @common.require_accepts_json
-def eumetsat_multicast(hostname=MX1_FRA):
+def eumetsat_multicast(hostname=None):
     """
     Handler for `/poller/eumetsat-multicast` which returns information about
     multicast subscriptions on mx1.fra.de.geant.net.
 
-    The hostname is optional, with default value mx1.fra.de.geant.net
+    The hostname parameter is optional.  If it is present, only hostnames
+    matching `hostname*` are returned.  If not present, data for all
+    `mx*` routers is returned.
 
     The response is a list of oid/router/community structures that all
     all subscription octet counters to be polled.
@@ -934,41 +977,53 @@ def eumetsat_multicast(hostname=MX1_FRA):
                 f'.{sub["subscription"]}.{sub["endpoint"]}'
                 '.255.255.255.255')
 
-    cache_key = f'classifier-cache:poller-eumetsat-multicast:{hostname}'
-
     r = common.get_current_redis()
 
+    cache_key = f'classifier-cache:poller-eumetsat-multicast'
+    if hostname:
+        cache_key = f'{cache_key}:{hostname}'
+
     result = r.get(cache_key)
     if result:
         result = result.decode('utf-8')
     else:
-        netconf = r.get(f'netconf:{hostname}')
-        if not netconf:
-            return Response(
-                status=503,
-                response=f'error loading netconf for {hostname}')
 
-        netconf_doc = etree.fromstring(netconf.decode('utf-8'))
-        community = juniper.snmp_community_string(netconf_doc)
-        if not community:
-            # HACKHACK: vpn source ip isn't in acl
-            # TODO: remove this when done testing!
-            community = '0pBiFbD'
-        if not community:
+        def _multicast_oids(router_info):
+
+            if not router_info['community']:
+                logger.error(f'this address can''t query {router["routers"]}')
+                return
+
+            def _rsp_element(sub):
+                result = {
+                    'router': router_info['router'],
+                    'oid': _oid(sub),
+                    'community': router_info['community']
+                }
+                result.update(sub)
+                return result
+
+            yield from map(_rsp_element, SUBSCRIPTIONS)
+
+        routers = list(_load_community_strings(
+            base_key_pattern=f'netconf:{hostname}*'
+            if hostname else 'netconf:mx*'))
+        errors = list(filter(lambda x: 'error' in x, routers))
+        if errors:
             return Response(
+                response=', '.join(errors),
                 status=503,
-                response=f'error extracting community string for {hostname}')
-
-        def _rsp_element(sub):
-            result = {
-                'router': hostname,
-                'oid': _oid(sub),
-                'community': community
-            }
-            result.update(sub)
-            return result
+                mimetype='text/html')
 
-        result = [_rsp_element(sub) for sub in SUBSCRIPTIONS]
+        result = list(map(_multicast_oids, routers))
+        result = itertools.chain(*result)
+        result = list(result)
+        if not result:
+            target = hostname or 'any routers!'
+            return Response(
+                response=f'no multicast config for {target}',
+                status=404,
+                mimetype='text/html')
 
         result = json.dumps(result)
         # cache this data for the next call