Finished feature derive-snmp-community-string.

changelog

@@ -22,4 +22,5 @@
 0.16<working>: flatten redis storage structure
      poller api
      classifier metadata api
-
+     read snmp community string from netconf
+     derive active router list from junosspace

inventory_provider/juniper.py

 import logging
 import re
+import ipaddress
 
 from jnpr.junos import Device
 from lxml import etree
+import netifaces
 import requests
 from requests.auth import HTTPBasicAuth
 
@@ -314,3 +316,44 @@ def load_routers_from_junosspace(config):
             "name": name,
             "hostname": hostname
         }
+
+
+def local_interfaces(
+        type=netifaces.AF_INET,
+        omit_link_local=True,
+        omit_loopback=True):
+    """
+    generator yielding IPv4Interface or IPv6Interface objects,
+    depending on the value of type
+    :param type: hopefully AF_INET or AF_INET6
+    :param omit_link_local: skip v6 fe80* addresses if true
+    :param omit_loopback: skip lo* interfaces if true
+    :return:
+    """
+    for n in netifaces.interfaces():
+        if omit_loopback and re.match(r'^lo\d+', n):
+            continue
+        am = netifaces.ifaddresses(n)
+        for a in am.get(type, []):
+            if omit_link_local and a['addr'].startswith('fe80:'):
+                continue
+            m = re.match(r'^(.+?)(%.*)?$', a['addr'])
+            assert m
+            addr = m.group(1)
+            m = re.match(r'.*/(\d+)$', a['netmask'])
+            if m:
+                mask = m.group(1)
+            else:
+                mask = a['netmask']
+            yield ipaddress.ip_interface('%s/%s' % (addr, mask))
+
+
+def snmp_community_string(netconf_config):
+    my_addressess = list([i.ip for i in local_interfaces()])
+    for community in netconf_config.xpath('//configuration/snmp/community'):
+        for subnet in community.xpath('./clients/name/text()'):
+            allowed_network = ipaddress.ip_network(subnet, strict=False)
+            for me in my_addressess:
+                if me in allowed_network:
+                    return community.xpath('./name/text()')[0]
+    return None

inventory_provider/tasks/worker.py

@@ -215,6 +215,53 @@ def update_junosspace_device_list():
     task_logger.debug('<<< update_junosspace_device_list')
 
 
+def load_netconf_data(hostname):
+    """
+    this method should only be called from a task
+
+    :param hostname:
+    :return:
+    """
+    r = get_redis(InventoryTask.config)
+    netconf = r.get('netconf:' + hostname)
+    if not netconf:
+        return None
+    return etree.fromstring(netconf.decode('utf-8'))
+
+
+def clear_cached_classifier_responses(hostname):
+    task_logger = logging.getLogger(constants.TASK_LOGGER_NAME)
+    task_logger.debug(
+        'removing cached classifier responses for %r' % hostname)
+    r = get_redis(InventoryTask.config)
+    for k in r.keys('classifier:cache:%s:*' % hostname):
+        r.delete(k)
+
+
+@app.task()
+def update_router_config(hostname):
+    task_logger = logging.getLogger(constants.TASK_LOGGER_NAME)
+    task_logger.debug('>>> update_router_config')
+
+    netconf_refresh_config.apply(hostname)
+
+    netconf_doc = load_netconf_data(hostname)
+    if not netconf_doc:
+        task_logger.error('no netconf data available for %r' % hostname)
+    else:
+        community = juniper.snmp_community_string(netconf_doc)
+        if not community:
+            task_logger.error(
+                'error extracting community string for %r' % hostname)
+        else:
+            snmp_refresh_interfaces.apply(hostname, community)
+
+            # TODO: move this out of else? (i.e. clear even if netconf fails?)
+            clear_cached_classifier_responses(hostname)
+
+    task_logger.debug('<<< update_router_config')
+
+
 def _derive_router_hostnames(config):
     r = get_redis(config)
     junosspace_equipment = set()
@@ -261,12 +308,6 @@ def start_refresh_cache_all(config):
     for hostname in _derive_router_hostnames(config):
         task_logger.debug(
             'queueing router refresh jobs for %r' % hostname)
-
-        # TODO: !!!! extract community string from netconf data
-        task_logger.error(
-            'TODO: !!!! extract community string from netconf data')
-        subtasks.append(netconf_refresh_config.s(hostname))
-        # TODO: these should be synchronous, and then cleanup classifier cache
-        subtasks.append(snmp_refresh_interfaces.s(hostname, '0pBiFbD'))
+        subtasks.append(update_router_config.s(hostname))
 
     return group(subtasks).apply_async()

requirements.txt

@@ -9,6 +9,7 @@ celery
 junos-eznc
 lxml
 requests
+netifaces
 
 pytest
 pytest-mock

setup.py

@@ -20,7 +20,8 @@ setup(
         'celery',
         'junos-eznc',
         'lxml',
-        'requests'
+        'requests',
+        'netifaces'
     ],
     include_package_data=True,
 )

test/per_router/test_juniper_data.py

+import ast
 import os
 
 import jsonschema
@@ -98,3 +99,22 @@ def test_bgp_list(netconf_doc):
 
     routes = list(juniper.list_bgp_routes(netconf_doc))
     jsonschema.validate(routes, schema)
+
+
+NETIFACES_TEST_DATA_STRING = """{
+    'lo0':  {2: [{'addr': '127.0.0.1', 'netmask': '255.0.0.0', 'peer': '127.0.0.1'}],
+             30: [{'addr': '::1', 'netmask': 'ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff/128', 'peer': '::1', 'flags': 0},
+                 {'addr': 'fe80::1%lo0', 'netmask': 'ffff:ffff:ffff:ffff::/64', 'flags': 0}]},
+    'eth0': {18: [{'addr': '78:4f:43:76:73:ba'}],
+             2: [{'addr': '83.97.92.239', 'netmask': '255.255.252.0', 'broadcast': '83.97.95.255'}],
+             30: [{'addr': 'fe80::250:56ff:fea1:8340', 'netmask': 'ffff:ffff:ffff:ffff::/64', 'flags': 1024},
+                 {'addr': '2001:798:3::104', 'netmask': 'ffff:ffff:ffff:ffff::/64', 'flags': 1088}]}
+}"""  # noqa E501
+
+NETIFACES_TEST_DATA = ast.literal_eval(NETIFACES_TEST_DATA_STRING)
+
+
+def test_snmp_community_string(mocker, netconf_doc):
+    mocker.patch('netifaces.interfaces', lambda: NETIFACES_TEST_DATA.keys())
+    mocker.patch('netifaces.ifaddresses', lambda n: NETIFACES_TEST_DATA[n])
+    assert juniper.snmp_community_string(netconf_doc) == '0pBiFbD'

test/test_juniper_data_global.py

+import ast
+import netifaces
+import ipaddress
+
+from inventory_provider import juniper
+
+NETIFACES_TEST_DATA_STRING = """{
+    'lo0': {2: [{'addr': '127.0.0.1', 'netmask': '255.0.0.0', 'peer': '127.0.0.1'}],
+            30: [{'addr': '::1', 'netmask': 'ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff/128', 'peer': '::1', 'flags': 0},
+                 {'addr': 'fe80::1%lo0', 'netmask': 'ffff:ffff:ffff:ffff::/64', 'flags': 0}]},
+    'gif0': {},
+    'stf0': {},
+    'XHC20': {},
+    'XHC4': {},
+    'XHC3': {},
+    'en3': {18: [{'addr': 'b6:00:24:b9:f0:01'}]},
+    'en8': {18: [{'addr': 'b6:00:24:b9:f0:00'}]},
+    'en4': {18: [{'addr': 'b6:00:24:b9:f0:05'}]},
+    'en9': {18: [{'addr': 'b6:00:24:b9:f0:04'}]},
+    'en0': {18: [{'addr': '78:4f:43:76:73:ba'}],
+            2: [{'addr': '195.169.24.149', 'netmask': '255.255.255.128', 'broadcast': '195.169.24.255'}],
+            30: [{'addr': 'fe80::1c97:ec77:3f32:cdfe%en0', 'netmask': 'ffff:ffff:ffff:ffff::/64', 'flags': 1024},
+                 {'addr': '2001:610:9d8:4:4d7:f763:9815:e78d', 'netmask': 'ffff:ffff:ffff:ffff::/64', 'flags': 1088},
+                 {'addr': '2001:610:9d8:4:492e:61b6:2c92:c387', 'netmask': 'ffff:ffff:ffff:ffff::/64', 'flags': 192}]},
+    'p2p0': {18: [{'addr': '0a:4f:43:76:73:ba'}]},
+    'awdl0': {18: [{'addr': '8e:87:e3:bb:92:1f'}],
+              30: [{'addr': 'fe80::8c87:e3ff:febb:921f%awdl0', 'netmask': 'ffff:ffff:ffff:ffff::/64', 'flags': 0}]},
+    'bridge0': {18: [{'addr': 'b6:00:24:b9:f0:01'}]},
+    'utun0': {30: [{'addr': 'fe80::8328:d0ef:52b4:d379%utun0', 'netmask': 'ffff:ffff:ffff:ffff::/64', 'flags': 0}]},
+    'utun1': {30: [{'addr': 'fe80::5a75:c789:2fa0:6ee4%utun1', 'netmask': 'ffff:ffff:ffff:ffff::/64', 'flags': 0}]},
+    'XHC0': {},
+    'XHC1': {},
+    'XHC2': {},
+    'en21': {18: [{'addr': '64:4b:f0:10:23:25'}],
+             2: [{'addr': '195.169.24.170', 'netmask': '255.255.255.128', 'broadcast': '195.169.24.255'}],
+             30: [{'addr': 'fe80::41c:798c:3fff:f8c9%en21', 'netmask': 'ffff:ffff:ffff:ffff::/64', 'flags': 1024},
+                  {'addr': '2001:610:9d8:4:c1e:4402:e7cf:547f', 'netmask': 'ffff:ffff:ffff:ffff::/64', 'flags': 1088},
+                  {'addr': '2001:610:9d8:4:911c:954d:d4e2:baef', 'netmask': 'ffff:ffff:ffff:ffff::/64', 'flags': 192}]},
+    'en5': {18: [{'addr': 'ac:de:48:00:11:22'}],
+            30: [{'addr': 'fe80::aede:48ff:fe00:1122%en5', 'netmask': 'ffff:ffff:ffff:ffff::/64', 'flags': 0}]},
+    'en18': {18: [{'addr': 'ca:3c:85:86:34:2a'}], 2: [{'addr': '169.254.184.83', 'netmask': '255.255.0.0'}]},
+}"""  # noqa: E501
+
+NETIFACES_TEST_DATA = ast.literal_eval(NETIFACES_TEST_DATA_STRING)
+
+
+def test_local_v4_interfaces(mocker):
+    mocker.patch('netifaces.interfaces', lambda: NETIFACES_TEST_DATA.keys())
+    mocker.patch('netifaces.ifaddresses', lambda n: NETIFACES_TEST_DATA[n])
+    addresses = list(juniper.local_interfaces())
+    assert len(addresses) == 3
+    for a in addresses:
+        assert isinstance(a, ipaddress.IPv4Interface)
+
+
+def test_local_v6_interfaces(mocker):
+    mocker.patch('netifaces.interfaces', lambda: NETIFACES_TEST_DATA.keys())
+    mocker.patch('netifaces.ifaddresses', lambda n: NETIFACES_TEST_DATA[n])
+    addresses = list(juniper.local_interfaces(netifaces.AF_INET6))
+    assert len(addresses) == 4
+    for a in addresses:
+        assert isinstance(a, ipaddress.IPv6Interface)