diff --git a/changelog b/changelog
index 4b04fcdd101c47623ff16b64290838c77b334d73..d4e390378b9398fce00ceaa70ebaf18c64c1283c 100644
--- a/changelog
+++ b/changelog
@@ -22,4 +22,5 @@
0.16<working>: flatten redis storage structure
poller api
classifier metadata api
-
+ read snmp community string from netconf
+ derive active router list from junosspace
diff --git a/inventory_provider/juniper.py b/inventory_provider/juniper.py
index 8dca8768b1e0ec76a1fc9968a5fa5ef85eddc70e..680648625818dee03abe9dcb11f204df55a2e59d 100644
--- a/inventory_provider/juniper.py
+++ b/inventory_provider/juniper.py
@@ -1,8 +1,10 @@
import logging
import re
+import ipaddress
from jnpr.junos import Device
from lxml import etree
+import netifaces
import requests
from requests.auth import HTTPBasicAuth
@@ -314,3 +316,44 @@ def load_routers_from_junosspace(config):
"name": name,
"hostname": hostname
}
+
+
+def local_interfaces(
+ type=netifaces.AF_INET,
+ omit_link_local=True,
+ omit_loopback=True):
+ """
+ generator yielding IPv4Interface or IPv6Interface objects,
+ depending on the value of type
+ :param type: hopefully AF_INET or AF_INET6
+ :param omit_link_local: skip v6 fe80* addresses if true
+ :param omit_loopback: skip lo* interfaces if true
+ :return:
+ """
+ for n in netifaces.interfaces():
+ if omit_loopback and re.match(r'^lo\d+', n):
+ continue
+ am = netifaces.ifaddresses(n)
+ for a in am.get(type, []):
+ if omit_link_local and a['addr'].startswith('fe80:'):
+ continue
+ m = re.match(r'^(.+?)(%.*)?$', a['addr'])
+ assert m
+ addr = m.group(1)
+ m = re.match(r'.*/(\d+)$', a['netmask'])
+ if m:
+ mask = m.group(1)
+ else:
+ mask = a['netmask']
+ yield ipaddress.ip_interface('%s/%s' % (addr, mask))
+
+
+def snmp_community_string(netconf_config):
+ my_addressess = list([i.ip for i in local_interfaces()])
+ for community in netconf_config.xpath('//configuration/snmp/community'):
+ for subnet in community.xpath('./clients/name/text()'):
+ allowed_network = ipaddress.ip_network(subnet, strict=False)
+ for me in my_addressess:
+ if me in allowed_network:
+ return community.xpath('./name/text()')[0]
+ return None
diff --git a/inventory_provider/tasks/worker.py b/inventory_provider/tasks/worker.py
index 5cc2be20438d1a8173828f2501f2b137dcc33a7f..685cf38174ddd71b8e42eb1e18520e4595c83fd2 100644
--- a/inventory_provider/tasks/worker.py
+++ b/inventory_provider/tasks/worker.py
@@ -215,6 +215,53 @@ def update_junosspace_device_list():
task_logger.debug('<<< update_junosspace_device_list')
+def load_netconf_data(hostname):
+ """
+ this method should only be called from a task
+
+ :param hostname:
+ :return:
+ """
+ r = get_redis(InventoryTask.config)
+ netconf = r.get('netconf:' + hostname)
+ if not netconf:
+ return None
+ return etree.fromstring(netconf.decode('utf-8'))
+
+
+def clear_cached_classifier_responses(hostname):
+ task_logger = logging.getLogger(constants.TASK_LOGGER_NAME)
+ task_logger.debug(
+ 'removing cached classifier responses for %r' % hostname)
+ r = get_redis(InventoryTask.config)
+ for k in r.keys('classifier:cache:%s:*' % hostname):
+ r.delete(k)
+
+
+@app.task()
+def update_router_config(hostname):
+ task_logger = logging.getLogger(constants.TASK_LOGGER_NAME)
+ task_logger.debug('>>> update_router_config')
+
+ netconf_refresh_config.apply(hostname)
+
+ netconf_doc = load_netconf_data(hostname)
+ if not netconf_doc:
+ task_logger.error('no netconf data available for %r' % hostname)
+ else:
+ community = juniper.snmp_community_string(netconf_doc)
+ if not community:
+ task_logger.error(
+ 'error extracting community string for %r' % hostname)
+ else:
+ snmp_refresh_interfaces.apply(hostname, community)
+
+ # TODO: move this out of else? (i.e. clear even if netconf fails?)
+ clear_cached_classifier_responses(hostname)
+
+ task_logger.debug('<<< update_router_config')
+
+
def _derive_router_hostnames(config):
r = get_redis(config)
junosspace_equipment = set()
@@ -261,12 +308,6 @@ def start_refresh_cache_all(config):
for hostname in _derive_router_hostnames(config):
task_logger.debug(
'queueing router refresh jobs for %r' % hostname)
-
- # TODO: !!!! extract community string from netconf data
- task_logger.error(
- 'TODO: !!!! extract community string from netconf data')
- subtasks.append(netconf_refresh_config.s(hostname))
- # TODO: these should be synchronous, and then cleanup classifier cache
- subtasks.append(snmp_refresh_interfaces.s(hostname, '0pBiFbD'))
+ subtasks.append(update_router_config.s(hostname))
return group(subtasks).apply_async()
diff --git a/requirements.txt b/requirements.txt
index ff5a3796d6da2c9e606022c04fc5806a5f353eb5..815ac1d5e52e62e71ff23f0a037d4e30be705bb3 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -9,6 +9,7 @@ celery
junos-eznc
lxml
requests
+netifaces
pytest
pytest-mock
diff --git a/setup.py b/setup.py
index cacec6f53c463e17d73d2a10471fa2a9f217daff..0989421fee7a8fe127d25d1176c7bce8b54abfe4 100644
--- a/setup.py
+++ b/setup.py
@@ -20,7 +20,8 @@ setup(
'celery',
'junos-eznc',
'lxml',
- 'requests'
+ 'requests',
+ 'netifaces'
],
include_package_data=True,
)
diff --git a/test/per_router/test_juniper_data.py b/test/per_router/test_juniper_data.py
index 32b31990b7c33690af3c27288b20b306061942d4..7847e322edd15ba2e85e87836a6a37ffc1322467 100644
--- a/test/per_router/test_juniper_data.py
+++ b/test/per_router/test_juniper_data.py
@@ -1,3 +1,4 @@
+import ast
import os
import jsonschema
@@ -98,3 +99,22 @@ def test_bgp_list(netconf_doc):
routes = list(juniper.list_bgp_routes(netconf_doc))
jsonschema.validate(routes, schema)
+
+
+NETIFACES_TEST_DATA_STRING = """{
+ 'lo0': {2: [{'addr': '127.0.0.1', 'netmask': '255.0.0.0', 'peer': '127.0.0.1'}],
+ 30: [{'addr': '::1', 'netmask': 'ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff/128', 'peer': '::1', 'flags': 0},
+ {'addr': 'fe80::1%lo0', 'netmask': 'ffff:ffff:ffff:ffff::/64', 'flags': 0}]},
+ 'eth0': {18: [{'addr': '78:4f:43:76:73:ba'}],
+ 2: [{'addr': '83.97.92.239', 'netmask': '255.255.252.0', 'broadcast': '83.97.95.255'}],
+ 30: [{'addr': 'fe80::250:56ff:fea1:8340', 'netmask': 'ffff:ffff:ffff:ffff::/64', 'flags': 1024},
+ {'addr': '2001:798:3::104', 'netmask': 'ffff:ffff:ffff:ffff::/64', 'flags': 1088}]}
+}""" # noqa E501
+
+NETIFACES_TEST_DATA = ast.literal_eval(NETIFACES_TEST_DATA_STRING)
+
+
+def test_snmp_community_string(mocker, netconf_doc):
+ mocker.patch('netifaces.interfaces', lambda: NETIFACES_TEST_DATA.keys())
+ mocker.patch('netifaces.ifaddresses', lambda n: NETIFACES_TEST_DATA[n])
+ assert juniper.snmp_community_string(netconf_doc) == '0pBiFbD'
diff --git a/test/test_juniper_data_global.py b/test/test_juniper_data_global.py
new file mode 100644
index 0000000000000000000000000000000000000000..74454f003e531ee2070a99c52cd3860997978355
--- /dev/null
+++ b/test/test_juniper_data_global.py
@@ -0,0 +1,62 @@
+import ast
+import netifaces
+import ipaddress
+
+from inventory_provider import juniper
+
+NETIFACES_TEST_DATA_STRING = """{
+ 'lo0': {2: [{'addr': '127.0.0.1', 'netmask': '255.0.0.0', 'peer': '127.0.0.1'}],
+ 30: [{'addr': '::1', 'netmask': 'ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff/128', 'peer': '::1', 'flags': 0},
+ {'addr': 'fe80::1%lo0', 'netmask': 'ffff:ffff:ffff:ffff::/64', 'flags': 0}]},
+ 'gif0': {},
+ 'stf0': {},
+ 'XHC20': {},
+ 'XHC4': {},
+ 'XHC3': {},
+ 'en3': {18: [{'addr': 'b6:00:24:b9:f0:01'}]},
+ 'en8': {18: [{'addr': 'b6:00:24:b9:f0:00'}]},
+ 'en4': {18: [{'addr': 'b6:00:24:b9:f0:05'}]},
+ 'en9': {18: [{'addr': 'b6:00:24:b9:f0:04'}]},
+ 'en0': {18: [{'addr': '78:4f:43:76:73:ba'}],
+ 2: [{'addr': '195.169.24.149', 'netmask': '255.255.255.128', 'broadcast': '195.169.24.255'}],
+ 30: [{'addr': 'fe80::1c97:ec77:3f32:cdfe%en0', 'netmask': 'ffff:ffff:ffff:ffff::/64', 'flags': 1024},
+ {'addr': '2001:610:9d8:4:4d7:f763:9815:e78d', 'netmask': 'ffff:ffff:ffff:ffff::/64', 'flags': 1088},
+ {'addr': '2001:610:9d8:4:492e:61b6:2c92:c387', 'netmask': 'ffff:ffff:ffff:ffff::/64', 'flags': 192}]},
+ 'p2p0': {18: [{'addr': '0a:4f:43:76:73:ba'}]},
+ 'awdl0': {18: [{'addr': '8e:87:e3:bb:92:1f'}],
+ 30: [{'addr': 'fe80::8c87:e3ff:febb:921f%awdl0', 'netmask': 'ffff:ffff:ffff:ffff::/64', 'flags': 0}]},
+ 'bridge0': {18: [{'addr': 'b6:00:24:b9:f0:01'}]},
+ 'utun0': {30: [{'addr': 'fe80::8328:d0ef:52b4:d379%utun0', 'netmask': 'ffff:ffff:ffff:ffff::/64', 'flags': 0}]},
+ 'utun1': {30: [{'addr': 'fe80::5a75:c789:2fa0:6ee4%utun1', 'netmask': 'ffff:ffff:ffff:ffff::/64', 'flags': 0}]},
+ 'XHC0': {},
+ 'XHC1': {},
+ 'XHC2': {},
+ 'en21': {18: [{'addr': '64:4b:f0:10:23:25'}],
+ 2: [{'addr': '195.169.24.170', 'netmask': '255.255.255.128', 'broadcast': '195.169.24.255'}],
+ 30: [{'addr': 'fe80::41c:798c:3fff:f8c9%en21', 'netmask': 'ffff:ffff:ffff:ffff::/64', 'flags': 1024},
+ {'addr': '2001:610:9d8:4:c1e:4402:e7cf:547f', 'netmask': 'ffff:ffff:ffff:ffff::/64', 'flags': 1088},
+ {'addr': '2001:610:9d8:4:911c:954d:d4e2:baef', 'netmask': 'ffff:ffff:ffff:ffff::/64', 'flags': 192}]},
+ 'en5': {18: [{'addr': 'ac:de:48:00:11:22'}],
+ 30: [{'addr': 'fe80::aede:48ff:fe00:1122%en5', 'netmask': 'ffff:ffff:ffff:ffff::/64', 'flags': 0}]},
+ 'en18': {18: [{'addr': 'ca:3c:85:86:34:2a'}], 2: [{'addr': '169.254.184.83', 'netmask': '255.255.0.0'}]},
+}""" # noqa: E501
+
+NETIFACES_TEST_DATA = ast.literal_eval(NETIFACES_TEST_DATA_STRING)
+
+
+def test_local_v4_interfaces(mocker):
+ mocker.patch('netifaces.interfaces', lambda: NETIFACES_TEST_DATA.keys())
+ mocker.patch('netifaces.ifaddresses', lambda n: NETIFACES_TEST_DATA[n])
+ addresses = list(juniper.local_interfaces())
+ assert len(addresses) == 3
+ for a in addresses:
+ assert isinstance(a, ipaddress.IPv4Interface)
+
+
+def test_local_v6_interfaces(mocker):
+ mocker.patch('netifaces.interfaces', lambda: NETIFACES_TEST_DATA.keys())
+ mocker.patch('netifaces.ifaddresses', lambda n: NETIFACES_TEST_DATA[n])
+ addresses = list(juniper.local_interfaces(netifaces.AF_INET6))
+ assert len(addresses) == 4
+ for a in addresses:
+ assert isinstance(a, ipaddress.IPv6Interface)