diff --git a/inventory_provider/juniper.py b/inventory_provider/juniper.py index 468eef553af60b98d553a4c9b756b1db5069536f..575388892d6fd24292436ab21e8624c2a52adc2d 100644 --- a/inventory_provider/juniper.py +++ b/inventory_provider/juniper.py @@ -213,12 +213,16 @@ def list_interfaces(netconf_config): assert name is not None, "expected interface 'name' child element" ifc = { 'name': name.text, - 'description': '' + 'description': '', + 'bundle': [] } description = e.find('description') if description is not None: ifc['description'] = description.text + for b in i.iterfind(".//bundle"): + ifc['bundle'].append(b.text) + ifc['ipv4'] = e.xpath('./family/inet/address/name/text()') ifc['ipv6'] = e.xpath('./family/inet6/address/name/text()') diff --git a/inventory_provider/routes/poller.py b/inventory_provider/routes/poller.py index 93179717b4ce524fac454964bacf2016e40d0ebd..a48de48bc17acc7e5e4536f3a3851277bed7517b 100644 --- a/inventory_provider/routes/poller.py +++ b/inventory_provider/routes/poller.py @@ -62,8 +62,14 @@ def poller_interface_oids(hostname): if not snmp_index: continue + bundle_parents = r.get('netconf-interface-bundles:%s:%s' % ( + hostname, ifc['name'].split('.')[0])) + ifc_data = { 'name': ifc['name'], + 'bundle': ifc['bundle'], + 'bundle-parents': + json.loads(bundle_parents) if bundle_parents else [], 'snmp-index': snmp_index, 'description': ifc['description'], 'circuits': [] diff --git a/inventory_provider/tasks/worker.py b/inventory_provider/tasks/worker.py index d1800813488a1042558beb66b5c30607ccdf5135..9515ce7b30ca80f6e280e425baa002eed12513ce 100644 --- a/inventory_provider/tasks/worker.py +++ b/inventory_provider/tasks/worker.py @@ -2,7 +2,7 @@ import json import logging import re -from celery import bootsteps, Task, group, states +from celery import bootsteps, Task, states from celery.result import AsyncResult from collections import defaultdict @@ -337,10 +337,23 @@ def refresh_juniper_interface_list(hostname, netconf): for k in r.keys('netconf-interfaces:%s:*' % hostname): r.delete(k) + for k in r.keys('netconf-interface-bundles:%s:*' % hostname): + r.delete(k) + + all_bundles = defaultdict(list) for ifc in juniper.list_interfaces(netconf): + bundles = ifc.get('bundle', None) + for bundle in bundles: + if bundle: + all_bundles[bundle].append(ifc['name']) + r.set( 'netconf-interfaces:%s:%s' % (hostname, ifc['name']), json.dumps(ifc)) + for k, v in all_bundles.items(): + r.set( + 'netconf-interface-bundles:%s:%s' % (hostname, k), + json.dumps(v)) @app.task(base=InventoryTask, bind=True) @@ -458,27 +471,28 @@ def launch_refresh_cache_all(config): # first batch of subtasks: refresh cached opsdb data subtasks = [ - update_junosspace_device_list.s(), - update_interfaces_to_services.s(), - update_geant_lambdas.s(), - update_circuit_hierarchy.s() + update_junosspace_device_list.apply_async(), + update_interfaces_to_services.apply_async(), + update_geant_lambdas.apply_async(), + update_circuit_hierarchy.apply_async() ] - - results = group(subtasks).apply_async() - results.join() + [x.get() for x in subtasks] # second batch of subtasks: # alarms db status cache # juniper netconf & snmp data subtasks = [ - update_equipment_locations.s(), + update_equipment_locations.apply_async(), ] for hostname in _derive_router_hostnames(config): logger.debug( 'queueing router refresh jobs for %r' % hostname) - subtasks.append(reload_router_config.s(hostname)) + subtasks.append(reload_router_config.apply_async(args=[hostname])) + break + + [x.get() for x in subtasks] - return [r.id for r in group(subtasks).apply_async()] + return [x.id for x in subtasks] def check_task_status(task_id): diff --git a/test/per_router/test_data_routes.py b/test/per_router/test_data_routes.py index aa47ad7d9f517381e7659e24c7d479bc498092bf..ec4762ccdced8a4d8b70f3eea262dd9716373a66 100644 --- a/test/per_router/test_data_routes.py +++ b/test/per_router/test_data_routes.py @@ -18,6 +18,10 @@ def test_router_interfaces(router, client_with_mocked_data): "properties": { "name": {"type": "string"}, "description": {"type": "string"}, + "bundle": { + "type": "array", + "items": {"type": "string"} + }, "ipv4": { "type": "array", "items": {"type": "string"} diff --git a/test/per_router/test_juniper_data.py b/test/per_router/test_juniper_data.py index 721477640e199595517261c8107d954c39160de5..3c99a0862cad888415eed692ba0b00e90e2503f6 100644 --- a/test/per_router/test_juniper_data.py +++ b/test/per_router/test_juniper_data.py @@ -52,6 +52,10 @@ def test_interface_list(netconf_doc): "properties": { "name": {"type": "string"}, "description": {"type": "string"}, + "bundle": { + "type": "array", + "items": {"type": "string"} + }, "ipv4": { "type": "array", "items": {"type": "string"} diff --git a/test/per_router/test_poller_routes.py b/test/per_router/test_poller_routes.py index be5ce39c49d8f2cd9c9c44cebaf9cba6459a945c..41ce0f2655db8205cfe7c3b242c85e98db1d213b 100644 --- a/test/per_router/test_poller_routes.py +++ b/test/per_router/test_poller_routes.py @@ -34,11 +34,25 @@ def test_router_interfaces(router, client_with_mocked_data): "type": "array", "items": {"$ref": "#/definitions/circuit"} }, + "bundle": { + "type": "array", + "items": {"type": "string"} + }, + "bundle-parents": { + "type": "array", + "items": {"type": "string"} + }, "description": {"type": "string"}, "name": {"type": "string"}, "snmp-index": {"type": "integer"} }, - "required": ["circuits", "description", "name", "snmp-index"], + "required": [ + "circuits", + "bundle", + "bundle-parents", + "description", + "name", + "snmp-index"], "additionalProperties": False } }