Merge branch 'feature/add-interface-bundle-data' into develop

inventory_provider/juniper.py

@@ -213,12 +213,16 @@ def list_interfaces(netconf_config):
         assert name is not None, "expected interface 'name' child element"
         ifc = {
             'name': name.text,
-            'description': ''
+            'description': '',
+            'bundle': []
         }
         description = e.find('description')
         if description is not None:
             ifc['description'] = description.text
 
+        for b in i.iterfind(".//bundle"):
+            ifc['bundle'].append(b.text)
+
         ifc['ipv4'] = e.xpath('./family/inet/address/name/text()')
         ifc['ipv6'] = e.xpath('./family/inet6/address/name/text()')
 

inventory_provider/routes/poller.py

@@ -62,8 +62,14 @@ def poller_interface_oids(hostname):
         if not snmp_index:
             continue
 
+        bundle_parents = r.get('netconf-interface-bundles:%s:%s' % (
+            hostname, ifc['name'].split('.')[0]))
+
         ifc_data = {
             'name': ifc['name'],
+            'bundle': ifc['bundle'],
+            'bundle-parents':
+                json.loads(bundle_parents) if bundle_parents else [],
             'snmp-index': snmp_index,
             'description': ifc['description'],
             'circuits': []

inventory_provider/tasks/worker.py

@@ -2,7 +2,7 @@ import json
 import logging
 import re
 
-from celery import bootsteps, Task, group, states
+from celery import bootsteps, Task, states
 from celery.result import AsyncResult
 
 from collections import defaultdict
@@ -337,10 +337,23 @@ def refresh_juniper_interface_list(hostname, netconf):
     for k in r.keys('netconf-interfaces:%s:*' % hostname):
         r.delete(k)
 
+    for k in r.keys('netconf-interface-bundles:%s:*' % hostname):
+        r.delete(k)
+
+    all_bundles = defaultdict(list)
     for ifc in juniper.list_interfaces(netconf):
+        bundles = ifc.get('bundle', None)
+        for bundle in bundles:
+            if bundle:
+                all_bundles[bundle].append(ifc['name'])
+
         r.set(
             'netconf-interfaces:%s:%s' % (hostname, ifc['name']),
             json.dumps(ifc))
+    for k, v in all_bundles.items():
+        r.set(
+            'netconf-interface-bundles:%s:%s' % (hostname, k),
+            json.dumps(v))
 
 
 @app.task(base=InventoryTask, bind=True)
@@ -458,27 +471,28 @@ def launch_refresh_cache_all(config):
 
     # first batch of subtasks: refresh cached opsdb data
     subtasks = [
-        update_junosspace_device_list.s(),
-        update_interfaces_to_services.s(),
-        update_geant_lambdas.s(),
-        update_circuit_hierarchy.s()
+        update_junosspace_device_list.apply_async(),
+        update_interfaces_to_services.apply_async(),
+        update_geant_lambdas.apply_async(),
+        update_circuit_hierarchy.apply_async()
     ]
-
-    results = group(subtasks).apply_async()
-    results.join()
+    [x.get() for x in subtasks]
 
     # second batch of subtasks:
     #   alarms db status cache
     #   juniper netconf & snmp data
     subtasks = [
-        update_equipment_locations.s(),
+        update_equipment_locations.apply_async(),
     ]
     for hostname in _derive_router_hostnames(config):
         logger.debug(
             'queueing router refresh jobs for %r' % hostname)
-        subtasks.append(reload_router_config.s(hostname))
+        subtasks.append(reload_router_config.apply_async(args=[hostname]))
+        break
+
+    [x.get() for x in subtasks]
 
-    return [r.id for r in group(subtasks).apply_async()]
+    return [x.id for x in subtasks]
 
 
 def check_task_status(task_id):

test/per_router/test_data_routes.py

@@ -18,6 +18,10 @@ def test_router_interfaces(router, client_with_mocked_data):
             "properties": {
                 "name": {"type": "string"},
                 "description": {"type": "string"},
+                "bundle": {
+                    "type": "array",
+                    "items": {"type": "string"}
+                },
                 "ipv4": {
                     "type": "array",
                     "items": {"type": "string"}

test/per_router/test_juniper_data.py

@@ -52,6 +52,10 @@ def test_interface_list(netconf_doc):
             "properties": {
                 "name": {"type": "string"},
                 "description": {"type": "string"},
+                "bundle": {
+                    "type": "array",
+                    "items": {"type": "string"}
+                },
                 "ipv4": {
                     "type": "array",
                     "items": {"type": "string"}

test/per_router/test_poller_routes.py

@@ -34,11 +34,25 @@ def test_router_interfaces(router, client_with_mocked_data):
                     "type": "array",
                     "items": {"$ref": "#/definitions/circuit"}
                 },
+                "bundle": {
+                    "type": "array",
+                    "items": {"type": "string"}
+                },
+                "bundle-parents": {
+                    "type": "array",
+                    "items": {"type": "string"}
+                },
                 "description": {"type": "string"},
                 "name": {"type": "string"},
                 "snmp-index": {"type": "integer"}
             },
-            "required": ["circuits", "description", "name", "snmp-index"],
+            "required": [
+                "circuits",
+                "bundle",
+                "bundle-parents",
+                "description",
+                "name",
+                "snmp-index"],
             "additionalProperties": False
         }
     }