move all cache updating to single task

inventory_provider/routes/jobs.py

@@ -4,42 +4,17 @@ from flask import Blueprint, Response, current_app
 
 from inventory_provider.tasks.app import app
 from inventory_provider.constants import TASK_LOGGER_NAME
-
 routes = Blueprint("inventory-data-job-routes", __name__)
 
 
 @routes.route("/update", methods=['GET', 'POST'])
 def update():
-
     task_logger = logging.getLogger(TASK_LOGGER_NAME)
-
-    config = current_app.config["INVENTORY_PROVIDER_CONFIG"]
-    for r in config["routers"]:
-
-        task_logger.info("fetching details for: %r" % r)
-
-        task_logger.debug(
-            'launching task: '
-            'inventory_provider.tasks.worker.netconf_refresh_config'
-            '(%s)' % r['hostname'])
-        app.send_task(
-            'inventory_provider.tasks.worker.netconf_refresh_config',
-            args=[r["hostname"]])
-
-        task_logger.debug(
-            'launching task: '
-            'inventory_provider.tasks.worker.snmp_refresh_interfaces'
-            '(%s)' % r['hostname'])
-        app.send_task(
-            'inventory_provider.tasks.worker.snmp_refresh_interfaces',
-            args=[r["hostname"], r["community"]])
-
     task_logger.debug(
         'launching task: '
-        'inventory_provider.tasks.worker.update_inventory_system_cache')
+        'inventory_provider.tasks.worker.refresh_cache_all')
     app.send_task(
-        'inventory_provider.tasks.worker.update_inventory_system_cache')
-
+        'inventory_provider.tasks.worker.refresh_cache_all')
     return Response("OK")
 
 
@@ -51,7 +26,7 @@ def update():
 #             'inventory_provider.tasks.worker.update_alarmsdb_cache')
 #     app.send_task(
 #         'inventory_provider.tasks.worker.update_alarmsdb_cache')
-#     return Response("OK")
+    return Response("OK")
 
 
 # @routes.route("update-interfaces-to-services", methods=['GET'])

inventory_provider/tasks/worker.py

 import json
 import logging
+import re
 
 from celery import bootsteps, Task, group
-
+from celery.result import allow_join_result
 
 from collections import defaultdict
 from lxml import etree
@@ -242,3 +243,61 @@ def update_junosspace_device_list():
         r.set(
             'junosspace:' + d['hostname'],
             json.dumps(d).encode('utf-8'))
+
+
+def _derive_router_hostnames(config):
+
+    r = get_redis(config)
+    junosspace_equipment = set()
+    for k in r.keys('junosspace:*'):
+        m = re.match('^junosspace:(.*)$', k.decode('utf-8'))
+        assert m
+        junosspace_equipment.add(m.group(1))
+
+    opsdb_equipment = set()
+    for k in r.keys('opsdb:interface_services:*'):
+        m = re.match(
+            'opsdb:interface_services:([^:]+):.*$',
+            k.decode('utf-8'))
+        opsdb_equipment.add(m.group(1))
+
+    return junosspace_equipment & opsdb_equipment
+
+
+@app.task()
+def refresh_cache_all():
+    task_logger = logging.getLogger(constants.TASK_LOGGER_NAME)
+
+    task_logger.debug('starting update_junosspace_device_list,'
+                 'update_inventory_system_cache')
+    g = group([
+        update_junosspace_device_list.s(),
+        update_inventory_system_cache.s()
+    ])
+
+    results = g.apply_async()
+    with allow_join_result():
+        results.join()
+
+    for hostname in _derive_router_hostnames(InventoryTask.config):
+        task_logger.info("fetching details for: %r" % hostname)
+
+        task_logger.debug(
+            'launching task: '
+            'inventory_provider.tasks.worker.netconf_refresh_config'
+            '(%s)' % hostname)
+        app.send_task(
+            'inventory_provider.tasks.worker.netconf_refresh_config',
+            args=[hostname])
+
+        task_logger.debug(
+            'launching task: '
+            'inventory_provider.tasks.worker.snmp_refresh_interfaces'
+            '(%s)' % hostname)
+        # TODO: !!!! extract community string from netconf data
+        app.send_task(
+            'inventory_provider.tasks.worker.snmp_refresh_interfaces',
+            args=[hostname, '0pBiFbD'])
+
+    return "OK"
+