initial flattening of redis structure

inventory_provider/juniper.py

@@ -111,7 +111,6 @@ def load_config(hostname, ssh_params):
     :return:
     """
     juniper_logger = logging.getLogger(JUNIPER_LOGGER_NAME)
-    config = _rpc(hostname, ssh_params).get_config()
 
     juniper_logger.info("capturing netconf data for '%s'" % hostname)
 
@@ -124,6 +123,8 @@ def load_config(hostname, ssh_params):
 
     schema_doc = etree.XML(CONFIG_SCHEMA.encode('utf-8'))
     config_schema = etree.XMLSchema(schema_doc)
+
+    config = _rpc(hostname, ssh_params).get_config()
     _validate(config_schema, config)
 
     # validate interfaces/interface/unit elements ...

inventory_provider/routes/jobs.py

@@ -12,8 +12,8 @@ routes = Blueprint("inventory-data-job-routes", __name__)
 def update():
 
     task_logger = logging.getLogger(TASK_LOGGER_NAME)
-    config = current_app.config["INVENTORY_PROVIDER_CONFIG"]
 
+    config = current_app.config["INVENTORY_PROVIDER_CONFIG"]
     for r in config["routers"]:
 
         task_logger.info("fetching details for: %r" % r)
@@ -34,6 +34,10 @@ def update():
             'inventory_provider.tasks.worker.snmp_refresh_interfaces',
             args=[r["hostname"], r["community"]])
 
+    task_logger.debug(
+        'launching task: '
+        'inventory_provider.tasks.worker.update_inventory_system_cache')
+
     return Response("OK")
 
 

inventory_provider/tasks/worker.py

@@ -29,18 +29,18 @@ class InventoryTask(Task):
     def __init__(self):
         pass
 
-    @staticmethod
-    def save_key(hostname, key, value):
-        assert isinstance(value, str), \
-            "sanity failure: expected string data as value"
-        r = get_redis(InventoryTask.config)
-        r.hset(
-            name=hostname,
-            key=key,
-            value=value)
-        InventoryTask.logger.debug(
-            "saved %s, key %s" % (hostname, key))
-        return "OK"
+    # @staticmethod
+    # def save_key(hostname, key, value):
+    #     assert isinstance(value, str), \
+    #         "sanity failure: expected string data as value"
+    #     r = get_redis(InventoryTask.config)
+    #     r.hset(
+    #         name=hostname,
+    #         key=key,
+    #         value=value)
+    #     InventoryTask.logger.debug(
+    #         "saved %s, key %s" % (hostname, key))
+    #     return "OK"
 
     @staticmethod
     def save_value(key, value):
@@ -60,19 +60,25 @@ class InventoryTask(Task):
             json.dumps(data_obj))
 
     @staticmethod
-    def save_key_json(hostname, key, data_obj):
-        InventoryTask.save_key(
-            hostname,
-            key,
-            json.dumps(data_obj))
-
-    @staticmethod
-    def save_key_etree(hostname, key, xml_doc):
-        InventoryTask.save_key(
-            hostname,
+    def save_value_etree(key, xml_doc):
+        InventoryTask.save_value(
             key,
             etree.tostring(xml_doc, encoding='unicode'))
 
+    # @staticmethod
+    # def save_key_json(hostname, key, data_obj):
+    #     InventoryTask.save_key(
+    #         hostname,
+    #         key,
+    #         json.dumps(data_obj))
+    #
+    # @staticmethod
+    # def save_key_etree(hostname, key, xml_doc):
+    #     InventoryTask.save_key(
+    #         hostname,
+    #         key,
+    #         etree.tostring(xml_doc, encoding='unicode'))
+
 
 class WorkerArgs(bootsteps.Step):
     def __init__(self, worker, config_filename, **options):
@@ -81,11 +87,11 @@ class WorkerArgs(bootsteps.Step):
         InventoryTask.logger = logging.getLogger(constants.TASK_LOGGER_NAME)
 
 
-interfaces_key = "interface_services"
-equipment_locations_key = "equipment_locations"
-service_child_to_parents_key = "child_to_parent_circuit_relations"
-service_parent_to_children_key = "parent_to_children_circuit_relations"
-interface_status_key = "interface_statuses"
+# interfaces_key = "interface_services"
+# equipment_locations_key = "equipment_locations"
+# service_child_to_parents_key = "child_to_parent_circuit_relations"
+# service_parent_to_children_key = "parent_to_children_circuit_relations"
+# interface_status_key = "interface_statuses"
 
 
 def worker_args(parser):
@@ -107,9 +113,8 @@ def snmp_refresh_interfaces(self, hostname, community):
     logger.debug('STARTING: snmp_refresh_interfaces(%r, %r)'
                  % (hostname, community))
 
-    InventoryTask.save_key_json(
-        hostname,
-        "snmp-interfaces",
+    InventoryTask.save_value_json(
+        'snmp-interfaces:' + hostname,
         list(snmp.get_router_interfaces(
             hostname,
             community,
@@ -124,9 +129,8 @@ def netconf_refresh_config(self, hostname):
     logger = logging.getLogger(constants.TASK_LOGGER_NAME)
     logger.debug('STARTING: netconf_refresh_config(%r)' % hostname)
 
-    InventoryTask.save_key_etree(
-        hostname,
-        "netconf",
+    InventoryTask.save_value_etree(
+        'netconf:' + hostname,
         juniper.load_config(hostname, InventoryTask.config["ssh"]))
 
     logger.debug('FINISHED: netconf_refresh_config(%r)' % hostname)
@@ -146,88 +150,69 @@ def netconf_refresh_config(self, hostname):
 
 @app.task()
 def update_interfaces_to_services():
-    logger = logging.getLogger(constants.TASK_LOGGER_NAME)
-    logger.error('HERE: update_interfaces_to_services')
+    interface_services = defaultdict(list)
+    with db.connection(InventoryTask.config["ops-db"]) as cx:
+        for service in opsdb.get_circuits(cx):
+            equipment_interface = '%s:%s' % (
+                service['equipment'], service['interface_name'])
+            interface_services[equipment_interface].append(service)
 
     r = get_redis(InventoryTask.config)
-    with db.connection(InventoryTask.config["ops-db"]) as cx:
-        services = opsdb.get_circuits(cx)
-
-    mapped_interfaces = defaultdict(list)
-    for service in services:
-        key = "{}::{}".format(
-            service["equipment"],
-            service["interface_name"]
-        )
-        mapped_interfaces[key].append(service)
-        # Puts lu services under the parent ae as well as their own interface
-        # eg. services on ae15.12 would be found under ae15 as well as ae15.12
-        if "." in service["interface_name"]:
-            key = "{}::{}".format(
-                service["equipment"],
-                service["interface_name"].split(".")[0]
-            )
-            mapped_interfaces[key].append(service)
-
-    r.delete(interfaces_key)
-    for key, value in mapped_interfaces.items():
-        r.hset(interfaces_key, key, json.dumps(value))
+    for key in r.scan_iter('opsdb:interface_services:*'):
+        r.delete(key)
+    for equipment_interface, services in interface_services.items():
+        r.set(
+            'opsdb:interface_services:' + equipment_interface,
+            json.dumps(services))
 
 
 @app.task()
 def update_equipment_locations():
     r = get_redis(InventoryTask.config)
-    r.delete(equipment_locations_key)
-
+    for key in r.scan_iter('opsdb:location:*'):
+        r.delete(key)
     with db.connection(InventoryTask.config["ops-db"]) as cx:
         for ld in opsdb.get_equipment_location_data(cx):
-            r.hset(
-                equipment_locations_key, ld["equipment_name"], json.dumps(ld))
+            r.set('opsdb:location:%s' % ld['equipment_name'], json.dumps(ld))
 
 
 @app.task()
 def update_circuit_hierarchy():
-    r = get_redis(InventoryTask.config)
-    children_to_parents = defaultdict(list)
-    parents_to_children = defaultdict(list)
     with db.connection(InventoryTask.config["ops-db"]) as cx:
-        records = opsdb.get_circuit_hierarchy(cx)
-        for relation in records:
+        child_to_parents = defaultdict(list)
+        parent_to_children = defaultdict(list)
+        for relation in opsdb.get_circuit_hierarchy(cx):
             parent_id = relation["parent_circuit_id"]
             child_id = relation["child_circuit_id"]
-            parents_to_children[parent_id].append(relation)
-            children_to_parents[child_id].append(relation)
+            parent_to_children[parent_id].append(relation)
+            child_to_parents[child_id].append(relation)
 
-        r.delete(service_child_to_parents_key)
-        for child, parents in children_to_parents.items():
-            r.hset(service_child_to_parents_key, child, json.dumps(parents))
+        r = get_redis(InventoryTask.config)
+        for key in r.scan_iter('opsdb:services:parents:*'):
+            r.delete(key)
+        for cid, parents in child_to_parents.items():
+            r.set('opsdb:services:parents:%d' % cid, json.dumps(parents))
 
-        r.delete(service_parent_to_children_key)
-        for parent, children in parents_to_children.items():
-            r.hset(
-                service_parent_to_children_key, parent, json.dumps(children))
+        for key in r.scan_iter('opsdb:services:children:*'):
+            r.delete(key)
+        for cid, children in child_to_parents.items():
+            r.set('opsdb:services:children:%d' % cid, json.dumps(children))
 
 
 @app.task()
 def update_interface_statuses():
-    r = get_redis(InventoryTask.config)
     with db.connection(InventoryTask.config["ops-db"]) as cx:
             services = opsdb.get_circuits(cx)
     with db.connection(InventoryTask.config["alarms-db"]) as cx:
         with db.cursor(cx) as csr:
             for service in services:
-                key = "{}::{}".format(
+                key = 'alarmsdb:interface_status:%s:%s' \
+                    % (service['equipment'], service['interface_name'])
+                status = alarmsdb.get_last_known_interface_status(
+                    csr,
                     service["equipment"],
-                    service["interface_name"]
-                )
-                if not r.hexists(interface_status_key, key):
-                    r.hset(interface_status_key,
-                           key,
-                           alarmsdb.get_last_known_interface_status(
-                               csr,
-                               service["equipment"],
-                               service["interface_name"]
-                           ))
+                    service["interface_name"])
+                InventoryTask.save_value(key, status)
 
 
 @app.task()