Finished feature redis-inventory-cache.

inventory_provider/config.py

@@ -29,9 +29,23 @@ CONFIG_SCHEMA = {
             },
             "required": ["private-key", "known-hosts"],
             "additionalProperties": False
+        },
+        "redis": {
+            "type": "object",
+            "properties": {
+                "hostname": {"type": "string"},
+                "port": {"type": "integer"}
+            },
+            "required": ["hostname", "port"],
+            "additionalProperties": False
         }
     },
-    "required": ["alarms-db", "oid_list.conf", "routers_community.conf"],
+    "required": [
+        "alarms-db",
+        "oid_list.conf",
+        "routers_community.conf",
+        "ssh",
+        "redis"],
     "additionalProperties": False
 }
 

inventory_provider/router_interfaces.py

@@ -3,6 +3,7 @@ import logging
 from multiprocessing import Process, Queue
 
 import click
+import redis
 
 from inventory_provider import constants
 from inventory_provider import snmp
@@ -24,7 +25,7 @@ def ssh_exec_commands_q(hostname, ssh_params, commands, q):
     threading_logger.debug("[<<EXIT] exec_router_commands_q: %r" % hostname)
 
 
-def get_router_details(router, params, q):
+def get_router_details(router, params):
 
     threading_logger = logging.getLogger(constants.THREADING_LOGGER_NAME)
 
@@ -52,21 +53,26 @@ def get_router_details(router, params, q):
     command_output = commands_proc_queue.get()
     assert len(command_output) == len(commands)
 
-    result = {}
+    r = redis.StrictRedis(
+        host=params["redis"]["hostname"],
+        port=params["redis"]["port"])
     for c, o in zip(commands, command_output):
         if c["key"]:
-            result[c["key"]] = c["parser"](o)
-
+            r.hset(
+                name=router["hostname"],
+                key=c["key"],
+                value=json.dumps(c["parser"](o)))
     commands_proc.join()
     threading_logger.debug("... got commands result & joined: %r" % router)
 
     threading_logger.debug("waiting for snmp ifc results: %r" % router)
-    result["snmp-interfaces"] = snmpifc_proc_queue.get()
+    r.hset(
+        name=router["hostname"],
+        key="snmp-interfaces",
+        value=json.dumps(snmpifc_proc_queue.get()))
     snmpifc_proc.join()
     threading_logger.debug("... got snmp ifc result & joined: %r" % router)
 
-    q.put(result)
-
     threading_logger.debug("[<<EXIT]get_router_details: %r" % router)
 
 
@@ -76,16 +82,14 @@ def load_network_details(params):
 
     processes = []
     for r in params["routers"]:
-        q = Queue()
-        p = Process(target=get_router_details, args=(r, params, q))
+        p = Process(target=get_router_details, args=(r, params))
         p.start()
-        processes.append({"router": r, "process": p, "queue": q})
+        processes.append({"router": r, "process": p})
 
     result = {}
     for p in processes:
         threading_logger.debug(
             "waiting for get_router_details result: %r" % p["router"])
-        result[p["router"]["hostname"]] = p["queue"].get()
         p["process"].join()
         threading_logger.debug(
             "got result and joined get_router_details proc: %r" % p["router"])
@@ -106,11 +110,25 @@ def _validate_config(ctx, param, value):
     default=open("config.json"),
     callback=_validate_config)
 def cli(params):
-    network_details = load_network_details(params)
+    load_network_details(params)
+
+    r = redis.StrictRedis(
+        host=params["redis"]["hostname"],
+        port=params["redis"]["port"])
+
+    result = {}
+    for hostname in r.keys():
+        host = {}
+        for key in r.hkeys(name=hostname):
+            host[key.decode("utf-8")] = json.loads(
+                r.hget(hostname, key).decode("utf-8")
+            )
+        result[hostname.decode("utf-8")] = host
+
     filename = "/tmp/router-info.json"
     logging.debug("writing output to: " + filename)
     with open(filename, "w") as f:
-        f.write(json.dumps(network_details))
+        f.write(json.dumps(result))
 
 
 if __name__ == "__main__":

requirements.txt

@@ -4,5 +4,6 @@ pysnmp
 jsonschema
 paramiko
 flask
+redis
 
 pytest

setup.py

@@ -15,6 +15,7 @@ setup(
         'pysnmp',
         'jsonschema',
         'paramiko',
-        'flask'
+        'flask',
+        'redis'
     ]
 )

test/test_data_routes.py

@@ -99,6 +99,10 @@ def data_config_filename(tmp_dir_name):
         "ssh": {
             "private-key": "private-key-filename",
             "known-hosts": "known-hosts=filename"
+        },
+        "redis": {
+            "hostname": "xxxxxx",
+            "port": 6379
         }
     }