pass raw console output from ssh process

inventory_provider/juniper.py

@@ -8,9 +8,12 @@ import paramiko
 from inventory_provider.constants import JUNIPER_LOGGER_NAME
 
 
-def neighbors(router, routing_instances=["IAS"], group_expression=None):
+def neighbors(
+        parsed_json_output,
+        routing_instances=["IAS"],
+        group_expression=r'^GEANT-IX(v6)?[\s-].*$'):
 
-    for config in router["bgp"]["configuration"]:
+    for config in parsed_json_output["configuration"]:
         for ri in config["routing-instances"]:
             for inst in ri["instance"]:
                 if inst["name"]["data"] not in routing_instances:
@@ -25,8 +28,8 @@ def neighbors(router, routing_instances=["IAS"], group_expression=None):
                                 yield n
 
 
-def interfaces(router):
-    for ifc_info in router["interfaces"]["interface-information"]:
+def interfaces(parsed_json_output):
+    for ifc_info in parsed_json_output["interface-information"]:
         for ifc_list in ifc_info["logical-interface"]:
             for ifc in ifc_list:
                 yield {
@@ -40,50 +43,66 @@ def interfaces(router):
 
 
 @contextlib.contextmanager
-def ssh_connection(router, ssh_params):
+def ssh_connection(hostname, ssh_params):
     import os
     key_filename = os.path.join(os.path.dirname(__file__), ssh_params["private-key"])
     known_hosts = os.path.join(os.path.dirname(__file__), ssh_params["known-hosts"])
     k = paramiko.DSSKey.from_private_key_file(key_filename)
 
-    router["hostname"] = "mx1.ams.nl.geant.net"
     with paramiko.SSHClient() as ssh:
         ssh.load_host_keys(known_hosts)
         ssh.connect(
-            hostname=router["hostname"],
+            hostname=hostname,
             username="Monit0r",
             pkey=k)
         yield ssh
 
 
-def exec_router_commands_json(router, ssh_params, commands):
+def ssh_exec_commands(hostname, ssh_params, commands):
     juniper_logger = logging.getLogger(JUNIPER_LOGGER_NAME)
-    with ssh_connection(router, ssh_params) as ssh:
+    with ssh_connection(hostname, ssh_params) as ssh:
 
         _, stdout, _ = ssh.exec_command("set cli screen-length 0")
         assert stdout.channel.recv_exit_status() == 0
 
-        def _dups_to_list(pairs):
-            counter_map = {}
-            for k, v in pairs:
-                counter_map.setdefault(k, []).append(v)
-            result = {}
-            for k, v in counter_map.items():
-                if len(v) == 1:
-                    result[k] = v[0]
-                else:
-                    result[k] = v
-            return result
-
         for c in commands:
-            juniper_logger.debug("command: '%s'" % (c + " | display json"))
-            _, stdout, _ = ssh.exec_command(c + " | display json")
+            juniper_logger.debug("command: '%s'" % c)
+            _, stdout, _ = ssh.exec_command(c)
             assert stdout.channel.recv_exit_status() == 0
             # TODO: error handling
-            output = stdout.read()
-            if output:
-                juniper_logger.debug("%r output: [%d] %r" % (router, len(output), output[:20]))
-                yield json.loads(output, object_pairs_hook=_dups_to_list)
+            yield stdout.read().decode("utf-8")
+
+
+def shell_commands():
+
+    def _dups_to_list(pairs):
+        counter_map = {}
+        for k, v in pairs:
+            counter_map.setdefault(k, []).append(v)
+        result = {}
+        for k, v in counter_map.items():
+            if len(v) == 1:
+                result[k] = v[0]
             else:
-                juniper_logger.debug("%r output empty" % router)
-                yield {}
+                result[k] = v
+        return result
+
+    yield {
+        "command": "set cli screen-length 0",
+        "parser": lambda _: (None, None)
+    }
+
+    yield {
+        "command": 'show configuration routing-instances IAS protocols bgp | display json',
+        "parser": lambda txt: ("bgp", list(neighbors(json.loads(txt))) if txt else {})
+    }
+
+    yield {
+        "command": 'show configuration logical-systems VRR protocols bgp | display json',
+        "parser": lambda txt: ("vrr", json.loads(txt) if txt else {})
+    }
+
+    yield {
+        "command": 'show interfaces descriptions | display json',
+        "parser": lambda txt: ("interfaces", list(interfaces(json.loads(txt, object_pairs_hook=_dups_to_list))) if txt else {})
+    }

inventory_provider/router_interfaces.py

@@ -17,11 +17,11 @@ def get_router_interfaces_q(router, params, q):
     threading_logger.debug("[<<EXIT]  get_router_interfaces_q: %r" % router)
 
 
-def exec_router_commands_json_q(router, ssh_params, commands, q):
+def ssh_exec_commands_q(hostname, ssh_params, commands, q):
     threading_logger = logging.getLogger(constants.THREADING_LOGGER_NAME)
-    threading_logger.debug("[ENTER>>] exec_router_commands_q: %r" % router)
-    q.put(list(juniper.exec_router_commands_json(router, ssh_params, commands)))
-    threading_logger.debug("[<<EXIT] exec_router_commands_q: %r" % router)
+    threading_logger.debug("[ENTER>>] exec_router_commands_q: %r" % hostname)
+    q.put(list(juniper.ssh_exec_commands(hostname, ssh_params, commands)))
+    threading_logger.debug("[<<EXIT] exec_router_commands_q: %r" % hostname)
 
 
 def get_router_details(router, params, q):
@@ -30,28 +30,38 @@ def get_router_details(router, params, q):
 
     threading_logger.debug("[ENTER>>]get_router_details: %r" % router)
 
-    commands = [
-        'show configuration routing-instances IAS protocols bgp',
-        # 'show configuration routing-instances IAS protocols bgp | display set | match neighbor | match description | match "GEANT-IX | GEANT-IX-"',
-        # 'show configuration routing-instances IAS protocols bgp | display set | match neighbor | match description | match "GEANT-IXv6 | GEANT-IXv6-"',
-        'show configuration logical-systems VRR protocols bgp',
-        # 'show configuration logical-systems VRR protocols bgp group VPN-RR-INTERNAL | match "neigh|desc"',
-        # 'show configuration logical-systems VRR protocols bgp group VPN-RR | match "neigh|desc"'
-        'show interfaces descriptions'
-    ]
+    commands = list(juniper.shell_commands())
 
     snmpifc_proc_queue = Queue()
-    snmpifc_proc = Process(target=get_router_interfaces_q, args=(router, params, snmpifc_proc_queue))
+    snmpifc_proc = Process(
+        target=get_router_interfaces_q,
+        args=(router, params, snmpifc_proc_queue))
     snmpifc_proc.start()
 
     commands_proc_queue = Queue()
-    commands_proc = Process(target=exec_router_commands_json_q, args=(router, params["ssh"], commands, commands_proc_queue))
+    commands_proc = Process(
+        target=ssh_exec_commands_q,
+        args=(
+            router["hostname"],
+            params["ssh"],
+            [c["command"] for c in commands],
+            commands_proc_queue))
     commands_proc.start()
 
     threading_logger.debug("waiting for commands result: %r" % router)
     command_output = commands_proc_queue.get()
     assert len(command_output) == len(commands)
-    result = dict(zip(["bgp", "vrr", "interfaces"], command_output))
+
+    for i, o in enumerate(command_output):
+        with open("/tmp/%s-%d.output" % (router["hostname"], i), "w") as f:
+            f.write(o)
+
+    result = {}
+    for c, o in zip(commands, command_output):
+        parsed = c["parser"](o)
+        if parsed[0]:
+            result[parsed[0]] = parsed[1]
+
     commands_proc.join()
     threading_logger.debug("... got commands result & joined: %r" % router)