added ssh cli output capture of bgp, interfaces, etc.

inventory_provider/router_interfaces.py

 import contextlib
 import json
 import re
-from multiprocessing import Pool
+from multiprocessing import Pool, Process, Queue
 
 import click
 import jsonschema
 import mysql.connector
+import paramiko
 from pysnmp.hlapi import nextCmd, SnmpEngine, CommunityData, \
         UdpTransportTarget, ContextData, ObjectType, ObjectIdentity
 
@@ -25,7 +26,16 @@ CONFIG_SCHEMA = {
             "additionalProperties": False
         },
         "oid_list.conf": {"type": "string"},
-        "routers_community.conf": {"type": "string"}
+        "routers_community.conf": {"type": "string"},
+        "ssh": {
+            "type": "object",
+            "properties": {
+                "private-key": {"type": "string"},
+                "known-hosts": {"type": "string"}
+            },
+            "required": ["private-key", "known-hosts"],
+            "additionalProperties": False
+        }
     },
     "required": ["alarms-db", "oid_list.conf", "routers_community.conf"],
     "additionalProperties": False
@@ -218,8 +228,93 @@ def get_router_interfaces(router):
         }
 
 
-def get_router_interfaces_l(router):
-    return list(get_router_interfaces(router))
+@contextlib.contextmanager
+def ssh_connection(router, ssh_params):
+    import os
+    key_filename = os.path.join(os.path.dirname(__file__), ssh_params["private-key"])
+    known_hosts = os.path.join(os.path.dirname(__file__), ssh_params["known-hosts"])
+    k = paramiko.DSSKey.from_private_key_file(key_filename)
+
+    router["hostname"] = "mx1.ams.nl.geant.net"
+    with paramiko.SSHClient() as ssh:
+        ssh.load_host_keys(known_hosts)
+        ssh.connect(
+            hostname=router["hostname"],
+            username="Monit0r",
+            pkey=k)
+        yield ssh
+
+
+def exec_router_commands_json(router, ssh_params, commands):
+    with ssh_connection(router, ssh_params) as ssh:
+
+        _, stdout, _ = ssh.exec_command("set cli screen-length 0")
+        assert stdout.channel.recv_exit_status() == 0
+
+        for c in commands:
+            print("command: '%s'" % (c + " | display json"))
+            _, stdout, _ = ssh.exec_command(c + " | display json")
+            assert stdout.channel.recv_exit_status() == 0
+            # TODO: error handling
+            # output = stdout.read()
+            output = stdout.read()
+            if output:
+                print("%r output: [%d] %r" % (router, len(output), output[:20]))
+                yield json.loads(output)
+            else:
+                print("%r output empty")
+                yield {}
+
+
+def get_router_interfaces_q(router, q):
+    print("[ENTER>>] get_router_interfaces_q: %r" % router)
+    q.put(list(get_router_interfaces(router)))
+    print("[<<EXIT]  get_router_interfaces_q: %r" % router)
+
+
+def exec_router_commands_json_q(router, ssh_params, commands, q):
+    print("[ENTER>>] exec_router_commands_q: %r" % router)
+    q.put(list(exec_router_commands_json(router, ssh_params, commands)))
+    print("[<<EXIT] exec_router_commands_q: %r" % router)
+
+
+def get_router_details(router, params, q):
+
+    print("get_router_details: %r" % router)
+
+    commands = [
+        'show configuration routing-instances IAS protocols bgp',
+        # 'show configuration routing-instances IAS protocols bgp | display set | match neighbor | match description | match "GEANT-IX | GEANT-IX-"',
+        # 'show configuration routing-instances IAS protocols bgp | display set | match neighbor | match description | match "GEANT-IXv6 | GEANT-IXv6-"',
+        'show configuration logical-systems VRR protocols bgp',
+        # 'show configuration logical-systems VRR protocols bgp group VPN-RR-INTERNAL | match "neigh|desc"',
+        # 'show configuration logical-systems VRR protocols bgp group VPN-RR | match "neigh|desc"'
+        'show interfaces descriptions'
+    ]
+
+    snmpifc_proc_queue = Queue()
+    snmpifc_proc = Process(target=get_router_interfaces_q, args=(router,snmpifc_proc_queue))
+    snmpifc_proc.start()
+
+    commands_proc_queue = Queue()
+    commands_proc = Process(target=exec_router_commands_json_q, args=(router, params["ssh"], commands, commands_proc_queue))
+    commands_proc.start()
+
+    result = {}
+
+    print("waiting for commands result: %r" % router)
+    command_output = commands_proc_queue.get()
+    assert len(command_output) == len(commands)
+    result = dict(zip(["bgp", "vrr", "interfaces"], command_output))
+    commands_proc.join()
+    print("... got commands result & joined: %r" % router)
+
+    print("waiting for snmp ifc results: %r" % router)
+    result["snmp-interfaces"] = snmpifc_proc_queue.get()
+    snmpifc_proc.join()
+    print("... got snmp ifc result & joined: %r" % router)
+
+    q.put(result)
 
 
 @click.command()
@@ -228,17 +323,53 @@ def get_router_interfaces_l(router):
 #    required=True,
     type=click.File(),
     help="Configuration filename",
+
     default=open("config.json"),
     callback=_validate_config)
 def cli(config):
-    pool = Pool(20)
     with open("routers_community.conf") as f:
         routers = list(load_routers(f))
-    for r, i in zip(routers, pool.map(get_router_interfaces_l, routers)):
-        print(r["hostname"])
-        for ifc in i:
-            print("\t%r" % ifc)
 
+    processes = []
+    for r in routers:
+        q = Queue()
+        p = Process(target=get_router_details, args=(r, config, q))
+        p.start()
+        processes.append({"router": r, "process": p, "queue": q})
+
+    result = {}
+    for p in processes:
+        print("waiting for get_router_details result: %r" % p["router"])
+        result[p["router"]["hostname"]] = p["queue"].get()
+        p["process"].join()
+        print("got result and joined get_router_details proc: %r" % p["router"])
+
+    filename = "/tmp/router-info.json"
+    with open(filename, "w") as f:
+        f.write(json.dumps(result))
+
+    print("writing output to: " + filename)
+
+    # for r, i in zip(routers, pool.map(get_router_details, args)):
+    #     print(r)
+    #     print(i)
+    #     print("")
+    #     break
+    #     # i = get_router_interfaces(r)
+    #     # print(r["hostname"])
+    #     # for ifc in i:
+    #     #     print("\t%r" % ifc)
+    #     # break
+    #     # # commands = [
+    #     # #     'show configuration routing-instances IAS protocols bgp | display set | match neighbor | match description | match "GEANT-IX | GEANT-IX-"',
+    #     # #     'show configuration routing-instances IAS protocols bgp | display set | match neighbor | match description | match "GEANT-IXv6 | GEANT-IXv6-"',
+    #     # #     'show configuration logical-systems VRR protocols bgp group VPN-RR-INTERNAL | match "neigh|desc"',
+    #     # #     'show configuration logical-systems VRR protocols bgp group VPN-RR | match "neigh|desc"'
+    #     # # ]
+    #     # # for lines in exec_router_commands(r, config["ssh"], commands):
+    #     # #     print("-------------")
+    #     # #     print("".join(lines))
+    #     # break
 
 if __name__ == "__main__":
     cli()

requirements.txt

@@ -2,4 +2,4 @@ click
 mysql-connector
 pysnmp
 jsonschema
-pika
+paramiko