diff --git a/inventory_provider/juniper.py b/inventory_provider/juniper.py
index 55b76058341ada30f32970b4cced00990ceffee2..9cd737b456d507f5586cb6dfb2a28d6b72ed1d39 100644
--- a/inventory_provider/juniper.py
+++ b/inventory_provider/juniper.py
@@ -111,7 +111,6 @@ def load_config(hostname, ssh_params):
:return:
"""
juniper_logger = logging.getLogger(JUNIPER_LOGGER_NAME)
- config = _rpc(hostname, ssh_params).get_config()
juniper_logger.info("capturing netconf data for '%s'" % hostname)
@@ -124,6 +123,8 @@ def load_config(hostname, ssh_params):
schema_doc = etree.XML(CONFIG_SCHEMA.encode('utf-8'))
config_schema = etree.XMLSchema(schema_doc)
+
+ config = _rpc(hostname, ssh_params).get_config()
_validate(config_schema, config)
# validate interfaces/interface/unit elements ...
diff --git a/inventory_provider/routes/jobs.py b/inventory_provider/routes/jobs.py
index eddba59bffc7c10e3b66d3ac90252388caceb96b..be887d82d4093a89ad17be0c0f70605a38028fa8 100644
--- a/inventory_provider/routes/jobs.py
+++ b/inventory_provider/routes/jobs.py
@@ -12,8 +12,8 @@ routes = Blueprint("inventory-data-job-routes", __name__)
def update():
task_logger = logging.getLogger(TASK_LOGGER_NAME)
- config = current_app.config["INVENTORY_PROVIDER_CONFIG"]
+ config = current_app.config["INVENTORY_PROVIDER_CONFIG"]
for r in config["routers"]:
task_logger.info("fetching details for: %r" % r)
@@ -34,6 +34,10 @@ def update():
'inventory_provider.tasks.worker.snmp_refresh_interfaces',
args=[r["hostname"], r["community"]])
+ task_logger.debug(
+ 'launching task: '
+ 'inventory_provider.tasks.worker.update_inventory_system_cache')
+
return Response("OK")
diff --git a/inventory_provider/tasks/worker.py b/inventory_provider/tasks/worker.py
index a069c08c0b779481326ed365deefdfb225bca8b2..7dcfbcbf63895d2032f56810c3b3e79d90f06552 100644
--- a/inventory_provider/tasks/worker.py
+++ b/inventory_provider/tasks/worker.py
@@ -29,18 +29,18 @@ class InventoryTask(Task):
def __init__(self):
pass
- @staticmethod
- def save_key(hostname, key, value):
- assert isinstance(value, str), \
- "sanity failure: expected string data as value"
- r = get_redis(InventoryTask.config)
- r.hset(
- name=hostname,
- key=key,
- value=value)
- InventoryTask.logger.debug(
- "saved %s, key %s" % (hostname, key))
- return "OK"
+ # @staticmethod
+ # def save_key(hostname, key, value):
+ # assert isinstance(value, str), \
+ # "sanity failure: expected string data as value"
+ # r = get_redis(InventoryTask.config)
+ # r.hset(
+ # name=hostname,
+ # key=key,
+ # value=value)
+ # InventoryTask.logger.debug(
+ # "saved %s, key %s" % (hostname, key))
+ # return "OK"
@staticmethod
def save_value(key, value):
@@ -60,19 +60,25 @@ class InventoryTask(Task):
json.dumps(data_obj))
@staticmethod
- def save_key_json(hostname, key, data_obj):
- InventoryTask.save_key(
- hostname,
- key,
- json.dumps(data_obj))
-
- @staticmethod
- def save_key_etree(hostname, key, xml_doc):
- InventoryTask.save_key(
- hostname,
+ def save_value_etree(key, xml_doc):
+ InventoryTask.save_value(
key,
etree.tostring(xml_doc, encoding='unicode'))
+ # @staticmethod
+ # def save_key_json(hostname, key, data_obj):
+ # InventoryTask.save_key(
+ # hostname,
+ # key,
+ # json.dumps(data_obj))
+ #
+ # @staticmethod
+ # def save_key_etree(hostname, key, xml_doc):
+ # InventoryTask.save_key(
+ # hostname,
+ # key,
+ # etree.tostring(xml_doc, encoding='unicode'))
+
class WorkerArgs(bootsteps.Step):
def __init__(self, worker, config_filename, **options):
@@ -81,11 +87,11 @@ class WorkerArgs(bootsteps.Step):
InventoryTask.logger = logging.getLogger(constants.TASK_LOGGER_NAME)
-interfaces_key = "interface_services"
-equipment_locations_key = "equipment_locations"
-service_child_to_parents_key = "child_to_parent_circuit_relations"
-service_parent_to_children_key = "parent_to_children_circuit_relations"
-interface_status_key = "interface_statuses"
+# interfaces_key = "interface_services"
+# equipment_locations_key = "equipment_locations"
+# service_child_to_parents_key = "child_to_parent_circuit_relations"
+# service_parent_to_children_key = "parent_to_children_circuit_relations"
+# interface_status_key = "interface_statuses"
def worker_args(parser):
@@ -107,9 +113,8 @@ def snmp_refresh_interfaces(self, hostname, community):
logger.debug('STARTING: snmp_refresh_interfaces(%r, %r)'
% (hostname, community))
- InventoryTask.save_key_json(
- hostname,
- "snmp-interfaces",
+ InventoryTask.save_value_json(
+ 'snmp-interfaces:' + hostname,
list(snmp.get_router_interfaces(
hostname,
community,
@@ -124,9 +129,8 @@ def netconf_refresh_config(self, hostname):
logger = logging.getLogger(constants.TASK_LOGGER_NAME)
logger.debug('STARTING: netconf_refresh_config(%r)' % hostname)
- InventoryTask.save_key_etree(
- hostname,
- "netconf",
+ InventoryTask.save_value_etree(
+ 'netconf:' + hostname,
juniper.load_config(hostname, InventoryTask.config["ssh"]))
logger.debug('FINISHED: netconf_refresh_config(%r)' % hostname)
@@ -146,88 +150,69 @@ def netconf_refresh_config(self, hostname):
@app.task()
def update_interfaces_to_services():
- logger = logging.getLogger(constants.TASK_LOGGER_NAME)
- logger.error('HERE: update_interfaces_to_services')
+ interface_services = defaultdict(list)
+ with db.connection(InventoryTask.config["ops-db"]) as cx:
+ for service in opsdb.get_circuits(cx):
+ equipment_interface = '%s:%s' % (
+ service['equipment'], service['interface_name'])
+ interface_services[equipment_interface].append(service)
r = get_redis(InventoryTask.config)
- with db.connection(InventoryTask.config["ops-db"]) as cx:
- services = opsdb.get_circuits(cx)
-
- mapped_interfaces = defaultdict(list)
- for service in services:
- key = "{}::{}".format(
- service["equipment"],
- service["interface_name"]
- )
- mapped_interfaces[key].append(service)
- # Puts lu services under the parent ae as well as their own interface
- # eg. services on ae15.12 would be found under ae15 as well as ae15.12
- if "." in service["interface_name"]:
- key = "{}::{}".format(
- service["equipment"],
- service["interface_name"].split(".")[0]
- )
- mapped_interfaces[key].append(service)
-
- r.delete(interfaces_key)
- for key, value in mapped_interfaces.items():
- r.hset(interfaces_key, key, json.dumps(value))
+ for key in r.scan_iter('opsdb:interface_services:*'):
+ r.delete(key)
+ for equipment_interface, services in interface_services.items():
+ r.set(
+ 'opsdb:interface_services:' + equipment_interface,
+ json.dumps(services))
@app.task()
def update_equipment_locations():
r = get_redis(InventoryTask.config)
- r.delete(equipment_locations_key)
-
+ for key in r.scan_iter('opsdb:location:*'):
+ r.delete(key)
with db.connection(InventoryTask.config["ops-db"]) as cx:
for ld in opsdb.get_equipment_location_data(cx):
- r.hset(
- equipment_locations_key, ld["equipment_name"], json.dumps(ld))
+ r.set('opsdb:location:%s' % ld['equipment_name'], json.dumps(ld))
@app.task()
def update_circuit_hierarchy():
- r = get_redis(InventoryTask.config)
- children_to_parents = defaultdict(list)
- parents_to_children = defaultdict(list)
with db.connection(InventoryTask.config["ops-db"]) as cx:
- records = opsdb.get_circuit_hierarchy(cx)
- for relation in records:
+ child_to_parents = defaultdict(list)
+ parent_to_children = defaultdict(list)
+ for relation in opsdb.get_circuit_hierarchy(cx):
parent_id = relation["parent_circuit_id"]
child_id = relation["child_circuit_id"]
- parents_to_children[parent_id].append(relation)
- children_to_parents[child_id].append(relation)
+ parent_to_children[parent_id].append(relation)
+ child_to_parents[child_id].append(relation)
- r.delete(service_child_to_parents_key)
- for child, parents in children_to_parents.items():
- r.hset(service_child_to_parents_key, child, json.dumps(parents))
+ r = get_redis(InventoryTask.config)
+ for key in r.scan_iter('opsdb:services:parents:*'):
+ r.delete(key)
+ for cid, parents in child_to_parents.items():
+ r.set('opsdb:services:parents:%d' % cid, json.dumps(parents))
- r.delete(service_parent_to_children_key)
- for parent, children in parents_to_children.items():
- r.hset(
- service_parent_to_children_key, parent, json.dumps(children))
+ for key in r.scan_iter('opsdb:services:children:*'):
+ r.delete(key)
+ for cid, children in child_to_parents.items():
+ r.set('opsdb:services:children:%d' % cid, json.dumps(children))
@app.task()
def update_interface_statuses():
- r = get_redis(InventoryTask.config)
with db.connection(InventoryTask.config["ops-db"]) as cx:
services = opsdb.get_circuits(cx)
with db.connection(InventoryTask.config["alarms-db"]) as cx:
with db.cursor(cx) as csr:
for service in services:
- key = "{}::{}".format(
+ key = 'alarmsdb:interface_status:%s:%s' \
+ % (service['equipment'], service['interface_name'])
+ status = alarmsdb.get_last_known_interface_status(
+ csr,
service["equipment"],
- service["interface_name"]
- )
- if not r.hexists(interface_status_key, key):
- r.hset(interface_status_key,
- key,
- alarmsdb.get_last_known_interface_status(
- csr,
- service["equipment"],
- service["interface_name"]
- ))
+ service["interface_name"])
+ InventoryTask.save_value(key, status)
@app.task()