diff --git a/changelog b/changelog
index b2af90b8e90a1bca48685ffcee83ee4da2b6deb1..52a582c38c54e861fbbf23e7e8a3d3e50d457429 100644
--- a/changelog
+++ b/changelog
@@ -13,3 +13,4 @@
logging levels configured from environment
0.12: added addresses to interface response
put actual module number in version response
+0.13: added external inventory caching
diff --git a/inventory_provider/alarmsdb.py b/inventory_provider/alarmsdb.py
index 164cbbdb92b4b4de22a79b79ba5d8c47c04591a6..b3256e10bb1b60c3511d6942dff4f061ecd75ed1 100644
--- a/inventory_provider/alarmsdb.py
+++ b/inventory_provider/alarmsdb.py
@@ -1,40 +1,13 @@
-import contextlib
-import mysql.connector
+from inventory_provider import db
-@contextlib.contextmanager
-def connection(alarmsdb): # pragma: no cover
- cx = None
- try:
- cx = mysql.connector.connect(
- host=alarmsdb["hostname"],
- user=alarmsdb["username"],
- passwd=alarmsdb["password"],
- db=alarmsdb["dbname"])
- yield cx
- finally:
- if cx:
- cx.close()
-
-
-@contextlib.contextmanager
-def cursor(cnx): # pragma: no cover
- csr = None
- try:
- csr = cnx.cursor()
- yield csr
- finally:
- if csr:
- csr.close()
-
-
-def get_last_known_infinera_interface_status(db, equipment, interface):
+def get_last_known_infinera_interface_status(connection, equipment, interface):
query = "SELECT status FROM infinera_alarms" \
" WHERE" \
" CONCAT(ne_name, '-', REPLACE(object_name, 'T', '')) = %s" \
" ORDER BY ne_init_time DESC, ne_clear_time DESC LIMIT 1"
search_string = equipment + "-" + interface
- with cursor(db) as crs:
+ with db.cursor(connection) as crs:
crs.execute(query, (search_string,))
result = crs.fetchone()
if not result:
@@ -45,11 +18,11 @@ def get_last_known_infinera_interface_status(db, equipment, interface):
return "up"
-def get_last_known_coriant_interface_status(db, equipment, interface):
+def get_last_known_coriant_interface_status(connection, equipment, interface):
query = "SELECT status FROM coriant_alarms" \
" WHERE ne_id_name = %s AND entity_string LIKE %s" \
" ORDER BY last_event_time DESC LIMIT 1"
- with cursor(db) as crs:
+ with db.cursor(connection) as crs:
crs.execute(query, (equipment, interface + "-%"))
result = crs.fetchone()
if not result:
@@ -60,12 +33,13 @@ def get_last_known_coriant_interface_status(db, equipment, interface):
return "up"
-def get_last_known_juniper_link_interface_status(db, equipment, interface):
+def get_last_known_juniper_link_interface_status(
+ connection, equipment, interface):
query = "SELECT IF(link_admin_status = 'up'" \
" AND link_oper_status = 'up', 1, 0) AS up FROM juniper_alarms" \
" WHERE equipment_name = %s AND link_interface_name = %s" \
" ORDER BY alarm_id DESC LIMIT 1"
- with cursor(db) as crs:
+ with db.cursor(connection) as crs:
crs.execute(query, ('lo0.' + equipment, interface))
result = crs.fetchone()
if not result:
@@ -76,13 +50,13 @@ def get_last_known_juniper_link_interface_status(db, equipment, interface):
return "up"
-def get_last_known_interface_status(db, equipment, interface):
+def get_last_known_interface_status(connection, equipment, interface):
result = get_last_known_infinera_interface_status(
- db, equipment, interface)
+ connection, equipment, interface)
if result == "unknown":
result = get_last_known_coriant_interface_status(
- db, equipment, interface)
+ connection, equipment, interface)
if result == "unknown":
result = get_last_known_juniper_link_interface_status(
- db, equipment, interface)
+ connection, equipment, interface)
return result
diff --git a/inventory_provider/db.py b/inventory_provider/db.py
new file mode 100644
index 0000000000000000000000000000000000000000..2de869b3bb39b7acaa46a624465b026934c228a4
--- /dev/null
+++ b/inventory_provider/db.py
@@ -0,0 +1,41 @@
+import contextlib
+import mysql.connector
+import redis
+
+from flask import current_app, g
+
+
+def get_redis(): # pragma: no cover
+ if 'redis_db' not in g:
+ config = current_app.config['INVENTORY_PROVIDER_CONFIG']
+ g.redis_db = redis.Redis(
+ host=config['redis']['hostname'],
+ port=config['redis']['port'])
+
+ return g.redis_db
+
+
+@contextlib.contextmanager
+def connection(db_params):
+ cx = None
+ try:
+ cx = mysql.connector.connect(
+ host=db_params["hostname"],
+ user=db_params["username"],
+ passwd=db_params["password"],
+ db=db_params["dbname"])
+ yield cx
+ finally:
+ if cx:
+ cx.close()
+
+
+@contextlib.contextmanager
+def cursor(cnx): # pragma: no cover
+ csr = None
+ try:
+ csr = cnx.cursor()
+ yield csr
+ finally:
+ if csr:
+ csr.close()
diff --git a/inventory_provider/opsdb.py b/inventory_provider/opsdb.py
index 47fe94f0ab8e3f358e85c7e077d5d73990105231..c0a6417abbc10b803ba4ee39053d8e1498f64533 100644
--- a/inventory_provider/opsdb.py
+++ b/inventory_provider/opsdb.py
@@ -1,43 +1,231 @@
-import contextlib
-import logging
+from inventory_provider import db
-import mysql.connector
-from inventory_provider.constants import DATABASE_LOGGER_NAME
+equipment_location_query = """SELECT
+ e.absid,
+ e.name AS equipment_name,
+ p.name AS pop_name,
+ p.abbreviation AS pop_abbreviation,
+ p.site_id AS pop_site_id,
+ p.country,
+ g.longitude,
+ g.latitude
+ FROM
+ equipment e
+ INNER JOIN pop p
+ ON p.absid = e.PTR_pop
+ INNER JOIN geocoding g
+ ON g.absid = p.PTR_geocoding
+ WHERE
+ e.status != 'terminated'
+ AND e.status != 'disposed'"""
-@contextlib.contextmanager
-def connection(opsdb): # pragma: no cover
- cx = None
+circuit_hierarchy_query = """SELECT
+ pc.name AS parent_circuit,
+ pc.absid AS parent_circuit_id,
+ LOWER(pc.status) AS parent_circuit_status,
+ cc.name AS child_circuit,
+ cc.absid AS child_circuit_id,
+ LOWER(cc.status) AS child_circuit_status,
+ cg.segment_group AS segment_group
+ FROM circuit_glue cg
+ INNER JOIN circuit pc ON pc.absid = cg.PTR_circuit
+ INNER JOIN circuit cc ON cc.absid = cg.PTR_component"""
+
+
+retrieve_services_query = """SELECT *
+ FROM (SELECT
+ c.absid AS id,
+ c.name,
+ LOWER(c.status) AS status,
+ LOWER(c.circuit_type) AS circuit_type,
+ LOWER(c.service_type) AS service_type,
+ events.short_descr AS project,
+ e.name AS equipment,
+ cc.port_a AS port,
+ cc.int_LU_a AS logical_unit,
+ LOWER(o.name) AS manufacturer,
+ LOWER(ec.card_id) AS card_id,
+ LOWER(
+ IF(pp.interface_name IS NULL,
+ '', pp.interface_name)) AS interface_name
+ FROM circuit c
+ INNER JOIN circuit_connections cc
+ ON cc.circ_absid = c.absid
+ INNER JOIN equipment e
+ ON e.absid = cc.PTR_equip_a
+ LEFT JOIN events
+ ON events.absid = cc.PTR_project
+ INNER JOIN equipment_card ec
+ ON ec.absid = cc.PTR_card_a
+ LEFT JOIN organisation o
+ ON o.absid = ec.manufacturer
+ LEFT JOIN port_plugin pp
+ ON pp.PTR_card = cc.PTR_card_a AND pp.port = cc.port_a
+ WHERE c.status != 'terminated' AND is_circuit = 1
+ UNION
+ SELECT
+ c.absid AS id,
+ c.name,
+ LOWER(c.status) AS status,
+ LOWER(c.circuit_type) AS circuit_type,
+ LOWER(c.service_type) AS service_type,
+ events.short_descr AS project,
+ e.name AS equipment,
+ cc.port_b AS port,
+ cc.int_LU_b AS logical_unit,
+ LOWER(o.name) AS manufacturer,
+ LOWER(ec.card_id) AS card_id,
+ LOWER(
+ IF(pp.interface_name IS NULL,
+ '', pp.interface_name)) AS interface_name
+ FROM circuit c
+ INNER JOIN circuit_connections cc
+ ON cc.circ_absid = c.absid
+ INNER JOIN equipment e
+ ON e.absid = cc.PTR_equip_b
+ LEFT JOIN events
+ ON events.absid = cc.PTR_project
+ INNER JOIN equipment_card ec
+ ON ec.absid = cc.PTR_card_b
+ LEFT JOIN organisation o
+ ON o.absid = ec.manufacturer
+ LEFT JOIN port_plugin pp
+ ON pp.PTR_card = cc.PTR_card_b AND pp.port = cc.port_b
+ WHERE c.status != 'terminated' AND is_circuit = 1
+ UNION
+ SELECT
+ c.absid AS id,
+ c.name,
+ LOWER(c.status) AS status,
+ LOWER(c.circuit_type) AS circuit_type,
+ LOWER(c.service_type) AS service_type,
+ events.short_descr AS project,
+ e.name AS equipment,
+ cc.port_a_OUT AS port,
+ cc.int_LU_a AS logical_unit,
+ LOWER(o.name) AS manufacturer,
+ LOWER(ec.card_id) AS card_id,
+ LOWER(
+ IF(pp.interface_name IS NULL,
+ '', pp.interface_name)) AS interface_name
+ FROM circuit c
+ INNER JOIN circuit_connections cc
+ ON cc.circ_absid = c.absid
+ INNER JOIN equipment e
+ ON e.absid = cc.PTR_equip_a
+ LEFT JOIN events
+ ON events.absid = cc.PTR_project
+ INNER JOIN equipment_card ec
+ ON ec.absid = cc.PTR_card_a_OUT
+ LEFT JOIN organisation o
+ ON o.absid = ec.manufacturer
+ LEFT JOIN port_plugin pp
+ ON pp.PTR_card = cc.PTR_card_a_OUT
+ AND pp.port = cc.port_a_OUT
+ WHERE c.status != 'terminated' AND is_circuit = 1
+ UNION
+ SELECT
+ c.absid AS id,
+ c.name,
+ LOWER(c.status) AS status,
+ LOWER(c.circuit_type) AS circuit_type,
+ LOWER(c.service_type) AS service_type,
+ events.short_descr AS project,
+ e.name AS equipment,
+ cc.port_b_OUT AS port,
+ cc.int_LU_b AS logical_unit,
+ LOWER(o.name) AS manufacturer,
+ LOWER(ec.card_id) AS card_id,
+ LOWER(
+ IF(pp.interface_name IS NULL,
+ '', pp.interface_name)) AS interface_name
+ FROM circuit c
+ INNER JOIN circuit_connections cc
+ ON cc.circ_absid = c.absid
+ INNER JOIN equipment e
+ ON e.absid = cc.PTR_equip_b
+ LEFT JOIN events
+ ON events.absid = cc.PTR_project
+ INNER JOIN equipment_card ec
+ ON ec.absid = cc.PTR_card_b_OUT
+ LEFT JOIN organisation o
+ ON o.absid = ec.manufacturer
+ LEFT JOIN port_plugin pp
+ ON pp.PTR_card = cc.PTR_card_b_OUT
+ AND pp.port = cc.port_b_OUT
+ WHERE
+ c.status != 'terminated'
+ AND is_circuit = 1)
+ AS inner_query
+ ORDER BY
+ FIELD(status,
+ 'spare',
+ 'planned',
+ 'ordered',
+ 'installed',
+ 'operational')"""
+
+
+def _convert_to_dict(crs):
+ return [dict((crs.description[i][0], "" if value is None else value)
+ for i, value in enumerate(row)) for row in crs.fetchall()]
+
+
+def _infinera_field_update(record):
+ equipment_parts = record["equipment"].rsplit("-", 1)
+ card_parts = record["card_id"].split("-", 1)
+ record["interface_name"] = ""
+ record["equipment"] = equipment_parts[0]
try:
- cx = mysql.connector.connect(
- host=opsdb["hostname"],
- user=opsdb["username"],
- passwd=opsdb["password"],
- db=opsdb["dbname"])
- yield cx
- finally:
- if cx:
- cx.close()
-
-
-@contextlib.contextmanager
-def cursor(cnx): # pragma: no cover
- csr = None
+ record["interface_name"] = equipment_parts[1] + "-"
+ except IndexError:
+ pass # Nothing to see here
try:
- csr = cnx.cursor()
- yield csr
- finally:
- if csr:
- csr.close()
-
-
-def _db_test(db, router):
- database_logger = logging.getLogger(DATABASE_LOGGER_NAME)
- with cursor(db) as crs:
- query = "select model, manufacturer from equipment where name = %s"
- crs.execute(query, (router['hostname'],))
- for (model, manufacturer) in crs:
- database_logger.debug("%s: %s %s" % (
- router['hostname'], model, manufacturer))
- yield {"model": model, "manufacturer": manufacturer}
+ record["interface_name"] += card_parts[1]
+ except IndexError:
+ record["interface_name"] += card_parts[0]
+ if record["port"] is not None and record["port"] != "":
+ record["interface_name"] += "-" + record["port"]
+ record["interface_name"] = record["interface_name"] \
+ .replace("--", "-").upper()
+ return record
+
+
+def _juniper_field_update(record):
+ if not record["interface_name"]:
+ record["interface_name"] = record["card_id"]
+ if record["port"] is not None and record["port"] != "":
+ separator = "/" if "-" in record["interface_name"] else ""
+ record["interface_name"] += separator + str(record["port"])
+ if record["logical_unit"] is not None and record["logical_unit"] != "":
+ record["interface_name"] += "." + str(record["logical_unit"])
+ return record
+
+
+def _update_fields(r):
+ func = globals().get("_" + r["manufacturer"] + "_field_update")
+ return func(r) if func else r
+
+
+def get_circuits(connection):
+ with db.cursor(connection) as crs:
+ crs.execute(retrieve_services_query)
+ r = _convert_to_dict(crs)
+ r = list(map(_update_fields, r))
+ return r
+
+
+def get_circuit_hierarchy(connection):
+ with db.cursor(connection) as crs:
+ crs.execute(circuit_hierarchy_query)
+ r = _convert_to_dict(crs)
+ return r
+
+
+def get_equipment_location_data(connection):
+ with db.cursor(connection) as crs:
+ crs.execute(equipment_location_query)
+ r = _convert_to_dict(crs)
+ return r
diff --git a/inventory_provider/routes/alarmsdb.py b/inventory_provider/routes/alarmsdb.py
index 8dddaa0b581be5799511270ef9c6b4e0a8187f76..ee1fda69bdeed171f78e2bdea561903562c7dfc0 100644
--- a/inventory_provider/routes/alarmsdb.py
+++ b/inventory_provider/routes/alarmsdb.py
@@ -2,7 +2,7 @@ import functools
import json
from flask import Blueprint, request, Response, current_app
-from inventory_provider import alarmsdb
+from inventory_provider import alarmsdb, db
routes = Blueprint("inventory-alarmsdb-query-routes", __name__)
@@ -34,9 +34,9 @@ def get_interface_status():
equipment = request.args.get("equipment")
interface = request.args.get("interface")
- with alarmsdb.connection(config['alarms-db']) as db:
+ with db.connection(config['alarms-db']) as connection:
result = {"status": alarmsdb.get_last_known_interface_status(
- db, equipment, interface)}
+ connection, equipment, interface)}
return Response(
json.dumps(result),
diff --git a/inventory_provider/routes/jobs.py b/inventory_provider/routes/jobs.py
index a81f9855e60b35178f60e2677227b24b908cf05e..b1a78c86065bc0a9446266c88aecd2a4e747f1f2 100644
--- a/inventory_provider/routes/jobs.py
+++ b/inventory_provider/routes/jobs.py
@@ -1,6 +1,8 @@
import logging
from flask import Blueprint, Response, current_app
+import inventory_provider.storage.external_inventory as external_inventory
+from inventory_provider import db, opsdb
from inventory_provider.tasks.app import app
from inventory_provider.constants import TASK_LOGGER_NAME
@@ -34,3 +36,58 @@ def update():
args=[r["hostname"], r["community"]])
return Response("OK")
+
+
+@routes.route("update-services", methods=['GET'])
+def update_service():
+ config = current_app.config['INVENTORY_PROVIDER_CONFIG']
+
+ with db.connection(config['ops-db']) as connection:
+ result = opsdb.get_circuits(connection)
+ external_inventory.update_services_to_monitor(result)
+ return Response("OK")
+
+
+@routes.route("update-interfaces", methods=['GET'])
+def update_interfaces():
+ config = current_app.config['INVENTORY_PROVIDER_CONFIG']
+
+ with db.connection(config['ops-db']) as connection:
+ result = opsdb.get_circuits(connection)
+ external_inventory.update_interfaces_to_services(result)
+ return Response("OK")
+
+
+@routes.route("update-service-hierarchy", methods=['GET'])
+def update_service_hierarchy():
+ config = current_app.config['INVENTORY_PROVIDER_CONFIG']
+
+ with db.connection(config['ops-db']) as connection:
+ result = opsdb.get_circuit_hierarchy(connection)
+ external_inventory.update_service_hierarchy(result)
+ return Response("OK")
+
+
+@routes.route("update-equipment-locations", methods=['GET'])
+def update_equipment_locations():
+ config = current_app.config['INVENTORY_PROVIDER_CONFIG']
+
+ with db.connection(config['ops-db']) as connection:
+ result = opsdb.get_equipment_location_data(connection)
+ external_inventory.update_equipment_locations(result)
+ return Response("OK")
+
+
+@routes.route("update-from-inventory-system", methods=['GET'])
+def update_from_inventory_system():
+ config = current_app.config['INVENTORY_PROVIDER_CONFIG']
+
+ with db.connection(config['ops-db']) as connection:
+ circuits = opsdb.get_circuits(connection)
+ hierarchy = opsdb.get_circuit_hierarchy(connection)
+ equipment_locations = opsdb.get_equipment_location_data(connection)
+ external_inventory.update_services_to_monitor(circuits)
+ external_inventory.update_interfaces_to_services(circuits)
+ external_inventory.update_service_hierarchy(hierarchy)
+ external_inventory.update_equipment_locations(equipment_locations)
+ return Response("OK")
diff --git a/inventory_provider/routes/opsdb.py b/inventory_provider/routes/opsdb.py
index 1b3195ecec04c04d87598918bc03d4862e07739a..9d176e6825704b1ddfe7d645a23bab021606f79e 100644
--- a/inventory_provider/routes/opsdb.py
+++ b/inventory_provider/routes/opsdb.py
@@ -1,11 +1,16 @@
import functools
import json
-from flask import Blueprint, request, Response, current_app
-from inventory_provider import opsdb
+from flask import Blueprint, request, Response
+from inventory_provider import db
+from inventory_provider.storage import external_inventory
routes = Blueprint("inventory-opsdb-query-routes", __name__)
+services_key = "inv_services"
+interfaces_key = "inv_interfaces"
+equipment_locations_key = "inv_eq_locations"
+
def require_accepts_json(f):
"""
@@ -26,16 +31,78 @@ def require_accepts_json(f):
return decorated_function
-@routes.route("/test", methods=['GET', 'POST'])
-@require_accepts_json
-def opsdb_test():
- config = current_app.config['INVENTORY_PROVIDER_CONFIG']
+def _decode_utf8_dict(d):
+ return {k.decode('utf8'): json.loads(v) for k, v in d.items()}
+
+
+@routes.route("/interfaces")
+def get_all_interface_details():
+ r = db.get_redis()
+ result = _decode_utf8_dict(
+ r.hgetall(interfaces_key))
+
+ return Response(
+ json.dumps(result),
+ mimetype="application/json")
+
+@routes.route("/interfaces/<equipment_name>")
+def get_interface_details_for_equipment(equipment_name):
+ r = db.get_redis()
result = {}
- with opsdb.connection(config['ops-db']) as db:
- for r in config['routers']:
- result[r['hostname']] = list(opsdb._db_test(db, r))
+ for t in r.hscan_iter(interfaces_key, "{}::*".format(equipment_name)):
+ result[t[0].decode("utf8")] = json.loads(t[1])
+
+ return Response(
+ json.dumps(result),
+ mimetype="application/json")
+
+
+@routes.route("/interfaces/<equipment_name>/<path:interface>")
+def get_interface_details(equipment_name, interface):
+ r = db.get_redis()
+ return Response(
+ r.hget(
+ interfaces_key,
+ "{}::{}".format(equipment_name, interface)),
+ mimetype="application/json")
+
+
+@routes.route("/equipment-location")
+def get_all_equipment_locations():
+ r = db.get_redis()
+ result = list(
+ _decode_utf8_dict(
+ r.hgetall(equipment_locations_key)).values())
return Response(
json.dumps(result),
mimetype="application/json")
+
+
+@routes.route("/equipment-location/<path:equipment_name>")
+def get_equipment_location(equipment_name):
+ r = db.get_redis()
+ return Response(
+ r.hget(equipment_locations_key, equipment_name),
+ mimetype="application/json")
+
+
+@routes.route("/circuit-hierarchy/children/<parent_id>")
+def get_children(parent_id):
+ r = db.get_redis()
+ return Response(
+ r.hget(
+ external_inventory.service_parent_to_children_key,
+ parent_id),
+ mimetype="application/json")
+
+
+@routes.route("/circuit-hierarchy/parents/<child_id>")
+def get_parents(child_id):
+ r = db.get_redis()
+ return Response(
+ r.hget(
+ external_inventory.service_child_to_parents_key,
+ child_id),
+ mimetype="application/json")
diff --git a/inventory_provider/static/juniper.html b/inventory_provider/static/juniper.html
index 5ccb25324d28e99a0aea763fe3536aa65201fcf4..80e70c607a0fc98730a0df46b52b397798e56c59 100644
--- a/inventory_provider/static/juniper.html
+++ b/inventory_provider/static/juniper.html
@@ -20,7 +20,11 @@
<p><b>interfaces</b></p>
<ul>
<li ng-repeat="i in interfaces">{{i.name}}
- <ul><li>{{i.description}}</li></ul>
+ <ul>
+ <li>{{i.description}}</li>
+ <li ng-repeat="v4 in i.ipv4">v4: {{v4}}</li>
+ <li ng-repeat="v6 in i.ipv6">v6: {{v6}}</li>
+ </ul>
</li>
</ul>
<div class="raw">{{interfaces}}</div>
diff --git a/inventory_provider/storage/__init__.py b/inventory_provider/storage/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/inventory_provider/storage/external_inventory.py b/inventory_provider/storage/external_inventory.py
new file mode 100644
index 0000000000000000000000000000000000000000..cc7bccf69f48c717202e22656f88f9f55f5c33aa
--- /dev/null
+++ b/inventory_provider/storage/external_inventory.py
@@ -0,0 +1,60 @@
+import json
+from collections import defaultdict
+from inventory_provider import db
+
+
+services_key = "inv_services"
+interfaces_key = "inv_interfaces"
+equipment_locations_key = "inv_eq_locations"
+service_child_to_parents_key = "inv_service_child_to_parents"
+service_parent_to_children_key = "inv_service_parent_to_children"
+
+
+def update_services_to_monitor(services):
+ r = db.get_redis()
+ relevant_types = ('path', 'service', 'l2circuit')
+ r.delete(services_key)
+ for service in services:
+ if service['circuit_type'].lower() in relevant_types:
+ r.hset(services_key, service['id'], json.dumps(service))
+
+
+def update_interfaces_to_services(services):
+ r = db.get_redis()
+ mapped_interfaces = defaultdict(list)
+ r.delete(interfaces_key)
+ for service in services:
+ key = "{}::{}".format(
+ service['equipment'],
+ service['interface_name']
+ )
+ mapped_interfaces[key].append(service)
+
+ for key, value in mapped_interfaces.items():
+ r.hset(interfaces_key, key, json.dumps(value))
+
+
+def update_service_hierarchy(records):
+ r = db.get_redis()
+ children_to_parents = defaultdict(list)
+ parents_to_children = defaultdict(list)
+ for relation in records:
+ parent_id = relation["parent_circuit_id"]
+ child_id = relation["child_circuit_id"]
+ parents_to_children[parent_id].append(relation)
+ children_to_parents[child_id].append(relation)
+
+ r.delete(service_child_to_parents_key)
+ for child, parents in children_to_parents.items():
+ r.hset(service_child_to_parents_key, child, json.dumps(parents))
+
+ r.delete(service_parent_to_children_key)
+ for parent, children in parents_to_children.items():
+ r.hset(service_parent_to_children_key, parent, json.dumps(children))
+
+
+def update_equipment_locations(equipment_location_data):
+ r = db.get_redis()
+ r.delete(equipment_locations_key)
+ for ld in equipment_location_data:
+ r.hset(equipment_locations_key, ld['equipment_name'], json.dumps(ld))
diff --git a/setup.py b/setup.py
index d8aae6b09389b832b27d55c2f26d3a121c4e253b..b041145b677577119497968f883a6ed70f59cb09 100644
--- a/setup.py
+++ b/setup.py
@@ -2,7 +2,7 @@ from setuptools import setup, find_packages
setup(
name='inventory-provider',
- version="0.12",
+ version="0.13",
author='GEANT',
author_email='swd@geant.org',
description='Dashboard inventory provider',
diff --git a/test/storage/test_external_inventory.py b/test/storage/test_external_inventory.py
new file mode 100644
index 0000000000000000000000000000000000000000..30c13e0bc7287fa2b118b875b05b707482028857
--- /dev/null
+++ b/test/storage/test_external_inventory.py
@@ -0,0 +1,150 @@
+from inventory_provider.storage import external_inventory
+
+
+def test_update_services_to_monitor(mocker):
+ mocked_redis = mocker.patch(
+ "inventory_provider.storage.external_inventory.db.get_redis")
+ mocked_hset = mocked_redis.return_value.hset
+
+ services = [
+ {"circuit_type": "path", "id": "test_id_0", "other": "stuff"},
+ {"circuit_type": "service", "id": "test_id_1", "other": "stuff"},
+ {"circuit_type": "l2circuit", "id": "test_id_2", "other": "stuff"},
+ {"circuit_type": "spam", "id": "test_id_4", "other": "stuff"}
+ ]
+ external_inventory.update_services_to_monitor(services)
+ mocked_hset.assert_any_call(
+ external_inventory.services_key, "test_id_0",
+ "{\"circuit_type\": \"path\","
+ " \"id\": \"test_id_0\","
+ " \"other\": \"stuff\"}")
+ mocked_hset.assert_any_call(
+ external_inventory.services_key, "test_id_1",
+ "{\"circuit_type\": \"service\","
+ " \"id\": \"test_id_1\","
+ " \"other\": \"stuff\"}")
+ mocked_hset.assert_any_call(
+ external_inventory.services_key, "test_id_2",
+ "{\"circuit_type\": \"l2circuit\","
+ " \"id\": \"test_id_2\","
+ " \"other\": \"stuff\"}")
+ assert mocked_hset.call_count == 3
+
+
+def test_update_interfaces_to_services(mocker):
+ mocked_redis = mocker.patch(
+ "inventory_provider.storage.external_inventory.db.get_redis")
+ mocked_hset = mocked_redis.return_value.hset
+ services = [
+ {"equipment": "eq_0", "interface_name": "if_0"},
+ {"equipment": "eq_1", "interface_name": "if_1"},
+ {"equipment": "eq_2", "interface_name": "if_2"},
+ {"equipment": "eq_3", "interface_name": "if_3"},
+ {"equipment": "eq_3", "interface_name": "if_3", "extra": "stuff"},
+ {"equipment": "eq_4", "interface_name": "if_4"}
+ ]
+ unique_keys = set(s["equipment"] + "::" + s["interface_name"]
+ for s in services)
+
+ external_inventory.update_interfaces_to_services(services)
+ assert mocked_hset.call_count == len(unique_keys)
+ mocked_hset.assert_any_call(external_inventory.interfaces_key,
+ "eq_2::if_2",
+ "[{\"equipment\": \"eq_2\","
+ " \"interface_name\": \"if_2\"}]")
+ mocked_hset.assert_any_call(external_inventory.interfaces_key,
+ "eq_3::if_3",
+ "[{\"equipment\": \"eq_3\","
+ " \"interface_name\": \"if_3\"},"
+ " {\"equipment\": \"eq_3\","
+ " \"interface_name\": \"if_3\","
+ " \"extra\": \"stuff\"}]")
+
+
+def test_update_service_hierarchy(mocker):
+ mocked_redis = mocker.patch(
+ "inventory_provider.storage.external_inventory.db.get_redis")
+ mocked_hset = mocked_redis.return_value.hset
+ data = [
+ {"parent_circuit": "PC_1",
+ "parent_circuit_id": "1001",
+ "parent_circuit_status": "operational",
+ "child_circuit": "CC_1",
+ "child_circuit_id": "2001",
+ "segment_group": 1},
+ {"parent_circuit": "PC_2",
+ "parent_circuit_id": "1002",
+ "parent_circuit_status": "operational",
+ "child_circuit": "CC_1",
+ "child_circuit_id": "2001",
+ "segment_group": 1},
+ {"parent_circuit": "PC_3",
+ "parent_circuit_id": "1003",
+ "parent_circuit_status": "operational",
+ "child_circuit": "CC_2",
+ "child_circuit_id": "2002",
+ "segment_group": 1},
+ {"parent_circuit": "PC_3",
+ "parent_circuit_id": "1003",
+ "parent_circuit_status": "operational",
+ "child_circuit": "CC_3",
+ "child_circuit_id": "2003",
+ "segment_group": 1}
+ ]
+ external_inventory.update_service_hierarchy(data)
+ u_parent_keys = set([d["parent_circuit_id"] for d in data])
+ u_child_keys = set([d["child_circuit_id"] for d in data])
+ assert mocked_hset.call_count == len(u_parent_keys) + len(u_child_keys)
+ mocked_hset.assert_any_call(
+ external_inventory.service_child_to_parents_key,
+ "2001",
+ "[{\"parent_circuit\": \"PC_1\","
+ " \"parent_circuit_id\": \"1001\","
+ " \"parent_circuit_status\": \"operational\","
+ " \"child_circuit\": \"CC_1\","
+ " \"child_circuit_id\": \"2001\","
+ " \"segment_group\": 1},"
+ " {\"parent_circuit\": \"PC_2\","
+ " \"parent_circuit_id\": \"1002\","
+ " \"parent_circuit_status\": \"operational\","
+ " \"child_circuit\": \"CC_1\","
+ " \"child_circuit_id\": \"2001\","
+ " \"segment_group\": 1}]"
+ )
+ mocked_hset.assert_any_call(
+ external_inventory.service_child_to_parents_key,
+ "2002",
+ "[{\"parent_circuit\": \"PC_3\","
+ " \"parent_circuit_id\": \"1003\","
+ " \"parent_circuit_status\": \"operational\","
+ " \"child_circuit\": \"CC_2\","
+ " \"child_circuit_id\": \"2002\","
+ " \"segment_group\": 1}]"
+ )
+
+ mocked_hset.assert_any_call(
+ external_inventory.service_parent_to_children_key,
+ "1003",
+ "[{\"parent_circuit\": \"PC_3\","
+ " \"parent_circuit_id\": \"1003\","
+ " \"parent_circuit_status\": \"operational\","
+ " \"child_circuit\": \"CC_2\","
+ " \"child_circuit_id\": \"2002\","
+ " \"segment_group\": 1},"
+ " {\"parent_circuit\": \"PC_3\","
+ " \"parent_circuit_id\": \"1003\","
+ " \"parent_circuit_status\": \"operational\","
+ " \"child_circuit\": \"CC_3\","
+ " \"child_circuit_id\": \"2003\","
+ " \"segment_group\": 1}]"
+ )
+ mocked_hset.assert_any_call(
+ external_inventory.service_parent_to_children_key,
+ "1002",
+ "[{\"parent_circuit\": \"PC_2\","
+ " \"parent_circuit_id\": \"1002\","
+ " \"parent_circuit_status\": \"operational\","
+ " \"child_circuit\": \"CC_1\","
+ " \"child_circuit_id\": \"2001\","
+ " \"segment_group\": 1}]"
+ )
diff --git a/test/test_alarmdb_routes.py b/test/test_alarmdb_routes.py
index 2e16e72d8fb59c170b43299b9a0809ea8af97baf..82522455868b48bdda22a82131c606dcdd8f0d72 100644
--- a/test/test_alarmdb_routes.py
+++ b/test/test_alarmdb_routes.py
@@ -9,7 +9,7 @@ DEFAULT_REQUEST_HEADERS = {
def test_get_interface_status(mocker, client):
mocked_conn = mocker.patch('inventory_provider.routes.alarmsdb'
- '.alarmsdb.connection')
+ '.db.connection')
mocked_conn.return_value.__enter__.return_value = None
mocked_inteface_status = mocker.patch(
diff --git a/test/test_alarmsdb.py b/test/test_alarmsdb.py
index c993a762106481a77983df66efdc1276b1a78d2e..7eb707c1210bca0a21cd36ebdc3b658290cca426 100644
--- a/test/test_alarmsdb.py
+++ b/test/test_alarmsdb.py
@@ -2,7 +2,7 @@ import inventory_provider.alarmsdb as alarmsdb
def test_infinera_interface_status(mocker):
- mocked_get_cursor = mocker.patch('inventory_provider.alarmsdb.cursor')
+ mocked_get_cursor = mocker.patch('inventory_provider.alarmsdb.db.cursor')
mocked_execute = mocked_get_cursor. \
return_value.__enter__.return_value.execute
mocked_fetchone = mocked_get_cursor.return_value.__enter__. \
@@ -30,7 +30,7 @@ def test_infinera_interface_status(mocker):
def test_coriant_interface_status(mocker):
- mocked_get_cursor = mocker.patch('inventory_provider.alarmsdb.cursor')
+ mocked_get_cursor = mocker.patch('inventory_provider.alarmsdb.db.cursor')
mocked_execute = mocked_get_cursor. \
return_value.__enter__.return_value.execute
mocked_fetchone = mocked_get_cursor.return_value.__enter__. \
@@ -58,7 +58,7 @@ def test_coriant_interface_status(mocker):
def test_juniper_interface_status(mocker):
- mocked_get_cursor = mocker.patch('inventory_provider.alarmsdb.cursor')
+ mocked_get_cursor = mocker.patch('inventory_provider.alarmsdb.db.cursor')
mocked_execute = mocked_get_cursor. \
return_value.__enter__.return_value.execute
mocked_fetchone = mocked_get_cursor.return_value.__enter__. \
diff --git a/test/test_external_inventory_routes.py b/test/test_external_inventory_routes.py
new file mode 100644
index 0000000000000000000000000000000000000000..4adc7d5890795740626e7a696fe51f6c1a89a6fc
--- /dev/null
+++ b/test/test_external_inventory_routes.py
@@ -0,0 +1,134 @@
+import json
+from inventory_provider.storage import external_inventory
+
+DEFAULT_REQUEST_HEADERS = {
+ "Content-type": "application/json",
+ "Accept": ["application/json"]
+}
+
+
+def test_get_one_equipment_location(mocker, client):
+ mocked_redis = mocker.patch(
+ "inventory_provider.routes.opsdb.db.get_redis")
+ mocked_hget = mocked_redis.return_value.hget
+ dummy_data = {
+ "absid": 1404,
+ "equipment_name": "pp-cz-e13b",
+ "pop_name": "Prague",
+ "pop_abbreviation": "pra",
+ "pop_site_id": "",
+ "country": "Czech Republic",
+ "longitude": 14.391738888889,
+ "latitude": 50.101847222222
+ }
+ mocked_hget.return_value = json.dumps(dummy_data)
+
+ rv = client.get(
+ '/opsdb/equipment-location/dummy-equipment',
+ headers=DEFAULT_REQUEST_HEADERS)
+ assert rv.status_code == 200
+ assert rv.is_json
+ assert dummy_data == json.loads(rv.data.decode("utf-8"))
+
+ mocked_hget.assert_called_with(
+ external_inventory.equipment_locations_key,
+ "dummy-equipment"
+ )
+
+
+def test_get_equipment_location(mocker, client):
+ mocked_redis = mocker.patch(
+ "inventory_provider.routes.opsdb.db.get_redis")
+ mocked_hgetall = mocked_redis.return_value.hgetall
+
+ rv = client.get(
+ '/opsdb/equipment-location',
+ headers=DEFAULT_REQUEST_HEADERS)
+ assert rv.status_code == 200
+ assert rv.is_json
+
+ mocked_hgetall.assert_called_with(
+ external_inventory.equipment_locations_key
+ )
+
+
+def test_get_interface_info(mocker, client):
+ mocked_redis = mocker.patch(
+ "inventory_provider.routes.opsdb.db.get_redis")
+ mocked_hgetall = mocked_redis.return_value.hgetall
+
+ rv = client.get(
+ '/opsdb/interfaces',
+ headers=DEFAULT_REQUEST_HEADERS)
+ assert rv.status_code == 200
+ assert rv.is_json
+
+ mocked_hgetall.assert_called_with(
+ external_inventory.interfaces_key
+ )
+
+
+def test_get_interface_info_for_equipment(mocker, client):
+ mocked_redis = mocker.patch(
+ "inventory_provider.routes.opsdb.db.get_redis")
+ mocked_hscan_iter = mocked_redis.return_value.hscan_iter
+
+ rv = client.get(
+ '/opsdb/interfaces/dummy-equipment',
+ headers=DEFAULT_REQUEST_HEADERS)
+ assert rv.status_code == 200
+ assert rv.is_json
+
+ mocked_hscan_iter.assert_called_with(
+ external_inventory.interfaces_key, "dummy-equipment::*"
+ )
+
+
+def test_get_interface_info_for_equipment_and_interface(mocker, client):
+ mocked_redis = mocker.patch(
+ "inventory_provider.routes.opsdb.db.get_redis")
+ mocked_hget = mocked_redis.return_value.hget
+
+ rv = client.get(
+ '/opsdb/interfaces/dummy-equipment/xe-2/3/1',
+ headers=DEFAULT_REQUEST_HEADERS)
+ assert rv.status_code == 200
+ assert rv.is_json
+
+ mocked_hget.assert_called_with(
+ external_inventory.interfaces_key, "dummy-equipment::xe-2/3/1"
+ )
+
+
+def test_get_children(mocker, client):
+ mocked_redis = mocker.patch(
+ "inventory_provider.routes.opsdb.db.get_redis")
+ mocked_hget = mocked_redis.return_value.hget
+
+ rv = client.get(
+ '/opsdb/circuit-hierarchy/children/22987',
+ headers=DEFAULT_REQUEST_HEADERS)
+ assert rv.status_code == 200
+ assert rv.is_json
+
+ mocked_hget.assert_called_with(
+ external_inventory.service_parent_to_children_key,
+ "22987"
+ )
+
+
+def test_get_parents(mocker, client):
+ mocked_redis = mocker.patch(
+ "inventory_provider.routes.opsdb.db.get_redis")
+ mocked_hget = mocked_redis.return_value.hget
+
+ rv = client.get(
+ '/opsdb/circuit-hierarchy/children/22987',
+ headers=DEFAULT_REQUEST_HEADERS)
+ assert rv.status_code == 200
+ assert rv.is_json
+
+ mocked_hget.assert_called_with(
+ external_inventory.service_parent_to_children_key,
+ "22987"
+ )
diff --git a/test/test_opsdb.py b/test/test_opsdb.py
new file mode 100644
index 0000000000000000000000000000000000000000..3507bdc0bcb80e341586cd2d8d966a5591da94fc
--- /dev/null
+++ b/test/test_opsdb.py
@@ -0,0 +1,116 @@
+import inventory_provider.opsdb
+
+
+def test_update_fields(mocker):
+ mocker.patch("inventory_provider.opsdb._juniper_field_update")
+ t = {"manufacturer": "juniper"}
+ inventory_provider.opsdb._update_fields(t)
+ inventory_provider.opsdb._juniper_field_update.assert_called_once_with(t)
+
+ mocker.patch("inventory_provider.opsdb._infinera_field_update")
+ t = {"manufacturer": "infinera"}
+ inventory_provider.opsdb._update_fields(t)
+ inventory_provider.opsdb._infinera_field_update.assert_called_once_with(t)
+
+ f = {"manufacturer": "non-existent"}
+ r = inventory_provider.opsdb._update_fields(f)
+ assert f == r
+
+
+def test_infinera_field_update():
+ i = {
+ "equipment": "AMS01-DTNX10-1-1",
+ "card_id": "tim-b-5-7",
+ "port": "1"
+ }
+ r = inventory_provider.opsdb._infinera_field_update(i)
+ assert r["equipment"] == "AMS01-DTNX10-1"
+ assert r["interface_name"] == "1-B-5-7-1"
+
+ i = {
+ "equipment": "BUD01_CX_01",
+ "card_id": "tim-1/2",
+ "port": "1"
+ }
+ r = inventory_provider.opsdb._infinera_field_update(i)
+ assert r["equipment"] == "BUD01_CX_01"
+ assert r["interface_name"] == "1/2-1"
+
+ i = {
+ "equipment": "irrelevant",
+ "card_id": "tim_1/2",
+ "port": "1"
+ }
+ r = inventory_provider.opsdb._infinera_field_update(i)
+ assert r["interface_name"] == "TIM_1/2-1"
+
+
+def test_juniper_field_update():
+ i = {
+ "interface_name": "xe-1/2",
+ "logical_unit": None
+ }
+ r = inventory_provider.opsdb._juniper_field_update(i)
+ assert r["interface_name"] == "xe-1/2"
+
+ i["interface_name"] = "xe-1/2"
+ i["logical_unit"] = 101
+ r = inventory_provider.opsdb._juniper_field_update(i)
+ assert r["interface_name"] == "xe-1/2.101"
+
+ i["interface_name"] = "xe-1/2"
+ i["logical_unit"] = 0
+ r = inventory_provider.opsdb._juniper_field_update(i)
+ assert r["interface_name"] == "xe-1/2.0"
+
+ i["interface_name"] = "xe-1/2"
+ i["logical_unit"] = None
+ i["port"] = 0
+ r = inventory_provider.opsdb._juniper_field_update(i)
+ assert r["interface_name"] == "xe-1/2"
+
+ i["interface_name"] = None
+ i["card_id"] = "xe-2/0"
+ i["logical_unit"] = None
+ i["port"] = None
+ r = inventory_provider.opsdb._juniper_field_update(i)
+ assert r["interface_name"] == "xe-2/0"
+
+ i["interface_name"] = None
+ i["port"] = "0"
+ r = inventory_provider.opsdb._juniper_field_update(i)
+ assert r["interface_name"] == "xe-2/0/0"
+
+ i["interface_name"] = None
+ i["port"] = 0
+ r = inventory_provider.opsdb._juniper_field_update(i)
+ assert r["interface_name"] == "xe-2/0/0"
+
+ i["interface_name"] = None
+ i["logical_unit"] = "123"
+ r = inventory_provider.opsdb._juniper_field_update(i)
+ assert r["interface_name"] == "xe-2/0/0.123"
+
+ i["interface_name"] = None
+ i["logical_unit"] = 123
+ r = inventory_provider.opsdb._juniper_field_update(i)
+ assert r["interface_name"] == "xe-2/0/0.123"
+
+
+def test_get_circuits(mocker):
+ mocker.patch("inventory_provider.opsdb.db.cursor")
+ mocked_convert_to_dict = mocker.patch(
+ "inventory_provider.opsdb._convert_to_dict")
+ i = {"manufacturer": "infinera"}
+ j = {"manufacturer": "juniper"}
+ mocked_convert_to_dict.return_value = [i, j]
+
+ mocked_infinera_update = mocker.patch(
+ "inventory_provider.opsdb._infinera_field_update")
+ mocked_juniper_update = mocker.patch(
+ "inventory_provider.opsdb._juniper_field_update")
+
+ inventory_provider.opsdb.get_circuits(None)
+
+ mocked_infinera_update.assert_called_once_with(i)
+ mocked_juniper_update.assert_called_once_with(j)
diff --git a/tox.ini b/tox.ini
index 1f203d8810d2613924649bd99f8c67c77007d627..08605507f9962a1b519d7779d9bbefcc8a82519f 100644
--- a/tox.ini
+++ b/tox.ini
@@ -10,7 +10,7 @@ deps =
commands =
coverage erase
- coverage run --source inventory_provider -m py.test
+ coverage run --source inventory_provider -m py.test {posargs}
coverage xml
coverage html
coverage report --fail-under 75