Finished release 0.13.

changelog

@@ -13,3 +13,4 @@
       logging levels configured from environment
 0.12: added addresses to interface response
       put actual module number in version response
+0.13: added external inventory caching

inventory_provider/alarmsdb.py

-import contextlib
-import mysql.connector
+from inventory_provider import db
 
 
-@contextlib.contextmanager
-def connection(alarmsdb):  # pragma: no cover
-    cx = None
-    try:
-        cx = mysql.connector.connect(
-            host=alarmsdb["hostname"],
-            user=alarmsdb["username"],
-            passwd=alarmsdb["password"],
-            db=alarmsdb["dbname"])
-        yield cx
-    finally:
-        if cx:
-            cx.close()
-
-
-@contextlib.contextmanager
-def cursor(cnx):  # pragma: no cover
-    csr = None
-    try:
-        csr = cnx.cursor()
-        yield csr
-    finally:
-        if csr:
-            csr.close()
-
-
-def get_last_known_infinera_interface_status(db, equipment, interface):
+def get_last_known_infinera_interface_status(connection, equipment, interface):
     query = "SELECT status FROM infinera_alarms" \
             " WHERE" \
             " CONCAT(ne_name, '-', REPLACE(object_name, 'T', '')) = %s" \
             " ORDER BY ne_init_time DESC, ne_clear_time DESC LIMIT 1"
     search_string = equipment + "-" + interface
-    with cursor(db) as crs:
+    with db.cursor(connection) as crs:
         crs.execute(query, (search_string,))
         result = crs.fetchone()
     if not result:
@@ -45,11 +18,11 @@ def get_last_known_infinera_interface_status(db, equipment, interface):
         return "up"
 
 
-def get_last_known_coriant_interface_status(db, equipment, interface):
+def get_last_known_coriant_interface_status(connection, equipment, interface):
     query = "SELECT status FROM coriant_alarms" \
             " WHERE ne_id_name = %s AND entity_string LIKE %s" \
             " ORDER BY last_event_time DESC LIMIT 1"
-    with cursor(db) as crs:
+    with db.cursor(connection) as crs:
         crs.execute(query, (equipment, interface + "-%"))
         result = crs.fetchone()
     if not result:
@@ -60,12 +33,13 @@ def get_last_known_coriant_interface_status(db, equipment, interface):
         return "up"
 
 
-def get_last_known_juniper_link_interface_status(db, equipment, interface):
+def get_last_known_juniper_link_interface_status(
+        connection, equipment, interface):
     query = "SELECT IF(link_admin_status = 'up'" \
             " AND link_oper_status = 'up', 1, 0) AS up FROM juniper_alarms" \
             " WHERE equipment_name = %s AND link_interface_name = %s" \
             " ORDER BY alarm_id DESC LIMIT 1"
-    with cursor(db) as crs:
+    with db.cursor(connection) as crs:
         crs.execute(query, ('lo0.' + equipment, interface))
         result = crs.fetchone()
     if not result:
@@ -76,13 +50,13 @@ def get_last_known_juniper_link_interface_status(db, equipment, interface):
         return "up"
 
 
-def get_last_known_interface_status(db, equipment, interface):
+def get_last_known_interface_status(connection, equipment, interface):
     result = get_last_known_infinera_interface_status(
-        db, equipment, interface)
+        connection, equipment, interface)
     if result == "unknown":
         result = get_last_known_coriant_interface_status(
-                db, equipment, interface)
+            connection, equipment, interface)
     if result == "unknown":
         result = get_last_known_juniper_link_interface_status(
-            db, equipment, interface)
+            connection, equipment, interface)
     return result

inventory_provider/db.py

+import contextlib
+import mysql.connector
+import redis
+
+from flask import current_app, g
+
+
+def get_redis():  # pragma: no cover
+    if 'redis_db' not in g:
+        config = current_app.config['INVENTORY_PROVIDER_CONFIG']
+        g.redis_db = redis.Redis(
+            host=config['redis']['hostname'],
+            port=config['redis']['port'])
+
+    return g.redis_db
+
+
+@contextlib.contextmanager
+def connection(db_params):
+    cx = None
+    try:
+        cx = mysql.connector.connect(
+            host=db_params["hostname"],
+            user=db_params["username"],
+            passwd=db_params["password"],
+            db=db_params["dbname"])
+        yield cx
+    finally:
+        if cx:
+            cx.close()
+
+
+@contextlib.contextmanager
+def cursor(cnx):  # pragma: no cover
+    csr = None
+    try:
+        csr = cnx.cursor()
+        yield csr
+    finally:
+        if csr:
+            csr.close()

inventory_provider/opsdb.py

-import contextlib
-import logging
+from inventory_provider import db
 
-import mysql.connector
 
-from inventory_provider.constants import DATABASE_LOGGER_NAME
+equipment_location_query = """SELECT
+                          e.absid,
+                          e.name AS equipment_name,
+                          p.name AS pop_name,
+                          p.abbreviation AS pop_abbreviation,
+                          p.site_id AS pop_site_id,
+                          p.country,
+                          g.longitude,
+                          g.latitude
+                        FROM
+                          equipment e
+                        INNER JOIN pop p
+                          ON p.absid = e.PTR_pop
+                        INNER JOIN geocoding g
+                          ON g.absid = p.PTR_geocoding
+                        WHERE
+                          e.status != 'terminated'
+                          AND e.status != 'disposed'"""
 
 
-@contextlib.contextmanager
-def connection(opsdb):  # pragma: no cover
-    cx = None
+circuit_hierarchy_query = """SELECT
+                       pc.name AS parent_circuit,
+                       pc.absid AS parent_circuit_id,
+                       LOWER(pc.status) AS parent_circuit_status,
+                       cc.name AS child_circuit,
+                       cc.absid AS child_circuit_id,
+                       LOWER(cc.status) AS child_circuit_status,
+                       cg.segment_group AS segment_group
+                     FROM circuit_glue cg
+                     INNER JOIN circuit pc ON pc.absid = cg.PTR_circuit
+                     INNER JOIN circuit cc ON cc.absid = cg.PTR_component"""
+
+
+retrieve_services_query = """SELECT *
+                    FROM (SELECT
+                      c.absid AS id,
+                      c.name,
+                      LOWER(c.status) AS status,
+                      LOWER(c.circuit_type) AS circuit_type,
+                      LOWER(c.service_type) AS service_type,
+                      events.short_descr AS project,
+                      e.name AS equipment,
+                      cc.port_a AS port,
+                      cc.int_LU_a AS logical_unit,
+                      LOWER(o.name) AS manufacturer,
+                      LOWER(ec.card_id) AS card_id,
+                      LOWER(
+                        IF(pp.interface_name IS NULL,
+                        '', pp.interface_name)) AS interface_name
+                    FROM circuit c
+                      INNER JOIN circuit_connections cc
+                        ON cc.circ_absid = c.absid
+                      INNER JOIN equipment e
+                        ON e.absid = cc.PTR_equip_a
+                      LEFT JOIN events
+                        ON events.absid = cc.PTR_project
+                      INNER JOIN equipment_card ec
+                        ON ec.absid = cc.PTR_card_a
+                      LEFT JOIN organisation o
+                        ON o.absid = ec.manufacturer
+                      LEFT JOIN port_plugin pp
+                        ON pp.PTR_card = cc.PTR_card_a AND pp.port = cc.port_a
+                    WHERE c.status != 'terminated' AND is_circuit = 1
+                    UNION
+                    SELECT
+                      c.absid AS id,
+                      c.name,
+                      LOWER(c.status) AS status,
+                      LOWER(c.circuit_type) AS circuit_type,
+                      LOWER(c.service_type) AS service_type,
+                      events.short_descr AS project,
+                      e.name AS equipment,
+                      cc.port_b AS port,
+                      cc.int_LU_b AS logical_unit,
+                      LOWER(o.name) AS manufacturer,
+                      LOWER(ec.card_id) AS card_id,
+                      LOWER(
+                        IF(pp.interface_name IS NULL,
+                        '', pp.interface_name)) AS interface_name
+                    FROM circuit c
+                      INNER JOIN circuit_connections cc
+                        ON cc.circ_absid = c.absid
+                      INNER JOIN equipment e
+                        ON e.absid = cc.PTR_equip_b
+                      LEFT JOIN events
+                        ON events.absid = cc.PTR_project
+                      INNER JOIN equipment_card ec
+                        ON ec.absid = cc.PTR_card_b
+                      LEFT JOIN organisation o
+                        ON o.absid = ec.manufacturer
+                      LEFT JOIN port_plugin pp
+                        ON pp.PTR_card = cc.PTR_card_b AND pp.port = cc.port_b
+                    WHERE c.status != 'terminated' AND is_circuit = 1
+                    UNION
+                    SELECT
+                      c.absid AS id,
+                      c.name,
+                      LOWER(c.status) AS status,
+                      LOWER(c.circuit_type) AS circuit_type,
+                      LOWER(c.service_type) AS service_type,
+                      events.short_descr AS project,
+                      e.name AS equipment,
+                      cc.port_a_OUT AS port,
+                      cc.int_LU_a AS logical_unit,
+                      LOWER(o.name) AS manufacturer,
+                      LOWER(ec.card_id) AS card_id,
+                      LOWER(
+                        IF(pp.interface_name IS NULL,
+                        '', pp.interface_name)) AS interface_name
+                    FROM circuit c
+                      INNER JOIN circuit_connections cc
+                        ON cc.circ_absid = c.absid
+                      INNER JOIN equipment e
+                        ON e.absid = cc.PTR_equip_a
+                      LEFT JOIN events
+                        ON events.absid = cc.PTR_project
+                      INNER JOIN equipment_card ec
+                        ON ec.absid = cc.PTR_card_a_OUT
+                      LEFT JOIN organisation o
+                        ON o.absid = ec.manufacturer
+                      LEFT JOIN port_plugin pp
+                        ON pp.PTR_card = cc.PTR_card_a_OUT
+                         AND pp.port = cc.port_a_OUT
+                    WHERE c.status != 'terminated' AND is_circuit = 1
+                    UNION
+                    SELECT
+                      c.absid AS id,
+                      c.name,
+                      LOWER(c.status) AS status,
+                      LOWER(c.circuit_type) AS circuit_type,
+                      LOWER(c.service_type) AS service_type,
+                      events.short_descr AS project,
+                      e.name AS equipment,
+                      cc.port_b_OUT AS port,
+                      cc.int_LU_b AS logical_unit,
+                      LOWER(o.name) AS manufacturer,
+                      LOWER(ec.card_id) AS card_id,
+                      LOWER(
+                        IF(pp.interface_name IS NULL,
+                        '', pp.interface_name)) AS interface_name
+                    FROM circuit c
+                      INNER JOIN circuit_connections cc
+                        ON cc.circ_absid = c.absid
+                      INNER JOIN equipment e
+                        ON e.absid = cc.PTR_equip_b
+                      LEFT JOIN events
+                        ON events.absid = cc.PTR_project
+                      INNER JOIN equipment_card ec
+                        ON ec.absid = cc.PTR_card_b_OUT
+                      LEFT JOIN organisation o
+                        ON o.absid = ec.manufacturer
+                      LEFT JOIN port_plugin pp
+                        ON pp.PTR_card = cc.PTR_card_b_OUT
+                         AND pp.port = cc.port_b_OUT
+                    WHERE
+                     c.status != 'terminated'
+                      AND is_circuit = 1)
+                    AS inner_query
+                    ORDER BY
+                     FIELD(status,
+                      'spare',
+                      'planned',
+                      'ordered',
+                      'installed',
+                      'operational')"""
+
+
+def _convert_to_dict(crs):
+    return [dict((crs.description[i][0], "" if value is None else value)
+                 for i, value in enumerate(row)) for row in crs.fetchall()]
+
+
+def _infinera_field_update(record):
+    equipment_parts = record["equipment"].rsplit("-", 1)
+    card_parts = record["card_id"].split("-", 1)
+    record["interface_name"] = ""
+    record["equipment"] = equipment_parts[0]
     try:
-        cx = mysql.connector.connect(
-            host=opsdb["hostname"],
-            user=opsdb["username"],
-            passwd=opsdb["password"],
-            db=opsdb["dbname"])
-        yield cx
-    finally:
-        if cx:
-            cx.close()
-
-
-@contextlib.contextmanager
-def cursor(cnx):  # pragma: no cover
-    csr = None
+        record["interface_name"] = equipment_parts[1] + "-"
+    except IndexError:
+        pass  # Nothing to see here
     try:
-        csr = cnx.cursor()
-        yield csr
-    finally:
-        if csr:
-            csr.close()
-
-
-def _db_test(db, router):
-    database_logger = logging.getLogger(DATABASE_LOGGER_NAME)
-    with cursor(db) as crs:
-        query = "select model, manufacturer from equipment where name = %s"
-        crs.execute(query, (router['hostname'],))
-        for (model, manufacturer) in crs:
-            database_logger.debug("%s: %s %s" % (
-                router['hostname'], model, manufacturer))
-            yield {"model": model, "manufacturer": manufacturer}
+        record["interface_name"] += card_parts[1]
+    except IndexError:
+        record["interface_name"] += card_parts[0]
+    if record["port"] is not None and record["port"] != "":
+        record["interface_name"] += "-" + record["port"]
+    record["interface_name"] = record["interface_name"] \
+        .replace("--", "-").upper()
+    return record
+
+
+def _juniper_field_update(record):
+    if not record["interface_name"]:
+        record["interface_name"] = record["card_id"]
+        if record["port"] is not None and record["port"] != "":
+            separator = "/" if "-" in record["interface_name"] else ""
+            record["interface_name"] += separator + str(record["port"])
+    if record["logical_unit"] is not None and record["logical_unit"] != "":
+        record["interface_name"] += "." + str(record["logical_unit"])
+    return record
+
+
+def _update_fields(r):
+    func = globals().get("_" + r["manufacturer"] + "_field_update")
+    return func(r) if func else r
+
+
+def get_circuits(connection):
+    with db.cursor(connection) as crs:
+        crs.execute(retrieve_services_query)
+        r = _convert_to_dict(crs)
+    r = list(map(_update_fields, r))
+    return r
+
+
+def get_circuit_hierarchy(connection):
+    with db.cursor(connection) as crs:
+        crs.execute(circuit_hierarchy_query)
+        r = _convert_to_dict(crs)
+    return r
+
+
+def get_equipment_location_data(connection):
+    with db.cursor(connection) as crs:
+        crs.execute(equipment_location_query)
+        r = _convert_to_dict(crs)
+    return r

inventory_provider/routes/alarmsdb.py

@@ -2,7 +2,7 @@ import functools
 import json
 
 from flask import Blueprint, request, Response, current_app
-from inventory_provider import alarmsdb
+from inventory_provider import alarmsdb, db
 
 routes = Blueprint("inventory-alarmsdb-query-routes", __name__)
 
@@ -34,9 +34,9 @@ def get_interface_status():
     equipment = request.args.get("equipment")
     interface = request.args.get("interface")
 
-    with alarmsdb.connection(config['alarms-db']) as db:
+    with db.connection(config['alarms-db']) as connection:
         result = {"status": alarmsdb.get_last_known_interface_status(
-            db, equipment, interface)}
+            connection, equipment, interface)}
 
     return Response(
         json.dumps(result),

inventory_provider/routes/jobs.py

 import logging
 from flask import Blueprint, Response, current_app
 
+import inventory_provider.storage.external_inventory as external_inventory
+from inventory_provider import db, opsdb
 from inventory_provider.tasks.app import app
 from inventory_provider.constants import TASK_LOGGER_NAME
 
@@ -34,3 +36,58 @@ def update():
             args=[r["hostname"], r["community"]])
 
     return Response("OK")
+
+
+@routes.route("update-services", methods=['GET'])
+def update_service():
+    config = current_app.config['INVENTORY_PROVIDER_CONFIG']
+
+    with db.connection(config['ops-db']) as connection:
+        result = opsdb.get_circuits(connection)
+    external_inventory.update_services_to_monitor(result)
+    return Response("OK")
+
+
+@routes.route("update-interfaces", methods=['GET'])
+def update_interfaces():
+    config = current_app.config['INVENTORY_PROVIDER_CONFIG']
+
+    with db.connection(config['ops-db']) as connection:
+        result = opsdb.get_circuits(connection)
+    external_inventory.update_interfaces_to_services(result)
+    return Response("OK")
+
+
+@routes.route("update-service-hierarchy", methods=['GET'])
+def update_service_hierarchy():
+    config = current_app.config['INVENTORY_PROVIDER_CONFIG']
+
+    with db.connection(config['ops-db']) as connection:
+        result = opsdb.get_circuit_hierarchy(connection)
+    external_inventory.update_service_hierarchy(result)
+    return Response("OK")
+
+
+@routes.route("update-equipment-locations", methods=['GET'])
+def update_equipment_locations():
+    config = current_app.config['INVENTORY_PROVIDER_CONFIG']
+
+    with db.connection(config['ops-db']) as connection:
+        result = opsdb.get_equipment_location_data(connection)
+    external_inventory.update_equipment_locations(result)
+    return Response("OK")
+
+
+@routes.route("update-from-inventory-system", methods=['GET'])
+def update_from_inventory_system():
+    config = current_app.config['INVENTORY_PROVIDER_CONFIG']
+
+    with db.connection(config['ops-db']) as connection:
+        circuits = opsdb.get_circuits(connection)
+        hierarchy = opsdb.get_circuit_hierarchy(connection)
+        equipment_locations = opsdb.get_equipment_location_data(connection)
+    external_inventory.update_services_to_monitor(circuits)
+    external_inventory.update_interfaces_to_services(circuits)
+    external_inventory.update_service_hierarchy(hierarchy)
+    external_inventory.update_equipment_locations(equipment_locations)
+    return Response("OK")

inventory_provider/routes/opsdb.py

 import functools
 import json
 
-from flask import Blueprint, request, Response, current_app
-from inventory_provider import opsdb
+from flask import Blueprint, request, Response
+from inventory_provider import db
+from inventory_provider.storage import external_inventory
 
 routes = Blueprint("inventory-opsdb-query-routes", __name__)
 
+services_key = "inv_services"
+interfaces_key = "inv_interfaces"
+equipment_locations_key = "inv_eq_locations"
+
 
 def require_accepts_json(f):
     """
@@ -26,16 +31,78 @@ def require_accepts_json(f):
     return decorated_function
 
 
-@routes.route("/test", methods=['GET', 'POST'])
-@require_accepts_json
-def opsdb_test():
-    config = current_app.config['INVENTORY_PROVIDER_CONFIG']
+def _decode_utf8_dict(d):
+    return {k.decode('utf8'): json.loads(v) for k, v in d.items()}
+
+
+@routes.route("/interfaces")
+def get_all_interface_details():
+    r = db.get_redis()
+    result = _decode_utf8_dict(
+            r.hgetall(interfaces_key))
+
+    return Response(
+        json.dumps(result),
+        mimetype="application/json")
+
 
+@routes.route("/interfaces/<equipment_name>")
+def get_interface_details_for_equipment(equipment_name):
+    r = db.get_redis()
     result = {}
-    with opsdb.connection(config['ops-db']) as db:
-        for r in config['routers']:
-            result[r['hostname']] = list(opsdb._db_test(db, r))
+    for t in r.hscan_iter(interfaces_key, "{}::*".format(equipment_name)):
+        result[t[0].decode("utf8")] = json.loads(t[1])
+
+    return Response(
+        json.dumps(result),
+        mimetype="application/json")
+
+
+@routes.route("/interfaces/<equipment_name>/<path:interface>")
+def get_interface_details(equipment_name, interface):
+    r = db.get_redis()
+    return Response(
+        r.hget(
+            interfaces_key,
+            "{}::{}".format(equipment_name, interface)),
+        mimetype="application/json")
+
+
+@routes.route("/equipment-location")
+def get_all_equipment_locations():
+    r = db.get_redis()
+    result = list(
+        _decode_utf8_dict(
+            r.hgetall(equipment_locations_key)).values())
 
     return Response(
         json.dumps(result),
         mimetype="application/json")
+
+
+@routes.route("/equipment-location/<path:equipment_name>")
+def get_equipment_location(equipment_name):
+    r = db.get_redis()
+    return Response(
+        r.hget(equipment_locations_key, equipment_name),
+        mimetype="application/json")
+
+
+@routes.route("/circuit-hierarchy/children/<parent_id>")
+def get_children(parent_id):
+    r = db.get_redis()
+    return Response(
+        r.hget(
+            external_inventory.service_parent_to_children_key,
+            parent_id),
+        mimetype="application/json")
+
+
+@routes.route("/circuit-hierarchy/parents/<child_id>")
+def get_parents(child_id):
+    r = db.get_redis()
+    return Response(
+        r.hget(
+            external_inventory.service_child_to_parents_key,
+            child_id),
+        mimetype="application/json")

inventory_provider/static/juniper.html

@@ -20,7 +20,11 @@
         <p><b>interfaces</b></p>
         <ul>
           <li ng-repeat="i in interfaces">{{i.name}}
-            <ul><li>{{i.description}}</li></ul>
+            <ul>
+              <li>{{i.description}}</li>
+              <li ng-repeat="v4 in i.ipv4">v4: {{v4}}</li>
+              <li ng-repeat="v6 in i.ipv6">v6: {{v6}}</li>
+            </ul>
           </li>
         </ul>
         <div class="raw">{{interfaces}}</div>

inventory_provider/storage/external_inventory.py

+import json
+from collections import defaultdict
+from inventory_provider import db
+
+
+services_key = "inv_services"
+interfaces_key = "inv_interfaces"
+equipment_locations_key = "inv_eq_locations"
+service_child_to_parents_key = "inv_service_child_to_parents"
+service_parent_to_children_key = "inv_service_parent_to_children"
+
+
+def update_services_to_monitor(services):
+    r = db.get_redis()
+    relevant_types = ('path', 'service', 'l2circuit')
+    r.delete(services_key)
+    for service in services:
+        if service['circuit_type'].lower() in relevant_types:
+            r.hset(services_key, service['id'], json.dumps(service))
+
+
+def update_interfaces_to_services(services):
+    r = db.get_redis()
+    mapped_interfaces = defaultdict(list)
+    r.delete(interfaces_key)
+    for service in services:
+        key = "{}::{}".format(
+            service['equipment'],
+            service['interface_name']
+        )
+        mapped_interfaces[key].append(service)
+
+    for key, value in mapped_interfaces.items():
+        r.hset(interfaces_key, key, json.dumps(value))
+
+
+def update_service_hierarchy(records):
+    r = db.get_redis()
+    children_to_parents = defaultdict(list)
+    parents_to_children = defaultdict(list)
+    for relation in records:
+        parent_id = relation["parent_circuit_id"]
+        child_id = relation["child_circuit_id"]
+        parents_to_children[parent_id].append(relation)
+        children_to_parents[child_id].append(relation)
+
+    r.delete(service_child_to_parents_key)
+    for child, parents in children_to_parents.items():
+        r.hset(service_child_to_parents_key, child, json.dumps(parents))
+
+    r.delete(service_parent_to_children_key)
+    for parent, children in parents_to_children.items():
+        r.hset(service_parent_to_children_key, parent, json.dumps(children))
+
+
+def update_equipment_locations(equipment_location_data):
+    r = db.get_redis()
+    r.delete(equipment_locations_key)
+    for ld in equipment_location_data:
+        r.hset(equipment_locations_key, ld['equipment_name'], json.dumps(ld))

setup.py

@@ -2,7 +2,7 @@ from setuptools import setup, find_packages
 
 setup(
     name='inventory-provider',
-    version="0.12",
+    version="0.13",
     author='GEANT',
     author_email='swd@geant.org',
     description='Dashboard inventory provider',

test/storage/test_external_inventory.py

+from inventory_provider.storage import external_inventory
+
+
+def test_update_services_to_monitor(mocker):
+    mocked_redis = mocker.patch(
+        "inventory_provider.storage.external_inventory.db.get_redis")
+    mocked_hset = mocked_redis.return_value.hset
+
+    services = [
+        {"circuit_type": "path", "id": "test_id_0", "other": "stuff"},
+        {"circuit_type": "service", "id": "test_id_1", "other": "stuff"},
+        {"circuit_type": "l2circuit", "id": "test_id_2", "other": "stuff"},
+        {"circuit_type": "spam", "id": "test_id_4", "other": "stuff"}
+    ]
+    external_inventory.update_services_to_monitor(services)
+    mocked_hset.assert_any_call(
+        external_inventory.services_key, "test_id_0",
+        "{\"circuit_type\": \"path\","
+        " \"id\": \"test_id_0\","
+        " \"other\": \"stuff\"}")
+    mocked_hset.assert_any_call(
+        external_inventory.services_key, "test_id_1",
+        "{\"circuit_type\": \"service\","
+        " \"id\": \"test_id_1\","
+        " \"other\": \"stuff\"}")
+    mocked_hset.assert_any_call(
+        external_inventory.services_key, "test_id_2",
+        "{\"circuit_type\": \"l2circuit\","
+        " \"id\": \"test_id_2\","
+        " \"other\": \"stuff\"}")
+    assert mocked_hset.call_count == 3
+
+
+def test_update_interfaces_to_services(mocker):
+    mocked_redis = mocker.patch(
+        "inventory_provider.storage.external_inventory.db.get_redis")
+    mocked_hset = mocked_redis.return_value.hset
+    services = [
+        {"equipment": "eq_0", "interface_name": "if_0"},
+        {"equipment": "eq_1", "interface_name": "if_1"},
+        {"equipment": "eq_2", "interface_name": "if_2"},
+        {"equipment": "eq_3", "interface_name": "if_3"},
+        {"equipment": "eq_3", "interface_name": "if_3", "extra": "stuff"},
+        {"equipment": "eq_4", "interface_name": "if_4"}
+    ]
+    unique_keys = set(s["equipment"] + "::" + s["interface_name"]
+                      for s in services)
+
+    external_inventory.update_interfaces_to_services(services)
+    assert mocked_hset.call_count == len(unique_keys)
+    mocked_hset.assert_any_call(external_inventory.interfaces_key,
+                                "eq_2::if_2",
+                                "[{\"equipment\": \"eq_2\","
+                                " \"interface_name\": \"if_2\"}]")
+    mocked_hset.assert_any_call(external_inventory.interfaces_key,
+                                "eq_3::if_3",
+                                "[{\"equipment\": \"eq_3\","
+                                " \"interface_name\": \"if_3\"},"
+                                " {\"equipment\": \"eq_3\","
+                                " \"interface_name\": \"if_3\","
+                                " \"extra\": \"stuff\"}]")
+
+
+def test_update_service_hierarchy(mocker):
+    mocked_redis = mocker.patch(
+        "inventory_provider.storage.external_inventory.db.get_redis")
+    mocked_hset = mocked_redis.return_value.hset
+    data = [
+        {"parent_circuit": "PC_1",
+         "parent_circuit_id": "1001",
+         "parent_circuit_status": "operational",
+         "child_circuit": "CC_1",
+         "child_circuit_id": "2001",
+         "segment_group": 1},
+        {"parent_circuit": "PC_2",
+         "parent_circuit_id": "1002",
+         "parent_circuit_status": "operational",
+         "child_circuit": "CC_1",
+         "child_circuit_id": "2001",
+         "segment_group": 1},
+        {"parent_circuit": "PC_3",
+         "parent_circuit_id": "1003",
+         "parent_circuit_status": "operational",
+         "child_circuit": "CC_2",
+         "child_circuit_id": "2002",
+         "segment_group": 1},
+        {"parent_circuit": "PC_3",
+         "parent_circuit_id": "1003",
+         "parent_circuit_status": "operational",
+         "child_circuit": "CC_3",
+         "child_circuit_id": "2003",
+         "segment_group": 1}
+    ]
+    external_inventory.update_service_hierarchy(data)
+    u_parent_keys = set([d["parent_circuit_id"] for d in data])
+    u_child_keys = set([d["child_circuit_id"] for d in data])
+    assert mocked_hset.call_count == len(u_parent_keys) + len(u_child_keys)
+    mocked_hset.assert_any_call(
+        external_inventory.service_child_to_parents_key,
+        "2001",
+        "[{\"parent_circuit\": \"PC_1\","
+        " \"parent_circuit_id\": \"1001\","
+        " \"parent_circuit_status\": \"operational\","
+        " \"child_circuit\": \"CC_1\","
+        " \"child_circuit_id\": \"2001\","
+        " \"segment_group\": 1},"
+        " {\"parent_circuit\": \"PC_2\","
+        " \"parent_circuit_id\": \"1002\","
+        " \"parent_circuit_status\": \"operational\","
+        " \"child_circuit\": \"CC_1\","
+        " \"child_circuit_id\": \"2001\","
+        " \"segment_group\": 1}]"
+    )
+    mocked_hset.assert_any_call(
+        external_inventory.service_child_to_parents_key,
+        "2002",
+        "[{\"parent_circuit\": \"PC_3\","
+        " \"parent_circuit_id\": \"1003\","
+        " \"parent_circuit_status\": \"operational\","
+        " \"child_circuit\": \"CC_2\","
+        " \"child_circuit_id\": \"2002\","
+        " \"segment_group\": 1}]"
+    )
+
+    mocked_hset.assert_any_call(
+        external_inventory.service_parent_to_children_key,
+        "1003",
+        "[{\"parent_circuit\": \"PC_3\","
+        " \"parent_circuit_id\": \"1003\","
+        " \"parent_circuit_status\": \"operational\","
+        " \"child_circuit\": \"CC_2\","
+        " \"child_circuit_id\": \"2002\","
+        " \"segment_group\": 1},"
+        " {\"parent_circuit\": \"PC_3\","
+        " \"parent_circuit_id\": \"1003\","
+        " \"parent_circuit_status\": \"operational\","
+        " \"child_circuit\": \"CC_3\","
+        " \"child_circuit_id\": \"2003\","
+        " \"segment_group\": 1}]"
+    )
+    mocked_hset.assert_any_call(
+        external_inventory.service_parent_to_children_key,
+        "1002",
+        "[{\"parent_circuit\": \"PC_2\","
+        " \"parent_circuit_id\": \"1002\","
+        " \"parent_circuit_status\": \"operational\","
+        " \"child_circuit\": \"CC_1\","
+        " \"child_circuit_id\": \"2001\","
+        " \"segment_group\": 1}]"
+    )

test/test_alarmdb_routes.py

@@ -9,7 +9,7 @@ DEFAULT_REQUEST_HEADERS = {
 
 def test_get_interface_status(mocker, client):
     mocked_conn = mocker.patch('inventory_provider.routes.alarmsdb'
-                               '.alarmsdb.connection')
+                               '.db.connection')
     mocked_conn.return_value.__enter__.return_value = None
 
     mocked_inteface_status = mocker.patch(

test/test_alarmsdb.py

@@ -2,7 +2,7 @@ import inventory_provider.alarmsdb as alarmsdb
 
 
 def test_infinera_interface_status(mocker):
-    mocked_get_cursor = mocker.patch('inventory_provider.alarmsdb.cursor')
+    mocked_get_cursor = mocker.patch('inventory_provider.alarmsdb.db.cursor')
     mocked_execute = mocked_get_cursor. \
         return_value.__enter__.return_value.execute
     mocked_fetchone = mocked_get_cursor.return_value.__enter__. \
@@ -30,7 +30,7 @@ def test_infinera_interface_status(mocker):
 
 
 def test_coriant_interface_status(mocker):
-    mocked_get_cursor = mocker.patch('inventory_provider.alarmsdb.cursor')
+    mocked_get_cursor = mocker.patch('inventory_provider.alarmsdb.db.cursor')
     mocked_execute = mocked_get_cursor. \
         return_value.__enter__.return_value.execute
     mocked_fetchone = mocked_get_cursor.return_value.__enter__. \
@@ -58,7 +58,7 @@ def test_coriant_interface_status(mocker):
 
 
 def test_juniper_interface_status(mocker):
-    mocked_get_cursor = mocker.patch('inventory_provider.alarmsdb.cursor')
+    mocked_get_cursor = mocker.patch('inventory_provider.alarmsdb.db.cursor')
     mocked_execute = mocked_get_cursor. \
         return_value.__enter__.return_value.execute
     mocked_fetchone = mocked_get_cursor.return_value.__enter__. \

test/test_external_inventory_routes.py

+import json
+from inventory_provider.storage import external_inventory
+
+DEFAULT_REQUEST_HEADERS = {
+    "Content-type": "application/json",
+    "Accept": ["application/json"]
+}
+
+
+def test_get_one_equipment_location(mocker, client):
+    mocked_redis = mocker.patch(
+        "inventory_provider.routes.opsdb.db.get_redis")
+    mocked_hget = mocked_redis.return_value.hget
+    dummy_data = {
+        "absid": 1404,
+        "equipment_name": "pp-cz-e13b",
+        "pop_name": "Prague",
+        "pop_abbreviation": "pra",
+        "pop_site_id": "",
+        "country": "Czech Republic",
+        "longitude": 14.391738888889,
+        "latitude": 50.101847222222
+    }
+    mocked_hget.return_value = json.dumps(dummy_data)
+
+    rv = client.get(
+        '/opsdb/equipment-location/dummy-equipment',
+        headers=DEFAULT_REQUEST_HEADERS)
+    assert rv.status_code == 200
+    assert rv.is_json
+    assert dummy_data == json.loads(rv.data.decode("utf-8"))
+
+    mocked_hget.assert_called_with(
+        external_inventory.equipment_locations_key,
+        "dummy-equipment"
+    )
+
+
+def test_get_equipment_location(mocker, client):
+    mocked_redis = mocker.patch(
+        "inventory_provider.routes.opsdb.db.get_redis")
+    mocked_hgetall = mocked_redis.return_value.hgetall
+
+    rv = client.get(
+        '/opsdb/equipment-location',
+        headers=DEFAULT_REQUEST_HEADERS)
+    assert rv.status_code == 200
+    assert rv.is_json
+
+    mocked_hgetall.assert_called_with(
+        external_inventory.equipment_locations_key
+    )
+
+
+def test_get_interface_info(mocker, client):
+    mocked_redis = mocker.patch(
+        "inventory_provider.routes.opsdb.db.get_redis")
+    mocked_hgetall = mocked_redis.return_value.hgetall
+
+    rv = client.get(
+        '/opsdb/interfaces',
+        headers=DEFAULT_REQUEST_HEADERS)
+    assert rv.status_code == 200
+    assert rv.is_json
+
+    mocked_hgetall.assert_called_with(
+        external_inventory.interfaces_key
+    )
+
+
+def test_get_interface_info_for_equipment(mocker, client):
+    mocked_redis = mocker.patch(
+        "inventory_provider.routes.opsdb.db.get_redis")
+    mocked_hscan_iter = mocked_redis.return_value.hscan_iter
+
+    rv = client.get(
+        '/opsdb/interfaces/dummy-equipment',
+        headers=DEFAULT_REQUEST_HEADERS)
+    assert rv.status_code == 200
+    assert rv.is_json
+
+    mocked_hscan_iter.assert_called_with(
+        external_inventory.interfaces_key, "dummy-equipment::*"
+    )
+
+
+def test_get_interface_info_for_equipment_and_interface(mocker, client):
+    mocked_redis = mocker.patch(
+        "inventory_provider.routes.opsdb.db.get_redis")
+    mocked_hget = mocked_redis.return_value.hget
+
+    rv = client.get(
+        '/opsdb/interfaces/dummy-equipment/xe-2/3/1',
+        headers=DEFAULT_REQUEST_HEADERS)
+    assert rv.status_code == 200
+    assert rv.is_json
+
+    mocked_hget.assert_called_with(
+        external_inventory.interfaces_key, "dummy-equipment::xe-2/3/1"
+    )
+
+
+def test_get_children(mocker, client):
+    mocked_redis = mocker.patch(
+        "inventory_provider.routes.opsdb.db.get_redis")
+    mocked_hget = mocked_redis.return_value.hget
+
+    rv = client.get(
+        '/opsdb/circuit-hierarchy/children/22987',
+        headers=DEFAULT_REQUEST_HEADERS)
+    assert rv.status_code == 200
+    assert rv.is_json
+
+    mocked_hget.assert_called_with(
+        external_inventory.service_parent_to_children_key,
+        "22987"
+    )
+
+
+def test_get_parents(mocker, client):
+    mocked_redis = mocker.patch(
+        "inventory_provider.routes.opsdb.db.get_redis")
+    mocked_hget = mocked_redis.return_value.hget
+
+    rv = client.get(
+        '/opsdb/circuit-hierarchy/children/22987',
+        headers=DEFAULT_REQUEST_HEADERS)
+    assert rv.status_code == 200
+    assert rv.is_json
+
+    mocked_hget.assert_called_with(
+        external_inventory.service_parent_to_children_key,
+        "22987"
+    )

test/test_opsdb.py

+import inventory_provider.opsdb
+
+
+def test_update_fields(mocker):
+    mocker.patch("inventory_provider.opsdb._juniper_field_update")
+    t = {"manufacturer": "juniper"}
+    inventory_provider.opsdb._update_fields(t)
+    inventory_provider.opsdb._juniper_field_update.assert_called_once_with(t)
+
+    mocker.patch("inventory_provider.opsdb._infinera_field_update")
+    t = {"manufacturer": "infinera"}
+    inventory_provider.opsdb._update_fields(t)
+    inventory_provider.opsdb._infinera_field_update.assert_called_once_with(t)
+
+    f = {"manufacturer": "non-existent"}
+    r = inventory_provider.opsdb._update_fields(f)
+    assert f == r
+
+
+def test_infinera_field_update():
+    i = {
+        "equipment": "AMS01-DTNX10-1-1",
+        "card_id": "tim-b-5-7",
+        "port": "1"
+         }
+    r = inventory_provider.opsdb._infinera_field_update(i)
+    assert r["equipment"] == "AMS01-DTNX10-1"
+    assert r["interface_name"] == "1-B-5-7-1"
+
+    i = {
+        "equipment": "BUD01_CX_01",
+        "card_id": "tim-1/2",
+        "port": "1"
+         }
+    r = inventory_provider.opsdb._infinera_field_update(i)
+    assert r["equipment"] == "BUD01_CX_01"
+    assert r["interface_name"] == "1/2-1"
+
+    i = {
+        "equipment": "irrelevant",
+        "card_id": "tim_1/2",
+        "port": "1"
+         }
+    r = inventory_provider.opsdb._infinera_field_update(i)
+    assert r["interface_name"] == "TIM_1/2-1"
+
+
+def test_juniper_field_update():
+    i = {
+        "interface_name": "xe-1/2",
+        "logical_unit": None
+         }
+    r = inventory_provider.opsdb._juniper_field_update(i)
+    assert r["interface_name"] == "xe-1/2"
+
+    i["interface_name"] = "xe-1/2"
+    i["logical_unit"] = 101
+    r = inventory_provider.opsdb._juniper_field_update(i)
+    assert r["interface_name"] == "xe-1/2.101"
+
+    i["interface_name"] = "xe-1/2"
+    i["logical_unit"] = 0
+    r = inventory_provider.opsdb._juniper_field_update(i)
+    assert r["interface_name"] == "xe-1/2.0"
+
+    i["interface_name"] = "xe-1/2"
+    i["logical_unit"] = None
+    i["port"] = 0
+    r = inventory_provider.opsdb._juniper_field_update(i)
+    assert r["interface_name"] == "xe-1/2"
+
+    i["interface_name"] = None
+    i["card_id"] = "xe-2/0"
+    i["logical_unit"] = None
+    i["port"] = None
+    r = inventory_provider.opsdb._juniper_field_update(i)
+    assert r["interface_name"] == "xe-2/0"
+
+    i["interface_name"] = None
+    i["port"] = "0"
+    r = inventory_provider.opsdb._juniper_field_update(i)
+    assert r["interface_name"] == "xe-2/0/0"
+
+    i["interface_name"] = None
+    i["port"] = 0
+    r = inventory_provider.opsdb._juniper_field_update(i)
+    assert r["interface_name"] == "xe-2/0/0"
+
+    i["interface_name"] = None
+    i["logical_unit"] = "123"
+    r = inventory_provider.opsdb._juniper_field_update(i)
+    assert r["interface_name"] == "xe-2/0/0.123"
+
+    i["interface_name"] = None
+    i["logical_unit"] = 123
+    r = inventory_provider.opsdb._juniper_field_update(i)
+    assert r["interface_name"] == "xe-2/0/0.123"
+
+
+def test_get_circuits(mocker):
+    mocker.patch("inventory_provider.opsdb.db.cursor")
+    mocked_convert_to_dict = mocker.patch(
+        "inventory_provider.opsdb._convert_to_dict")
+    i = {"manufacturer": "infinera"}
+    j = {"manufacturer": "juniper"}
+    mocked_convert_to_dict.return_value = [i, j]
+
+    mocked_infinera_update = mocker.patch(
+        "inventory_provider.opsdb._infinera_field_update")
+    mocked_juniper_update = mocker.patch(
+        "inventory_provider.opsdb._juniper_field_update")
+
+    inventory_provider.opsdb.get_circuits(None)
+
+    mocked_infinera_update.assert_called_once_with(i)
+    mocked_juniper_update.assert_called_once_with(j)

tox.ini

@@ -10,7 +10,7 @@ deps =
 
 commands =
     coverage erase
-    coverage run --source inventory_provider -m py.test
+    coverage run --source inventory_provider -m py.test {posargs}
     coverage xml
     coverage html
     coverage report --fail-under 75