Merge branch 'develop' into feature/classifier-support

changelog

@@ -14,5 +14,6 @@
 0.12: added addresses to interface response
       put actual module number in version response
 0.13: added external inventory caching
-0.14<in progress>: added sample route for startup-only tasks
+0.14: added sample route for startup-only tasks
       added method for caching selected alarmsdb tables
+      added caching of last known interface status from alarmsdb

inventory_provider/alarmsdb.py

 from inventory_provider import db
 
 
-def get_last_known_infinera_interface_status(connection, equipment, interface):
+def get_last_known_infinera_interface_status(crs, equipment, interface):
     query = "SELECT status FROM infinera_alarms" \
             " WHERE" \
             " CONCAT(ne_name, '-', REPLACE(object_name, 'T', '')) = %s" \
             " ORDER BY ne_init_time DESC, ne_clear_time DESC LIMIT 1"
     search_string = equipment + "-" + interface
-    with db.cursor(connection) as crs:
-        crs.execute(query, (search_string,))
-        result = crs.fetchone()
+    crs.execute(query, (search_string,))
+    result = crs.fetchone()
     if not result:
         return "unknown"
     elif result[0] == "Raised":
@@ -18,13 +17,12 @@ def get_last_known_infinera_interface_status(connection, equipment, interface):
         return "up"
 
 
-def get_last_known_coriant_interface_status(connection, equipment, interface):
+def get_last_known_coriant_interface_status(crs, equipment, interface):
     query = "SELECT status FROM coriant_alarms" \
             " WHERE ne_id_name = %s AND entity_string LIKE %s" \
             " ORDER BY last_event_time DESC LIMIT 1"
-    with db.cursor(connection) as crs:
-        crs.execute(query, (equipment, interface + "-%"))
-        result = crs.fetchone()
+    crs.execute(query, (equipment, interface + "-%"))
+    result = crs.fetchone()
     if not result:
         return "unknown"
     elif result[0] == "Raised":
@@ -34,14 +32,13 @@ def get_last_known_coriant_interface_status(connection, equipment, interface):
 
 
 def get_last_known_juniper_link_interface_status(
-        connection, equipment, interface):
+        crs, equipment, interface):
     query = "SELECT IF(link_admin_status = 'up'" \
             " AND link_oper_status = 'up', 1, 0) AS up FROM juniper_alarms" \
             " WHERE equipment_name = %s AND link_interface_name = %s" \
             " ORDER BY alarm_id DESC LIMIT 1"
-    with db.cursor(connection) as crs:
-        crs.execute(query, ('lo0.' + equipment, interface))
-        result = crs.fetchone()
+    crs.execute(query, ('lo0.' + equipment, interface))
+    result = crs.fetchone()
     if not result:
         return "unknown"
     elif result[0] == 0:
@@ -50,15 +47,15 @@ def get_last_known_juniper_link_interface_status(
         return "up"
 
 
-def get_last_known_interface_status(connection, equipment, interface):
+def get_last_known_interface_status(crs, equipment, interface):
     result = get_last_known_infinera_interface_status(
-        connection, equipment, interface)
+        crs, equipment, interface)
     if result == "unknown":
         result = get_last_known_coriant_interface_status(
-            connection, equipment, interface)
+            crs, equipment, interface)
     if result == "unknown":
         result = get_last_known_juniper_link_interface_status(
-            connection, equipment, interface)
+            crs, equipment, interface)
     return result
 
 

inventory_provider/routes/data.py

@@ -2,11 +2,12 @@ import functools
 import json
 import pkg_resources
 
-from flask import Blueprint, request, Response, current_app
+from flask import Blueprint, jsonify, request, Response, current_app
 from lxml import etree
 import redis
 
-from inventory_provider import juniper
+from inventory_provider import db, juniper
+from inventory_provider.storage import external_inventory
 
 routes = Blueprint("inventory-data-query-routes", __name__)
 
@@ -162,3 +163,18 @@ def bgp_configs(hostname):
     return Response(
         json.dumps(routes),
         mimetype="application/json")
+
+
+@routes.route("/interfaces/status/<hostname>/<path:interface>",
+              methods=['GET', 'POST'])
+@require_accepts_json
+def interface_statuses(hostname, interface):
+    r = db.get_redis()
+    result = r.hget(external_inventory.interface_status_key,
+                    "{}::{}".format(hostname, interface))
+    if not result:
+        return Response(
+            response="no available info for {} {}".format(hostname, interface),
+            status=404,
+            mimetype="text/html")
+    return jsonify({"status": result.decode('utf-8')})

inventory_provider/storage/external_inventory.py

 import json
 from collections import defaultdict
-from inventory_provider import db
+from flask import current_app
+from inventory_provider import alarmsdb, db
 
 
 services_key = "inv_services"
@@ -8,27 +9,38 @@ interfaces_key = "inv_interfaces"
 equipment_locations_key = "inv_eq_locations"
 service_child_to_parents_key = "inv_service_child_to_parents"
 service_parent_to_children_key = "inv_service_parent_to_children"
+interface_status_key = "service_status"
 
 
 def update_services_to_monitor(services):
     r = db.get_redis()
-    relevant_types = ('path', 'service', 'l2circuit')
+    relevant_types = ("path", "service", "l2circuit")
     r.delete(services_key)
     for service in services:
-        if service['circuit_type'].lower() in relevant_types:
-            r.hset(services_key, service['id'], json.dumps(service))
+        if service["circuit_type"].lower() in relevant_types:
+            r.hset(services_key, service["id"], json.dumps(service))
 
 
 def update_interfaces_to_services(services):
     r = db.get_redis()
-    mapped_interfaces = defaultdict(list)
-    r.delete(interfaces_key)
-    for service in services:
-        key = "{}::{}".format(
-            service['equipment'],
-            service['interface_name']
-        )
-        mapped_interfaces[key].append(service)
+    config = current_app.config["INVENTORY_PROVIDER_CONFIG"]
+    with db.connection(config["alarms-db"]) as cnx:
+        with db.cursor(cnx) as csr:
+            mapped_interfaces = defaultdict(list)
+            r.delete(interfaces_key)
+            for service in services:
+                key = "{}::{}".format(
+                    service["equipment"],
+                    service["interface_name"]
+                )
+                mapped_interfaces[key].append(service)
+                if not r.hexists(interface_status_key, key):
+                    r.hset(interface_status_key, key,
+                           alarmsdb.get_last_known_interface_status(
+                               csr,
+                               service["equipment"],
+                               service["interface_name"]
+                           ))
 
     for key, value in mapped_interfaces.items():
         r.hset(interfaces_key, key, json.dumps(value))
@@ -57,4 +69,4 @@ def update_equipment_locations(equipment_location_data):
     r = db.get_redis()
     r.delete(equipment_locations_key)
     for ld in equipment_location_data:
-        r.hset(equipment_locations_key, ld['equipment_name'], json.dumps(ld))
+        r.hset(equipment_locations_key, ld["equipment_name"], json.dumps(ld))

setup.py

@@ -2,7 +2,7 @@ from setuptools import setup, find_packages
 
 setup(
     name='inventory-provider',
-    version="0.14",
+    version="0.15",
     author='GEANT',
     author_email='swd@geant.org',
     description='Dashboard inventory provider',

test/storage/test_external_inventory.py

@@ -34,6 +34,12 @@ def test_update_services_to_monitor(mocker):
 def test_update_interfaces_to_services(mocker):
     mocked_redis = mocker.patch(
         "inventory_provider.storage.external_inventory.db.get_redis")
+    mocker.patch(
+        "inventory_provider.storage.external_inventory.current_app")
+    mocker.patch(
+        "inventory_provider.storage.external_inventory.db.connection")
+    mocker.patch(
+        "inventory_provider.storage.external_inventory.db.cursor")
     mocked_hset = mocked_redis.return_value.hset
     services = [
         {"equipment": "eq_0", "interface_name": "if_0"},

test/test_alarmsdb.py

@@ -2,27 +2,23 @@ import inventory_provider.alarmsdb as alarmsdb
 
 
 def test_infinera_interface_status(mocker):
-    mocked_get_cursor = mocker.patch('inventory_provider.alarmsdb.db.cursor')
-    mocked_execute = mocked_get_cursor. \
-        return_value.__enter__.return_value.execute
-    mocked_fetchone = mocked_get_cursor.return_value.__enter__. \
-        return_value.fetchone
-    mocked_fetchone.return_value = ('Raised',)
+    mock = mocker.Mock()
+    mock.fetchone.return_value = ('Raised',)
     assert alarmsdb.get_last_known_infinera_interface_status(
-        None, 'eq1', 'intfc1'
+        mock, 'eq1', 'intfc1'
     ) == "down"
 
-    mocked_fetchone.return_value = ("Clear",)
+    mock.fetchone.return_value = ("Clear",)
     assert alarmsdb.get_last_known_infinera_interface_status(
-            None, 'eq1', 'intfc1'
+        mock, 'eq1', 'intfc1'
     ) == "up"
 
-    mocked_fetchone.return_value = ()
+    mock.fetchone.return_value = ()
     assert alarmsdb.get_last_known_infinera_interface_status(
-            None, 'eq1', 'intfc1'
+        mock, 'eq1', 'intfc1'
     ) == "unknown"
 
-    mocked_execute.assert_called_with(
+    mock.execute.assert_called_with(
         "SELECT status FROM infinera_alarms WHERE"
         " CONCAT(ne_name, '-', REPLACE(object_name, 'T', '')) = %s"
         " ORDER BY ne_init_time DESC, ne_clear_time DESC LIMIT 1",
@@ -30,27 +26,23 @@ def test_infinera_interface_status(mocker):
 
 
 def test_coriant_interface_status(mocker):
-    mocked_get_cursor = mocker.patch('inventory_provider.alarmsdb.db.cursor')
-    mocked_execute = mocked_get_cursor. \
-        return_value.__enter__.return_value.execute
-    mocked_fetchone = mocked_get_cursor.return_value.__enter__. \
-        return_value.fetchone
-    mocked_fetchone.return_value = ('Raised',)
+    mock = mocker.Mock()
+    mock.fetchone.return_value = ('Raised',)
     assert alarmsdb.get_last_known_coriant_interface_status(
-        None, 'eq1', 'intfc1'
+        mock, 'eq1', 'intfc1'
     ) == "down"
 
-    mocked_fetchone.return_value = ("Clear",)
+    mock.fetchone.return_value = ("Clear",)
     assert alarmsdb.get_last_known_coriant_interface_status(
-            None, 'eq1', 'intfc1'
+        mock, 'eq1', 'intfc1'
     ) == "up"
 
-    mocked_fetchone.return_value = ()
+    mock.fetchone.return_value = ()
     assert alarmsdb.get_last_known_coriant_interface_status(
-            None, 'eq1', 'intfc1'
+        mock, 'eq1', 'intfc1'
     ) == "unknown"
 
-    mocked_execute.assert_called_with(
+    mock.execute.assert_called_with(
         "SELECT status FROM coriant_alarms"
         " WHERE ne_id_name = %s AND entity_string LIKE %s"
         " ORDER BY last_event_time DESC LIMIT 1",
@@ -58,27 +50,23 @@ def test_coriant_interface_status(mocker):
 
 
 def test_juniper_interface_status(mocker):
-    mocked_get_cursor = mocker.patch('inventory_provider.alarmsdb.db.cursor')
-    mocked_execute = mocked_get_cursor. \
-        return_value.__enter__.return_value.execute
-    mocked_fetchone = mocked_get_cursor.return_value.__enter__. \
-        return_value.fetchone
-    mocked_fetchone.return_value = (0,)
+    mock = mocker.Mock()
+    mock.fetchone.return_value = (0,)
     assert alarmsdb.get_last_known_juniper_link_interface_status(
-        None, 'eq1', 'intfc1'
+        mock, 'eq1', 'intfc1'
     ) == "down"
 
-    mocked_fetchone.return_value = (1,)
+    mock.fetchone.return_value = (1,)
     assert alarmsdb.get_last_known_juniper_link_interface_status(
-            None, 'eq1', 'intfc1'
+        mock, 'eq1', 'intfc1'
     ) == "up"
 
-    mocked_fetchone.return_value = ()
+    mock.fetchone.return_value = ()
     assert alarmsdb.get_last_known_juniper_link_interface_status(
-            None, 'eq1', 'intfc1'
+        mock, 'eq1', 'intfc1'
     ) == "unknown"
 
-    mocked_execute.assert_called_with(
+    mock.execute.assert_called_with(
                 "SELECT IF(link_admin_status = 'up'"
                 " AND link_oper_status = 'up', 1, 0)"
                 " AS up FROM juniper_alarms"