Finished release 0.14.

README.md

@@ -252,3 +252,7 @@ Any non-empty responses are JSON formatted messages.
 
   This resource updates the inventory network data for juniper devices.
 
+* /jobs/update-startup
+
+  This resource updates data that should only be refreshed
+  in case of system restart.

changelog

@@ -14,3 +14,6 @@
 0.12: added addresses to interface response
       put actual module number in version response
 0.13: added external inventory caching
+0.14: added sample route for startup-only tasks
+      added method for caching selected alarmsdb tables
+      added caching of last known interface status from alarmsdb

inventory_provider/alarmsdb.py

 from inventory_provider import db
 
 
-def get_last_known_infinera_interface_status(connection, equipment, interface):
+def get_last_known_infinera_interface_status(crs, equipment, interface):
     query = "SELECT status FROM infinera_alarms" \
             " WHERE" \
             " CONCAT(ne_name, '-', REPLACE(object_name, 'T', '')) = %s" \
             " ORDER BY ne_init_time DESC, ne_clear_time DESC LIMIT 1"
     search_string = equipment + "-" + interface
-    with db.cursor(connection) as crs:
-        crs.execute(query, (search_string,))
-        result = crs.fetchone()
+    crs.execute(query, (search_string,))
+    result = crs.fetchone()
     if not result:
         return "unknown"
     elif result[0] == "Raised":
@@ -18,13 +17,12 @@ def get_last_known_infinera_interface_status(connection, equipment, interface):
         return "up"
 
 
-def get_last_known_coriant_interface_status(connection, equipment, interface):
+def get_last_known_coriant_interface_status(crs, equipment, interface):
     query = "SELECT status FROM coriant_alarms" \
             " WHERE ne_id_name = %s AND entity_string LIKE %s" \
             " ORDER BY last_event_time DESC LIMIT 1"
-    with db.cursor(connection) as crs:
-        crs.execute(query, (equipment, interface + "-%"))
-        result = crs.fetchone()
+    crs.execute(query, (equipment, interface + "-%"))
+    result = crs.fetchone()
     if not result:
         return "unknown"
     elif result[0] == "Raised":
@@ -34,14 +32,13 @@ def get_last_known_coriant_interface_status(connection, equipment, interface):
 
 
 def get_last_known_juniper_link_interface_status(
-        connection, equipment, interface):
+        crs, equipment, interface):
     query = "SELECT IF(link_admin_status = 'up'" \
             " AND link_oper_status = 'up', 1, 0) AS up FROM juniper_alarms" \
             " WHERE equipment_name = %s AND link_interface_name = %s" \
             " ORDER BY alarm_id DESC LIMIT 1"
-    with db.cursor(connection) as crs:
-        crs.execute(query, ('lo0.' + equipment, interface))
-        result = crs.fetchone()
+    crs.execute(query, ('lo0.' + equipment, interface))
+    result = crs.fetchone()
     if not result:
         return "unknown"
     elif result[0] == 0:
@@ -50,13 +47,27 @@ def get_last_known_juniper_link_interface_status(
         return "up"
 
 
-def get_last_known_interface_status(connection, equipment, interface):
+def get_last_known_interface_status(crs, equipment, interface):
     result = get_last_known_infinera_interface_status(
-        connection, equipment, interface)
+        crs, equipment, interface)
     if result == "unknown":
         result = get_last_known_coriant_interface_status(
-            connection, equipment, interface)
+            crs, equipment, interface)
     if result == "unknown":
         result = get_last_known_juniper_link_interface_status(
-            connection, equipment, interface)
+            crs, equipment, interface)
     return result
+
+
+def _load_juniper_servers_table(connection):
+    with db.cursor(connection) as crs:
+        crs.execute('select ip_address, project_name from juniper_servers')
+        for row in crs.fetchall():
+            yield {
+                'ip_address': row[0],
+                'project_name': row[1]
+            }
+
+
+def load_cache(connection):
+    yield "juniper_servers", list(_load_juniper_servers_table(connection))

inventory_provider/routes/data.py

@@ -2,11 +2,12 @@ import functools
 import json
 import pkg_resources
 
-from flask import Blueprint, request, Response, current_app
+from flask import Blueprint, jsonify, request, Response, current_app
 from lxml import etree
 import redis
 
-from inventory_provider import juniper
+from inventory_provider import db, juniper
+from inventory_provider.storage import external_inventory
 
 routes = Blueprint("inventory-data-query-routes", __name__)
 
@@ -162,3 +163,18 @@ def bgp_configs(hostname):
     return Response(
         json.dumps(routes),
         mimetype="application/json")
+
+
+@routes.route("/interfaces/status/<hostname>/<path:interface>",
+              methods=['GET', 'POST'])
+@require_accepts_json
+def interface_statuses(hostname, interface):
+    r = db.get_redis()
+    result = r.hget(external_inventory.interface_status_key,
+                    "{}::{}".format(hostname, interface))
+    if not result:
+        return Response(
+            response="no available info for {} {}".format(hostname, interface),
+            status=404,
+            mimetype="text/html")
+    return jsonify({"status": result.decode('utf-8')})

inventory_provider/routes/jobs.py

@@ -38,6 +38,17 @@ def update():
     return Response("OK")
 
 
+@routes.route("/update-startup", methods=['GET', 'POST'])
+def startup_update():
+    task_logger = logging.getLogger(TASK_LOGGER_NAME)
+    task_logger.debug(
+            'launching task: '
+            'inventory_provider.tasks.worker.update_alarmsdb_cache')
+    app.send_task(
+        'inventory_provider.tasks.worker.update_alarmsdb_cache')
+    return Response("OK")
+
+
 @routes.route("update-services", methods=['GET'])
 def update_service():
     config = current_app.config['INVENTORY_PROVIDER_CONFIG']

inventory_provider/storage/external_inventory.py

 import json
 from collections import defaultdict
-from inventory_provider import db
+from flask import current_app
+from inventory_provider import alarmsdb, db
 
 
 services_key = "inv_services"
@@ -8,27 +9,38 @@ interfaces_key = "inv_interfaces"
 equipment_locations_key = "inv_eq_locations"
 service_child_to_parents_key = "inv_service_child_to_parents"
 service_parent_to_children_key = "inv_service_parent_to_children"
+interface_status_key = "service_status"
 
 
 def update_services_to_monitor(services):
     r = db.get_redis()
-    relevant_types = ('path', 'service', 'l2circuit')
+    relevant_types = ("path", "service", "l2circuit")
     r.delete(services_key)
     for service in services:
-        if service['circuit_type'].lower() in relevant_types:
-            r.hset(services_key, service['id'], json.dumps(service))
+        if service["circuit_type"].lower() in relevant_types:
+            r.hset(services_key, service["id"], json.dumps(service))
 
 
 def update_interfaces_to_services(services):
     r = db.get_redis()
-    mapped_interfaces = defaultdict(list)
-    r.delete(interfaces_key)
-    for service in services:
-        key = "{}::{}".format(
-            service['equipment'],
-            service['interface_name']
-        )
-        mapped_interfaces[key].append(service)
+    config = current_app.config["INVENTORY_PROVIDER_CONFIG"]
+    with db.connection(config["alarms-db"]) as cnx:
+        with db.cursor(cnx) as csr:
+            mapped_interfaces = defaultdict(list)
+            r.delete(interfaces_key)
+            for service in services:
+                key = "{}::{}".format(
+                    service["equipment"],
+                    service["interface_name"]
+                )
+                mapped_interfaces[key].append(service)
+                if not r.hexists(interface_status_key, key):
+                    r.hset(interface_status_key, key,
+                           alarmsdb.get_last_known_interface_status(
+                               csr,
+                               service["equipment"],
+                               service["interface_name"]
+                           ))
 
     for key, value in mapped_interfaces.items():
         r.hset(interfaces_key, key, json.dumps(value))
@@ -57,4 +69,4 @@ def update_equipment_locations(equipment_location_data):
     r = db.get_redis()
     r.delete(equipment_locations_key)
     for ld in equipment_location_data:
-        r.hset(equipment_locations_key, ld['equipment_name'], json.dumps(ld))
+        r.hset(equipment_locations_key, ld["equipment_name"], json.dumps(ld))

inventory_provider/tasks/worker.py

@@ -6,8 +6,10 @@ import redis
 from lxml import etree
 
 from inventory_provider.tasks.app import app
+from inventory_provider import alarmsdb
 from inventory_provider import config
 from inventory_provider import constants
+from inventory_provider import db
 from inventory_provider import environment
 from inventory_provider import snmp
 from inventory_provider import juniper
@@ -38,6 +40,25 @@ class InventoryTask(Task):
             "saved %s, key %s" % (hostname, key))
         return "OK"
 
+    @staticmethod
+    def save_value(key, value):
+        assert isinstance(value, str), \
+            "sanity failure: expected string data as value"
+        r = redis.StrictRedis(
+            host=InventoryTask.config["redis"]["hostname"],
+            port=InventoryTask.config["redis"]["port"])
+        r.set(
+            name=key,
+            value=value)
+        InventoryTask.logger.debug("saved %s" % key)
+        return "OK"
+
+    @staticmethod
+    def save_value_json(key, data_obj):
+        InventoryTask.save_value(
+            key,
+            json.dumps(data_obj))
+
     @staticmethod
     def save_key_json(hostname, key, data_obj):
         InventoryTask.save_key(
@@ -102,3 +123,15 @@ def netconf_refresh_config(self, hostname):
         juniper.load_config(hostname, InventoryTask.config["ssh"]))
 
     logger.debug('FINISHED: netconf_refresh_config(%r)' % hostname)
+
+
+@app.task(bind=InventoryTask)
+def update_alarmsdb_cache(self):
+    logger = logging.getLogger(constants.TASK_LOGGER_NAME)
+    logger.debug('STARTING: update_alarmsdb_cache')
+
+    with db.connection(InventoryTask.config["alarms-db"]) as cx:
+        for table_name, data in alarmsdb.load_cache(cx):
+            InventoryTask.save_value_json('alarmsdb:%s' % table_name, data)
+
+    logger.debug('FINISHED: update_alarmsdb_cache')

setup.py

@@ -2,7 +2,7 @@ from setuptools import setup, find_packages
 
 setup(
     name='inventory-provider',
-    version="0.13",
+    version="0.14",
     author='GEANT',
     author_email='swd@geant.org',
     description='Dashboard inventory provider',

test/storage/test_external_inventory.py

@@ -34,6 +34,12 @@ def test_update_services_to_monitor(mocker):
 def test_update_interfaces_to_services(mocker):
     mocked_redis = mocker.patch(
         "inventory_provider.storage.external_inventory.db.get_redis")
+    mocker.patch(
+        "inventory_provider.storage.external_inventory.current_app")
+    mocker.patch(
+        "inventory_provider.storage.external_inventory.db.connection")
+    mocker.patch(
+        "inventory_provider.storage.external_inventory.db.cursor")
     mocked_hset = mocked_redis.return_value.hset
     services = [
         {"equipment": "eq_0", "interface_name": "if_0"},

test/test_alarmsdb.py

@@ -2,27 +2,23 @@ import inventory_provider.alarmsdb as alarmsdb
 
 
 def test_infinera_interface_status(mocker):
-    mocked_get_cursor = mocker.patch('inventory_provider.alarmsdb.db.cursor')
-    mocked_execute = mocked_get_cursor. \
-        return_value.__enter__.return_value.execute
-    mocked_fetchone = mocked_get_cursor.return_value.__enter__. \
-        return_value.fetchone
-    mocked_fetchone.return_value = ('Raised',)
+    mock = mocker.Mock()
+    mock.fetchone.return_value = ('Raised',)
     assert alarmsdb.get_last_known_infinera_interface_status(
-        None, 'eq1', 'intfc1'
+        mock, 'eq1', 'intfc1'
     ) == "down"
 
-    mocked_fetchone.return_value = ("Clear",)
+    mock.fetchone.return_value = ("Clear",)
     assert alarmsdb.get_last_known_infinera_interface_status(
-            None, 'eq1', 'intfc1'
+        mock, 'eq1', 'intfc1'
     ) == "up"
 
-    mocked_fetchone.return_value = ()
+    mock.fetchone.return_value = ()
     assert alarmsdb.get_last_known_infinera_interface_status(
-            None, 'eq1', 'intfc1'
+        mock, 'eq1', 'intfc1'
     ) == "unknown"
 
-    mocked_execute.assert_called_with(
+    mock.execute.assert_called_with(
         "SELECT status FROM infinera_alarms WHERE"
         " CONCAT(ne_name, '-', REPLACE(object_name, 'T', '')) = %s"
         " ORDER BY ne_init_time DESC, ne_clear_time DESC LIMIT 1",
@@ -30,27 +26,23 @@ def test_infinera_interface_status(mocker):
 
 
 def test_coriant_interface_status(mocker):
-    mocked_get_cursor = mocker.patch('inventory_provider.alarmsdb.db.cursor')
-    mocked_execute = mocked_get_cursor. \
-        return_value.__enter__.return_value.execute
-    mocked_fetchone = mocked_get_cursor.return_value.__enter__. \
-        return_value.fetchone
-    mocked_fetchone.return_value = ('Raised',)
+    mock = mocker.Mock()
+    mock.fetchone.return_value = ('Raised',)
     assert alarmsdb.get_last_known_coriant_interface_status(
-        None, 'eq1', 'intfc1'
+        mock, 'eq1', 'intfc1'
     ) == "down"
 
-    mocked_fetchone.return_value = ("Clear",)
+    mock.fetchone.return_value = ("Clear",)
     assert alarmsdb.get_last_known_coriant_interface_status(
-            None, 'eq1', 'intfc1'
+        mock, 'eq1', 'intfc1'
     ) == "up"
 
-    mocked_fetchone.return_value = ()
+    mock.fetchone.return_value = ()
     assert alarmsdb.get_last_known_coriant_interface_status(
-            None, 'eq1', 'intfc1'
+        mock, 'eq1', 'intfc1'
     ) == "unknown"
 
-    mocked_execute.assert_called_with(
+    mock.execute.assert_called_with(
         "SELECT status FROM coriant_alarms"
         " WHERE ne_id_name = %s AND entity_string LIKE %s"
         " ORDER BY last_event_time DESC LIMIT 1",
@@ -58,27 +50,23 @@ def test_coriant_interface_status(mocker):
 
 
 def test_juniper_interface_status(mocker):
-    mocked_get_cursor = mocker.patch('inventory_provider.alarmsdb.db.cursor')
-    mocked_execute = mocked_get_cursor. \
-        return_value.__enter__.return_value.execute
-    mocked_fetchone = mocked_get_cursor.return_value.__enter__. \
-        return_value.fetchone
-    mocked_fetchone.return_value = (0,)
+    mock = mocker.Mock()
+    mock.fetchone.return_value = (0,)
     assert alarmsdb.get_last_known_juniper_link_interface_status(
-        None, 'eq1', 'intfc1'
+        mock, 'eq1', 'intfc1'
     ) == "down"
 
-    mocked_fetchone.return_value = (1,)
+    mock.fetchone.return_value = (1,)
     assert alarmsdb.get_last_known_juniper_link_interface_status(
-            None, 'eq1', 'intfc1'
+        mock, 'eq1', 'intfc1'
     ) == "up"
 
-    mocked_fetchone.return_value = ()
+    mock.fetchone.return_value = ()
     assert alarmsdb.get_last_known_juniper_link_interface_status(
-            None, 'eq1', 'intfc1'
+        mock, 'eq1', 'intfc1'
     ) == "unknown"
 
-    mocked_execute.assert_called_with(
+    mock.execute.assert_called_with(
                 "SELECT IF(link_admin_status = 'up'"
                 " AND link_oper_status = 'up', 1, 0)"
                 " AS up FROM juniper_alarms"