diff --git a/README.md b/README.md
index 1f9360ce53b68f77c88348a50dc26a6c876bc512..142aacf6f32c6ee28f219f19618b3e77bdb31972 100644
--- a/README.md
+++ b/README.md
@@ -252,3 +252,7 @@ Any non-empty responses are JSON formatted messages.
This resource updates the inventory network data for juniper devices.
+* /jobs/update-startup
+
+ This resource updates data that should only be refreshed
+ in case of system restart.
diff --git a/changelog b/changelog
index 52a582c38c54e861fbbf23e7e8a3d3e50d457429..73a599d671ad205bf199e5b7a6cb2257f87fe409 100644
--- a/changelog
+++ b/changelog
@@ -14,3 +14,6 @@
0.12: added addresses to interface response
put actual module number in version response
0.13: added external inventory caching
+0.14: added sample route for startup-only tasks
+ added method for caching selected alarmsdb tables
+ added caching of last known interface status from alarmsdb
diff --git a/inventory_provider/alarmsdb.py b/inventory_provider/alarmsdb.py
index b3256e10bb1b60c3511d6942dff4f061ecd75ed1..7eb2cba54356fffcf55ae51151e9d0e78c03210d 100644
--- a/inventory_provider/alarmsdb.py
+++ b/inventory_provider/alarmsdb.py
@@ -1,15 +1,14 @@
from inventory_provider import db
-def get_last_known_infinera_interface_status(connection, equipment, interface):
+def get_last_known_infinera_interface_status(crs, equipment, interface):
query = "SELECT status FROM infinera_alarms" \
" WHERE" \
" CONCAT(ne_name, '-', REPLACE(object_name, 'T', '')) = %s" \
" ORDER BY ne_init_time DESC, ne_clear_time DESC LIMIT 1"
search_string = equipment + "-" + interface
- with db.cursor(connection) as crs:
- crs.execute(query, (search_string,))
- result = crs.fetchone()
+ crs.execute(query, (search_string,))
+ result = crs.fetchone()
if not result:
return "unknown"
elif result[0] == "Raised":
@@ -18,13 +17,12 @@ def get_last_known_infinera_interface_status(connection, equipment, interface):
return "up"
-def get_last_known_coriant_interface_status(connection, equipment, interface):
+def get_last_known_coriant_interface_status(crs, equipment, interface):
query = "SELECT status FROM coriant_alarms" \
" WHERE ne_id_name = %s AND entity_string LIKE %s" \
" ORDER BY last_event_time DESC LIMIT 1"
- with db.cursor(connection) as crs:
- crs.execute(query, (equipment, interface + "-%"))
- result = crs.fetchone()
+ crs.execute(query, (equipment, interface + "-%"))
+ result = crs.fetchone()
if not result:
return "unknown"
elif result[0] == "Raised":
@@ -34,14 +32,13 @@ def get_last_known_coriant_interface_status(connection, equipment, interface):
def get_last_known_juniper_link_interface_status(
- connection, equipment, interface):
+ crs, equipment, interface):
query = "SELECT IF(link_admin_status = 'up'" \
" AND link_oper_status = 'up', 1, 0) AS up FROM juniper_alarms" \
" WHERE equipment_name = %s AND link_interface_name = %s" \
" ORDER BY alarm_id DESC LIMIT 1"
- with db.cursor(connection) as crs:
- crs.execute(query, ('lo0.' + equipment, interface))
- result = crs.fetchone()
+ crs.execute(query, ('lo0.' + equipment, interface))
+ result = crs.fetchone()
if not result:
return "unknown"
elif result[0] == 0:
@@ -50,13 +47,27 @@ def get_last_known_juniper_link_interface_status(
return "up"
-def get_last_known_interface_status(connection, equipment, interface):
+def get_last_known_interface_status(crs, equipment, interface):
result = get_last_known_infinera_interface_status(
- connection, equipment, interface)
+ crs, equipment, interface)
if result == "unknown":
result = get_last_known_coriant_interface_status(
- connection, equipment, interface)
+ crs, equipment, interface)
if result == "unknown":
result = get_last_known_juniper_link_interface_status(
- connection, equipment, interface)
+ crs, equipment, interface)
return result
+
+
+def _load_juniper_servers_table(connection):
+ with db.cursor(connection) as crs:
+ crs.execute('select ip_address, project_name from juniper_servers')
+ for row in crs.fetchall():
+ yield {
+ 'ip_address': row[0],
+ 'project_name': row[1]
+ }
+
+
+def load_cache(connection):
+ yield "juniper_servers", list(_load_juniper_servers_table(connection))
diff --git a/inventory_provider/routes/data.py b/inventory_provider/routes/data.py
index 3cc9e58fcda1a6dce3ebfd509204347f369895df..30810e45b074fbf599e5d26ace622675e7e7ca53 100644
--- a/inventory_provider/routes/data.py
+++ b/inventory_provider/routes/data.py
@@ -2,11 +2,12 @@ import functools
import json
import pkg_resources
-from flask import Blueprint, request, Response, current_app
+from flask import Blueprint, jsonify, request, Response, current_app
from lxml import etree
import redis
-from inventory_provider import juniper
+from inventory_provider import db, juniper
+from inventory_provider.storage import external_inventory
routes = Blueprint("inventory-data-query-routes", __name__)
@@ -162,3 +163,18 @@ def bgp_configs(hostname):
return Response(
json.dumps(routes),
mimetype="application/json")
+
+
+@routes.route("/interfaces/status/<hostname>/<path:interface>",
+ methods=['GET', 'POST'])
+@require_accepts_json
+def interface_statuses(hostname, interface):
+ r = db.get_redis()
+ result = r.hget(external_inventory.interface_status_key,
+ "{}::{}".format(hostname, interface))
+ if not result:
+ return Response(
+ response="no available info for {} {}".format(hostname, interface),
+ status=404,
+ mimetype="text/html")
+ return jsonify({"status": result.decode('utf-8')})
diff --git a/inventory_provider/routes/jobs.py b/inventory_provider/routes/jobs.py
index b1a78c86065bc0a9446266c88aecd2a4e747f1f2..315c3e8d7c960228304a6a5020497cde0bbe1f42 100644
--- a/inventory_provider/routes/jobs.py
+++ b/inventory_provider/routes/jobs.py
@@ -38,6 +38,17 @@ def update():
return Response("OK")
+@routes.route("/update-startup", methods=['GET', 'POST'])
+def startup_update():
+ task_logger = logging.getLogger(TASK_LOGGER_NAME)
+ task_logger.debug(
+ 'launching task: '
+ 'inventory_provider.tasks.worker.update_alarmsdb_cache')
+ app.send_task(
+ 'inventory_provider.tasks.worker.update_alarmsdb_cache')
+ return Response("OK")
+
+
@routes.route("update-services", methods=['GET'])
def update_service():
config = current_app.config['INVENTORY_PROVIDER_CONFIG']
diff --git a/inventory_provider/storage/external_inventory.py b/inventory_provider/storage/external_inventory.py
index cc7bccf69f48c717202e22656f88f9f55f5c33aa..e721a348d52a83ca82121dd299f674f630d38392 100644
--- a/inventory_provider/storage/external_inventory.py
+++ b/inventory_provider/storage/external_inventory.py
@@ -1,6 +1,7 @@
import json
from collections import defaultdict
-from inventory_provider import db
+from flask import current_app
+from inventory_provider import alarmsdb, db
services_key = "inv_services"
@@ -8,27 +9,38 @@ interfaces_key = "inv_interfaces"
equipment_locations_key = "inv_eq_locations"
service_child_to_parents_key = "inv_service_child_to_parents"
service_parent_to_children_key = "inv_service_parent_to_children"
+interface_status_key = "service_status"
def update_services_to_monitor(services):
r = db.get_redis()
- relevant_types = ('path', 'service', 'l2circuit')
+ relevant_types = ("path", "service", "l2circuit")
r.delete(services_key)
for service in services:
- if service['circuit_type'].lower() in relevant_types:
- r.hset(services_key, service['id'], json.dumps(service))
+ if service["circuit_type"].lower() in relevant_types:
+ r.hset(services_key, service["id"], json.dumps(service))
def update_interfaces_to_services(services):
r = db.get_redis()
- mapped_interfaces = defaultdict(list)
- r.delete(interfaces_key)
- for service in services:
- key = "{}::{}".format(
- service['equipment'],
- service['interface_name']
- )
- mapped_interfaces[key].append(service)
+ config = current_app.config["INVENTORY_PROVIDER_CONFIG"]
+ with db.connection(config["alarms-db"]) as cnx:
+ with db.cursor(cnx) as csr:
+ mapped_interfaces = defaultdict(list)
+ r.delete(interfaces_key)
+ for service in services:
+ key = "{}::{}".format(
+ service["equipment"],
+ service["interface_name"]
+ )
+ mapped_interfaces[key].append(service)
+ if not r.hexists(interface_status_key, key):
+ r.hset(interface_status_key, key,
+ alarmsdb.get_last_known_interface_status(
+ csr,
+ service["equipment"],
+ service["interface_name"]
+ ))
for key, value in mapped_interfaces.items():
r.hset(interfaces_key, key, json.dumps(value))
@@ -57,4 +69,4 @@ def update_equipment_locations(equipment_location_data):
r = db.get_redis()
r.delete(equipment_locations_key)
for ld in equipment_location_data:
- r.hset(equipment_locations_key, ld['equipment_name'], json.dumps(ld))
+ r.hset(equipment_locations_key, ld["equipment_name"], json.dumps(ld))
diff --git a/inventory_provider/tasks/worker.py b/inventory_provider/tasks/worker.py
index 78f8ef788dd3b1f9a50b5b8c9e41126d24626fec..8f2efb84f0badfac81a87851d541f9297c6a1bd0 100644
--- a/inventory_provider/tasks/worker.py
+++ b/inventory_provider/tasks/worker.py
@@ -6,8 +6,10 @@ import redis
from lxml import etree
from inventory_provider.tasks.app import app
+from inventory_provider import alarmsdb
from inventory_provider import config
from inventory_provider import constants
+from inventory_provider import db
from inventory_provider import environment
from inventory_provider import snmp
from inventory_provider import juniper
@@ -38,6 +40,25 @@ class InventoryTask(Task):
"saved %s, key %s" % (hostname, key))
return "OK"
+ @staticmethod
+ def save_value(key, value):
+ assert isinstance(value, str), \
+ "sanity failure: expected string data as value"
+ r = redis.StrictRedis(
+ host=InventoryTask.config["redis"]["hostname"],
+ port=InventoryTask.config["redis"]["port"])
+ r.set(
+ name=key,
+ value=value)
+ InventoryTask.logger.debug("saved %s" % key)
+ return "OK"
+
+ @staticmethod
+ def save_value_json(key, data_obj):
+ InventoryTask.save_value(
+ key,
+ json.dumps(data_obj))
+
@staticmethod
def save_key_json(hostname, key, data_obj):
InventoryTask.save_key(
@@ -102,3 +123,15 @@ def netconf_refresh_config(self, hostname):
juniper.load_config(hostname, InventoryTask.config["ssh"]))
logger.debug('FINISHED: netconf_refresh_config(%r)' % hostname)
+
+
+@app.task(bind=InventoryTask)
+def update_alarmsdb_cache(self):
+ logger = logging.getLogger(constants.TASK_LOGGER_NAME)
+ logger.debug('STARTING: update_alarmsdb_cache')
+
+ with db.connection(InventoryTask.config["alarms-db"]) as cx:
+ for table_name, data in alarmsdb.load_cache(cx):
+ InventoryTask.save_value_json('alarmsdb:%s' % table_name, data)
+
+ logger.debug('FINISHED: update_alarmsdb_cache')
diff --git a/setup.py b/setup.py
index b041145b677577119497968f883a6ed70f59cb09..d48ac903b5b8d269c01cbe3d2f8b600d35383380 100644
--- a/setup.py
+++ b/setup.py
@@ -2,7 +2,7 @@ from setuptools import setup, find_packages
setup(
name='inventory-provider',
- version="0.13",
+ version="0.14",
author='GEANT',
author_email='swd@geant.org',
description='Dashboard inventory provider',
diff --git a/test/storage/test_external_inventory.py b/test/storage/test_external_inventory.py
index 30c13e0bc7287fa2b118b875b05b707482028857..b0caf9684824c9a701e67a05f25ceca5468723ea 100644
--- a/test/storage/test_external_inventory.py
+++ b/test/storage/test_external_inventory.py
@@ -34,6 +34,12 @@ def test_update_services_to_monitor(mocker):
def test_update_interfaces_to_services(mocker):
mocked_redis = mocker.patch(
"inventory_provider.storage.external_inventory.db.get_redis")
+ mocker.patch(
+ "inventory_provider.storage.external_inventory.current_app")
+ mocker.patch(
+ "inventory_provider.storage.external_inventory.db.connection")
+ mocker.patch(
+ "inventory_provider.storage.external_inventory.db.cursor")
mocked_hset = mocked_redis.return_value.hset
services = [
{"equipment": "eq_0", "interface_name": "if_0"},
diff --git a/test/test_alarmsdb.py b/test/test_alarmsdb.py
index 7eb707c1210bca0a21cd36ebdc3b658290cca426..7236721fc5f43d4931214a98a8d610049429c8a1 100644
--- a/test/test_alarmsdb.py
+++ b/test/test_alarmsdb.py
@@ -2,27 +2,23 @@ import inventory_provider.alarmsdb as alarmsdb
def test_infinera_interface_status(mocker):
- mocked_get_cursor = mocker.patch('inventory_provider.alarmsdb.db.cursor')
- mocked_execute = mocked_get_cursor. \
- return_value.__enter__.return_value.execute
- mocked_fetchone = mocked_get_cursor.return_value.__enter__. \
- return_value.fetchone
- mocked_fetchone.return_value = ('Raised',)
+ mock = mocker.Mock()
+ mock.fetchone.return_value = ('Raised',)
assert alarmsdb.get_last_known_infinera_interface_status(
- None, 'eq1', 'intfc1'
+ mock, 'eq1', 'intfc1'
) == "down"
- mocked_fetchone.return_value = ("Clear",)
+ mock.fetchone.return_value = ("Clear",)
assert alarmsdb.get_last_known_infinera_interface_status(
- None, 'eq1', 'intfc1'
+ mock, 'eq1', 'intfc1'
) == "up"
- mocked_fetchone.return_value = ()
+ mock.fetchone.return_value = ()
assert alarmsdb.get_last_known_infinera_interface_status(
- None, 'eq1', 'intfc1'
+ mock, 'eq1', 'intfc1'
) == "unknown"
- mocked_execute.assert_called_with(
+ mock.execute.assert_called_with(
"SELECT status FROM infinera_alarms WHERE"
" CONCAT(ne_name, '-', REPLACE(object_name, 'T', '')) = %s"
" ORDER BY ne_init_time DESC, ne_clear_time DESC LIMIT 1",
@@ -30,27 +26,23 @@ def test_infinera_interface_status(mocker):
def test_coriant_interface_status(mocker):
- mocked_get_cursor = mocker.patch('inventory_provider.alarmsdb.db.cursor')
- mocked_execute = mocked_get_cursor. \
- return_value.__enter__.return_value.execute
- mocked_fetchone = mocked_get_cursor.return_value.__enter__. \
- return_value.fetchone
- mocked_fetchone.return_value = ('Raised',)
+ mock = mocker.Mock()
+ mock.fetchone.return_value = ('Raised',)
assert alarmsdb.get_last_known_coriant_interface_status(
- None, 'eq1', 'intfc1'
+ mock, 'eq1', 'intfc1'
) == "down"
- mocked_fetchone.return_value = ("Clear",)
+ mock.fetchone.return_value = ("Clear",)
assert alarmsdb.get_last_known_coriant_interface_status(
- None, 'eq1', 'intfc1'
+ mock, 'eq1', 'intfc1'
) == "up"
- mocked_fetchone.return_value = ()
+ mock.fetchone.return_value = ()
assert alarmsdb.get_last_known_coriant_interface_status(
- None, 'eq1', 'intfc1'
+ mock, 'eq1', 'intfc1'
) == "unknown"
- mocked_execute.assert_called_with(
+ mock.execute.assert_called_with(
"SELECT status FROM coriant_alarms"
" WHERE ne_id_name = %s AND entity_string LIKE %s"
" ORDER BY last_event_time DESC LIMIT 1",
@@ -58,27 +50,23 @@ def test_coriant_interface_status(mocker):
def test_juniper_interface_status(mocker):
- mocked_get_cursor = mocker.patch('inventory_provider.alarmsdb.db.cursor')
- mocked_execute = mocked_get_cursor. \
- return_value.__enter__.return_value.execute
- mocked_fetchone = mocked_get_cursor.return_value.__enter__. \
- return_value.fetchone
- mocked_fetchone.return_value = (0,)
+ mock = mocker.Mock()
+ mock.fetchone.return_value = (0,)
assert alarmsdb.get_last_known_juniper_link_interface_status(
- None, 'eq1', 'intfc1'
+ mock, 'eq1', 'intfc1'
) == "down"
- mocked_fetchone.return_value = (1,)
+ mock.fetchone.return_value = (1,)
assert alarmsdb.get_last_known_juniper_link_interface_status(
- None, 'eq1', 'intfc1'
+ mock, 'eq1', 'intfc1'
) == "up"
- mocked_fetchone.return_value = ()
+ mock.fetchone.return_value = ()
assert alarmsdb.get_last_known_juniper_link_interface_status(
- None, 'eq1', 'intfc1'
+ mock, 'eq1', 'intfc1'
) == "unknown"
- mocked_execute.assert_called_with(
+ mock.execute.assert_called_with(
"SELECT IF(link_admin_status = 'up'"
" AND link_oper_status = 'up', 1, 0)"
" AS up FROM juniper_alarms"