diff --git a/inventory_provider/routes/common.py b/inventory_provider/routes/common.py
index 7a84df255501f5d225279d696b71747325bb0348..e1821391fdd386e9e3314777716d44130ea30beb 100644
--- a/inventory_provider/routes/common.py
+++ b/inventory_provider/routes/common.py
@@ -1,6 +1,10 @@
+from collections import OrderedDict
import functools
+import json
import logging
-from collections import OrderedDict
+import queue
+import random
+import threading
import requests
from flask import request, Response, current_app, g
@@ -103,3 +107,92 @@ def after_request(response):
data,
str(response.status_code)))
return response
+
+
+def _redis_client_proc(key_queue, value_queue, config_params):
+ """
+ create a local redis connection with the current db index,
+ lookup the values of the keys that come from key_queue
+ and put them o=n value_queue
+
+ i/o contract:
+ None arriving on key_queue means no more keys are coming
+ put None in value_queue means we are finished
+
+ :param key_queue:
+ :param value_queue:
+ :param config_params: app config
+ :return: yields dicts like {'key': str, 'value': dict}
+ """
+ try:
+ r = tasks_common.get_current_redis(config_params)
+ while True:
+ key = key_queue.get()
+
+ # contract is that None means no more requests
+ if not key:
+ break
+
+ value = r.get(key).decode('utf-8')
+ value_queue.put({
+ 'key': key,
+ 'value': json.loads(value)
+ })
+
+ except json.JSONDecodeError:
+ logger.exception(f'error decoding entry for {key}')
+
+ finally:
+ # contract is to return None when finished
+ value_queue.put(None)
+
+
+def load_json_docs(config_params, key_pattern, num_threads=10):
+ """
+ load all json docs from redis
+
+ the loading is done with multiple connections in parallel, since this
+ method is called from an api handler and when the client is far from
+ the redis master the cumulative latency causes nginx/gunicorn timeouts
+
+ :param config_params: app config
+ :param pattern: key pattern to load
+ :param num_threads: number of client threads to create
+ :return: yields dicts like {'key': str, 'value': dict}
+ """
+ response_queue = queue.Queue()
+
+ threads = []
+ for _ in range(num_threads):
+ q = queue.Queue()
+ t = threading.Thread(
+ target=_redis_client_proc,
+ args=[q, response_queue, config_params])
+ t.start()
+ threads.append({'thread': t, 'queue': q})
+
+ r = tasks_common.get_current_redis(config_params)
+ # scan with bigger batches, to mitigate network latency effects
+ for k in r.scan_iter(key_pattern, count=1000):
+ k = k.decode('utf-8')
+ t = random.choice(threads)
+ t['queue'].put(k)
+
+ # tell all threads there are no more keys coming
+ for t in threads:
+ t['queue'].put(None)
+
+ num_finished = 0
+ # read values from response_queue until we receive
+ # None len(threads) times
+ while num_finished < len(threads):
+ value = response_queue.get()
+ if not value:
+ num_finished += 1
+ logger.debug('one worker thread finished')
+ continue
+ yield value
+
+ # cleanup like we're supposed to, even though it's python
+ for t in threads:
+ t['thread'].join(timeout=0.5) # timeout, for sanity
diff --git a/inventory_provider/routes/data.py b/inventory_provider/routes/data.py
index f5f780cebb9365eec28c8af7b0524931c78d0fba..f25bd80dae5374c1b43b3802e697a07327b6c137 100644
--- a/inventory_provider/routes/data.py
+++ b/inventory_provider/routes/data.py
@@ -27,23 +27,41 @@ def routers():
return jsonify(result)
+@routes.route("/interfaces", methods=['GET', 'POST'])
@routes.route("/interfaces/<hostname>", methods=['GET', 'POST'])
@common.require_accepts_json
-def router_interfaces(hostname):
- r = common.get_current_redis()
- interfaces = []
- for k in r.keys('netconf-interfaces:%s:*' % hostname):
- ifc = r.get(k.decode('utf-8'))
- if ifc:
- interfaces.append(json.loads(ifc.decode('utf-8')))
+def router_interfaces(hostname=None):
- if not interfaces:
- return Response(
- response="no available interface info for '%s'" % hostname,
- status=404,
- mimetype="text/html")
+ cache_key = f'classifier-cache:netconf-interfaces:{hostname}' \
+ if hostname else 'classifier-cache:netconf-interfaces:all'
- return jsonify(interfaces)
+ r = common.get_current_redis()
+ result = r.get(cache_key)
+ if result:
+ result = result.decode('utf-8')
+
+ else:
+ key_pattern = f'netconf-interfaces:{hostname}:*' \
+ if hostname else 'netconf-interfaces:*'
+ config = current_app.config['INVENTORY_PROVIDER_CONFIG']
+
+ result = []
+ for ifc in common.load_json_docs(config, key_pattern):
+ key_fields = ifc['key'].split(':')
+ ifc['value']['router'] = key_fields[1]
+ result.append(ifc['value'])
+
+ if not result:
+ return Response(
+ response="no available interface info for '%s'" % hostname,
+ status=404,
+ mimetype="text/html")
+
+ result = json.dumps(result)
+ # cache this data for the next call
+ r.set(cache_key, result.encode('utf-8'))
+
+ return Response(result, mimetype="application/json")
@routes.route("/pop/<equipment_name>", methods=['GET', 'POST'])
diff --git a/inventory_provider/static/interfaces.html b/inventory_provider/static/interfaces.html
index 80ca1c39c1566406bc9acd27198ea2f4a15c4cf9..e4c59af9ef7a17d4d536de2eb833e69d291a8b0e 100644
--- a/inventory_provider/static/interfaces.html
+++ b/inventory_provider/static/interfaces.html
@@ -9,18 +9,19 @@
<body>
<div ng-controller="interfaces">
- <h2>Interfaces</h2>
- <div>
- <select
- ng-options="r for r in routers"
- ng-change="update_interfaces()"
- ng-model="router"></select>
- <select
- ng-options="i for i in interfaces"
- ng-change="update_status()"
- ng-model="interface">
- </select>
- </div>
+ <div class="column">
+ <p><strong>interfaces</strong></p>
+ <ul>
+ <li ng-repeat="i in interfaces">{{i.router}}:{{i.name}}
+ <ul>
+ <li>{{i.description}}</li>
+ <li ng-repeat="v4 in i.ipv4">v4: {{v4}}</li>
+ <li ng-repeat="v6 in i.ipv6">v6: {{v6}}</li>
+ </ul>
+ </li>
+ </ul>
+ <!--div class="raw">{{interfaces}}</div-->
+ </div>
<div>
STATUS: {{status}}
diff --git a/inventory_provider/static/interfaces.js b/inventory_provider/static/interfaces.js
index 499350b2567bc0295ad6576d283359bdff6c1912..63ab9939fdf58148bd49631002ff40552d966c28 100644
--- a/inventory_provider/static/interfaces.js
+++ b/inventory_provider/static/interfaces.js
@@ -12,12 +12,13 @@ myApp.controller('interfaces', function($scope, $http) {
$http({
method: 'GET',
- url: window.location.origin + "/data/routers"
+ url: window.location.origin + "/data/interfaces"
}).then(
- function(rsp) {$scope.routers = rsp.data;},
+ function(rsp) {$scope.interfaces = rsp.data;},
function(rsp) {$scope.routers = ['error'];}
);
+ /*
$scope.update_interfaces = function() {
$http({
@@ -47,5 +48,6 @@ myApp.controller('interfaces', function($scope, $http) {
function(rsp) {$scope.interfaces = 'query error';}
);
}
+ */
});
\ No newline at end of file
diff --git a/inventory_provider/static/style.css b/inventory_provider/static/style.css
index a72ff44af879d3d178cce21668484bca5bae43fa..b234cbe2d3b0a09e2a4a2efef909588cd0ddedc1 100644
--- a/inventory_provider/static/style.css
+++ b/inventory_provider/static/style.css
@@ -1,6 +1,6 @@
.column {
float: left;
- width: 33.33%;
+ width: 100%%;
}
/* Clear floats after the columns */
diff --git a/test/per_router/test_data_routes.py b/test/per_router/test_data_routes.py
index 063aa2a3e13b49ff9df6fc3d07953edfbaa4806c..85fa9bae5600f70bca58615dd67b117f33145c62 100644
--- a/test/per_router/test_data_routes.py
+++ b/test/per_router/test_data_routes.py
@@ -18,6 +18,7 @@ def test_router_interfaces(router, client):
"properties": {
"name": {"type": "string"},
"description": {"type": "string"},
+ "router": {"type": "string"},
"bundle": {
"type": "array",
"items": {"type": "string"}
@@ -31,7 +32,7 @@ def test_router_interfaces(router, client):
"items": {"type": "string"}
}
},
- "required": ["name", "description", "ipv4", "ipv6"],
+ "required": ["name", "description", "ipv4", "router", "ipv6"],
"additionalProperties": False
}
}
diff --git a/test/test_general_data_routes.py b/test/test_general_data_routes.py
index fa8c385aa5866a4a77b583f43cb0c940831c676e..a618f8fb2c7ff6a0e01530511ce3fba166ef9a23 100644
--- a/test/test_general_data_routes.py
+++ b/test/test_general_data_routes.py
@@ -75,3 +75,42 @@ def test_pop_not_found(client, mocker):
headers=DEFAULT_REQUEST_HEADERS)
assert rv.status_code == 404
+
+
+def test_router_interfaces_all(client):
+
+ interfaces_list_schema = {
+ "$schema": "http://json-schema.org/draft-07/schema#",
+ "type": "array",
+ "items": {
+ "type": "object",
+ "properties": {
+ "name": {"type": "string"},
+ "description": {"type": "string"},
+ "router": {"type": "string"},
+ "bundle": {
+ "type": "array",
+ "items": {"type": "string"}
+ },
+ "ipv4": {
+ "type": "array",
+ "items": {"type": "string"}
+ },
+ "ipv6": {
+ "type": "array",
+ "items": {"type": "string"}
+ }
+ },
+ "required": ["name", "description", "ipv4", "router", "ipv6"],
+ "additionalProperties": False
+ }
+ }
+
+ rv = client.post(
+ '/data/interfaces',
+ headers=DEFAULT_REQUEST_HEADERS)
+
+ assert rv.status_code == 200
+ response = json.loads(rv.data.decode("utf-8"))
+ jsonschema.validate(response, interfaces_list_schema)
+ assert response # at least shouldn't be empty
diff --git a/test/test_general_routes.py b/test/test_general_routes.py
index a6a915e95e0412641280c155f09d0d31f519c90f..116e89f412329b2d053bcafe6531f55e12398e40 100644
--- a/test/test_general_routes.py
+++ b/test/test_general_routes.py
@@ -1,6 +1,8 @@
import json
import jsonschema
+from inventory_provider.routes import common
+
DEFAULT_REQUEST_HEADERS = {
"Content-type": "application/json",
"Accept": ["application/json"]
@@ -50,3 +52,46 @@ def test_version_request(client, mocked_redis):
jsonschema.validate(
json.loads(rv.data.decode("utf-8")),
version_schema)
+
+
+def test_load_json_docs(data_config, mocked_redis):
+
+ INTERFACE_SCHEMA = {
+ "$schema": "http://json-schema.org/draft-07/schema#",
+
+ "definitions": {
+ "interface": {
+ "type": "object",
+ "properties": {
+ "name": {"type": "string"},
+ "description": {"type": "string"},
+ "bundle": {
+ "type": "array",
+ "items": {"type": "string"}
+ },
+ "ipv4": {
+ "type": "array",
+ "items": {"type": "string"}
+ },
+ "ipv6": {
+ "type": "array",
+ "items": {"type": "string"}
+ }
+ },
+ "required": ["name", "description", "ipv4", "ipv6"],
+ "additionalProperties": False
+ }
+ },
+
+ "type": "object",
+ "properties": {
+ "key": {"type": "string"},
+ "value": {"$ref": "#/definitions/interface"}
+ },
+ "required": ["key", "value"],
+ "additionalProperties": False
+ }
+
+ for ifc in common.load_json_docs(
+ data_config, 'netconf-interfaces:*', num_threads=20):
+ jsonschema.validate(ifc, INTERFACE_SCHEMA)