diff --git a/Changelog.md b/Changelog.md
index 31e6c80a0033635261027312ca20fc96147da482..853920da13387fff19262ec1e1296afe7d4688f7 100644
--- a/Changelog.md
+++ b/Changelog.md
@@ -2,6 +2,13 @@
All notable changes to this project will be documented in this file.
+## [0.86] - 2022-03-22
+- POL1-552: neteng pop api
+- POL1-571: poller/interfaces cache bug fix
+
+## [0.85] - 2022-03-15
+- POL1-569: return all services by default for BRIAN, not only monitored
+
## [0.84] - 2022-03-07
- DBOARD3-536: added /ping endpoint
diff --git a/inventory_provider/db/ims_data.py b/inventory_provider/db/ims_data.py
index c644d13c5d63fe88709b7f8630affdd8baffb496..ec9717ffec56b98c807fd7266a105deb0d87008b 100644
--- a/inventory_provider/db/ims_data.py
+++ b/inventory_provider/db/ims_data.py
@@ -26,28 +26,35 @@ IMS_OPSDB_STATUS_MAP = {
STATUSES_TO_IGNORE = \
[InventoryStatus.OUT_OF_SERVICE.value]
+_POP_LOCATION_SCHEMA_STRUCT = {
+ 'type': 'object',
+ 'properties': {
+ 'name': {'type': 'string'},
+ 'city': {'type': 'string'},
+ 'country': {'type': 'string'},
+ 'abbreviation': {'type': 'string'},
+ 'longitude': {'type': 'number'},
+ 'latitude': {'type': 'number'}
+ },
+ 'required': [
+ 'name',
+ 'city',
+ 'country',
+ 'abbreviation',
+ 'longitude',
+ 'latitude'],
+ 'additionalProperties': False
+}
+
+POP_LOCATION_SCHEMA = {
+ '$schema': 'http://json-schema.org/draft-07/schema#',
+ **_POP_LOCATION_SCHEMA_STRUCT
+}
+
NODE_LOCATION_SCHEMA = {
'$schema': 'http://json-schema.org/draft-07/schema#',
'definitions': {
- 'pop-location': {
- 'type': 'object',
- 'properties': {
- 'name': {'type': 'string'},
- 'city': {'type': 'string'},
- 'country': {'type': 'string'},
- 'abbreviation': {'type': 'string'},
- 'longitude': {'type': 'number'},
- 'latitude': {'type': 'number'}
- },
- 'required': [
- 'name',
- 'city',
- 'country',
- 'abbreviation',
- 'longitude',
- 'latitude'],
- 'additionalProperties': False
- }
+ 'pop-location': _POP_LOCATION_SCHEMA_STRUCT
},
'type': 'object',
diff --git a/inventory_provider/routes/common.py b/inventory_provider/routes/common.py
index 6dda30525fb971c0c46987d62e17846710ef985e..c74e5dfea1e2c0ff926db8764004f2b132727e91 100644
--- a/inventory_provider/routes/common.py
+++ b/inventory_provider/routes/common.py
@@ -137,7 +137,8 @@ def after_request(response):
return response
-def _redis_client_proc(key_queue, value_queue, config_params, doc_type):
+def _redis_client_proc(
+ key_queue, value_queue, config_params, doc_type, use_next_redis=False):
"""
create a local redis connection with the current db index,
lookup the values of the keys that come from key_queue
@@ -165,7 +166,10 @@ def _redis_client_proc(key_queue, value_queue, config_params, doc_type):
return etree.XML(value)
try:
- r = tasks_common.get_current_redis(config_params)
+ if use_next_redis:
+ r = tasks_common.get_next_redis(config_params)
+ else:
+ r = tasks_common.get_current_redis(config_params)
while True:
key = key_queue.get()
@@ -215,7 +219,7 @@ def _load_redis_docs(
q = queue.Queue()
t = threading.Thread(
target=_redis_client_proc,
- args=[q, response_queue, config_params, doc_type])
+ args=[q, response_queue, config_params, doc_type, use_next_redis])
t.start()
threads.append({'thread': t, 'queue': q})
diff --git a/inventory_provider/routes/jobs.py b/inventory_provider/routes/jobs.py
index 44e7e8ec9fb7eee700b6de2a4b1ed086bc5a7ee9..1f76e4309ef0f436b6771434b064a82764687ee5 100644
--- a/inventory_provider/routes/jobs.py
+++ b/inventory_provider/routes/jobs.py
@@ -87,22 +87,6 @@ INDIVIDUAL_TASK_STATUS_RESPONSE_SCHEMA = {
}
-# INDIVIDUAL_TASK_STATUS_RESPONSE_SCHEMA = {
-# "$schema": "http://json-schema.org/draft-07/schema#",
-# "type": "object",
-# "properties": {
-# "id": {"type": "string"},
-# "status": {"type": "string"},
-# "exception": {"type": "boolean"},
-# "ready": {"type": "boolean"},
-# "success": {"type": "boolean"},
-# "result": {"type": "object"}
-# },
-# "required": ["id", "status", "exception", "ready", "success"],
-# "additionalProperties": False
-# }
-
-
@routes.after_request
def after_request(resp):
return common.after_request(resp)
diff --git a/inventory_provider/routes/neteng.py b/inventory_provider/routes/neteng.py
index 29f8abcd9c963c06ac40fed5063d32f401435a28..dce7b2dc432e7caa6df58345441b14517603aa36 100644
--- a/inventory_provider/routes/neteng.py
+++ b/inventory_provider/routes/neteng.py
@@ -12,9 +12,20 @@ These endpoints are intended for use by neteng tools.
.. autofunction:: inventory_provider.routes.neteng.get_location
+/neteng/pops
+---------------------------------
+
+.. autofunction:: inventory_provider.routes.neteng.get_pop_names
+
+/neteng/pop/name
+---------------------------------
+
+.. autofunction:: inventory_provider.routes.neteng.get_pop_location
+
"""
import json
import logging
+import re
import threading
from flask import Blueprint, Response, jsonify
@@ -25,6 +36,12 @@ routes = Blueprint('neteng-query-routes', __name__)
logger = logging.getLogger(__name__)
_subnet_lookup_semaphore = threading.Semaphore()
+STRING_LIST_SCHEMA = {
+ '$schema': 'http://json-schema.org/draft-07/schema#',
+ 'type': 'array',
+ 'items': {'type': 'string'}
+}
+
@routes.after_request
def after_request(resp):
@@ -35,9 +52,9 @@ def after_request(resp):
@common.require_accepts_json
def get_location(equipment):
"""
- Handler for `/neteng/location/equipment-name`
+ Handler for `/neteng/location/<equipment-name>`
- This method will pop location information for the IMS node
+ This method will return pop location information for the IMS node
with name = `equipment-name`.
404 is returned if the IMS node name is not known.
@@ -54,15 +71,78 @@ def get_location(equipment):
value = r.get(f'ims:location:{equipment}')
if not value:
return Response(
- response='no location information available for "{equipment}"',
+ response=f'no location information available for "{equipment}"',
status=404,
mimetype='text/html')
value = json.loads(value.decode('utf-8'))
if not value:
return Response(
- response='unexpected empty cached data for "{equipment}"',
+ response=f'unexpected empty cached data for "{equipment}"',
status=500,
mimetype='text/html')
return jsonify(value[0])
+
+
+@routes.route('/pops', methods=['GET', 'POST'])
+@common.require_accepts_json
+def get_pop_names():
+ """
+ Handler for `/neteng/pops`
+
+ This method will return a list of defined pop
+ abbreviations. Elements from this list can be used
+ with `/neteng/pop`.
+
+ .. asjson::
+ inventory_provider.routes.neteng.STRING_LIST_SCHEMA
+
+ :return: as above
+ """
+
+ def _pops():
+ r = common.get_current_redis()
+ for k in r.scan_iter('ims:pop:*', count=1000):
+ k = k.decode('utf-8')
+ m = re.match('^ims:pop:(.+)$', k)
+ yield m.group(1)
+
+ return jsonify(sorted(list(_pops())))
+
+
+@routes.route('/pop/<abbreviation>', methods=['GET', 'POST'])
+@common.require_accepts_json
+def get_pop_location(abbreviation):
+ """
+ Handler for `/neteng/pop/<name>`
+
+ This method will return location information for the POP
+ with abbreviation = `abbreviation` in IMS.
+
+ 404 is returned if the POP name is not known.
+ Otherwise the return value will be formatted as:
+
+ .. asjson::
+ inventory_provider.db.ims_data.POP_LOCATION_SCHEMA
+
+ :return: as above
+ """
+
+ r = common.get_current_redis()
+
+ value = r.get(f'ims:pop:{abbreviation}')
+ if not value:
+ return Response(
+ response=f'no location information available for "{abbreviation}"',
+ status=404,
+ mimetype='text/html')
+
+ value = json.loads(value.decode('utf-8'))
+ if not value:
+ return Response(
+ response=f'unexpected empty cached data for "{abbreviation}"',
+ status=500,
+ mimetype='text/html')
+
+ return jsonify(value)
diff --git a/inventory_provider/routes/poller.py b/inventory_provider/routes/poller.py
index 97145a72eb74d86cf3b0e2c2536f006480a26741..525564e86ab84ae0cb9e54544bf380a9ad7aa48a 100644
--- a/inventory_provider/routes/poller.py
+++ b/inventory_provider/routes/poller.py
@@ -72,6 +72,7 @@ from flask import Blueprint, Response, current_app, request, jsonify
from inventory_provider import juniper
from inventory_provider.routes import common
from inventory_provider.tasks.common import ims_sorted_service_type_key
+from inventory_provider.tasks import common as tasks_common
from inventory_provider.routes.classifier import get_ims_equipment_name, \
get_ims_interface
from inventory_provider.routes.common import _ignore_cache_or_retrieve
@@ -524,20 +525,6 @@ def _get_dashboard_data(ifc):
}
-def _add_dashboards(interfaces):
- """
- generator that dashboards to each interfaces.
-
- :param interfaces: result of _load_interfaces
- :return: generator with `dashboards` populated in each element
- """
-
- for ifc in interfaces:
- dashboards = _get_dashboards(ifc)
- ifc['dashboards'] = sorted([d.name for d in dashboards])
- yield _get_dashboard_data(ifc)
-
-
def _load_interface_bundles(config, hostname=None, use_next_redis=False):
result = dict()
@@ -568,8 +555,8 @@ def _load_interface_bundles(config, hostname=None, use_next_redis=False):
def _load_services(config, hostname=None, use_next_redis=False):
- # if hostname:
- # hostname = get_ims_equipment_name(hostname)
+ if hostname:
+ hostname = get_ims_equipment_name(hostname)
result = dict()
key_pattern = f'ims:interface_services:{hostname}:*' \
@@ -654,6 +641,9 @@ def _load_interfaces(
:return:
"""
def _load_docs(key_pattern):
+ # print('')
+ # logger.debug(f'docs Key: {key_pattern}')
+ # print('')
for doc in _load_netconf_docs(config, key_pattern, use_next_redis):
@@ -673,6 +663,7 @@ def _load_interfaces(
base_key_pattern = f'netconf:{hostname}*' if hostname else 'netconf:*'
yield from _load_docs(base_key_pattern)
if not no_lab:
+ logger.debug('lab')
yield from _load_docs(f'lab:{base_key_pattern}')
@@ -695,78 +686,42 @@ def _add_bundle_parents(interfaces, hostname=None):
yield ifc
-def _add_circuits(interfaces, hostname=None):
- """
- generator that adds service info to each interface.
-
- :param interfaces: result of _load_interfaces
- :param hostname: hostname or None for all
- :return: generator with 'circuits' populated in each element, if present
- """
-
- if hostname:
- hostname = get_ims_equipment_name(hostname)
- services = _load_services(
- current_app.config['INVENTORY_PROVIDER_CONFIG'], hostname=hostname)
- for ifc in interfaces:
- router_services = services.get(
- get_ims_equipment_name(ifc['router']), None)
- if router_services:
- ifc['circuits'] = router_services.get(
- get_ims_interface(ifc['name']), []
- )
-
- yield ifc
-
-
-def _add_snmp_indexes(interfaces, hostname=None):
- """
- generator that adds snmp info to each interface, if available
-
- :param interfaces: result of _load_interfaces
- :param hostname: hostname or None for all
- :return: generator with 'snmp-index' optionally added to each element
- """
- snmp_indexes = common.load_snmp_indexes(
- current_app.config['INVENTORY_PROVIDER_CONFIG'], hostname)
- for ifc in interfaces:
- router_snmp = snmp_indexes.get(ifc['router'], None)
- if router_snmp and ifc['name'] in router_snmp:
- ifc['snmp-index'] = router_snmp[ifc['name']]['index']
- # TODO: uncomment this when it won't break poller-admin-service
- # not urgent ... it looks empirically like all logical-system
- # interfaces are repeated for both communities
- # ifc['snmp-community'] = router_snmp[ifc['name']]['community']
- yield ifc
-
-
-def _load_interfaces_to_poll(hostname=None):
- """
- prepares the result of a call to /interfaces
-
- :param hostname: hostname or None for all
- :return: generator yielding interface elements
- """
-
- no_lab = common.get_bool_request_arg('no-lab', False)
- basic_interfaces = _load_interfaces(
- current_app.config['INVENTORY_PROVIDER_CONFIG'],
- hostname,
- no_lab=no_lab)
- # basic_interfaces = list(basic_interfaces)
- with_bundles = _add_bundle_parents(basic_interfaces, hostname)
- with_circuits = _add_circuits(with_bundles, hostname)
- # with_circuits = list(with_circuits)
- with_snmp = _add_snmp_indexes(with_circuits, hostname)
- # with_snmp = list(with_snmp)
-
- # only return interfaces that can be polled
- def _has_snmp_index(ifc):
- return 'snmp-index' in ifc
-
- to_poll = filter(_has_snmp_index, with_snmp)
-
- return _add_dashboards(to_poll)
+def load_interfaces_to_poll(
+ config, hostname=None, no_lab=False, use_next_redis=False):
+ basic_interfaces = \
+ list(_load_interfaces(config, hostname, no_lab, use_next_redis))
+ bundles = _load_interface_bundles(config, hostname, use_next_redis)
+ services = _load_services(config, hostname, use_next_redis)
+ snmp_indexes = common.load_snmp_indexes(config, hostname, use_next_redis)
+
+ def _get_populated_interfaces(all_interfaces):
+ if use_next_redis:
+ r = tasks_common.get_next_redis(config)
+ else:
+ r = tasks_common.get_current_redis(config)
+ for ifc in all_interfaces:
+ router_snmp = snmp_indexes.get(ifc['router'], None)
+ if router_snmp and ifc['name'] in router_snmp:
+ ifc['snmp-index'] = router_snmp[ifc['name']]['index']
+
+ router_bundle = bundles.get(ifc['router'], None)
+ if router_bundle:
+ base_ifc = ifc['name'].split('.')[0]
+ ifc['bundle-parents'] = router_bundle.get(base_ifc, [])
+
+ router_services = services.get(
+ get_ims_equipment_name(ifc['router'], r), None)
+ if router_services:
+ ifc['circuits'] = router_services.get(
+ get_ims_interface(ifc['name']), []
+ )
+
+ dashboards = _get_dashboards(ifc)
+ ifc['dashboards'] = sorted([d.name for d in dashboards])
+ yield _get_dashboard_data(ifc)
+ else:
+ continue
+ return _get_populated_interfaces(basic_interfaces)
@routes.route("/interfaces", methods=['GET', 'POST'])
@@ -808,7 +763,8 @@ def interfaces(hostname=None):
result = _ignore_cache_or_retrieve(request, cache_key, r)
if not result:
- result = list(_load_interfaces_to_poll(hostname))
+ result = list(load_interfaces_to_poll(
+ current_app.config['INVENTORY_PROVIDER_CONFIG'], hostname, no_lab))
if not result:
return Response(
response='no interfaces found',
@@ -1394,9 +1350,9 @@ def gws_direct_config():
:return:
"""
- format = request.args.get('format', default='json', type=str)
- format = format.lower()
- if format not in ('html', 'json'):
+ wanted = request.args.get('format', default='json', type=str)
+ wanted = wanted.lower()
+ if wanted not in ('html', 'json'):
return Response(
response='format must be one of: html, json',
status=400,
@@ -1420,7 +1376,7 @@ def gws_direct_config():
'info': ifc.get('info', '')
}
- if format == 'json':
+ if wanted == 'json':
if not request.accept_mimetypes.accept_json:
return Response(
response="response will be json",
diff --git a/inventory_provider/snmp.py b/inventory_provider/snmp.py
index 6e974f2822ed5ad4bef26c5d8a698907a1e6a12c..d53eb6265dd25dabaafb4568a8fb6cb566534b36 100644
--- a/inventory_provider/snmp.py
+++ b/inventory_provider/snmp.py
@@ -72,13 +72,13 @@ def walk(agent_hostname, community, base_oid): # pragma: no cover
:return:
"""
- mibBuilder = builder.MibBuilder()
+ mib_builder = builder.MibBuilder()
# mibViewController = view.MibViewController(mibBuilder)
compiler.addMibCompiler(
- mibBuilder,
+ mib_builder,
sources=['http://mibs.snmplabs.com/asn1/@mib@'])
# Pre-load MIB modules we expect to work with
- mibBuilder.loadModules(
+ mib_builder.loadModules(
'SNMPv2-MIB',
'SNMP-COMMUNITY-MIB',
'RFC1213-MIB')
@@ -86,10 +86,10 @@ def walk(agent_hostname, community, base_oid): # pragma: no cover
logger.debug("walking %s: %s" % (agent_hostname, base_oid))
try:
- for (engineErrorIndication,
- pduErrorIndication,
- errorIndex,
- varBinds) in nextCmd(
+ for (engine_error_indication,
+ pdu_error_indication,
+ error_index,
+ var_binds) in nextCmd(
SnmpEngine(),
CommunityData(community),
UdpTransportTarget((agent_hostname, 161)),
@@ -101,17 +101,17 @@ def walk(agent_hostname, community, base_oid): # pragma: no cover
# cf. http://snmplabs.com/
# pysnmp/examples/hlapi/asyncore/sync/contents.html
- if engineErrorIndication:
+ if engine_error_indication:
raise SNMPWalkError(
f'snmp response engine error indication: '
- f'{str(engineErrorIndication)} - {agent_hostname}')
- if pduErrorIndication:
+ f'{str(engine_error_indication)} - {agent_hostname}')
+ if pdu_error_indication:
raise SNMPWalkError(
'snmp response pdu error %r at %r' % (
- pduErrorIndication,
- errorIndex
- and varBinds[int(errorIndex) - 1][0] or '?'))
- if errorIndex != 0:
+ pdu_error_indication,
+ error_index
+ and var_binds[int(error_index) - 1][0] or '?'))
+ if error_index != 0:
raise SNMPWalkError(
'sanity failure: errorIndex != 0, '
'but no error indication')
@@ -120,7 +120,7 @@ def walk(agent_hostname, community, base_oid): # pragma: no cover
# rfc1902.ObjectType(rfc1902.ObjectIdentity(x[0]),x[1])
# .resolveWithMib(mibViewController)
# for x in varBinds]
- for oid, val in varBinds:
+ for oid, val in var_binds:
result = {
"oid": _canonify_oid(oid),
"value": _cast_snmp_value(val)
diff --git a/inventory_provider/static/interfaces.html b/inventory_provider/static/interfaces.html
deleted file mode 100644
index e4c59af9ef7a17d4d536de2eb833e69d291a8b0e..0000000000000000000000000000000000000000
--- a/inventory_provider/static/interfaces.html
+++ /dev/null
@@ -1,31 +0,0 @@
-<!doctype html>
-<html ng-app="inventoryApp" lang="en">
- <head>
- <title>interfaces</title>
- <script src="https://ajax.googleapis.com/ajax/libs/angularjs/1.7.5/angular.min.js"></script>
- <script src="interfaces.js"></script>
- <link rel="stylesheet" href="style.css">
- </head>
- <body>
-
- <div ng-controller="interfaces">
- <div class="column">
- <p><strong>interfaces</strong></p>
- <ul>
- <li ng-repeat="i in interfaces">{{i.router}}:{{i.name}}
- <ul>
- <li>{{i.description}}</li>
- <li ng-repeat="v4 in i.ipv4">v4: {{v4}}</li>
- <li ng-repeat="v6 in i.ipv6">v6: {{v6}}</li>
- </ul>
- </li>
- </ul>
- <!--div class="raw">{{interfaces}}</div-->
- </div>
-
- <div>
- STATUS: {{status}}
- </div>
- </div>
- </body>
-</html>
\ No newline at end of file
diff --git a/inventory_provider/static/interfaces.js b/inventory_provider/static/interfaces.js
deleted file mode 100644
index 63ab9939fdf58148bd49631002ff40552d966c28..0000000000000000000000000000000000000000
--- a/inventory_provider/static/interfaces.js
+++ /dev/null
@@ -1,53 +0,0 @@
-var myApp = angular.module('inventoryApp', []);
-
-myApp.controller('interfaces', function($scope, $http) {
-
- $scope.routers = [];
- $scope.router = '';
-
- $scope.interfaces = [];
- $scope.interface = '';
-
- $scope.status = 'not yet loaded';
-
- $http({
- method: 'GET',
- url: window.location.origin + "/data/interfaces"
- }).then(
- function(rsp) {$scope.interfaces = rsp.data;},
- function(rsp) {$scope.routers = ['error'];}
- );
-
- /*
- $scope.update_interfaces = function() {
-
- $http({
- method: 'GET',
- url: window.location.origin + "/data/interfaces/" + $scope.router
- }).then(
- function(rsp) {
- $scope.interfaces = rsp.data.map(function(x){
- return x.name
- });
- },
- function(rsp) {$scope.interfaces = ['error'];}
- );
- }
-
- $scope.update_status = function() {
-
- $http({
- method: 'GET',
- url: window.location.origin
- + "/alarmsdb/interface-status?equipment="
- + $scope.router
- + "&interface="
- + $scope.interface
- }).then(
- function(rsp) {$scope.status = rsp.data.status;},
- function(rsp) {$scope.interfaces = 'query error';}
- );
- }
- */
-
-});
\ No newline at end of file
diff --git a/inventory_provider/static/juniper.html b/inventory_provider/static/juniper.html
deleted file mode 100644
index 212276b8004a62fba92d844e654f50e1b1338e32..0000000000000000000000000000000000000000
--- a/inventory_provider/static/juniper.html
+++ /dev/null
@@ -1,56 +0,0 @@
-<!doctype html>
-<html ng-app="inventoryApp" lang="en">
- <head>
- <title>juniper</title>
- <script src="https://ajax.googleapis.com/ajax/libs/angularjs/1.7.5/angular.min.js"></script>
- <script src="juniper.js"></script>
- <link rel="stylesheet" href="style.css">
- </head>
- <body>
-
- <div ng-controller="juniper">
- <h2>Interfaces</h2>
- <div>
- <select
- ng-options="r for r in routers"
- ng-change="update_interface()"
- ng-model="router"></select>
- </div>
-
- <div class="column">
- <p><strong>interfaces</strong></p>
- <ul>
- <li ng-repeat="i in interfaces">{{i.name}}
- <ul>
- <li>{{i.description}}</li>
- <li ng-repeat="v4 in i.ipv4">v4: {{v4}}</li>
- <li ng-repeat="v6 in i.ipv6">v6: {{v6}}</li>
- </ul>
- </li>
- </ul>
- <div class="raw">{{interfaces}}</div>
- </div>
- <div class="column">
- <p><strong>bgp</strong></p>
- <ul>
- <li ng-repeat="p in bgp">{{p.description}}
- <ul>
- <li>local as: {{p.as.local}}</li>
- <li>peer as: {{p.as.peer}}</li>
- </ul>
- </li>
- </ul>
- <div class="raw">{{bgp}}</div>
- </div>
- <div class="column">
- <p><strong>snmp</strong></p>
- <ul>
- <li ng-repeat="i in snmp">{{i.name}}
- <ul><li>index: {{i.index}}</li></ul>
- </li>
- </ul>
- <div class="raw">{{snmp}}</div>
- </div>
- </div>
- </body>
-</html>
\ No newline at end of file
diff --git a/inventory_provider/static/juniper.js b/inventory_provider/static/juniper.js
deleted file mode 100644
index 035a0b442dc07c63ebc427cc4523585b21bbd085..0000000000000000000000000000000000000000
--- a/inventory_provider/static/juniper.js
+++ /dev/null
@@ -1,48 +0,0 @@
-var myApp = angular.module('inventoryApp', []);
-
-myApp.controller('juniper', function($scope, $http) {
-
- $scope.routers = [];
- $scope.router = '';
-
- $scope.interfaces = 'not yet loaded';
- $scope.bgp = 'not yet loaded';
- $scope.snmp = 'not yet loaded';
-
- $http({
- method: 'GET',
- url: window.location.origin + "/data/routers"
- }).then(
- function(rsp) {$scope.routers = rsp.data;},
- function(rsp) {$scope.routers = ['error'];}
- );
-
- $scope.update_interface = function() {
-
- $http({
- method: 'GET',
- url: window.location.origin + "/data/interfaces/" + $scope.router
- }).then(
- function(rsp) {$scope.interfaces = rsp.data;},
- function(rsp) {$scope.interfaces = 'error';}
- );
-
- $http({
- method: 'GET',
- url: window.location.origin + "/data/bgp/" + $scope.router
- }).then(
- function(rsp) {$scope.bgp = rsp.data;},
- function(rsp) {$scope.bgp = 'error';}
- );
-
- $http({
- method: 'GET',
- url: window.location.origin + "/data/snmp/" + $scope.router
- }).then(
- function(rsp) {$scope.snmp = rsp.data;},
- function(rsp) {$scope.snmp = 'error';}
- );
-
- }
-
-});
\ No newline at end of file
diff --git a/inventory_provider/static/style.css b/inventory_provider/static/style.css
deleted file mode 100644
index b234cbe2d3b0a09e2a4a2efef909588cd0ddedc1..0000000000000000000000000000000000000000
--- a/inventory_provider/static/style.css
+++ /dev/null
@@ -1,17 +0,0 @@
-.column {
- float: left;
- width: 100%%;
-}
-
-/* Clear floats after the columns */
-.row:after {
- content: "";
- display: table;
- clear: both;
-}
-
-.raw {
- font-style: italic;
- font-size: 10px;
- font-family: Courier
-}
\ No newline at end of file
diff --git a/inventory_provider/tasks/worker.py b/inventory_provider/tasks/worker.py
index e06c7feb7a5b8ad1701d7e64416044ff8ef9db46..c6057747dfb24b55f1f938d92ed680a68762768f 100644
--- a/inventory_provider/tasks/worker.py
+++ b/inventory_provider/tasks/worker.py
@@ -17,12 +17,7 @@ from lxml import etree
from inventory_provider.db import ims_data
from inventory_provider.db.ims import IMS
-from inventory_provider.routes.classifier import get_ims_interface, \
- get_ims_equipment_name
-from inventory_provider.routes.common import load_snmp_indexes
-from inventory_provider.routes.poller import _load_interfaces, \
- _load_interface_bundles, _get_dashboard_data, _get_dashboards, \
- _load_services
+from inventory_provider.routes.poller import load_interfaces_to_poll
from inventory_provider.tasks.app import app
from inventory_provider.tasks.common \
import get_next_redis, get_current_redis, \
@@ -521,7 +516,8 @@ def retrieve_and_persist_neteng_managed_device_list(
'No equipment retrieved from previous list')
except Exception as e:
warning_callback(str(e))
- update_latch_status(pending=False, failure=True)
+ update_latch_status(
+ InventoryTask.config, pending=False, failure=True)
raise e
try:
@@ -530,7 +526,7 @@ def retrieve_and_persist_neteng_managed_device_list(
info_callback(f'saved {len(netdash_equipment)} managed routers')
except Exception as e:
warning_callback(str(e))
- update_latch_status(pending=False, failure=True)
+ update_latch_status(InventoryTask.config, pending=False, failure=True)
raise e
return netdash_equipment
@@ -1075,12 +1071,23 @@ def persist_ims_data(data, use_current=False):
node_pair_services = data['node_pair_services']
sid_services = data['sid_services']
+ def _get_pops():
+ # de-dupe the sites (by abbreviation)
+ pops = {
+ equip['pop']['abbreviation']: equip['pop']
+ for equip in locations.values()
+ if equip['pop']['abbreviation']}
+ return pops.values()
+
if use_current:
r = get_current_redis(InventoryTask.config)
+ r.delete('ims:sid_services')
+
# only need to delete the individual keys if it's just an IMS update
# rather than a complete update (the db will have been flushed)
for key_pattern in [
+ 'ims:pop:*',
'ims:location:*',
'ims:lg:*',
'ims:circuit_hierarchy:*',
@@ -1089,8 +1096,6 @@ def persist_ims_data(data, use_current=False):
'ims:gws_indirect:*',
'ims:node_pair_services:*'
]:
-
- r.delete('ims:sid_services')
rp = r.pipeline()
for k in r.scan_iter(key_pattern, count=1000):
rp.delete(k)
@@ -1101,6 +1106,8 @@ def persist_ims_data(data, use_current=False):
rp = r.pipeline()
for h, d in locations.items():
rp.set(f'ims:location:{h}', json.dumps([d]))
+ for pop in _get_pops():
+ rp.set(f'ims:pop:{pop["abbreviation"]}', json.dumps(pop))
rp.execute()
rp = r.pipeline()
for router in lg_routers:
@@ -1191,47 +1198,9 @@ def populate_poller_interfaces_cache(warning_callback=lambda s: None):
lab_keys_pattern = 'lab:netconf-interfaces-hosts:*'
lab_equipment = [h.decode('utf-8')[len(lab_keys_pattern) - 1:]
for h in r.keys(lab_keys_pattern)]
- standard_interfaces = _load_interfaces(
- InventoryTask.config,
- no_lab=False,
- use_next_redis=True)
-
- bundles = _load_interface_bundles(
- InventoryTask.config,
- use_next_redis=True
- )
- snmp_indexes = load_snmp_indexes(
- InventoryTask.config, use_next_redis=True)
-
- services = _load_services(InventoryTask.config, use_next_redis=True)
-
- def _get_populated_interfaces(interfaces):
-
- for ifc in interfaces:
- router_snmp = snmp_indexes.get(ifc['router'], None)
- if router_snmp and ifc['name'] in router_snmp:
- ifc['snmp-index'] = router_snmp[ifc['name']]['index']
-
- router_bundle = bundles.get(ifc['router'], None)
- if router_bundle:
- base_ifc = ifc['name'].split('.')[0]
- ifc['bundle-parents'] = router_bundle.get(base_ifc, [])
-
- router_services = services.get(
- get_ims_equipment_name(ifc['router'], r), None)
- if router_services:
- ifc['circuits'] = router_services.get(
- get_ims_interface(ifc['name']), []
- )
-
- dashboards = _get_dashboards(ifc)
- ifc['dashboards'] = sorted([d.name for d in dashboards])
- yield _get_dashboard_data(ifc)
- else:
- continue
- all_populated_interfaces = \
- list(_get_populated_interfaces(standard_interfaces))
+ all_populated_interfaces = list(
+ load_interfaces_to_poll(InventoryTask.config, use_next_redis=True))
non_lab_populated_interfaces = [x for x in all_populated_interfaces
if x['router'] not in lab_equipment]
diff --git a/setup.py b/setup.py
index 402d60e24a134bdd44bce44e1b3ca57fc24ade95..9508ab83c38fddb17b8ce8db5c5fdcb6925aae00 100644
--- a/setup.py
+++ b/setup.py
@@ -2,7 +2,7 @@ from setuptools import setup, find_packages
setup(
name='inventory-provider',
- version="0.85",
+ version="0.86",
author='GEANT',
author_email='swd@geant.org',
description='Dashboard inventory provider',
diff --git a/test/data/router-info.json b/test/data/router-info.json
index 11cf50f26a517892c3177aaebce341a8dd8207d0..213f34b48238c27b9d813ccddb8c914280e64405 100644
Binary files a/test/data/router-info.json and b/test/data/router-info.json differ
diff --git a/test/test_general_poller_routes.py b/test/test_general_poller_routes.py
index f97ca9552bd7f60fb9c9932d0d450a515428378b..4a40cb4f4ad971370700b45c5ae51faddb286917 100644
--- a/test/test_general_poller_routes.py
+++ b/test/test_general_poller_routes.py
@@ -155,10 +155,13 @@ def test_gws_indirect_snmp(client):
response_data = json.loads(rv.data.decode('utf-8'))
jsonschema.validate(response_data, poller.SERVICES_LIST_SCHEMA)
- assert response_data # test data is non-empty
- # all access services should have snmp info
- assert all('snmp' in s for s in response_data)
- assert all('counters' in s['snmp'] for s in response_data)
+ assert response_data # sanity: test data is non-empty
+ # all operational access services should have snmp info
+ operational = list(filter(
+ lambda s: s['status'] == 'operational', response_data))
+ assert operational # sanity: test data contains operational services
+ assert all('snmp' in s for s in operational)
+ assert all('counters' in s['snmp'] for s in operational)
def test_get_services_default(client):
@@ -201,9 +204,12 @@ def test_services_snmp(client, service_type):
response_data = json.loads(rv.data.decode('utf-8'))
jsonschema.validate(response_data, poller.SERVICES_LIST_SCHEMA)
- assert response_data # test data is non-empty
- # all access services should have snmp info
- assert all('snmp' in s for s in response_data)
+ assert response_data # sanity: test data is non-empty
+ # operational services should have snmp info
+ operational = list(filter(
+ lambda s: s['status'] == 'operational', response_data))
+ assert operational # sanity: test data contains operational services
+ assert all('snmp' in s for s in operational)
@pytest.mark.parametrize('uri_type,expected_type', [
diff --git a/test/test_neteng_routes.py b/test/test_neteng_routes.py
index 1e95322a3be4cd43a83caeea12c2a9f6d35e034e..7b23b5a2510650e0c8a96b6a9a74168926ad9396 100644
--- a/test/test_neteng_routes.py
+++ b/test/test_neteng_routes.py
@@ -2,7 +2,9 @@ import json
import jsonschema
import pytest
-from inventory_provider.db.ims_data import NODE_LOCATION_SCHEMA
+from inventory_provider.db.ims_data \
+ import NODE_LOCATION_SCHEMA, POP_LOCATION_SCHEMA
+from inventory_provider.routes.neteng import STRING_LIST_SCHEMA
@pytest.mark.parametrize('equipment_name', [
@@ -25,3 +27,35 @@ def test_location_not_found(client, mocked_redis):
'/neteng/location/BOGUS.EQUIPMENT.NAME',
headers={'Accept': ['application/json']})
assert rv.status_code == 404
+
+
+def test_get_pops(client, mocked_redis):
+ rv = client.get(
+ '/neteng/pops',
+ headers={'Accept': ['application/json']})
+ assert rv.status_code == 200
+ jsonschema.validate(
+ json.loads(rv.data.decode('utf-8')),
+ STRING_LIST_SCHEMA)
+
+
+@pytest.mark.parametrize('pop_name', [
+ 'AMS', 'LON', 'LON2', 'ORB', 'ORBE'
+])
+def test_pop_location(client, mocked_redis, pop_name):
+ rv = client.post(
+ f'/neteng/pop/{pop_name}',
+ headers={'Accept': ['application/json']})
+ assert rv.status_code == 200
+ jsonschema.validate(
+ json.loads(rv.data.decode('utf-8')),
+ POP_LOCATION_SCHEMA)
+ s = json.loads(rv.data.decode('utf-8'))
+ print(s)
+
+
+def test_pop_not_found(client, mocked_redis):
+ rv = client.post(
+ '/neteng/pop/BOGUS.POP.ABBREV',
+ headers={'Accept': ['application/json']})
+ assert rv.status_code == 404
diff --git a/test/test_worker.py b/test/test_worker.py
index 032768c53599c125efb946f019a4e7b5c7201d03..2ff0ce69d32d2bc8da47ea5eb728489565c3a9f5 100644
--- a/test/test_worker.py
+++ b/test/test_worker.py
@@ -392,7 +392,10 @@ def test_persist_ims_data(mocker, data_config, mocked_redis):
return_value=r)
data = {
- "locations": {"loc_a": "LOC A", "loc_b": "LOC B"},
+ "locations": {
+ "loc_a": {'pop': {'name': "LOC A", 'abbreviation': 'aaa'}},
+ "loc_b": {'pop': {'name': "LOC B", 'abbreviation': 'bbb'}},
+ },
"lg_routers": [
{"equipment name": "lg_eq1"}, {"equipment name": "lg_eq2"}
],
@@ -484,6 +487,11 @@ def test_retrieve_and_persist_neteng_managed_device_list(
def test_populate_poller_interfaces_cache(
mocker, data_config, mocked_redis):
r = common._get_redis(data_config)
+
+ mocker.patch('inventory_provider.tasks.common.get_next_redis',
+ return_value=r)
+ mocker.patch('inventory_provider.tasks.worker.get_next_redis',
+ return_value=r)
all_interfaces = [
{
"router": "router_a.geant.net",
@@ -638,19 +646,17 @@ def test_populate_poller_interfaces_cache(
r.set("lab:netconf-interfaces-hosts:lab_router_a.geant.net", "dummy")
r.set("lab:netconf-interfaces-hosts:lab_router_b.geant.net", "dummy")
- mocker.patch('inventory_provider.tasks.worker._load_interfaces',
+ mocker.patch('inventory_provider.routes.poller._load_interfaces',
side_effect=[all_interfaces, ])
- mocker.patch('inventory_provider.tasks.worker._load_interface_bundles',
+ mocker.patch('inventory_provider.routes.poller._load_interface_bundles',
return_value=bundles)
- mocker.patch('inventory_provider.tasks.worker.load_snmp_indexes',
+ mocker.patch('inventory_provider.routes.common.load_snmp_indexes',
return_value=snmp_indexes)
- mocker.patch('inventory_provider.tasks.worker._load_services',
+ mocker.patch('inventory_provider.routes.poller._load_services',
return_value=services)
mocker.patch(
'inventory_provider.tasks.worker.InventoryTask.config'
)
- mocker.patch('inventory_provider.tasks.worker.get_next_redis',
- return_value=r)
populate_poller_interfaces_cache()
assert r.exists("classifier-cache:poller-interfaces:all:no_lab")