diff --git a/Changelog.md b/Changelog.md
index 7e3a39738561186d4e619e5ca6e679450d9bc26d..055c1ebd34e5bec0ad59c4ba2805e0a9be024d18 100644
--- a/Changelog.md
+++ b/Changelog.md
@@ -2,6 +2,9 @@
All notable changes to this project will be documented in this file.
+## [0.62] - 2021-03-24
+- POL1-392: added latch timestamp to version response
+
## [0.61] - 2021-03-05
- POL1-380: added /poller/speeds route
diff --git a/circuit_tree.py b/circuit_tree.py
new file mode 100644
index 0000000000000000000000000000000000000000..96b7928e6629f7715fcb94d37979c684e399bbfb
--- /dev/null
+++ b/circuit_tree.py
@@ -0,0 +1,63 @@
+import os
+from operator import itemgetter
+
+import click
+from tree_format import format_tree
+
+from inventory_provider.db.ims import IMS, CIRCUIT_PROPERTIES
+
+username = 'TEST05'
+password = ''
+bt = os.getenv('IMS_BT')
+
+ds = IMS('http://83.97.94.128:2080/api', username, password, bt)
+
+NAV_PROPS = [
+ CIRCUIT_PROPERTIES['Speed'],
+ CIRCUIT_PROPERTIES['Product']
+]
+
+
+@click.command()
+@click.option('-c', 'carriers', is_flag=True)
+@click.argument('root_circuit_id', type=click.INT, required=True)
+def cli(carriers, root_circuit_id):
+ if carriers:
+ children_prop = 'carriercircuits'
+ childid = 'carriercircuitid'
+ NAV_PROPS.append(CIRCUIT_PROPERTIES['CarrierCircuits'])
+ else:
+ children_prop = 'subcircuits'
+ childid = 'subcircuitid'
+ NAV_PROPS.append(CIRCUIT_PROPERTIES['SubCircuits'])
+
+ def _get_childcircuit_tree(circuit_id):
+
+ circuit = ds.get_entity_by_id(
+ 'circuit', circuit_id, navigation_properties=NAV_PROPS)
+ _tree = [
+ f'{circuit["id"]} -- {circuit["name"]} -- '
+ f'prod. {circuit["product"]["name"]} -- '
+ f'spd. {circuit["speed"]["name"]}'
+ ]
+
+ if circuit[children_prop]:
+
+ children = []
+ for child_circuit in circuit[children_prop]:
+ children.append(_get_childcircuit_tree(
+ child_circuit[childid]
+ ))
+ _tree.append(children)
+ else:
+ _tree.append([])
+ return _tree
+ tree = _get_childcircuit_tree(root_circuit_id)
+ print(format_tree(
+ tree, format_node=itemgetter(0), get_children=itemgetter(1))
+ )
+
+
+if __name__ == '__main__':
+ # 659386
+ cli()
diff --git a/inventory_provider/routes/default.py b/inventory_provider/routes/default.py
index f65d6124835c09f70459f9d2defe01a36e23dbd6..047cc10e3534d092879545c3f3a3ab8cf0e882e8 100644
--- a/inventory_provider/routes/default.py
+++ b/inventory_provider/routes/default.py
@@ -4,42 +4,43 @@ from flask import Blueprint, jsonify, current_app
from inventory_provider.routes import common
from inventory_provider.tasks.common import get_current_redis, get_latch
-routes = Blueprint("inventory-data-default-routes", __name__)
+routes = Blueprint('inventory-data-default-routes', __name__)
API_VERSION = '0.1'
VERSION_SCHEMA = {
- "$schema": "http://json-schema.org/draft-07/schema#",
+ '$schema': 'http://json-schema.org/draft-07/schema#',
- "definitions": {
- "latch": {
- "type": "object",
- "properties": {
- "current": {"type": "integer"},
- "next": {"type": "integer"},
- "this": {"type": "integer"},
- "failure": {"type": "boolean"},
- "pending": {"type": "boolean"},
+ 'definitions': {
+ 'latch': {
+ 'type': 'object',
+ 'properties': {
+ 'current': {'type': 'integer'},
+ 'next': {'type': 'integer'},
+ 'this': {'type': 'integer'},
+ 'failure': {'type': 'boolean'},
+ 'pending': {'type': 'boolean'},
+ 'timestamp': {'type': 'number'}
},
- "required": ["current", "next", "this", "pending", "failure"],
- "additionalProperties": False
+ 'required': ['current', 'next', 'this', 'pending', 'failure'],
+ 'additionalProperties': False
}
},
- "type": "object",
- "properties": {
- "api": {
- "type": "string",
- "pattern": r'\d+\.\d+'
+ 'type': 'object',
+ 'properties': {
+ 'api': {
+ 'type': 'string',
+ 'pattern': r'\d+\.\d+'
},
- "module": {
- "type": "string",
- "pattern": r'\d+\.\d+'
+ 'module': {
+ 'type': 'string',
+ 'pattern': r'\d+\.\d+'
},
- "latch": {"$ref": "#/definitions/latch"}
+ 'latch': {'$ref': '#/definitions/latch'}
},
- "required": ["api", "module"],
- "additionalProperties": False
+ 'required': ['api', 'module'],
+ 'additionalProperties': False
}
@@ -48,7 +49,7 @@ def after_request(resp):
return common.after_request(resp)
-@routes.route("/version", methods=['GET', 'POST'])
+@routes.route('/version', methods=['GET', 'POST'])
@common.require_accepts_json
def version():
"""
diff --git a/inventory_provider/tasks/common.py b/inventory_provider/tasks/common.py
index ac85afb7de7094517c95cf2ed5f2929b211f6afc..427d51d8a18e4128f3dd3b174274e65fa17f5f8f 100644
--- a/inventory_provider/tasks/common.py
+++ b/inventory_provider/tasks/common.py
@@ -1,5 +1,6 @@
import json
import logging
+import time
import jsonschema
import redis
@@ -11,17 +12,18 @@ DEFAULT_REDIS_SENTINEL_TIMEOUT = 0.1
DEFAULT_SENTINEL_SOCKET_TIMEOUT = 0.1
DB_LATCH_SCHEMA = {
- "$schema": "http://json-schema.org/draft-07/schema#",
- "type": "object",
- "properties": {
- "current": {"type": "integer"},
- "next": {"type": "integer"},
- "this": {"type": "integer"},
- "pending": {"type": "boolean"},
- "failure": {"type": "boolean"}
+ '$schema': 'http://json-schema.org/draft-07/schema#',
+ 'type': 'object',
+ 'properties': {
+ 'current': {'type': 'integer'},
+ 'next': {'type': 'integer'},
+ 'this': {'type': 'integer'},
+ 'pending': {'type': 'boolean'},
+ 'failure': {'type': 'boolean'},
+ 'timestamp': {'type': 'number'}
},
- "required": ["current", "next", "this"],
- "additionalProperties": False
+ 'required': ['current', 'next', 'this'],
+ 'additionalProperties': False
}
TASK_LOG_SCHEMA = {
@@ -86,17 +88,24 @@ def update_latch_status(config, pending=False, failure=False):
logger.debug('updating latch status: pending={}, failure={}'.format(
pending, failure))
+ now = time.time()
for db in config['redis-databases']:
r = _get_redis(config, dbid=db)
latch = get_latch(r)
if not latch:
continue
+ if not pending and not failure:
+ if not latch['pending'] and not latch['failure']:
+ logger.error(
+ 'updating latch for db {db} with pending=failure=True, '
+ f'but latch is already {latch}')
+ latch['timestamp'] = now
latch['pending'] = pending
latch['failure'] = failure
r.set('db:latch', json.dumps(latch))
-def set_latch(config, new_current, new_next):
+def set_latch(config, new_current, new_next, timestamp):
logger.debug('setting latch: new current={}, new next={}'.format(
new_current, new_next))
@@ -107,7 +116,8 @@ def set_latch(config, new_current, new_next):
'next': new_next,
'this': db,
'pending': False,
- 'failure': False
+ 'failure': False,
+ 'timestamp': timestamp
}
r = _get_redis(config, dbid=db)
@@ -129,7 +139,11 @@ def latch_db(config):
next_idx = db_ids.index(latch['next'])
next_idx = (next_idx + 1) % len(db_ids)
- set_latch(config, new_current=latch['next'], new_next=db_ids[next_idx])
+ set_latch(
+ config,
+ new_current=latch['next'],
+ new_next=db_ids[next_idx],
+ timestamp=time.time())
def _get_redis(config, dbid=None):
diff --git a/inventory_provider/tasks/worker.py b/inventory_provider/tasks/worker.py
index 8a85d7e6ad1cac9db894b2730755addb6057b00f..9102731490f31548ccb06818d5e109196635dce2 100644
--- a/inventory_provider/tasks/worker.py
+++ b/inventory_provider/tasks/worker.py
@@ -199,7 +199,7 @@ def update_interfaces_to_services(self):
rp.execute()
-def _unmanaged_interfaces(self):
+def _unmanaged_interfaces():
def _convert(d):
# the config file keys are more readable than
@@ -215,18 +215,6 @@ def _unmanaged_interfaces(self):
_convert,
InventoryTask.config.get('unmanaged-interfaces', []))
- # if interfaces:
- # r = get_next_redis(InventoryTask.config)
- # rp = r.pipeline()
- # for ifc in interfaces:
- # rp.set(
- # f'reverse_interface_addresses:{ifc["name"]}',
- # json.dumps(ifc))
- # rp.set(
- # f'subnets:{ifc["interface address"]}',
- # json.dumps([ifc]))
- # rp.execute()
-
@app.task(base=InventoryTask, bind=True, name='update_access_services')
@log_task_entry_and_exit
@@ -553,7 +541,8 @@ def _erase_next_db(config):
set_latch(
config,
new_current=saved_latch['current'],
- new_next=saved_latch['next'])
+ new_next=saved_latch['next'],
+ timestamp=saved_latch.get('timestamp', 0))
@app.task(base=InventoryTask, bind=True, name='internal_refresh_phase_2')
@@ -710,6 +699,10 @@ def _build_subnet_db(update_callback=lambda s: None):
entry = subnets.setdefault(ifc['interface address'], [])
entry.append(ifc)
+ for ifc in _unmanaged_interfaces():
+ entry = subnets.setdefault(ifc['interface address'], [])
+ entry.append(ifc)
+
update_callback('saving {} subnets'.format(len(subnets)))
rp = r.pipeline()
diff --git a/setup.py b/setup.py
index 74ed25b87921f2629418f0fdfd3966ffc3a83bdf..4832a99f1912f06f8d5044f585c61a108fac9c9a 100644
--- a/setup.py
+++ b/setup.py
@@ -2,7 +2,7 @@ from setuptools import setup, find_packages
setup(
name='inventory-provider',
- version="0.61",
+ version="0.62",
author='GEANT',
author_email='swd@geant.org',
description='Dashboard inventory provider',
diff --git a/test/conftest.py b/test/conftest.py
index 4bafdb12d30e117bc0de5cd261e1d1bdec8f240a..2984ae3785845bc100471a30a207177601be2da3 100644
--- a/test/conftest.py
+++ b/test/conftest.py
@@ -54,6 +54,20 @@ def data_config_filename():
"password": "ims_password"
},
"managed-routers": "bogus url",
+ "unmanaged-interfaces": [
+ {
+ "address": "99.99.99.99",
+ "network": "99.99.99.0/24",
+ "interface": "ABC/0/0/0",
+ "router": "bogus.host.name"
+ },
+ {
+ "address": "999.999.999.99",
+ "network": "999.999.999.0/24",
+ "interface": "ZZZ/9/a/x:0.123",
+ "router": "another.bogus.host.name"
+ }
+ ]
}
f.write(json.dumps(config).encode('utf-8'))
diff --git a/test/test_celery_worker_global.py b/test/test_celery_worker_global.py
index 330f8d815e4040eb06abef7d2d0594075e72e41c..ecd1d25e192ecaa06ee4528684292366a8472e2a 100644
--- a/test/test_celery_worker_global.py
+++ b/test/test_celery_worker_global.py
@@ -77,7 +77,7 @@ def test_next_redis(data_config, mocked_redis):
:param mocked_redis:
:return:
"""
- common.set_latch(data_config, 10, 20)
+ common.set_latch(data_config, 10, 20, 100)
r = common.get_next_redis(data_config)
assert r
@@ -85,6 +85,7 @@ def test_next_redis(data_config, mocked_redis):
latch = common.get_latch(r)
assert latch['current'] == 10
assert latch['next'] == 20
+ assert latch['timestamp'] == 100
def test_next_redis_with_none(data_config, mocked_redis):
diff --git a/test/test_general_routes.py b/test/test_general_routes.py
index 254d9db122b4d944bdd66df67f8a41e5ca67712c..8bc82db71c96426f68d502f8fe4dac4e2414cd40 100644
--- a/test/test_general_routes.py
+++ b/test/test_general_routes.py
@@ -5,59 +5,59 @@ from inventory_provider.routes import common
from inventory_provider.routes.default import VERSION_SCHEMA
DEFAULT_REQUEST_HEADERS = {
- "Content-type": "application/json",
- "Accept": ["application/json"]
+ 'Content-type': 'application/json',
+ 'Accept': ['application/json']
}
def test_version_request(client, mocked_redis):
rv = client.post(
- "version",
+ 'version',
headers=DEFAULT_REQUEST_HEADERS)
assert rv.status_code == 200
jsonschema.validate(
- json.loads(rv.data.decode("utf-8")),
+ json.loads(rv.data.decode('utf-8')),
VERSION_SCHEMA)
def test_load_json_docs(data_config, mocked_redis):
INTERFACE_SCHEMA = {
- "$schema": "http://json-schema.org/draft-07/schema#",
+ '$schema': 'http://json-schema.org/draft-07/schema#',
- "definitions": {
- "interface": {
- "type": "object",
- "properties": {
- "logical-system": {"type": "string"},
- "name": {"type": "string"},
- "description": {"type": "string"},
- "bundle": {
- "type": "array",
- "items": {"type": "string"}
+ 'definitions': {
+ 'interface': {
+ 'type': 'object',
+ 'properties': {
+ 'logical-system': {'type': 'string'},
+ 'name': {'type': 'string'},
+ 'description': {'type': 'string'},
+ 'bundle': {
+ 'type': 'array',
+ 'items': {'type': 'string'}
},
- "ipv4": {
- "type": "array",
- "items": {"type": "string"}
+ 'ipv4': {
+ 'type': 'array',
+ 'items': {'type': 'string'}
},
- "ipv6": {
- "type": "array",
- "items": {"type": "string"}
+ 'ipv6': {
+ 'type': 'array',
+ 'items': {'type': 'string'}
}
},
- "required": ["name", "description", "ipv4", "ipv6"],
- "additionalProperties": False
+ 'required': ['name', 'description', 'ipv4', 'ipv6'],
+ 'additionalProperties': False
}
},
- "type": "object",
- "properties": {
- "key": {"type": "string"},
- "value": {"$ref": "#/definitions/interface"}
+ 'type': 'object',
+ 'properties': {
+ 'key': {'type': 'string'},
+ 'value': {'$ref': '#/definitions/interface'}
},
- "required": ["key", "value"],
- "additionalProperties": False
+ 'required': ['key', 'value'],
+ 'additionalProperties': False
}
for ifc in common.load_json_docs(
diff --git a/test/test_worker_utils.py b/test/test_worker_utils.py
index 198bc43b2591fe5f40eb381c9763adcaf73a939f..f18c826456595a9e084787c8cb982eb47561dcb6 100644
--- a/test/test_worker_utils.py
+++ b/test/test_worker_utils.py
@@ -11,6 +11,7 @@ import jsonschema
from inventory_provider.tasks import worker
from inventory_provider.tasks import common
from inventory_provider.routes import msr
+from inventory_provider import config
def backend_db():
@@ -23,7 +24,7 @@ def backend_db():
}).db
-def test_build_subnet_db(mocked_worker_module):
+def test_build_subnet_db(mocked_worker_module, data_config_filename):
"""
Verify that valid reverse subnet objects are created.
@@ -52,6 +53,16 @@ def test_build_subnet_db(mocked_worker_module):
'items': {"$ref": "#/definitions/interface"},
}
+ all_subnet_interfaces = set()
+ unmanaged_interfaces = set()
+ with open(data_config_filename) as f:
+ params = config.load(f)
+ for ifc in params.get('unmanaged-interfaces', []):
+ ifc_key = (f'{ifc["router"].lower()}'
+ f':{ifc["interface"].lower()}'
+ f':{ifc["network"]}')
+ unmanaged_interfaces.add(ifc_key)
+
db = backend_db() # also forces initialization
def _x(k):
@@ -80,8 +91,16 @@ def test_build_subnet_db(mocked_worker_module):
for ifc in value:
assert ifc['interface address'] == address
+ ifc_key = (f'{ifc["router"]}'
+ f':{ifc["interface name"]}'
+ f':{ifc["interface address"]}')
+
+ all_subnet_interfaces.add(ifc_key)
+
assert found_record
+ assert unmanaged_interfaces <= all_subnet_interfaces
+
def test_build_juniper_peering_db(mocked_worker_module):
"""