Finished release 0.62.

Changelog.md

@@ -2,6 +2,9 @@
 
 All notable changes to this project will be documented in this file.
 
+## [0.62] - 2021-03-24
+- POL1-392: added latch timestamp to version response
+
 ## [0.61] - 2021-03-05
 - POL1-380: added /poller/speeds route
 

circuit_tree.py

+import os
+from operator import itemgetter
+
+import click
+from tree_format import format_tree
+
+from inventory_provider.db.ims import IMS, CIRCUIT_PROPERTIES
+
+username = 'TEST05'
+password = ''
+bt = os.getenv('IMS_BT')
+
+ds = IMS('http://83.97.94.128:2080/api', username, password, bt)
+
+NAV_PROPS = [
+    CIRCUIT_PROPERTIES['Speed'],
+    CIRCUIT_PROPERTIES['Product']
+]
+
+
+@click.command()
+@click.option('-c', 'carriers', is_flag=True)
+@click.argument('root_circuit_id', type=click.INT, required=True)
+def cli(carriers, root_circuit_id):
+    if carriers:
+        children_prop = 'carriercircuits'
+        childid = 'carriercircuitid'
+        NAV_PROPS.append(CIRCUIT_PROPERTIES['CarrierCircuits'])
+    else:
+        children_prop = 'subcircuits'
+        childid = 'subcircuitid'
+        NAV_PROPS.append(CIRCUIT_PROPERTIES['SubCircuits'])
+
+    def _get_childcircuit_tree(circuit_id):
+
+        circuit = ds.get_entity_by_id(
+            'circuit', circuit_id, navigation_properties=NAV_PROPS)
+        _tree = [
+            f'{circuit["id"]} -- {circuit["name"]} -- '
+            f'prod. {circuit["product"]["name"]} -- '
+            f'spd. {circuit["speed"]["name"]}'
+        ]
+
+        if circuit[children_prop]:
+
+            children = []
+            for child_circuit in circuit[children_prop]:
+                children.append(_get_childcircuit_tree(
+                    child_circuit[childid]
+                ))
+            _tree.append(children)
+        else:
+            _tree.append([])
+        return _tree
+    tree = _get_childcircuit_tree(root_circuit_id)
+    print(format_tree(
+        tree, format_node=itemgetter(0), get_children=itemgetter(1))
+    )
+
+
+if __name__ == '__main__':
+    # 659386
+    cli()

inventory_provider/routes/default.py

@@ -4,42 +4,43 @@ from flask import Blueprint, jsonify, current_app
 from inventory_provider.routes import common
 from inventory_provider.tasks.common import get_current_redis, get_latch
 
-routes = Blueprint("inventory-data-default-routes", __name__)
+routes = Blueprint('inventory-data-default-routes', __name__)
 
 API_VERSION = '0.1'
 
 VERSION_SCHEMA = {
-    "$schema": "http://json-schema.org/draft-07/schema#",
+    '$schema': 'http://json-schema.org/draft-07/schema#',
 
-    "definitions": {
-        "latch": {
-            "type": "object",
-            "properties": {
-                "current": {"type": "integer"},
-                "next": {"type": "integer"},
-                "this": {"type": "integer"},
-                "failure": {"type": "boolean"},
-                "pending": {"type": "boolean"},
+    'definitions': {
+        'latch': {
+            'type': 'object',
+            'properties': {
+                'current': {'type': 'integer'},
+                'next': {'type': 'integer'},
+                'this': {'type': 'integer'},
+                'failure': {'type': 'boolean'},
+                'pending': {'type': 'boolean'},
+                'timestamp': {'type': 'number'}
             },
-            "required": ["current", "next", "this", "pending", "failure"],
-            "additionalProperties": False
+            'required': ['current', 'next', 'this', 'pending', 'failure'],
+            'additionalProperties': False
         }
     },
 
-    "type": "object",
-    "properties": {
-        "api": {
-            "type": "string",
-            "pattern": r'\d+\.\d+'
+    'type': 'object',
+    'properties': {
+        'api': {
+            'type': 'string',
+            'pattern': r'\d+\.\d+'
         },
-        "module": {
-            "type": "string",
-            "pattern": r'\d+\.\d+'
+        'module': {
+            'type': 'string',
+            'pattern': r'\d+\.\d+'
         },
-        "latch": {"$ref": "#/definitions/latch"}
+        'latch': {'$ref': '#/definitions/latch'}
     },
-    "required": ["api", "module"],
-    "additionalProperties": False
+    'required': ['api', 'module'],
+    'additionalProperties': False
 }
 
 
@@ -48,7 +49,7 @@ def after_request(resp):
     return common.after_request(resp)
 
 
-@routes.route("/version", methods=['GET', 'POST'])
+@routes.route('/version', methods=['GET', 'POST'])
 @common.require_accepts_json
 def version():
     """

inventory_provider/tasks/common.py

 import json
 import logging
+import time
 
 import jsonschema
 import redis
@@ -11,17 +12,18 @@ DEFAULT_REDIS_SENTINEL_TIMEOUT = 0.1
 DEFAULT_SENTINEL_SOCKET_TIMEOUT = 0.1
 
 DB_LATCH_SCHEMA = {
-    "$schema": "http://json-schema.org/draft-07/schema#",
-    "type": "object",
-    "properties": {
-        "current": {"type": "integer"},
-        "next": {"type": "integer"},
-        "this": {"type": "integer"},
-        "pending": {"type": "boolean"},
-        "failure": {"type": "boolean"}
+    '$schema': 'http://json-schema.org/draft-07/schema#',
+    'type': 'object',
+    'properties': {
+        'current': {'type': 'integer'},
+        'next': {'type': 'integer'},
+        'this': {'type': 'integer'},
+        'pending': {'type': 'boolean'},
+        'failure': {'type': 'boolean'},
+        'timestamp': {'type': 'number'}
     },
-    "required": ["current", "next", "this"],
-    "additionalProperties": False
+    'required': ['current', 'next', 'this'],
+    'additionalProperties': False
 }
 
 TASK_LOG_SCHEMA = {
@@ -86,17 +88,24 @@ def update_latch_status(config, pending=False, failure=False):
     logger.debug('updating latch status: pending={}, failure={}'.format(
         pending, failure))
 
+    now = time.time()
     for db in config['redis-databases']:
         r = _get_redis(config, dbid=db)
         latch = get_latch(r)
         if not latch:
             continue
+        if not pending and not failure:
+            if not latch['pending'] and not latch['failure']:
+                logger.error(
+                    'updating latch for db {db} with pending=failure=True, '
+                    f'but latch is already {latch}')
+            latch['timestamp'] = now
         latch['pending'] = pending
         latch['failure'] = failure
         r.set('db:latch', json.dumps(latch))
 
 
-def set_latch(config, new_current, new_next):
+def set_latch(config, new_current, new_next, timestamp):
 
     logger.debug('setting latch: new current={}, new next={}'.format(
         new_current, new_next))
@@ -107,7 +116,8 @@ def set_latch(config, new_current, new_next):
             'next': new_next,
             'this': db,
             'pending': False,
-            'failure': False
+            'failure': False,
+            'timestamp': timestamp
         }
 
         r = _get_redis(config, dbid=db)
@@ -129,7 +139,11 @@ def latch_db(config):
     next_idx = db_ids.index(latch['next'])
     next_idx = (next_idx + 1) % len(db_ids)
 
-    set_latch(config, new_current=latch['next'], new_next=db_ids[next_idx])
+    set_latch(
+        config,
+        new_current=latch['next'],
+        new_next=db_ids[next_idx],
+        timestamp=time.time())
 
 
 def _get_redis(config, dbid=None):

inventory_provider/tasks/worker.py

@@ -199,7 +199,7 @@ def update_interfaces_to_services(self):
     rp.execute()
 
 
-def _unmanaged_interfaces(self):
+def _unmanaged_interfaces():
 
     def _convert(d):
         # the config file keys are more readable than
@@ -215,18 +215,6 @@ def _unmanaged_interfaces(self):
         _convert,
         InventoryTask.config.get('unmanaged-interfaces', []))
 
-    # if interfaces:
-    #     r = get_next_redis(InventoryTask.config)
-    #     rp = r.pipeline()
-    #     for ifc in interfaces:
-    #         rp.set(
-    #             f'reverse_interface_addresses:{ifc["name"]}',
-    #             json.dumps(ifc))
-    #         rp.set(
-    #             f'subnets:{ifc["interface address"]}',
-    #             json.dumps([ifc]))
-    #     rp.execute()
-
 
 @app.task(base=InventoryTask, bind=True, name='update_access_services')
 @log_task_entry_and_exit
@@ -553,7 +541,8 @@ def _erase_next_db(config):
         set_latch(
             config,
             new_current=saved_latch['current'],
-            new_next=saved_latch['next'])
+            new_next=saved_latch['next'],
+            timestamp=saved_latch.get('timestamp', 0))
 
 
 @app.task(base=InventoryTask, bind=True, name='internal_refresh_phase_2')
@@ -710,6 +699,10 @@ def _build_subnet_db(update_callback=lambda s: None):
             entry = subnets.setdefault(ifc['interface address'], [])
             entry.append(ifc)
 
+    for ifc in _unmanaged_interfaces():
+        entry = subnets.setdefault(ifc['interface address'], [])
+        entry.append(ifc)
+
     update_callback('saving {} subnets'.format(len(subnets)))
 
     rp = r.pipeline()

setup.py

@@ -2,7 +2,7 @@ from setuptools import setup, find_packages
 
 setup(
     name='inventory-provider',
-    version="0.61",
+    version="0.62",
     author='GEANT',
     author_email='swd@geant.org',
     description='Dashboard inventory provider',

test/conftest.py

@@ -54,6 +54,20 @@ def data_config_filename():
                 "password": "ims_password"
               },
             "managed-routers": "bogus url",
+            "unmanaged-interfaces": [
+                {
+                    "address": "99.99.99.99",
+                    "network": "99.99.99.0/24",
+                    "interface": "ABC/0/0/0",
+                    "router": "bogus.host.name"
+                },
+                {
+                    "address": "999.999.999.99",
+                    "network": "999.999.999.0/24",
+                    "interface": "ZZZ/9/a/x:0.123",
+                    "router": "another.bogus.host.name"
+                }
+            ]
         }
 
         f.write(json.dumps(config).encode('utf-8'))

test/test_celery_worker_global.py

@@ -77,7 +77,7 @@ def test_next_redis(data_config, mocked_redis):
     :param mocked_redis:
     :return:
     """
-    common.set_latch(data_config, 10, 20)
+    common.set_latch(data_config, 10, 20, 100)
     r = common.get_next_redis(data_config)
     assert r
 
@@ -85,6 +85,7 @@ def test_next_redis(data_config, mocked_redis):
     latch = common.get_latch(r)
     assert latch['current'] == 10
     assert latch['next'] == 20
+    assert latch['timestamp'] == 100
 
 
 def test_next_redis_with_none(data_config, mocked_redis):

test/test_general_routes.py

@@ -5,59 +5,59 @@ from inventory_provider.routes import common
 from inventory_provider.routes.default import VERSION_SCHEMA
 
 DEFAULT_REQUEST_HEADERS = {
-    "Content-type": "application/json",
-    "Accept": ["application/json"]
+    'Content-type': 'application/json',
+    'Accept': ['application/json']
 }
 
 
 def test_version_request(client, mocked_redis):
 
     rv = client.post(
-        "version",
+        'version',
         headers=DEFAULT_REQUEST_HEADERS)
     assert rv.status_code == 200
     jsonschema.validate(
-        json.loads(rv.data.decode("utf-8")),
+        json.loads(rv.data.decode('utf-8')),
         VERSION_SCHEMA)
 
 
 def test_load_json_docs(data_config, mocked_redis):
 
     INTERFACE_SCHEMA = {
-        "$schema": "http://json-schema.org/draft-07/schema#",
+        '$schema': 'http://json-schema.org/draft-07/schema#',
 
-        "definitions": {
-            "interface": {
-                "type": "object",
-                "properties": {
-                    "logical-system": {"type": "string"},
-                    "name": {"type": "string"},
-                    "description": {"type": "string"},
-                    "bundle": {
-                        "type": "array",
-                        "items": {"type": "string"}
+        'definitions': {
+            'interface': {
+                'type': 'object',
+                'properties': {
+                    'logical-system': {'type': 'string'},
+                    'name': {'type': 'string'},
+                    'description': {'type': 'string'},
+                    'bundle': {
+                        'type': 'array',
+                        'items': {'type': 'string'}
                     },
-                    "ipv4": {
-                        "type": "array",
-                        "items": {"type": "string"}
+                    'ipv4': {
+                        'type': 'array',
+                        'items': {'type': 'string'}
                     },
-                    "ipv6": {
-                        "type": "array",
-                        "items": {"type": "string"}
+                    'ipv6': {
+                        'type': 'array',
+                        'items': {'type': 'string'}
                     }
                 },
-                "required": ["name", "description", "ipv4", "ipv6"],
-                "additionalProperties": False
+                'required': ['name', 'description', 'ipv4', 'ipv6'],
+                'additionalProperties': False
             }
         },
 
-        "type": "object",
-        "properties": {
-            "key": {"type": "string"},
-            "value": {"$ref": "#/definitions/interface"}
+        'type': 'object',
+        'properties': {
+            'key': {'type': 'string'},
+            'value': {'$ref': '#/definitions/interface'}
         },
-        "required": ["key", "value"],
-        "additionalProperties": False
+        'required': ['key', 'value'],
+        'additionalProperties': False
     }
 
     for ifc in common.load_json_docs(

test/test_worker_utils.py

@@ -11,6 +11,7 @@ import jsonschema
 from inventory_provider.tasks import worker
 from inventory_provider.tasks import common
 from inventory_provider.routes import msr
+from inventory_provider import config
 
 
 def backend_db():
@@ -23,7 +24,7 @@ def backend_db():
     }).db
 
 
-def test_build_subnet_db(mocked_worker_module):
+def test_build_subnet_db(mocked_worker_module, data_config_filename):
     """
     Verify that valid reverse subnet objects are created.
 
@@ -52,6 +53,16 @@ def test_build_subnet_db(mocked_worker_module):
         'items': {"$ref": "#/definitions/interface"},
     }
 
+    all_subnet_interfaces = set()
+    unmanaged_interfaces = set()
+    with open(data_config_filename) as f:
+        params = config.load(f)
+        for ifc in params.get('unmanaged-interfaces', []):
+            ifc_key = (f'{ifc["router"].lower()}'
+                       f':{ifc["interface"].lower()}'
+                       f':{ifc["network"]}')
+            unmanaged_interfaces.add(ifc_key)
+
     db = backend_db()  # also forces initialization
 
     def _x(k):
@@ -80,8 +91,16 @@ def test_build_subnet_db(mocked_worker_module):
         for ifc in value:
             assert ifc['interface address'] == address
 
+        ifc_key = (f'{ifc["router"]}'
+                   f':{ifc["interface name"]}'
+                   f':{ifc["interface address"]}')
+
+        all_subnet_interfaces.add(ifc_key)
+
     assert found_record
 
+    assert unmanaged_interfaces <= all_subnet_interfaces
+
 
 def test_build_juniper_peering_db(mocked_worker_module):
     """