Merge branch 'develop' into feature/DBOARD3-306

Changelog.md

@@ -2,6 +2,11 @@
 
 All notable changes to this project will be documented in this file.
 
+## [0.59] - 2021-01-27
+- DBOARD3-386: allow transfer-on-commit in netconf
+- DBOARD3-384: added pivoted asn group info to bgp peer-info responses
+- moved documentation into sphinx rst
+
 ## [0.58] - 2021-01-23
 - DBOARD3-385: use cached netconf data in case of schema validation errors
 

docs/source/protocol/msr.rst

@@ -6,7 +6,41 @@ MSR Support Endpoints
 
 These endpoints are intended for use by MSR.
 
+.. contents:: :local:
+
 /msr/access-services
 ---------------------------------
 
 .. autofunction:: inventory_provider.routes.msr.access_services
+
+
+/msr/bgp/logical-systems
+-------------------------------------
+
+.. autofunction:: inventory_provider.routes.msr.get_logical_systems
+
+
+/msr/bgp/logical-system-peerings</name>
+------------------------------------------
+
+.. autofunction:: inventory_provider.routes.msr.logical_system_peerings
+
+
+/msr/bgp/groups
+-------------------------------------
+
+.. autofunction:: inventory_provider.routes.msr.get_peering_groups
+
+
+/msr/bgp/group-peerings</name>
+-------------------------------------
+
+.. autofunction:: inventory_provider.routes.msr.bgp_group_peerings
+
+
+helpers
+-------------------------------------
+
+.. autofunction:: inventory_provider.routes.msr._handle_peering_group_list_request
+
+.. autofunction:: inventory_provider.routes.msr._handle_peering_group_request

install-sdist-to-test.py

+from getpass import getpass
+import os
+import logging
+import threading
+
+import click
+from paramiko import SSHClient
+from scp import SCPClient
+
+DEFAULT_HOSTNAMES = [
+    'test-inventory-provider01.geant.org',
+    'test-inventory-provider02.geant.org'
+]
+
+
+def _install_proc(hostname, username, password, sdist):
+
+    logging.info(f'installing on {hostname}')
+
+    scp_destination = f'/tmp/{os.path.basename(sdist)}'
+
+    ssh = SSHClient()
+    ssh.load_system_host_keys()
+    ssh.connect(hostname=hostname, username=username, password=password)
+    scp = SCPClient(ssh.get_transport())
+    scp.put(sdist, scp_destination)
+
+    channel = ssh.invoke_shell()
+    stdin = channel.makefile('wb')
+    stdout = channel.makefile('rb')
+
+    pip = '/home/inventory/venv/bin/pip'
+    commands = [
+        f'sudo su - -c \'{pip} uninstall -y inventory\'',
+        f'sudo su - -c \'{pip} install {scp_destination}\'',
+        f'rm {scp_destination}',
+        'chown -R inventory.inventory /home/inventory/venv',
+        'exit'
+    ]
+    stdin.write('\n'.join(commands) + '\n')
+    print('\n'.join(commands))
+    print(stdout.read())
+    stdout.close()
+    stdin.close()
+
+    processes = [
+        'inventory-provider.service',
+        'inventory-worker.service',
+        'inventory-monitor.service'
+    ]
+    restart_cmd = 'sudo su - -c \'systemctl restart ' \
+        + ' '.join(processes) + '\''
+    stdin, stdout, _ = ssh.exec_command(restart_cmd)
+    output = stdout.read()
+    status = stdout.channel.recv_exit_status()
+    logging.debug(f'status: {status}, output: {output}')
+
+    ssh.close()
+
+    logging.info(f'finished installing on {hostname}')
+
+
+@click.command()
+@click.option(
+    "--hostname",
+    multiple=True,
+    default=DEFAULT_HOSTNAMES,
+    type=click.STRING,
+    help="hostname [%r]"
+         % str(DEFAULT_HOSTNAMES))
+@click.option(
+    "--user",
+    default=None,
+    type=click.STRING,
+    help="ssh username")
+@click.option(
+    "--sdist",
+    required=True,
+    type=click.Path(exists=True, dir_okay=False),
+    help="sdist filename")
+def cli(hostname, user, sdist):
+    password = getpass(prompt='Password: ', stream=None)
+
+    def _make_thread(h):
+        t = threading.Thread(
+            target=_install_proc,
+            args=[h, user, password, sdist])
+        t.start()
+        return t
+
+    threads = [_make_thread(h) for h in hostname]
+
+    for t in threads:
+        t.join()
+
+
+if __name__ == '__main__':
+    logging.basicConfig(level=logging.INFO)
+    cli()

inventory_provider/routes/msr.py

+import itertools
+import json
+
 from flask import Blueprint, jsonify, Response
 
 from inventory_provider.routes import common
@@ -35,6 +38,43 @@ ACCESS_SERVICES_LIST_SCHEMA = {
     "items": {"$ref": "#/definitions/service"}
 }
 
+PEERING_GROUP_LIST_SCHEMA = {
+    "$schema": "http://json-schema.org/draft-07/schema#",
+    "type": "array",
+    "items": {"type": "string"}
+
+}
+
+PEERING_LIST_SCHEMA = {
+    "$schema": "http://json-schema.org/draft-07/schema#",
+    "definitions": {
+        "peering-instance": {
+            "type": "object",
+            "properties": {
+                "address": {"type": "string"},
+                "description": {"type": "string"},
+                "logical-system": {"type": "string"},
+                "group": {"type": "string"},
+                "hostname": {"type": "string"},
+                "remote-asn": {"type": "integer"},
+                "local-asn": {"type": "integer"},
+                "instance": {"type": "string"}
+            },
+            # only vrr peerings have remote-asn
+            # only group peerings have local-asn or instance
+            # not all group peerings have 'description'
+            # and only vrr or vpn-proxy peerings are within a logical system
+            "required": [
+                "address",
+                "group",
+                "hostname"],
+            "additionalProperties": False
+        }
+    },
+    "type": "array",
+    "items": {"$ref": "#/definitions/peering-instance"}
+}
+
 
 @routes.after_request
 def after_request(resp):
@@ -54,12 +94,209 @@ def access_services():
     .. asjson::
        inventory_provider.routes.msr.ACCESS_SERVICES_LIST_SCHEMA
 
+    :param name:
     :return:
     """
-
     # todo - replace with IMS implementation
-
     return Response(
         response='no access services found',
         status=404,
         mimetype="text/html")
+    # redis = common.get_current_redis()
+    #
+    # def _services():
+    #     for k in redis.scan_iter('opsdb:access_services:*'):
+    #         service = redis.get(k.decode('utf-8')).decode('utf-8')
+    #         yield json.loads(service)
+    #
+    # cache_key = 'classifier-cache:msr:access-services'
+    # result = redis.get(cache_key)
+    #
+    # if result:
+    #     result = json.loads(result.decode('utf-8'))
+    # else:
+    #     result = list(_services())
+    #     # cache this data for the next call
+    #     redis.set(cache_key, json.dumps(result).encode('utf-8'))
+    #
+    # if not result:
+    #     return Response(
+    #         response='no access services found',
+    #         status=404,
+    #         mimetype="text/html")
+    #
+    # return jsonify(result)
+
+
+def _handle_peering_group_request(name, cache_key, group_key_base):
+    """
+    Common method for used by
+    :meth:`inventory_provider.routes.msr.logical_system_peerings` and
+    :meth:`inventory_provider.routes.msr.bgp_group_peerings`.
+
+    This method will return a list of all peerings configured
+    for the specified group `name on any router,
+    or for all group names if `name` None.
+
+    The response will be formatted according to the following schema:
+
+    .. asjson::
+       inventory_provider.routes.msr.PEERING_LIST_SCHEMA
+
+    :param name: group/logical-system name, or None
+    :param cache_key: base cache key for this type of request
+    :param group_key_base: key above which the peerings are grouped
+    :return: a json list, formatted as above
+    """
+
+    r = common.get_current_redis()
+
+    def _get_all_subkeys():
+        keys = []
+        for k in r.scan_iter(f'{group_key_base}:*', count=1000):
+            keys.append(k.decode('utf-8'))
+        return keys
+
+    def _load_list_items(key):
+        value = r.get(key)
+        if value:
+            yield from json.loads(value.decode('utf-8'))
+
+    if name:
+        cache_key = f'{cache_key}:{name}'
+
+    items = r.get(cache_key)
+
+    if items:
+        items = json.loads(items.decode('utf-8'))
+    else:
+        if name:
+            items = _load_list_items(f'{group_key_base}:{name}')
+        else:
+            gen_list = list(map(_load_list_items, _get_all_subkeys()))
+            items = itertools.chain(*gen_list)
+
+        items = list(items)
+        if not items:
+            return Response(
+                response='no peerings found',
+                status=404,
+                mimetype="text/html")
+
+        r.set(cache_key, json.dumps(items).encode('utf-8'))
+
+    return jsonify(items)
+
+
+@routes.route("/bgp/logical-system-peerings", methods=['GET', 'POST'])
+@routes.route("/bgp/logical-system-peerings/<name>", methods=['GET', 'POST'])
+@common.require_accepts_json
+def logical_system_peerings(name=None):
+    """
+    Handler for `/msr/bgp/logical-system-peerings`
+
+    This method will return a list of all peerings configured
+    for the requested logical-system name on any router, or for any
+    logical system if no parameter is given.
+
+    :return: see :meth:`inventory_provider.routes.msr._handle_peering_group_request`
+    """  # noqa: E501
+    return _handle_peering_group_request(
+        name=name,
+        cache_key='classifier-cache:msr:logical-system-peerings',
+        group_key_base='juniper-peerings:logical-system')
+
+
+@routes.route("/bgp/group-peerings", methods=['GET', 'POST'])
+@routes.route("/bgp/group-peerings/<name>", methods=['GET', 'POST'])
+@common.require_accepts_json
+def bgp_group_peerings(name=None):
+    """
+    Handler for `/msr/bgp/group-peerings`
+
+    This method will return a list of all peerings configured
+    for the requested logical-system name on any router, or for any
+    logical system if no parameter is given.
+
+    :return: see :meth:`inventory_provider.routes.msr._handle_peering_group_request`
+    """  # noqa: E501
+    return _handle_peering_group_request(
+        name=name,
+        cache_key='classifier-cache:msr:group-peerings',
+        group_key_base='juniper-peerings:group')
+
+
+def _handle_peering_group_list_request(cache_key, group_key_base):
+    """
+    Common method for used by
+    :meth:`inventory_provider.routes.msr.get_logical_systems` and
+    :meth:`inventory_provider.routes.msr.get_peering_groups`.
+
+    This method will return a list of all immediate subkeys of
+     `group_key_base`.
+
+    The response will be formatted according to the following schema:
+
+    .. asjson::
+       inventory_provider.routes.msr.PEERING_GROUP_LIST_SCHEMA
+
+    :param cache_key: base cache key for this type of request
+    :param group_key_base: key above which the peerings are grouped
+    :return: a json list, formatted as above
+    """
+
+    r = common.get_current_redis()
+
+    def _get_all_subkeys():
+        for k in r.scan_iter(f'{group_key_base}:*', count=1000):
+            k = k.decode('utf-8')
+            yield k[len(group_key_base) + 1:]
+
+    names = r.get(cache_key)
+
+    if names:
+        names = json.loads(names.decode('utf-8'))
+    else:
+        names = list(_get_all_subkeys())
+        if not names:
+            return Response(
+                response='no groups found',
+                status=404,
+                mimetype="text/html")
+        names = sorted(names)
+
+        r.set(cache_key, json.dumps(names).encode('utf-8'))
+
+    return jsonify(names)
+
+
+@routes.route("/bgp/logical-systems", methods=['GET', 'POST'])
+@common.require_accepts_json
+def get_logical_systems():
+    """
+    Handler for `/msr/bgp/logical-systems`
+
+    Returns a list of logical system names for which peering
+    information is available.
+
+    :return: see :meth:`inventory_provider.routes.msr._handle_peering_group_list_request`
+    """  # noqa: E501
+    return _handle_peering_group_list_request(
+        cache_key='classifier-cache:msr:logical-systems',
+        group_key_base='juniper-peerings:logical-system')
+
+
+@routes.route("/bgp/groups", methods=['GET', 'POST'])
+@common.require_accepts_json
+def get_peering_groups():
+    """
+    Handler for `/msr/bgp/groups`
+
+    Returns a list of group names for which peering
+    information is available.
+
+    :return: see :meth:`inventory_provider.routes.msr._handle_peering_group_list_request`
+    """  # noqa: E501
+    return _handle_peering_group_list_request(
+        cache_key='classifier-cache:msr:peering-groups',
+        group_key_base='juniper-peerings:group')

inventory_provider/tasks/worker.py

@@ -797,6 +797,8 @@ def _build_juniper_peering_db(update_callback=lambda s: None):
     peerings_per_address = {}
     ix_peerings = []
     peerings_per_asn = {}
+    peerings_per_logical_system = {}
+    peerings_per_group = {}
 
     # scan with bigger batches, to mitigate network latency effects
     key_prefix = 'juniper-peerings:hosts:'
@@ -813,6 +815,13 @@ def _build_juniper_peering_db(update_callback=lambda s: None):
             asn = p.get('remote-asn', None)
             if asn:
                 peerings_per_asn.setdefault(asn, []).append(p)
+            logical_system = p.get('logical-system', None)
+            if logical_system:
+                peerings_per_logical_system.setdefault(
+                    logical_system, []).append(p)
+            group = p.get('group', None)
+            if group:
+                peerings_per_group.setdefault(group, []).append(p)
 
     # sort ix peerings by group
     ix_groups = {}
@@ -839,6 +848,19 @@ def _build_juniper_peering_db(update_callback=lambda s: None):
     for k, v in peerings_per_asn.items():
         rp.set(f'juniper-peerings:peer-asn:{k}', json.dumps(v))
 
+    # create pivoted logical-systems peering lists
+    update_callback(
+        f'saving {len(peerings_per_logical_system)}'
+        ' logical-system peering lists')
+    for k, v in peerings_per_logical_system.items():
+        rp.set(f'juniper-peerings:logical-system:{k}', json.dumps(v))
+
+    # create pivoted group peering lists
+    update_callback(
+        f'saving {len(peerings_per_group)} group peering lists')
+    for k, v in peerings_per_group.items():
+        rp.set(f'juniper-peerings:group:{k}', json.dumps(v))
+
     rp.execute()
 
 

test/test_msr_routes.py

 import json
 import jsonschema
-from inventory_provider.routes.msr import ACCESS_SERVICES_LIST_SCHEMA
+
+import pytest
+
+from inventory_provider.routes.msr import ACCESS_SERVICES_LIST_SCHEMA, \
+    PEERING_LIST_SCHEMA, PEERING_GROUP_LIST_SCHEMA
 
 DEFAULT_REQUEST_HEADERS = {
     "Content-type": "application/json",
@@ -16,3 +20,97 @@ def test_access_services(client):
         '/msr/access-services',
         headers=DEFAULT_REQUEST_HEADERS)
     assert rv.status_code == 404
+    # assert rv.status_code == 200
+    # assert rv.is_json
+    # response_data = json.loads(rv.data.decode('utf-8'))
+    # jsonschema.validate(response_data, ACCESS_SERVICES_LIST_SCHEMA)
+    #
+    # assert response_data  # test data is non-empty
+
+
+def test_logical_system_peerings_all(client):
+    rv = client.get(
+        '/msr/bgp/logical-system-peerings',
+        headers=DEFAULT_REQUEST_HEADERS)
+    assert rv.status_code == 200
+    assert rv.is_json
+    response_data = json.loads(rv.data.decode('utf-8'))
+    jsonschema.validate(response_data, PEERING_LIST_SCHEMA)
+
+    assert response_data  # test data is non-empty
+    assert all('logical-system' in p for p in response_data)
+
+
+@pytest.mark.parametrize('name', ['VRR', 'VPN-PROXY'])
+def test_logical_system_peerings_specific(client, name):
+    rv = client.get(
+        f'/msr/bgp/logical-system-peerings/{name}',
+        headers=DEFAULT_REQUEST_HEADERS)
+    assert rv.status_code == 200
+    assert rv.is_json
+    response_data = json.loads(rv.data.decode('utf-8'))
+    jsonschema.validate(response_data, PEERING_LIST_SCHEMA)
+
+    assert response_data  # test data is non-empty
+    assert all(p['logical-system'] == name for p in response_data)
+
+
+@pytest.mark.parametrize('name', [
+    'VRR1',
+    'VPNPROXY',
+    'vrr',
+    ' vrr',
+    'VPN PROXY'
+])
+def test_logical_system_peerings_404(client, name):
+    rv = client.get(
+        f'/msr/bgp/logical-system-peerings/{name}',
+        headers=DEFAULT_REQUEST_HEADERS)
+    assert rv.status_code == 404
+
+
+def test_group_peerings_all(client):
+    rv = client.get(
+        '/msr/bgp/group-peerings',
+        headers=DEFAULT_REQUEST_HEADERS)
+    assert rv.status_code == 200
+    assert rv.is_json
+    response_data = json.loads(rv.data.decode('utf-8'))
+    jsonschema.validate(response_data, PEERING_LIST_SCHEMA)
+
+    assert response_data  # test data is non-empty
+
+
+@pytest.mark.parametrize('name', ['BGPLU', 'eGEANT', 'eGEANT-mcast'])
+def test_group_peerings_specific(client, name):
+    rv = client.get(
+        f'/msr/bgp/group-peerings/{name}',
+        headers=DEFAULT_REQUEST_HEADERS)
+    assert rv.status_code == 200
+    assert rv.is_json
+    response_data = json.loads(rv.data.decode('utf-8'))
+    jsonschema.validate(response_data, PEERING_LIST_SCHEMA)
+
+    assert response_data  # test data is non-empty
+    assert all(p['group'] == name for p in response_data)
+
+
+@pytest.mark.parametrize('name', ['EGEANT', 'eGEANT mcast'])
+def test_group_peerings_404(client, name):
+    rv = client.get(
+        f'/msr/bgp/logical-system-peerings/{name}',
+        headers=DEFAULT_REQUEST_HEADERS)
+    assert rv.status_code == 404
+
+
+@pytest.mark.parametrize('uri', [
+    '/msr/bgp/logical-systems',
+    '/msr/bgp/groups'])
+def test_peerings_group_list(client, uri):
+    rv = client.get(uri, headers=DEFAULT_REQUEST_HEADERS)
+    assert rv.status_code == 200
+    assert rv.is_json
+    response_data = json.loads(rv.data.decode('utf-8'))
+    jsonschema.validate(response_data, PEERING_GROUP_LIST_SCHEMA)
+
+    assert response_data  # test data is non-empty

test/test_worker_utils.py

@@ -9,6 +9,7 @@ import jsonschema
 
 from inventory_provider.tasks import worker
 from inventory_provider.tasks import common
+from inventory_provider.routes import msr
 
 
 def backend_db():
@@ -91,55 +92,61 @@ def test_build_juniper_peering_db(mocked_worker_module):
     # same as inventory_provider.juniper.PEERING_LIST_SCHEMA,
     # but with "hostname" in every returned record
 
-    PEERING_LIST_SCHEMA = {
+    LOGICAL_SYSTEM_PEERING_SCHEMA = {
+        "type": "object",
+        "properties": {
+            "logical-system": {"type": "string"},
+            "group": {"type": "string"},
+            "description": {"type": "string"},
+            "address": {"type": "string"},
+            "remote-asn": {"type": "integer"},
+            "local-asn": {"type": "integer"},
+            "hostname": {"type": "string"}
+        },
+        # local/remote-asn and/or description are not always present,
+        # just based on empirical tests - not a problem
+        "required": ["logical-system", "group", "address"],
+        "additionalProperties": False
+    }
+
+    TOP_LEVEL_PEERING_SCHEMA = {
+        "type": "object",
+        "properties": {
+            "group": {"type": "string"},
+            "description": {"type": "string"},
+            "address": {"type": "string"},
+            "remote-asn": {"type": "integer"},
+            "local-asn": {"type": "integer"},
+            "hostname": {"type": "string"}
+        },
+        # lots of internal peerings - so maybe no explicit asn's
+        "required": ["group", "address"],
+        "additionalProperties": False
+    }
+
+    INSTANCE_PEERING = {
+        "type": "object",
+        "properties": {
+            "instance": {"type": "string"},
+            "group": {"type": "string"},
+            "description": {"type": "string"},
+            "address": {"type": "string"},
+            "remote-asn": {"type": "integer"},
+            "local-asn": {"type": "integer"},
+            "hostname": {"type": "string"}
+        },
+        # description and-or local-asn is not always present,
+        # just based on empirical tests - not a problem
+        "required": ["instance", "group", "address", "remote-asn"],
+        "additionalProperties": False
+    }
+
+    DETAILED_PEERING_LIST_SCHEMA = {
         "$schema": "http://json-schema.org/draft-07/schema#",
         "definitions": {
-            "top-level-peering": {
-                "type": "object",
-                "properties": {
-                    "group": {"type": "string"},
-                    "description": {"type": "string"},
-                    "address": {"type": "string"},
-                    "remote-asn": {"type": "integer"},
-                    "local-asn": {"type": "integer"},
-                    "hostname": {"type": "string"}
-                },
-                # lots of internal peerings - so maybe no explicit asn's
-                "required": ["group", "address"],
-                "additionalProperties": False
-            },
-            "instance-peering": {
-                "type": "object",
-                "properties": {
-                    "instance": {"type": "string"},
-                    "group": {"type": "string"},
-                    "description": {"type": "string"},
-                    "address": {"type": "string"},
-                    "remote-asn": {"type": "integer"},
-                    "local-asn": {"type": "integer"},
-                    "hostname": {"type": "string"}
-                },
-                # description and-or local-asn is not always present,
-                # just based on empirical tests - not a problem
-                "required": ["instance", "group", "address", "remote-asn"],
-                "additionalProperties": False
-            },
-            "logical-system-peering": {
-                "type": "object",
-                "properties": {
-                    "logical-system": {"type": "string"},
-                    "group": {"type": "string"},
-                    "description": {"type": "string"},
-                    "address": {"type": "string"},
-                    "remote-asn": {"type": "integer"},
-                    "local-asn": {"type": "integer"},
-                    "hostname": {"type": "string"}
-                },
-                # local/remote-asn and/or description are not always present,
-                # just based on empirical tests - not a problem
-                "required": ["logical-system", "group", "address"],
-                "additionalProperties": False
-            },
+            "top-level-peering": TOP_LEVEL_PEERING_SCHEMA,
+            "instance-peering": INSTANCE_PEERING,
+            "logical-system-peering": LOGICAL_SYSTEM_PEERING_SCHEMA,
             "peering": {
                 "oneOf": [
                     {"$ref": "#/definitions/top-level-peering"},
@@ -154,6 +161,8 @@ def test_build_juniper_peering_db(mocked_worker_module):
 
     db = backend_db()  # also forces initialization
 
+    # remove the juniper-peerings:* items that
+    # will be created by _build_juniper_peering_db
     def _x(k):
         if not k.startswith('juniper-peerings'):
             return False
@@ -167,6 +176,8 @@ def test_build_juniper_peering_db(mocked_worker_module):
     worker._build_juniper_peering_db()
 
     found_record = False
+    found_logical_system = False
+    found_group = False
     for key, value in db.items():
 
         if not _x(key):
@@ -183,9 +194,23 @@ def test_build_juniper_peering_db(mocked_worker_module):
                 assert address == canonical
             continue
 
-        jsonschema.validate(value, PEERING_LIST_SCHEMA)
+        jsonschema.validate(value, DETAILED_PEERING_LIST_SCHEMA)
+
+        if 'logical-system:' in key:
+            jsonschema.validate(value, msr.PEERING_LIST_SCHEMA)
+            m = re.match(r'.*logical-system:(.+)$', key)
+            assert all(p['logical-system'] == m.group(1) for p in value)
+            found_logical_system = True
+
+        if 'group:' in key:
+            jsonschema.validate(value, msr.PEERING_LIST_SCHEMA)
+            m = re.match(r'.*group:(.+)$', key)
+            assert all(p['group'] == m.group(1) for p in value)
+            found_group = True
 
     assert found_record
+    assert found_logical_system
+    assert found_group
 
 
 def test_build_snmp_peering_db(mocked_worker_module):