diff --git a/Changelog.md b/Changelog.md index b08a15dd10356ff17b8be1033a86e1a5c8d105b9..f473dcc2dc32ee5892e03b8d0d023210fd6c3a8e 100644 --- a/Changelog.md +++ b/Changelog.md @@ -2,6 +2,11 @@ All notable changes to this project will be documented in this file. +## [0.59] - 2021-01-27 +- DBOARD3-386: allow transfer-on-commit in netconf +- DBOARD3-384: added pivoted asn group info to bgp peer-info responses +- moved documentation into sphinx rst + ## [0.58] - 2021-01-23 - DBOARD3-385: use cached netconf data in case of schema validation errors diff --git a/docs/source/protocol/msr.rst b/docs/source/protocol/msr.rst index c73074bb6b5b6bb8ca59e05ff298e4ab5ae01a12..12eb41eefaaf3fc24995b1dac69591a808860c62 100644 --- a/docs/source/protocol/msr.rst +++ b/docs/source/protocol/msr.rst @@ -6,7 +6,41 @@ MSR Support Endpoints These endpoints are intended for use by MSR. +.. contents:: :local: + /msr/access-services --------------------------------- .. autofunction:: inventory_provider.routes.msr.access_services + + +/msr/bgp/logical-systems +------------------------------------- + +.. autofunction:: inventory_provider.routes.msr.get_logical_systems + + +/msr/bgp/logical-system-peerings</name> +------------------------------------------ + +.. autofunction:: inventory_provider.routes.msr.logical_system_peerings + + +/msr/bgp/groups +------------------------------------- + +.. autofunction:: inventory_provider.routes.msr.get_peering_groups + + +/msr/bgp/group-peerings</name> +------------------------------------- + +.. autofunction:: inventory_provider.routes.msr.bgp_group_peerings + + +helpers +------------------------------------- + +.. autofunction:: inventory_provider.routes.msr._handle_peering_group_list_request + +.. autofunction:: inventory_provider.routes.msr._handle_peering_group_request diff --git a/install-sdist-to-test.py b/install-sdist-to-test.py new file mode 100644 index 0000000000000000000000000000000000000000..e4a58b91298c55fe7704d7a55f79aa59eb618dca --- /dev/null +++ b/install-sdist-to-test.py @@ -0,0 +1,99 @@ +from getpass import getpass +import os +import logging +import threading + +import click +from paramiko import SSHClient +from scp import SCPClient + +DEFAULT_HOSTNAMES = [ + 'test-inventory-provider01.geant.org', + 'test-inventory-provider02.geant.org' +] + + +def _install_proc(hostname, username, password, sdist): + + logging.info(f'installing on {hostname}') + + scp_destination = f'/tmp/{os.path.basename(sdist)}' + + ssh = SSHClient() + ssh.load_system_host_keys() + ssh.connect(hostname=hostname, username=username, password=password) + scp = SCPClient(ssh.get_transport()) + scp.put(sdist, scp_destination) + + channel = ssh.invoke_shell() + stdin = channel.makefile('wb') + stdout = channel.makefile('rb') + + pip = '/home/inventory/venv/bin/pip' + commands = [ + f'sudo su - -c \'{pip} uninstall -y inventory\'', + f'sudo su - -c \'{pip} install {scp_destination}\'', + f'rm {scp_destination}', + 'chown -R inventory.inventory /home/inventory/venv', + 'exit' + ] + stdin.write('\n'.join(commands) + '\n') + print('\n'.join(commands)) + print(stdout.read()) + stdout.close() + stdin.close() + + processes = [ + 'inventory-provider.service', + 'inventory-worker.service', + 'inventory-monitor.service' + ] + restart_cmd = 'sudo su - -c \'systemctl restart ' \ + + ' '.join(processes) + '\'' + stdin, stdout, _ = ssh.exec_command(restart_cmd) + output = stdout.read() + status = stdout.channel.recv_exit_status() + logging.debug(f'status: {status}, output: {output}') + + ssh.close() + + logging.info(f'finished installing on {hostname}') + + +@click.command() +@click.option( + "--hostname", + multiple=True, + default=DEFAULT_HOSTNAMES, + type=click.STRING, + help="hostname [%r]" + % str(DEFAULT_HOSTNAMES)) +@click.option( + "--user", + default=None, + type=click.STRING, + help="ssh username") +@click.option( + "--sdist", + required=True, + type=click.Path(exists=True, dir_okay=False), + help="sdist filename") +def cli(hostname, user, sdist): + password = getpass(prompt='Password: ', stream=None) + + def _make_thread(h): + t = threading.Thread( + target=_install_proc, + args=[h, user, password, sdist]) + t.start() + return t + + threads = [_make_thread(h) for h in hostname] + + for t in threads: + t.join() + + +if __name__ == '__main__': + logging.basicConfig(level=logging.INFO) + cli() diff --git a/inventory_provider/routes/msr.py b/inventory_provider/routes/msr.py index 433107661bbe5f7eee5df60b018954dd4ccb9cbd..3c864acd0b415f1f3045e309f5255e88df722e98 100644 --- a/inventory_provider/routes/msr.py +++ b/inventory_provider/routes/msr.py @@ -1,3 +1,6 @@ +import itertools +import json + from flask import Blueprint, jsonify, Response from inventory_provider.routes import common @@ -35,6 +38,43 @@ ACCESS_SERVICES_LIST_SCHEMA = { "items": {"$ref": "#/definitions/service"} } +PEERING_GROUP_LIST_SCHEMA = { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "array", + "items": {"type": "string"} + +} + +PEERING_LIST_SCHEMA = { + "$schema": "http://json-schema.org/draft-07/schema#", + "definitions": { + "peering-instance": { + "type": "object", + "properties": { + "address": {"type": "string"}, + "description": {"type": "string"}, + "logical-system": {"type": "string"}, + "group": {"type": "string"}, + "hostname": {"type": "string"}, + "remote-asn": {"type": "integer"}, + "local-asn": {"type": "integer"}, + "instance": {"type": "string"} + }, + # only vrr peerings have remote-asn + # only group peerings have local-asn or instance + # not all group peerings have 'description' + # and only vrr or vpn-proxy peerings are within a logical system + "required": [ + "address", + "group", + "hostname"], + "additionalProperties": False + } + }, + "type": "array", + "items": {"$ref": "#/definitions/peering-instance"} +} + @routes.after_request def after_request(resp): @@ -54,12 +94,209 @@ def access_services(): .. asjson:: inventory_provider.routes.msr.ACCESS_SERVICES_LIST_SCHEMA + :param name: :return: """ - # todo - replace with IMS implementation - return Response( response='no access services found', status=404, mimetype="text/html") + # redis = common.get_current_redis() + # + # def _services(): + # for k in redis.scan_iter('opsdb:access_services:*'): + # service = redis.get(k.decode('utf-8')).decode('utf-8') + # yield json.loads(service) + # + # cache_key = 'classifier-cache:msr:access-services' + # result = redis.get(cache_key) + # + # if result: + # result = json.loads(result.decode('utf-8')) + # else: + # result = list(_services()) + # # cache this data for the next call + # redis.set(cache_key, json.dumps(result).encode('utf-8')) + # + # if not result: + # return Response( + # response='no access services found', + # status=404, + # mimetype="text/html") + # + # return jsonify(result) + + +def _handle_peering_group_request(name, cache_key, group_key_base): + """ + Common method for used by + :meth:`inventory_provider.routes.msr.logical_system_peerings` and + :meth:`inventory_provider.routes.msr.bgp_group_peerings`. + + This method will return a list of all peerings configured + for the specified group `name on any router, + or for all group names if `name` None. + + The response will be formatted according to the following schema: + + .. asjson:: + inventory_provider.routes.msr.PEERING_LIST_SCHEMA + + :param name: group/logical-system name, or None + :param cache_key: base cache key for this type of request + :param group_key_base: key above which the peerings are grouped + :return: a json list, formatted as above + """ + + r = common.get_current_redis() + + def _get_all_subkeys(): + keys = [] + for k in r.scan_iter(f'{group_key_base}:*', count=1000): + keys.append(k.decode('utf-8')) + return keys + + def _load_list_items(key): + value = r.get(key) + if value: + yield from json.loads(value.decode('utf-8')) + + if name: + cache_key = f'{cache_key}:{name}' + + items = r.get(cache_key) + + if items: + items = json.loads(items.decode('utf-8')) + else: + if name: + items = _load_list_items(f'{group_key_base}:{name}') + else: + gen_list = list(map(_load_list_items, _get_all_subkeys())) + items = itertools.chain(*gen_list) + + items = list(items) + if not items: + return Response( + response='no peerings found', + status=404, + mimetype="text/html") + + r.set(cache_key, json.dumps(items).encode('utf-8')) + + return jsonify(items) + + +@routes.route("/bgp/logical-system-peerings", methods=['GET', 'POST']) +@routes.route("/bgp/logical-system-peerings/<name>", methods=['GET', 'POST']) +@common.require_accepts_json +def logical_system_peerings(name=None): + """ + Handler for `/msr/bgp/logical-system-peerings` + + This method will return a list of all peerings configured + for the requested logical-system name on any router, or for any + logical system if no parameter is given. + + :return: see :meth:`inventory_provider.routes.msr._handle_peering_group_request` + """ # noqa: E501 + return _handle_peering_group_request( + name=name, + cache_key='classifier-cache:msr:logical-system-peerings', + group_key_base='juniper-peerings:logical-system') + + +@routes.route("/bgp/group-peerings", methods=['GET', 'POST']) +@routes.route("/bgp/group-peerings/<name>", methods=['GET', 'POST']) +@common.require_accepts_json +def bgp_group_peerings(name=None): + """ + Handler for `/msr/bgp/group-peerings` + + This method will return a list of all peerings configured + for the requested logical-system name on any router, or for any + logical system if no parameter is given. + + :return: see :meth:`inventory_provider.routes.msr._handle_peering_group_request` + """ # noqa: E501 + return _handle_peering_group_request( + name=name, + cache_key='classifier-cache:msr:group-peerings', + group_key_base='juniper-peerings:group') + + +def _handle_peering_group_list_request(cache_key, group_key_base): + """ + Common method for used by + :meth:`inventory_provider.routes.msr.get_logical_systems` and + :meth:`inventory_provider.routes.msr.get_peering_groups`. + + This method will return a list of all immediate subkeys of + `group_key_base`. + + The response will be formatted according to the following schema: + + .. asjson:: + inventory_provider.routes.msr.PEERING_GROUP_LIST_SCHEMA + + :param cache_key: base cache key for this type of request + :param group_key_base: key above which the peerings are grouped + :return: a json list, formatted as above + """ + + r = common.get_current_redis() + + def _get_all_subkeys(): + for k in r.scan_iter(f'{group_key_base}:*', count=1000): + k = k.decode('utf-8') + yield k[len(group_key_base) + 1:] + + names = r.get(cache_key) + + if names: + names = json.loads(names.decode('utf-8')) + else: + names = list(_get_all_subkeys()) + if not names: + return Response( + response='no groups found', + status=404, + mimetype="text/html") + names = sorted(names) + + r.set(cache_key, json.dumps(names).encode('utf-8')) + + return jsonify(names) + + +@routes.route("/bgp/logical-systems", methods=['GET', 'POST']) +@common.require_accepts_json +def get_logical_systems(): + """ + Handler for `/msr/bgp/logical-systems` + + Returns a list of logical system names for which peering + information is available. + + :return: see :meth:`inventory_provider.routes.msr._handle_peering_group_list_request` + """ # noqa: E501 + return _handle_peering_group_list_request( + cache_key='classifier-cache:msr:logical-systems', + group_key_base='juniper-peerings:logical-system') + + +@routes.route("/bgp/groups", methods=['GET', 'POST']) +@common.require_accepts_json +def get_peering_groups(): + """ + Handler for `/msr/bgp/groups` + + Returns a list of group names for which peering + information is available. + + :return: see :meth:`inventory_provider.routes.msr._handle_peering_group_list_request` + """ # noqa: E501 + return _handle_peering_group_list_request( + cache_key='classifier-cache:msr:peering-groups', + group_key_base='juniper-peerings:group') diff --git a/inventory_provider/tasks/worker.py b/inventory_provider/tasks/worker.py index 585698e7dfa5d70cdd5ddc63110bf54ebb6f8332..8c18412962ed07d373fe1f06778c29afd39cae00 100644 --- a/inventory_provider/tasks/worker.py +++ b/inventory_provider/tasks/worker.py @@ -797,6 +797,8 @@ def _build_juniper_peering_db(update_callback=lambda s: None): peerings_per_address = {} ix_peerings = [] peerings_per_asn = {} + peerings_per_logical_system = {} + peerings_per_group = {} # scan with bigger batches, to mitigate network latency effects key_prefix = 'juniper-peerings:hosts:' @@ -813,6 +815,13 @@ def _build_juniper_peering_db(update_callback=lambda s: None): asn = p.get('remote-asn', None) if asn: peerings_per_asn.setdefault(asn, []).append(p) + logical_system = p.get('logical-system', None) + if logical_system: + peerings_per_logical_system.setdefault( + logical_system, []).append(p) + group = p.get('group', None) + if group: + peerings_per_group.setdefault(group, []).append(p) # sort ix peerings by group ix_groups = {} @@ -839,6 +848,19 @@ def _build_juniper_peering_db(update_callback=lambda s: None): for k, v in peerings_per_asn.items(): rp.set(f'juniper-peerings:peer-asn:{k}', json.dumps(v)) + # create pivoted logical-systems peering lists + update_callback( + f'saving {len(peerings_per_logical_system)}' + ' logical-system peering lists') + for k, v in peerings_per_logical_system.items(): + rp.set(f'juniper-peerings:logical-system:{k}', json.dumps(v)) + + # create pivoted group peering lists + update_callback( + f'saving {len(peerings_per_group)} group peering lists') + for k, v in peerings_per_group.items(): + rp.set(f'juniper-peerings:group:{k}', json.dumps(v)) + rp.execute() diff --git a/test/test_msr_routes.py b/test/test_msr_routes.py index d2f28de6780db4911ef04d1627a529bce5f8317a..ff43a4d95cd7bf661e6ca9781564825f71f670c7 100644 --- a/test/test_msr_routes.py +++ b/test/test_msr_routes.py @@ -1,6 +1,10 @@ import json import jsonschema -from inventory_provider.routes.msr import ACCESS_SERVICES_LIST_SCHEMA + +import pytest + +from inventory_provider.routes.msr import ACCESS_SERVICES_LIST_SCHEMA, \ + PEERING_LIST_SCHEMA, PEERING_GROUP_LIST_SCHEMA DEFAULT_REQUEST_HEADERS = { "Content-type": "application/json", @@ -16,3 +20,97 @@ def test_access_services(client): '/msr/access-services', headers=DEFAULT_REQUEST_HEADERS) assert rv.status_code == 404 + # assert rv.status_code == 200 + # assert rv.is_json + # response_data = json.loads(rv.data.decode('utf-8')) + # jsonschema.validate(response_data, ACCESS_SERVICES_LIST_SCHEMA) + # + # assert response_data # test data is non-empty + + +def test_logical_system_peerings_all(client): + rv = client.get( + '/msr/bgp/logical-system-peerings', + headers=DEFAULT_REQUEST_HEADERS) + assert rv.status_code == 200 + assert rv.is_json + response_data = json.loads(rv.data.decode('utf-8')) + jsonschema.validate(response_data, PEERING_LIST_SCHEMA) + + assert response_data # test data is non-empty + assert all('logical-system' in p for p in response_data) + + +@pytest.mark.parametrize('name', ['VRR', 'VPN-PROXY']) +def test_logical_system_peerings_specific(client, name): + rv = client.get( + f'/msr/bgp/logical-system-peerings/{name}', + headers=DEFAULT_REQUEST_HEADERS) + assert rv.status_code == 200 + assert rv.is_json + response_data = json.loads(rv.data.decode('utf-8')) + jsonschema.validate(response_data, PEERING_LIST_SCHEMA) + + assert response_data # test data is non-empty + assert all(p['logical-system'] == name for p in response_data) + + +@pytest.mark.parametrize('name', [ + 'VRR1', + 'VPNPROXY', + 'vrr', + ' vrr', + 'VPN PROXY' +]) +def test_logical_system_peerings_404(client, name): + rv = client.get( + f'/msr/bgp/logical-system-peerings/{name}', + headers=DEFAULT_REQUEST_HEADERS) + assert rv.status_code == 404 + + +def test_group_peerings_all(client): + rv = client.get( + '/msr/bgp/group-peerings', + headers=DEFAULT_REQUEST_HEADERS) + assert rv.status_code == 200 + assert rv.is_json + response_data = json.loads(rv.data.decode('utf-8')) + jsonschema.validate(response_data, PEERING_LIST_SCHEMA) + + assert response_data # test data is non-empty + + +@pytest.mark.parametrize('name', ['BGPLU', 'eGEANT', 'eGEANT-mcast']) +def test_group_peerings_specific(client, name): + rv = client.get( + f'/msr/bgp/group-peerings/{name}', + headers=DEFAULT_REQUEST_HEADERS) + assert rv.status_code == 200 + assert rv.is_json + response_data = json.loads(rv.data.decode('utf-8')) + jsonschema.validate(response_data, PEERING_LIST_SCHEMA) + + assert response_data # test data is non-empty + assert all(p['group'] == name for p in response_data) + + +@pytest.mark.parametrize('name', ['EGEANT', 'eGEANT mcast']) +def test_group_peerings_404(client, name): + rv = client.get( + f'/msr/bgp/logical-system-peerings/{name}', + headers=DEFAULT_REQUEST_HEADERS) + assert rv.status_code == 404 + + +@pytest.mark.parametrize('uri', [ + '/msr/bgp/logical-systems', + '/msr/bgp/groups']) +def test_peerings_group_list(client, uri): + rv = client.get(uri, headers=DEFAULT_REQUEST_HEADERS) + assert rv.status_code == 200 + assert rv.is_json + response_data = json.loads(rv.data.decode('utf-8')) + jsonschema.validate(response_data, PEERING_GROUP_LIST_SCHEMA) + + assert response_data # test data is non-empty diff --git a/test/test_worker_utils.py b/test/test_worker_utils.py index 00c9e7a87c0cac299c2cfb26c125d21636089e45..9e952e4f195c04756bd3c832b9a509c6ae7b5114 100644 --- a/test/test_worker_utils.py +++ b/test/test_worker_utils.py @@ -9,6 +9,7 @@ import jsonschema from inventory_provider.tasks import worker from inventory_provider.tasks import common +from inventory_provider.routes import msr def backend_db(): @@ -91,55 +92,61 @@ def test_build_juniper_peering_db(mocked_worker_module): # same as inventory_provider.juniper.PEERING_LIST_SCHEMA, # but with "hostname" in every returned record - PEERING_LIST_SCHEMA = { + LOGICAL_SYSTEM_PEERING_SCHEMA = { + "type": "object", + "properties": { + "logical-system": {"type": "string"}, + "group": {"type": "string"}, + "description": {"type": "string"}, + "address": {"type": "string"}, + "remote-asn": {"type": "integer"}, + "local-asn": {"type": "integer"}, + "hostname": {"type": "string"} + }, + # local/remote-asn and/or description are not always present, + # just based on empirical tests - not a problem + "required": ["logical-system", "group", "address"], + "additionalProperties": False + } + + TOP_LEVEL_PEERING_SCHEMA = { + "type": "object", + "properties": { + "group": {"type": "string"}, + "description": {"type": "string"}, + "address": {"type": "string"}, + "remote-asn": {"type": "integer"}, + "local-asn": {"type": "integer"}, + "hostname": {"type": "string"} + }, + # lots of internal peerings - so maybe no explicit asn's + "required": ["group", "address"], + "additionalProperties": False + } + + INSTANCE_PEERING = { + "type": "object", + "properties": { + "instance": {"type": "string"}, + "group": {"type": "string"}, + "description": {"type": "string"}, + "address": {"type": "string"}, + "remote-asn": {"type": "integer"}, + "local-asn": {"type": "integer"}, + "hostname": {"type": "string"} + }, + # description and-or local-asn is not always present, + # just based on empirical tests - not a problem + "required": ["instance", "group", "address", "remote-asn"], + "additionalProperties": False + } + + DETAILED_PEERING_LIST_SCHEMA = { "$schema": "http://json-schema.org/draft-07/schema#", "definitions": { - "top-level-peering": { - "type": "object", - "properties": { - "group": {"type": "string"}, - "description": {"type": "string"}, - "address": {"type": "string"}, - "remote-asn": {"type": "integer"}, - "local-asn": {"type": "integer"}, - "hostname": {"type": "string"} - }, - # lots of internal peerings - so maybe no explicit asn's - "required": ["group", "address"], - "additionalProperties": False - }, - "instance-peering": { - "type": "object", - "properties": { - "instance": {"type": "string"}, - "group": {"type": "string"}, - "description": {"type": "string"}, - "address": {"type": "string"}, - "remote-asn": {"type": "integer"}, - "local-asn": {"type": "integer"}, - "hostname": {"type": "string"} - }, - # description and-or local-asn is not always present, - # just based on empirical tests - not a problem - "required": ["instance", "group", "address", "remote-asn"], - "additionalProperties": False - }, - "logical-system-peering": { - "type": "object", - "properties": { - "logical-system": {"type": "string"}, - "group": {"type": "string"}, - "description": {"type": "string"}, - "address": {"type": "string"}, - "remote-asn": {"type": "integer"}, - "local-asn": {"type": "integer"}, - "hostname": {"type": "string"} - }, - # local/remote-asn and/or description are not always present, - # just based on empirical tests - not a problem - "required": ["logical-system", "group", "address"], - "additionalProperties": False - }, + "top-level-peering": TOP_LEVEL_PEERING_SCHEMA, + "instance-peering": INSTANCE_PEERING, + "logical-system-peering": LOGICAL_SYSTEM_PEERING_SCHEMA, "peering": { "oneOf": [ {"$ref": "#/definitions/top-level-peering"}, @@ -154,6 +161,8 @@ def test_build_juniper_peering_db(mocked_worker_module): db = backend_db() # also forces initialization + # remove the juniper-peerings:* items that + # will be created by _build_juniper_peering_db def _x(k): if not k.startswith('juniper-peerings'): return False @@ -167,6 +176,8 @@ def test_build_juniper_peering_db(mocked_worker_module): worker._build_juniper_peering_db() found_record = False + found_logical_system = False + found_group = False for key, value in db.items(): if not _x(key): @@ -183,9 +194,23 @@ def test_build_juniper_peering_db(mocked_worker_module): assert address == canonical continue - jsonschema.validate(value, PEERING_LIST_SCHEMA) + jsonschema.validate(value, DETAILED_PEERING_LIST_SCHEMA) + + if 'logical-system:' in key: + jsonschema.validate(value, msr.PEERING_LIST_SCHEMA) + m = re.match(r'.*logical-system:(.+)$', key) + assert all(p['logical-system'] == m.group(1) for p in value) + found_logical_system = True + + if 'group:' in key: + jsonschema.validate(value, msr.PEERING_LIST_SCHEMA) + m = re.match(r'.*group:(.+)$', key) + assert all(p['group'] == m.group(1) for p in value) + found_group = True assert found_record + assert found_logical_system + assert found_group def test_build_snmp_peering_db(mocked_worker_module):