diff --git a/inventory_provider/routes/msr.py b/inventory_provider/routes/msr.py index 1a21b7f38d30b425c3613e400d2d46a88c87a914..942f400f92d2f4b0b83db3f01ac4207f0f2c8f66 100644 --- a/inventory_provider/routes/msr.py +++ b/inventory_provider/routes/msr.py @@ -62,6 +62,7 @@ helpers .. autofunction:: inventory_provider.routes.msr._handle_peering_group_request """ # noqa E501 +import functools import itertools import json import ipaddress @@ -83,6 +84,7 @@ from inventory_provider.tasks import common as tasks_common routes = Blueprint('msr-query-routes', __name__) logger = logging.getLogger(__name__) +_subnet_lookup_semaphore = threading.Semaphore() PEERING_GROUP_LIST_SCHEMA = { '$schema': 'http://json-schema.org/draft-07/schema#', @@ -434,15 +436,41 @@ def _find_subnet_keys(addresses): continue +@functools.lru_cache(100) +def _get_subnets(r): + result = {} + for k in r.scan_iter('subnets:*', count=1000): + k = k.decode('utf-8') + m = re.match(r'^subnets:(.+)$', k) + assert m + result[k] = ipaddress.ip_interface(m.group(1)).network + return result + + def _get_subnet_interfaces(address, r): - exploded = ipaddress.ip_address(address).exploded - for k in r.scan_iter(f'subnets:{exploded}/*', count=1000): - value = r.get(k.decode('utf-8')) + + # synchronize calls to _get_subnets, so we don't + # call it many times together when running in + # multi-thread mode + _subnet_lookup_semaphore.acquire() + try: + all_subnets = _get_subnets(r) + except: + logger.exception('error looking up subnets') + all_subnets = {} + finally: + _subnet_lookup_semaphore.release() + + address = ipaddress.ip_address(address) + for key, network in all_subnets.items(): + if address not in network: + continue + value = r.get(key) if not value: - return None - value = value.decode('utf-8') - return json.loads(value) - return [] + logger.error(f'no value for for redis key "{key}"') + continue + + yield from json.loads(value.decode('utf-8')) def _get_services_for_address(address: str, r: 'StrictRedis'): diff --git a/test/test_msr_routes.py b/test/test_msr_routes.py index c2f213200957200120d744292b10bf9543576d0f..ba10ce0fbe404a78c29985baf96179dc986cdf92 100644 --- a/test/test_msr_routes.py +++ b/test/test_msr_routes.py @@ -140,8 +140,16 @@ def test_peerings_group_list(client, uri): @pytest.mark.parametrize('address', [ - '62.40.127.141', - '62.40.127.139' + # hand-chosen, should have services + '62.40.127.141', # from sample peering outage response + '62.40.127.139', # from sample peering outage response + '62.40.98.201', # AMS-AMS IP TRUNK (mx1.ams/ae0.1) + '62.40.127.134', # BELNET AP3 (mx1.ams/ae13.2) + '62.40.124.38', # SURF AP1 (mx1.ams/ae15.1103) + '62.40.125.57', # JISC AP2 (mx1.lon2/ae12.0) + '2001:0798:0099:0001:0000:0000:0000:0026', # v6 peering with Internet2 + '2001:0798:0099:0001:0000:0000:0000:0056', # v6 peering with Canarie + '2001:0798:0018:10aa:0000:0000:0000:0016', # v6 peering with HEANET ]) def test_lookup_services_for_address(address, mocked_redis): _redis_instance = _get_redis({ @@ -156,7 +164,8 @@ def test_lookup_services_for_address(address, mocked_redis): jsonschema.validate(info, PEERING_ADDRESS_SERVICES_LIST) # sanity check to be sure we have interesting test data - assert all(x['services'] for x in info) + assert info + assert any(x['services'] for x in info) _OUTAGE_PEER_ADDRESSES = [