Skip to content
Snippets Groups Projects
Commit 4bcafeb6 authored by Erik Reid's avatar Erik Reid
Browse files

updated endpoint to return pops instead of equipment for services

parent 725960bc
No related branches found
No related tags found
No related merge requests found
...@@ -11,69 +11,64 @@ from mapping_provider.backends import services ...@@ -11,69 +11,64 @@ from mapping_provider.backends import services
router = APIRouter() router = APIRouter()
class Pop(BaseModel):
class Site(BaseModel): latitude: float | None
latitude: float longitude: float | None
longitude: float
name: str name: str
abbreviation: str
city: str
country: str
@classmethod
def from_inprov_site(cls, site: dict[str, Any]) -> 'Site':
return cls(
latitude=site['latitude'],
longitude=site['longitude'],
name=site['name']
)
class SiteList(BaseModel): class PopList(BaseModel):
sites: list[Site] pops: list[Pop]
class Router(BaseModel):
fqdn: str
site: str
@classmethod class Equipment(BaseModel):
def from_inprov_router(cls, router: dict[str, Any]) -> 'Router': name: str
return cls( pop: str
fqdn = router['fqdn'], status: str
site = router['site']
)
class RouterList(BaseModel):
routers: list[Router]
class EquipmentList(BaseModel):
equipment: list[Equipment]
INPROV_SITE_LIST_SCHEMA = {
INPROV_POP_LIST_SCHEMA = {
'$schema': 'https://json-schema.org/draft/2020-12/schema', '$schema': 'https://json-schema.org/draft/2020-12/schema',
'definitions': { 'definitions': {
'site': { 'pop': {
'type': 'object', 'type': 'object',
'properties': { 'properties': {
'name': {'type': 'string'}, 'name': {'type': 'string'},
'latitude': {'type': 'number'}, 'abbreviation': {'type': 'string'},
'longitude': {'type': 'number'}, 'city': {'type': 'string'},
'country': {'type': 'string'},
'latitude': {'type': ['number', 'null']},
'longitude': {'type': ['number', 'null']},
}, },
'required': ['name', 'latitude', 'longitude'], 'required': ['name', 'abbreviation', 'city', 'country', 'latitude', 'longitude'],
'additionalProperties': True, 'additionalProperties': True,
}, },
}, },
'type': 'array', 'type': 'array',
'items': {'$ref': '#/definitions/site'} 'items': {'$ref': '#/definitions/pop'}
} }
INPROV_ROUTER_LIST_SCHEMA = { INPROV_EQUIPMENT_LIST_SCHEMA = {
'$schema': 'https://json-schema.org/draft/2020-12/schema', '$schema': 'https://json-schema.org/draft/2020-12/schema',
'definitions': { 'definitions': {
'router': { 'router': {
'type': 'object', 'type': 'object',
'properties': { 'properties': {
'fqdn': {'type': 'string'}, 'name': {'type': 'string'},
'site': {'type': 'string'}, 'pop': {'type': 'string'},
'status': {'type': 'string'},
}, },
'required': ['fqdn', 'site'], 'required': ['name', 'pop', 'status'],
}, },
}, },
...@@ -81,81 +76,94 @@ INPROV_ROUTER_LIST_SCHEMA = { ...@@ -81,81 +76,94 @@ INPROV_ROUTER_LIST_SCHEMA = {
'items': {'$ref': '#/definitions/router'} 'items': {'$ref': '#/definitions/router'}
} }
INPROV_SERVICE_LIST_SCHEMA = { # INPROV_SERVICE_LIST_SCHEMA = {
'$schema': 'https://json-schema.org/draft/2020-12/schema', # '$schema': 'https://json-schema.org/draft/2020-12/schema',
'definitions': { # 'definitions': {
'endpoint': { # 'endpoint': {
'type': 'object', # 'type': 'object',
'properties': { # 'properties': {
'hostname': {'type': 'string'}, # 'hostname': {'type': 'string'},
'interface': {'type': 'string'}, # 'interface': {'type': 'string'},
}, # },
}, # },
'service': { # 'service': {
'type': 'object', # 'type': 'object',
'properties': { # 'properties': {
'sid': {'type': 'string'}, # 'sid': {'type': 'string'},
'name': {'type': 'string'}, # 'name': {'type': 'string'},
'type': {'type': 'string'}, # 'type': {'type': 'string'},
'endpoints': { # 'endpoints': {
'type': 'array', # 'type': 'array',
'items': {'$ref': '#/definitions/endpoint'}, # 'items': {'$ref': '#/definitions/endpoint'},
'minItems': 1, # 'minItems': 1,
}, # },
'overlays': { # 'overlays': {
'type': 'object', 'properties': { # 'type': 'object', 'properties': {
'speed': {'type': 'number'}, # 'speed': {'type': 'number'},
}, # },
'required': ['speed'], # 'required': ['speed'],
}, # },
}, # },
'required': ['sid', 'name', 'type', 'endpoints', 'overlays'], # 'required': ['sid', 'name', 'type', 'endpoints', 'overlays'],
}, # },
}, # },
'type': 'array', # 'type': 'array',
'items': {'$ref': '#/definitions/service'} # 'items': {'$ref': '#/definitions/service'}
} # }
INPROV_API_URL_TODO = 'https://test-inprov01.geant.org' INPROV_API_URL_TODO = 'https://test-inprov01.geant.org'
@router.get("/sites")
def get_sites() -> SiteList: @router.get("/pops")
def get_pops() -> PopList:
""" """
handler for /sites handler for /pops
""" """
# TODO: catch/handle the usual exceptions # TODO: catch/handle the usual exceptions
app_params = config.load() app_params = config.load()
rv = requests.get( rv = requests.get(
f'{app_params.inventory}/map/sites', f'{app_params.inventory}/map/pops',
headers={'Accept': 'application/json'}) headers={'Accept': 'application/json'})
rv.raise_for_status() rv.raise_for_status()
site_list_json = rv.json() pop_list_obj = rv.json()
jsonschema.validate(site_list_json, INPROV_SITE_LIST_SCHEMA) jsonschema.validate(pop_list_obj, INPROV_POP_LIST_SCHEMA)
rsp_sites = map(Site.from_inprov_site, site_list_json) def _make_pop(pop_dict: dict[str, Any]) -> Pop:
return SiteList(sites=list(rsp_sites)) return Pop(
latitude=pop_dict['latitude'],
longitude=pop_dict['longitude'],
name=pop_dict['name'],
abbreviation=pop_dict['abbreviation'],
city=pop_dict['city'],
country=pop_dict['country'],
)
return PopList(pops=map(_make_pop, pop_list_obj))
@router.get("/routers") @router.get("/equipment")
def get_routers() -> RouterList: def get_equipment() -> EquipmentList:
""" """
handler for /sites handler for /equipment
""" """
# TODO: catch/handle the usual exceptions # TODO: catch/handle the usual exceptions
app_params = config.load() app_params = config.load()
rv = requests.get( rv = requests.get(
f'{app_params.inventory}/map/routers', f'{app_params.inventory}/map/equipment',
headers={'Accept': 'application/json'}) headers={'Accept': 'application/json'})
rv.raise_for_status() rv.raise_for_status()
router_list_json = rv.json() equipment_list_obj = rv.json()
jsonschema.validate(router_list_json, INPROV_ROUTER_LIST_SCHEMA) jsonschema.validate(equipment_list_obj, INPROV_EQUIPMENT_LIST_SCHEMA)
rsp_routers = map(Router.from_inprov_router, router_list_json) def _make_equipment(equipment_dict: dict[str, Any]) -> Equipment:
return RouterList(routers=list(rsp_routers)) return Equipment(
name=equipment_dict['name'],
pop=equipment_dict['pop'],
status=equipment_dict['status'],
)
return EquipmentList(equipment=map(_make_equipment, equipment_list_obj))
@router.get("/trunks") @router.get("/trunks")
......
...@@ -25,24 +25,22 @@ REPORTING_SCID_CURRENT_CACHE_SCHEMA = { ...@@ -25,24 +25,22 @@ REPORTING_SCID_CURRENT_CACHE_SCHEMA = {
'type': 'object', 'type': 'object',
'properties': { 'properties': {
'hostname': {'type': 'string'}, 'hostname': {'type': 'string'},
# 'interface': {'type': 'string'}, 'interface': {'type': 'string'},
# 'addresses': { # 'addresses': {
# 'type': 'array', # 'type': 'array',
# 'items': {'type': 'string'} # 'items': {'type': 'string'}
# } # }
}, },
'required': ['hostname'] 'required': ['hostname', 'interface']
# 'required': ['hostname', 'interface']
}, },
'lambda_interface': { 'lambda_interface': {
'type': 'object', 'type': 'object',
'properties': { 'properties': {
'equipment': {'type': 'string'}, 'equipment': {'type': 'string'},
# 'port': {'type': 'string'}, 'port': {'type': 'string'},
}, },
'required': ['equipment'] 'required': ['equipment', 'port']
# 'required': ['equipment', 'port']
}, },
'service': { 'service': {
......
import functools
import logging import logging
from collections.abc import Generator from collections.abc import Generator
import re
from typing import Any
from pydantic import BaseModel from pydantic import BaseModel
...@@ -8,11 +11,6 @@ from . import brian, cache, correlator, inventory ...@@ -8,11 +11,6 @@ from . import brian, cache, correlator, inventory
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
class Endpoint(BaseModel):
equipment: str
interface: str
class BitRates(BaseModel): class BitRates(BaseModel):
ingress: float | None ingress: float | None
egress: float | None egress: float | None
...@@ -31,13 +29,39 @@ class Service(BaseModel): ...@@ -31,13 +29,39 @@ class Service(BaseModel):
scid: str scid: str
name: str name: str
type: str type: str
endpoints: list[Endpoint] pops: list[str]
overlays: Overlays overlays: Overlays
class ServiceList(BaseModel): class ServiceList(BaseModel):
services: list[Service] services: list[Service]
def endpoint_to_pop(endpoint: dict[str, Any], inprov_equipment_dict: list[dict[str, str]]) -> str:
def _hostname_to_equipment(_hn):
m = re.match(r'^(.+)\.geant\.net$', _hn)
if not m:
logger.error(f'unexpected hostname pattern: {_hn}')
return '?'
return m.group(1).upper()
if 'hostname' in endpoint:
eq_name = _hostname_to_equipment(endpoint['hostname'])
elif 'equipment' in endpoint:
eq_name = endpoint['equipment']
else:
# should already be validated
raise AssertionError(f'no equipment or hostname in endpoint: {endpoint}')
if eq_name not in inprov_equipment_dict:
# TODO: is this really possible if all data is read from IMS at the same time?
logger.error(f'unknown endpoint equipment: {eq_name}')
return '?'
return inprov_equipment_dict[eq_name]['pop']
def _services(service_type: str | None = None) -> Generator[Service]: def _services(service_type: str | None = None) -> Generator[Service]:
""" """
...@@ -52,6 +76,7 @@ def _services(service_type: str | None = None) -> Generator[Service]: ...@@ -52,6 +76,7 @@ def _services(service_type: str | None = None) -> Generator[Service]:
# poller_interfaces = cache.get(inventory.INPROV_POLLER_INTERFACES_CACHE_FILENAME) # poller_interfaces = cache.get(inventory.INPROV_POLLER_INTERFACES_CACHE_FILENAME)
correlator_state = cache.get(correlator.CACHED_CORRELATOR_STATE_FILENAME) correlator_state = cache.get(correlator.CACHED_CORRELATOR_STATE_FILENAME)
brian_rates = cache.get(brian.CACHED_BRIAN_SCID_RATES_FILENAME) brian_rates = cache.get(brian.CACHED_BRIAN_SCID_RATES_FILENAME)
equipment_list = cache.get(inventory.INPROV_EQUIPMENT_CACHE_FILENAME)
except FileNotFoundError: except FileNotFoundError:
logger.exception('not enough data available to build the service list') logger.exception('not enough data available to build the service list')
return return
...@@ -68,6 +93,9 @@ def _services(service_type: str | None = None) -> Generator[Service]: ...@@ -68,6 +93,9 @@ def _services(service_type: str | None = None) -> Generator[Service]:
down_service_sids = set(_get_down_correlator_services()) down_service_sids = set(_get_down_correlator_services())
brian_scid_rates = {r['scid']: r['values'] for r in brian_rates} brian_scid_rates = {r['scid']: r['values'] for r in brian_rates}
equipment_dict = {_x['name']: _x for _x in equipment_list}
_endpoint_to_pop = functools.partial(endpoint_to_pop, inprov_equipment_dict=equipment_dict)
for _s in scid_current: for _s in scid_current:
if _s['status'] != 'operational': if _s['status'] != 'operational':
...@@ -76,6 +104,8 @@ def _services(service_type: str | None = None) -> Generator[Service]: ...@@ -76,6 +104,8 @@ def _services(service_type: str | None = None) -> Generator[Service]:
if service_type and _s['service_type'] != service_type: if service_type and _s['service_type'] != service_type:
continue continue
pops = sorted(set(map(_endpoint_to_pop, _s['endpoints'])))
rates = brian_scid_rates.get(_s['scid'], {}) rates = brian_scid_rates.get(_s['scid'], {})
overlays = Overlays( overlays = Overlays(
speed = _s['speed'], speed = _s['speed'],
...@@ -94,18 +124,12 @@ def _services(service_type: str | None = None) -> Generator[Service]: ...@@ -94,18 +124,12 @@ def _services(service_type: str | None = None) -> Generator[Service]:
), ),
) )
endpoints = []
for _e in _s['endpoints']:
equipment = _e['hostname'] if 'hostname' in _e else _e['equipment']
interface = _e['interface'] if 'interface' in _e else _e['port']
endpoints.append(Endpoint(equipment=equipment, interface=interface))
yield Service( yield Service(
sid = _s['sid'], sid = _s['sid'],
scid = _s['scid'], scid = _s['scid'],
name = _s['name'], name = _s['name'],
type = _s['service_type'], type = _s['service_type'],
endpoints = endpoints, pops = pops,
overlays = overlays, overlays = overlays,
) )
......
...@@ -58,9 +58,10 @@ def client(dummy_config_filename): ...@@ -58,9 +58,10 @@ def client(dummy_config_filename):
# there's no rmq in the test config data, so cache won't be initialized # there's no rmq in the test config data, so cache won't be initialized
cache.init(tmp_dir) cache.init(tmp_dir)
cache.set(inventory.INPROV_MAP_SERVICES_CACHE_FILENAME, load_test_data('inprov-services.json')) # cache.set(inventory.INPROV_MAP_SERVICES_CACHE_FILENAME, load_test_data('inprov-services.json'))
cache.set(inventory.REPORTING_SCID_CURRENT_CACHE_FILENAME, load_test_data('scid-current.json')) cache.set(inventory.REPORTING_SCID_CURRENT_CACHE_FILENAME, load_test_data('scid-current.json'))
cache.set(inventory.INPROV_POLLER_INTERFACES_CACHE_FILENAME, load_test_data('poller-interfaces.json')) cache.set(inventory.INPROV_EQUIPMENT_CACHE_FILENAME, load_test_data('inprov-equipment.json'))
# cache.set(inventory.INPROV_POLLER_INTERFACES_CACHE_FILENAME, load_test_data('poller-interfaces.json'))
cache.set(correlator.CACHED_CORRELATOR_STATE_FILENAME, load_test_data('correlator-state.json')) cache.set(correlator.CACHED_CORRELATOR_STATE_FILENAME, load_test_data('correlator-state.json'))
cache.set(brian.CACHED_BRIAN_SCID_RATES_FILENAME, load_test_data('brian-scid-rates.json')) cache.set(brian.CACHED_BRIAN_SCID_RATES_FILENAME, load_test_data('brian-scid-rates.json'))
......
...@@ -36,8 +36,8 @@ def test_inventory_service_download(): ...@@ -36,8 +36,8 @@ def test_inventory_service_download():
cache.init(tmp_dir) cache.init(tmp_dir)
inventory._load_all_inventory( inventory._load_all_inventory(
inventory_base_uri='https://dummy-hostname.dummy.domain', inventory_base_uri=inventory_base_uri,
reporting_base_uri='https://another-dummy-hostname.dummy.domain') reporting_base_uri=reporting_base_uri)
# assert os.path.exists(os.path.join(tmp_dir, services.POLLER_INTERFACES_CACHE_FILENAME)) # assert os.path.exists(os.path.join(tmp_dir, services.POLLER_INTERFACES_CACHE_FILENAME))
......
...@@ -2,40 +2,41 @@ import re ...@@ -2,40 +2,41 @@ import re
import responses import responses
from mapping_provider.api.map import RouterList, SiteList from mapping_provider.api.map import EquipmentList, PopList
from mapping_provider.backends.services import ServiceList from mapping_provider.backends.services import ServiceList
from .common import load_test_data from .common import load_test_data
@responses.activate @responses.activate
def test_get_sites(client): def test_get_pops(client):
responses.add( responses.add(
method=responses.GET, method=responses.GET,
url=re.compile(r'.*/map/sites$'), url=re.compile(r'.*/map/pops$'),
json=load_test_data('inprov-sites.json') json=load_test_data('inprov-pops.json')
) )
rv = client.get("/map/sites") rv = client.get("/map/pops")
assert rv.status_code == 200 assert rv.status_code == 200
site_list = SiteList.model_validate(rv.json()) pop_list = PopList.model_validate(rv.json())
assert site_list.sites, 'test data should not be empty' assert pop_list.pops, 'test data should not be empty'
@responses.activate @responses.activate
def test_get_routers(client): def test_get_equipment(client):
responses.add( responses.add(
method=responses.GET, method=responses.GET,
url=re.compile(r'.*/map/routers$'), url=re.compile(r'.*/map/equipment$'),
json=load_test_data('inprov-routers.json') json=load_test_data('inprov-equipment.json')
) )
rv = client.get("/map/routers") rv = client.get("/map/equipment")
assert rv.status_code == 200 assert rv.status_code == 200
router_list = RouterList.model_validate(rv.json()) equipment_list = EquipmentList.model_validate(rv.json())
assert router_list.routers, 'test data should not be empty' assert equipment_list.equipment, 'test data should not be empty'
@responses.activate @responses.activate
def test_get_trunks(client): def test_get_trunks(client):
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment