diff --git a/Changelog.md b/Changelog.md
index 33899bee32a7cb71760225be423e7dafa94e4880..db9c93808bcda05ecc040fd654f472d50222144a 100644
--- a/Changelog.md
+++ b/Changelog.md
@@ -2,6 +2,9 @@
All notable changes to this project will be documented in this file.
+## [0.145] - 2025-05-27
+- NGM-15/19/20: added /map/pops & /map/equipment
+
## [0.145] - 2025-05-20
- DBOARD3-1238: Updated OID for Juniper Active State Checking
diff --git a/inventory_provider/__init__.py b/inventory_provider/__init__.py
index 3155d6cce97b98ca09e7942da6806f23292a2181..fb6aa000e7510036de79c6f532c68192a4f62a1f 100644
--- a/inventory_provider/__init__.py
+++ b/inventory_provider/__init__.py
@@ -93,6 +93,9 @@ def create_app(setup_logging=True):
from inventory_provider.routes import state_checker
app.register_blueprint(state_checker.routes, url_prefix='/state-checker')
+ from inventory_provider.routes import map
+ app.register_blueprint(map.routes, url_prefix='/map')
+
if app.config.get('ENABLE_TESTING_ROUTES', False):
from inventory_provider.routes import testing
app.register_blueprint(testing.routes, url_prefix='/testing')
diff --git a/inventory_provider/gap.py b/inventory_provider/gap.py
index 0a7b1de811f712cf2051029ea97fc59406ac472c..7c5f72e745f0a3579b81e08452f628e7a720b932 100644
--- a/inventory_provider/gap.py
+++ b/inventory_provider/gap.py
@@ -75,20 +75,7 @@ def make_request(body: dict, token: str, app_config: dict) -> Dict:
return response.json()
-def extract_router_info(device: dict, token: str, app_config: dict) -> Optional[dict]:
- tag_to_key_map = {
- "RTR": "router",
- "OFFICE_ROUTER": "officeRouter",
- "Super_POP_SWITCH": "superPopSwitch"
- }
-
- tag = device.get("product", {}).get("tag")
- key = tag_to_key_map.get(tag)
- subscription_id = device.get("subscriptionId")
-
- if key is None or subscription_id is None:
- logger.warning(f"Skipping device with invalid tag or subscription ID: {device}")
- return None
+def load_subscription_product_blocks(subscription_id: str, token: str, app_config: dict) -> dict:
query = f"""
query {{
@@ -109,25 +96,107 @@ def extract_router_info(device: dict, token: str, app_config: dict) -> Optional[
page_data = response.get('data', {}).get('subscriptions', {}).get('page')
if not page_data:
+ return {}
+
+ def _product_block_instances():
+ for _page_chunk in page_data:
+ yield from _page_chunk.get('productBlockInstances')
+ # yield from _page_chunk.get('productBlockInstances', [{}])
+
+ def _product_block_items():
+
+ for _pb_instance in _product_block_instances():
+ _pb_instance_values = _pb_instance.get('productBlockInstanceValues')
+ # wfo format sanity check ...
+ assert len(_pb_instance_values) >= 2, \
+ f'Expected at least 2 items in product block instance, got {len(_pb_instance_values)}'
+ _iterator = iter(_pb_instance_values)
+ name_item = next(_iterator)
+ assert name_item.get('field') == 'name'
+
+ # unused, but sanity check ...
+ _unused_label_item = next(_iterator)
+ assert _unused_label_item.get('field') == 'label'
+
+ value_dict = {_it.get('field'): _it.get('value') for _it in _iterator}
+ yield name_item.get('value'), value_dict
+
+ return dict(_product_block_items())
+
+
+def extract_site_info(subscription_id: str, token: str, app_config: dict) -> Optional[dict]:
+ product_blocks = load_subscription_product_blocks(
+ subscription_id=subscription_id,
+ token=token,
+ app_config=app_config)
+
+ if not product_blocks:
logger.warning(f"No data for subscription ID: {subscription_id}")
+ return {}
+
+ if not product_blocks:
+ logger.warning(f"No data for subscription ID: {subscription_id}")
+ return {}
+
+ def _find_pb_value(field):
+ for _pb_values in product_blocks.values():
+ if field in _pb_values:
+ return _pb_values[field]
return None
- instance_values = page_data[0].get('productBlockInstances', [{}])[0].get('productBlockInstanceValues', [])
+ return {
+ 'city': _find_pb_value('siteCity'),
+ 'country': _find_pb_value('siteCountry'),
+ 'country_code': _find_pb_value('siteCountryCode'),
+ 'name': _find_pb_value('siteName'),
+ 'longitude': float(_find_pb_value('siteLongitude')),
+ 'latitude': float(_find_pb_value('siteLatitude')),
+ 'tier': int(_find_pb_value('siteTier')),
+ }
+
- fqdn = next((item.get('value') for item in instance_values if item.get('field') == f'{key}Fqdn'), None)
- vendor = next((item.get('value') for item in instance_values if item.get('field') == 'vendor'), None)
+def extract_router_info(device_subscription: dict, token: str, app_config: dict) -> Optional[dict]:
+ tag_to_key_map = {
+ "RTR": "router",
+ "OFFICE_ROUTER": "officeRouter",
+ "Super_POP_SWITCH": "superPopSwitch"
+ }
- if fqdn and vendor:
- return {'fqdn': fqdn, 'vendor': vendor}
+ tag = device_subscription.get("product", {}).get("tag")
+ key = tag_to_key_map.get(tag)
+ subscription_id = device_subscription.get("subscriptionId")
+
+ if key is None or subscription_id is None:
+ logger.warning(f"Skipping device with invalid tag or subscription ID: {device_subscription}")
+ return None
+
+ product_blocks = load_subscription_product_blocks(
+ subscription_id=subscription_id,
+ token=token,
+ app_config=app_config)
+
+ if not product_blocks:
+ logger.warning(f"No data for subscription ID: {subscription_id}")
+ return {}
+
+ def _find_pb_value(field):
+ for _pb_values in product_blocks.values():
+ if field in _pb_values:
+ return _pb_values[field]
+ return None
+
+ fqdn = _find_pb_value(f'{key}Fqdn',)
+ vendor = _find_pb_value('vendor')
+ site = _find_pb_value('siteName')
+
+ if fqdn and vendor and site:
+ return {'fqdn': fqdn, 'vendor': vendor, 'site': site}
else:
- logger.warning(f"Skipping device with missing FQDN or vendor: {device}")
+ logger.warning(f"Skipping device with missing FQDN or vendor: {device_subscription}")
return None
-def load_routers_from_orchestrator(app_config: dict) -> Dict:
- """Gets devices from the orchestrator and returns a dictionary of FQDNs and vendors."""
- token = get_token(app_config['aai'])
- routers = {}
+def _retrieve_subscriptions(token: str, app_config: dict, filter_by: str, subscription_query: str):
end_cursor = 0
has_next_page = True
@@ -135,10 +204,10 @@ def load_routers_from_orchestrator(app_config: dict) -> Dict:
query = f"""
{{
subscriptions(
- filterBy: {{field: "status", value: "PROVISIONING|ACTIVE"}},
+ filterBy: {filter_by},
first: 100,
after: {end_cursor},
- query: "tag:(RTR|OFFICE_ROUTER|Super_POP_SWITCH)"
+ query: "{subscription_query}"
) {{
pageInfo {{
hasNextPage
@@ -164,11 +233,60 @@ def load_routers_from_orchestrator(app_config: dict) -> Dict:
devices = []
has_next_page = False
- with concurrent.futures.ThreadPoolExecutor() as executor:
- futures = [executor.submit(extract_router_info, device, token, app_config) for device in devices]
- for future in concurrent.futures.as_completed(futures):
- router_info = future.result()
- if router_info is not None:
- routers[router_info['fqdn']] = router_info['vendor']
+ yield from devices
+
+
+def load_sites_from_orchestrator(app_config: dict) -> Dict:
+ """Gets sites from the orchestrator and returns a dictionary of FQDNs and vendors."""
+ token = get_token(app_config['aai'])
+ subscriptions = _retrieve_subscriptions(
+ token=token,
+ app_config=app_config,
+ filter_by='{field: "status", value: "PROVISIONING|ACTIVE"}',
+ subscription_query="tag:SITE")
+
+ sites = []
+ with concurrent.futures.ThreadPoolExecutor() as executor:
+
+ futures = [
+ executor.submit(
+ extract_site_info,
+ subscription.get("subscriptionId"),
+ token,
+ app_config)
+ for subscription in subscriptions]
+
+ for future in concurrent.futures.as_completed(futures):
+ site_info = future.result()
+ if site_info is not None:
+ sites.append(site_info)
+ # routers[router_info['fqdn']] = router_info['vendor']
+
+ return sites
+
+
+def load_routers_from_orchestrator(app_config: dict) -> Dict:
+ """Gets devices from the orchestrator and returns a dictionary of FQDNs and vendors."""
+
+ token = get_token(app_config['aai'])
+
+ subscriptions = _retrieve_subscriptions(
+ token=token,
+ app_config=app_config,
+ filter_by='{field: "status", value: "PROVISIONING|ACTIVE"}',
+ subscription_query="tag:(RTR|OFFICE_ROUTER|Super_POP_SWITCH)")
+ routers = []
+ with concurrent.futures.ThreadPoolExecutor() as executor:
+ futures = [
+ executor.submit(
+ extract_router_info,
+ subscription,
+ token,
+ app_config)
+ for subscription in subscriptions]
+ for future in concurrent.futures.as_completed(futures):
+ router_info = future.result()
+ if router_info is not None:
+ routers.append(router_info)
return routers
diff --git a/inventory_provider/routes/__init__.py b/inventory_provider/routes/__init__.py
index 4cd55dfe6fb4089cb72a0b3ebed1c07d0c7eddb3..bbf6f098e71c4a5e4e62f9f4857e2e9709c06214 100644
--- a/inventory_provider/routes/__init__.py
+++ b/inventory_provider/routes/__init__.py
@@ -36,4 +36,5 @@ and www.json.org for more details.
.. automodule:: inventory_provider.routes.neteng
+.. automodule:: inventory_provider.routes.map
"""
diff --git a/inventory_provider/routes/classifier.py b/inventory_provider/routes/classifier.py
index f0e28417e062be97855b859d86b07dbc4b2c1c12..e22c923f4df871f9264136fe8eed3af2a4be140f 100644
--- a/inventory_provider/routes/classifier.py
+++ b/inventory_provider/routes/classifier.py
@@ -1224,12 +1224,12 @@ def get_all_routers() -> Response:
def _get_router_list(redis):
- all_routers_raw = redis.get("netdash")
+ all_routers_raw = redis.get("gso:routers")
all_routers = json.loads(all_routers_raw) if all_routers_raw else {}
return [
- {"hostname": hostname, "vendor": vendor}
- for hostname, vendor in all_routers.items()
- if get_ims_equipment_name_or_none(hostname)
+ {"hostname": _r['fqdn'], "vendor": _r['vendor']}
+ for _r in all_routers
+ if get_ims_equipment_name_or_none(_r['fqdn'])
]
@@ -1266,11 +1266,11 @@ def _get_router_info(ims_source_equipment, redis):
contacts = set()
vendor = "unknown"
- all_routers = redis.get("netdash")
+ all_routers = redis.get("gso:routers")
- for name, v in (json.loads(all_routers) if all_routers else {}).items():
- if name.startswith(ims_source_equipment.lower()):
- vendor = v
+ for _r in (json.loads(all_routers) if all_routers else {}):
+ if _r['fqdn'].startswith(ims_source_equipment.lower()):
+ vendor = _r['vendor']
break
for key in redis.scan_iter(f"ims:interface_services:{ims_source_equipment}:*"):
diff --git a/inventory_provider/routes/map.py b/inventory_provider/routes/map.py
new file mode 100644
index 0000000000000000000000000000000000000000..ad4a3821ee987c4218be72c2e0ccea43289f5d03
--- /dev/null
+++ b/inventory_provider/routes/map.py
@@ -0,0 +1,115 @@
+"""
+mapping support endpoints
+=========================
+
+These endpoints are intended for use by the mapping-provider for fetching
+static inventory information relevant to maps.
+
+.. contents:: :local:
+
+/map/pops
+---------------------------------
+
+.. autofunction:: inventory_provider.routes.map.pops
+
+
+/map/equipment
+---------------------------------
+
+.. autofunction:: inventory_provider.routes.map.equipment
+
+
+"""
+import json
+import logging
+
+from flask import Blueprint, jsonify
+import jsonschema
+
+from inventory_provider.routes import common
+
+logger = logging.getLogger(__name__)
+routes = Blueprint('map-support-routes', __name__)
+
+EQUIPMENT_LIST_SCHEMA = {
+ '$schema': 'https://json-schema.org/draft/2020-12/schema',
+
+ 'definitions': {
+ 'equipment': {
+ 'type': 'object',
+ 'properties': {
+ 'name': {'type': 'string'},
+ 'pop': {'type': 'string'},
+ 'status': {'type': 'string'},
+ },
+ 'required': ['name', 'pop', 'status'],
+ 'additionalProperties': False
+ },
+ },
+
+ 'type': 'array',
+ 'items': {'$ref': '#/definitions/equipment'}
+}
+
+POP_LIST_SCHEMA = {
+ '$schema': 'https://json-schema.org/draft/2020-12/schema',
+
+ 'definitions': {
+ 'pop': {
+ 'type': 'object',
+ 'properties': {
+ 'name': {'type': 'string'},
+ 'abbreviation': {'type': 'string'},
+ 'city': {'type': 'string'},
+ 'country': {'type': 'string'},
+ 'latitude': {'type': ['number', 'null']},
+ 'longitude': {'type': ['number', 'null']},
+ },
+ 'required': ['city', 'country', 'name', 'abbreviation', 'latitude', 'longitude'],
+ 'additionalProperties': False
+ },
+ },
+
+ 'type': 'array',
+ 'items': {'$ref': '#/definitions/pop'}
+}
+
+
+@routes.route("/equipment", methods=['GET'])
+@common.require_accepts_json
+def equipment(service_type=None):
+ """
+ Handler for `/map/equipment`
+ which returns information for all equipment in the IMS inventory
+
+ .. asjson::
+ inventory_provider.routes.map.EQUIPMENT_LIST_SCHEMA
+
+ :return: list of equipment
+ """
+ r = common.get_current_redis()
+ equipment_list = json.loads(r.get('ims:map:equipment').decode('utf-8'))
+ # ensure the internal data is formatted correctly
+ # TODO: handle validation gracefully? or just return an error?
+ jsonschema.validate(equipment_list, EQUIPMENT_LIST_SCHEMA)
+ return jsonify(equipment_list)
+
+
+@routes.route("/pops", methods=['GET'])
+@common.require_accepts_json
+def pops(service_type=None):
+ """
+ Handler for `/map/pops`
+ which returns information for all pops in the IMS inventory
+
+ .. asjson::
+ inventory_provider.routes.map.POP_LIST_SCHEMA
+
+ :return: list of pops
+ """
+ r = common.get_current_redis()
+ pop_list = json.loads(r.get('ims:map:pops').decode('utf-8'))
+ # ensure the internal data is formatted correctly
+ # TODO: handle validation gracefully? or just return an error?
+ jsonschema.validate(pop_list, POP_LIST_SCHEMA)
+ return jsonify(pop_list)
diff --git a/inventory_provider/routes/msr.py b/inventory_provider/routes/msr.py
index 9ccdb3409777c363a8d4d39c091b759390c97a59..2068656cdd9cb6c3e9dd7176c859f3533720b29a 100644
--- a/inventory_provider/routes/msr.py
+++ b/inventory_provider/routes/msr.py
@@ -108,7 +108,7 @@ import threading
from collections import defaultdict
from typing import Dict
-from flask import Blueprint, Response, request, current_app
+from flask import Blueprint, Response, request, current_app, jsonify
import jsonschema
from inventory_provider.routes import common
@@ -1062,23 +1062,12 @@ def _endpoint_extractor(all_interfaces: Dict, endpoint_details: Dict):
}
-@routes.route('/services', methods=['GET'])
-@common.require_accepts_json
-def get_system_correlation_services():
+def load_all_msr_services():
"""
- Handler for `/msr/services`
-
- This method returns all known services with with information required
- by the reporting tool stack.
-
- cf. https://jira.software.geant.org/browse/POL1-530
-
- The response will be formatted as follows:
+ utility method used to construct the response for /msr/services,
+ as well as other endpoints that construct similar responses
- .. asjson::
- inventory_provider.routes.msr.SYSTEM_CORRELATION_SERVICES_LIST_SCHEMA
-
- :return:
+ :return: List of all reportable services
"""
def _get_redundancy_asn(endpoints):
@@ -1099,14 +1088,16 @@ def get_system_correlation_services():
cache_key = 'classifier-cache:msr:services'
r = common.get_current_redis()
- response = _ignore_cache_or_retrieve(request, cache_key, r)
- if not response:
+ all_msr_services = _ignore_cache_or_retrieve(request, cache_key, r)
+ if all_msr_services:
+ all_msr_services = json.loads(all_msr_services)
+ else:
all_interfaces = _load_all_interfaces()
sid_services = json.loads(r.get('ims:sid_services').decode('utf-8'))
- response = []
+ all_msr_services = []
for sid, details in sid_services.items():
service_info = {'endpoints': []}
for d in details:
@@ -1128,14 +1119,39 @@ def get_system_correlation_services():
asn = _get_redundancy_asn(service_info['endpoints'])
if asn:
service_info['redundant_asn'] = asn
- response.append(service_info)
+ all_msr_services.append(service_info)
+ # sanity
jsonschema.validate(
- response, SYSTEM_CORRELATION_SERVICES_LIST_SCHEMA)
+ all_msr_services, SYSTEM_CORRELATION_SERVICES_LIST_SCHEMA)
+
+ if all_msr_services:
+ # expected to always be non-empty
+ r.set(cache_key, json.dumps(all_msr_services).encode('utf-8'))
+
+ return all_msr_services
- if response:
- response = json.dumps(response, indent=2)
- # r.set(cache_key, response.encode('utf-8'))
+
+@routes.route('/services', methods=['GET'])
+@common.require_accepts_json
+def get_system_correlation_services():
+ """
+ Handler for `/msr/services`
+
+ This method returns all known services with with information required
+ by the reporting tool stack.
+
+ cf. https://jira.software.geant.org/browse/POL1-530
+
+ The response will be formatted as follows:
+
+ .. asjson::
+ inventory_provider.routes.msr.SYSTEM_CORRELATION_SERVICES_LIST_SCHEMA
+
+ :return:
+ """
+
+ response = load_all_msr_services()
if not response:
return Response(
@@ -1143,7 +1159,7 @@ def get_system_correlation_services():
status=404,
mimetype="text/html")
- return Response(response, mimetype="application/json")
+ return jsonify(response)
@routes.route('/bgp', methods=['GET'])
diff --git a/inventory_provider/routes/poller.py b/inventory_provider/routes/poller.py
index 99d108f82db80757f7570cd8a7b52d4ec81b74bb..72e6292c8fb4cc5e57938b3820f8e411febbfaca 100644
--- a/inventory_provider/routes/poller.py
+++ b/inventory_provider/routes/poller.py
@@ -1019,20 +1019,17 @@ def interfaces(hostname=None):
return Response(result, mimetype="application/json")
-def get_netdash_equipment(config, use_next_redis=False) -> Dict[str, str]:
- """Get the netdash equipment mapping from redis."""
+def vendor_getter(config, use_next_redis=False):
if use_next_redis:
r = tasks_common.get_next_redis(config)
else:
r = tasks_common.get_current_redis(config)
- return json.loads(r.get('netdash').decode('utf-8'))
-
-def vendor_getter(config, use_next_redis=False):
- netdash_equipment = get_netdash_equipment(config, use_next_redis)
+ managed_router_list = json.loads(r.get('gso:routers').decode('utf-8'))
+ managed_routers = {_r['fqdn']: _r['vendor'] for _r in managed_router_list}
def get_vendor(router: str):
- return netdash_equipment.get(
+ return managed_routers.get(
router, "nokia" if router.startswith("rt0") else "juniper"
)
diff --git a/inventory_provider/routes/testing.py b/inventory_provider/routes/testing.py
index 5fcb7cce47a48eaa86ac4794d098448929ff69e4..5104e66038ce843bd600ebc96db90a4e6460010b 100644
--- a/inventory_provider/routes/testing.py
+++ b/inventory_provider/routes/testing.py
@@ -52,10 +52,10 @@ def get_circuit_hierarchy_and_port_id_services():
def juniper_addresses():
# TODO: this route (and corant, infinera routes) can be removed
r = common.get_current_redis()
- routers = r.get('netdash')
+ routers = r.get('gso:routers')
assert routers # sanity: value shouldn't be empty
routers = json.loads(routers.decode('utf-8'))
- juniper_routers = [k for k, v in routers.items() if v == 'juniper']
+ juniper_routers = [_r['fqdn'] for _r in routers if _r['vendor'] == 'juniper']
return jsonify(juniper_routers)
diff --git a/inventory_provider/tasks/worker.py b/inventory_provider/tasks/worker.py
index 516a88ae59db20fd2629033a288bb7e5eef2fed2..f1b66f176732034956233de1d04d460de0bf7068 100644
--- a/inventory_provider/tasks/worker.py
+++ b/inventory_provider/tasks/worker.py
@@ -6,6 +6,7 @@ import json
import logging
import os
import re
+from typing import Callable
import ncclient.operations
import ncclient.transport.errors
@@ -468,42 +469,56 @@ def update_entry_point(self):
update_latch_status(InventoryTask.config, pending=True)
self.log_info("Starting update")
- routers = retrieve_and_persist_neteng_managed_device_list(
+ retrieve_and_persist_gap_site_list(
info_callback=self.log_info,
- warning_callback=self.log_warning
- )
+ warning_callback=self.log_warning)
+
+ routers = retrieve_and_persist_gap_managed_device_list(
+ info_callback=self.log_info,
+ warning_callback=self.log_warning)
lab_routers = InventoryTask.config.get('lab-routers', {})
+
+ juniper_prod_fqdns = [r['fqdn'] for r in routers if r['vendor'] == 'juniper']
+ self.log_info(f'juniper prod fqdns: {juniper_prod_fqdns}')
+ juniper_lab_fqdns = [r for r, v in lab_routers.items() if v == 'juniper']
+ self.log_info(f'juniper lab fqdns: {juniper_lab_fqdns}')
+ nokia_prod_fqdns = [r['fqdn'] for r in routers if r['vendor'] == 'nokia']
+ self.log_info(f'nokia prod fqdns: {nokia_prod_fqdns}')
+ nokia_lab_fqdns = [r for r, v in lab_routers.items() if v == 'nokia']
+ self.log_info(f'nokia lab fqdns: {nokia_lab_fqdns}')
+ unknown_prod_fqdns = [r['fqdn'] for r in routers if r['vendor'] == 'unknown']
+ if unknown_prod_fqdns:
+ self.log_info(f'unknown prod fqdns: {unknown_prod_fqdns}')
+ unknown_lab_fqdns = [r for r, v in lab_routers.items() if v == 'unknown']
+ if unknown_lab_fqdns:
+ self.log_info(f'unknown lab fqdns: {unknown_lab_fqdns}')
+
chord(
(
ims_task.s().on_error(task_error_handler.s()),
+
chord(
- (reload_router_config_juniper.s(r) for r, v in routers.items()
- if v == 'juniper'),
+ map(reload_router_config_juniper.s, juniper_prod_fqdns),
empty_task.si('juniper router tasks complete')
),
chord(
- (reload_lab_router_config_juniper.s(r)
- for r, v in lab_routers.items() if v == 'juniper'),
+ map(reload_router_config_juniper.s, juniper_lab_fqdns),
empty_task.si('juniper lab router tasks complete')
),
chord(
- (reload_router_config_nokia.s(r) for r, v in routers.items()
- if v == 'nokia'),
+ map(reload_router_config_nokia.s, nokia_prod_fqdns),
empty_task.si('nokia router tasks complete')
),
chord(
- (reload_router_config_nokia.s(r, True)
- for r, v in lab_routers.items() if v == 'nokia'),
+ map(reload_router_config_nokia.s, nokia_lab_fqdns),
empty_task.si('nokia lab router tasks complete')
),
chord(
- (reload_router_config_try_all.s(r) for r, v in routers.items()
- if v == 'unknown'),
+ map(reload_router_config_try_all.s, unknown_prod_fqdns),
empty_task.si('unknown router tasks complete')
),
chord(
- (reload_router_config_try_all.s(r, True)
- for r, v in lab_routers.items() if v == 'unknown'),
+ map(reload_router_config_try_all.s, unknown_lab_fqdns),
empty_task.si('unknown lab router tasks complete')
)
),
@@ -527,27 +542,44 @@ def empty_task(self, message):
logger.warning(f'message from empty task: {message}')
-def retrieve_and_persist_neteng_managed_device_list(
- info_callback=lambda s: None,
- warning_callback=lambda s: None):
- netdash_equipment = None
+def _retrieve_and_persist_value(
+ key: str,
+ retrieve_callback: Callable,
+ info_callback: Callable,
+ warning_callback: Callable,
+ info_keyword: str = 'data items'):
+ """
+ utility method, not super general, but mostly will save a simple
+ result returned from `retrieve_callback` to the redis key `key`
+
+ if `retrieve_callback` doesn't return anything, tries to get the
+ previous value
+
+ :param key: redis key to save the result to
+ :param retrieve_callback: function to retrieve the data (takes app config dict)
+ :param info_callback: for logging progress callbacks
+ :param warning_callback: for logging warnings/errors
+ :param info_keyword: keyword to use in various info messages
+ :return: the data retrieved from `retrieve_callback`
+ """
+
try:
- info_callback('querying netdash for managed routers')
- netdash_equipment = gap.load_routers_from_orchestrator(InventoryTask.config)
+ info_callback('querying for managed {info_keyword}s')
+ retreived_data = retrieve_callback(InventoryTask.config)
except Exception as e:
- warning_callback(f'Error retrieving device list: {e}')
+ warning_callback(f'Error retrieving {info_keyword} list: {e}')
- if netdash_equipment:
- info_callback(f'found {len(netdash_equipment)} routers')
+ if retreived_data:
+ info_callback(f'found {len(retreived_data)} {info_keyword}s')
else:
- warning_callback('No devices retrieved, using previous list')
+ warning_callback(f'No {info_keyword}s retrieved, using previous list')
try:
current_r = get_current_redis(InventoryTask.config)
- netdash_equipment = current_r.get('netdash')
- netdash_equipment = json.loads(netdash_equipment.decode('utf-8'))
- if not netdash_equipment:
+ retreived_data = current_r.get('gso:routers')
+ retreived_data = json.loads(retreived_data.decode('utf-8'))
+ if not retreived_data:
raise InventoryTaskError(
- 'No equipment retrieved from previous list')
+ 'No {info_keyword}s retrieved from previous list')
except Exception as e:
warning_callback(str(e))
update_latch_status(
@@ -556,13 +588,46 @@ def retrieve_and_persist_neteng_managed_device_list(
try:
next_r = get_next_redis(InventoryTask.config)
- next_r.set('netdash', json.dumps(netdash_equipment))
- info_callback(f'saved {len(netdash_equipment)} managed routers')
+ next_r.set(key, json.dumps(retreived_data))
+ info_callback(f'saved {len(retreived_data)} managed {info_keyword}s')
except Exception as e:
warning_callback(str(e))
update_latch_status(InventoryTask.config, pending=False, failure=True)
raise e
- return netdash_equipment
+
+ return retreived_data
+
+
+def retrieve_and_persist_gap_managed_device_list(
+ info_callback=lambda s: None,
+ warning_callback=lambda s: None):
+ """
+ loads the list of managed routers from GSO, saves
+ in gso:routers redis key
+ """
+
+ return _retrieve_and_persist_value(
+ key='gso:routers',
+ retrieve_callback=gap.load_routers_from_orchestrator,
+ info_callback=info_callback,
+ warning_callback=warning_callback,
+ info_keyword='router')
+
+
+def retrieve_and_persist_gap_site_list(
+ info_callback=lambda s: None,
+ warning_callback=lambda s: None):
+ """
+ loads the list of sites from GSO, saves
+ in gso:sites redis key
+ """
+
+ return _retrieve_and_persist_value(
+ key='gso:sites',
+ retrieve_callback=gap.load_sites_from_orchestrator,
+ info_callback=info_callback,
+ warning_callback=warning_callback,
+ info_keyword='site')
@app.task(base=InventoryTask, bind=True, name='reload_router_config_try_all')
@@ -1743,6 +1808,23 @@ def persist_ims_data(data, use_current=False):
return sites.values()
+ def _get_map_equipment():
+ # simple list of equipment from locations
+ def _fmt_elem(_x):
+ return {
+ 'name': _x['equipment-name'],
+ 'status': _x['status'],
+ 'pop': _x['pop']['name']
+ }
+ return list(map(_fmt_elem, locations.values()))
+
+ def _get_map_pops():
+ # get a de-duped list of sites with actual equipment
+ _pops = {}
+ for _x in locations.values():
+ _pops[_x['pop']['name']] = _x['pop']
+ return list(_pops.values())
+
if use_current:
r = get_current_redis(InventoryTask.config)
@@ -1758,7 +1840,8 @@ def persist_ims_data(data, use_current=False):
'ims:access_services:*',
'ims:gws_indirect:*',
'ims:node_pair_services:*',
- 'ims:pop_nodes:*'
+ 'ims:pop_nodes:*',
+ 'ims:map:*',
]:
rp = r.pipeline()
for k in r.scan_iter(key_pattern, count=1000):
@@ -1766,7 +1849,10 @@ def persist_ims_data(data, use_current=False):
else:
r = get_next_redis(InventoryTask.config)
+ r.set('ims:map:pops', json.dumps(_get_map_pops()))
+ r.set('ims:map:equipment', json.dumps(_get_map_equipment()))
r.set('ims:sid_services', json.dumps(sid_services))
+
rp = r.pipeline()
for h, d in locations.items():
rp.set(f'ims:location:{h}', json.dumps([d]))
@@ -2046,11 +2132,11 @@ def _populate_equipment_vendors():
def populate_equipment_vendors(r):
equipment_details = json.loads(r.get('ims:cache:equipment_details').decode('utf-8'))
- router_vendors = json.loads(r.get('netdash').decode('utf-8'))
+ routers = json.loads(r.get('gso:routers').decode('utf-8'))
equipment_details.extend({
- 'name': r,
+ 'name': r['fqdn'],
'model': 'UNKNOWN',
- 'vendor': v} for r, v in router_vendors.items())
+ 'vendor': r['vendor']} for r in routers)
for ed in equipment_details:
r.set(f'state-checker:equipment-vendors:{ed["name"]}', json.dumps(ed))
diff --git a/setup.py b/setup.py
index e50a345ba133ec0043f54b61c542ee2fbab96898..d99f1e76be04064f12c3888f48cf5e8fdb25a9ac 100644
--- a/setup.py
+++ b/setup.py
@@ -2,7 +2,7 @@ from setuptools import setup, find_packages
setup(
name='inventory-provider',
- version="0.145",
+ version="0.146",
author='GEANT',
author_email='swd@geant.org',
description='Dashboard inventory provider',
diff --git a/test/data/router-info.json b/test/data/router-info.json
index a46964473af6b4a5af9f8210b18eac9f62f1dace..51ac638c56fe834130b414f1cd8cfa5f16dd2b6b 100644
Binary files a/test/data/router-info.json and b/test/data/router-info.json differ
diff --git a/test/test_classifier_routes.py b/test/test_classifier_routes.py
index e4a5b8a73dfc54e6f6870da43777f6a8c25269c4..e5cca53d2aaab00f3b54f656c0ef243709a65364 100644
--- a/test/test_classifier_routes.py
+++ b/test/test_classifier_routes.py
@@ -238,9 +238,13 @@ def test_router_info_all_routers(client):
def test_all_routers_skips_routers_not_in_ims(client, mocked_redis):
- all_routers = json.loads(mocked_redis.get('netdash'))
- all_routers['invalid.router'] = 'juniper'
- mocked_redis.set('netdash', json.dumps(all_routers))
+ all_routers = json.loads(mocked_redis.get('gso:routers'))
+ all_routers.append({
+ 'fqdn': 'invalid.router',
+ 'vendor': 'juniper',
+ 'site': 'AAA'
+ })
+ mocked_redis.set('gso:routers', json.dumps(all_routers))
result = client.get("/classifier/router-info", headers=DEFAULT_REQUEST_HEADERS).json
assert not any(r['hostname'] == 'invalid.router' for r in result)
diff --git a/test/test_gap.py b/test/test_gap.py
index ea8d0503f77ecdea91a5bbe991c6c96f64269627..dca58e73efe6941b49dc89f1eaf1c02411b15fb6 100644
--- a/test/test_gap.py
+++ b/test/test_gap.py
@@ -40,8 +40,11 @@ def test_extract_router_info(mocker):
'page': [{
'productBlockInstances': [{
'productBlockInstanceValues': [
+ {'field': 'name', 'value': 'block name'},
+ {'field': 'label', 'value': 'ignored label'},
{'field': 'routerFqdn', 'value': 'test_fqdn'},
- {'field': 'vendor', 'value': 'test_vendor'}
+ {'field': 'vendor', 'value': 'test_vendor'},
+ {'field': 'siteName', 'value': 'test_site'}
]
}]
}]
@@ -61,7 +64,11 @@ def test_extract_router_info(mocker):
}
}
router_info = gap.extract_router_info(device, 'test_token', config)
- assert router_info == {'fqdn': 'test_fqdn', 'vendor': 'test_vendor'}
+ assert router_info == {
+ 'fqdn': 'test_fqdn',
+ 'vendor': 'test_vendor',
+ 'site': 'test_site'
+ }
def test_load_routers_from_orchestrator(mocker):
@@ -83,8 +90,8 @@ def test_load_routers_from_orchestrator(mocker):
}
})
mocker.patch('inventory_provider.gap.extract_router_info', side_effect=[
- {'fqdn': 'fqdn1', 'vendor': 'vendor1'},
- {'fqdn': 'fqdn2', 'vendor': 'vendor2'}
+ {'fqdn': 'fqdn1', 'vendor': 'vendor1', 'site': 'AAA'},
+ {'fqdn': 'fqdn2', 'vendor': 'vendor2', 'site': 'BBB'}
])
mocker.patch('inventory_provider.gap.get_token', return_value='test_token')
config = {
@@ -101,4 +108,7 @@ def test_load_routers_from_orchestrator(mocker):
}
routers = gap.load_routers_from_orchestrator(config)
- assert routers == {'fqdn1': 'vendor1', 'fqdn2': 'vendor2'}
+ assert routers == [
+ {'fqdn': 'fqdn1', 'vendor': 'vendor1', 'site': 'AAA'},
+ {'fqdn': 'fqdn2', 'vendor': 'vendor2', 'site': 'BBB'}
+ ]
diff --git a/test/test_map_routes.py b/test/test_map_routes.py
new file mode 100644
index 0000000000000000000000000000000000000000..cfc8237d5a018a0f52d75af435e53b0c94848be9
--- /dev/null
+++ b/test/test_map_routes.py
@@ -0,0 +1,28 @@
+import json
+import jsonschema
+
+from inventory_provider.routes import map
+
+
+def test_get_equipment(client):
+ rv = client.get(
+ '/map/equipment',
+ headers={'Accept': ['application/json']})
+ assert rv.status_code == 200
+ assert rv.is_json
+ response_data = json.loads(rv.data.decode('utf-8'))
+ jsonschema.validate(response_data, map.EQUIPMENT_LIST_SCHEMA)
+ assert response_data # test data is non-empty
+ # print(json.dumps(response_data, indent=2))
+
+
+def test_get_pops(client):
+ rv = client.get(
+ '/map/pops',
+ headers={'Accept': ['application/json']})
+ assert rv.status_code == 200
+ assert rv.is_json
+ response_data = json.loads(rv.data.decode('utf-8'))
+ jsonschema.validate(response_data, map.POP_LIST_SCHEMA)
+ assert response_data # test data is non-empty
+ # print(json.dumps(response_data, indent=2))
diff --git a/test/test_worker.py b/test/test_worker.py
index 541c508ba1d06aec7450b9cf6dbd995f2f0c967a..aea808b45ff65f6eb81e5dd7cd5d09fc5fb658a3 100644
--- a/test/test_worker.py
+++ b/test/test_worker.py
@@ -8,7 +8,8 @@ from inventory_provider.tasks.worker import (
transform_ims_data,
extract_ims_data,
persist_ims_data,
- retrieve_and_persist_neteng_managed_device_list,
+ retrieve_and_persist_gap_managed_device_list,
+ retrieve_and_persist_gap_site_list,
populate_poller_interfaces_cache,
refresh_nokia_interface_list,
retrieve_and_persist_config_nokia,
@@ -211,6 +212,7 @@ def test_transform_ims_data():
locations = {
"eq_a": {
"equipment-name": "eq_a",
+ "status": "operational",
"pop": {
"name": "pop_loc_a",
"abbreviation": "pla",
@@ -218,6 +220,7 @@ def test_transform_ims_data():
},
"eq_b": {
"equipment-name": "eq_b",
+ "status": "bogus",
"pop": {
"name": "pop_loc_b",
"abbreviation": "plb",
@@ -225,6 +228,7 @@ def test_transform_ims_data():
},
"UNKNOWN_LOC": {
"equipment-name": "UNKNOWN_LOC",
+ "status": "blah blah",
"pop": {
"name": "UNKNOWN",
"abbreviation": "UNKNOWN",
@@ -575,10 +579,12 @@ def test_persist_ims_data(mocker, data_config, mocked_redis):
"locations": {
"eq_a": {
'equipment-name': 'eq_a',
+ 'status': 'operational',
'pop': {'name': "LOC A", 'abbreviation': 'aaa'}
},
"eq_b": {
'equipment-name': 'eq_b',
+ 'status': 'blah blah',
'pop': {'name': "LOC B", 'abbreviation': 'bbb'}
},
},
@@ -674,21 +680,61 @@ def test_persist_ims_data(mocker, data_config, mocked_redis):
data["sid_services"]
-def test_retrieve_and_persist_neteng_managed_device_list(
+def test_retrieve_and_persist_gap_managed_device_list(
mocker, data_config, mocked_redis):
- device_list = [{'abc': 'juniper'}, {'def': 'nokia'}]
+
+ device_list = [
+ {'fqdn': 'hostname1', 'site': 'AAA', 'vendor': 'juniper'},
+ {'fqdn': 'hostname2', 'site': 'BBB', 'vendor': 'nokia'},
+ ]
r = common._get_redis(data_config)
mocker.patch(
'inventory_provider.tasks.worker.InventoryTask.config'
)
mocker.patch('inventory_provider.tasks.worker.get_next_redis', return_value=r)
- r.delete('netdash')
+ r.delete('gso:routers')
mocker.patch('inventory_provider.tasks.worker.get_current_redis', return_value=r)
mocker.patch('inventory_provider.gap.load_routers_from_orchestrator', return_value=device_list)
- result = retrieve_and_persist_neteng_managed_device_list()
+
+ result = retrieve_and_persist_gap_managed_device_list()
assert result == device_list
- assert json.loads(r.get('netdash')) == device_list
+ assert json.loads(r.get('gso:routers')) == device_list
+
+
+def test_retrieve_and_persist_gap_site_list(
+ mocker, data_config, mocked_redis):
+ site_list = [
+ {
+ 'city': 'city#1',
+ 'country': 'country#1',
+ 'country_code': 'AA',
+ 'name': 'AAA',
+ 'longitude': 0.0,
+ 'latitude': 1.1,
+ 'tier': 3
+ },
+ {
+ 'city': 'city #2',
+ 'country': 'country #2',
+ 'country_code': 'ZZ',
+ 'name': 'ZZZ',
+ 'longitude': 99.89,
+ 'latitude': 123.456,
+ 'tier': 1
+ },
+ ]
+
+ r = common._get_redis(data_config)
+ mocker.patch('inventory_provider.tasks.common._get_redis', return_value=r)
+
+ mocker.patch('inventory_provider.tasks.worker.InventoryTask.config')
+ r.delete('gso:sites')
+ mocker.patch('inventory_provider.gap.load_sites_from_orchestrator', return_value=site_list)
+
+ result = retrieve_and_persist_gap_site_list()
+ assert result == site_list
+ assert json.loads(r.get('gso:sites')) == site_list
def test_populate_poller_interfaces_cache(
@@ -1010,8 +1056,8 @@ def test_refresh_nokia_interface_list(mocked_redis, data_config, load_nokia_netc
def test_populate_error_report_interfaces_cache(mocker, data_config, mocked_redis):
r = common._get_redis(data_config)
+ mocker.patch('inventory_provider.tasks.common._get_redis', return_value=r)
- mocker.patch('inventory_provider.tasks.worker.get_next_redis', return_value=r)
all_interfaces = [
{
"router": "router_a.geant.net",
@@ -1069,12 +1115,21 @@ def test_populate_error_report_interfaces_cache(mocker, data_config, mocked_redi
'inventory_provider.tasks.worker.InventoryTask.config'
)
- netdash_equipment = {
- "router_a.geant.net": "juniper",
- "rt0.geant.net": "nokia"
- }
+ test_routers = [
+ {
+ 'fqdn': 'router_a.geant.net',
+ 'vendor': 'juniper',
+ 'site': 'AAA'
+ },
+ {
+ 'fqdn': 'rt0.geant.net',
+ 'vendor': 'nokia',
+ 'site': 'BBB'
+ }
+ ]
+
+ mocked_redis.set('gso:routers', json.dumps(test_routers))
- mocker.patch('inventory_provider.routes.poller.get_netdash_equipment', return_value=netdash_equipment)
exp_router_a_interfaces = [
{
"router": "router_a.geant.net",