diff --git a/Changelog.md b/Changelog.md
index 77862a31572c8463decb38e79c2c162ca0cf821c..6d142aa67cc2a5cfdb5b1b6f876e2439831b422c 100644
--- a/Changelog.md
+++ b/Changelog.md
@@ -2,6 +2,9 @@
All notable changes to this project will be documented in this file.
+## [0.101] - 2023-03-28
+- DBOARD3-713: MIC endpoint - only include services that are monitored in Geant NMS
+
## [0.100] - 2022-12-07
- POL1-646: Changed BRIAN interface description parsing to expect whitespace
- POL1-643: Added port_type field to interfaces to distinguish access/service
diff --git a/README.md b/README.md
index e8216dd6425d784fec8fd47e769f0ce8452ed0ce..69a4990dbe570d66142a8fd7c5976bc7258b03f6 100644
--- a/README.md
+++ b/README.md
@@ -1,15 +1,15 @@
-# Inventory Provider
-
-The Inventory Provider
-acts as a single point of truth for
-information about the GÉANT network, exposed by
-by an HTTP API.
-
-Documentation can be generated by running sphinx:
-
-```bash
-sphinx-build -M html docs/source docs/build
-```
-
-The documents should be viewable in the
-workspace of the most recent [Jenkins job](https://jenkins.geant.org/job/inventory-provider%20(develop)/ws/docs/build/html/index.html).
\ No newline at end of file
+# Inventory Provider
+
+The Inventory Provider
+acts as a single point of truth for
+information about the GÉANT network, exposed by
+by an HTTP API.
+
+Documentation can be generated by running sphinx in the commandline:
+
+```bash
+sphinx-build -M html docs/source docs/build
+```
+
+The documents should be viewable in the
+workspace of the most recent [Jenkins job](https://jenkins.geant.org/job/inventory-provider%20(develop)/ws/docs/build/html/index.html).
diff --git a/inventory_provider/db/ims.py b/inventory_provider/db/ims.py
index d7b4eebd7297576c0936b5b2461bfca3117702f8..07aa68c7f759e735fbbb7bdb78716e90421613c0 100644
--- a/inventory_provider/db/ims.py
+++ b/inventory_provider/db/ims.py
@@ -1,9 +1,11 @@
+from enum import Enum
+import functools
import logging
+import time
import requests
-import time
-from enum import Enum
+from inventory_provider import environment
# Navigation Properties
@@ -11,6 +13,9 @@ from enum import Enum
from requests import HTTPError
logger = logging.getLogger(__name__)
+log_entry_and_exit = functools.partial(
+ environment.log_entry_and_exit, logger=logger)
+
# http://149.210.162.190:81/ImsVersions/21.9/html/50e6a1b1-3910-2091-63d5-e13777b2194e.htm # noqa
CIRCUIT_CUSTOMER_RELATION = {
@@ -235,7 +240,7 @@ class IMS(object):
response.raise_for_status()
break
- def _get_entity(
+ def _get_ims_data(
self,
url,
params=None,
@@ -317,6 +322,7 @@ class IMS(object):
IMS.cache[cache_key] = return_value
return return_value
+ @log_entry_and_exit
def get_entity_by_id(
self,
entity_type,
@@ -326,8 +332,9 @@ class IMS(object):
url = f'{entity_type}/{entity_id}'
return \
- self._get_entity(url, None, navigation_properties, use_cache)
+ self._get_ims_data(url, None, navigation_properties, use_cache)
+ @log_entry_and_exit
def get_entity_by_name(
self,
entity,
@@ -335,8 +342,9 @@ class IMS(object):
navigation_properties=None,
use_cache=False):
url = f'{entity}/byname/"{name}"'
- return self._get_entity(url, None, navigation_properties, use_cache)
+ return self._get_ims_data(url, None, navigation_properties, use_cache)
+ @log_entry_and_exit
def get_filtered_entities(
self,
entity,
@@ -344,17 +352,31 @@ class IMS(object):
navigation_properties=None,
use_cache=False,
step_count=50):
+ url = f'{entity}/filtered/{filter_string}'
+ yield from self.get_entities(
+ url,
+ navigation_properties,
+ use_cache,
+ step_count
+ )
+
+ def get_entities(
+ self,
+ url,
+ navigation_properties=None,
+ use_cache=False,
+ step_count=50):
more_to_come = True
start_entity = 0
+ gateway_error_count = 0
while more_to_come:
params = {
'paginatorStartElement': start_entity,
'paginatorNumberOfElements': step_count
}
- url = f'{entity}/filtered/{filter_string}'
try:
more_to_come = False
- entities = self._get_entity(
+ entities = self._get_ims_data(
url,
params,
navigation_properties,
@@ -364,6 +386,17 @@ class IMS(object):
if r.status_code == requests.codes.not_found \
and NO_FILTERED_RESULTS_MESSAGE in r.text.lower():
entities = None
+ elif r.status_code == 504:
+ gateway_error_count += 1
+ logger.debug(
+ f"GATEWAY TIME-OUT for {url}"
+ f" -- COUNT: {gateway_error_count}")
+ if gateway_error_count > 4:
+ raise e
+ time.sleep(5)
+ logger.debug("WAKING UP")
+ more_to_come = True
+ continue
else:
raise e
if entities:
@@ -372,6 +405,7 @@ class IMS(object):
start_entity += step_count
yield from entities
+ @log_entry_and_exit
def get_all_entities(
self,
entity,
@@ -379,9 +413,9 @@ class IMS(object):
use_cache=False,
step_count=50
):
- yield from self.get_filtered_entities(
- entity,
- 'Id <> 0',
+ url = f'{entity}/all'
+ yield from self.get_entities(
+ url,
navigation_properties,
use_cache,
step_count
diff --git a/inventory_provider/db/ims_data.py b/inventory_provider/db/ims_data.py
index 371e2f2cfc537d4509bc75d1f8a1222c4493bf9e..81cf11b7e430398537410e29ec27c2e085f871b6 100644
--- a/inventory_provider/db/ims_data.py
+++ b/inventory_provider/db/ims_data.py
@@ -1,3 +1,4 @@
+import functools
import logging
import re
from collections import defaultdict
@@ -9,10 +10,9 @@ from inventory_provider.db import ims
from inventory_provider.db.ims import InventoryStatus, IMS, \
CUSTOMER_RELATED_CONTACT_PROPERTIES, EXTRA_FIELD_VALUE_PROPERTIES
-environment.setup_logging()
logger = logging.getLogger(__name__)
-
-# Dashboard V3
+log_entry_and_exit = functools.partial(
+ environment.log_entry_and_exit, logger=logger)
IMS_OPSDB_STATUS_MAP = {
InventoryStatus.PLANNED: 'planned',
@@ -83,6 +83,7 @@ def get_flexils_by_circuitid(ds: IMS):
return dict(by_circuit)
+@log_entry_and_exit
def get_non_monitored_circuit_ids(ds: IMS):
# note the id for the relevant field is hard-coded. I didn't want to use
# the name of the field as this can be changed by users
@@ -94,6 +95,7 @@ def get_non_monitored_circuit_ids(ds: IMS):
yield d['extrafieldvalueobjectinfo']['objectid']
+@log_entry_and_exit
def get_monitored_circuit_ids(ds: IMS):
# note the id for the relevant field is hard-coded. I didn't want to use
# the name of the field as this can be changed by users
@@ -106,6 +108,7 @@ def get_monitored_circuit_ids(ds: IMS):
yield d['extrafieldvalueobjectinfo']['objectid']
+@log_entry_and_exit
def get_ids_and_sids(ds: IMS):
for sid_circuit in ds.get_filtered_entities(
'ExtraFieldValue',
@@ -115,6 +118,7 @@ def get_ids_and_sids(ds: IMS):
yield sid_circuit['objectid'], sid_circuit['value']
+@log_entry_and_exit
def get_service_types(ds: IMS):
for d in ds.get_filtered_entities(
'ComboBoxData',
@@ -122,6 +126,7 @@ def get_service_types(ds: IMS):
yield d['selection']
+@log_entry_and_exit
def get_customer_tts_contacts(ds: IMS):
customer_contacts = defaultdict(set)
@@ -137,6 +142,7 @@ def get_customer_tts_contacts(ds: IMS):
yield k, sorted(list(v))
+@log_entry_and_exit
def get_customer_planned_work_contacts(ds: IMS):
customer_contacts = defaultdict(set)
@@ -152,6 +158,7 @@ def get_customer_planned_work_contacts(ds: IMS):
yield k, sorted(list(v))
+@log_entry_and_exit
def get_circuit_related_customers(ds: IMS):
return_value = defaultdict(list)
@@ -169,6 +176,7 @@ def get_circuit_related_customers(ds: IMS):
return return_value
+@log_entry_and_exit
def get_port_id_services(ds: IMS):
circuit_nav_props = [
ims.CIRCUIT_PROPERTIES['Ports'],
@@ -300,6 +308,7 @@ def get_port_id_services(ds: IMS):
}
+@log_entry_and_exit
def get_port_sids(ds: IMS):
"""
This function fetches SIDs for external ports that have them defined,
@@ -313,6 +322,7 @@ def get_port_sids(ds: IMS):
step_count=10000)}
+@log_entry_and_exit
def get_internal_port_sids(ds: IMS):
"""
This function fetches SIDs for external ports that have them defined,
@@ -326,6 +336,7 @@ def get_internal_port_sids(ds: IMS):
step_count=10000)}
+@log_entry_and_exit
def get_port_details(ds: IMS):
port_nav_props = [
ims.PORT_PROPERTIES['Node'],
@@ -381,6 +392,7 @@ def get_port_details(ds: IMS):
'internalport', internal_port_nav_props, step_count=2000), 'internal')
+@log_entry_and_exit
def get_circuit_hierarchy(ds: IMS):
circuit_nav_props = [
ims.CIRCUIT_PROPERTIES['Customer'],
@@ -425,6 +437,7 @@ def get_circuit_hierarchy(ds: IMS):
}
+@log_entry_and_exit
def get_node_locations(ds: IMS):
"""
return location info for all Site nodes
@@ -498,6 +511,7 @@ INTERNAL_POP_NAMES = {
}
+@log_entry_and_exit
def lookup_lg_routers(ds: IMS):
pattern = re.compile("vpn-proxy|vrr|taas", re.IGNORECASE)
@@ -562,6 +576,7 @@ def lookup_lg_routers(ds: IMS):
yield eq
+@log_entry_and_exit
def lookup_geant_nodes(ds: IMS):
return (n["name"]for n in ds.get_filtered_entities(
diff --git a/inventory_provider/environment.py b/inventory_provider/environment.py
index 989c0a1355ebb7b0e44f110842b25f045868a47d..ea0c5bff839abe503bf24c6b60d3e79d6c0172f4 100644
--- a/inventory_provider/environment.py
+++ b/inventory_provider/environment.py
@@ -1,6 +1,24 @@
+import functools
import json
import logging.config
import os
+import time
+
+
+def log_entry_and_exit(f, logger):
+ # cf. https://stackoverflow.com/a/47663642
+ # cf. https://stackoverflow.com/a/59098957/6708581
+ @functools.wraps(f)
+ def _w(*args, **kwargs):
+ logger.debug(f'>>> {f.__name__}{args}')
+ start_time = time.time()
+ try:
+ return f(*args, **kwargs)
+ finally:
+ end_time = time.time()
+ duration = end_time - start_time
+ logger.debug(f'<<< {f.__name__}{args} -- duration {duration}')
+ return _w
def setup_logging():
diff --git a/inventory_provider/routes/mic.py b/inventory_provider/routes/mic.py
index 43484ef028e2053da82566e7d42fc9537586ea85..a335ae3eb8ee176bba4de1bf93c3f9153d3683e0 100644
--- a/inventory_provider/routes/mic.py
+++ b/inventory_provider/routes/mic.py
@@ -222,7 +222,9 @@ def get_everything():
site = f'{d["pop_name"]} ({d["pop_abbreviation"]})'
eq_name = d['equipment']
if_name = d['port']
- all_data[site][eq_name][if_name] = d['related-services']
+ all_data[site][eq_name][if_name] = \
+ [rs for rs in d['related-services']
+ if rs['status'] == 'operational']
result = json.dumps(all_data)
r.set(cache_key, result.encode('utf-8'))
diff --git a/inventory_provider/tasks/worker.py b/inventory_provider/tasks/worker.py
index 2a58d54adeb272fc831e7c5621f0398091c30131..f9b1e36b0ece9857c254da5a4c14e628fda41bb9 100644
--- a/inventory_provider/tasks/worker.py
+++ b/inventory_provider/tasks/worker.py
@@ -4,7 +4,6 @@ import json
import logging
import os
import re
-import time
from typing import List
from celery import Task, states, chord
@@ -36,24 +35,9 @@ FINALIZER_TIMEOUT_S = 300
# TODO: error callback (cf. http://docs.celeryproject.org/en/latest/userguide/calling.html#linking-callbacks-errbacks) # noqa: E501
environment.setup_logging()
-
logger = logging.getLogger(__name__)
-
-
-def log_task_entry_and_exit(f):
- # cf. https://stackoverflow.com/a/47663642
- # cf. https://stackoverflow.com/a/59098957/6708581
- @functools.wraps(f)
- def _w(*args, **kwargs):
- logger.debug(f'>>> {f.__name__}{args}')
- start_time = time.time()
- try:
- return f(*args, **kwargs)
- finally:
- end_time = time.time()
- duration = end_time - start_time
- logger.debug(f'<<< {f.__name__}{args} -- duration {duration}')
- return _w
+log_task_entry_and_exit = functools.partial(
+ environment.log_entry_and_exit, logger=logger)
class InventoryTaskError(Exception):
@@ -863,7 +847,8 @@ def _extract_ims_data(ims_api_url, ims_username, ims_password):
executor.submit(_populate_hierarchy): 'hierarchy',
executor.submit(_populate_port_id_details): 'port_id_details',
executor.submit(_populate_circuit_info): 'circuit_info',
- executor.submit(_populate_flexils_data): 'flexils_data'
+ executor.submit(_populate_flexils_data): 'flexils_data',
+ executor.submit(_populate_customers): 'customers'
}
for future in concurrent.futures.as_completed(futures):
@@ -1105,8 +1090,13 @@ def transform_ims_data(data):
for tlc in circ['related-services']:
# why were these removed?
# contacts.update(tlc.pop('contacts'))
- contacts.update(tlc.get('contacts'))
- pw_contacts.update(tlc.get('planned_work_contacts', []))
+ if circ['status'] == 'operational' \
+ and circ['id'] in circuit_ids_to_monitor \
+ and tlc['status'] == 'operational' \
+ and tlc['id'] in circuit_ids_to_monitor:
+ contacts.update(tlc.get('contacts'))
+ pw_contacts.update(
+ tlc.get('planned_work_contacts', []))
circ['contacts'] = sorted(list(contacts))
circ['planned_work_contacts'] = sorted(list(pw_contacts))
diff --git a/setup.py b/setup.py
index 2dd9d8101f8408a6febaf8937a60c4f3b0997219..0cc25fe475947f1c97f1efc86dcfe6f036e76679 100644
--- a/setup.py
+++ b/setup.py
@@ -2,7 +2,7 @@ from setuptools import setup, find_packages
setup(
name='inventory-provider',
- version="0.100",
+ version="0.101",
author='GEANT',
author_email='swd@geant.org',
description='Dashboard inventory provider',
diff --git a/test/test_ims.py b/test/test_ims.py
index 23f3066007be08ab543d51192e8000a4f66c2801..fc8c1c219d755659fe09fbd23a06659d5fa027d6 100644
--- a/test/test_ims.py
+++ b/test/test_ims.py
@@ -139,7 +139,7 @@ def test_ims_class_get_all_entities(mocker):
list(ds.get_all_entities('Node', step_count=10))
mock_get.assert_called_once_with(
- 'dummy_base/ims/Node/filtered/Id <> 0',
+ 'dummy_base/ims/Node/all',
headers={'Authorization': 'Bearer dummy_bt'},
params={
'paginatorStartElement': 0,
diff --git a/tox.ini b/tox.ini
index 88bf0caa1b18a625125d67275de52f68333f0f17..86a7ce2a3e4293a0f2b668507c7850acee255ae4 100644
--- a/tox.ini
+++ b/tox.ini
@@ -5,7 +5,7 @@ envlist = py36
exclude = venv,.tox,build
[testenv]
-passenv = TEST_OPSDB_HOSTNAME TEST_OPSDB_DBNAME TEST_OPSDB_USERNAME TEST_OPSDB_PASSWORD
+passenv = TEST_OPSDB_HOSTNAME,TEST_OPSDB_DBNAME,TEST_OPSDB_USERNAME,TEST_OPSDB_PASSWORD
deps =
coverage
flake8