Finished release 0.101.

Changelog.md

@@ -2,6 +2,9 @@
 
 All notable changes to this project will be documented in this file.
 
+## [0.101] - 2023-03-28
+- DBOARD3-713: MIC endpoint - only include services that are monitored in Geant NMS
+
 ## [0.100] - 2022-12-07
 - POL1-646: Changed BRIAN interface description parsing to expect whitespace
 - POL1-643: Added port_type field to interfaces to distinguish access/service

README.md

-# Inventory Provider
-
-The Inventory Provider
-acts as a single point of truth for
-information about the GÉANT network, exposed by
-by an HTTP API.
-
-Documentation can be generated by running sphinx:
-
-```bash
-sphinx-build -M html docs/source docs/build
-```
-
-The documents should be viewable in the
-workspace of the most recent [Jenkins job](https://jenkins.geant.org/job/inventory-provider%20(develop)/ws/docs/build/html/index.html).
\ No newline at end of file
+# Inventory Provider
+
+The Inventory Provider
+acts as a single point of truth for
+information about the GÉANT network, exposed by
+by an HTTP API.
+
+Documentation can be generated by running sphinx in the commandline:
+
+```bash
+sphinx-build -M html docs/source docs/build
+```
+
+The documents should be viewable in the
+workspace of the most recent [Jenkins job](https://jenkins.geant.org/job/inventory-provider%20(develop)/ws/docs/build/html/index.html).

inventory_provider/db/ims.py

+from enum import Enum
+import functools
 import logging
+import time
 
 import requests
-import time
 
-from enum import Enum
+from inventory_provider import environment
 
 
 # Navigation Properties
@@ -11,6 +13,9 @@ from enum import Enum
 from requests import HTTPError
 
 logger = logging.getLogger(__name__)
+log_entry_and_exit = functools.partial(
+    environment.log_entry_and_exit, logger=logger)
+
 
 # http://149.210.162.190:81/ImsVersions/21.9/html/50e6a1b1-3910-2091-63d5-e13777b2194e.htm  # noqa
 CIRCUIT_CUSTOMER_RELATION = {
@@ -235,7 +240,7 @@ class IMS(object):
             response.raise_for_status()
             break
 
-    def _get_entity(
+    def _get_ims_data(
             self,
             url,
             params=None,
@@ -317,6 +322,7 @@ class IMS(object):
                     IMS.cache[cache_key] = return_value
                 return return_value
 
+    @log_entry_and_exit
     def get_entity_by_id(
             self,
             entity_type,
@@ -326,8 +332,9 @@ class IMS(object):
 
         url = f'{entity_type}/{entity_id}'
         return \
-            self._get_entity(url, None, navigation_properties, use_cache)
+            self._get_ims_data(url, None, navigation_properties, use_cache)
 
+    @log_entry_and_exit
     def get_entity_by_name(
             self,
             entity,
@@ -335,8 +342,9 @@ class IMS(object):
             navigation_properties=None,
             use_cache=False):
         url = f'{entity}/byname/"{name}"'
-        return self._get_entity(url, None, navigation_properties, use_cache)
+        return self._get_ims_data(url, None, navigation_properties, use_cache)
 
+    @log_entry_and_exit
     def get_filtered_entities(
             self,
             entity,
@@ -344,17 +352,31 @@ class IMS(object):
             navigation_properties=None,
             use_cache=False,
             step_count=50):
+        url = f'{entity}/filtered/{filter_string}'
+        yield from self.get_entities(
+            url,
+            navigation_properties,
+            use_cache,
+            step_count
+        )
+
+    def get_entities(
+            self,
+            url,
+            navigation_properties=None,
+            use_cache=False,
+            step_count=50):
         more_to_come = True
         start_entity = 0
+        gateway_error_count = 0
         while more_to_come:
             params = {
                 'paginatorStartElement': start_entity,
                 'paginatorNumberOfElements': step_count
             }
-            url = f'{entity}/filtered/{filter_string}'
             try:
                 more_to_come = False
-                entities = self._get_entity(
+                entities = self._get_ims_data(
                     url,
                     params,
                     navigation_properties,
@@ -364,6 +386,17 @@ class IMS(object):
                 if r.status_code == requests.codes.not_found \
                         and NO_FILTERED_RESULTS_MESSAGE in r.text.lower():
                     entities = None
+                elif r.status_code == 504:
+                    gateway_error_count += 1
+                    logger.debug(
+                        f"GATEWAY TIME-OUT for {url}"
+                        f"  --  COUNT: {gateway_error_count}")
+                    if gateway_error_count > 4:
+                        raise e
+                    time.sleep(5)
+                    logger.debug("WAKING UP")
+                    more_to_come = True
+                    continue
                 else:
                     raise e
             if entities:
@@ -372,6 +405,7 @@ class IMS(object):
                 start_entity += step_count
                 yield from entities
 
+    @log_entry_and_exit
     def get_all_entities(
             self,
             entity,
@@ -379,9 +413,9 @@ class IMS(object):
             use_cache=False,
             step_count=50
     ):
-        yield from self.get_filtered_entities(
-            entity,
-            'Id <> 0',
+        url = f'{entity}/all'
+        yield from self.get_entities(
+            url,
             navigation_properties,
             use_cache,
             step_count

inventory_provider/db/ims_data.py

+import functools
 import logging
 import re
 from collections import defaultdict
@@ -9,10 +10,9 @@ from inventory_provider.db import ims
 from inventory_provider.db.ims import InventoryStatus, IMS, \
     CUSTOMER_RELATED_CONTACT_PROPERTIES, EXTRA_FIELD_VALUE_PROPERTIES
 
-environment.setup_logging()
 logger = logging.getLogger(__name__)
-
-# Dashboard V3
+log_entry_and_exit = functools.partial(
+    environment.log_entry_and_exit, logger=logger)
 
 IMS_OPSDB_STATUS_MAP = {
     InventoryStatus.PLANNED: 'planned',
@@ -83,6 +83,7 @@ def get_flexils_by_circuitid(ds: IMS):
     return dict(by_circuit)
 
 
+@log_entry_and_exit
 def get_non_monitored_circuit_ids(ds: IMS):
     # note the id for the relevant field is hard-coded. I didn't want to use
     # the name of the field as this can be changed by users
@@ -94,6 +95,7 @@ def get_non_monitored_circuit_ids(ds: IMS):
         yield d['extrafieldvalueobjectinfo']['objectid']
 
 
+@log_entry_and_exit
 def get_monitored_circuit_ids(ds: IMS):
     # note the id for the relevant field is hard-coded. I didn't want to use
     # the name of the field as this can be changed by users
@@ -106,6 +108,7 @@ def get_monitored_circuit_ids(ds: IMS):
         yield d['extrafieldvalueobjectinfo']['objectid']
 
 
+@log_entry_and_exit
 def get_ids_and_sids(ds: IMS):
     for sid_circuit in ds.get_filtered_entities(
         'ExtraFieldValue',
@@ -115,6 +118,7 @@ def get_ids_and_sids(ds: IMS):
         yield sid_circuit['objectid'], sid_circuit['value']
 
 
+@log_entry_and_exit
 def get_service_types(ds: IMS):
     for d in ds.get_filtered_entities(
             'ComboBoxData',
@@ -122,6 +126,7 @@ def get_service_types(ds: IMS):
         yield d['selection']
 
 
+@log_entry_and_exit
 def get_customer_tts_contacts(ds: IMS):
 
     customer_contacts = defaultdict(set)
@@ -137,6 +142,7 @@ def get_customer_tts_contacts(ds: IMS):
         yield k, sorted(list(v))
 
 
+@log_entry_and_exit
 def get_customer_planned_work_contacts(ds: IMS):
 
     customer_contacts = defaultdict(set)
@@ -152,6 +158,7 @@ def get_customer_planned_work_contacts(ds: IMS):
         yield k, sorted(list(v))
 
 
+@log_entry_and_exit
 def get_circuit_related_customers(ds: IMS):
 
     return_value = defaultdict(list)
@@ -169,6 +176,7 @@ def get_circuit_related_customers(ds: IMS):
     return return_value
 
 
+@log_entry_and_exit
 def get_port_id_services(ds: IMS):
     circuit_nav_props = [
         ims.CIRCUIT_PROPERTIES['Ports'],
@@ -300,6 +308,7 @@ def get_port_id_services(ds: IMS):
             }
 
 
+@log_entry_and_exit
 def get_port_sids(ds: IMS):
     """
     This function fetches SIDs for external ports that have them defined,
@@ -313,6 +322,7 @@ def get_port_sids(ds: IMS):
             step_count=10000)}
 
 
+@log_entry_and_exit
 def get_internal_port_sids(ds: IMS):
     """
     This function fetches SIDs for external ports that have them defined,
@@ -326,6 +336,7 @@ def get_internal_port_sids(ds: IMS):
             step_count=10000)}
 
 
+@log_entry_and_exit
 def get_port_details(ds: IMS):
     port_nav_props = [
         ims.PORT_PROPERTIES['Node'],
@@ -381,6 +392,7 @@ def get_port_details(ds: IMS):
         'internalport', internal_port_nav_props, step_count=2000), 'internal')
 
 
+@log_entry_and_exit
 def get_circuit_hierarchy(ds: IMS):
     circuit_nav_props = [
         ims.CIRCUIT_PROPERTIES['Customer'],
@@ -425,6 +437,7 @@ def get_circuit_hierarchy(ds: IMS):
         }
 
 
+@log_entry_and_exit
 def get_node_locations(ds: IMS):
     """
     return location info for all Site nodes
@@ -498,6 +511,7 @@ INTERNAL_POP_NAMES = {
 }
 
 
+@log_entry_and_exit
 def lookup_lg_routers(ds: IMS):
     pattern = re.compile("vpn-proxy|vrr|taas", re.IGNORECASE)
 
@@ -562,6 +576,7 @@ def lookup_lg_routers(ds: IMS):
             yield eq
 
 
+@log_entry_and_exit
 def lookup_geant_nodes(ds: IMS):
 
     return (n["name"]for n in ds.get_filtered_entities(

inventory_provider/environment.py

+import functools
 import json
 import logging.config
 import os
+import time
+
+
+def log_entry_and_exit(f, logger):
+    # cf. https://stackoverflow.com/a/47663642
+    # cf. https://stackoverflow.com/a/59098957/6708581
+    @functools.wraps(f)
+    def _w(*args, **kwargs):
+        logger.debug(f'>>> {f.__name__}{args}')
+        start_time = time.time()
+        try:
+            return f(*args, **kwargs)
+        finally:
+            end_time = time.time()
+            duration = end_time - start_time
+            logger.debug(f'<<< {f.__name__}{args} -- duration {duration}')
+    return _w
 
 
 def setup_logging():

inventory_provider/routes/mic.py

@@ -222,7 +222,9 @@ def get_everything():
                     site = f'{d["pop_name"]} ({d["pop_abbreviation"]})'
                     eq_name = d['equipment']
                     if_name = d['port']
-                    all_data[site][eq_name][if_name] = d['related-services']
+                    all_data[site][eq_name][if_name] = \
+                        [rs for rs in d['related-services']
+                         if rs['status'] == 'operational']
         result = json.dumps(all_data)
 
         r.set(cache_key, result.encode('utf-8'))

inventory_provider/tasks/worker.py

@@ -4,7 +4,6 @@ import json
 import logging
 import os
 import re
-import time
 from typing import List
 
 from celery import Task, states, chord
@@ -36,24 +35,9 @@ FINALIZER_TIMEOUT_S = 300
 # TODO: error callback (cf. http://docs.celeryproject.org/en/latest/userguide/calling.html#linking-callbacks-errbacks)  # noqa: E501
 
 environment.setup_logging()
-
 logger = logging.getLogger(__name__)
-
-
-def log_task_entry_and_exit(f):
-    # cf. https://stackoverflow.com/a/47663642
-    # cf. https://stackoverflow.com/a/59098957/6708581
-    @functools.wraps(f)
-    def _w(*args, **kwargs):
-        logger.debug(f'>>> {f.__name__}{args}')
-        start_time = time.time()
-        try:
-            return f(*args, **kwargs)
-        finally:
-            end_time = time.time()
-            duration = end_time - start_time
-            logger.debug(f'<<< {f.__name__}{args} -- duration {duration}')
-    return _w
+log_task_entry_and_exit = functools.partial(
+    environment.log_entry_and_exit, logger=logger)
 
 
 class InventoryTaskError(Exception):
@@ -863,7 +847,8 @@ def _extract_ims_data(ims_api_url, ims_username, ims_password):
             executor.submit(_populate_hierarchy): 'hierarchy',
             executor.submit(_populate_port_id_details): 'port_id_details',
             executor.submit(_populate_circuit_info): 'circuit_info',
-            executor.submit(_populate_flexils_data): 'flexils_data'
+            executor.submit(_populate_flexils_data): 'flexils_data',
+            executor.submit(_populate_customers): 'customers'
         }
 
         for future in concurrent.futures.as_completed(futures):
@@ -1105,8 +1090,13 @@ def transform_ims_data(data):
                 for tlc in circ['related-services']:
                     # why were these removed?
                     # contacts.update(tlc.pop('contacts'))
-                    contacts.update(tlc.get('contacts'))
-                    pw_contacts.update(tlc.get('planned_work_contacts', []))
+                    if circ['status'] == 'operational' \
+                            and circ['id'] in circuit_ids_to_monitor \
+                            and tlc['status'] == 'operational' \
+                            and tlc['id'] in circuit_ids_to_monitor:
+                        contacts.update(tlc.get('contacts'))
+                        pw_contacts.update(
+                            tlc.get('planned_work_contacts', []))
                 circ['contacts'] = sorted(list(contacts))
                 circ['planned_work_contacts'] = sorted(list(pw_contacts))
 

setup.py

@@ -2,7 +2,7 @@ from setuptools import setup, find_packages
 
 setup(
     name='inventory-provider',
-    version="0.100",
+    version="0.101",
     author='GEANT',
     author_email='swd@geant.org',
     description='Dashboard inventory provider',

test/test_ims.py

@@ -139,7 +139,7 @@ def test_ims_class_get_all_entities(mocker):
 
     list(ds.get_all_entities('Node', step_count=10))
     mock_get.assert_called_once_with(
-        'dummy_base/ims/Node/filtered/Id <> 0',
+        'dummy_base/ims/Node/all',
         headers={'Authorization': 'Bearer dummy_bt'},
         params={
             'paginatorStartElement': 0,

tox.ini

@@ -5,7 +5,7 @@ envlist = py36
 exclude = venv,.tox,build
 
 [testenv]
-passenv = TEST_OPSDB_HOSTNAME TEST_OPSDB_DBNAME TEST_OPSDB_USERNAME TEST_OPSDB_PASSWORD
+passenv = TEST_OPSDB_HOSTNAME,TEST_OPSDB_DBNAME,TEST_OPSDB_USERNAME,TEST_OPSDB_PASSWORD
 deps =
     coverage
     flake8