Finished release 0.48.

Changelog.md

@@ -2,6 +2,11 @@
 
 All notable changes to this project will be documented in this file.
 
+## [0.48] - 2020-07-02
+- recover update gracefully in case of Kombu exceptions
+- update for IMS api changes
+- call otrs-exports in the web server thread, not as a celery task
+
 ## [0.47] - 2020-06-06
 - bugfix when catching junos rpc exceptions
 

Jenkinsfile

+// make sure the pip cache is accessible
+def container_args = "-v ${env.JENKINS_HOME}/.cache:/cache:rw,z -e XDG_CACHE_HOME=/cache"
+
+pipeline {
+    agent none
+    stages {
+        stage('init') {
+            agent any
+            steps {
+                script {
+                    def matcher = env.GIT_URL =~ /\/(.*)\.git/
+
+                    if(matcher.find()) {
+                        gitProjectId = matcher[0][1];
+                    }
+                }
+                echo gitProjectId
+            }
+        }
+        stage('test') {
+            parallel {
+
+                stage('test Python 3.6.8 (Centos 7)') {
+                    agent {
+                        docker {
+                            image 'python:3.6.8'
+                            args container_args
+                        }
+                    }
+                    steps {
+                        run_unit_tests('py36')
+                    }
+                }
+
+                stage('test Python 3.7.6') {
+                    agent {
+                        docker {
+                            image 'python:3.7.6'
+                            args container_args
+                        }
+                    }
+                    steps {
+                        run_unit_tests('py37')
+                    }
+                }
+
+                stage('test Python 3.8.3') {
+                    agent {
+                        docker {
+                            image 'python:3.8.3'
+                            args container_args
+                        }
+                    }
+                    steps {
+                        run_unit_tests('py38')
+                    }
+                }
+
+            }
+        }
+        stage('SonarQube analysis') {
+            agent any
+            steps {
+                script {
+                    // must match 'Name' from Jenkins 'Global Tool Configuration'
+                    scannerHome = tool 'sonar-scaner';
+                }
+                // must match 'Name' from Jenkins 'Configure System'
+                withSonarQubeEnv('Project SonarQube') {
+                    sh "${scannerHome}/bin/sonar-scanner -Dproject.settings=./sonar.properties"
+                }
+            }
+
+        }
+    }
+}
+
+void run_unit_tests(tox_env) {
+    sh 'python -m venv venv'
+    sh 'venv/bin/pip install tox'
+    sh "venv/bin/tox -e $tox_env"
+}
+
+

inventory_provider/db/ims.py

@@ -6,8 +6,15 @@ from enum import Enum
 
 # Navigation Properties
 # http://149.210.162.190:81/ImsVersions/4.19.9/html/86d07a57-fa45-835e-d4a2-a789c4acbc96.htm  # noqa
+from requests import HTTPError
+
 CIRCUIT_PROPERTIES = {
-    'InternalPorts': 1024
+    'Ports': 512,
+    'InternalPorts': 1024,
+    'SubCircuits': 131072,
+    'PortsFullDetails': 262144,
+    'PortA': 34359738368,
+    'PortB': 68719476736
 }
 # http://149.210.162.190:81/ImsVersions/4.19.9/html/dbc969d0-e735-132e-6281-f724c6d7da64.htm  # NOQA
 CONTACT_PROPERTIES = {
@@ -21,22 +28,37 @@ CUSTOMER_PROPERTIES = {
     'CustomerRelatedContacts': 32768,
     'CustomerType': 262144
 }
+# http://149.210.162.190:81/ImsVersions/4.19.9/html/347cb410-8c05-47bd-ceb0-d1dd05bf98a4.htm  # noqa
+CITY_PROPERTIES = {
+    'Country': 8
+}
+# http://149.210.162.190:81/ImsVersions/4.19.9/html/a8dc6266-d934-8162-4a55-9e1648187f2c.htm  # noqa
+EQUIP_DEF_PROPERTIES = {
+    'Nodes': 4096
+}
+# http://149.210.162.190:81/ImsVersions/4.19.9/html/f18222e3-e353-0abe-b89c-820db87940ac.htm  # noqa
+INTERNAL_PORT_PROPERTIES = {
+    'Node': 8,
+    'Shelf': 64,
+    'Circuit': 256,
+}
 # http://149.210.162.190:81/ImsVersions/4.19.9/html/2d27d509-77cb-537d-3ffa-796de7e82af8.htm  # noqa
 NODE_PROPERTIES = {
     'EquipmentDefinition': 16,
     'ManagementSystem': 128,
     'Site': 256,
+    'Shelves': 1024,
+    'Ports': 4096,
+    'InternalPorts': 8192,
     'NodeCity': 32768,
     'Order': 67108864,
     'NodeCountry': 1073741824
 }
-# http://149.210.162.190:81/ImsVersions/4.19.9/html/347cb410-8c05-47bd-ceb0-d1dd05bf98a4.htm  # noqa
-CITY_PROPERTIES = {
-    'Country': 8
-}
-# http://149.210.162.190:81/ImsVersions/4.19.9/html/a8dc6266-d934-8162-4a55-9e1648187f2c.htm  # noqa
-EQUIP_DEF_PROPERTIES = {
-    'Nodes': 4096
+# http://149.210.162.190:81/ImsVersions/4.19.9/html/199f32b5-5104-fec7-8787-0d730113e902.htm  # noqa
+PORT_PROPERTIES = {
+    'Node': 16,
+    'Shelf': 32,
+    'Circuit': 512,
 }
 # http://149.210.162.190:81/ImsVersions/4.19.9/html/9c8d50f0-842c-5959-0fa6-14e1720669ec.htm  # noqa
 SITE_PROPERTIES = {
@@ -55,6 +77,8 @@ VENDOR_RELATED_CONTACT_PROPERTIES = {
     'Contact': 16
 }
 
+NO_FILTERED_RESULTS_MESSAGE = 'no records found for entity:'
+
 
 class InventoryStatus(Enum):
     PLANNED = 1
@@ -81,7 +105,6 @@ class IMSError(Exception):
 class IMS(object):
 
     TIMEOUT_THRESHOLD = 1200
-    NO_OF_ELEMENTS_PER_ITERATION = 50
     PERMITTED_RECONNECT_ATTEMPTS = 3
     LOGIN_PATH = '/login'
     IMS_PATH = '/ims'
@@ -147,7 +170,12 @@ class IMS(object):
             if response_.status_code == requests.codes.unauthorized:
                 return True
 
-            if response_.status_code == requests.codes.ok:
+            if response_.status_code in (requests.codes.ok,
+                                         requests.codes.not_found):
+
+                if NO_FILTERED_RESULTS_MESSAGE in response_.text.lower():
+                    return False
+
                 r = response_.json()
                 if r and 'HasErrors' in r and r['HasErrors']:
                     for e in r['Errors']:
@@ -209,34 +237,47 @@ class IMS(object):
             entity,
             filter_string,
             navigation_properties=None,
-            use_cache=False):
+            use_cache=False,
+            step_count=50):
         more_to_come = True
         start_entity = 0
         while more_to_come:
             params = {
                 'paginatorStartElement': start_entity,
-                'paginatorNumberOfElements': IMS.NO_OF_ELEMENTS_PER_ITERATION
+                'paginatorNumberOfElements': step_count
             }
             url = f'{entity}/filtered/{filter_string}'
-            entities = self._get_entity(
-                url,
-                params,
-                navigation_properties,
-                use_cache)
-            more_to_come = False
+            try:
+                more_to_come = False
+                entities = self._get_entity(
+                    url,
+                    params,
+                    navigation_properties,
+                    use_cache)
+            except HTTPError as e:
+                r = e.response
+                if r.status_code == requests.codes.not_found \
+                        and NO_FILTERED_RESULTS_MESSAGE in r.text.lower():
+                    entities = None
+                else:
+                    raise e
             if entities:
                 more_to_come = \
-                    len(entities) >= IMS.NO_OF_ELEMENTS_PER_ITERATION
-                start_entity += IMS.NO_OF_ELEMENTS_PER_ITERATION
+                    len(entities) >= step_count
+                start_entity += step_count
                 yield from entities
 
     def get_all_entities(
             self,
             entity,
             navigation_properties=None,
-            use_cache=False):
+            use_cache=False,
+            step_count=50
+    ):
         yield from self.get_filtered_entities(
             entity,
             'Id <> 0',
             navigation_properties,
-            use_cache)
+            use_cache,
+            step_count
+        )

inventory_provider/routes/ims_otrs.py

@@ -63,9 +63,9 @@ def send_exports():
                 response=html.escape(f'Bad value for <files> {files_value}'),
                 status=requests.codes.bad_request,
                 mimetype="text/html")
-    task = export_data_for_otrs.delay(
+    debug_uuid = export_data_for_otrs(
         files_to_export=files_value, export_duplicates=duplicates)
     return Response(
-        response=task.id,
+        response=f'Exports sent, search logs for {debug_uuid} for details',
         status=requests.codes.ok,
         mimetype="text/html")

inventory_provider/tasks/ims_worker.py

@@ -6,6 +6,7 @@ import tempfile
 from datetime import datetime
 from enum import IntFlag
 from pathlib import Path
+from uuid import uuid4
 
 from inventory_provider.db import ims_data
 from inventory_provider.db.ims import IMS
@@ -48,6 +49,8 @@ class OTRSFiles(IntFlag):
 @app.task(base=InventoryTask, bind=True, name='export_data_for_otrs')
 @log_task_entry_and_exit
 def export_data_for_otrs(self, files_to_export=None, export_duplicates=False):
+    debug_uuid = uuid4()
+    logger.debug(f'debug uuid: {debug_uuid}')
     if files_to_export:
         files_to_export = OTRSFiles(files_to_export)
     else:
@@ -56,7 +59,7 @@ def export_data_for_otrs(self, files_to_export=None, export_duplicates=False):
     ims_config = InventoryTask.config["ims"]
     otrs_config = InventoryTask.config["otrs-export"]
 
-    command_template = 'rsync -aP --rsh="ssh -l {user} -p 22 -i {key_file} -o \'UserKnownHostsFile {known_hosts}\'" {source_dir}/* {destination}'  # noqa
+    command_template = 'rsync -aPq --rsh="ssh -l {user} -p 22 -i {key_file} -o \'UserKnownHostsFile {known_hosts}\'" {source_dir}/* {destination}'  # noqa
 
     with tempfile.TemporaryDirectory() as temp_dir:
         temp_path = Path(temp_dir)
@@ -88,3 +91,4 @@ def export_data_for_otrs(self, files_to_export=None, export_duplicates=False):
             destination=otrs_config['destination']
         )
         subprocess.run(command, shell=True, check=True)
+    return debug_uuid

inventory_provider/tasks/worker.py

+import enum
 import json
 import logging
 import os
 import time
 
 from redis.exceptions import RedisError
+from kombu.exceptions import KombuError
 
 from celery import Task, states
 from celery.result import AsyncResult
@@ -534,20 +536,29 @@ def internal_refresh_phase_2(self):
     # second batch of subtasks:
     #   alarms db status cache
     #   juniper netconf & snmp data
-    subtasks = [
-        update_equipment_locations.apply_async(),
-        update_lg_routers.apply_async(),
-        update_access_services.apply_async(),
-        import_unmanaged_interfaces.apply_async()
-    ]
-    for hostname in data.derive_router_hostnames(InventoryTask.config):
-        logger.debug('queueing router refresh jobs for %r' % hostname)
-        subtasks.append(reload_router_config.apply_async(args=[hostname]))
+    try:
 
-    pending_task_ids = [x.id for x in subtasks]
+        subtasks = [
+            update_equipment_locations.apply_async(),
+            update_lg_routers.apply_async(),
+            update_access_services.apply_async(),
+            import_unmanaged_interfaces.apply_async()
+        ]
+        for hostname in data.derive_router_hostnames(InventoryTask.config):
+            logger.debug('queueing router refresh jobs for %r' % hostname)
+            subtasks.append(reload_router_config.apply_async(args=[hostname]))
 
-    t = refresh_finalizer.apply_async(args=[json.dumps(pending_task_ids)])
-    pending_task_ids.append(t.id)
+        pending_task_ids = [x.id for x in subtasks]
+
+        refresh_finalizer.apply_async(args=[json.dumps(pending_task_ids)])
+
+    except KombuError:
+        # TODO: possible race condition here
+        # e.g. if one of these tasks takes a long time and another
+        # update is started, we could end up with strange data
+        update_latch_status(config, pending=False, failure=True)
+        logger.exception('error launching refresh phase 2 subtasks')
+        raise
 
 
 @log_task_entry_and_exit
@@ -578,7 +589,7 @@ def launch_refresh_cache_all(config):
         t = internal_refresh_phase_2.apply_async()
         return t.id
 
-    except RedisError:
+    except (RedisError, KombuError):
         update_latch_status(config, pending=False, failure=True)
         logger.exception('error launching refresh subtasks')
         raise
@@ -646,7 +657,8 @@ def refresh_finalizer(self, pending_task_ids_json):
     except (jsonschema.ValidationError,
             json.JSONDecodeError,
             InventoryTaskError,
-            RedisError) as e:
+            RedisError,
+            KombuError) as e:
         update_latch_status(InventoryTask.config, failure=True)
         raise e
 
@@ -654,15 +666,19 @@ def refresh_finalizer(self, pending_task_ids_json):
     self.log_info('latched current/next dbs')
 
 
+class PollerServiceCategory(str, enum.Enum):
+    MDVPN = 'mdvpn'
+    LHCONE = 'lhcone'
+
+
 @log_task_entry_and_exit
 def _build_service_category_interface_list(update_callback=lambda s: None):
 
     def _classify(ifc):
         if ifc['description'].startswith('SRV_MDVPN'):
-            return 'mdvpn'
+            yield PollerServiceCategory.MDVPN
         if 'LHCONE' in ifc['description']:
-            return 'lhcone'
-        return None
+            yield PollerServiceCategory.LHCONE
 
     update_callback('loading all known interfaces')
     interfaces = data.build_service_interface_user_list(InventoryTask.config)
@@ -674,13 +690,11 @@ def _build_service_category_interface_list(update_callback=lambda s: None):
     rp = r.pipeline()
 
     for ifc in interfaces:
-        service_type = _classify(ifc)
-        if not service_type:
-            continue
-        rp.set(
-            f'interface-services:{service_type}'
-            f':{ifc["router"]}:{ifc["interface"]}',
-            json.dumps(ifc))
+        for service_category in _classify(ifc):
+            rp.set(
+                f'interface-services:{service_category.value}'
+                f':{ifc["router"]}:{ifc["interface"]}',
+                json.dumps(ifc))
 
     rp.execute()
 

setup.py

@@ -2,7 +2,7 @@ from setuptools import setup, find_packages
 
 setup(
     name='inventory-provider',
-    version="0.47",
+    version="0.48",
     author='GEANT',
     author_email='swd@geant.org',
     description='Dashboard inventory provider',

sonar.properties

+sonar.projectKey=inventory-provider
+sonar.projectName=Inventory Provider
+sonar.projectVersion=0.x
+sonar.sources=inventory_provider
+sonar.python.coverage.reportPaths=coverage.xml

test/test_ims.py

+from requests import HTTPError
+
 import inventory_provider
 
 
@@ -5,6 +7,7 @@ class MockResponse:
     def __init__(self, json_data, status_code):
         self.json_data = json_data
         self.status_code = status_code
+        self.text = '' if json_data else 'No records found for Entity:XXXXX'
 
     def json(self):
         return self.json_data
@@ -59,18 +62,16 @@ def test_ims_class_filtered_entities(mocker):
     ds = inventory_provider.db.ims.IMS(
         'dummy_base', 'dummy_username', 'dummy_password', 'dummy_bt')
 
-    list(ds.get_filtered_entities('Node', 'dummy_param=dummy value'))
+    list(ds.get_filtered_entities(
+        'Node', 'dummy_param=dummy value', step_count=50))
     mock_get.assert_called_once_with(
         'dummy_base/ims/Node/filtered/dummy_param=dummy value',
         headers={'Authorization': 'Bearer dummy_bt'},
         params={
             'paginatorStartElement': 0,
-            'paginatorNumberOfElements':
-                inventory_provider.db.ims.IMS.NO_OF_ELEMENTS_PER_ITERATION
+            'paginatorNumberOfElements': 50
         })
 
-    inventory_provider.db.ims.IMS.NO_OF_ELEMENTS_PER_ITERATION = 2
-
     def side_effect(*args, **kargs):
         if kargs['params']['paginatorStartElement'] == 0:
             return MockResponse([1, 2], 200)
@@ -79,7 +80,8 @@ def test_ims_class_filtered_entities(mocker):
     mock_multi_get = mocker.patch(
         'inventory_provider.db.ims.requests.get', side_effect=side_effect)
 
-    res = list(ds.get_filtered_entities('Node', 'dummy_param=dummy value'))
+    res = list(ds.get_filtered_entities(
+        'Node', 'dummy_param=dummy value', step_count=2))
     mock_multi_get.assert_any_call(
         'dummy_base/ims/Node/filtered/dummy_param=dummy value',
         headers={'Authorization': 'Bearer dummy_bt'},
@@ -97,20 +99,33 @@ def test_ims_class_filtered_entities(mocker):
     assert mock_multi_get.call_count == 2
     assert res == [1, 2, 3]
 
+    def side_effect_no_recs(*args, **kargs):
+        if kargs['params']['paginatorStartElement'] == 0:
+            return MockResponse([1, 2], 200)
+        e = HTTPError()
+        e.response = MockResponse('', 404)
+        raise e
+
+    mocker.patch('inventory_provider.db.ims.requests.get',
+                 side_effect=side_effect_no_recs)
+
+    res = list(ds.get_filtered_entities(
+        'Node', 'dummy_param=dummy value', step_count=2))
+    assert res == [1, 2]
+
 
 def test_ims_class_get_all_entities(mocker):
     mock_get = mocker.patch('inventory_provider.db.ims.requests.get')
     ds = inventory_provider.db.ims.IMS(
         'dummy_base', 'dummy_username', 'dummy_password', 'dummy_bt')
 
-    list(ds.get_all_entities('Node'))
+    list(ds.get_all_entities('Node', step_count=10))
     mock_get.assert_called_once_with(
         'dummy_base/ims/Node/filtered/Id <> 0',
         headers={'Authorization': 'Bearer dummy_bt'},
         params={
             'paginatorStartElement': 0,
-            'paginatorNumberOfElements':
-                inventory_provider.db.ims.IMS.NO_OF_ELEMENTS_PER_ITERATION
+            'paginatorNumberOfElements': 10
         })
 
 

test/test_ims_worker.py

@@ -33,7 +33,7 @@ def test_otrs_exports(data_config_filename, data_config, mocker):
     args, kwargs = mocked_run.call_args
     called_with = args[0]
 
-    t = r'^rsync -aP --rsh="ssh -l {user} -p 22 -i {key_file} -o \'UserKnownHostsFile {known_hosts}\'" /\S+/\* {destination}$'  # noqa
+    t = r'^rsync -aPq --rsh="ssh -l {user} -p 22 -i {key_file} -o \'UserKnownHostsFile {known_hosts}\'" /\S+/\* {destination}$'  # noqa
 
     p = t.format(
         user=otrs_config['username'],