Merge branch 'feature/DBOARD3-305' into develop

inventory_provider/__init__.py

@@ -48,6 +48,9 @@ def create_app():
     from inventory_provider.routes import ims_lg
     app.register_blueprint(ims_lg.routes, url_prefix='/ims-lg')
 
+    from inventory_provider.routes import ims_data
+    app.register_blueprint(ims_data.routes, url_prefix='/ims-data')
+
     # end of IMS based routes
 
     # OTRS routes

inventory_provider/db/ims.py

@@ -64,7 +64,8 @@ PORT_PROPERTIES = {
 SITE_PROPERTIES = {
     'City': 2,
     'SiteAliases': 64,
-    'Country': 256
+    'Country': 256,
+    'Nodes': 32768
 }
 # http://149.210.162.190:81/ImsVersions/4.19.9/html/8ce06cb7-7707-46c4-f02f-86083310d81b.htm  # noqa
 VENDOR_PROPERTIES = {

inventory_provider/db/ims_data.py

@@ -10,37 +10,46 @@ logger = logging.getLogger(__name__)
 
 # Dashboard V3
 
+IMS_OPSDB_STATUS_MAP = {
+    InventoryStatus.PLANNED: 'Planned',
+    InventoryStatus.READY_FOR_SERVICE: 'Installed',
+    InventoryStatus.IN_SERVICE: 'Operational',
+    InventoryStatus.MIGRATION: 'Planned',
+    InventoryStatus.OUT_OF_SERVICE: 'Terminated',
+    InventoryStatus.READY_FOR_CEASURE: 'Disposed'
+}
+
 
-def lookup_pop_info(ds, hostname):
+def get_node_locations(ds):
     site_nav_props = [
         ims.SITE_PROPERTIES['City'],
         ims.SITE_PROPERTIES['SiteAliases'],
-        ims.SITE_PROPERTIES['Country']
+        ims.SITE_PROPERTIES['Country'],
+        ims.SITE_PROPERTIES['Nodes']
     ]
-
-    node = ds.get_entity_by_name('Node', hostname)
-    if not node:
-        return None
-    site = ds.get_entity_by_id('Site', node['SiteId'], site_nav_props, True)
-    city = site['City']
-    abbreviation = ''
-    try:
-        abbreviation = site['SiteAliases'][0]['AliasName']
-    except IndexError:
-        pass  # no alias - ignore silently
-    eq = {
-            'equipment-name': node['Name'],
-            'status': InventoryStatus(node['InventoryStatusId']).name,
-            'pop': {
-                'name': site['Name'],
-                'city': city['Name'],
-                'country': city['Country']['Name'],
-                'abbreviation': abbreviation,
-                'longitude': site['Longitude'],
-                'latitude': site['Latitude'],
-            }
-        }
-    return eq
+    sites = ds.get_all_entities('Site', site_nav_props, step_count=500)
+    for site in sites:
+        city = site['city']
+        abbreviation = ''
+        try:
+            abbreviation = site['sitealiases'][0]['aliasname']
+        except IndexError:
+            pass  # no alias - ignore silently
+
+        for node in site['nodes']:
+            yield (node['name'], {
+                'equipment-name': node['name'],
+                'status': IMS_OPSDB_STATUS_MAP.get(
+                    InventoryStatus(node['inventorystatusid']), 'unknown'),
+                'pop': {
+                    'name': site['name'],
+                    'city': city['name'],
+                    'country': city['country']['name'],
+                    'abbreviation': abbreviation,
+                    'longitude': site['longitude'],
+                    'latitude': site['latitude'],
+                }
+            })
 
 
 # End of Dashboard V3 stuff
@@ -114,7 +123,7 @@ def lookup_lg_routers(ds):
                         'latitude': site['latitude'],
                     }
                 }
-            yield(eq)
+            yield eq
 
 
 def otrs_get_customer_company_rows(ds):

inventory_provider/routes/common.py

 import functools
 import logging
+from collections import OrderedDict
 
+import requests
 from flask import request, Response, current_app, g
 from inventory_provider.tasks import common as tasks_common
 
 logger = logging.getLogger(__name__)
 
 
+def ims_hostname_decorator(field):
+    """
+    Decorator to convert host names to various formats to try to match what is
+    found in IMS before executing the decorated function.
+    :param field: name of the field containing hostname
+    :return: result of decorated function
+    """
+
+    suffix = '.geant.net'
+
+    def wrapper(func):
+        def inner(*args, **kwargs):
+            orig_val = kwargs[field]
+            values_to_try = []
+            if orig_val.endswith(suffix):
+                values_to_try.append(orig_val[:-len(suffix)].upper())
+            values_to_try.append(orig_val.upper())
+            values_to_try.append(orig_val)
+            values_to_try.append(orig_val.lower())
+
+            for val in list(OrderedDict.fromkeys(values_to_try)):
+                kwargs[field] = val
+                res = func(*args, **kwargs)
+                if res.status_code != requests.codes.not_found:
+                    return res
+            return res
+        return inner
+    return wrapper
+
+
 def get_current_redis():
     if 'current_redis_db' in g:
         latch = tasks_common.get_latch(g.current_redis_db)

inventory_provider/routes/ims_data.py

+from flask import Blueprint, Response
+
+from inventory_provider.routes import common
+
+routes = Blueprint("ims-inventory-data-query-routes", __name__)
+
+
+@routes.after_request
+def after_request(resp):
+    return common.after_request(resp)
+
+
+@routes.route("/pop/<equipment_name>", methods=['GET', 'POST'])
+@common.require_accepts_json
+@common.ims_hostname_decorator('equipment_name')
+def equipment_location(equipment_name):
+    redis = common.get_current_redis()
+    result = redis.get(f'ims:location:{equipment_name}')
+
+    if not result:
+        return Response(
+            response="no available info for {}".format(equipment_name),
+            status=404,
+            mimetype="text/html")
+
+    return Response(result, mimetype="application/json")

inventory_provider/routes/testing.py

@@ -21,6 +21,12 @@ def flushdb():
 
 # IMS routes
 
+@routes.route("update-equipment-locations-ims", methods=['GET', 'POST'])
+def update_equipment_locations_ims():
+    ims_worker.update_equipment_locations_ims.delay(use_current=True)
+    return Response('OK')
+
+
 @routes.route("update-lg-routers-ims", methods=['GET', 'POST'])
 def update_lg_routers_ims():
     ims_worker.update_lg_routers_ims.delay(use_current=True)

inventory_provider/tasks/ims_worker.py

@@ -21,6 +21,34 @@ environment.setup_logging()
 logger = logging.getLogger(__name__)
 
 
+@app.task(base=InventoryTask, bind=True, name='update_equipment_locations_ims')
+@log_task_entry_and_exit
+def update_equipment_locations_ims(self, use_current=False):
+
+    if use_current:
+        r = get_current_redis(InventoryTask.config)
+        # scan with bigger batches, to mitigate network latency effects
+    else:
+        r = get_next_redis(InventoryTask.config)
+    rp = r.pipeline()
+    for k in r.scan_iter('ims:location:*', count=1000):
+        rp.delete(k)
+    rp.execute()
+
+    c = InventoryTask.config["ims"]
+    ds = IMS(c['api'], c['username'], c['password'])
+
+    rp = r.pipeline()
+    hostnames_found = set()
+    for h, d in ims_data.get_node_locations(ds):
+        # put into a list to match non-IMS version
+        rp.set(f'ims:location:{h}', json.dumps([d]))
+        if h in hostnames_found:
+            print(f'Multiple entries for {h}')
+        hostnames_found.add(h)
+    rp.execute()
+
+
 @app.task(base=InventoryTask, bind=True, name='update_lg_routers_ims')
 @log_task_entry_and_exit
 def update_lg_routers_ims(self, use_current=False):

test/test_ims_data.py

@@ -2,9 +2,9 @@ import json
 
 import inventory_provider
 from inventory_provider.db.ims import InventoryStatus
-from inventory_provider.db.ims_data import lookup_pop_info, \
-    lookup_lg_routers, otrs_get_customer_company_rows, \
-    otrs_get_customer_users_rows
+from inventory_provider.db.ims_data import lookup_lg_routers, \
+    otrs_get_customer_company_rows, \
+    otrs_get_customer_users_rows, get_node_locations, IMS_OPSDB_STATUS_MAP
 
 
 def test_lookup_lg_routers(mocker):
@@ -59,32 +59,28 @@ def test_lookup_lg_routers(mocker):
     }
 
 
-def test_lookup_pop_info(mocker):
-
+def test_get_node_location(mocker):
     ims = mocker.patch('inventory_provider.db.ims.IMS')
-    with open('test/data/ims_pop_info_mx1_pra_cz.json') as data:
+    with open('test/data/ims_nodes_data.json') as data:
         resp_data = json.load(data)
-    ims.return_value.get_entity_by_name.return_value = resp_data['router']
-    ims.return_value.get_entity_by_id.return_value = resp_data['site']
+    ims.return_value.get_all_entities.return_value = resp_data
 
     ds = inventory_provider.db.ims.IMS(
         'dummy_base', 'dummy_username', 'dummy_password')
-    res = lookup_pop_info(ds, 'dummy_host')
-    ds.get_entity_by_name.assert_called_once_with('Node', 'dummy_host')
-    ds.get_entity_by_id.assert_called_once_with(
-        'Site', 445312, [2, 64, 256], True)
-    assert res == {
-        'equipment-name': 'MX1.PRA.CZ',
-        'status': InventoryStatus.IN_SERVICE.name,
+    res = list(get_node_locations(ds))
+    assert len(res) == 35
+    assert res[0] == ('LON3_CX_01', {
+        'equipment-name': 'LON3_CX_01',
+        'status': IMS_OPSDB_STATUS_MAP[InventoryStatus.IN_SERVICE],
         'pop': {
-            'name': 'PRAGUE',
-            'city': 'PRAGUE',
-            'country': 'CZECH REPUBLIC',
-            'abbreviation': 'PRA',
-            'longitude': 14.39166667,
-            'latitude': 50.10166667,
+            'name': 'LONDON 3 POWERGATE',
+            'city': 'LONDON',
+            'country': 'UNITED KINGDOM',
+            'abbreviation': 'LON3',
+            'longitude': -0.257712,
+            'latitude': 51.5308142,
         }
-    }
+    })
 
 
 def test_otrs_get_customer_company_rows(mocker):