weeded duplicates and provided option to exclude remaining duplicates

inventory_provider/db/ims_data.py

@@ -5,10 +5,46 @@ from collections import OrderedDict
 from inventory_provider import environment
 from inventory_provider.db import ims
 from inventory_provider.db.ims import InventoryStatus
-
 environment.setup_logging()
 logger = logging.getLogger(__name__)
 
+# Dashboard V3
+
+
+def lookup_pop_info(ds, hostname):
+    site_nav_props = [
+        ims.SITE_PROPERTIES['City'],
+        ims.SITE_PROPERTIES['SiteAliases'],
+        ims.SITE_PROPERTIES['Country']
+    ]
+
+    node = ds.get_entity_by_name('Node', hostname)
+    if not node:
+        return None
+    site = ds.get_entity_by_id('Site', node['SiteId'], site_nav_props, True)
+    city = site['City']
+    abbreviation = ''
+    try:
+        abbreviation = site['SiteAliases'][0]['AliasName']
+    except IndexError:
+        pass  # no alias - ignore silently
+    eq = {
+            'equipment-name': node['Name'],
+            'status': InventoryStatus(node['InventoryStatusId']).name,
+            'pop': {
+                'name': site['Name'],
+                'city': city['Name'],
+                'country': city['Country']['Name'],
+                'abbreviation': abbreviation,
+                'longitude': site['Longitude'],
+                'latitude': site['Latitude'],
+            }
+        }
+    return eq
+
+
+# End of Dashboard V3 stuff
+
 INTERNAL_POP_NAMES = {
     'Cambridge OC',
     'DANTE Lab',
@@ -81,38 +117,6 @@ def lookup_lg_routers(ds):
             yield(eq)
 
 
-def lookup_pop_info(ds, hostname):
-    site_nav_props = [
-        ims.SITE_PROPERTIES['City'],
-        ims.SITE_PROPERTIES['SiteAliases'],
-        ims.SITE_PROPERTIES['Country']
-    ]
-
-    node = ds.get_entity_by_name('Node', hostname)
-    if not node:
-        return None
-    site = ds.get_entity_by_id('Site', node['SiteId'], site_nav_props, True)
-    city = site['City']
-    abbreviation = ''
-    try:
-        abbreviation = site['SiteAliases'][0]['AliasName']
-    except IndexError:
-        pass  # no alias - ignore silently
-    eq = {
-            'equipment-name': node['Name'],
-            'status': InventoryStatus(node['InventoryStatusId']).name,
-            'pop': {
-                'name': site['Name'],
-                'city': city['Name'],
-                'country': city['Country']['Name'],
-                'abbreviation': abbreviation,
-                'longitude': site['Longitude'],
-                'latitude': site['Latitude'],
-            }
-        }
-    return eq
-
-
 def otrs_get_customer_company_rows(ds):
     yield ['customer_id', 'name', 'street', 'zip', 'city', 'country', 'url',
            'comments']
@@ -215,7 +219,7 @@ def otrs_get_vendor_contacts(ds):
         yield t_customer_user
 
 
-def otrs_get_customer_users_rows(ds):
+def otrs_get_customer_users_rows(ds, return_duplicates=False):
     yield ['email', 'username', 'customer_id', 'customer_id_2', 'title',
            'firstname', 'lastname', 'phone', 'fax', 'mobile', 'street', 'zip',
            'city', 'country', 'comments']
@@ -229,18 +233,59 @@ def otrs_get_customer_users_rows(ds):
             if c['email'] not in yielded_customer_emails:
                 yield c
 
-    sorted_cus = sorted(get_all_cus_user_rows(), key=lambda x: x['email'])
+    def weed_duplicates(duplicates):
+        # this is here to allow for additional rules
+        id_rank = {
+            'geant': 1
+        }
+        top_rank = -1
+        top_ranked = None
+        for d in duplicates:
+            rank = id_rank.get(d['customer_id'].lower(), 0)
+            if rank > top_rank:
+                top_rank = rank
+                top_ranked = []
+            if rank == top_rank:
+                top_ranked.append(d)
+        return top_ranked
+
+    cus_by_email = {}
     duplicate_emails = set()
-    previous_email = None
-    for cu in sorted_cus:
-        if cu['email'] == previous_email:
-            duplicate_emails.add(previous_email)
-        previous_email = cu['email']
+    for cu in get_all_cus_user_rows():
+        email = cu['email']
+        cus_for_email = cus_by_email.get(email, [])
+        if cus_for_email:
+            duplicate_emails.add(email)
+        cus_for_email.append(cu)
+        cus_by_email[email] = cus_for_email
+
+    remaining_duplicates = []
     if duplicate_emails:
-        logger.error('Duplicate emails found in OTRS customer-user export: '
-                     f'{duplicate_emails}')
-        # raise KeyError('Duplicate emails found in OTRS customer-user export: '  # noqa
-        #                f'{duplicate_emails}')
-
-    for cu in sorted_cus:
-        yield list(cu.values())
+        logger.info('Duplicate emails found in OTRS customer-user export: '
+                    f'{duplicate_emails} - attempting to weed')
+        # weeded = []
+        for email in duplicate_emails:
+            weeded = weed_duplicates(cus_by_email.pop(email))
+            if len(weeded) == 1:
+                cus_by_email[email] = weeded
+            else:
+                remaining_duplicates.extend(
+                    sorted(
+                        [list(w.values()) for w in weeded], key=lambda x: x[2])
+                )
+
+        if remaining_duplicates:
+            # need guidance what to do if this occurs, should we pick the first
+            # one for each, or ignore them completely? Ignoring them for now
+            logger.error('Duplicate emails remain after weeding, '
+                         f'{"including" if return_duplicates else "excluding"}'
+                         ' duplicates in returned data: ')
+
+            for rd in remaining_duplicates:
+                logger.debug(rd)
+
+    for email, details in sorted(cus_by_email.items()):
+        yield list(details[0].values())
+
+    if return_duplicates:
+        yield from remaining_duplicates

inventory_provider/routes/ims_otrs.py

@@ -18,16 +18,11 @@ def after_request(resp):
     return common.after_request(resp)
 
 
-def get_otrs_data(fn):
-    ims_config = current_app.config['INVENTORY_PROVIDER_CONFIG']["ims"]
-    ds = IMS(
-        ims_config['api'],
-        ims_config['username'],
-        ims_config['password'])
+def get_otrs_output(result):
 
     with StringIO() as sio:
         writer = csv.writer(sio, delimiter='^')
-        writer.writerows(fn(ds))
+        writer.writerows(result)
         data = sio.getvalue()
     return Response(
         response=data,
@@ -37,17 +32,24 @@ def get_otrs_data(fn):
 
 @routes.route('customer-companies')
 def get_customer_companies_data():
-    return get_otrs_data(ims_data.otrs_get_customer_company_rows)
+    ims_config = current_app.config['INVENTORY_PROVIDER_CONFIG']["ims"]
+    ds = IMS(ims_config['api'], ims_config['username'], ims_config['password'])
+    return get_otrs_output(ims_data.otrs_get_customer_company_rows(ds))
 
 
 @routes.route('customer-users')
 def get_customer_users_data():
-    return get_otrs_data(ims_data.otrs_get_customer_users_rows)
+    ims_config = current_app.config['INVENTORY_PROVIDER_CONFIG']["ims"]
+    ds = IMS(ims_config['api'], ims_config['username'], ims_config['password'])
+    return_duplicates = request.args.get('duplicates', 'f').lower() == 'true'
+    return get_otrs_output(ims_data.otrs_get_customer_users_rows(
+        ds, return_duplicates=return_duplicates))
 
 
 @routes.route('export')
 def send_exports():
     files_value = request.args.get('files', None)
+    duplicates = request.args.get('duplicates', 'f').lower() == 'true'
     if files_value:
         try:
             files_value = int(files_value)
@@ -61,7 +63,8 @@ def send_exports():
                 response=html.escape(f'Bad value for <files> {files_value}'),
                 status=requests.codes.bad_request,
                 mimetype="text/html")
-    task = export_data_for_otrs.delay(files_value)
+    task = export_data_for_otrs.delay(
+        files_to_export=files_value, export_duplicates=duplicates)
     return Response(
         response=task.id,
         status=requests.codes.ok,

inventory_provider/routes/otrs_jobs.py

-import html
-
-import requests
-from flask import Blueprint, request, Response
-
-from inventory_provider.routes import common
-from inventory_provider.tasks.ims_worker import OTRSFiles, export_data_for_otrs
-
-routes = Blueprint("otrs", __name__)
-
-
-@routes.after_request
-def after_request(resp):
-    return common.after_request(resp)
-
-
-@routes.route('export')
-def send_exports():
-    files_value = request.args.get('files', None)
-    if files_value:
-        try:
-            files_value = int(files_value)
-        except ValueError:
-            return Response(
-                response=html.escape('<files> should be an Integer'),
-                status=requests.codes.bad_request,
-                mimetype="text/html")
-        if files_value < 0 or files_value > sum(OTRSFiles):
-            return Response(
-                response=html.escape(f'Bad value for <files> {files_value}'),
-                status=requests.codes.bad_request,
-                mimetype="text/html")
-    task = export_data_for_otrs.delay(files_value)
-    return Response(
-        response=task.id,
-        status=requests.codes.ok,
-        mimetype="text/html")

inventory_provider/tasks/ims_worker.py

@@ -47,13 +47,12 @@ class OTRSFiles(IntFlag):
 
 @app.task(base=InventoryTask, bind=True, name='export_data_for_otrs')
 @log_task_entry_and_exit
-def export_data_for_otrs(self, files_to_export=None):
+def export_data_for_otrs(self, files_to_export=None, export_duplicates=False):
     if files_to_export:
         files_to_export = OTRSFiles(files_to_export)
     else:
         files_to_export = set(OTRSFiles)
 
-    logger.debug('>>> export_data_for_otrs_ims')
     ims_config = InventoryTask.config["ims"]
     otrs_config = InventoryTask.config["otrs-export"]
 
@@ -78,7 +77,8 @@ def export_data_for_otrs(self, files_to_export=None):
             cus_usr_path = temp_path.joinpath(f'{prefix}customer_user.csv')
             with open(cus_usr_path, 'w+') as f:
                 writer = csv.writer(f, delimiter='^')
-                writer.writerows(ims_data.otrs_get_customer_users_rows(ds))
+                writer.writerows(ims_data.otrs_get_customer_users_rows(
+                    ds, return_duplicates=export_duplicates))
 
         command = command_template.format(
             user=otrs_config['username'],
@@ -88,5 +88,3 @@ def export_data_for_otrs(self, files_to_export=None):
             destination=otrs_config['destination']
         )
         subprocess.run(command, shell=True, check=True)
-
-    logger.debug('<<< export_data_for_otrs_ims')