diff --git a/inventory_provider/db/ims_data.py b/inventory_provider/db/ims_data.py
index 11dc04dfe2199df5201052fb82c30b40d6e4b3d5..48ab115c15711abc3b7a14659a80e3139748250f 100644
--- a/inventory_provider/db/ims_data.py
+++ b/inventory_provider/db/ims_data.py
@@ -5,10 +5,46 @@ from collections import OrderedDict
from inventory_provider import environment
from inventory_provider.db import ims
from inventory_provider.db.ims import InventoryStatus
-
environment.setup_logging()
logger = logging.getLogger(__name__)
+# Dashboard V3
+
+
+def lookup_pop_info(ds, hostname):
+ site_nav_props = [
+ ims.SITE_PROPERTIES['City'],
+ ims.SITE_PROPERTIES['SiteAliases'],
+ ims.SITE_PROPERTIES['Country']
+ ]
+
+ node = ds.get_entity_by_name('Node', hostname)
+ if not node:
+ return None
+ site = ds.get_entity_by_id('Site', node['SiteId'], site_nav_props, True)
+ city = site['City']
+ abbreviation = ''
+ try:
+ abbreviation = site['SiteAliases'][0]['AliasName']
+ except IndexError:
+ pass # no alias - ignore silently
+ eq = {
+ 'equipment-name': node['Name'],
+ 'status': InventoryStatus(node['InventoryStatusId']).name,
+ 'pop': {
+ 'name': site['Name'],
+ 'city': city['Name'],
+ 'country': city['Country']['Name'],
+ 'abbreviation': abbreviation,
+ 'longitude': site['Longitude'],
+ 'latitude': site['Latitude'],
+ }
+ }
+ return eq
+
+
+# End of Dashboard V3 stuff
+
INTERNAL_POP_NAMES = {
'Cambridge OC',
'DANTE Lab',
@@ -81,38 +117,6 @@ def lookup_lg_routers(ds):
yield(eq)
-def lookup_pop_info(ds, hostname):
- site_nav_props = [
- ims.SITE_PROPERTIES['City'],
- ims.SITE_PROPERTIES['SiteAliases'],
- ims.SITE_PROPERTIES['Country']
- ]
-
- node = ds.get_entity_by_name('Node', hostname)
- if not node:
- return None
- site = ds.get_entity_by_id('Site', node['SiteId'], site_nav_props, True)
- city = site['City']
- abbreviation = ''
- try:
- abbreviation = site['SiteAliases'][0]['AliasName']
- except IndexError:
- pass # no alias - ignore silently
- eq = {
- 'equipment-name': node['Name'],
- 'status': InventoryStatus(node['InventoryStatusId']).name,
- 'pop': {
- 'name': site['Name'],
- 'city': city['Name'],
- 'country': city['Country']['Name'],
- 'abbreviation': abbreviation,
- 'longitude': site['Longitude'],
- 'latitude': site['Latitude'],
- }
- }
- return eq
-
-
def otrs_get_customer_company_rows(ds):
yield ['customer_id', 'name', 'street', 'zip', 'city', 'country', 'url',
'comments']
@@ -215,7 +219,7 @@ def otrs_get_vendor_contacts(ds):
yield t_customer_user
-def otrs_get_customer_users_rows(ds):
+def otrs_get_customer_users_rows(ds, return_duplicates=False):
yield ['email', 'username', 'customer_id', 'customer_id_2', 'title',
'firstname', 'lastname', 'phone', 'fax', 'mobile', 'street', 'zip',
'city', 'country', 'comments']
@@ -229,18 +233,59 @@ def otrs_get_customer_users_rows(ds):
if c['email'] not in yielded_customer_emails:
yield c
- sorted_cus = sorted(get_all_cus_user_rows(), key=lambda x: x['email'])
+ def weed_duplicates(duplicates):
+ # this is here to allow for additional rules
+ id_rank = {
+ 'geant': 1
+ }
+ top_rank = -1
+ top_ranked = None
+ for d in duplicates:
+ rank = id_rank.get(d['customer_id'].lower(), 0)
+ if rank > top_rank:
+ top_rank = rank
+ top_ranked = []
+ if rank == top_rank:
+ top_ranked.append(d)
+ return top_ranked
+
+ cus_by_email = {}
duplicate_emails = set()
- previous_email = None
- for cu in sorted_cus:
- if cu['email'] == previous_email:
- duplicate_emails.add(previous_email)
- previous_email = cu['email']
+ for cu in get_all_cus_user_rows():
+ email = cu['email']
+ cus_for_email = cus_by_email.get(email, [])
+ if cus_for_email:
+ duplicate_emails.add(email)
+ cus_for_email.append(cu)
+ cus_by_email[email] = cus_for_email
+
+ remaining_duplicates = []
if duplicate_emails:
- logger.error('Duplicate emails found in OTRS customer-user export: '
- f'{duplicate_emails}')
- # raise KeyError('Duplicate emails found in OTRS customer-user export: ' # noqa
- # f'{duplicate_emails}')
-
- for cu in sorted_cus:
- yield list(cu.values())
+ logger.info('Duplicate emails found in OTRS customer-user export: '
+ f'{duplicate_emails} - attempting to weed')
+ # weeded = []
+ for email in duplicate_emails:
+ weeded = weed_duplicates(cus_by_email.pop(email))
+ if len(weeded) == 1:
+ cus_by_email[email] = weeded
+ else:
+ remaining_duplicates.extend(
+ sorted(
+ [list(w.values()) for w in weeded], key=lambda x: x[2])
+ )
+
+ if remaining_duplicates:
+ # need guidance what to do if this occurs, should we pick the first
+ # one for each, or ignore them completely? Ignoring them for now
+ logger.error('Duplicate emails remain after weeding, '
+ f'{"including" if return_duplicates else "excluding"}'
+ ' duplicates in returned data: ')
+
+ for rd in remaining_duplicates:
+ logger.debug(rd)
+
+ for email, details in sorted(cus_by_email.items()):
+ yield list(details[0].values())
+
+ if return_duplicates:
+ yield from remaining_duplicates
diff --git a/inventory_provider/routes/ims_otrs.py b/inventory_provider/routes/ims_otrs.py
index cbae9e9e740f7de090b20b542e9f4439cc89af88..b03fec117fd2e6e70408e923d5a855f5d35e8cc8 100644
--- a/inventory_provider/routes/ims_otrs.py
+++ b/inventory_provider/routes/ims_otrs.py
@@ -18,16 +18,11 @@ def after_request(resp):
return common.after_request(resp)
-def get_otrs_data(fn):
- ims_config = current_app.config['INVENTORY_PROVIDER_CONFIG']["ims"]
- ds = IMS(
- ims_config['api'],
- ims_config['username'],
- ims_config['password'])
+def get_otrs_output(result):
with StringIO() as sio:
writer = csv.writer(sio, delimiter='^')
- writer.writerows(fn(ds))
+ writer.writerows(result)
data = sio.getvalue()
return Response(
response=data,
@@ -37,17 +32,24 @@ def get_otrs_data(fn):
@routes.route('customer-companies')
def get_customer_companies_data():
- return get_otrs_data(ims_data.otrs_get_customer_company_rows)
+ ims_config = current_app.config['INVENTORY_PROVIDER_CONFIG']["ims"]
+ ds = IMS(ims_config['api'], ims_config['username'], ims_config['password'])
+ return get_otrs_output(ims_data.otrs_get_customer_company_rows(ds))
@routes.route('customer-users')
def get_customer_users_data():
- return get_otrs_data(ims_data.otrs_get_customer_users_rows)
+ ims_config = current_app.config['INVENTORY_PROVIDER_CONFIG']["ims"]
+ ds = IMS(ims_config['api'], ims_config['username'], ims_config['password'])
+ return_duplicates = request.args.get('duplicates', 'f').lower() == 'true'
+ return get_otrs_output(ims_data.otrs_get_customer_users_rows(
+ ds, return_duplicates=return_duplicates))
@routes.route('export')
def send_exports():
files_value = request.args.get('files', None)
+ duplicates = request.args.get('duplicates', 'f').lower() == 'true'
if files_value:
try:
files_value = int(files_value)
@@ -61,7 +63,8 @@ def send_exports():
response=html.escape(f'Bad value for <files> {files_value}'),
status=requests.codes.bad_request,
mimetype="text/html")
- task = export_data_for_otrs.delay(files_value)
+ task = export_data_for_otrs.delay(
+ files_to_export=files_value, export_duplicates=duplicates)
return Response(
response=task.id,
status=requests.codes.ok,
diff --git a/inventory_provider/routes/otrs_jobs.py b/inventory_provider/routes/otrs_jobs.py
deleted file mode 100644
index 6de4678de1b8d27c5cb373d9f5208bf3ea6a5e0b..0000000000000000000000000000000000000000
--- a/inventory_provider/routes/otrs_jobs.py
+++ /dev/null
@@ -1,37 +0,0 @@
-import html
-
-import requests
-from flask import Blueprint, request, Response
-
-from inventory_provider.routes import common
-from inventory_provider.tasks.ims_worker import OTRSFiles, export_data_for_otrs
-
-routes = Blueprint("otrs", __name__)
-
-
-@routes.after_request
-def after_request(resp):
- return common.after_request(resp)
-
-
-@routes.route('export')
-def send_exports():
- files_value = request.args.get('files', None)
- if files_value:
- try:
- files_value = int(files_value)
- except ValueError:
- return Response(
- response=html.escape('<files> should be an Integer'),
- status=requests.codes.bad_request,
- mimetype="text/html")
- if files_value < 0 or files_value > sum(OTRSFiles):
- return Response(
- response=html.escape(f'Bad value for <files> {files_value}'),
- status=requests.codes.bad_request,
- mimetype="text/html")
- task = export_data_for_otrs.delay(files_value)
- return Response(
- response=task.id,
- status=requests.codes.ok,
- mimetype="text/html")
diff --git a/inventory_provider/tasks/ims_worker.py b/inventory_provider/tasks/ims_worker.py
index fdab630d459dbe9b7b082e3d920eae58996562ce..26f7f09dccb3a2be7544604d73116d0228a7eaf7 100644
--- a/inventory_provider/tasks/ims_worker.py
+++ b/inventory_provider/tasks/ims_worker.py
@@ -47,13 +47,12 @@ class OTRSFiles(IntFlag):
@app.task(base=InventoryTask, bind=True, name='export_data_for_otrs')
@log_task_entry_and_exit
-def export_data_for_otrs(self, files_to_export=None):
+def export_data_for_otrs(self, files_to_export=None, export_duplicates=False):
if files_to_export:
files_to_export = OTRSFiles(files_to_export)
else:
files_to_export = set(OTRSFiles)
- logger.debug('>>> export_data_for_otrs_ims')
ims_config = InventoryTask.config["ims"]
otrs_config = InventoryTask.config["otrs-export"]
@@ -78,7 +77,8 @@ def export_data_for_otrs(self, files_to_export=None):
cus_usr_path = temp_path.joinpath(f'{prefix}customer_user.csv')
with open(cus_usr_path, 'w+') as f:
writer = csv.writer(f, delimiter='^')
- writer.writerows(ims_data.otrs_get_customer_users_rows(ds))
+ writer.writerows(ims_data.otrs_get_customer_users_rows(
+ ds, return_duplicates=export_duplicates))
command = command_template.format(
user=otrs_config['username'],
@@ -88,5 +88,3 @@ def export_data_for_otrs(self, files_to_export=None):
destination=otrs_config['destination']
)
subprocess.run(command, shell=True, check=True)
-
- logger.debug('<<< export_data_for_otrs_ims')