diff --git a/inventory_provider/__init__.py b/inventory_provider/__init__.py
index 877944d598d7f3731b2022174a03f7424eadcb1c..d350c9ebbcd98a251485f562b3259f9980ae8721 100644
--- a/inventory_provider/__init__.py
+++ b/inventory_provider/__init__.py
@@ -62,16 +62,6 @@ def create_app():
# end of IMS based routes
- # OTRS routes
-
- from inventory_provider.routes import ims_otrs
- app.register_blueprint(ims_otrs.routes, url_prefix='/otrs')
-
- # from inventory_provider.routes import otrs_jobs
- # app.register_blueprint(otrs_jobs.routes, url_prefix='/otrs')
-
- # end of OTRS routes
-
from inventory_provider.routes import default
app.register_blueprint(default.routes, url_prefix='/')
diff --git a/inventory_provider/config.py b/inventory_provider/config.py
index 733877cc12a078b0579ae783dc9585d490471e63..7bce596156579ca8075a97e4c751669b1eac368a 100644
--- a/inventory_provider/config.py
+++ b/inventory_provider/config.py
@@ -41,22 +41,6 @@ CONFIG_SCHEMA = {
'required': ['api', 'username', 'password'],
'additionalProperties': False
},
- 'otrs-export': {
- 'type': 'object',
- 'properties': {
- 'username': {'type': 'string'},
- 'private-key': {'type': 'string'},
- 'known-hosts': {'type': 'string'},
- 'destination': {'type': 'string'}
- },
- 'required': [
- 'username',
- 'private-key',
- 'known-hosts',
- 'destination'
- ],
- 'additionalProperties': False
- },
'redis-credentials': {
'type': 'object',
'properties': {
@@ -161,7 +145,6 @@ CONFIG_SCHEMA = {
'redis': {'$ref': '#/definitions/redis-credentials'},
'sentinel': {'$ref': '#/definitions/redis-sentinel-config'},
'ims': {'$ref': '#/definitions/ims'},
- 'otrs-export': {'$ref': '#/definitions/otrs-export'},
'redis-databases': {
'type': 'array',
'minItems': 1,
@@ -186,7 +169,6 @@ CONFIG_SCHEMA = {
'ssh',
'redis',
'redis-databases',
- 'otrs-export',
'ims',
'managed-routers',
'gws-direct']
@@ -197,7 +179,6 @@ CONFIG_SCHEMA = {
'ssh',
'sentinel',
'redis-databases',
- 'otrs-export',
'ims',
'managed-routers',
'gws-direct']
diff --git a/inventory_provider/db/ims_data.py b/inventory_provider/db/ims_data.py
index 0283754bf0f3367fdb6a1bf9911dcf3ecb88970d..f0f9ddb516115a1e0435b9b390a933832122c340 100644
--- a/inventory_provider/db/ims_data.py
+++ b/inventory_provider/db/ims_data.py
@@ -1,6 +1,6 @@
import logging
import re
-from collections import OrderedDict, defaultdict
+from collections import defaultdict
from copy import copy
from itertools import chain, groupby
from operator import itemgetter
@@ -420,174 +420,3 @@ def lookup_lg_routers(ds: IMS):
}
}
yield eq
-
-
-def otrs_get_customer_company_rows(ds: IMS):
- yield ['customer_id', 'name', 'street', 'zip', 'city', 'country', 'url',
- 'comments']
- all_cus_comps = set()
- for customer in ds.get_all_entities('Customer'):
- all_cus_comps.add(customer['name'])
- yield [customer['name'].replace(' ', ''), customer['name'],
- '', '', '', '', '', '']
- for vendor in ds.get_all_entities('Vendor'):
- if vendor['name'] not in all_cus_comps:
- all_cus_comps.add(vendor['name'])
- yield [vendor['name'].replace(' ', ''), vendor['name'],
- '', '', '', '', '', '']
-
-
-# TODO - check the rules for validation once model has been confirmed
-def _is_valid_customer(cus):
- if not cus['email']:
- return False
- return True
-
-
-def otrs_get_customer_contacts(ds: IMS):
-
- def _get_customer_id2(t):
- if t['id'] == 3 or t['name'] == 'EU NREN':
- return 'OTRS-GEANT-NREN'
- return ''
-
- for customer in ds.get_all_entities(
- 'Customer',
- [
- ims.CUSTOMER_PROPERTIES['CustomerRelatedContacts'],
- ims.CUSTOMER_PROPERTIES['CustomerType']
- ]):
-
- if customer['customerrelatedcontacts']:
-
- for contact in customer['customerrelatedcontacts']:
- t_customer_user = OrderedDict({
- 'email': contact['contact']['mail'],
- 'username': contact['contact']['mail'], # TODO if tal_id is going to be present use that # noqa
- 'customer_id': customer['name'].replace(' ', ''),
- 'customer_id_2':
- _get_customer_id2(customer['customertype']),
- 'title': '',
- 'firstname': contact['contact']['name'],
- 'lastname':
- ' '.join(filter(None, [
- contact['contact']['infix'],
- contact['contact']['lastname']
- ])),
- 'phone': '',
- 'fax': '',
- 'mobile': '',
- 'street': '',
- 'zip': '',
- 'city': '',
- 'country': '',
- 'comments': '',
- })
- if not _is_valid_customer(t_customer_user):
- continue
-
- yield t_customer_user
-
-
-def otrs_get_vendor_contacts(ds: IMS):
-
- for vrc in ds.get_all_entities(
- 'VendorRelatedContact',
- [
- ims.VENDOR_RELATED_CONTACT_PROPERTIES['Vendor'],
- ims.VENDOR_RELATED_CONTACT_PROPERTIES['Contact']
- ]):
- t_customer_user = OrderedDict({
- 'email': vrc['contact']['mail'],
- 'username': vrc['contact']['mail'], # TODO if tal_id is going to be present use that # noqa
- 'customer_id': vrc['vendor']['name'].replace(' ', ''),
- 'customer_id_2': '',
- 'title': '',
- 'firstname': vrc['contact']['name'],
- 'lastname':
- ' '.join(filter(None, [
- vrc['contact']['infix'],
- vrc['contact']['lastname']
- ])),
- 'phone': '',
- 'fax': '',
- 'mobile': '',
- 'street': '',
- 'zip': '',
- 'city': '',
- 'country': '',
- 'comments': '',
- })
- if not _is_valid_customer(t_customer_user):
- continue
-
- yield t_customer_user
-
-
-def otrs_get_customer_users_rows(ds: IMS, return_duplicates: bool = False):
- yield ['email', 'username', 'customer_id', 'customer_id_2', 'title',
- 'firstname', 'lastname', 'phone', 'fax', 'mobile', 'street', 'zip',
- 'city', 'country', 'comments']
-
- def get_all_cus_user_rows():
- yielded_customer_emails = set()
- for c in otrs_get_customer_contacts(ds):
- yielded_customer_emails.add(c['email'])
- yield c
- for c in otrs_get_vendor_contacts(ds):
- if c['email'] not in yielded_customer_emails:
- yield c
-
- def weed_duplicates(duplicates):
- # this is here to allow for additional rules
- id_rank = {
- 'geant': 1
- }
- top_rank = -1
- top_ranked = None
- for d in duplicates:
- rank = id_rank.get(d['customer_id'].lower(), 0)
- if rank > top_rank:
- top_rank = rank
- top_ranked = []
- if rank == top_rank:
- top_ranked.append(d)
- return top_ranked
-
- cus_by_email = {}
- duplicate_emails = set()
- for cu in get_all_cus_user_rows():
- email = cu['email']
- cus_for_email = cus_by_email.get(email, [])
- if cus_for_email:
- duplicate_emails.add(email)
- cus_for_email.append(cu)
- cus_by_email[email] = cus_for_email
-
- remaining_duplicates = []
- if duplicate_emails:
- logger.info('Duplicate emails found in OTRS customer-user export: '
- f'{duplicate_emails} - attempting to weed')
- for email in duplicate_emails:
- weeded = weed_duplicates(cus_by_email.pop(email))
- if len(weeded) == 1:
- cus_by_email[email] = weeded
- else:
- remaining_duplicates.extend(
- sorted(
- [list(w.values()) for w in weeded], key=lambda x: x[2])
- )
-
- if remaining_duplicates:
- logger.error('Duplicate emails remain after weeding, '
- f'{"including" if return_duplicates else "excluding"}'
- ' duplicates in returned data: ')
-
- for rd in remaining_duplicates:
- logger.debug(rd)
-
- for email, details in sorted(cus_by_email.items()):
- yield list(details[0].values())
-
- if return_duplicates:
- yield from remaining_duplicates
diff --git a/inventory_provider/routes/ims_otrs.py b/inventory_provider/routes/ims_otrs.py
deleted file mode 100644
index b5143b657327bcb63100205fb4a8a7f3dc96dd02..0000000000000000000000000000000000000000
--- a/inventory_provider/routes/ims_otrs.py
+++ /dev/null
@@ -1,71 +0,0 @@
-import csv
-import html
-from io import StringIO
-
-import requests
-from flask import Blueprint, request, Response, current_app
-
-from inventory_provider.db import ims_data
-from inventory_provider.db.ims import IMS
-from inventory_provider.routes import common
-from inventory_provider.tasks.worker import OTRSFiles, export_data_for_otrs
-
-routes = Blueprint("otrs", __name__)
-
-
-@routes.after_request
-def after_request(resp):
- return common.after_request(resp)
-
-
-def get_otrs_output(result):
-
- with StringIO() as sio:
- writer = csv.writer(sio, delimiter='^')
- writer.writerows(result)
- data = sio.getvalue()
- return Response(
- response=data,
- status=requests.codes.ok,
- mimetype="text/html")
-
-
-@routes.route('customer-companies')
-def get_customer_companies_data():
- ims_config = current_app.config['INVENTORY_PROVIDER_CONFIG']["ims"]
- ds = IMS(ims_config['api'], ims_config['username'], ims_config['password'])
- return get_otrs_output(ims_data.otrs_get_customer_company_rows(ds))
-
-
-@routes.route('customer-users')
-def get_customer_users_data():
- ims_config = current_app.config['INVENTORY_PROVIDER_CONFIG']["ims"]
- ds = IMS(ims_config['api'], ims_config['username'], ims_config['password'])
- return_duplicates = request.args.get('duplicates', 'f').lower() == 'true'
- return get_otrs_output(ims_data.otrs_get_customer_users_rows(
- ds, return_duplicates=return_duplicates))
-
-
-@routes.route('export')
-def send_exports():
- files_value = request.args.get('files', None)
- duplicates = request.args.get('duplicates', 'f').lower() == 'true'
- if files_value:
- try:
- files_value = int(files_value)
- except ValueError:
- return Response(
- response=html.escape('<files> should be an Integer'),
- status=requests.codes.bad_request,
- mimetype="text/html")
- if files_value < 0 or files_value > sum(OTRSFiles):
- return Response(
- response=html.escape(f'Bad value for <files> {files_value}'),
- status=requests.codes.bad_request,
- mimetype="text/html")
- debug_uuid = export_data_for_otrs(
- files_to_export=files_value, export_duplicates=duplicates)
- return Response(
- response=f'Exports sent, search logs for {debug_uuid} for details',
- status=requests.codes.ok,
- mimetype="text/html")
diff --git a/inventory_provider/tasks/worker.py b/inventory_provider/tasks/worker.py
index 544e10c098db1343ae249718c6011d50102b629b..f496e544957e593a385be2c3d679d2663413ae4c 100644
--- a/inventory_provider/tasks/worker.py
+++ b/inventory_provider/tasks/worker.py
@@ -1,18 +1,11 @@
-import csv
import functools
import json
import logging
import os
import re
-import subprocess
-import tempfile
import threading
import time
-from datetime import datetime
-from enum import IntFlag
-from pathlib import Path
from typing import List
-from uuid import uuid4
from redis.exceptions import RedisError
from kombu.exceptions import KombuError
@@ -881,59 +874,6 @@ def update_lg_routers(self, use_current=False):
r.set(f'ims:lg:{router["equipment name"]}', json.dumps(router))
-class OTRSFiles(IntFlag):
- CUSTOMER_COMPANIES = 1
- CUSTOMER_USERS = 2
-
-
-@app.task(base=InventoryTask, bind=True, name='export_data_for_otrs')
-@log_task_entry_and_exit
-def export_data_for_otrs(self, files_to_export=None, export_duplicates=False):
- debug_uuid = uuid4()
- logger.debug(f'debug uuid: {debug_uuid}')
- if files_to_export:
- files_to_export = OTRSFiles(files_to_export)
- else:
- files_to_export = set(OTRSFiles)
-
- ims_config = InventoryTask.config["ims"]
- otrs_config = InventoryTask.config["otrs-export"]
-
- command_template = 'rsync -aPq --no-perms --rsh="ssh -l {user} -p 22 -i {key_file} -o \'UserKnownHostsFile {known_hosts}\'" {source_dir}/* {destination}' # noqa
-
- with tempfile.TemporaryDirectory() as temp_dir:
- temp_path = Path(temp_dir)
- ds = IMS(
- ims_config['api'],
- ims_config['username'],
- ims_config['password'])
-
- prefix = datetime.now().strftime('%Y%m%d') + '_'
-
- if OTRSFiles.CUSTOMER_COMPANIES in files_to_export:
- cus_co_path = temp_path.joinpath(f'{prefix}customer_company.csv')
- with open(cus_co_path, 'w+') as f:
- writer = csv.writer(f, delimiter='^')
- writer.writerows(ims_data.otrs_get_customer_company_rows(ds))
-
- if OTRSFiles.CUSTOMER_USERS in files_to_export:
- cus_usr_path = temp_path.joinpath(f'{prefix}customer_user.csv')
- with open(cus_usr_path, 'w+') as f:
- writer = csv.writer(f, delimiter='^')
- writer.writerows(ims_data.otrs_get_customer_users_rows(
- ds, return_duplicates=export_duplicates))
-
- command = command_template.format(
- user=otrs_config['username'],
- key_file=otrs_config['private-key'],
- known_hosts=otrs_config['known-hosts'],
- source_dir=temp_dir,
- destination=otrs_config['destination']
- )
- subprocess.run(command, shell=True, check=True)
- return debug_uuid
-
-
def _wait_for_tasks(task_ids, update_callback=lambda s: None):
all_successful = True
diff --git a/test/conftest.py b/test/conftest.py
index bdfe95676907fa170cde88260eadc5385ffdb9a4..c71ce4daff1c3b6c1fd569576e0b9de89b4bd628 100644
--- a/test/conftest.py
+++ b/test/conftest.py
@@ -45,12 +45,6 @@ def data_config_filename():
"socket_timeout": 2.8
},
"redis-databases": [0, 7],
- "otrs-export": {
- "username": "otrs_username",
- "private-key": "otrs_ky_loc",
- "destination": "otrs_dest",
- "known-hosts": "otrs_known_hosts"
- },
"ims": {
"api": "ims_api",
"username": "ims_username",
diff --git a/test/test_ims_data.py b/test/test_ims_data.py
index 74fa71c3df0ec3d797511b72ac077048d1142f49..cd8d7e421eb8750f077bf74a22c8f6b3ee3d019c 100644
--- a/test/test_ims_data.py
+++ b/test/test_ims_data.py
@@ -3,8 +3,7 @@ import json
import inventory_provider
from inventory_provider.db.ims import InventoryStatus
from inventory_provider.db.ims_data import lookup_lg_routers, \
- otrs_get_customer_company_rows, \
- otrs_get_customer_users_rows, get_node_locations, IMS_OPSDB_STATUS_MAP, \
+ get_node_locations, IMS_OPSDB_STATUS_MAP, \
get_port_id_services, get_port_details, \
get_circuit_hierarchy
@@ -287,89 +286,3 @@ def test_get_node_location(mocker):
'latitude': 51.5308142,
}
})
-
-
-def test_otrs_get_customer_company_rows(mocker):
- ims = mocker.patch('inventory_provider.db.ims.IMS')
- with open('test/data/ims_otrs_customers.json') as data:
- resp_data = json.load(data)
-
- def se(*args, **kargs):
- return resp_data[args[0]]
- mocked_get_all_entities = ims.return_value.get_all_entities
- mocked_get_all_entities.side_effect = se
- ds = inventory_provider.db.ims.IMS(
- 'dummy_base', 'dummy_username', 'dummy_password')
- cus_comp_rows = list(otrs_get_customer_company_rows(ds))
- assert len(cus_comp_rows) == 6
- mocked_get_all_entities.assert_any_call('Customer')
- mocked_get_all_entities.assert_any_call('Vendor')
- assert cus_comp_rows[0] == ['customer_id', 'name', 'street', 'zip',
- 'city', 'country', 'url', 'comments']
- ids = []
- names = []
- for row in cus_comp_rows[1:]:
- assert len(row) == 8
- ids.append(row[0])
- names.append(row[1])
- assert ids == ['DUMMY1', 'DUMMY2', 'DUMMY3', 'DUMMY4', 'DUMMY5']
- assert names == ['DUMMY 1', 'DUMMY 2', 'DUMMY 3', 'DUMMY 4', 'DUMMY 5']
-
-
-def test_otrs_get_customer_users(mocker):
- ims = mocker.patch('inventory_provider.db.ims.IMS')
- resp_data = {}
- with open('test/data/ims_otrs_customers_extra.json') as data:
- resp_data['Customer'] = json.load(data)
- with open('test/data/ims_otrs_vendor_contact_extra.json') as data:
- resp_data['VendorRelatedContact'] = json.load(data)
-
- def se(*args, **kargs):
- return resp_data.get(args[0], [])
-
- mocked_get_all_entities = ims.return_value.get_all_entities
- mocked_get_all_entities.side_effect = se
-
- mocked_get_all_entities = ims.return_value.get_all_entities
- ds = inventory_provider.db.ims.IMS(
- 'dummy_base', 'dummy_username', 'dummy_password')
-
- customer_users = list(otrs_get_customer_users_rows(ds))
- mocked_get_all_entities.assert_any_call('Customer', [32768, 262144])
- mocked_get_all_entities.assert_any_call('VendorRelatedContact', [8, 16])
-
- assert customer_users[0] == [
- 'email',
- 'username',
- 'customer_id',
- 'customer_id_2',
- 'title',
- 'firstname',
- 'lastname',
- 'phone',
- 'fax',
- 'mobile',
- 'street',
- 'zip',
- 'city',
- 'country',
- 'comments'
- ]
-
- assert len(customer_users) == 13
- assert customer_users[1] == [
- 'BANOC@DUMMY2.COM', 'BANOC@DUMMY2.COM', 'DUMMY2', '', '',
- 'DUMMY 2 NOC', '-', '', '', '',
- '', '', '', '', ''
- ]
- assert customer_users[6] == [
- 'HNOC@DUMMY8.COM', 'HNOC@DUMMY8.COM', 'DUMMY8', 'OTRS-GEANT-NREN', '',
- 'H D_FIRST', 'H D_INFIX H D_LAST', '', '', '',
- '', '', '', '', ''
- ]
- assert customer_users[10] == [
- 'K@DUMMY10.FR', 'K@DUMMY10.FR', 'DUMMY10', '', '',
- 'K D_FIRST', 'K D_INFIX K D_LAST', '', '', '',
- '', '', '', '', ''
- ]
- assert customer_users[12][6] == 'M LAST'
diff --git a/test/test_worker.py b/test/test_worker.py
index 4a2a53e1b466a8f7f4e7338567996546d2d94ba0..4ee32c2d74be5ddd274991c9a78fcbd6a5e68dd0 100644
--- a/test/test_worker.py
+++ b/test/test_worker.py
@@ -1,44 +1,4 @@
-import os
-import re
-from inventory_provider.tasks.worker import export_data_for_otrs, OTRSFiles
-
-def test_otrs_exports(data_config_filename, data_config, mocker):
- os.environ['INVENTORY_PROVIDER_CONFIG_FILENAME'] = data_config_filename
- otrs_config = data_config["otrs-export"]
- mocked_writer = \
- mocker.patch('inventory_provider.tasks.worker.csv.writer')
- mocked_row_writer = mocked_writer.return_value.writerows
- mocked_run = mocker.patch(
- 'inventory_provider.tasks.worker.subprocess.run')
- mocker.patch(
- 'inventory_provider.tasks.worker.ims_data.otrs_get_customer_company_rows', # noqa
- return_value=[[1, 2, 3, 4], [5, 6, 7, 8]]
- )
- mocker.patch(
- 'inventory_provider.tasks.worker.ims_data.otrs_get_customer_users_rows', # noqa
- return_value=[[9, 10, 11, 12], [13, 14, 15, 16]]
- )
-
- export_data_for_otrs(OTRSFiles.CUSTOMER_COMPANIES)
- mocked_row_writer.assert_called_with([[1, 2, 3, 4], [5, 6, 7, 8]])
-
- export_data_for_otrs(OTRSFiles.CUSTOMER_USERS)
- mocked_row_writer.assert_called_with([[9, 10, 11, 12], [13, 14, 15, 16]])
-
- export_data_for_otrs()
- assert mocked_row_writer.call_count == 4
-
- args, kwargs = mocked_run.call_args
- called_with = args[0]
-
- t = r'^rsync -aPq --no-perms --rsh="ssh -l {user} -p 22 -i {key_file} -o \'UserKnownHostsFile {known_hosts}\'" /\S+/\* {destination}$' # noqa
-
- p = t.format(
- user=otrs_config['username'],
- key_file=otrs_config['private-key'],
- destination=otrs_config['destination'],
- known_hosts=otrs_config['known-hosts']
- )
- assert bool(re.match(p, called_with))
+def test_place_holder():
+ assert True