diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000000000000000000000000000000000000..8d6fe3ccb931c4867538b49a6bea8be51087f6c6
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,3 @@
+*.log
+*config.sample.json
+__pycache__
diff --git a/ims-data-example.py b/ims-data-example.py
new file mode 100644
index 0000000000000000000000000000000000000000..445cb6414d5e47cbaf5ba505eb6a72d359b5987c
--- /dev/null
+++ b/ims-data-example.py
@@ -0,0 +1,20 @@
+import json
+import os
+
+from inventory_provider.ims import IMS
+from inventory_provider import config
+from inventory_provider import ims_data
+
+CONFIG_FILENAME = os.path.abspath(os.path.join(
+ os.path.dirname(__file__), 'config.sample.json'))
+
+with open(CONFIG_FILENAME) as f:
+ params = config.load(f)
+ ims_params = params['ims']
+
+ds = IMS(ims_params['api'], ims_params['username'], ims_params['password'])
+
+services = ims_data.lookup_lg_routers(ds=ds)
+services = ims_data.get_port_id_services(ds=ds)
+services = list(services)
+print(json.dumps(services, indent=2))
diff --git a/inventory_provider/config.py b/inventory_provider/config.py
new file mode 100644
index 0000000000000000000000000000000000000000..f332f5bb2bdf60b66c99c231addf5f0c23fcc94e
--- /dev/null
+++ b/inventory_provider/config.py
@@ -0,0 +1,265 @@
+import json
+import jsonschema
+
+CONFIG_SCHEMA = {
+ '$schema': 'http://json-schema.org/draft-07/schema#',
+
+ 'definitions': {
+ 'timeout': {
+ 'type': 'number',
+ 'maximum': 60, # sanity
+ 'exclusiveMinimum': 0
+ },
+ 'database-credentials': {
+ 'type': 'object',
+ 'properties': {
+ 'hostname': {'type': 'string'},
+ 'dbname': {'type': 'string'},
+ 'username': {'type': 'string'},
+ 'password': {'type': 'string'}
+ },
+ 'required': ['hostname', 'dbname', 'username', 'password'],
+ 'additionalProperties': False
+ },
+ 'ssh-credentials': {
+ 'type': 'object',
+ 'properties': {
+ 'username': {'type': 'string'},
+ 'private-key': {'type': 'string'},
+ 'known-hosts': {'type': 'string'}
+ },
+ 'required': ['private-key', 'known-hosts'],
+ 'additionalProperties': False
+ },
+ 'ims': {
+ 'type': 'object',
+ 'properties': {
+ 'api': {'type': 'string'},
+ 'username': {'type': 'string'},
+ 'password': {'type': 'string'}
+ },
+ 'required': ['api', 'username', 'password'],
+ 'additionalProperties': False
+ },
+ 'otrs-export': {
+ 'type': 'object',
+ 'properties': {
+ 'username': {'type': 'string'},
+ 'private-key': {'type': 'string'},
+ 'known-hosts': {'type': 'string'},
+ 'destination': {'type': 'string'}
+ },
+ 'required': [
+ 'username',
+ 'private-key',
+ 'known-hosts',
+ 'destination'
+ ],
+ 'additionalProperties': False
+ },
+ 'redis-credentials': {
+ 'type': 'object',
+ 'properties': {
+ 'hostname': {'type': 'string'},
+ 'port': {'type': 'integer'},
+ 'celery-db-index': {'type': 'integer'},
+ 'socket_timeout': {'$ref': '#/definitions/timeout'}
+ },
+ 'required': ['hostname', 'port'],
+ 'additionalProperties': False
+ },
+ 'redis-sentinel-config': {
+ 'type': 'object',
+ 'properties': {
+ 'hostname': {'type': 'string'},
+ 'port': {'type': 'integer'},
+ 'celery-db-index': {'type': 'integer'},
+ 'name': {'type': 'string'},
+ 'redis_socket_timeout': {'$ref': '#/definitions/timeout'},
+ 'sentinel_socket_timeout': {'$ref': '#/definitions/timeout'}
+ },
+ 'required': ['hostname', 'port', 'name'],
+ 'additionalProperties': False
+ },
+ 'interface-address': {
+ 'type': 'object',
+ 'properties': {
+ 'address': {'type': 'string'},
+ 'network': {'type': 'string'},
+ 'interface': {'type': 'string'},
+ 'router': {'type': 'string'}
+ },
+ 'required': ['address', 'network', 'interface', 'router'],
+ 'additionalProperties': False
+ },
+ 'oid': {
+ 'type': 'string',
+ 'pattern': r'^(\d+\.)*\d+$'
+ },
+ 'gws-direct-counters': {
+ 'type': 'object',
+ 'properties': {
+ 'discards_in': {'$ref': '#/definitions/oid'},
+ 'discards_out': {'$ref': '#/definitions/oid'},
+ 'errors_in': {'$ref': '#/definitions/oid'},
+ 'errors_out': {'$ref': '#/definitions/oid'},
+ 'traffic_in': {'$ref': '#/definitions/oid'},
+ 'traffic_out': {'$ref': '#/definitions/oid'},
+ },
+ 'additionalProperties': False
+ },
+ 'gws-direct-interface': {
+ 'type': 'object',
+ 'properties': {
+ 'info': {'type': 'string'},
+ 'tag': {'type': 'string'},
+ 'counters': {'$ref': '#/definitions/gws-direct-counters'}
+ },
+ 'required': ['tag', 'counters'],
+ 'additionalProperties': False
+ },
+ 'gws-direct-host-v2': {
+ 'type': 'object',
+ 'properties': {
+ 'hostname': {'type': 'string'},
+ 'community': {'type': 'string'},
+ 'interfaces': {
+ 'type': 'array',
+ 'items': {'$ref': '#/definitions/gws-direct-interface'},
+ 'minItems': 1
+ }
+ },
+ 'required': ['hostname', 'community', 'interfaces'],
+ 'additionalProperties': False
+ },
+ 'snmp-v3-cred': {
+ 'type': 'object',
+ 'properties': {
+ 'protocol': {'enum': ['MD5', 'DES']},
+ 'password': {'type': 'string'}
+ },
+ 'required': ['protocol', 'password'],
+ 'additionalProperties': False
+ },
+ 'gws-direct-host-v3': {
+ 'type': 'object',
+ 'properties': {
+ 'hostname': {'type': 'string'},
+ 'sec-name': {'type': 'string'},
+ 'auth': {'$ref': '#/definitions/snmp-v3-cred'},
+ 'priv': {'$ref': '#/definitions/snmp-v3-cred'},
+ 'interfaces': {
+ 'type': 'array',
+ 'items': {'$ref': '#/definitions/gws-direct-interface'},
+ 'minItems': 1
+ }
+ },
+ 'required': ['hostname', 'sec-name', 'interfaces'],
+ 'additionalProperties': False
+ },
+ 'gws-direct-nren-isp': {
+ 'type': 'object',
+ 'properties': {
+ 'nren': {'type': 'string'},
+ 'isp': {
+ 'type': 'string',
+ 'enum': ['Cogent', 'Telia', 'CenturyLink']
+ },
+ 'hosts': {
+ 'type': 'array',
+ 'items': {
+ 'oneOf': [
+ {'$ref': '#/definitions/gws-direct-host-v2'},
+ {'$ref': '#/definitions/gws-direct-host-v3'},
+ ]
+ },
+ 'minItems': 1
+ }
+ },
+ 'required': ['nren', 'isp', 'hosts'],
+ 'additionalProperties': False
+ },
+ 'gws-direct': {
+ 'type': 'array',
+ 'items': {'$ref': '#/definitions/gws-direct-nren-isp'}
+ },
+ 'nren-asn-map': {
+ 'type': 'array',
+ 'items': {
+ 'type': 'object',
+ 'properties': {
+ 'nren': {'type': 'string'},
+ 'asn': {'type': 'integer'}
+ },
+ 'required': ['nren', 'asn'],
+ 'additionalProperties': False
+ },
+ }
+ },
+
+ 'type': 'object',
+ 'properties': {
+ 'ops-db': {'$ref': '#/definitions/database-credentials'},
+ 'ssh': {'$ref': '#/definitions/ssh-credentials'},
+ 'redis': {'$ref': '#/definitions/redis-credentials'},
+ 'sentinel': {'$ref': '#/definitions/redis-sentinel-config'},
+ 'ims': {'$ref': '#/definitions/ims'},
+ 'otrs-export': {'$ref': '#/definitions/otrs-export'},
+ 'redis-databases': {
+ 'type': 'array',
+ 'minItems': 1,
+ 'items': {'type': 'integer'}
+ },
+ 'managed-routers': {'type': 'string'},
+ 'lab-routers': {
+ 'type': 'array',
+ 'items': {'type': 'string'}
+ },
+ 'unmanaged-interfaces': {
+ 'type': 'array',
+ 'items': {'$ref': '#/definitions/interface-address'}
+ },
+ 'gws-direct': {'$ref': '#/definitions/gws-direct'},
+ 'nren-asn-map': {'$ref': '#/definitions/nren-asn-map'}
+
+ },
+ 'oneOf': [
+ {
+ 'required': [
+ 'ssh',
+ 'redis',
+ 'redis-databases',
+ 'ims',
+ 'managed-routers',
+ 'gws-direct',
+ 'nren-asn-map']
+ },
+ {
+ 'required': [
+ 'ssh',
+ 'sentinel',
+ 'redis-databases',
+ 'ims',
+ 'managed-routers',
+ 'gws-direct',
+ 'nren-asn-map']
+ }
+ ],
+ 'additionalProperties': False
+}
+
+
+def load(f):
+ """
+ Loads, validates and returns configuration parameters.
+
+ Input is validated against this jsonschema:
+
+ .. asjson:: inventory_provider.config.CONFIG_SCHEMA
+
+ :param f: file-like object that produces the config file
+ :return: a dict containing the parsed configuration parameters
+ """
+ config = json.loads(f.read())
+ jsonschema.validate(config, CONFIG_SCHEMA)
+ return config
diff --git a/inventory_provider/environment.py b/inventory_provider/environment.py
new file mode 100644
index 0000000000000000000000000000000000000000..989c0a1355ebb7b0e44f110842b25f045868a47d
--- /dev/null
+++ b/inventory_provider/environment.py
@@ -0,0 +1,24 @@
+import json
+import logging.config
+import os
+
+
+def setup_logging():
+ """
+ set up logging using the configured filename
+
+ if LOGGING_CONFIG is defined in the environment, use this for
+ the filename, otherwise use logging_default_config.json
+ """
+ default_filename = os.path.join(
+ os.path.dirname(__file__),
+ 'logging_default_config.json')
+ filename = os.getenv('LOGGING_CONFIG', default_filename)
+ with open(filename) as f:
+ # TODO: this mac workaround should be removed ...
+ d = json.loads(f.read())
+ import platform
+ if platform.system() == 'Darwin':
+ d['handlers']['syslog_handler']['address'] = '/var/run/syslog'
+ logging.config.dictConfig(d)
+ # logging.config.dictConfig(json.loads(f.read()))
diff --git a/inventory_provider/ims.py b/inventory_provider/ims.py
new file mode 100644
index 0000000000000000000000000000000000000000..d7b4eebd7297576c0936b5b2461bfca3117702f8
--- /dev/null
+++ b/inventory_provider/ims.py
@@ -0,0 +1,388 @@
+import logging
+
+import requests
+import time
+
+from enum import Enum
+
+
+# Navigation Properties
+# http://149.210.162.190:81/ImsVersions/4.19.9/html/86d07a57-fa45-835e-d4a2-a789c4acbc96.htm # noqa
+from requests import HTTPError
+
+logger = logging.getLogger(__name__)
+
+# http://149.210.162.190:81/ImsVersions/21.9/html/50e6a1b1-3910-2091-63d5-e13777b2194e.htm # noqa
+CIRCUIT_CUSTOMER_RELATION = {
+ "Circuit": 2,
+ "Customer": 4
+}
+# http://149.210.162.190:81/ImsVersions/20.1/html/86d07a57-fa45-835e-d4a2-a789c4acbc96.htm # noqa
+CIRCUIT_PROPERTIES = {
+ 'Site': 8,
+ 'Speed': 16,
+ 'Customer': 32,
+ 'Product': 128,
+ 'CalculatedNode': 256,
+ 'Ports': 512,
+ 'InternalPorts': 1024,
+ 'CarrierCircuits': 65536,
+ 'SubCircuits': 131072,
+ 'PortsFullDetails': 262144,
+ 'InternalPortsFullDetails': 524288,
+ 'PortA': 34359738368,
+ 'PortB': 68719476736
+}
+# http://149.210.162.190:81/ImsVersions/4.19.9/html/347cb410-8c05-47bd-ceb0-d1dd05bf98a4.htm # noqa
+CITY_PROPERTIES = {
+ 'Country': 8
+}
+# http://149.210.162.190:81/ImsVersions/4.19.9/html/dbc969d0-e735-132e-6281-f724c6d7da64.htm # noqa
+CONTACT_PROPERTIES = {
+ 'SiteRelatedContacts': 8,
+ 'CustomerRelatedContacts': 16,
+ 'GroupRelatedContacts': 32,
+ 'VendorRelatedContacts': 64
+}
+# http://149.210.162.190:81/ImsVersions/4.19.9/html/5a40472e-48ee-c120-0a36-52a85d52127c.htm # noqa
+CUSTOMER_PROPERTIES = {
+ 'CustomerRelatedContacts': 32768,
+ 'CustomerType': 262144
+}
+# http://149.210.162.190:81/ImsVersions/20.1/html/4f3e1d5e-53c3-1beb-cb16-65b5308dcb81.htm
+CUSTOMER_RELATED_CONTACT_PROPERTIES = {
+ 'Customer': 8,
+ 'Contact': 16
+}
+# http://149.210.162.190:81/ImsVersions/4.19.9/html/a8dc6266-d934-8162-4a55-9e1648187f2c.htm # noqa
+EQUIP_DEF_PROPERTIES = {
+ 'Nodes': 4096
+}
+# http://149.210.162.190:81/ImsVersions/20.1/html/6fd3a968-26e2-e40f-e3cd-c99afa34c3e6.htm
+EXTRA_FIELD_VALUE_PROPERTIES = {
+ 'ExtraFieldValueObjectInfo': 64
+}
+# http://149.210.162.190:81/ImsVersions/4.19.9/html/f18222e3-e353-0abe-b89c-820db87940ac.htm # noqa
+INTERNAL_PORT_PROPERTIES = {
+ 'Node': 8,
+ 'Shelf': 64,
+ 'Circuit': 256,
+}
+# http://149.210.162.190:81/ImsVersions/4.19.9/html/2d27d509-77cb-537d-3ffa-796de7e82af8.htm # noqa
+NODE_PROPERTIES = {
+ 'EquipmentDefinition': 16,
+ 'ManagementSystem': 128,
+ 'Site': 256,
+ 'Shelves': 1024,
+ 'Ports': 4096,
+ 'InternalPorts': 8192,
+ 'NodeCity': 32768,
+ 'Order': 67108864,
+ 'NodeCountry': 1073741824
+}
+# http://149.210.162.190:81/ImsVersions/4.19.9/html/199f32b5-5104-fec7-8787-0d730113e902.htm # noqa
+PORT_PROPERTIES = {
+ 'Node': 16,
+ 'Shelf': 32,
+ 'Circuit': 512,
+}
+# http://149.210.162.190:81/ImsVersions/4.19.9/html/9c8d50f0-842c-5959-0fa6-14e1720669ec.htm # noqa
+SITE_PROPERTIES = {
+ 'City': 2,
+ 'SiteAliases': 64,
+ 'Country': 256,
+ 'Nodes': 32768
+}
+# http://149.210.162.190:81/ImsVersions/4.19.9/html/8ce06cb7-7707-46c4-f02f-86083310d81b.htm # noqa
+VENDOR_PROPERTIES = {
+ 'VendorRelatedContacts': 64,
+ 'VendorType': 1024
+}
+# http://149.210.162.190:81/ImsVersions/4.19.9/html/62e7aa63-aff0-6992-7697-377ace239c4f.htm # noqa
+VENDOR_RELATED_CONTACT_PROPERTIES = {
+ 'Vendor': 8,
+ 'Contact': 16
+}
+
+VM_PORT_RELATE_PROPERTIES = {
+ 'Circuit': 32
+}
+
+VM_INTERNAL_PORT_RELATE_PROPERTIES = {
+ 'Circuit': 32
+}
+
+NO_FILTERED_RESULTS_MESSAGE = 'no records found for entity:'
+
+# this will be obsolete as soon as Inventory Provider update is done, but is
+# here for between the time of the roll out and the Inventory Update
+IMS_SERVICE_NAMES = {
+ 'EUMETSAT GRE',
+ 'EUMETSAT INTERNATIONAL',
+ 'EUMETSAT TERRESTRIAL',
+ 'EXPRESS ROUTE',
+ 'GEANT - GBS',
+ 'GEANT CLOUD PEERING',
+ 'GEANT IP',
+ 'GEANT LAMBDA',
+ 'GEANT OPEN CROSS CONNECT',
+ 'GEANT OPEN PORT',
+ 'GEANT PEERING',
+ 'GEANT PLUS',
+ 'GTS',
+ 'GWS - BROKERED',
+ 'GWS - DIRECT',
+ 'GWS - INDIRECT',
+ 'GWS - UPSTREAM',
+ 'IP PEERING - NON R&E (PRIVATE)',
+ 'IP PEERING - NON R&E (PUBLIC)',
+ 'IP PEERING - R&E',
+ 'IP TRUNK',
+ 'L2SERVICES',
+ 'L3-VPN',
+ 'MD-VPN (INTERNAL)',
+ 'MD-VPN (NATIVE)',
+ 'MD-VPN (PROXY)',
+ 'POP LAN LINK',
+ 'SERVER LINK'
+}
+
+
+class CustomerType(Enum):
+ COMMERCIAL_PEER = 1
+ CONNECTIVITY_SUPPLIER = 2
+ EU_NREN = 3
+ HOUSING_SUPPLIER = 4
+ IX_SUPPLIER = 5
+ OTHER = 6
+ R_AND_E_PEER = 7
+ UNKNOWN = 'UNKNOWN'
+
+
+class InventoryStatus(Enum):
+ PLANNED = 1
+ READY_FOR_SERVICE = 2
+ IN_SERVICE = 3
+ MIGRATION = 4
+ OUT_OF_SERVICE = 6
+ READY_FOR_CEASURE = 7
+
+
+class RelateType(Enum):
+ VENDOR = 1
+ CONTACT = 2
+ GROUP = 3
+ CONTRACT = 4
+ CUSTOMER = 5
+ SITE = 6
+
+
+class IMSError(Exception):
+ pass
+
+
+class IMS(object):
+
+ TIMEOUT_THRESHOLD = 1200
+ PERMITTED_RECONNECT_ATTEMPTS = 3
+ LOGIN_PATH = '/login'
+ IMS_PATH = '/ims'
+
+ cache = {}
+ base_url = None
+ bearer_token = None
+ bearer_token_init_time = 0
+ reconnect_attempts = 0
+
+ def __init__(self, base_url, username, password, bearer_token=None):
+ IMS.base_url = base_url
+ self.username = username
+ self.password = password
+ IMS.bearer_token = bearer_token
+
+ @classmethod
+ def _init_bearer_token(cls, username, password):
+ re_init_time = time.time()
+ if not cls.bearer_token or \
+ re_init_time - cls.bearer_token_init_time \
+ > cls.TIMEOUT_THRESHOLD:
+ cls.reconnect_attempts = 0
+ else:
+ cls.reconnect_attempts += 1
+ if cls.reconnect_attempts > cls.PERMITTED_RECONNECT_ATTEMPTS:
+ raise IMSError('Too many reconnection attempts made')
+
+ logger.debug(f'Logging in - Username: {username}'
+ f' - URL: {cls.base_url + cls.LOGIN_PATH}')
+ response = requests.post(
+ cls.base_url + cls.LOGIN_PATH,
+ auth=(username, password))
+ response.raise_for_status()
+ cls.bearer_token_init_time = re_init_time
+ cls.bearer_token = response.text
+
+ def clear_dynamic_context_cache(self):
+ if not IMS.bearer_token:
+ IMS._init_bearer_token(self.username, self.password)
+ while True:
+ logger.info('Clearing Dynamic Context Cache')
+ response = requests.put(
+ f'{self.base_url + IMS.IMS_PATH}/ClearDynamicContextCache',
+ headers={'Authorization': f'Bearer {self.bearer_token}'})
+ if response.status_code == 401:
+ IMS._init_bearer_token(self.username, self.password)
+ continue
+ response.raise_for_status()
+ break
+
+ def _get_entity(
+ self,
+ url,
+ params=None,
+ navigation_properties=None,
+ use_cache=False):
+ url = f'{self.base_url + IMS.IMS_PATH}/{url}'
+ cache_key = url
+ if navigation_properties:
+ params = params if params else {}
+ params['navigationproperty'] = navigation_properties if isinstance(
+ navigation_properties, int) else sum(navigation_properties)
+
+ if use_cache:
+ if params:
+ s = '-'.join(
+ [f'{t}::{params[t]}'
+ for t in sorted(params)])
+ cache_key = f'{cache_key}::{s}'
+ entity = IMS.cache.get(cache_key, None)
+ if entity:
+ return entity
+
+ if not IMS.bearer_token:
+ IMS._init_bearer_token(self.username, self.password)
+
+ def _is_invalid_login_state(response_):
+ if response_.status_code == requests.codes.unauthorized:
+ return True
+
+ if response_.status_code in (requests.codes.ok,
+ requests.codes.not_found):
+
+ if NO_FILTERED_RESULTS_MESSAGE in response_.text.lower():
+ return False
+ try:
+ r = _convert_keys(response_.json())
+ except Exception as e:
+ t = response_.text
+ if len(t) > 100:
+ message_text = f"{t[:50]} ... {t[-50:]}"
+ else:
+ message_text = t
+ logger.debug(f"unexpected response: {message_text}\n{e}"
+ "\nre-raising")
+ raise e
+ if r and 'haserrors' in r and r['haserrors']:
+ for e in r['Errors']:
+ if e['errormessage'] and \
+ 'guid expired' in e['errormessage'].lower():
+ return True
+
+ return False
+
+ def _convert_keys(source):
+ if isinstance(source, list):
+ return [_convert_keys(x) for x in source]
+ elif isinstance(source, dict):
+ new = {}
+ for k, v in source.items():
+ if isinstance(v, (dict, list)):
+ v = _convert_keys(v)
+ new[k.lower()] = v
+ return new
+ return source
+
+ while True:
+ response = requests.get(
+ url,
+ headers={'Authorization': f'Bearer {self.bearer_token}'},
+ params=params)
+ if _is_invalid_login_state(response):
+ IMS._init_bearer_token(self.username, self.password)
+ else:
+ response.raise_for_status()
+ orig = response.json()
+ return_value = _convert_keys(orig)
+
+ if use_cache:
+ IMS.cache[cache_key] = return_value
+ return return_value
+
+ def get_entity_by_id(
+ self,
+ entity_type,
+ entity_id,
+ navigation_properties=None,
+ use_cache=False):
+
+ url = f'{entity_type}/{entity_id}'
+ return \
+ self._get_entity(url, None, navigation_properties, use_cache)
+
+ def get_entity_by_name(
+ self,
+ entity,
+ name,
+ navigation_properties=None,
+ use_cache=False):
+ url = f'{entity}/byname/"{name}"'
+ return self._get_entity(url, None, navigation_properties, use_cache)
+
+ def get_filtered_entities(
+ self,
+ entity,
+ filter_string,
+ navigation_properties=None,
+ use_cache=False,
+ step_count=50):
+ more_to_come = True
+ start_entity = 0
+ while more_to_come:
+ params = {
+ 'paginatorStartElement': start_entity,
+ 'paginatorNumberOfElements': step_count
+ }
+ url = f'{entity}/filtered/{filter_string}'
+ try:
+ more_to_come = False
+ entities = self._get_entity(
+ url,
+ params,
+ navigation_properties,
+ use_cache)
+ except HTTPError as e:
+ r = e.response
+ if r.status_code == requests.codes.not_found \
+ and NO_FILTERED_RESULTS_MESSAGE in r.text.lower():
+ entities = None
+ else:
+ raise e
+ if entities:
+ more_to_come = \
+ len(entities) >= step_count
+ start_entity += step_count
+ yield from entities
+
+ def get_all_entities(
+ self,
+ entity,
+ navigation_properties=None,
+ use_cache=False,
+ step_count=50
+ ):
+ yield from self.get_filtered_entities(
+ entity,
+ 'Id <> 0',
+ navigation_properties,
+ use_cache,
+ step_count
+ )
diff --git a/inventory_provider/ims_data.py b/inventory_provider/ims_data.py
new file mode 100644
index 0000000000000000000000000000000000000000..28392fb60c27d80f2c7a2d142970d7b0ae492aa4
--- /dev/null
+++ b/inventory_provider/ims_data.py
@@ -0,0 +1,570 @@
+import logging
+import re
+from collections import defaultdict
+from copy import copy
+from itertools import chain
+
+from inventory_provider import environment
+from inventory_provider import ims
+from inventory_provider.ims import InventoryStatus, IMS, \
+ CUSTOMER_RELATED_CONTACT_PROPERTIES, EXTRA_FIELD_VALUE_PROPERTIES
+
+environment.setup_logging()
+logger = logging.getLogger(__name__)
+
+# Dashboard V3
+
+IMS_OPSDB_STATUS_MAP = {
+ InventoryStatus.PLANNED: 'planned',
+ InventoryStatus.READY_FOR_SERVICE: 'installed',
+ InventoryStatus.IN_SERVICE: 'operational',
+ InventoryStatus.MIGRATION: 'planned',
+ InventoryStatus.OUT_OF_SERVICE: 'terminated',
+ InventoryStatus.READY_FOR_CEASURE: 'terminated'
+}
+STATUSES_TO_IGNORE = \
+ [InventoryStatus.OUT_OF_SERVICE.value]
+
+_POP_LOCATION_SCHEMA_STRUCT = {
+ 'type': 'object',
+ 'properties': {
+ 'name': {'type': 'string'},
+ 'city': {'type': 'string'},
+ 'country': {'type': 'string'},
+ 'abbreviation': {'type': 'string'},
+ 'longitude': {'type': 'number'},
+ 'latitude': {'type': 'number'}
+ },
+ 'required': [
+ 'name',
+ 'city',
+ 'country',
+ 'abbreviation',
+ 'longitude',
+ 'latitude'],
+ 'additionalProperties': False
+}
+
+POP_LOCATION_SCHEMA = {
+ '$schema': 'http://json-schema.org/draft-07/schema#',
+ **_POP_LOCATION_SCHEMA_STRUCT
+}
+
+NODE_LOCATION_SCHEMA = {
+ '$schema': 'http://json-schema.org/draft-07/schema#',
+ 'definitions': {
+ 'pop-location': _POP_LOCATION_SCHEMA_STRUCT
+ },
+
+ 'type': 'object',
+ 'properties': {
+ 'equipment-name': {'type': 'string'},
+ 'status': {'type': 'string'},
+ 'pop': {'$ref': '#/definitions/pop-location'}
+ },
+ 'required': ['equipment-name', 'status', 'pop'],
+ 'additionalProperties': False
+}
+
+
+def get_flexils_by_circuitid(ds: IMS):
+ by_circuit = defaultdict(list)
+ found_keys = set()
+ for entity in ds.get_all_entities('FLEXILS_SCHF_SUBINTERFACES'):
+ k = f'{entity["nodename"]}:{entity["port_ref"]}'
+ if k in found_keys:
+ continue
+ found_keys.add(k)
+ by_circuit[entity['circuitid']].append({
+ 'node_name': entity['nodename'],
+ 'full_port_name': entity['port_ref'],
+ 'key': k
+ })
+ return dict(by_circuit)
+
+
+def get_non_monitored_circuit_ids(ds: IMS):
+ # note the id for the relevant field is hard-coded. I didn't want to use
+ # the name of the field as this can be changed by users
+ for d in ds.get_filtered_entities(
+ 'ExtraFieldValue',
+ 'extrafield.id == 2898 | value == 0',
+ EXTRA_FIELD_VALUE_PROPERTIES['ExtraFieldValueObjectInfo']
+ ):
+ yield d['extrafieldvalueobjectinfo']['objectid']
+
+
+def get_monitored_circuit_ids(ds: IMS):
+ # note the id for the relevant field is hard-coded. I didn't want to use
+ # the name of the field as this can be changed by users
+ for d in ds.get_filtered_entities(
+ 'ExtraFieldValue',
+ 'extrafield.id == 2898 | value == 1',
+ EXTRA_FIELD_VALUE_PROPERTIES['ExtraFieldValueObjectInfo'],
+ step_count=10000
+ ):
+ yield d['extrafieldvalueobjectinfo']['objectid']
+
+
+def get_ids_and_sids(ds: IMS):
+ for sid_circuit in ds.get_filtered_entities(
+ 'ExtraFieldValue',
+ 'extrafieldid == 3209 | value <> ""',
+ step_count=10000
+ ):
+ yield sid_circuit['objectid'], sid_circuit['value']
+
+
+def get_service_types(ds: IMS):
+ for d in ds.get_filtered_entities(
+ 'ComboBoxData',
+ 'name == "PW_INFORM_PRODUCTCODE"'):
+ yield d['selection']
+
+
+def get_customer_tts_contacts(ds: IMS):
+
+ customer_contacts = defaultdict(set)
+
+ for x in ds.get_filtered_entities(
+ 'customerrelatedcontact',
+ "contact.troubleticketMail != ''",
+ CUSTOMER_RELATED_CONTACT_PROPERTIES['Contact']
+ ):
+ customer_contacts[x['customerid']].add(
+ x['contact']['troubleticketmail'])
+ for k, v in customer_contacts.items():
+ yield k, sorted(list(v))
+
+
+def get_customer_planned_work_contacts(ds: IMS):
+
+ customer_contacts = defaultdict(set)
+
+ for x in ds.get_filtered_entities(
+ 'customerrelatedcontact',
+ "contact.PlannedworkMail != ''",
+ CUSTOMER_RELATED_CONTACT_PROPERTIES['Contact']
+ ):
+ customer_contacts[x['customerid']].add(
+ x['contact']['plannedworkmail'])
+ for k, v in customer_contacts.items():
+ yield k, sorted(list(v))
+
+
+def get_circuit_related_customers(ds: IMS):
+
+ return_value = defaultdict(list)
+ for ccr in ds.get_filtered_entities(
+ 'CircuitCustomerRelation',
+ 'circuit.inventoryStatusId== 3',
+ ims.CIRCUIT_CUSTOMER_RELATION['Customer']):
+ return_value[ccr['circuitid']].append(
+ {
+ 'id': ccr['customer']['id'],
+ 'name': ccr['customer']['name'],
+ 'type': ccr['customer']['customertypeid']
+ }
+ )
+ return return_value
+
+
+def get_port_id_services(ds: IMS):
+ circuit_nav_props = [
+ ims.CIRCUIT_PROPERTIES['Ports'],
+ ims.CIRCUIT_PROPERTIES['InternalPorts'],
+ ]
+ ims_service_names = list(get_service_types(ds))
+ customers = {c['id']: c['name'] for c in ds.get_all_entities('customer')}
+ products = {p['id']: p['name'] for p in ds.get_all_entities('product')}
+
+ def _get_circuit_type(c):
+ if products[c['productid']] in ims_service_names:
+ return 'service'
+ else:
+ return 'circuit'
+
+ def _get_circuits():
+
+ _ignore_status_str = ' | '.join([
+ f'inventoryStatusId != {s}' for s in STATUSES_TO_IGNORE
+ ])
+ for c in ds.get_filtered_entities(
+ 'Circuit',
+ _ignore_status_str,
+ circuit_nav_props,
+ step_count=2000):
+ c['circuit_type'] = _get_circuit_type(c)
+ yield c
+
+ circuits = _get_circuits()
+
+ # order of preference
+ # internalports (first and last sequencenumbers)
+ # ports (first and last sequencenumbers)
+ # internal port a / internal port b
+ # port a / port b
+
+ # if there are more than two ports we'll yield a circuit of the ends of the
+ # sequence and the reverse; and then single ended circuits for all ports
+ # between
+ # e.g. four ports are reported [a,b,c,d] the 4 circs would have endpoints
+ # a and d
+ # d and a
+ # b only
+ # c only
+ def _populate_end_info(_circuit, _port_ids):
+ port_ids = [p for p in _port_ids if p]
+ if not port_ids:
+ return []
+ port_a_id = port_ids[0]
+ port_b_id = port_ids[-1]
+ _circuit['port_a_id'] = port_a_id
+ if port_a_id != port_b_id:
+ _circuit['port_b_id'] = port_b_id
+ yield copy(_circuit)
+ _circuit['port_a_id'], _circuit['port_b_id'] = \
+ _circuit['port_b_id'], _circuit['port_a_id']
+ yield copy(_circuit)
+ if len(port_ids) > 2:
+ _circuit.pop('port_b_id', None)
+ for p in port_ids[1:-1]:
+ _circuit['port_a_id'] = p
+ yield copy(_circuit)
+
+ for circuit in circuits:
+ cd = {
+ 'id': circuit['id'],
+ 'name': circuit['name'],
+ 'status': IMS_OPSDB_STATUS_MAP.get(
+ InventoryStatus(circuit['inventorystatusid']), 'unknown'),
+ 'circuit_type': circuit['circuit_type'],
+ 'service_type': products[circuit['productid']],
+ 'project': customers[circuit['customerid']],
+ 'customer': customers[circuit['customerid']],
+ 'customerid': circuit['customerid']
+ }
+ ports = []
+ cd['port_type'] = 'unknowm'
+ if circuit['internalports']:
+ ports = sorted(
+ circuit['internalports'], key=lambda x: x['sequencenumber'])
+ ports = [p['id'] for p in ports]
+ cd['port_type'] = 'internal'
+ elif circuit['ports']:
+ ports = sorted(
+ circuit['ports'], key=lambda x: x['sequencenumber'])
+ ports = [p['id'] for p in ports]
+ cd['port_type'] = 'ports'
+ elif circuit['portaid'] or circuit['portbid']:
+ ports = [circuit['portaid'], circuit['portbid']]
+ cd['port_type'] = 'ab'
+ yield from _populate_end_info(cd, ports)
+
+ ignore_status_str = ''.join([
+ f'circuit.inventoryStatusId != {s} | ' for s in STATUSES_TO_IGNORE
+ ])
+ for portrelate in chain(
+ ds.get_filtered_entities(
+ 'vmportrelate',
+ ignore_status_str +
+ 'circuitId != 0',
+ ims.VM_PORT_RELATE_PROPERTIES['Circuit'],
+ step_count=2000
+ ),
+ ds.get_filtered_entities(
+ 'vminternalportrelate',
+ ignore_status_str +
+ 'circuitId != 0',
+ ims.VM_INTERNAL_PORT_RELATE_PROPERTIES['Circuit'],
+ step_count=2000
+ )
+ ):
+ circuit = portrelate['circuit']
+ if circuit:
+ yield {
+ 'id': circuit['id'],
+ 'name': circuit['name'],
+ 'status': IMS_OPSDB_STATUS_MAP.get(
+ InventoryStatus(circuit['inventorystatusid']),
+ 'unknown'),
+ 'circuit_type': _get_circuit_type(circuit),
+ 'service_type': products[circuit['productid']],
+ 'project': customers[circuit['customerid']],
+ 'customer': customers[circuit['customerid']],
+ 'customerid': circuit['customerid'],
+ 'port_a_id': portrelate.get(
+ 'portid',
+ portrelate.get('internalportid', '')),
+ 'port_type': 'port relate'
+ }
+
+
+def get_port_sids(ds: IMS):
+ """
+ This function fetches SIDs for external ports that have them defined,
+
+ :param ds: IMS datasource object
+ :returns: Dict mapping external port IDs to the SID assigned to the port
+ """
+ return {
+ p['objectid']: p['value'] for p in ds.get_filtered_entities(
+ 'ExtraFieldValue', 'extrafieldid == 3249 | value <> ""',
+ step_count=10000)}
+
+
+def get_internal_port_sids(ds: IMS):
+ """
+ This function fetches SIDs for external ports that have them defined,
+
+ :param ds: IMS datasource object
+ :returns: Dict mapping internal port IDs to the SID assigned to the port
+ """
+ return {
+ p['objectid']: p['value'] for p in ds.get_filtered_entities(
+ 'ExtraFieldValue', 'extrafieldid == 3250 | value <> ""',
+ step_count=10000)}
+
+
+def get_port_details(ds: IMS):
+ port_nav_props = [
+ ims.PORT_PROPERTIES['Node'],
+ ims.PORT_PROPERTIES['Shelf']
+ ]
+ internal_port_nav_props = {
+ ims.INTERNAL_PORT_PROPERTIES['Node'],
+ ims.PORT_PROPERTIES['Shelf']
+ }
+
+ # this is here instead of chaining to make debugging easier
+ def _process_ports(ports, p_type):
+ _port_sids = {}
+
+ if p_type == 'external':
+ _port_sids = get_port_sids(ds)
+ else:
+ _port_sids = get_internal_port_sids(ds)
+
+ for p in ports:
+ vendor = None
+ interface_name = None
+ try:
+ vendor = \
+ p['node']['equipmentdefinition']['vendor']['name'].lower()
+ except (TypeError, KeyError):
+ pass
+ # if there become more exceptions we will need to abstract this
+ if vendor == 'infinera' and p.get('shelf', None):
+ try:
+ interface_name = \
+ f"{p['shelf']['sequencenumber']}-{p['name']}"
+ except KeyError:
+ pass
+
+ if not interface_name:
+ interface_name = p['name']
+
+ data = {
+ 'port_id': p['id'],
+ 'equipment_name': p['node']['name'],
+ 'interface_name': interface_name
+ }
+
+ if p['id'] in _port_sids:
+ data['sid'] = _port_sids[p['id']]
+
+ yield data
+
+ yield from _process_ports(ds.get_all_entities(
+ 'port', port_nav_props, step_count=2000), 'external')
+ yield from _process_ports(ds.get_all_entities(
+ 'internalport', internal_port_nav_props, step_count=2000), 'internal')
+
+
+def get_circuit_hierarchy(ds: IMS):
+ circuit_nav_props = [
+ ims.CIRCUIT_PROPERTIES['Customer'],
+ ims.CIRCUIT_PROPERTIES['Product'],
+ ims.CIRCUIT_PROPERTIES['Speed'],
+ ims.CIRCUIT_PROPERTIES['SubCircuits'],
+ ims.CIRCUIT_PROPERTIES['CarrierCircuits']
+ ]
+
+ ignore_status_str = ' | '.join([
+ f'inventoryStatusId != {s}' for s in STATUSES_TO_IGNORE
+ ])
+ circuits = ds.get_filtered_entities(
+ 'Circuit',
+ ignore_status_str,
+ circuit_nav_props,
+ step_count=1000)
+ # circuits = ds.get_all_entities(
+ # 'Circuit', circuit_nav_props, step_count=1000)
+ service_types = list(get_service_types(ds))
+ for circuit in circuits:
+ if circuit['product']['name'] in service_types:
+ circuit_type = 'service'
+ else:
+ circuit_type = 'circuit'
+
+ sub_circuits = [c['subcircuitid'] for c in circuit['subcircuits']]
+ carrier_circuits = \
+ [c['carriercircuitid'] for c in circuit['carriercircuits']]
+ yield {
+ 'id': circuit['id'],
+ 'name': circuit['name'],
+ 'status': IMS_OPSDB_STATUS_MAP.get(
+ InventoryStatus(circuit['inventorystatusid']), 'unknown'),
+ 'product': circuit['product']['name'],
+ 'speed': circuit['speed']['name'],
+ 'project': circuit['customer']['name'],
+ 'circuit-type': circuit_type,
+ 'sub-circuits': sub_circuits,
+ 'carrier-circuits': carrier_circuits,
+ 'customerid': circuit['customerid']
+ }
+
+
+def get_node_locations(ds: IMS):
+ """
+ return location info for all Site nodes
+
+ yields dictionaries formatted as:
+
+ .. as_json::
+ inventory_provider.db.ims_data.NODE_LOCATION_SCHEMA
+
+ :param ds:
+ :return: yields dicts as above
+ """
+ site_nav_props = [
+ ims.SITE_PROPERTIES['City'],
+ ims.SITE_PROPERTIES['SiteAliases'],
+ ims.SITE_PROPERTIES['Country'],
+ ims.SITE_PROPERTIES['Nodes']
+ ]
+ sites = ds.get_all_entities('Site', site_nav_props, step_count=500)
+ for site in sites:
+ city = site['city']
+ abbreviation = ''
+ try:
+ abbreviation = site['sitealiases'][0]['aliasname']
+ except IndexError:
+ pass # no alias - ignore silently
+
+ for node in site['nodes']:
+ if node['inventorystatusid'] in STATUSES_TO_IGNORE:
+ continue
+
+ yield (node['name'], {
+ 'equipment-name': node['name'],
+ 'status': IMS_OPSDB_STATUS_MAP.get(
+ InventoryStatus(node['inventorystatusid']), 'unknown'),
+ 'pop': {
+ 'name': site['name'],
+ 'city': city['name'],
+ 'country': city['country']['name'],
+ 'abbreviation': abbreviation,
+ 'longitude': site['longitude'],
+ 'latitude': site['latitude'],
+ }
+ })
+
+ yield ('UNKNOWN_LOC', {
+ 'equipment-name': 'UNKNOWN',
+ 'status': 'unknown',
+ 'pop': {
+ 'name': 'UNKNOWN',
+ 'city': 'UNKNOWN',
+ 'country': 'UNKNOWN',
+ 'abbreviation': 'UNKNOWN',
+ 'longitude': 0,
+ 'latitude': 0,
+ }
+ })
+
+
+# End of Dashboard V3 stuff
+
+INTERNAL_POP_NAMES = {
+ 'Cambridge OC',
+ 'DANTE Lab',
+ 'GÉANT LAB',
+ 'GEANT LAB',
+ 'Amsterdam GEANT Office',
+ 'Amsterdam GÉANT Office',
+ 'CAMBRIDGE CITY HOUSE',
+ 'AMSTERDAM GEANT OFFICE',
+}
+
+
+def lookup_lg_routers(ds: IMS):
+ pattern = re.compile("vpn-proxy|vrr|taas", re.IGNORECASE)
+
+ def _matching_node(node_):
+ # [LG-46]
+ if InventoryStatus(node_['inventorystatusid']) \
+ != InventoryStatus.IN_SERVICE:
+ return False
+
+ if pattern.match(node_['name']):
+ return False
+
+ return True
+
+ site_nav_props = [
+ ims.SITE_PROPERTIES['SiteAliases'],
+ ims.SITE_PROPERTIES['City'],
+ ims.SITE_PROPERTIES['Country']
+ ]
+
+ eq_definitions = ds.get_filtered_entities(
+ 'EquipmentDefinition',
+ 'Name like MX',
+ ims.EQUIP_DEF_PROPERTIES['Nodes'])
+
+ for eq_def in eq_definitions:
+ nodes = eq_def['nodes']
+
+ for node in nodes:
+ if not _matching_node(node):
+ continue
+ # [LG - 46]
+ if node['inventorystatusid'] != InventoryStatus.IN_SERVICE.value:
+ continue
+
+ site = ds.get_entity_by_id('Site', node['siteid'], site_nav_props,
+ True)
+ city = site['city']
+
+ abbreviation = ''
+ try:
+ abbreviation = site['sitealiases'][0]['aliasname']
+ except IndexError:
+ pass # no alias - ignore silently
+
+ eq = {
+ 'equipment name': node['name'],
+ 'type':
+ 'INTERNAL'
+ if site['name'] in INTERNAL_POP_NAMES
+ else 'CORE',
+ 'pop': {
+ 'name': site['name'],
+ 'city': city['name'],
+ 'country': city['country']['name'],
+ 'country code': city['country']['abbreviation'],
+ 'abbreviation': abbreviation,
+ 'longitude': site['longitude'],
+ 'latitude': site['latitude'],
+ }
+ }
+ yield eq
+
+
+def lookup_geant_nodes(ds: IMS):
+
+ return (n["name"]for n in ds.get_filtered_entities(
+ 'Node',
+ 'customer.Name == "GEANT"',
+ ims.EQUIP_DEF_PROPERTIES['Nodes']))
diff --git a/inventory_provider/logging_default_config.json b/inventory_provider/logging_default_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..592693384664dc5882f683caacd49977986dcfb8
--- /dev/null
+++ b/inventory_provider/logging_default_config.json
@@ -0,0 +1,62 @@
+{
+ "version": 1,
+ "disable_existing_loggers": false,
+ "formatters": {
+ "simple": {
+ "format": "%(asctime)s - %(name)s (%(lineno)d) - %(levelname)s - %(message)s"
+ }
+ },
+
+ "handlers": {
+ "console": {
+ "class": "logging.StreamHandler",
+ "level": "DEBUG",
+ "formatter": "simple",
+ "stream": "ext://sys.stdout"
+ },
+
+ "syslog_handler": {
+ "class": "logging.handlers.SysLogHandler",
+ "level": "DEBUG",
+ "address": "/dev/log",
+ "facility": "user",
+ "formatter": "simple"
+ },
+
+ "info_file_handler": {
+ "class": "logging.handlers.RotatingFileHandler",
+ "level": "INFO",
+ "formatter": "simple",
+ "filename": "info.log",
+ "maxBytes": 10485760,
+ "backupCount": 20,
+ "encoding": "utf8"
+ },
+
+ "error_file_handler": {
+ "class": "logging.handlers.RotatingFileHandler",
+ "level": "ERROR",
+ "formatter": "simple",
+ "filename": "errors.log",
+ "maxBytes": 10485760,
+ "backupCount": 20,
+ "encoding": "utf8"
+ }
+ },
+
+ "loggers": {
+ "inventory_provider": {
+ "level": "DEBUG",
+ "handlers": ["console", "syslog_handler"],
+ "propagate": false
+ },
+ "inventory_provider.tasks": {
+ "level": "DEBUG"
+ }
+ },
+
+ "root": {
+ "level": "DEBUG",
+ "handlers": ["console", "syslog_handler"]
+ }
+}