ims_data.py

import logging
import re
from collections import defaultdict
from copy import copy
from itertools import chain

from inventory_provider import environment
from inventory_provider.db import ims
from inventory_provider.db.ims import InventoryStatus, IMS, \
    CUSTOMER_RELATED_CONTACT_PROPERTIES, EXTRA_FIELD_VALUE_PROPERTIES

environment.setup_logging()
logger = logging.getLogger(__name__)

# Dashboard V3

IMS_OPSDB_STATUS_MAP = {
    InventoryStatus.PLANNED: 'planned',
    InventoryStatus.READY_FOR_SERVICE: 'installed',
    InventoryStatus.IN_SERVICE: 'operational',
    InventoryStatus.MIGRATION: 'planned',
    InventoryStatus.OUT_OF_SERVICE: 'terminated',
    InventoryStatus.READY_FOR_CEASURE: 'terminated'
}
STATUSES_TO_IGNORE = \
    [InventoryStatus.OUT_OF_SERVICE.value]

_POP_LOCATION_SCHEMA_STRUCT = {
    'type': 'object',
    'properties': {
        'name': {'type': 'string'},
        'city': {'type': 'string'},
        'country': {'type': 'string'},
        'abbreviation': {'type': 'string'},
        'longitude': {'type': 'number'},
        'latitude': {'type': 'number'}
    },
    'required': [
        'name',
        'city',
        'country',
        'abbreviation',
        'longitude',
        'latitude'],
    'additionalProperties': False
}

POP_LOCATION_SCHEMA = {
    '$schema': 'http://json-schema.org/draft-07/schema#',
    **_POP_LOCATION_SCHEMA_STRUCT
}

NODE_LOCATION_SCHEMA = {
    '$schema': 'http://json-schema.org/draft-07/schema#',
    'definitions': {
        'pop-location': _POP_LOCATION_SCHEMA_STRUCT
    },

    'type': 'object',
    'properties': {
        'equipment-name': {'type': 'string'},
        'status': {'type': 'string'},
        'pop':  {'$ref': '#/definitions/pop-location'}
    },
    'required': ['equipment-name', 'status', 'pop'],
    'additionalProperties': False
}


def get_non_monitored_circuit_ids(ds: IMS):
    # note the id for the relevant field is hard-coded. I didn't want to use
    # the name of the field as this can be changed by users
    for d in ds.get_filtered_entities(
            'ExtraFieldValue',
            'extrafield.id == 2898 | value == 0',
            EXTRA_FIELD_VALUE_PROPERTIES['ExtraFieldValueObjectInfo']
    ):
        yield d['extrafieldvalueobjectinfo']['objectid']


def get_monitored_circuit_ids(ds: IMS):
    # note the id for the relevant field is hard-coded. I didn't want to use
    # the name of the field as this can be changed by users
    for d in ds.get_filtered_entities(
            'ExtraFieldValue',
            'extrafield.id == 2898 | value == 1',
            EXTRA_FIELD_VALUE_PROPERTIES['ExtraFieldValueObjectInfo'],
            step_count=10000
    ):
        yield d['extrafieldvalueobjectinfo']['objectid']


def get_ids_and_sids(ds: IMS):
    for sid_circuit in ds.get_filtered_entities(
        'ExtraFieldValue',
        'extrafieldid == 3209 | value <> ""',
        step_count=10000
    ):
        yield sid_circuit['objectid'], sid_circuit['value']


def get_service_types(ds: IMS):
    for d in ds.get_filtered_entities(
            'ComboBoxData',
            'name == "PW_INFORM_PRODUCTCODE"'):
        yield d['selection']


def get_customer_service_emails(ds: IMS):

    customer_contacts = defaultdict(set)
    for x in ds.get_filtered_entities(
        'customerrelatedcontact',
        "contact.plannedworkmail != ''",
        CUSTOMER_RELATED_CONTACT_PROPERTIES['Contact']
    ):
        customer_contacts[x['customerid']].add(x['contact']['mail'])
    for x in ds.get_filtered_entities(
        'customerrelatedcontact',
        "contact.troubleticketMail != ''",
        CUSTOMER_RELATED_CONTACT_PROPERTIES['Contact']
    ):
        customer_contacts[x['customerid']].add(x['contact']['mail'])
    for k, v in customer_contacts.items():
        yield k, sorted(list(v))


def get_circuit_related_customers(ds: IMS):

    return_value = defaultdict(list)
    for ccr in ds.get_filtered_entities(
                'CircuitCustomerRelation',
                'circuit.inventoryStatusId== 3',
                ims.CIRCUIT_CUSTOMER_RELATION['Customer']):
        return_value[ccr['circuitid']].append(
            {
                'id': ccr['customer']['id'],
                'name': ccr['customer']['name'],
                'type': ccr['customer']['customertypeid']
            }
        )
    return return_value


def get_port_id_services(ds: IMS):
    circuit_nav_props = [
        ims.CIRCUIT_PROPERTIES['Ports'],
        ims.CIRCUIT_PROPERTIES['InternalPorts'],
    ]
    ims_service_names = list(get_service_types(ds))
    customers = {c['id']: c['name'] for c in ds.get_all_entities('customer')}
    products = {p['id']: p['name'] for p in ds.get_all_entities('product')}

    def _get_circuit_type(c):
        if products[c['productid']] in ims_service_names:
            return 'service'
        else:
            return 'circuit'

    def _get_circuits():

        _ignore_status_str = ' | '.join([
            f'inventoryStatusId != {s}' for s in STATUSES_TO_IGNORE
        ])
        for c in ds.get_filtered_entities(
                'Circuit',
                _ignore_status_str,
                circuit_nav_props,
                step_count=2000):
            c['circuit_type'] = _get_circuit_type(c)
            yield c

    circuits = _get_circuits()

    # order of preference
    # internalports (first and last sequencenumbers)
    # ports (first and last sequencenumbers)
    # internal port a / internal port b
    # port a / port b

    # if there are more than two ports we'll yield a circuit of the ends of the
    # sequence and the reverse; and then single ended circuits for all ports
    # between
    # e.g. four ports are reported [a,b,c,d] the 4 circs would have endpoints
    # a and d
    # d and a
    # b only
    # c only
    def _populate_end_info(_circuit, _port_ids):
        port_ids = [p for p in _port_ids if p]
        if not port_ids:
            return []
        port_a_id = port_ids[0]
        port_b_id = port_ids[-1]
        _circuit['port_a_id'] = port_a_id
        if port_a_id != port_b_id:
            _circuit['port_b_id'] = port_b_id
            yield copy(_circuit)
            _circuit['port_a_id'], _circuit['port_b_id'] = \
                _circuit['port_b_id'], _circuit['port_a_id']
        yield copy(_circuit)
        if len(port_ids) > 2:
            _circuit.pop('port_b_id', None)
            for p in port_ids[1:-1]:
                _circuit['port_a_id'] = p
                yield copy(_circuit)

    for circuit in circuits:
        cd = {
            'id': circuit['id'],
            'name': circuit['name'],
            'status': IMS_OPSDB_STATUS_MAP.get(
                    InventoryStatus(circuit['inventorystatusid']), 'unknown'),
            'circuit_type': circuit['circuit_type'],
            'service_type': products[circuit['productid']],
            'project': customers[circuit['customerid']],
            'customer': customers[circuit['customerid']],
            'customerid': circuit['customerid']
        }
        ports = []
        cd['port_type'] = 'unknowm'
        if circuit['internalports']:
            ports = sorted(
                circuit['internalports'], key=lambda x: x['sequencenumber'])
            ports = [p['id'] for p in ports]
            cd['port_type'] = 'internal'
        elif circuit['ports']:
            ports = sorted(
                circuit['ports'], key=lambda x: x['sequencenumber'])
            ports = [p['id'] for p in ports]
            cd['port_type'] = 'ports'
        elif circuit['portaid'] or circuit['portbid']:
            ports = [circuit['portaid'], circuit['portbid']]
            cd['port_type'] = 'ab'
        yield from _populate_end_info(cd, ports)

    ignore_status_str = ''.join([
        f'circuit.inventoryStatusId != {s} | ' for s in STATUSES_TO_IGNORE
    ])
    for portrelate in chain(
            ds.get_filtered_entities(
                'vmportrelate',
                ignore_status_str +
                'circuitId != 0',
                ims.VM_PORT_RELATE_PROPERTIES['Circuit'],
                step_count=2000
            ),
            ds.get_filtered_entities(
                'vminternalportrelate',
                ignore_status_str +
                'circuitId != 0',
                ims.VM_INTERNAL_PORT_RELATE_PROPERTIES['Circuit'],
                step_count=2000
            )
    ):
        circuit = portrelate['circuit']
        if circuit:
            yield {
                'id': circuit['id'],
                'name': circuit['name'],
                'status': IMS_OPSDB_STATUS_MAP.get(
                    InventoryStatus(circuit['inventorystatusid']),
                    'unknown'),
                'circuit_type': _get_circuit_type(circuit),
                'service_type': products[circuit['productid']],
                'project': customers[circuit['customerid']],
                'customer': customers[circuit['customerid']],
                'customerid': circuit['customerid'],
                'port_a_id': portrelate.get(
                    'portid',
                    portrelate.get('internalportid', '')),
                'port_type': 'port relate'
            }


def get_port_details(ds: IMS):
    port_nav_props = [
        ims.PORT_PROPERTIES['Node'],
        ims.PORT_PROPERTIES['Shelf']
    ]
    internal_port_nav_props = {
        ims.INTERNAL_PORT_PROPERTIES['Node'],
        ims.PORT_PROPERTIES['Shelf']
    }

    # this is here instead of chaining to make debugging easier
    def _process_ports(ports, p_type):
        for p in ports:
            vendor = None
            interface_name = None
            try:
                vendor = \
                    p['node']['equipmentdefinition']['vendor']['name'].lower()
            except (TypeError, KeyError):
                pass
            # if there become more exceptions we will need to abstract this
            if vendor == 'infinera' and p.get('shelf', None):
                try:
                    interface_name = \
                        f"{p['shelf']['sequencenumber']}-{p['name']}"
                except KeyError:
                    pass

            if not interface_name:
                interface_name = p['name']
            data = {
                'port_id': p['id'],
                'equipment_name': p['node']['name'],
                'interface_name': interface_name
            }
            yield data

    yield from _process_ports(ds.get_all_entities(
        'port', port_nav_props, step_count=2000), 'external')
    yield from _process_ports(ds.get_all_entities(
        'internalport', internal_port_nav_props, step_count=2000), 'internal')


def get_circuit_hierarchy(ds: IMS):
    circuit_nav_props = [
        ims.CIRCUIT_PROPERTIES['Customer'],
        ims.CIRCUIT_PROPERTIES['Product'],
        ims.CIRCUIT_PROPERTIES['Speed'],
        ims.CIRCUIT_PROPERTIES['SubCircuits'],
        ims.CIRCUIT_PROPERTIES['CarrierCircuits']
    ]

    ignore_status_str = ' | '.join([
        f'inventoryStatusId != {s}' for s in STATUSES_TO_IGNORE
    ])
    circuits = ds.get_filtered_entities(
        'Circuit',
        ignore_status_str,
        circuit_nav_props,
        step_count=1000)
    # circuits = ds.get_all_entities(
    #     'Circuit', circuit_nav_props, step_count=1000)
    service_types = list(get_service_types(ds))
    for circuit in circuits:
        if circuit['product']['name'] in service_types:
            circuit_type = 'service'
        else:
            circuit_type = 'circuit'

        sub_circuits = [c['subcircuitid'] for c in circuit['subcircuits']]
        carrier_circuits = \
            [c['carriercircuitid'] for c in circuit['carriercircuits']]
        yield {
            'id': circuit['id'],
            'name': circuit['name'],
            'status': IMS_OPSDB_STATUS_MAP.get(
                InventoryStatus(circuit['inventorystatusid']), 'unknown'),
            'product': circuit['product']['name'],
            'speed': circuit['speed']['name'],
            'project': circuit['customer']['name'],
            'circuit-type': circuit_type,
            'sub-circuits': sub_circuits,
            'carrier-circuits': carrier_circuits,
            'customerid': circuit['customerid']
        }


def get_node_locations(ds: IMS):
    """
    return location info for all Site nodes

    yields dictionaries formatted as:

    .. as_json::
        inventory_provider.db.ims_data.NODE_LOCATION_SCHEMA

    :param ds:
    :return: yields dicts as above
    """
    site_nav_props = [
        ims.SITE_PROPERTIES['City'],
        ims.SITE_PROPERTIES['SiteAliases'],
        ims.SITE_PROPERTIES['Country'],
        ims.SITE_PROPERTIES['Nodes']
    ]
    sites = ds.get_all_entities('Site', site_nav_props, step_count=500)
    for site in sites:
        city = site['city']
        abbreviation = ''
        try:
            abbreviation = site['sitealiases'][0]['aliasname']
        except IndexError:
            pass  # no alias - ignore silently

        for node in site['nodes']:
            if node['inventorystatusid'] in STATUSES_TO_IGNORE:
                continue

            yield (node['name'], {
                'equipment-name': node['name'],
                'status': IMS_OPSDB_STATUS_MAP.get(
                    InventoryStatus(node['inventorystatusid']), 'unknown'),
                'pop': {
                    'name': site['name'],
                    'city': city['name'],
                    'country': city['country']['name'],
                    'abbreviation': abbreviation,
                    'longitude': site['longitude'],
                    'latitude': site['latitude'],
                }
            })

    yield ('UNKNOWN_LOC', {
        'equipment-name': 'UNKNOWN',
        'status': 'unknown',
        'pop': {
            'name': 'UNKNOWN',
            'city': 'UNKNOWN',
            'country': 'UNKNOWN',
            'abbreviation': 'UNKNOWN',
            'longitude': 0,
            'latitude': 0,
        }
    })


# End of Dashboard V3 stuff

INTERNAL_POP_NAMES = {
    'Cambridge OC',
    'DANTE Lab',
    'GÉANT LAB',
    'GEANT LAB',
    'Amsterdam GEANT Office',
    'Amsterdam GÉANT Office',
    'CAMBRIDGE CITY HOUSE',
    'AMSTERDAM GEANT OFFICE',
}


def lookup_lg_routers(ds: IMS):
    pattern = re.compile("vpn-proxy|vrr|taas", re.IGNORECASE)

    def _matching_node(node_):
        if InventoryStatus(node_['inventorystatusid']) in STATUSES_TO_IGNORE:
            return False

        if pattern.match(node_['name']):
            return False

        return True

    site_nav_props = [
        ims.SITE_PROPERTIES['SiteAliases'],
        ims.SITE_PROPERTIES['City'],
        ims.SITE_PROPERTIES['Country']
    ]

    eq_definitions = ds.get_filtered_entities(
        'EquipmentDefinition',
        'Name like MX',
        ims.EQUIP_DEF_PROPERTIES['Nodes'])

    for eq_def in eq_definitions:
        nodes = eq_def['nodes']

        for node in nodes:
            if not _matching_node(node):
                continue
            if node['inventorystatusid'] in STATUSES_TO_IGNORE:
                continue

            site = ds.get_entity_by_id('Site', node['siteid'], site_nav_props,
                                       True)
            city = site['city']

            abbreviation = ''
            try:
                abbreviation = site['sitealiases'][0]['aliasname']
            except IndexError:
                pass  # no alias - ignore silently

            eq = {
                    'equipment name': node['name'],
                    'type':
                    'INTERNAL'
                    if site['name'] in INTERNAL_POP_NAMES
                    else 'CORE',
                    'pop': {
                        'name': site['name'],
                        'city': city['name'],
                        'country': city['country']['name'],
                        'country code': city['country']['abbreviation'],
                        'abbreviation': abbreviation,
                        'longitude': site['longitude'],
                        'latitude': site['latitude'],
                    }
                }
            yield eq


def lookup_geant_nodes(ds: IMS):

    return (n["name"]for n in ds.get_filtered_entities(
        'Node',
        'customer.Name == "GEANT"',
        ims.EQUIP_DEF_PROPERTIES['Nodes']))