Skip to content
Snippets Groups Projects
Commit 2cc91c9e authored by geant-release-service's avatar geant-release-service
Browse files

Finished release 0.101.

parents 14a5de33 cbdfaf67
No related branches found
No related tags found
No related merge requests found
...@@ -2,6 +2,9 @@ ...@@ -2,6 +2,9 @@
All notable changes to this project will be documented in this file. All notable changes to this project will be documented in this file.
## [0.101] - 2023-03-28
- DBOARD3-713: MIC endpoint - only include services that are monitored in Geant NMS
## [0.100] - 2022-12-07 ## [0.100] - 2022-12-07
- POL1-646: Changed BRIAN interface description parsing to expect whitespace - POL1-646: Changed BRIAN interface description parsing to expect whitespace
- POL1-643: Added port_type field to interfaces to distinguish access/service - POL1-643: Added port_type field to interfaces to distinguish access/service
......
# Inventory Provider # Inventory Provider
The Inventory Provider The Inventory Provider
acts as a single point of truth for acts as a single point of truth for
information about the GÉANT network, exposed by information about the GÉANT network, exposed by
by an HTTP API. by an HTTP API.
Documentation can be generated by running sphinx: Documentation can be generated by running sphinx in the commandline:
```bash ```bash
sphinx-build -M html docs/source docs/build sphinx-build -M html docs/source docs/build
``` ```
The documents should be viewable in the The documents should be viewable in the
workspace of the most recent [Jenkins job](https://jenkins.geant.org/job/inventory-provider%20(develop)/ws/docs/build/html/index.html). workspace of the most recent [Jenkins job](https://jenkins.geant.org/job/inventory-provider%20(develop)/ws/docs/build/html/index.html).
\ No newline at end of file
from enum import Enum
import functools
import logging import logging
import time
import requests import requests
import time
from enum import Enum from inventory_provider import environment
# Navigation Properties # Navigation Properties
...@@ -11,6 +13,9 @@ from enum import Enum ...@@ -11,6 +13,9 @@ from enum import Enum
from requests import HTTPError from requests import HTTPError
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
log_entry_and_exit = functools.partial(
environment.log_entry_and_exit, logger=logger)
# http://149.210.162.190:81/ImsVersions/21.9/html/50e6a1b1-3910-2091-63d5-e13777b2194e.htm # noqa # http://149.210.162.190:81/ImsVersions/21.9/html/50e6a1b1-3910-2091-63d5-e13777b2194e.htm # noqa
CIRCUIT_CUSTOMER_RELATION = { CIRCUIT_CUSTOMER_RELATION = {
...@@ -235,7 +240,7 @@ class IMS(object): ...@@ -235,7 +240,7 @@ class IMS(object):
response.raise_for_status() response.raise_for_status()
break break
def _get_entity( def _get_ims_data(
self, self,
url, url,
params=None, params=None,
...@@ -317,6 +322,7 @@ class IMS(object): ...@@ -317,6 +322,7 @@ class IMS(object):
IMS.cache[cache_key] = return_value IMS.cache[cache_key] = return_value
return return_value return return_value
@log_entry_and_exit
def get_entity_by_id( def get_entity_by_id(
self, self,
entity_type, entity_type,
...@@ -326,8 +332,9 @@ class IMS(object): ...@@ -326,8 +332,9 @@ class IMS(object):
url = f'{entity_type}/{entity_id}' url = f'{entity_type}/{entity_id}'
return \ return \
self._get_entity(url, None, navigation_properties, use_cache) self._get_ims_data(url, None, navigation_properties, use_cache)
@log_entry_and_exit
def get_entity_by_name( def get_entity_by_name(
self, self,
entity, entity,
...@@ -335,8 +342,9 @@ class IMS(object): ...@@ -335,8 +342,9 @@ class IMS(object):
navigation_properties=None, navigation_properties=None,
use_cache=False): use_cache=False):
url = f'{entity}/byname/"{name}"' url = f'{entity}/byname/"{name}"'
return self._get_entity(url, None, navigation_properties, use_cache) return self._get_ims_data(url, None, navigation_properties, use_cache)
@log_entry_and_exit
def get_filtered_entities( def get_filtered_entities(
self, self,
entity, entity,
...@@ -344,17 +352,31 @@ class IMS(object): ...@@ -344,17 +352,31 @@ class IMS(object):
navigation_properties=None, navigation_properties=None,
use_cache=False, use_cache=False,
step_count=50): step_count=50):
url = f'{entity}/filtered/{filter_string}'
yield from self.get_entities(
url,
navigation_properties,
use_cache,
step_count
)
def get_entities(
self,
url,
navigation_properties=None,
use_cache=False,
step_count=50):
more_to_come = True more_to_come = True
start_entity = 0 start_entity = 0
gateway_error_count = 0
while more_to_come: while more_to_come:
params = { params = {
'paginatorStartElement': start_entity, 'paginatorStartElement': start_entity,
'paginatorNumberOfElements': step_count 'paginatorNumberOfElements': step_count
} }
url = f'{entity}/filtered/{filter_string}'
try: try:
more_to_come = False more_to_come = False
entities = self._get_entity( entities = self._get_ims_data(
url, url,
params, params,
navigation_properties, navigation_properties,
...@@ -364,6 +386,17 @@ class IMS(object): ...@@ -364,6 +386,17 @@ class IMS(object):
if r.status_code == requests.codes.not_found \ if r.status_code == requests.codes.not_found \
and NO_FILTERED_RESULTS_MESSAGE in r.text.lower(): and NO_FILTERED_RESULTS_MESSAGE in r.text.lower():
entities = None entities = None
elif r.status_code == 504:
gateway_error_count += 1
logger.debug(
f"GATEWAY TIME-OUT for {url}"
f" -- COUNT: {gateway_error_count}")
if gateway_error_count > 4:
raise e
time.sleep(5)
logger.debug("WAKING UP")
more_to_come = True
continue
else: else:
raise e raise e
if entities: if entities:
...@@ -372,6 +405,7 @@ class IMS(object): ...@@ -372,6 +405,7 @@ class IMS(object):
start_entity += step_count start_entity += step_count
yield from entities yield from entities
@log_entry_and_exit
def get_all_entities( def get_all_entities(
self, self,
entity, entity,
...@@ -379,9 +413,9 @@ class IMS(object): ...@@ -379,9 +413,9 @@ class IMS(object):
use_cache=False, use_cache=False,
step_count=50 step_count=50
): ):
yield from self.get_filtered_entities( url = f'{entity}/all'
entity, yield from self.get_entities(
'Id <> 0', url,
navigation_properties, navigation_properties,
use_cache, use_cache,
step_count step_count
......
import functools
import logging import logging
import re import re
from collections import defaultdict from collections import defaultdict
...@@ -9,10 +10,9 @@ from inventory_provider.db import ims ...@@ -9,10 +10,9 @@ from inventory_provider.db import ims
from inventory_provider.db.ims import InventoryStatus, IMS, \ from inventory_provider.db.ims import InventoryStatus, IMS, \
CUSTOMER_RELATED_CONTACT_PROPERTIES, EXTRA_FIELD_VALUE_PROPERTIES CUSTOMER_RELATED_CONTACT_PROPERTIES, EXTRA_FIELD_VALUE_PROPERTIES
environment.setup_logging()
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
log_entry_and_exit = functools.partial(
# Dashboard V3 environment.log_entry_and_exit, logger=logger)
IMS_OPSDB_STATUS_MAP = { IMS_OPSDB_STATUS_MAP = {
InventoryStatus.PLANNED: 'planned', InventoryStatus.PLANNED: 'planned',
...@@ -83,6 +83,7 @@ def get_flexils_by_circuitid(ds: IMS): ...@@ -83,6 +83,7 @@ def get_flexils_by_circuitid(ds: IMS):
return dict(by_circuit) return dict(by_circuit)
@log_entry_and_exit
def get_non_monitored_circuit_ids(ds: IMS): def get_non_monitored_circuit_ids(ds: IMS):
# note the id for the relevant field is hard-coded. I didn't want to use # note the id for the relevant field is hard-coded. I didn't want to use
# the name of the field as this can be changed by users # the name of the field as this can be changed by users
...@@ -94,6 +95,7 @@ def get_non_monitored_circuit_ids(ds: IMS): ...@@ -94,6 +95,7 @@ def get_non_monitored_circuit_ids(ds: IMS):
yield d['extrafieldvalueobjectinfo']['objectid'] yield d['extrafieldvalueobjectinfo']['objectid']
@log_entry_and_exit
def get_monitored_circuit_ids(ds: IMS): def get_monitored_circuit_ids(ds: IMS):
# note the id for the relevant field is hard-coded. I didn't want to use # note the id for the relevant field is hard-coded. I didn't want to use
# the name of the field as this can be changed by users # the name of the field as this can be changed by users
...@@ -106,6 +108,7 @@ def get_monitored_circuit_ids(ds: IMS): ...@@ -106,6 +108,7 @@ def get_monitored_circuit_ids(ds: IMS):
yield d['extrafieldvalueobjectinfo']['objectid'] yield d['extrafieldvalueobjectinfo']['objectid']
@log_entry_and_exit
def get_ids_and_sids(ds: IMS): def get_ids_and_sids(ds: IMS):
for sid_circuit in ds.get_filtered_entities( for sid_circuit in ds.get_filtered_entities(
'ExtraFieldValue', 'ExtraFieldValue',
...@@ -115,6 +118,7 @@ def get_ids_and_sids(ds: IMS): ...@@ -115,6 +118,7 @@ def get_ids_and_sids(ds: IMS):
yield sid_circuit['objectid'], sid_circuit['value'] yield sid_circuit['objectid'], sid_circuit['value']
@log_entry_and_exit
def get_service_types(ds: IMS): def get_service_types(ds: IMS):
for d in ds.get_filtered_entities( for d in ds.get_filtered_entities(
'ComboBoxData', 'ComboBoxData',
...@@ -122,6 +126,7 @@ def get_service_types(ds: IMS): ...@@ -122,6 +126,7 @@ def get_service_types(ds: IMS):
yield d['selection'] yield d['selection']
@log_entry_and_exit
def get_customer_tts_contacts(ds: IMS): def get_customer_tts_contacts(ds: IMS):
customer_contacts = defaultdict(set) customer_contacts = defaultdict(set)
...@@ -137,6 +142,7 @@ def get_customer_tts_contacts(ds: IMS): ...@@ -137,6 +142,7 @@ def get_customer_tts_contacts(ds: IMS):
yield k, sorted(list(v)) yield k, sorted(list(v))
@log_entry_and_exit
def get_customer_planned_work_contacts(ds: IMS): def get_customer_planned_work_contacts(ds: IMS):
customer_contacts = defaultdict(set) customer_contacts = defaultdict(set)
...@@ -152,6 +158,7 @@ def get_customer_planned_work_contacts(ds: IMS): ...@@ -152,6 +158,7 @@ def get_customer_planned_work_contacts(ds: IMS):
yield k, sorted(list(v)) yield k, sorted(list(v))
@log_entry_and_exit
def get_circuit_related_customers(ds: IMS): def get_circuit_related_customers(ds: IMS):
return_value = defaultdict(list) return_value = defaultdict(list)
...@@ -169,6 +176,7 @@ def get_circuit_related_customers(ds: IMS): ...@@ -169,6 +176,7 @@ def get_circuit_related_customers(ds: IMS):
return return_value return return_value
@log_entry_and_exit
def get_port_id_services(ds: IMS): def get_port_id_services(ds: IMS):
circuit_nav_props = [ circuit_nav_props = [
ims.CIRCUIT_PROPERTIES['Ports'], ims.CIRCUIT_PROPERTIES['Ports'],
...@@ -300,6 +308,7 @@ def get_port_id_services(ds: IMS): ...@@ -300,6 +308,7 @@ def get_port_id_services(ds: IMS):
} }
@log_entry_and_exit
def get_port_sids(ds: IMS): def get_port_sids(ds: IMS):
""" """
This function fetches SIDs for external ports that have them defined, This function fetches SIDs for external ports that have them defined,
...@@ -313,6 +322,7 @@ def get_port_sids(ds: IMS): ...@@ -313,6 +322,7 @@ def get_port_sids(ds: IMS):
step_count=10000)} step_count=10000)}
@log_entry_and_exit
def get_internal_port_sids(ds: IMS): def get_internal_port_sids(ds: IMS):
""" """
This function fetches SIDs for external ports that have them defined, This function fetches SIDs for external ports that have them defined,
...@@ -326,6 +336,7 @@ def get_internal_port_sids(ds: IMS): ...@@ -326,6 +336,7 @@ def get_internal_port_sids(ds: IMS):
step_count=10000)} step_count=10000)}
@log_entry_and_exit
def get_port_details(ds: IMS): def get_port_details(ds: IMS):
port_nav_props = [ port_nav_props = [
ims.PORT_PROPERTIES['Node'], ims.PORT_PROPERTIES['Node'],
...@@ -381,6 +392,7 @@ def get_port_details(ds: IMS): ...@@ -381,6 +392,7 @@ def get_port_details(ds: IMS):
'internalport', internal_port_nav_props, step_count=2000), 'internal') 'internalport', internal_port_nav_props, step_count=2000), 'internal')
@log_entry_and_exit
def get_circuit_hierarchy(ds: IMS): def get_circuit_hierarchy(ds: IMS):
circuit_nav_props = [ circuit_nav_props = [
ims.CIRCUIT_PROPERTIES['Customer'], ims.CIRCUIT_PROPERTIES['Customer'],
...@@ -425,6 +437,7 @@ def get_circuit_hierarchy(ds: IMS): ...@@ -425,6 +437,7 @@ def get_circuit_hierarchy(ds: IMS):
} }
@log_entry_and_exit
def get_node_locations(ds: IMS): def get_node_locations(ds: IMS):
""" """
return location info for all Site nodes return location info for all Site nodes
...@@ -498,6 +511,7 @@ INTERNAL_POP_NAMES = { ...@@ -498,6 +511,7 @@ INTERNAL_POP_NAMES = {
} }
@log_entry_and_exit
def lookup_lg_routers(ds: IMS): def lookup_lg_routers(ds: IMS):
pattern = re.compile("vpn-proxy|vrr|taas", re.IGNORECASE) pattern = re.compile("vpn-proxy|vrr|taas", re.IGNORECASE)
...@@ -562,6 +576,7 @@ def lookup_lg_routers(ds: IMS): ...@@ -562,6 +576,7 @@ def lookup_lg_routers(ds: IMS):
yield eq yield eq
@log_entry_and_exit
def lookup_geant_nodes(ds: IMS): def lookup_geant_nodes(ds: IMS):
return (n["name"]for n in ds.get_filtered_entities( return (n["name"]for n in ds.get_filtered_entities(
......
import functools
import json import json
import logging.config import logging.config
import os import os
import time
def log_entry_and_exit(f, logger):
# cf. https://stackoverflow.com/a/47663642
# cf. https://stackoverflow.com/a/59098957/6708581
@functools.wraps(f)
def _w(*args, **kwargs):
logger.debug(f'>>> {f.__name__}{args}')
start_time = time.time()
try:
return f(*args, **kwargs)
finally:
end_time = time.time()
duration = end_time - start_time
logger.debug(f'<<< {f.__name__}{args} -- duration {duration}')
return _w
def setup_logging(): def setup_logging():
......
...@@ -222,7 +222,9 @@ def get_everything(): ...@@ -222,7 +222,9 @@ def get_everything():
site = f'{d["pop_name"]} ({d["pop_abbreviation"]})' site = f'{d["pop_name"]} ({d["pop_abbreviation"]})'
eq_name = d['equipment'] eq_name = d['equipment']
if_name = d['port'] if_name = d['port']
all_data[site][eq_name][if_name] = d['related-services'] all_data[site][eq_name][if_name] = \
[rs for rs in d['related-services']
if rs['status'] == 'operational']
result = json.dumps(all_data) result = json.dumps(all_data)
r.set(cache_key, result.encode('utf-8')) r.set(cache_key, result.encode('utf-8'))
......
...@@ -4,7 +4,6 @@ import json ...@@ -4,7 +4,6 @@ import json
import logging import logging
import os import os
import re import re
import time
from typing import List from typing import List
from celery import Task, states, chord from celery import Task, states, chord
...@@ -36,24 +35,9 @@ FINALIZER_TIMEOUT_S = 300 ...@@ -36,24 +35,9 @@ FINALIZER_TIMEOUT_S = 300
# TODO: error callback (cf. http://docs.celeryproject.org/en/latest/userguide/calling.html#linking-callbacks-errbacks) # noqa: E501 # TODO: error callback (cf. http://docs.celeryproject.org/en/latest/userguide/calling.html#linking-callbacks-errbacks) # noqa: E501
environment.setup_logging() environment.setup_logging()
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
log_task_entry_and_exit = functools.partial(
environment.log_entry_and_exit, logger=logger)
def log_task_entry_and_exit(f):
# cf. https://stackoverflow.com/a/47663642
# cf. https://stackoverflow.com/a/59098957/6708581
@functools.wraps(f)
def _w(*args, **kwargs):
logger.debug(f'>>> {f.__name__}{args}')
start_time = time.time()
try:
return f(*args, **kwargs)
finally:
end_time = time.time()
duration = end_time - start_time
logger.debug(f'<<< {f.__name__}{args} -- duration {duration}')
return _w
class InventoryTaskError(Exception): class InventoryTaskError(Exception):
...@@ -863,7 +847,8 @@ def _extract_ims_data(ims_api_url, ims_username, ims_password): ...@@ -863,7 +847,8 @@ def _extract_ims_data(ims_api_url, ims_username, ims_password):
executor.submit(_populate_hierarchy): 'hierarchy', executor.submit(_populate_hierarchy): 'hierarchy',
executor.submit(_populate_port_id_details): 'port_id_details', executor.submit(_populate_port_id_details): 'port_id_details',
executor.submit(_populate_circuit_info): 'circuit_info', executor.submit(_populate_circuit_info): 'circuit_info',
executor.submit(_populate_flexils_data): 'flexils_data' executor.submit(_populate_flexils_data): 'flexils_data',
executor.submit(_populate_customers): 'customers'
} }
for future in concurrent.futures.as_completed(futures): for future in concurrent.futures.as_completed(futures):
...@@ -1105,8 +1090,13 @@ def transform_ims_data(data): ...@@ -1105,8 +1090,13 @@ def transform_ims_data(data):
for tlc in circ['related-services']: for tlc in circ['related-services']:
# why were these removed? # why were these removed?
# contacts.update(tlc.pop('contacts')) # contacts.update(tlc.pop('contacts'))
contacts.update(tlc.get('contacts')) if circ['status'] == 'operational' \
pw_contacts.update(tlc.get('planned_work_contacts', [])) and circ['id'] in circuit_ids_to_monitor \
and tlc['status'] == 'operational' \
and tlc['id'] in circuit_ids_to_monitor:
contacts.update(tlc.get('contacts'))
pw_contacts.update(
tlc.get('planned_work_contacts', []))
circ['contacts'] = sorted(list(contacts)) circ['contacts'] = sorted(list(contacts))
circ['planned_work_contacts'] = sorted(list(pw_contacts)) circ['planned_work_contacts'] = sorted(list(pw_contacts))
......
...@@ -2,7 +2,7 @@ from setuptools import setup, find_packages ...@@ -2,7 +2,7 @@ from setuptools import setup, find_packages
setup( setup(
name='inventory-provider', name='inventory-provider',
version="0.100", version="0.101",
author='GEANT', author='GEANT',
author_email='swd@geant.org', author_email='swd@geant.org',
description='Dashboard inventory provider', description='Dashboard inventory provider',
......
...@@ -139,7 +139,7 @@ def test_ims_class_get_all_entities(mocker): ...@@ -139,7 +139,7 @@ def test_ims_class_get_all_entities(mocker):
list(ds.get_all_entities('Node', step_count=10)) list(ds.get_all_entities('Node', step_count=10))
mock_get.assert_called_once_with( mock_get.assert_called_once_with(
'dummy_base/ims/Node/filtered/Id <> 0', 'dummy_base/ims/Node/all',
headers={'Authorization': 'Bearer dummy_bt'}, headers={'Authorization': 'Bearer dummy_bt'},
params={ params={
'paginatorStartElement': 0, 'paginatorStartElement': 0,
......
...@@ -5,7 +5,7 @@ envlist = py36 ...@@ -5,7 +5,7 @@ envlist = py36
exclude = venv,.tox,build exclude = venv,.tox,build
[testenv] [testenv]
passenv = TEST_OPSDB_HOSTNAME TEST_OPSDB_DBNAME TEST_OPSDB_USERNAME TEST_OPSDB_PASSWORD passenv = TEST_OPSDB_HOSTNAME,TEST_OPSDB_DBNAME,TEST_OPSDB_USERNAME,TEST_OPSDB_PASSWORD
deps = deps =
coverage coverage
flake8 flake8
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment