diff --git a/gso/oss-params-example.json b/gso/oss-params-example.json index 9c48d63db1dd48d0fbc489b8260aae536ee3b641..1c062376d840c95155a861f4fe697a4f7673d763 100644 --- a/gso/oss-params-example.json +++ b/gso/oss-params-example.json @@ -1,6 +1,7 @@ { "GENERAL": { - "public_hostname": "https://gap.geant.org" + "public_hostname": "https://gap.geant.org", + "environment": "lab" }, "RESOURCE_MANAGER_API_PREFIX": "http://localhost:44444", "IPAM": { @@ -42,6 +43,30 @@ "dns_view": "default" } }, + "MONITORING": { + "LIBRENMS": { + "endpoint": "https://librenms.lab.office.geant.net/", + "token": "<token>", + "DEVICE_GROUPS": { + "routers_lab": "lab_routers", + "routers_prod": "prod_routers" + } + }, + "SNMP": { + "version": "v2c", + "V2": { + "community": "librenms-community" + }, + "V3": { + "authlevel": "AuthPriv", + "authname": "librenms", + "authpass": "<password1>", + "authalgo": "sha", + "cryptopass": "<password2>", + "cryptoalgo": "aes" + } + } + }, "PROVISIONING_PROXY": { "scheme": "https", "api_base": "localhost:44444", diff --git a/gso/services/librenms.py b/gso/services/librenms.py new file mode 100644 index 0000000000000000000000000000000000000000..1ab11514c3972389d4f01c34165aee79503e25f8 --- /dev/null +++ b/gso/services/librenms.py @@ -0,0 +1,176 @@ +""" +The LibreNMS module interacts with the LibreNMS instance when +- Creating a device. +- Validating the input of a device. +- Terminating a device. +""" +import json +import logging +import requests +from gso import settings + + +logger = logging.getLogger(__name__) + + +class CfgStruct(object): + pass + + +def _get_cfg(): + """ + Internal function to retrieve all needed configuration. + """ + oss = settings.load_oss_params() + cfg = CfgStruct() + # Hack for later ease: 1st setattr will fill in the inner's dict + setattr(cfg, "_hack", "") + # Update inner dict + cfg.__dict__.update(oss.MONITORING) + assert cfg.__dict__ is not None + + # Add parameters on-the-fly + cfg.headers = {"X-Auth-Token": cfg.LIBRENMS.token} + + sep = "/" + if cfg.LIBRENMS.endpoint.endswith("/"): + sep = "" + cfg.base_url = f"{cfg.LIBRENMS.endpoint}{sep}api/v0" + cfg.url_devices = f"{cfg.base_url}/devices" + cfg.url_switches = f"{cfg.base_url}/devicegroups/switches" + cfg.device_groups = cfg.LIBRENMS.DEVICE_GROUPS + cfg.environment = oss.GENERAL.environment + if cfg.environment.startswith("lab"): + cfg_dg_rtr_lab = cfg.device_groups.routers_lab + cfg.url_routers = f"{cfg.base_url}/devicegroups/{cfg_dg_rtr_lab}" + elif cfg.environment.startswith("prod"): + cfg_dg_rtr_prod = cfg.device_groups.routers_prod + cfg.url_routers = f"{cfg.base_url}/devicegroups/{cfg_dg_rtr_prod}" + + return cfg + + +def validate_device(fqdn: str): + """ + Function that validates the existence of a device in LibreNMS. + + :param FQDN of the device to validate. + """ + CFG = _get_cfg() + + # Validate existence + nms_result = requests.get( + CFG.url_devices, headers=CFG.headers) + assert nms_result is not None + + device_id = list(map( + lambda x: x.get("device_id"), + filter(lambda x: x.get("hostname") == fqdn, + nms_result.json().get("devices")))) + + if len(device_id) != 1 or device_id[0] is None: + error_msg = f"Device with FQDN={fqdn} is not registered in LibreNMS" + print(error_msg) + raise AssertionError(error_msg) + + # Validate correctness + device_id = device_id[0] + url_device = f"{CFG.url_devices}/{device_id}" + logger.debug(f"Connecting to URL: {url_device}" + f"with headers: {CFG.headers}") + nms_result = requests.get( + url_device, headers=CFG.headers) + logger.debug(f"LibreNMS response={nms_result.content}") + + if nms_result.status_code != 200: + print(nms_result.content) + raise AssertionError(nms_result.content) + + # nms_dev_sysname = nms_result.json().get("sysName") + nms_dev_hostname = nms_result.json().get("devices")[0].get("hostname") + if fqdn != nms_dev_hostname: + error_msg = f"Device with FQDN={fqdn} may not be correctly "\ + f"registered in LibreNMS (expected FQDN: {nms_dev_hostname})" + print(error_msg) + raise AssertionError(error_msg) + + +def register_device(fqdn: str): + """ + Function that registers a new device in LibreNMS. + + :param FQDN of the device to register. + """ + CFG = _get_cfg() + logger.debug(f"Registering FQDN={fqdn} in LibreNMS") + + device_data = { + "display": fqdn, + "hostname": fqdn, + "sysName": fqdn, + # "override_icmp_disable": "true", + # IMPORTANT: uncomment if testing with FQDNs that are not reachable + # from LibreNMS (e.g. ContainerLab routers) + # "force_add": "true" + } + if CFG.SNMP.version == "v2c": + device_data.update({ + "community": CFG.SNMP.V2.community + }) + + elif CFG.SNMP.version == "v3": + for key in [ + "authlevel", "authname", "authpass", "authalgo", + "cryptopass", "cryptoalgo"]: + device_data.update({key: getattr(CFG.SNMP.V3, key)}) + + logger.debug(f"Connecting to URL: {CFG.url_devices}" + f"with headers: {CFG.headers} and" + f"payload: {device_data}") + nms_result = requests.post( + CFG.url_devices, headers=CFG.headers, + data=json.dumps(device_data)) + logger.debug(f"LibreNMS response={nms_result.content}") + + if nms_result.status_code != 200: + print(nms_result.content) + raise AssertionError(nms_result.content) + + +def deregister_device(fqdn: str): + """ + Function that reregisters a device from LibreNMS. + + :param FQDN of the device to deregister. + """ + CFG = _get_cfg() + logger.debug(f"Deregistering FQDN={fqdn} from LibreNMS") + + nms_result = requests.get( + CFG.url_devices, headers=CFG.headers) + assert nms_result is not None + device_id = list(map( + lambda x: x.get("device_id"), + filter(lambda x: x.get("hostname") == fqdn, + nms_result.json().get("devices")))) + if len(device_id) != 1: + return + device_id = device_id[0] + + # https://docs.librenms.org/API/Devices/#endpoint-categories + device_data = { + "field": "disabled", + "data": "1" + } + url_device = f"{CFG.url_devices}/{device_id}" + logger.debug(f"Connecting to URL: {url_device}" + f"with headers: {CFG.headers} and" + f"payload: {device_data}") + nms_result = requests.patch( + url_device, headers=CFG.headers, + data=json.dumps(device_data)) + logger.debug(f"LibreNMS response={nms_result.content}") + + # Fail silently if device was not registered + if nms_result.status_code != 200: + print(nms_result.content) diff --git a/gso/settings.py b/gso/settings.py index a0be09616096e5916526878f0875a27108eb8fc3..dfea62fdb3bbfabcf1a7ffee5c1303675c4f6f22 100644 --- a/gso/settings.py +++ b/gso/settings.py @@ -4,7 +4,7 @@ GSO settings, ensuring that the required parameters are set correctly. import ipaddress import json import os -from pydantic import BaseSettings, Field +from pydantic import BaseSettings class GeneralParams(BaseSettings): @@ -14,6 +14,7 @@ class GeneralParams(BaseSettings): #: The hostname that GSO is publicly served at, used for building the #: callback URL that the provisioning proxy uses. public_hostname: str + environment: str class InfoBloxParams(BaseSettings): @@ -68,6 +69,59 @@ class IPAMParams(BaseSettings): LT_IAS: ServiceNetworkParams +class MonitoringLibreNMSDevGroupsParams(BaseSettings): + """ + Parameters related to LibreNMS' devicegroups. + """ + routers_lab: str + routers_prod: str + + +class MonitoringSNMPV2Params(BaseSettings): + """ + Parameters related to SNMPv2. + """ + community: str + + +class MonitoringSNMPV3Params(BaseSettings): + """ + Parameters related to SNMPv3. + """ + authlevel: str + authname: str + authpass: str + authalgo: str + cryptopass: str + cryptoalgo: str + + +class MonitoringSNMPParams(BaseSettings): + """ + Parameters related to SNMP. + """ + version: str + V2: MonitoringSNMPV2Params + V3: MonitoringSNMPV3Params + + +class MonitoringLibreNMSParams(BaseSettings): + """ + Parameters related to LibreNMS. + """ + endpoint: str + token: str + DEVICE_GROUPS: MonitoringLibreNMSDevGroupsParams + + +class MonitoringParams(BaseSettings): + """ + Parameters related to the monitoring. + """ + LIBRENMS: MonitoringLibreNMSParams + SNMP: MonitoringSNMPParams + + class ProvisioningProxyParams(BaseSettings): """ Parameters for the provisioning proxy. @@ -85,6 +139,7 @@ class OSSParams(BaseSettings): GENERAL: GeneralParams IPAM: IPAMParams RESOURCE_MANAGER_API_PREFIX: str + MONITORING: MonitoringParams PROVISIONING_PROXY: ProvisioningProxyParams diff --git a/gso/workflows/__init__.py b/gso/workflows/__init__.py index e1dfe7e35c76296279ea9e85e065a5d0cfeb6ec0..ee83743d98e070260f6704b37a4adc7c0295ea74 100644 --- a/gso/workflows/__init__.py +++ b/gso/workflows/__init__.py @@ -4,6 +4,7 @@ init class that imports all workflows into GSO. from orchestrator.workflows import LazyWorkflowInstance LazyWorkflowInstance("gso.workflows.device.create_device", "create_device") +LazyWorkflowInstance("gso.workflows.device.validate_device", "validate_device") LazyWorkflowInstance("gso.workflows.device.terminate_device", "terminate_device") LazyWorkflowInstance("gso.workflows.device.get_facts", "get_facts") @@ -15,3 +16,4 @@ LazyWorkflowInstance("gso.workflows.iptrunk.modify_iptrunk_interface", LazyWorkflowInstance("gso.workflows.iptrunk.modify_iptrunk_isis_metric", "modify_iptrunk_isis_metric") LazyWorkflowInstance("gso.workflows.site.create_site", "create_site") +LazyWorkflowInstance("gso.workflows.site.terminate_site", "terminate_site") diff --git a/gso/workflows/device/create_device.py b/gso/workflows/device/create_device.py index af4549679ce6d1b8a787230fdf8421ad7481f5f2..ab7944cef9affa497f2b85fc3135e0628e5d1595 100644 --- a/gso/workflows/device/create_device.py +++ b/gso/workflows/device/create_device.py @@ -16,10 +16,11 @@ from orchestrator.workflows.utils import wrap_create_initial_input_form from gso.products.product_blocks import device as device_pb from gso.products.product_types import device -from gso.products.product_types.device import DeviceInactive, \ +from gso.products.product_types.device import Device, DeviceInactive, \ DeviceProvisioning from gso.products.product_types.site import Site from gso.services import _ipam +from gso.services import librenms from gso.services import provisioning_proxy from gso.services.provisioning_proxy import await_pp_results, \ confirm_pp_results @@ -66,6 +67,10 @@ def initial_input_form_generator(product_name: str) -> FormGenerator: return user_input.dict() +def _fqdn_from_subscription(subscription: Device) -> str: + return subscription.device.device_fqdn + + @step('Create subscription') def create_subscription(product: UUIDstr) -> State: subscription = DeviceInactive.from_product_id(product, uuid4()) @@ -94,13 +99,17 @@ def get_info_from_ipam(subscription: DeviceProvisioning) -> State: subscription.device.device_lo_ipv4_address = lo0_addr.v4 subscription.device.device_lo_ipv6_address = lo0_addr.v6 subscription.device.device_lo_iso_address \ - = iso_from_ipv4(str(subscription.device.device_lo_ipv4_address)) + = iso_from_ipv4( + str(subscription.device.device_lo_ipv4_address)) subscription.device.device_si_ipv4_network \ - = _ipam.allocate_service_ipv4_network(service_type='SI', comment=f"SI for {lo0_name}").v4 + = _ipam.allocate_service_ipv4_network( + service_type='SI', comment=f"SI for {lo0_name}").v4 subscription.device.device_ias_lt_ipv4_network \ - = _ipam.allocate_service_ipv4_network(service_type='LT_IAS', comment=f"LT for {lo0_name}").v4 + = _ipam.allocate_service_ipv4_network( + service_type='LT_IAS', comment=f"LT for {lo0_name}").v4 subscription.device.device_ias_lt_ipv6_network \ - = _ipam.allocate_service_ipv6_network(service_type='LT_IAS', comment=f"LT for {lo0_name}").v6 + = _ipam.allocate_service_ipv6_network( + service_type='LT_IAS', comment=f"LT for {lo0_name}").v6 return {'subscription': subscription} @@ -162,6 +171,17 @@ def provision_device_real(subscription: DeviceProvisioning, } +@step('Register device in LibreNMS') +def register_librenms(subscription: Device) -> State: + fqdn = _fqdn_from_subscription(subscription) + _ = librenms.register_device(fqdn) + return { + 'subscription': subscription, + # TBC: wait for results to be returned from LibreNMS or not + # 'result_text': result, + } + + @workflow( 'Create device', initial_input_form=wrap_create_initial_input_form( @@ -181,6 +201,7 @@ def create_device(): >> provision_device_real >> await_pp_results >> confirm_pp_results + >> register_librenms >> set_status(SubscriptionLifecycle.ACTIVE) >> resync >> done diff --git a/gso/workflows/device/terminate_device.py b/gso/workflows/device/terminate_device.py index 30da64eb0dab8abc251b48d056c325b51abd9d6a..298e5cc75d92547a0b2583036f229742215abdfd 100644 --- a/gso/workflows/device/terminate_device.py +++ b/gso/workflows/device/terminate_device.py @@ -12,6 +12,7 @@ from orchestrator.workflows.steps import ( from orchestrator.workflows.utils import wrap_modify_initial_input_form from gso.products.product_types.device import Device +from gso.services import librenms def initial_input_form_generator(subscription_id: UUIDstr) -> InputForm: @@ -19,12 +20,16 @@ def initial_input_form_generator(subscription_id: UUIDstr) -> InputForm: class TerminateForm(FormPage): are_you_sure: Label = ( - f'Are you sure you want to remove {subscription.description}?' + f"Are you sure you want to remove {subscription.description}?" ) return TerminateForm +def _fqdn_from_subscription(subscription: Device) -> str: + return subscription.device.device_fqdn + + def _deprovision_in_user_management_system(fqdn: str) -> str: pass @@ -35,6 +40,17 @@ def deprovision_user(subscription: Device) -> None: pass +@step("Deregister device from LibreNMS") +def deregister_librenms(subscription: Device) -> None: + fqdn = _fqdn_from_subscription(subscription) + _ = librenms.deregister_device(fqdn) + return { + "subscription": subscription, + # TBC: wait for results to be returned from LibreNMS or not + # "result_text": result, + } + + @workflow( "Terminate device", initial_input_form=wrap_modify_initial_input_form( @@ -47,6 +63,7 @@ def terminate_device(): >> store_process_subscription(Target.TERMINATE) >> unsync >> deprovision_user + >> deregister_librenms >> set_status(SubscriptionLifecycle.TERMINATED) >> resync >> done diff --git a/gso/workflows/device/validate_device.py b/gso/workflows/device/validate_device.py new file mode 100644 index 0000000000000000000000000000000000000000..ee1a7490d1262744b35adb697adb4278f8f6e384 --- /dev/null +++ b/gso/workflows/device/validate_device.py @@ -0,0 +1,48 @@ +# noinspection PyProtectedMember +from orchestrator.forms import FormPage +from orchestrator.targets import Target +from orchestrator.types import InputForm, UUIDstr +from orchestrator.forms.validators import Label +from orchestrator.workflow import done, init, step, workflow +from orchestrator.workflows.steps import resync # , set_status +from orchestrator.workflows.utils import wrap_create_initial_input_form + +from gso.products.product_types.device import Device +# noinspection PyProtectedMember +from gso.services import librenms + + +def initial_input_form_generator(subscription_id: UUIDstr) -> InputForm: + subscription = Device.from_subscription(subscription_id) + + class ValidateForm(FormPage): + are_you_sure: Label = ( + f"Are you sure you want to validate {subscription.description}?" + ) + + return ValidateForm + + +@step("Validate existence of device in LibreNMS") +def device_registered_in_librenms(subscription: Device) -> None: + _ = librenms.validate_device(subscription) + return { + "subscription": subscription, + # TBC: wait for results to be returned from LibreNMS or not + # "result_text": result, + } + + +@workflow( + "Validate device", + initial_input_form=wrap_create_initial_input_form( + initial_input_form_generator), + target=Target.SYSTEM, +) +def validate_device(): + return ( + init + >> device_registered_in_librenms + >> resync + >> done + ) diff --git a/gso/workflows/site/terminate_site.py b/gso/workflows/site/terminate_site.py new file mode 100644 index 0000000000000000000000000000000000000000..d08d5f44463b2612ae44ad8901505ec0a1fcb1be --- /dev/null +++ b/gso/workflows/site/terminate_site.py @@ -0,0 +1,53 @@ +from orchestrator.forms import FormPage +from orchestrator.forms.validators import Label +from orchestrator.targets import Target +from orchestrator.types import InputForm, SubscriptionLifecycle, UUIDstr +from orchestrator.workflow import done, init, step, workflow +from orchestrator.workflows.steps import ( + resync, + set_status, + store_process_subscription, + unsync, +) +from orchestrator.workflows.utils import wrap_modify_initial_input_form + +from gso.products.product_types.site import Site + + +def initial_input_form_generator(subscription_id: UUIDstr) -> InputForm: + subscription = Site.from_subscription(subscription_id) + + class TerminateForm(FormPage): + are_you_sure: Label = ( + f'Are you sure you want to remove {subscription.description}?' + ) + + return TerminateForm + + +def _deprovision_in_user_management_system(fqdn: str) -> str: + pass + + +@step("Deprovision Site") +def deprovision_site(subscription: Site) -> None: + # _deprovision_in_user_management_system(subscription.user.user_id) + pass + + +@workflow( + "Terminate Site", + initial_input_form=wrap_modify_initial_input_form( + initial_input_form_generator), + target=Target.TERMINATE, +) +def terminate_site(): + return ( + init + >> store_process_subscription(Target.TERMINATE) + >> unsync + >> deprovision_site + >> set_status(SubscriptionLifecycle.TERMINATED) + >> resync + >> done + )