NAT-181: interactions with LibreNMS to register, validate and disable a device

gso/oss-params-example.json

 {
   "GENERAL": {
-    "public_hostname": "https://gap.geant.org"
+    "public_hostname": "https://gap.geant.org",
+    "environment": "lab"
   },
   "RESOURCE_MANAGER_API_PREFIX": "http://localhost:44444",
   "IPAM": {
@@ -42,6 +43,30 @@
       "dns_view": "default"
     }
   },
+  "MONITORING": {
+    "LIBRENMS": {
+      "endpoint": "https://librenms.lab.office.geant.net/",
+      "token": "<token>",
+      "DEVICE_GROUPS": {
+        "routers_lab": "lab_routers",
+        "routers_prod": "prod_routers"
+      }
+    },
+    "SNMP": {
+      "version": "v2c",
+      "V2": {
+        "community": "librenms-community"
+      },
+      "V3": {
+        "authlevel": "AuthPriv",
+        "authname": "librenms",
+        "authpass": "<password1>",
+        "authalgo": "sha",
+        "cryptopass": "<password2>",
+        "cryptoalgo": "aes"
+      }
+    }
+  },
   "PROVISIONING_PROXY": {
     "scheme": "https",
     "api_base": "localhost:44444",

gso/services/librenms.py

+"""
+The LibreNMS module interacts with the LibreNMS instance when
+- Creating a device.
+- Validating the input of a device.
+- Terminating a device.
+"""
+import json
+import logging
+import requests
+from gso import settings
+
+
+logger = logging.getLogger(__name__)
+
+
+class CfgStruct(object):
+    pass
+
+
+def _get_cfg():
+    """
+    Internal function to retrieve all needed configuration.
+    """
+    oss = settings.load_oss_params()
+    cfg = CfgStruct()
+    # Hack for later ease: 1st setattr will fill in the inner's dict
+    setattr(cfg, "_hack", "")
+    # Update inner dict
+    cfg.__dict__.update(oss.MONITORING)
+    assert cfg.__dict__ is not None
+
+    # Add parameters on-the-fly
+    cfg.headers = {"X-Auth-Token": cfg.LIBRENMS.token}
+
+    sep = "/"
+    if cfg.LIBRENMS.endpoint.endswith("/"):
+        sep = ""
+    cfg.base_url = f"{cfg.LIBRENMS.endpoint}{sep}api/v0"
+    cfg.url_devices = f"{cfg.base_url}/devices"
+    cfg.url_switches = f"{cfg.base_url}/devicegroups/switches"
+    cfg.device_groups = cfg.LIBRENMS.DEVICE_GROUPS
+    cfg.environment = oss.GENERAL.environment
+    if cfg.environment.startswith("lab"):
+        cfg_dg_rtr_lab = cfg.device_groups.routers_lab
+        cfg.url_routers = f"{cfg.base_url}/devicegroups/{cfg_dg_rtr_lab}"
+    elif cfg.environment.startswith("prod"):
+        cfg_dg_rtr_prod = cfg.device_groups.routers_prod
+        cfg.url_routers = f"{cfg.base_url}/devicegroups/{cfg_dg_rtr_prod}"
+
+    return cfg
+
+
+def validate_device(fqdn: str):
+    """
+    Function that validates the existence of a device in LibreNMS.
+
+    :param FQDN of the device to validate.
+    """
+    CFG = _get_cfg()
+
+    # Validate existence
+    nms_result = requests.get(
+        CFG.url_devices, headers=CFG.headers)
+    assert nms_result is not None
+
+    device_id = list(map(
+        lambda x: x.get("device_id"),
+        filter(lambda x: x.get("hostname") == fqdn,
+               nms_result.json().get("devices"))))
+
+    if len(device_id) != 1 or device_id[0] is None:
+        error_msg = f"Device with FQDN={fqdn} is not registered in LibreNMS"
+        print(error_msg)
+        raise AssertionError(error_msg)
+
+    # Validate correctness
+    device_id = device_id[0]
+    url_device = f"{CFG.url_devices}/{device_id}"
+    logger.debug(f"Connecting to URL: {url_device}"
+                 f"with headers: {CFG.headers}")
+    nms_result = requests.get(
+        url_device, headers=CFG.headers)
+    logger.debug(f"LibreNMS response={nms_result.content}")
+
+    if nms_result.status_code != 200:
+        print(nms_result.content)
+        raise AssertionError(nms_result.content)
+
+    # nms_dev_sysname = nms_result.json().get("sysName")
+    nms_dev_hostname = nms_result.json().get("devices")[0].get("hostname")
+    if fqdn != nms_dev_hostname:
+        error_msg = f"Device with FQDN={fqdn} may not be correctly "\
+                f"registered in LibreNMS (expected FQDN: {nms_dev_hostname})"
+        print(error_msg)
+        raise AssertionError(error_msg)
+
+
+def register_device(fqdn: str):
+    """
+    Function that registers a new device in LibreNMS.
+
+    :param FQDN of the device to register.
+    """
+    CFG = _get_cfg()
+    logger.debug(f"Registering FQDN={fqdn} in LibreNMS")
+
+    device_data = {
+        "display": fqdn,
+        "hostname": fqdn,
+        "sysName": fqdn,
+        # "override_icmp_disable": "true",
+        # IMPORTANT: uncomment if testing with FQDNs that are not reachable
+        # from LibreNMS (e.g. ContainerLab routers)
+        # "force_add": "true"
+    }
+    if CFG.SNMP.version == "v2c":
+        device_data.update({
+            "community": CFG.SNMP.V2.community
+            })
+
+    elif CFG.SNMP.version == "v3":
+        for key in [
+                "authlevel", "authname", "authpass", "authalgo",
+                "cryptopass", "cryptoalgo"]:
+            device_data.update({key: getattr(CFG.SNMP.V3, key)})
+
+    logger.debug(f"Connecting to URL: {CFG.url_devices}"
+                 f"with headers: {CFG.headers} and"
+                 f"payload: {device_data}")
+    nms_result = requests.post(
+        CFG.url_devices, headers=CFG.headers,
+        data=json.dumps(device_data))
+    logger.debug(f"LibreNMS response={nms_result.content}")
+
+    if nms_result.status_code != 200:
+        print(nms_result.content)
+        raise AssertionError(nms_result.content)
+
+
+def deregister_device(fqdn: str):
+    """
+    Function that reregisters a device from LibreNMS.
+
+    :param FQDN of the device to deregister.
+    """
+    CFG = _get_cfg()
+    logger.debug(f"Deregistering FQDN={fqdn} from LibreNMS")
+
+    nms_result = requests.get(
+        CFG.url_devices, headers=CFG.headers)
+    assert nms_result is not None
+    device_id = list(map(
+        lambda x: x.get("device_id"),
+        filter(lambda x: x.get("hostname") == fqdn,
+               nms_result.json().get("devices"))))
+    if len(device_id) != 1:
+        return
+    device_id = device_id[0]
+
+    # https://docs.librenms.org/API/Devices/#endpoint-categories
+    device_data = {
+        "field": "disabled",
+        "data": "1"
+    }
+    url_device = f"{CFG.url_devices}/{device_id}"
+    logger.debug(f"Connecting to URL: {url_device}"
+                 f"with headers: {CFG.headers} and"
+                 f"payload: {device_data}")
+    nms_result = requests.patch(
+        url_device, headers=CFG.headers,
+        data=json.dumps(device_data))
+    logger.debug(f"LibreNMS response={nms_result.content}")
+
+    # Fail silently if device was not registered
+    if nms_result.status_code != 200:
+        print(nms_result.content)

gso/settings.py

@@ -4,7 +4,7 @@ GSO settings, ensuring that the required parameters are set correctly.
 import ipaddress
 import json
 import os
-from pydantic import BaseSettings, Field
+from pydantic import BaseSettings
 
 
 class GeneralParams(BaseSettings):
@@ -14,6 +14,7 @@ class GeneralParams(BaseSettings):
     #: The hostname that GSO is publicly served at, used for building the
     #: callback URL that the provisioning proxy uses.
     public_hostname: str
+    environment: str
 
 
 class InfoBloxParams(BaseSettings):
@@ -68,6 +69,59 @@ class IPAMParams(BaseSettings):
     LT_IAS: ServiceNetworkParams
 
 
+class MonitoringLibreNMSDevGroupsParams(BaseSettings):
+    """
+    Parameters related to LibreNMS' devicegroups.
+    """
+    routers_lab: str
+    routers_prod: str
+
+
+class MonitoringSNMPV2Params(BaseSettings):
+    """
+    Parameters related to SNMPv2.
+    """
+    community: str
+
+
+class MonitoringSNMPV3Params(BaseSettings):
+    """
+    Parameters related to SNMPv3.
+    """
+    authlevel: str
+    authname: str
+    authpass: str
+    authalgo: str
+    cryptopass: str
+    cryptoalgo: str
+
+
+class MonitoringSNMPParams(BaseSettings):
+    """
+    Parameters related to SNMP.
+    """
+    version: str
+    V2: MonitoringSNMPV2Params
+    V3: MonitoringSNMPV3Params
+
+
+class MonitoringLibreNMSParams(BaseSettings):
+    """
+    Parameters related to LibreNMS.
+    """
+    endpoint: str
+    token: str
+    DEVICE_GROUPS: MonitoringLibreNMSDevGroupsParams
+
+
+class MonitoringParams(BaseSettings):
+    """
+    Parameters related to the monitoring.
+    """
+    LIBRENMS: MonitoringLibreNMSParams
+    SNMP: MonitoringSNMPParams
+
+
 class ProvisioningProxyParams(BaseSettings):
     """
     Parameters for the provisioning proxy.
@@ -85,6 +139,7 @@ class OSSParams(BaseSettings):
     GENERAL: GeneralParams
     IPAM: IPAMParams
     RESOURCE_MANAGER_API_PREFIX: str
+    MONITORING: MonitoringParams
     PROVISIONING_PROXY: ProvisioningProxyParams
 
 

gso/workflows/__init__.py

@@ -4,6 +4,7 @@ init class that imports all workflows into GSO.
 from orchestrator.workflows import LazyWorkflowInstance
 
 LazyWorkflowInstance("gso.workflows.device.create_device", "create_device")
+LazyWorkflowInstance("gso.workflows.device.validate_device", "validate_device")
 LazyWorkflowInstance("gso.workflows.device.terminate_device",
                      "terminate_device")
 LazyWorkflowInstance("gso.workflows.device.get_facts", "get_facts")
@@ -15,3 +16,4 @@ LazyWorkflowInstance("gso.workflows.iptrunk.modify_iptrunk_interface",
 LazyWorkflowInstance("gso.workflows.iptrunk.modify_iptrunk_isis_metric",
                      "modify_iptrunk_isis_metric")
 LazyWorkflowInstance("gso.workflows.site.create_site", "create_site")
+LazyWorkflowInstance("gso.workflows.site.terminate_site", "terminate_site")

gso/workflows/device/create_device.py

@@ -16,10 +16,11 @@ from orchestrator.workflows.utils import wrap_create_initial_input_form
 
 from gso.products.product_blocks import device as device_pb
 from gso.products.product_types import device
-from gso.products.product_types.device import DeviceInactive, \
+from gso.products.product_types.device import Device, DeviceInactive, \
     DeviceProvisioning
 from gso.products.product_types.site import Site
 from gso.services import _ipam
+from gso.services import librenms
 from gso.services import provisioning_proxy
 from gso.services.provisioning_proxy import await_pp_results, \
     confirm_pp_results
@@ -66,6 +67,10 @@ def initial_input_form_generator(product_name: str) -> FormGenerator:
     return user_input.dict()
 
 
+def _fqdn_from_subscription(subscription: Device) -> str:
+    return subscription.device.device_fqdn
+
+
 @step('Create subscription')
 def create_subscription(product: UUIDstr) -> State:
     subscription = DeviceInactive.from_product_id(product, uuid4())
@@ -94,13 +99,17 @@ def get_info_from_ipam(subscription: DeviceProvisioning) -> State:
     subscription.device.device_lo_ipv4_address = lo0_addr.v4
     subscription.device.device_lo_ipv6_address = lo0_addr.v6
     subscription.device.device_lo_iso_address \
-        = iso_from_ipv4(str(subscription.device.device_lo_ipv4_address))
+        = iso_from_ipv4(
+            str(subscription.device.device_lo_ipv4_address))
     subscription.device.device_si_ipv4_network \
-        = _ipam.allocate_service_ipv4_network(service_type='SI', comment=f"SI for {lo0_name}").v4
+        = _ipam.allocate_service_ipv4_network(
+            service_type='SI', comment=f"SI for {lo0_name}").v4
     subscription.device.device_ias_lt_ipv4_network \
-        = _ipam.allocate_service_ipv4_network(service_type='LT_IAS', comment=f"LT for {lo0_name}").v4
+        = _ipam.allocate_service_ipv4_network(
+            service_type='LT_IAS', comment=f"LT for {lo0_name}").v4
     subscription.device.device_ias_lt_ipv6_network \
-        = _ipam.allocate_service_ipv6_network(service_type='LT_IAS', comment=f"LT for {lo0_name}").v6
+        = _ipam.allocate_service_ipv6_network(
+            service_type='LT_IAS', comment=f"LT for {lo0_name}").v6
     return {'subscription': subscription}
 
 
@@ -162,6 +171,17 @@ def provision_device_real(subscription: DeviceProvisioning,
             }
 
 
+@step('Register device in LibreNMS')
+def register_librenms(subscription: Device) -> State:
+    fqdn = _fqdn_from_subscription(subscription)
+    _ = librenms.register_device(fqdn)
+    return {
+        'subscription': subscription,
+        # TBC: wait for results to be returned from LibreNMS or not
+        # 'result_text': result,
+        }
+
+
 @workflow(
     'Create device',
     initial_input_form=wrap_create_initial_input_form(
@@ -181,6 +201,7 @@ def create_device():
             >> provision_device_real
             >> await_pp_results
             >> confirm_pp_results
+            >> register_librenms
             >> set_status(SubscriptionLifecycle.ACTIVE)
             >> resync
             >> done

gso/workflows/device/terminate_device.py

@@ -12,6 +12,7 @@ from orchestrator.workflows.steps import (
 from orchestrator.workflows.utils import wrap_modify_initial_input_form
 
 from gso.products.product_types.device import Device
+from gso.services import librenms
 
 
 def initial_input_form_generator(subscription_id: UUIDstr) -> InputForm:
@@ -19,12 +20,16 @@ def initial_input_form_generator(subscription_id: UUIDstr) -> InputForm:
 
     class TerminateForm(FormPage):
         are_you_sure: Label = (
-            f'Are you sure you want to remove {subscription.description}?'
+            f"Are you sure you want to remove {subscription.description}?"
         )
 
     return TerminateForm
 
 
+def _fqdn_from_subscription(subscription: Device) -> str:
+    return subscription.device.device_fqdn
+
+
 def _deprovision_in_user_management_system(fqdn: str) -> str:
     pass
 
@@ -35,6 +40,17 @@ def deprovision_user(subscription: Device) -> None:
     pass
 
 
+@step("Deregister device from LibreNMS")
+def deregister_librenms(subscription: Device) -> None:
+    fqdn = _fqdn_from_subscription(subscription)
+    _ = librenms.deregister_device(fqdn)
+    return {
+        "subscription": subscription,
+        # TBC: wait for results to be returned from LibreNMS or not
+        # "result_text": result,
+        }
+
+
 @workflow(
     "Terminate device",
     initial_input_form=wrap_modify_initial_input_form(
@@ -47,6 +63,7 @@ def terminate_device():
         >> store_process_subscription(Target.TERMINATE)
         >> unsync
         >> deprovision_user
+        >> deregister_librenms
         >> set_status(SubscriptionLifecycle.TERMINATED)
         >> resync
         >> done

gso/workflows/device/validate_device.py

+# noinspection PyProtectedMember
+from orchestrator.forms import FormPage
+from orchestrator.targets import Target
+from orchestrator.types import InputForm, UUIDstr
+from orchestrator.forms.validators import Label
+from orchestrator.workflow import done, init, step, workflow
+from orchestrator.workflows.steps import resync  # , set_status
+from orchestrator.workflows.utils import wrap_create_initial_input_form
+
+from gso.products.product_types.device import Device
+# noinspection PyProtectedMember
+from gso.services import librenms
+
+
+def initial_input_form_generator(subscription_id: UUIDstr) -> InputForm:
+    subscription = Device.from_subscription(subscription_id)
+
+    class ValidateForm(FormPage):
+        are_you_sure: Label = (
+            f"Are you sure you want to validate {subscription.description}?"
+        )
+
+    return ValidateForm
+
+
+@step("Validate existence of device in LibreNMS")
+def device_registered_in_librenms(subscription: Device) -> None:
+    _ = librenms.validate_device(subscription)
+    return {
+        "subscription": subscription,
+        # TBC: wait for results to be returned from LibreNMS or not
+        # "result_text": result,
+        }
+
+
+@workflow(
+    "Validate device",
+    initial_input_form=wrap_create_initial_input_form(
+                       initial_input_form_generator),
+    target=Target.SYSTEM,
+)
+def validate_device():
+    return (
+        init
+        >> device_registered_in_librenms
+        >> resync
+        >> done
+    )

gso/workflows/site/terminate_site.py

+from orchestrator.forms import FormPage
+from orchestrator.forms.validators import Label
+from orchestrator.targets import Target
+from orchestrator.types import InputForm, SubscriptionLifecycle, UUIDstr
+from orchestrator.workflow import done, init, step, workflow
+from orchestrator.workflows.steps import (
+    resync,
+    set_status,
+    store_process_subscription,
+    unsync,
+)
+from orchestrator.workflows.utils import wrap_modify_initial_input_form
+
+from gso.products.product_types.site import Site
+
+
+def initial_input_form_generator(subscription_id: UUIDstr) -> InputForm:
+    subscription = Site.from_subscription(subscription_id)
+
+    class TerminateForm(FormPage):
+        are_you_sure: Label = (
+            f'Are you sure you want to remove {subscription.description}?'
+        )
+
+    return TerminateForm
+
+
+def _deprovision_in_user_management_system(fqdn: str) -> str:
+    pass
+
+
+@step("Deprovision Site")
+def deprovision_site(subscription: Site) -> None:
+    #    _deprovision_in_user_management_system(subscription.user.user_id)
+    pass
+
+
+@workflow(
+    "Terminate Site",
+    initial_input_form=wrap_modify_initial_input_form(
+                       initial_input_form_generator),
+    target=Target.TERMINATE,
+)
+def terminate_site():
+    return (
+        init
+        >> store_process_subscription(Target.TERMINATE)
+        >> unsync
+        >> deprovision_site
+        >> set_status(SubscriptionLifecycle.TERMINATED)
+        >> resync
+        >> done
+    )