Merge branch 'develop' into NAT-185

.pylintrc

+[MAIN]
+extension-pkg-whitelist=pydantic
+
+[MISCELLANEOUS]
+
+# List of note tags to take in consideration, separated by a comma.
+# Note that it does not contain TODO, only the default FIXME and XXX
+notes=FIXME,
+      XXX

gso/main.py

+"""
+The main module, from where GSO is run.
+"""
 from orchestrator import OrchestratorCore
 from orchestrator.cli.main import app as core_cli
 from orchestrator.settings import AppSettings

gso/products/__init__.py

+"""
+Module that updates the domain model of GSO. Should contain all types of
+subscriptions.
+"""
 from orchestrator.domain import SUBSCRIPTION_MODEL_REGISTRY
 
 from gso.products.product_types.device import Device

gso/products/product_blocks/__init__.py

+from enum import IntEnum
+
+
+class PhyPortCapacity(IntEnum):
+    ONE = 1
+    TEN = 10
+    HUNDRED = 100
+    FOUR_HUNDRED = 400

gso/products/product_blocks/iptrunk.py

@@ -21,6 +21,7 @@ class IptrunkBlockInactive(ProductBlockModel,
     iptrunk_type: Optional[IptrunkType] = None
     iptrunk_speed: Optional[str] = None
     iptrunk_minimum_links: Optional[int] = None
+    iptrunk_isis_metric: Optional[int] = None
     iptrunk_ipv4_network: Optional[ipaddress.IPv4Network] = None
     iptrunk_ipv6_network: Optional[ipaddress.IPv6Network] = None
     #
@@ -46,6 +47,7 @@ class IptrunkBlockProvisioning(IptrunkBlockInactive,
     iptrunk_type: Optional[IptrunkType] = None
     iptrunk_speed: Optional[str] = None
     iptrunk_minimum_links: Optional[int] = None
+    iptrunk_isis_metric: Optional[int] = None
     iptrunk_ipv4_network: Optional[ipaddress.IPv4Network] = None
     iptrunk_ipv6_network: Optional[ipaddress.IPv6Network] = None
     #
@@ -71,6 +73,7 @@ class IptrunkBlock(IptrunkBlockProvisioning,
     iptrunk_type: IptrunkType
     iptrunk_speed: str
     iptrunk_minimum_links: int
+    iptrunk_isis_metric: int
     iptrunk_ipv4_network: ipaddress.IPv4Network
     iptrunk_ipv6_network: ipaddress.IPv6Network
     #

gso/services/provisioning_proxy.py

@@ -115,7 +115,7 @@ def provision_ip_trunk(subscription: IptrunkProvisioning,
     parameters = {
         'subscription': json.loads(json_dumps(subscription)),
         'dry_run': dry_run,
-        'verb': "deploy",
+        'verb': 'deploy',
         'object': config_object
     }
 
@@ -162,22 +162,20 @@ def deprovision_ip_trunk(subscription: Iptrunk,
     parameters = {
         'subscription': json.loads(json_dumps(subscription)),
         'dry_run': dry_run,
-        'verb': "remove"
+        'verb': 'terminate'
     }
 
     _send_request('ip_trunk', parameters, process_id, CUDOperation.DELETE)
 
 
 @inputstep('Await provisioning proxy results', assignee=Assignee('SYSTEM'))
-def await_pp_results(subscription: SubscriptionModel) -> State:
+def await_pp_results(subscription: SubscriptionModel,
+                     label_text: str) -> State:
     class ProvisioningResultPage(FormPage):
         class Config:
             title = f'Deploying {subscription.product.name}...'
 
-        warning_label: Label = f'{subscription.product.description} is being' \
-                               f' deployed right now. Feel free to refresh ' \
-                               f'this page every now and again. Just be ' \
-                               f'sure that you do NOT click submit!'
+        warning_label: Label = label_text
         pp_run_results: dict = None
         confirm: Accept = Accept('INCOMPLETE')
 

gso/settings.py

+"""
+GSO settings, ensuring that the required parameters are set correctly.
+"""
 import ipaddress
 import json
 import os
-from pydantic import BaseSettings
+from pydantic import BaseSettings, Field
 
 
 class GeneralParams(BaseSettings):
+    """
+    General parameters for a GSO configuration file.
+    """
+    #: The hostname that GSO is publicly served at, used for building the
+    #: callback URL that the provisioning proxy uses.
     public_hostname: str
 
 
 class InfoBloxParams(BaseSettings):
+    """
+    Parameters related to InfoBlox.
+    """
     scheme: str
     wapi_version: str
     host: str
@@ -17,18 +28,28 @@ class InfoBloxParams(BaseSettings):
 
 
 class V4NetworkParams(BaseSettings):
+    """
+    A set of parameters that describe an IPv4 network in InfoBlox.
+    """
     containers: list[ipaddress.IPv4Network]
     networks: list[ipaddress.IPv4Network]
     mask: int  # TODO: validation on mask?
 
 
 class V6NetworkParams(BaseSettings):
+    """
+    A set of parameters that describe an IPv6 network in InfoBlox.
+    """
     containers: list[ipaddress.IPv6Network]
     networks: list[ipaddress.IPv6Network]
     mask: int  # TODO: validation on mask?
 
 
 class ServiceNetworkParams(BaseSettings):
+    """
+    Parameters for InfoBlox that describe IPv4 and v6 networks, and the
+    corresponding domain name that should be used as a suffix.
+    """
     V4: V4NetworkParams
     V6: V6NetworkParams
     domain_name: str
@@ -36,6 +57,9 @@ class ServiceNetworkParams(BaseSettings):
 
 
 class IPAMParams(BaseSettings):
+    """
+    A set of parameters related to IPAM.
+    """
     INFOBLOX: InfoBloxParams
     LO: ServiceNetworkParams
     TRUNK: ServiceNetworkParams
@@ -45,6 +69,9 @@ class IPAMParams(BaseSettings):
 
 
 class ProvisioningProxyParams(BaseSettings):
+    """
+    Parameters for the provisioning proxy.
+    """
     scheme: str
     api_base: str
     auth: str  # FIXME: unfinished
@@ -52,16 +79,19 @@ class ProvisioningProxyParams(BaseSettings):
 
 
 class OSSParams(BaseSettings):
+    """
+    The set of parameters required for running GSO.
+    """
     GENERAL: GeneralParams
     IPAM: IPAMParams
-    RESOURCE_MANAGER_API_PREFIX: str  # api prefix
+    RESOURCE_MANAGER_API_PREFIX: str
     PROVISIONING_PROXY: ProvisioningProxyParams
 
 
 def load_oss_params() -> OSSParams:
     """
-    look for OSS_PARAMS_FILENAME in the environment and load the
-    parameters from that file
+    look for OSS_PARAMS_FILENAME in the environment and load the parameters
+    from that file.
     """
     with open(os.environ['OSS_PARAMS_FILENAME'], encoding='utf-8') as file:
         return OSSParams(**json.loads(file.read()))

gso/workflows/__init__.py

@@ -10,4 +10,8 @@ LazyWorkflowInstance("gso.workflows.device.get_facts", "get_facts")
 LazyWorkflowInstance("gso.workflows.iptrunk.create_iptrunk", "create_iptrunk")
 LazyWorkflowInstance("gso.workflows.iptrunk.terminate_iptrunk",
                      "terminate_iptrunk")
+LazyWorkflowInstance("gso.workflows.iptrunk.modify_iptrunk_interface",
+                     "modify_iptrunk_interface")
+LazyWorkflowInstance("gso.workflows.iptrunk.modify_iptrunk_isis_metric",
+                     "modify_iptrunk_isis_metric")
 LazyWorkflowInstance("gso.workflows.site.create_site", "create_site")

gso/workflows/device/create_device.py

@@ -139,7 +139,13 @@ def provision_device_dry(subscription: DeviceProvisioning,
                          process_id: UUIDstr) -> State:
     provisioning_proxy.provision_device(subscription, process_id)
 
-    return {'subscription': subscription}
+    return {'subscription': subscription,
+            'label_text': f'This is a dry run for the deployment of a new '
+                          f'{subscription.device_type}. Deployment is being '
+                          f'taken care of by the provisioning proxy, please '
+                          f'wait for the results to come back before '
+                          f'continuing.'
+            }
 
 
 @step('Provision device [FOR REAL]')
@@ -147,7 +153,13 @@ def provision_device_real(subscription: DeviceProvisioning,
                           process_id: UUIDstr) -> State:
     provisioning_proxy.provision_device(subscription, process_id, False)
 
-    return {'subscription': subscription}
+    return {'subscription': subscription,
+            'label_text': f'This is a live deployment of a new '
+                          f'{subscription.device_type}. Deployment is being '
+                          f'taken care of by the provisioning proxy, please '
+                          f'wait for the results to come back before '
+                          f'continuing.'
+            }
 
 
 @workflow(
@@ -161,8 +173,8 @@ def create_device():
             init
             >> create_subscription
             >> store_process_subscription(Target.CREATE)
-            >> get_info_from_ipam
             >> initialize_subscription
+            >> get_info_from_ipam
             >> provision_device_dry
             >> await_pp_results
             >> confirm_pp_results

gso/workflows/iptrunk/create_iptrunk.py

@@ -3,7 +3,7 @@ from uuid import uuid4
 from orchestrator.db.models import ProductTable, SubscriptionTable
 # noinspection PyProtectedMember
 from orchestrator.forms import FormPage
-from orchestrator.forms.validators import Choice, choice_list
+from orchestrator.forms.validators import Choice, UniqueConstrainedList
 from orchestrator.targets import Target
 from orchestrator.types import FormGenerator, State
 from orchestrator.types import SubscriptionLifecycle, UUIDstr
@@ -12,6 +12,7 @@ from orchestrator.workflows.steps import resync, set_status
 from orchestrator.workflows.steps import store_process_subscription
 from orchestrator.workflows.utils import wrap_create_initial_input_form
 
+from gso.products.product_blocks import PhyPortCapacity
 from gso.products.product_blocks.iptrunk import IptrunkType
 from gso.products.product_types.device import Device
 from gso.products.product_types.iptrunk import IptrunkInactive, \
@@ -21,11 +22,14 @@ from gso.services.provisioning_proxy import confirm_pp_results, \
     await_pp_results
 
 
-def device_selector(choice_value: str) -> list:
-    device_subscriptions = {}
+def initial_input_form_generator(product_name: str) -> FormGenerator:
+    # TODO: we need additional validation:
+    # * interface names must be validated
+
+    devices = {}
     for device_id, device_description in (
             SubscriptionTable.query.join(ProductTable)
-                                   .filter(
+            .filter(
                 ProductTable.product_type == 'Device',
                 SubscriptionTable.status == 'active',
             )
@@ -33,18 +37,8 @@ def device_selector(choice_value: str) -> list:
                            SubscriptionTable.description)
             .all()
     ):
-        device_subscriptions[str(device_id)] = device_description
-
-    # noinspection PyTypeChecker
-    return choice_list(
-        Choice(choice_value, zip(device_subscriptions.keys(),
-                                 device_subscriptions.items())),  # type:ignore
-        min_items=1,
-        max_items=1,
-    )
+        devices[str(device_id)] = device_description
 
-
-def initial_input_form_generator(product_name: str) -> FormGenerator:
     class CreateIptrunkForm(FormPage):
         class Config:
             title = product_name
@@ -52,31 +46,51 @@ def initial_input_form_generator(product_name: str) -> FormGenerator:
         geant_s_sid: str
         iptrunk_description: str
         iptrunk_type: IptrunkType
-        iptrunk_speed: str  # This should be an enum: 1/10/100/400
+        iptrunk_speed: PhyPortCapacity
         iptrunk_minimum_links: int
 
-        iptrunk_sideA_node_id: device_selector(
-            choice_value='DeviceEnumA')  # noqa: F821
+    initial_user_input = yield CreateIptrunkForm
+
+    class AeMembersListA(UniqueConstrainedList[str]):
+        min_items = initial_user_input.iptrunk_minimum_links
+
+    DeviceEnumA = Choice('Device A', zip(devices.keys(), devices.items()))
+
+    class CreateIptrunkSideAForm(FormPage):
+        class Config:
+            title = 'Provide subscription details for side A of the trunk.'
+
+        iptrunk_sideA_node_id: DeviceEnumA
         iptrunk_sideA_ae_iface: str
         iptrunk_sideA_ae_geant_a_sid: str
-        iptrunk_sideA_ae_members: list[str]
-        iptrunk_sideA_ae_members_descriptions: list[str]
+        iptrunk_sideA_ae_members: AeMembersListA
+        iptrunk_sideA_ae_members_descriptions: AeMembersListA
+
+    user_input_side_a = yield CreateIptrunkSideAForm
+
+    # We remove the selected device for side A, to prevent any loops
+    devices.pop(str(user_input_side_a.iptrunk_sideA_node_id.name))
+    DeviceEnumB = Choice('Device B', zip(devices.keys(), devices.items()))
 
-        iptrunk_sideB_node_id: device_selector(
-            choice_value='DeviceEnumB')  # noqa: F821
+    class AeMembersListB(UniqueConstrainedList[str]):
+        min_items = len(user_input_side_a.iptrunk_sideA_ae_members)
+        max_items = len(user_input_side_a.iptrunk_sideA_ae_members)
+
+    class CreateIptrunkSideBForm(FormPage):
+        class Config:
+            title = 'Provide subscription details for side B of the trunk.'
+
+        iptrunk_sideB_node_id: DeviceEnumB
         iptrunk_sideB_ae_iface: str
         iptrunk_sideB_ae_geant_a_sid: str
-        iptrunk_sideB_ae_members: list[str]
-        iptrunk_sideB_ae_members_descriptions: list[str]
-        # TODO: we need additional validation:
-        # * sideA fqdn must be different from sideB fqdn
-        # * the length of iptrunk_sideA_ae_members should
-        # be the same as iptrunk_sideB_ae_members
-        # * interface names must be validated
+        iptrunk_sideB_ae_members: AeMembersListB
+        iptrunk_sideB_ae_members_descriptions: AeMembersListB
 
-    user_input = yield CreateIptrunkForm
+    user_input_side_b = yield CreateIptrunkSideBForm
 
-    return user_input.dict()
+    return initial_user_input.dict() | \
+        user_input_side_a.dict() | \
+        user_input_side_b.dict()
 
 
 @step('Create subscription')
@@ -122,10 +136,11 @@ def initialize_subscription(
     subscription.iptrunk.iptrunk_description = iptrunk_description
     subscription.iptrunk.iptrunk_type = iptrunk_type
     subscription.iptrunk.iptrunk_speed = iptrunk_speed
+    subscription.iptrunk.iptrunk_isis_metric = 9000
     subscription.iptrunk.iptrunk_minimum_links = iptrunk_minimum_links
 
-    subscription.iptrunk.iptrunk_sideA_node = Device.from_subscription(
-        iptrunk_sideA_node_id[0]).device
+    subscription.iptrunk.iptrunk_sideA_node = \
+        Device.from_subscription(iptrunk_sideA_node_id).device
     subscription.iptrunk.iptrunk_sideA_ae_iface = iptrunk_sideA_ae_iface
     subscription.iptrunk.iptrunk_sideA_ae_geant_a_sid \
         = iptrunk_sideA_ae_geant_a_sid
@@ -133,8 +148,8 @@ def initialize_subscription(
     subscription.iptrunk.iptrunk_sideA_ae_members_description \
         = iptrunk_sideA_ae_members_descriptions
 
-    subscription.iptrunk.iptrunk_sideB_node = Device.from_subscription(
-        iptrunk_sideB_node_id[0]).device
+    subscription.iptrunk.iptrunk_sideB_node = \
+        Device.from_subscription(iptrunk_sideB_node_id).device
     subscription.iptrunk.iptrunk_sideB_ae_iface \
         = iptrunk_sideB_ae_iface
     subscription.iptrunk.iptrunk_sideB_ae_geant_a_sid \
@@ -157,7 +172,12 @@ def provision_ip_trunk_iface_dry(subscription: IptrunkProvisioning,
     provisioning_proxy.provision_ip_trunk(subscription, process_id,
                                           'trunk_interface')
 
-    return {'subscription': subscription}
+    return {'subscription': subscription,
+            'label_text': 'This is a dry run for the deployment of a new IP '
+                          'trunk. Deployment is being taken care of by the '
+                          'provisioning proxy, please wait for the results to '
+                          'come back before continuing.'
+            }
 
 
 @step('Provision IP trunk interface [FOR REAL]')
@@ -166,7 +186,12 @@ def provision_ip_trunk_iface_real(subscription: IptrunkProvisioning,
     provisioning_proxy.provision_ip_trunk(subscription, process_id,
                                           'trunk_interface', False)
 
-    return {'subscription': subscription}
+    return {'subscription': subscription,
+            'label_text': 'This is a live deployment of a new IP trunk. '
+                          'Deployment is being taken care of by the '
+                          'provisioning proxy, please wait for the results to '
+                          'come back before continuing.'
+            }
 
 
 @step('Provision IP trunk ISIS interface [DRY RUN]')
@@ -175,7 +200,13 @@ def provision_ip_trunk_isis_iface_dry(subscription: IptrunkProvisioning,
     provisioning_proxy.provision_ip_trunk(subscription, process_id,
                                           'isis_interface')
 
-    return {'subscription': subscription}
+    return {'subscription': subscription,
+            'label_text': 'This is a dry run for the deployment of a new IP '
+                          'trunk ISIS interface. '
+                          'Deployment is being taken care of by the '
+                          'provisioning proxy, please wait for the results to '
+                          'come back before continuing.'
+            }
 
 
 @step('Provision IP trunk ISIS interface [FOR REAL]')
@@ -184,7 +215,13 @@ def provision_ip_trunk_isis_iface_real(subscription: IptrunkProvisioning,
     provisioning_proxy.provision_ip_trunk(subscription, process_id,
                                           'isis_interface', False)
 
-    return {'subscription': subscription}
+    return {'subscription': subscription,
+            'label_text': 'This is a live deployment of a new IP trunk '
+                          'ISIS interface. '
+                          'Deployment is being taken care of by the '
+                          'provisioning proxy, please wait for the results to '
+                          'come back before continuing.'
+            }
 
 
 @step('Provision IP trunk LDP interface [DRY RUN]')
@@ -193,7 +230,13 @@ def provision_ip_trunk_ldp_iface_dry(subscription: IptrunkProvisioning,
     provisioning_proxy.provision_ip_trunk(subscription, process_id,
                                           'ldp_interface')
 
-    return {'subscription': subscription}
+    return {'subscription': subscription,
+            'label_text': 'This is a dry run for the deployment of a new IP '
+                          'trunk LDP interface. '
+                          'Deployment is being taken care of by the '
+                          'provisioning proxy, please wait for the results to '
+                          'come back before continuing.'
+            }
 
 
 @step('Provision IP trunk LDP interface [FOR REAL]')
@@ -202,7 +245,13 @@ def provision_ip_trunk_ldp_iface_real(subscription: IptrunkProvisioning,
     provisioning_proxy.provision_ip_trunk(subscription, process_id,
                                           'ldp_interface', False)
 
-    return {'subscription': subscription}
+    return {'subscription': subscription,
+            'label_text': 'This is a live deployment of a new IP trunk '
+                          'LDP interface. '
+                          'Deployment is being taken care of by the '
+                          'provisioning proxy, please wait for the results to '
+                          'come back before continuing.'
+            }
 
 
 @step('Provision IP trunk LLDP interface [DRY RUN]')
@@ -211,7 +260,13 @@ def provision_ip_trunk_lldp_iface_dry(subscription: IptrunkProvisioning,
     provisioning_proxy.provision_ip_trunk(subscription, process_id,
                                           'lldp_interface')
 
-    return {'subscription': subscription}
+    return {'subscription': subscription,
+            'label_text': 'This is a dry run for the deployment of a new IP '
+                          'trunk LLDP interface. '
+                          'Deployment is being taken care of by the '
+                          'provisioning proxy, please wait for the results to '
+                          'come back before continuing.'
+            }
 
 
 @step('Provision IP trunk LLDP interface [FOR REAL]')
@@ -220,7 +275,13 @@ def provision_ip_trunk_lldp_iface_real(subscription: IptrunkProvisioning,
     provisioning_proxy.provision_ip_trunk(subscription, process_id,
                                           'lldp_interface', False)
 
-    return {'subscription': subscription}
+    return {'subscription': subscription,
+            'label_text': 'This is a live deployment of a new IP trunk '
+                          'LLDP interface. '
+                          'Deployment is being taken care of by the '
+                          'provisioning proxy, please wait for the results to '
+                          'come back before continuing.'
+            }
 
 
 @workflow(

gso/workflows/iptrunk/terminate_iptrunk.py

@@ -35,7 +35,13 @@ def deprovision_ip_trunk_dry(subscription: Iptrunk,
                              process_id: UUIDstr) -> State:
     provisioning_proxy.deprovision_ip_trunk(subscription, process_id)
 
-    return {'subscription': subscription}
+    return {'subscription': subscription,
+            'label_text': 'This is a dry run for the termination of an IP '
+                          'trunk. '
+                          'Termination is being taken care of by the '
+                          'provisioning proxy, please wait for the results to '
+                          'come back before continuing.'
+            }
 
 
 @step('Deprovision IP trunk [FOR REAL]')
@@ -43,7 +49,12 @@ def deprovision_ip_trunk_real(subscription: Iptrunk,
                               process_id: UUIDstr) -> State:
     provisioning_proxy.deprovision_ip_trunk(subscription, process_id, False)
 
-    return {'subscription': subscription}
+    return {'subscription': subscription,
+            'label_text': 'This is a termination of an IP trunk. '
+                          'Termination is being taken care of by the '
+                          'provisioning proxy, please wait for the results to '
+                          'come back before continuing.'
+            }
 
 
 @workflow(

setup.py

@@ -6,7 +6,7 @@ setup(
     author='GEANT',
     author_email='swd@geant.org',
     description='GEANT Service Orchestrator',
-    url=('https://gitlab.geant.org/goat/geant-service-orchestrator'),
+    url='https://gitlab.geant.org/goat/geant-service-orchestrator',
     packages=find_packages(),
     install_requires=[
         'orchestrator-core==1.0.0',