.gitlab-ci.yml

@@ -15,7 +15,7 @@ run-tox-pipeline:
   image: python:3.12
 
   services:
-    - postgres:15.4
+    - postgres:14
 
   # Change pip's cache directory to be inside the project directory since we can
   # only cache local items.

Changelog.md

 # Changelog
 
+## [2.24] - 2024-11-07
+- Add support for Moodi dashboard in Edge Port creation workflow
+- Fix a bug where ISIS metric restoration did not work correctly
+
+## [2.23] - 2024-11-05
+- Added new workflows and updated the products of Swich and LAN Swith Interconnect
+- Upgraded orchestrator-core to 2.8.0 
+
 ## [2.22] - 2024-10-31
 - Added EdgePort, IAS and GEANT IP products and required workflows
 - Refactored pydantic models for maintainability

docs/source/glossary.rst

@@ -116,3 +116,6 @@ Glossary of terms
 
   WFO
     `Workflow Orchestrator <https://workfloworchestrator.org/>`_
+
+  Moodi
+    A service that does monitoring on demand during a workflow execution.
\ No newline at end of file

gso/oss-params-example.json

@@ -125,6 +125,7 @@
     "DSN": "https://sentry-dsn-url"
   },
   "MOODI": {
-    "host": "moodi.test.gap.geant.org"
+    "host": "moodi.test.gap.geant.org",
+    "moodi_enabled": false
   }
 }

gso/settings.py

@@ -209,6 +209,7 @@ class MoodiParams(BaseSettings):
     """Settings for Moodi."""
 
     host: str
+    moodi_enabled: bool = False
 
 
 class OSSParams(BaseSettings):

gso/utils/workflow_steps.py

@@ -6,7 +6,9 @@ from typing import Any
 from orchestrator import inputstep, step
 from orchestrator.config.assignee import Assignee
 from orchestrator.types import State, UUIDstr
+from orchestrator.utils.errors import ProcessFailureError
 from orchestrator.utils.json import json_dumps
+from orchestrator.workflow import StepList, conditional
 from pydantic import ConfigDict
 from pydantic_forms.core import FormPage
 from pydantic_forms.types import FormGenerator
@@ -14,7 +16,8 @@ from pydantic_forms.validators import Label
 
 from gso.products.product_blocks.router import RouterRole
 from gso.products.product_types.iptrunk import Iptrunk
-from gso.services.lso_client import LSOState
+from gso.services.kentik_client import KentikClient, NewKentikDevice
+from gso.services.lso_client import LSOState, indifferent_lso_interaction
 from gso.settings import load_oss_params
 from gso.utils.helpers import generate_inventory_for_active_routers
 from gso.utils.shared_enums import Vendor
@@ -391,25 +394,59 @@ def prompt_sharepoint_checklist_url(checklist_url: str) -> FormGenerator:
     return {}
 
 
-@step("Start Moodi")
-def start_moodi(subscription: dict[str, Any]) -> LSOState:
-    """Start monitoring on demand using Moodi Telemetry stack."""
-    params = load_oss_params()
+_is_moodi_enabled = conditional(lambda _: load_oss_params().MOODI.moodi_enabled)
 
-    return {
-        "playbook_name": "moodi_telemetry/playbooks/start_moodi.yaml",
-        "inventory": {"all": {"hosts": {params.MOODI.host: None}}},
-        "extra_vars": {"subscription": subscription},
-    }
 
+def start_moodi() -> StepList:
+    """Start monitoring on demand using :term:`Moodi` Telemetry stack."""
+    host = load_oss_params().MOODI.host
 
-@step("Stop Moodi")
-def stop_moodi() -> LSOState:
-    """Stop monitoring on demand."""
-    params = load_oss_params()
+    @step("Start Moodi")
+    def _start_moodi(subscription: dict[str, Any]) -> LSOState:
+        return {
+            "playbook_name": "moodi_telemetry/playbooks/start_moodi.yaml",
+            "inventory": {"all": {"hosts": {host: None}}},
+            "extra_vars": {"subscription": subscription},
+        }
 
-    return {
-        "playbook_name": "moodi_telemetry/playbooks/stop_moodi.yaml",
-        "inventory": {"all": {"hosts": {params.MOODI.host: None}}},
-        "extra_vars": None,
-    }
+    return _is_moodi_enabled(indifferent_lso_interaction(_start_moodi))
+
+
+def stop_moodi() -> StepList:
+    """Stop :term:`Moodi` Telemetry monitoring on demand."""
+    host = load_oss_params().MOODI.host
+
+    @step("Stop Moodi")
+    def _stop_moodi() -> LSOState:
+        return {
+            "playbook_name": "moodi_telemetry/playbooks/stop_moodi.yaml",
+            "inventory": {"all": {"hosts": {host: None}}},
+            "extra_vars": None,
+        }
+
+    return _is_moodi_enabled(indifferent_lso_interaction(_stop_moodi))
+
+
+@step("Create Kentik device")
+def create_kentik_device(state: State) -> State:
+    """Create a new device in Kentik."""
+    kentik_client = KentikClient()
+    kentik_site = kentik_client.get_site_by_name(state["subscription"]["router"]["router_site"]["site_name"])
+
+    if not kentik_site:
+        msg = "Site could not be found in Kentik."
+        raise ProcessFailureError(msg, details=state["subscription"]["router"]["router_site"]["site_name"])
+
+    new_device = NewKentikDevice(
+        device_name=state["subscription"]["router"]["router_fqdn"],
+        device_description=str(state["subscription"]["subscription_id"]),
+        sending_ips=[str(state["subscription"]["router"]["router_lo_ipv4_address"])],
+        site_id=kentik_site["id"],
+        device_snmp_ip=str(state["subscription"]["router"]["router_lo_ipv4_address"]),
+        device_bgp_flowspec=False,
+        device_bgp_neighbor_ip=str(state["subscription"]["router"]["router_lo_ipv4_address"]),
+        device_bgp_neighbor_ip6=str(state["subscription"]["router"]["router_lo_ipv6_address"]),
+    )
+    kentik_device = kentik_client.create_device(new_device)
+
+    return {"kentik_device": kentik_device}

gso/workflows/edge_port/create_edge_port.py

@@ -31,6 +31,7 @@ from gso.utils.helpers import (
 )
 from gso.utils.types.interfaces import LAGMember, PhysicalPortCapacity
 from gso.utils.types.tt_number import TTNumber
+from gso.utils.workflow_steps import start_moodi, stop_moodi
 from gso.workflows.shared import create_summary_form
 
 
@@ -267,11 +268,13 @@ def create_edge_port() -> StepList:
         >> create_subscription
         >> store_process_subscription(Target.CREATE)
         >> initialize_subscription
+        >> start_moodi()
         >> reserve_interfaces_in_netbox
         >> lso_interaction(create_edge_port_dry)
         >> lso_interaction(create_edge_port_real)
         >> allocate_interfaces_in_netbox
         >> set_status(SubscriptionLifecycle.ACTIVE)
         >> resync
+        >> stop_moodi()
         >> done
     )

gso/workflows/iptrunk/migrate_iptrunk.py

@@ -631,6 +631,7 @@ def restore_isis_metric(
     }
 
     return {
+        "subscription": subscription,
         "playbook_name": "gap_ansible/playbooks/iptrunks.yaml",
         "inventory": {
             "all": {
@@ -761,6 +762,15 @@ def update_subscription_model(
             IptrunkInterfaceBlock.new(subscription_id=uuid4(), **member),
         )
 
+    # Take the new site names, and update the subscription description
+    side_names = sorted([
+        subscription.iptrunk.iptrunk_sides[0].iptrunk_side_node.router_site.site_name,
+        subscription.iptrunk.iptrunk_sides[1].iptrunk_side_node.router_site.site_name,
+    ])
+    subscription.description = (
+        f"IP trunk {side_names[0]} {side_names[1]}, geant_s_sid:{subscription.iptrunk.geant_s_sid}"
+    )
+
     return {"subscription": subscription}
 
 

gso/workflows/router/create_router.py

@@ -28,6 +28,7 @@ from gso.utils.shared_enums import Vendor
 from gso.utils.types.ip_address import PortNumber
 from gso.utils.types.tt_number import TTNumber
 from gso.utils.workflow_steps import (
+    create_kentik_device,
     deploy_base_config_dry,
     deploy_base_config_real,
     prompt_sharepoint_checklist_url,
@@ -254,6 +255,7 @@ def create_router() -> StepList:
     * Create a new device in Netbox
     """
     router_is_nokia = conditional(lambda state: state["vendor"] == Vendor.NOKIA)
+    router_is_pe = conditional(lambda state: state["router_role"] == RouterRole.PE)
 
     return (
         begin
@@ -261,6 +263,7 @@ def create_router() -> StepList:
         >> store_process_subscription(Target.CREATE)
         >> initialize_subscription
         >> ipam_allocate_loopback
+        >> router_is_pe(create_kentik_device)
         >> lso_interaction(deploy_base_config_dry)
         >> lso_interaction(deploy_base_config_real)
         >> verify_ipam_loopback

gso/workflows/router/promote_p_to_pe.py

@@ -8,7 +8,6 @@ from orchestrator.forms import FormPage
 from orchestrator.forms.validators import Label
 from orchestrator.targets import Target
 from orchestrator.types import FormGenerator, State, UUIDstr
-from orchestrator.utils.errors import ProcessFailureError
 from orchestrator.utils.json import json_dumps
 from orchestrator.workflow import StepList, begin, done, inputstep, step, workflow
 from orchestrator.workflows.steps import resync, store_process_subscription, unsync
@@ -17,7 +16,6 @@ from pydantic import ConfigDict, model_validator
 
 from gso.products.product_blocks.router import RouterRole
 from gso.products.product_types.router import Router
-from gso.services.kentik_client import KentikClient, NewKentikDevice
 from gso.services.lso_client import LSOState, lso_interaction
 from gso.services.subscriptions import get_all_active_sites
 from gso.utils.helpers import generate_inventory_for_active_routers
@@ -34,6 +32,7 @@ from gso.utils.workflow_steps import (
     add_pe_to_pe_mesh_real,
     check_l3_services,
     check_pe_ibgp,
+    create_kentik_device,
     update_sdp_mesh_dry,
     update_sdp_mesh_real,
 )
@@ -130,40 +129,6 @@ def prompt_insert_in_earl(subscription: dict[str, Any]) -> FormGenerator:
     return {}
 
 
-@step("Create Kentik device")
-def create_kentik_device(subscription: Router) -> State:
-    """Create a new device in Kentik."""
-    if not (
-        subscription.router.router_site
-        and subscription.router.router_site.site_name
-        and subscription.router.router_site.site_tier
-        and subscription.router.router_fqdn
-    ):
-        msg = "Router object is missing required properties."
-        raise ProcessFailureError(msg)
-
-    kentik_client = KentikClient()
-    kentik_site = kentik_client.get_site_by_name(subscription.router.router_site.site_name)
-
-    if not kentik_site:
-        msg = "Site could not be found in Kentik."
-        raise ProcessFailureError(msg, details=subscription.router.router_site.site_name)
-
-    new_device = NewKentikDevice(
-        device_name=subscription.router.router_fqdn,
-        device_description=str(subscription.subscription_id),
-        sending_ips=[str(subscription.router.router_lo_ipv4_address)],
-        site_id=kentik_site["id"],
-        device_snmp_ip=str(subscription.router.router_lo_ipv4_address),
-        device_bgp_flowspec=False,
-        device_bgp_neighbor_ip=str(subscription.router.router_lo_ipv4_address),
-        device_bgp_neighbor_ip6=str(subscription.router.router_lo_ipv6_address),
-    )
-    kentik_device = kentik_client.create_device(new_device)
-
-    return {"kentik_device": kentik_device}
-
-
 @step("[DRY RUN] Remove P from all PEs")
 def remove_p_from_pe_dry(subscription: dict[str, Any], tt_number: str, process_id: UUIDstr) -> LSOState:
     """Perform a dry run of removing the P router from all the PE routers."""

requirements.txt

-orchestrator-core==2.7.6
+orchestrator-core==2.8.0
 requests==2.31.0
 infoblox-client~=0.6.0
 pycountry==23.12.11

setup.py

@@ -4,14 +4,14 @@ from setuptools import find_packages, setup
 
 setup(
     name="geant-service-orchestrator",
-    version="2.23",
+    version="2.25",
     author="GÉANT Orchestration and Automation Team",
     author_email="goat@geant.org",
     description="GÉANT Service Orchestrator",
     url="https://gitlab.software.geant.org/goat/gap/geant-service-orchestrator",
     packages=find_packages(),
     install_requires=[
-        "orchestrator-core==2.7.6",
+        "orchestrator-core==2.8.0",
         "requests==2.31.0",
         "infoblox-client~=0.6.0",
         "pycountry==23.12.11",

test/workflows/iptrunk/test_migrate_iptrunk.py

@@ -109,6 +109,7 @@ def interface_lists_are_equal(list1, list2):
     [UseJuniperSide.NONE, UseJuniperSide.SIDE_A, UseJuniperSide.SIDE_B, UseJuniperSide.SIDE_BOTH],
     indirect=True,
 )
+@pytest.mark.parametrize("restore_isis_metric", [True, False])
 @pytest.mark.workflow()
 @patch("gso.services.infoblox.create_host_by_ip")
 @patch("gso.services.infoblox.delete_host_by_ip")
@@ -122,7 +123,7 @@ def interface_lists_are_equal(list1, list2):
 @patch("gso.services.netbox_client.NetboxClient.free_interface")
 @patch("gso.services.netbox_client.NetboxClient.delete_interface")
 @patch("gso.workflows.iptrunk.migrate_iptrunk.SharePointClient")
-def test_migrate_iptrunk_success(
+def test_migrate_iptrunk_success(  # noqa: PLR0915
     mock_sharepoint_client,
     mocked_delete_interface,
     mocked_free_interface,
@@ -136,6 +137,7 @@ def test_migrate_iptrunk_success(
     mock_delete_host_by_ip,
     mock_create_host_by_ip,
     migrate_form_input,
+    restore_isis_metric,
     data_config_filename: PathLike,
 ):
     #  Set up mock return values
@@ -150,6 +152,7 @@ def test_migrate_iptrunk_success(
     mocked_delete_interface.return_value = mocked_netbox.delete_interface()
     mock_sharepoint_client.return_value = MockedSharePointClient
 
+    migrate_form_input[1]["restore_isis_metric"] = restore_isis_metric
     result, process_stat, step_log = run_workflow("migrate_iptrunk", migrate_form_input)
 
     for _ in range(8):
@@ -161,10 +164,10 @@ def test_migrate_iptrunk_success(
     for _ in range(8):
         result, step_log = assert_lso_interaction_success(result, process_stat, step_log)
 
-    assert_suspended(result)
-    result, step_log = resume_workflow(process_stat, step_log, input_data=USER_CONFIRM_EMPTY_FORM)
+    if restore_isis_metric:
+        assert_suspended(result)
+        result, step_log = resume_workflow(process_stat, step_log, input_data=USER_CONFIRM_EMPTY_FORM)
 
-    for _ in range(1):
         result, step_log = assert_lso_interaction_success(result, process_stat, step_log)
 
     #  Continue workflow after it has displayed a checklist URL.
@@ -178,7 +181,7 @@ def test_migrate_iptrunk_success(
     subscription = Iptrunk.from_subscription(subscription_id)
 
     assert subscription.status == "active"
-    assert mock_execute_playbook.call_count == 17
+    assert mock_execute_playbook.call_count == 17 if restore_isis_metric else 16
     assert mock_create_host_by_ip.call_count == 1
     assert mock_delete_host_by_ip.call_count == 1
 
@@ -216,3 +219,4 @@ def test_migrate_iptrunk_success(
     assert subscription.iptrunk.iptrunk_sides[0].iptrunk_side_ae_iface == new_lag_interface
     existing_members = subscription.iptrunk.iptrunk_sides[0].iptrunk_side_ae_members
     assert interface_lists_are_equal(new_lag_member_interfaces, existing_members)
+    assert (subscription.iptrunk.iptrunk_isis_metric == 999999) != restore_isis_metric

test/workflows/router/test_create_router.py

@@ -10,7 +10,7 @@ from gso.products.product_types.site import Site
 from gso.services.subscriptions import get_product_id_by_name
 from gso.utils.shared_enums import Vendor
 from test import USER_CONFIRM_EMPTY_FORM
-from test.services.conftest import MockedSharePointClient
+from test.services.conftest import MockedKentikClient, MockedSharePointClient
 from test.workflows import (
     assert_complete,
     assert_lso_interaction_failure,
@@ -31,19 +31,21 @@ def router_creation_input_form_data(site_subscription_factory, faker):
         "router_site": router_site,
         "hostname": faker.pystr(),
         "ts_port": faker.pyint(),
-        "router_role": faker.random_choices(elements=(RouterRole.P, RouterRole.PE, RouterRole.AMT), length=1)[0],
         "vendor": Vendor.NOKIA,
     }
 
 
 @pytest.mark.workflow()
+@pytest.mark.parametrize("router_role", [RouterRole.P, RouterRole.PE])
 @patch("gso.services.lso_client._send_request")
 @patch("gso.workflows.router.create_router.NetboxClient.create_device")
 @patch("gso.workflows.router.create_router.infoblox.hostname_available")
 @patch("gso.workflows.router.create_router.infoblox.find_host_by_fqdn")
 @patch("gso.workflows.router.create_router.infoblox.allocate_host")
 @patch("gso.workflows.router.create_router.SharePointClient")
+@patch("gso.utils.workflow_steps.KentikClient")
 def test_create_nokia_router_success(
+    mock_kentik_client,
     mock_sharepoint_client,
     mock_allocate_host,
     mock_find_host_by_fqdn,
@@ -51,10 +53,11 @@ def test_create_nokia_router_success(
     mock_netbox_create_device,
     mock_provision_router,
     router_creation_input_form_data,
+    router_role,
     faker,
-    data_config_filename,
 ):
     #  Set up mock return values
+    mock_kentik_client.return_value = MockedKentikClient
     product_id = get_product_id_by_name(ProductName.ROUTER)
     mock_site = Site.from_subscription(router_creation_input_form_data["router_site"]).site
     mock_v4 = faker.ipv4()
@@ -68,7 +71,7 @@ def test_create_nokia_router_success(
     mock_sharepoint_client.return_value = MockedSharePointClient
 
     #  Run workflow
-    initial_router_data = [{"product": product_id}, router_creation_input_form_data, {}]
+    initial_router_data = [{"product": product_id}, router_creation_input_form_data | {"router_role": router_role}, {}]
     result, process_stat, step_log = run_workflow("create_router", initial_router_data)
 
     state = extract_state(result)
@@ -118,6 +121,7 @@ def test_create_nokia_router_success(
 
 
 @pytest.mark.workflow()
+@pytest.mark.parametrize("router_role", [RouterRole.P, RouterRole.PE])
 @patch("gso.services.lso_client._send_request")
 @patch("gso.workflows.router.create_router.NetboxClient.create_device")
 @patch("gso.workflows.router.create_router.infoblox.hostname_available")
@@ -126,7 +130,9 @@ def test_create_nokia_router_success(
 @patch("gso.workflows.router.create_router.infoblox.allocate_v6_network")
 @patch("gso.workflows.router.create_router.infoblox.allocate_v4_network")
 @patch("gso.workflows.router.create_router.infoblox.allocate_host")
+@patch("gso.utils.workflow_steps.KentikClient")
 def test_create_nokia_router_lso_failure(
+    mock_kentik_client,
     mock_allocate_host,
     mock_allocate_v4_network,
     mock_allocate_v6_network,
@@ -136,9 +142,11 @@ def test_create_nokia_router_lso_failure(
     mock_netbox_create_device,
     mock_provision_router,
     router_creation_input_form_data,
+    router_role,
     faker,
 ):
     #  Set up mock return values
+    mock_kentik_client.return_value = MockedKentikClient
     mock_site = Site.from_subscription(router_creation_input_form_data["router_site"]).site
     mock_v4 = faker.ipv4()
     mock_v4_net = faker.ipv4(network=True)
@@ -176,7 +184,7 @@ def test_create_nokia_router_lso_failure(
 
     #  Run workflow
     product_id = get_product_id_by_name(ProductName.ROUTER)
-    initial_router_data = [{"product": product_id}, router_creation_input_form_data, {}]
+    initial_router_data = [{"product": product_id}, router_creation_input_form_data | {"router_role": router_role}, {}]
     result, process_stat, step_log = run_workflow("create_router", initial_router_data)
 
     result, step_log = assert_lso_interaction_success(result, process_stat, step_log)

test/workflows/router/test_promote_p_to_pe.py

@@ -19,7 +19,7 @@ from test.workflows import (
 
 @pytest.mark.workflow()
 @patch("gso.services.lso_client._send_request")
-@patch("gso.workflows.router.promote_p_to_pe.KentikClient")
+@patch("gso.utils.workflow_steps.KentikClient")
 def test_promote_p_to_pe_success(
     mock_kentik_client,
     mock_execute_playbook,