Improved terminate router and update IBGP mesh workflows

gso/services/subscriptions.py

@@ -232,17 +232,3 @@ def get_site_by_name(site_name: str) -> Site:
 
     return Site.from_subscription(subscription[0].subscription_id)
 
-
-def get_active_pe_router_dict() -> dict[str, Any]:
-    """Generate an Ansible-compatible inventory for executing playbooks. Contains all active PE routers."""
-    all_routers = [Router.from_subscription(r["subscription_id"]) for r in get_active_router_subscriptions()]
-
-    return {
-        router.router.router_fqdn: {
-            "lo4": str(router.router.router_lo_ipv4_address),
-            "lo6": str(router.router.router_lo_ipv6_address),
-            "vendor": str(router.router.vendor),
-        }
-        for router in all_routers
-        if router.router.router_role == RouterRole.PE
-    }

gso/utils/helpers.py

@@ -13,10 +13,11 @@ from pydantic_forms.validators import Choice
 
 from gso import settings
 from gso.products.product_blocks.iptrunk import IptrunkInterfaceBlock
+from gso.products.product_blocks.router import RouterRole
 from gso.products.product_blocks.site import LatitudeCoordinate, LongitudeCoordinate, SiteTier
 from gso.products.product_types.router import Router
 from gso.services.netbox_client import NetboxClient
-from gso.services.subscriptions import get_active_subscriptions_by_field_and_value
+from gso.services.subscriptions import get_active_router_subscriptions, get_active_subscriptions_by_field_and_value
 from gso.utils.shared_enums import IPv4AddressType, Vendor
 
 
@@ -54,9 +55,9 @@ def available_interfaces_choices(router_id: UUID, speed: str) -> Choice | None:
 
 
 def available_interfaces_choices_including_current_members(
-    router_id: UUID,
-    speed: str,
-    interfaces: list[IptrunkInterfaceBlock],
+        router_id: UUID,
+        speed: str,
+        interfaces: list[IptrunkInterfaceBlock],
 ) -> Choice | None:
     """Return a list of available interfaces for a given router and speed including the current members.
 
@@ -300,3 +301,41 @@ def generate_fqdn(hostname: str, site_name: str, country_code: str) -> str:
     """Generate an :term:`FQDN` from a hostname, site name, and a country code."""
     oss = settings.load_oss_params()
     return f"{hostname}.{site_name.lower()}.{country_code.lower()}{oss.IPAM.LO.domain_name}"
+
+
+def generate_inventory_for_active_routers(
+        router_role: RouterRole,
+        exclude_routers: list[str] | None = None,
+) -> dict:
+    """Generate an Ansible-compatible inventory for executing playbooks.
+
+    Contains all active routers of a specific role. Optionally, routers can be excluded from the inventory.
+
+    Args:
+    ----
+        router_role (RouterRole): The role of the routers to include in the inventory.
+        exclude_routers (list): List of routers to exclude from the inventory.
+
+    Returns:
+    -------
+        dict[str, Any]: A dictionary representing the inventory of active routers.
+
+    """
+    all_routers = [Router.from_subscription(r["subscription_id"]) for r in get_active_router_subscriptions()]
+    exclude_routers = exclude_routers or []
+
+    return {
+        "all": {
+            "hosts": {
+                router.router.router_fqdn:
+                    {
+                        "lo4": str(router.router.router_lo_ipv4_address),
+                        "lo6": str(router.router.router_lo_ipv6_address),
+                        "vendor": str(router.router.vendor),
+                    }
+                for router in all_routers
+                if router.router.router_role == router_role
+                and router.router.router_fqdn not in exclude_routers
+            }
+        }
+    }

gso/utils/workflow_steps.py

@@ -12,10 +12,8 @@ from pydantic_forms.core import FormPage
 from pydantic_forms.types import FormGenerator
 from pydantic_forms.validators import Label
 
-from gso.products.product_blocks.router import RouterRole
 from gso.products.product_types.iptrunk import Iptrunk
-from gso.products.product_types.router import Router
-from gso.services import lso_client, subscriptions
+from gso.services import lso_client
 from gso.settings import load_oss_params
 
 
@@ -123,80 +121,3 @@ def prompt_sharepoint_checklist_url(checklist_url: str) -> FormGenerator:
     yield SharepointPrompt
 
     return {}
-
-
-@step("[DRY RUN] Add all PE routers to P router iBGP group")
-def add_all_pe_to_p_dry(subscription: dict[str, Any], callback_route: str) -> None:
-    """Perform a dry run of adding the list of all PE routers to the new P router."""
-    extra_vars = {
-        "dry_run": True,
-        "subscription": subscription,
-        "pe_router_list": subscriptions.get_active_pe_router_dict(),
-        "verb": "add_pe_to_p",
-    }
-
-    lso_client.execute_playbook(
-        playbook_name="update_ibgp_mesh.yaml",
-        callback_route=callback_route,
-        inventory=subscription["router"]["router_fqdn"],
-        extra_vars=extra_vars,
-    )
-
-
-@step("[FOR REAL] Add all PE routers to P router iBGP group")
-def add_all_pe_to_p_real(
-    subscription: dict[str, Any], callback_route: str, tt_number: str, process_id: UUIDstr
-) -> None:
-    """Add the list of all PE routers to the new P router."""
-    extra_vars = {
-        "dry_run": False,
-        "subscription": subscription,
-        "pe_router_list": subscriptions.get_active_pe_router_dict(),
-        "commit_comment": f"GSO_PROCESS_ID: {process_id} - TT_NUMBER: {tt_number} - Update iBGP mesh",
-        "verb": "add_pe_to_p",
-    }
-
-    lso_client.execute_playbook(
-        playbook_name="update_ibgp_mesh.yaml",
-        callback_route=callback_route,
-        inventory=subscription["router"]["router_fqdn"],
-        extra_vars=extra_vars,
-    )
-
-
-@step("Calculate list of all active PE routers")
-def calculate_pe_router_list() -> State:
-    """Calculate a list of all active PE routers in the network."""
-    all_routers = [
-        Router.from_subscription(r["subscription_id"]) for r in subscriptions.get_active_router_subscriptions()
-    ]
-    all_pe_routers = [router for router in all_routers if router.router.router_role == RouterRole.PE]
-
-    return {"pe_router_list": all_pe_routers}
-
-
-@step("Calculate list of all active P routers")
-def calculate_p_router_list() -> State:
-    """Calculate a list of all active P routers in the network."""
-    all_routers = [
-        Router.from_subscription(r["subscription_id"]) for r in subscriptions.get_active_router_subscriptions()
-    ]
-    all_pe_routers = [router for router in all_routers if router.router.router_role == RouterRole.P]
-
-    return {"pe_router_list": all_pe_routers}
-
-
-def generate_inventory(router_list: list[Router]) -> dict[str, Any]:
-    """Generate an Ansible-compatible inventory for executing playbooks. Contains all active routers."""
-    return {
-        "all": {
-            "hosts": {
-                router.router.router_fqdn: {
-                    "lo4": str(router.router.router_lo_ipv4_address),
-                    "lo6": str(router.router.router_lo_ipv6_address),
-                    "vendor": str(router.router.vendor),
-                }
-                for router in router_list
-            }
-        },
-    }

gso/workflows/router/terminate_router.py

@@ -18,13 +18,14 @@ from orchestrator.workflows.steps import (
 )
 from orchestrator.workflows.utils import wrap_modify_initial_input_form
 
+from gso.products.product_blocks.router import RouterRole
 from gso.products.product_types.router import Router
 from gso.services import infoblox, lso_client
 from gso.services.librenms_client import LibreNMSClient
 from gso.services.lso_client import execute_playbook, lso_interaction
 from gso.services.netbox_client import NetboxClient
+from gso.utils.helpers import generate_inventory_for_active_routers
 from gso.utils.shared_enums import Vendor
-from gso.utils.workflow_steps import calculate_p_router_list, calculate_pe_router_list, generate_inventory
 
 logger = logging.getLogger(__name__)
 
@@ -44,6 +45,8 @@ def initial_input_form_generator(subscription_id: UUIDstr) -> FormGenerator:
         tt_number: str
         termination_label: Label = "Please confirm whether configuration should get removed from the router."
         remove_configuration: bool = True
+        update_ibgp_mesh_label: Label = "Please confirm whether the iBGP mesh should get updated."
+        update_ibgp_mesh: bool = True
 
     user_input = yield TerminateForm
     return user_input.model_dump() | {
@@ -109,80 +112,75 @@ def remove_device_from_netbox(subscription: Router) -> dict[str, Router]:
     return {"subscription": subscription}
 
 
-@step("[DRY RUN] Remove P router from the mesh of PE routers")
-def remove_p_from_mesh_dry(
-    subscription: Router, callback_route: str, pe_router_list: list[Router], tt_number: str, process_id: UUIDstr
+@step("[DRY RUN] Remove P router from all the PE routers")
+def remove_p_from_all_pe_dry(
+    subscription: Router, callback_route: str, tt_number: str, process_id: UUIDstr
 ) -> None:
-    """Perform a dry run of removing the terminated router from the mesh of PE routers."""
+    """Perform a dry run of removing the terminated router from all the PE routers."""
     extra_vars = {
         "dry_run": True,
         "subscription": subscription,
         "commit_comment": f"GSO_PROCESS_ID: {process_id} - TT_NUMBER: {tt_number} - "
-        f"Remove {subscription.router.router_fqdn} from iBGP mesh",
+        f"Remove {subscription.router.router_fqdn} from all the PE routers",
         "verb": "remove_p_from_pe",
     }
 
     lso_client.execute_playbook(
         playbook_name="update_ibgp_mesh.yaml",
         callback_route=callback_route,
-        inventory=generate_inventory(pe_router_list),
+        inventory=generate_inventory_for_active_routers(RouterRole.PE),
         extra_vars=extra_vars,
     )
 
 
-@step("[REAL RUN] Remove P router from the mesh of PE routers")
-def remove_p_from_mesh_real(
-    subscription: Router, callback_route: str, pe_router_list: list[Router], tt_number: str, process_id: UUIDstr
+@step("[REAL RUN] Remove P router from all the PE routers")
+def remove_p_from_all_pe_real(
+    subscription: Router, callback_route: str, tt_number: str, process_id: UUIDstr
 ) -> None:
-    """Perform a real run of removing the terminated router from the mesh of PE routers."""
+    """Perform a real run of removing the terminated router from all the PE routers."""
     extra_vars = {
         "dry_run": False,
         "subscription": subscription,
         "commit_comment": f"GSO_PROCESS_ID: {process_id} - TT_NUMBER: {tt_number} - "
-        f"Remove {subscription.router.router_fqdn} from iBGP mesh",
+        f"Remove {subscription.router.router_fqdn} from all the PE routers",
         "verb": "remove_p_from_pe",
     }
 
     lso_client.execute_playbook(
         playbook_name="update_ibgp_mesh.yaml",
         callback_route=callback_route,
-        inventory=generate_inventory(pe_router_list),
+        inventory=generate_inventory_for_active_routers(RouterRole.PE),
         extra_vars=extra_vars,
     )
 
 
-@step("[DRY RUN] Remove all PE routers from P router iBGP table")
-def remove_all_pe_from_p_dry(
-    subscription: Router, callback_route: str, pe_router_list: list[Router], tt_number: str, process_id: UUIDstr
+@step("[DRY RUN] Remove PE router from all the PE routers")
+def remove_pe_from_all_pe_dry(
+    subscription: Router, callback_route: str, tt_number: str, process_id: UUIDstr
 ) -> None:
-    """Perform a dry run of removing all PE routers from P router iBGP table."""
-    pe_router_list_excluding_current_router = [
-        router for router in pe_router_list if router.router.router_fqdn != subscription.router.router_fqdn
-    ]
+    """Perform a dry run of removing the terminated router from all the PE routers."""
     extra_vars = {
         "dry_run": True,
         "subscription": subscription,
         "commit_comment": f"GSO_PROCESS_ID: {process_id} - TT_NUMBER: {tt_number} - "
-        f"Remove {subscription.router.router_fqdn} from iBGP mesh",
+        f"Remove {subscription.router.router_fqdn} from all the PE routers",
         "verb": "remove_p_from_net",
     }
 
     lso_client.execute_playbook(
         playbook_name="update_ibgp_mesh.yaml",
         callback_route=callback_route,
-        inventory=generate_inventory(pe_router_list_excluding_current_router),
+        inventory=generate_inventory_for_active_routers(
+            RouterRole.PE, exclude_routers=[subscription.router.router_fqdn]),
         extra_vars=extra_vars,
     )
 
 
-@step("[REAL RUN] Remove all PE routers from P router iBGP table")
-def remove_all_pe_from_p_real(
-    subscription: Router, callback_route: str, pe_router_list: list[Router], tt_number: str, process_id: UUIDstr
+@step("[REAL RUN] Remove all PE routers from all the PE routers")
+def remove_pe_from_all_pe_real(
+    subscription: Router, callback_route: str, tt_number: str, process_id: UUIDstr
 ) -> None:
     """Perform a real run of removing PE router from P router iBGP table."""
-    pe_router_list_excluding_current_router = [
-        router for router in pe_router_list if router.router.router_fqdn != subscription.router.router_fqdn
-    ]
     extra_vars = {
         "dry_run": False,
         "subscription": subscription,
@@ -194,49 +192,50 @@ def remove_all_pe_from_p_real(
     lso_client.execute_playbook(
         playbook_name="update_ibgp_mesh.yaml",
         callback_route=callback_route,
-        inventory=generate_inventory(pe_router_list_excluding_current_router),
+        inventory=generate_inventory_for_active_routers(
+            RouterRole.PE, exclude_routers=[subscription.router.router_fqdn]),
         extra_vars=extra_vars,
     )
 
 
-@step("[DRY RUN] Remove PE router from the mesh of P routers")
-def remove_pe_from_mesh_dry(
-    subscription: Router, callback_route: str, p_router_list: list[Router], tt_number: str, process_id: UUIDstr
+@step("[DRY RUN] Remove PE router from all the P routers")
+def remove_pe_from_all_p_dry(
+    subscription: Router, callback_route: str, tt_number: str, process_id: UUIDstr
 ) -> None:
-    """Perform a dry run of removing PE router from the mesh of P routers."""
+    """Perform a dry run of removing PE router from all P routers."""
     extra_vars = {
         "dry_run": True,
         "subscription": subscription,
         "commit_comment": f"GSO_PROCESS_ID: {process_id} - TT_NUMBER: {tt_number} - "
-        f"Remove {subscription.router.router_fqdn} from iBGP mesh",
+        f"Remove {subscription.router.router_fqdn} from all the P routers",
         "verb": "remove_pe_from_p",
     }
 
     lso_client.execute_playbook(
         playbook_name="update_ibgp_mesh.yaml",
         callback_route=callback_route,
-        inventory=generate_inventory(p_router_list),
+        inventory=generate_inventory_for_active_routers(RouterRole.P),
         extra_vars=extra_vars,
     )
 
 
-@step("[REAL RUN] Remove PE router from the mesh of P routers")
-def remove_pe_from_mesh_real(
-    subscription: Router, callback_route: str, p_router_list: list[Router], tt_number: str, process_id: UUIDstr
+@step("[REAL RUN] Remove PE router from all P routers")
+def remove_pe_from_all_p_real(
+    subscription: Router, callback_route: str, tt_number: str, process_id: UUIDstr
 ) -> None:
-    """Perform a dry run of removing PE router from the mesh of P routers."""
+    """Perform a dry run of removing PE router from all P routers."""
     extra_vars = {
         "dry_run": False,
         "subscription": subscription,
         "commit_comment": f"GSO_PROCESS_ID: {process_id} - TT_NUMBER: {tt_number} - "
-        f"Remove {subscription.router.router_fqdn} from iBGP mesh",
+        f"Remove {subscription.router.router_fqdn} from all the P routers",
         "verb": "remove_pe_from_p",
     }
 
     lso_client.execute_playbook(
         playbook_name="update_ibgp_mesh.yaml",
         callback_route=callback_route,
-        inventory=generate_inventory(p_router_list),
+        inventory=generate_inventory_for_active_routers(RouterRole.P),
         extra_vars=extra_vars,
     )
 
@@ -262,6 +261,7 @@ def terminate_router() -> StepList:
     * Mark the subscription as terminated in the service database
     """
     run_config_steps = conditional(lambda state: state["remove_configuration"])
+    update_ibgp_mesh = conditional(lambda state: state["update_ibgp_mesh"])
     router_is_nokia = conditional(lambda state: state["router_is_nokia"])
     router_is_pe = conditional(lambda state: state["router_is_pe"] == "PE")
     router_is_p = conditional(lambda state: state["router_is_p"] == "P")
@@ -270,14 +270,12 @@ def terminate_router() -> StepList:
         begin
         >> store_process_subscription(Target.TERMINATE)
         >> unsync
-        >> calculate_pe_router_list
-        >> router_is_p(lso_interaction(remove_p_from_mesh_dry))
-        >> router_is_p(lso_interaction(remove_p_from_mesh_real))
-        >> router_is_pe(calculate_p_router_list)
-        >> router_is_pe(lso_interaction(remove_all_pe_from_p_dry))
-        >> router_is_pe(lso_interaction(remove_all_pe_from_p_real))
-        >> router_is_pe(lso_interaction(remove_pe_from_mesh_dry))
-        >> router_is_pe(lso_interaction(remove_pe_from_mesh_real))
+        >> update_ibgp_mesh(router_is_p(lso_interaction(remove_p_from_all_pe_dry)))
+        >> update_ibgp_mesh(router_is_p(lso_interaction(remove_p_from_all_pe_real)))
+        >> update_ibgp_mesh(router_is_pe(lso_interaction(remove_pe_from_all_pe_dry)))
+        >> update_ibgp_mesh(router_is_pe(lso_interaction(remove_pe_from_all_pe_real)))
+        >> update_ibgp_mesh(router_is_pe(lso_interaction(remove_pe_from_all_p_dry)))
+        >> update_ibgp_mesh(router_is_pe(lso_interaction(remove_pe_from_all_p_real)))
         >> deprovision_loopback_ips
         >> run_config_steps(lso_interaction(remove_config_from_router_dry))
         >> run_config_steps(lso_interaction(remove_config_from_router_real))

gso/workflows/router/update_ibgp_mesh.py

@@ -12,12 +12,12 @@ from orchestrator.workflows.steps import resync, store_process_subscription, uns
 from orchestrator.workflows.utils import wrap_modify_initial_input_form
 from pydantic import ConfigDict, model_validator
 
+from gso.products.product_blocks.router import RouterRole
 from gso.products.product_types.router import Router
-from gso.services import librenms_client, lso_client, subscriptions
+from gso.services import librenms_client, lso_client
 from gso.services.lso_client import lso_interaction
 from gso.services.subscriptions import get_trunks_that_terminate_on_router
-from gso.utils.helpers import SNMPVersion
-from gso.utils.workflow_steps import add_all_pe_to_p_dry, add_all_pe_to_p_real
+from gso.utils.helpers import SNMPVersion, generate_inventory_for_active_routers
 
 
 def initial_input_form_generator(subscription_id: UUIDstr) -> FormGenerator:
@@ -62,7 +62,7 @@ def add_p_to_mesh_dry(subscription: dict[str, Any], callback_route: str, tt_numb
     lso_client.execute_playbook(
         playbook_name="update_ibgp_mesh.yaml",
         callback_route=callback_route,
-        inventory={"all": {"hosts": subscriptions.get_active_pe_router_dict()}},
+        inventory=generate_inventory_for_active_routers(RouterRole.PE),
         extra_vars=extra_vars,
     )
 
@@ -80,7 +80,46 @@ def add_p_to_mesh_real(subscription: dict[str, Any], callback_route: str, tt_num
     lso_client.execute_playbook(
         playbook_name="update_ibgp_mesh.yaml",
         callback_route=callback_route,
-        inventory={"all": {"hosts": subscriptions.get_active_pe_router_dict()}},
+        inventory=generate_inventory_for_active_routers(RouterRole.PE),
+        extra_vars=extra_vars,
+    )
+
+
+@step("[DRY RUN] Add all PE routers to P router iBGP group")
+def add_all_pe_to_p_dry(subscription: dict[str, Any], callback_route: str) -> None:
+    """Perform a dry run of adding the list of all PE routers to the new P router."""
+    extra_vars = {
+        "dry_run": True,
+        "subscription": subscription,
+        "pe_router_list": generate_inventory_for_active_routers(RouterRole.PE),
+        "verb": "add_pe_to_p",
+    }
+
+    lso_client.execute_playbook(
+        playbook_name="update_ibgp_mesh.yaml",
+        callback_route=callback_route,
+        inventory=subscription["router"]["router_fqdn"],
+        extra_vars=extra_vars,
+    )
+
+
+@step("[FOR REAL] Add all PE routers to P router iBGP group")
+def add_all_pe_to_p_real(
+    subscription: dict[str, Any], callback_route: str, tt_number: str, process_id: UUIDstr
+) -> None:
+    """Add the list of all PE routers to the new P router."""
+    extra_vars = {
+        "dry_run": False,
+        "subscription": subscription,
+        "pe_router_list": generate_inventory_for_active_routers(RouterRole.PE),
+        "commit_comment": f"GSO_PROCESS_ID: {process_id} - TT_NUMBER: {tt_number} - Update iBGP mesh",
+        "verb": "add_pe_to_p",
+    }
+
+    lso_client.execute_playbook(
+        playbook_name="update_ibgp_mesh.yaml",
+        callback_route=callback_route,
+        inventory=subscription["router"]["router_fqdn"],
         extra_vars=extra_vars,
     )