From cc80e140e33fedc26c7799842053980ce088a16f Mon Sep 17 00:00:00 2001
From: Neda Moeini <neda.moeini@geant.org>
Date: Mon, 22 Jul 2024 13:08:30 +0200
Subject: [PATCH] Improved terminate router and update IBGP mesh workflows
---
gso/services/subscriptions.py | 14 ----
gso/utils/helpers.py | 47 +++++++++++-
gso/utils/workflow_steps.py | 81 +-------------------
gso/workflows/router/terminate_router.py | 96 ++++++++++++------------
gso/workflows/router/update_ibgp_mesh.py | 49 ++++++++++--
5 files changed, 135 insertions(+), 152 deletions(-)
diff --git a/gso/services/subscriptions.py b/gso/services/subscriptions.py
index e09bfbab..143cd671 100644
--- a/gso/services/subscriptions.py
+++ b/gso/services/subscriptions.py
@@ -232,17 +232,3 @@ def get_site_by_name(site_name: str) -> Site:
return Site.from_subscription(subscription[0].subscription_id)
-
-def get_active_pe_router_dict() -> dict[str, Any]:
- """Generate an Ansible-compatible inventory for executing playbooks. Contains all active PE routers."""
- all_routers = [Router.from_subscription(r["subscription_id"]) for r in get_active_router_subscriptions()]
-
- return {
- router.router.router_fqdn: {
- "lo4": str(router.router.router_lo_ipv4_address),
- "lo6": str(router.router.router_lo_ipv6_address),
- "vendor": str(router.router.vendor),
- }
- for router in all_routers
- if router.router.router_role == RouterRole.PE
- }
diff --git a/gso/utils/helpers.py b/gso/utils/helpers.py
index edd01009..c72aae77 100644
--- a/gso/utils/helpers.py
+++ b/gso/utils/helpers.py
@@ -13,10 +13,11 @@ from pydantic_forms.validators import Choice
from gso import settings
from gso.products.product_blocks.iptrunk import IptrunkInterfaceBlock
+from gso.products.product_blocks.router import RouterRole
from gso.products.product_blocks.site import LatitudeCoordinate, LongitudeCoordinate, SiteTier
from gso.products.product_types.router import Router
from gso.services.netbox_client import NetboxClient
-from gso.services.subscriptions import get_active_subscriptions_by_field_and_value
+from gso.services.subscriptions import get_active_router_subscriptions, get_active_subscriptions_by_field_and_value
from gso.utils.shared_enums import IPv4AddressType, Vendor
@@ -54,9 +55,9 @@ def available_interfaces_choices(router_id: UUID, speed: str) -> Choice | None:
def available_interfaces_choices_including_current_members(
- router_id: UUID,
- speed: str,
- interfaces: list[IptrunkInterfaceBlock],
+ router_id: UUID,
+ speed: str,
+ interfaces: list[IptrunkInterfaceBlock],
) -> Choice | None:
"""Return a list of available interfaces for a given router and speed including the current members.
@@ -300,3 +301,41 @@ def generate_fqdn(hostname: str, site_name: str, country_code: str) -> str:
"""Generate an :term:`FQDN` from a hostname, site name, and a country code."""
oss = settings.load_oss_params()
return f"{hostname}.{site_name.lower()}.{country_code.lower()}{oss.IPAM.LO.domain_name}"
+
+
+def generate_inventory_for_active_routers(
+ router_role: RouterRole,
+ exclude_routers: list[str] | None = None,
+) -> dict:
+ """Generate an Ansible-compatible inventory for executing playbooks.
+
+ Contains all active routers of a specific role. Optionally, routers can be excluded from the inventory.
+
+ Args:
+ ----
+ router_role (RouterRole): The role of the routers to include in the inventory.
+ exclude_routers (list): List of routers to exclude from the inventory.
+
+ Returns:
+ -------
+ dict[str, Any]: A dictionary representing the inventory of active routers.
+
+ """
+ all_routers = [Router.from_subscription(r["subscription_id"]) for r in get_active_router_subscriptions()]
+ exclude_routers = exclude_routers or []
+
+ return {
+ "all": {
+ "hosts": {
+ router.router.router_fqdn:
+ {
+ "lo4": str(router.router.router_lo_ipv4_address),
+ "lo6": str(router.router.router_lo_ipv6_address),
+ "vendor": str(router.router.vendor),
+ }
+ for router in all_routers
+ if router.router.router_role == router_role
+ and router.router.router_fqdn not in exclude_routers
+ }
+ }
+ }
diff --git a/gso/utils/workflow_steps.py b/gso/utils/workflow_steps.py
index 8bc37c65..93891456 100644
--- a/gso/utils/workflow_steps.py
+++ b/gso/utils/workflow_steps.py
@@ -12,10 +12,8 @@ from pydantic_forms.core import FormPage
from pydantic_forms.types import FormGenerator
from pydantic_forms.validators import Label
-from gso.products.product_blocks.router import RouterRole
from gso.products.product_types.iptrunk import Iptrunk
-from gso.products.product_types.router import Router
-from gso.services import lso_client, subscriptions
+from gso.services import lso_client
from gso.settings import load_oss_params
@@ -123,80 +121,3 @@ def prompt_sharepoint_checklist_url(checklist_url: str) -> FormGenerator:
yield SharepointPrompt
return {}
-
-
-@step("[DRY RUN] Add all PE routers to P router iBGP group")
-def add_all_pe_to_p_dry(subscription: dict[str, Any], callback_route: str) -> None:
- """Perform a dry run of adding the list of all PE routers to the new P router."""
- extra_vars = {
- "dry_run": True,
- "subscription": subscription,
- "pe_router_list": subscriptions.get_active_pe_router_dict(),
- "verb": "add_pe_to_p",
- }
-
- lso_client.execute_playbook(
- playbook_name="update_ibgp_mesh.yaml",
- callback_route=callback_route,
- inventory=subscription["router"]["router_fqdn"],
- extra_vars=extra_vars,
- )
-
-
-@step("[FOR REAL] Add all PE routers to P router iBGP group")
-def add_all_pe_to_p_real(
- subscription: dict[str, Any], callback_route: str, tt_number: str, process_id: UUIDstr
-) -> None:
- """Add the list of all PE routers to the new P router."""
- extra_vars = {
- "dry_run": False,
- "subscription": subscription,
- "pe_router_list": subscriptions.get_active_pe_router_dict(),
- "commit_comment": f"GSO_PROCESS_ID: {process_id} - TT_NUMBER: {tt_number} - Update iBGP mesh",
- "verb": "add_pe_to_p",
- }
-
- lso_client.execute_playbook(
- playbook_name="update_ibgp_mesh.yaml",
- callback_route=callback_route,
- inventory=subscription["router"]["router_fqdn"],
- extra_vars=extra_vars,
- )
-
-
-@step("Calculate list of all active PE routers")
-def calculate_pe_router_list() -> State:
- """Calculate a list of all active PE routers in the network."""
- all_routers = [
- Router.from_subscription(r["subscription_id"]) for r in subscriptions.get_active_router_subscriptions()
- ]
- all_pe_routers = [router for router in all_routers if router.router.router_role == RouterRole.PE]
-
- return {"pe_router_list": all_pe_routers}
-
-
-@step("Calculate list of all active P routers")
-def calculate_p_router_list() -> State:
- """Calculate a list of all active P routers in the network."""
- all_routers = [
- Router.from_subscription(r["subscription_id"]) for r in subscriptions.get_active_router_subscriptions()
- ]
- all_pe_routers = [router for router in all_routers if router.router.router_role == RouterRole.P]
-
- return {"pe_router_list": all_pe_routers}
-
-
-def generate_inventory(router_list: list[Router]) -> dict[str, Any]:
- """Generate an Ansible-compatible inventory for executing playbooks. Contains all active routers."""
- return {
- "all": {
- "hosts": {
- router.router.router_fqdn: {
- "lo4": str(router.router.router_lo_ipv4_address),
- "lo6": str(router.router.router_lo_ipv6_address),
- "vendor": str(router.router.vendor),
- }
- for router in router_list
- }
- },
- }
diff --git a/gso/workflows/router/terminate_router.py b/gso/workflows/router/terminate_router.py
index 835837c7..b8a73dd2 100644
--- a/gso/workflows/router/terminate_router.py
+++ b/gso/workflows/router/terminate_router.py
@@ -18,13 +18,14 @@ from orchestrator.workflows.steps import (
)
from orchestrator.workflows.utils import wrap_modify_initial_input_form
+from gso.products.product_blocks.router import RouterRole
from gso.products.product_types.router import Router
from gso.services import infoblox, lso_client
from gso.services.librenms_client import LibreNMSClient
from gso.services.lso_client import execute_playbook, lso_interaction
from gso.services.netbox_client import NetboxClient
+from gso.utils.helpers import generate_inventory_for_active_routers
from gso.utils.shared_enums import Vendor
-from gso.utils.workflow_steps import calculate_p_router_list, calculate_pe_router_list, generate_inventory
logger = logging.getLogger(__name__)
@@ -44,6 +45,8 @@ def initial_input_form_generator(subscription_id: UUIDstr) -> FormGenerator:
tt_number: str
termination_label: Label = "Please confirm whether configuration should get removed from the router."
remove_configuration: bool = True
+ update_ibgp_mesh_label: Label = "Please confirm whether the iBGP mesh should get updated."
+ update_ibgp_mesh: bool = True
user_input = yield TerminateForm
return user_input.model_dump() | {
@@ -109,80 +112,75 @@ def remove_device_from_netbox(subscription: Router) -> dict[str, Router]:
return {"subscription": subscription}
-@step("[DRY RUN] Remove P router from the mesh of PE routers")
-def remove_p_from_mesh_dry(
- subscription: Router, callback_route: str, pe_router_list: list[Router], tt_number: str, process_id: UUIDstr
+@step("[DRY RUN] Remove P router from all the PE routers")
+def remove_p_from_all_pe_dry(
+ subscription: Router, callback_route: str, tt_number: str, process_id: UUIDstr
) -> None:
- """Perform a dry run of removing the terminated router from the mesh of PE routers."""
+ """Perform a dry run of removing the terminated router from all the PE routers."""
extra_vars = {
"dry_run": True,
"subscription": subscription,
"commit_comment": f"GSO_PROCESS_ID: {process_id} - TT_NUMBER: {tt_number} - "
- f"Remove {subscription.router.router_fqdn} from iBGP mesh",
+ f"Remove {subscription.router.router_fqdn} from all the PE routers",
"verb": "remove_p_from_pe",
}
lso_client.execute_playbook(
playbook_name="update_ibgp_mesh.yaml",
callback_route=callback_route,
- inventory=generate_inventory(pe_router_list),
+ inventory=generate_inventory_for_active_routers(RouterRole.PE),
extra_vars=extra_vars,
)
-@step("[REAL RUN] Remove P router from the mesh of PE routers")
-def remove_p_from_mesh_real(
- subscription: Router, callback_route: str, pe_router_list: list[Router], tt_number: str, process_id: UUIDstr
+@step("[REAL RUN] Remove P router from all the PE routers")
+def remove_p_from_all_pe_real(
+ subscription: Router, callback_route: str, tt_number: str, process_id: UUIDstr
) -> None:
- """Perform a real run of removing the terminated router from the mesh of PE routers."""
+ """Perform a real run of removing the terminated router from all the PE routers."""
extra_vars = {
"dry_run": False,
"subscription": subscription,
"commit_comment": f"GSO_PROCESS_ID: {process_id} - TT_NUMBER: {tt_number} - "
- f"Remove {subscription.router.router_fqdn} from iBGP mesh",
+ f"Remove {subscription.router.router_fqdn} from all the PE routers",
"verb": "remove_p_from_pe",
}
lso_client.execute_playbook(
playbook_name="update_ibgp_mesh.yaml",
callback_route=callback_route,
- inventory=generate_inventory(pe_router_list),
+ inventory=generate_inventory_for_active_routers(RouterRole.PE),
extra_vars=extra_vars,
)
-@step("[DRY RUN] Remove all PE routers from P router iBGP table")
-def remove_all_pe_from_p_dry(
- subscription: Router, callback_route: str, pe_router_list: list[Router], tt_number: str, process_id: UUIDstr
+@step("[DRY RUN] Remove PE router from all the PE routers")
+def remove_pe_from_all_pe_dry(
+ subscription: Router, callback_route: str, tt_number: str, process_id: UUIDstr
) -> None:
- """Perform a dry run of removing all PE routers from P router iBGP table."""
- pe_router_list_excluding_current_router = [
- router for router in pe_router_list if router.router.router_fqdn != subscription.router.router_fqdn
- ]
+ """Perform a dry run of removing the terminated router from all the PE routers."""
extra_vars = {
"dry_run": True,
"subscription": subscription,
"commit_comment": f"GSO_PROCESS_ID: {process_id} - TT_NUMBER: {tt_number} - "
- f"Remove {subscription.router.router_fqdn} from iBGP mesh",
+ f"Remove {subscription.router.router_fqdn} from all the PE routers",
"verb": "remove_p_from_net",
}
lso_client.execute_playbook(
playbook_name="update_ibgp_mesh.yaml",
callback_route=callback_route,
- inventory=generate_inventory(pe_router_list_excluding_current_router),
+ inventory=generate_inventory_for_active_routers(
+ RouterRole.PE, exclude_routers=[subscription.router.router_fqdn]),
extra_vars=extra_vars,
)
-@step("[REAL RUN] Remove all PE routers from P router iBGP table")
-def remove_all_pe_from_p_real(
- subscription: Router, callback_route: str, pe_router_list: list[Router], tt_number: str, process_id: UUIDstr
+@step("[REAL RUN] Remove all PE routers from all the PE routers")
+def remove_pe_from_all_pe_real(
+ subscription: Router, callback_route: str, tt_number: str, process_id: UUIDstr
) -> None:
"""Perform a real run of removing PE router from P router iBGP table."""
- pe_router_list_excluding_current_router = [
- router for router in pe_router_list if router.router.router_fqdn != subscription.router.router_fqdn
- ]
extra_vars = {
"dry_run": False,
"subscription": subscription,
@@ -194,49 +192,50 @@ def remove_all_pe_from_p_real(
lso_client.execute_playbook(
playbook_name="update_ibgp_mesh.yaml",
callback_route=callback_route,
- inventory=generate_inventory(pe_router_list_excluding_current_router),
+ inventory=generate_inventory_for_active_routers(
+ RouterRole.PE, exclude_routers=[subscription.router.router_fqdn]),
extra_vars=extra_vars,
)
-@step("[DRY RUN] Remove PE router from the mesh of P routers")
-def remove_pe_from_mesh_dry(
- subscription: Router, callback_route: str, p_router_list: list[Router], tt_number: str, process_id: UUIDstr
+@step("[DRY RUN] Remove PE router from all the P routers")
+def remove_pe_from_all_p_dry(
+ subscription: Router, callback_route: str, tt_number: str, process_id: UUIDstr
) -> None:
- """Perform a dry run of removing PE router from the mesh of P routers."""
+ """Perform a dry run of removing PE router from all P routers."""
extra_vars = {
"dry_run": True,
"subscription": subscription,
"commit_comment": f"GSO_PROCESS_ID: {process_id} - TT_NUMBER: {tt_number} - "
- f"Remove {subscription.router.router_fqdn} from iBGP mesh",
+ f"Remove {subscription.router.router_fqdn} from all the P routers",
"verb": "remove_pe_from_p",
}
lso_client.execute_playbook(
playbook_name="update_ibgp_mesh.yaml",
callback_route=callback_route,
- inventory=generate_inventory(p_router_list),
+ inventory=generate_inventory_for_active_routers(RouterRole.P),
extra_vars=extra_vars,
)
-@step("[REAL RUN] Remove PE router from the mesh of P routers")
-def remove_pe_from_mesh_real(
- subscription: Router, callback_route: str, p_router_list: list[Router], tt_number: str, process_id: UUIDstr
+@step("[REAL RUN] Remove PE router from all P routers")
+def remove_pe_from_all_p_real(
+ subscription: Router, callback_route: str, tt_number: str, process_id: UUIDstr
) -> None:
- """Perform a dry run of removing PE router from the mesh of P routers."""
+ """Perform a dry run of removing PE router from all P routers."""
extra_vars = {
"dry_run": False,
"subscription": subscription,
"commit_comment": f"GSO_PROCESS_ID: {process_id} - TT_NUMBER: {tt_number} - "
- f"Remove {subscription.router.router_fqdn} from iBGP mesh",
+ f"Remove {subscription.router.router_fqdn} from all the P routers",
"verb": "remove_pe_from_p",
}
lso_client.execute_playbook(
playbook_name="update_ibgp_mesh.yaml",
callback_route=callback_route,
- inventory=generate_inventory(p_router_list),
+ inventory=generate_inventory_for_active_routers(RouterRole.P),
extra_vars=extra_vars,
)
@@ -262,6 +261,7 @@ def terminate_router() -> StepList:
* Mark the subscription as terminated in the service database
"""
run_config_steps = conditional(lambda state: state["remove_configuration"])
+ update_ibgp_mesh = conditional(lambda state: state["update_ibgp_mesh"])
router_is_nokia = conditional(lambda state: state["router_is_nokia"])
router_is_pe = conditional(lambda state: state["router_is_pe"] == "PE")
router_is_p = conditional(lambda state: state["router_is_p"] == "P")
@@ -270,14 +270,12 @@ def terminate_router() -> StepList:
begin
>> store_process_subscription(Target.TERMINATE)
>> unsync
- >> calculate_pe_router_list
- >> router_is_p(lso_interaction(remove_p_from_mesh_dry))
- >> router_is_p(lso_interaction(remove_p_from_mesh_real))
- >> router_is_pe(calculate_p_router_list)
- >> router_is_pe(lso_interaction(remove_all_pe_from_p_dry))
- >> router_is_pe(lso_interaction(remove_all_pe_from_p_real))
- >> router_is_pe(lso_interaction(remove_pe_from_mesh_dry))
- >> router_is_pe(lso_interaction(remove_pe_from_mesh_real))
+ >> update_ibgp_mesh(router_is_p(lso_interaction(remove_p_from_all_pe_dry)))
+ >> update_ibgp_mesh(router_is_p(lso_interaction(remove_p_from_all_pe_real)))
+ >> update_ibgp_mesh(router_is_pe(lso_interaction(remove_pe_from_all_pe_dry)))
+ >> update_ibgp_mesh(router_is_pe(lso_interaction(remove_pe_from_all_pe_real)))
+ >> update_ibgp_mesh(router_is_pe(lso_interaction(remove_pe_from_all_p_dry)))
+ >> update_ibgp_mesh(router_is_pe(lso_interaction(remove_pe_from_all_p_real)))
>> deprovision_loopback_ips
>> run_config_steps(lso_interaction(remove_config_from_router_dry))
>> run_config_steps(lso_interaction(remove_config_from_router_real))
diff --git a/gso/workflows/router/update_ibgp_mesh.py b/gso/workflows/router/update_ibgp_mesh.py
index bf0d0416..58d20702 100644
--- a/gso/workflows/router/update_ibgp_mesh.py
+++ b/gso/workflows/router/update_ibgp_mesh.py
@@ -12,12 +12,12 @@ from orchestrator.workflows.steps import resync, store_process_subscription, uns
from orchestrator.workflows.utils import wrap_modify_initial_input_form
from pydantic import ConfigDict, model_validator
+from gso.products.product_blocks.router import RouterRole
from gso.products.product_types.router import Router
-from gso.services import librenms_client, lso_client, subscriptions
+from gso.services import librenms_client, lso_client
from gso.services.lso_client import lso_interaction
from gso.services.subscriptions import get_trunks_that_terminate_on_router
-from gso.utils.helpers import SNMPVersion
-from gso.utils.workflow_steps import add_all_pe_to_p_dry, add_all_pe_to_p_real
+from gso.utils.helpers import SNMPVersion, generate_inventory_for_active_routers
def initial_input_form_generator(subscription_id: UUIDstr) -> FormGenerator:
@@ -62,7 +62,7 @@ def add_p_to_mesh_dry(subscription: dict[str, Any], callback_route: str, tt_numb
lso_client.execute_playbook(
playbook_name="update_ibgp_mesh.yaml",
callback_route=callback_route,
- inventory={"all": {"hosts": subscriptions.get_active_pe_router_dict()}},
+ inventory=generate_inventory_for_active_routers(RouterRole.PE),
extra_vars=extra_vars,
)
@@ -80,7 +80,46 @@ def add_p_to_mesh_real(subscription: dict[str, Any], callback_route: str, tt_num
lso_client.execute_playbook(
playbook_name="update_ibgp_mesh.yaml",
callback_route=callback_route,
- inventory={"all": {"hosts": subscriptions.get_active_pe_router_dict()}},
+ inventory=generate_inventory_for_active_routers(RouterRole.PE),
+ extra_vars=extra_vars,
+ )
+
+
+@step("[DRY RUN] Add all PE routers to P router iBGP group")
+def add_all_pe_to_p_dry(subscription: dict[str, Any], callback_route: str) -> None:
+ """Perform a dry run of adding the list of all PE routers to the new P router."""
+ extra_vars = {
+ "dry_run": True,
+ "subscription": subscription,
+ "pe_router_list": generate_inventory_for_active_routers(RouterRole.PE),
+ "verb": "add_pe_to_p",
+ }
+
+ lso_client.execute_playbook(
+ playbook_name="update_ibgp_mesh.yaml",
+ callback_route=callback_route,
+ inventory=subscription["router"]["router_fqdn"],
+ extra_vars=extra_vars,
+ )
+
+
+@step("[FOR REAL] Add all PE routers to P router iBGP group")
+def add_all_pe_to_p_real(
+ subscription: dict[str, Any], callback_route: str, tt_number: str, process_id: UUIDstr
+) -> None:
+ """Add the list of all PE routers to the new P router."""
+ extra_vars = {
+ "dry_run": False,
+ "subscription": subscription,
+ "pe_router_list": generate_inventory_for_active_routers(RouterRole.PE),
+ "commit_comment": f"GSO_PROCESS_ID: {process_id} - TT_NUMBER: {tt_number} - Update iBGP mesh",
+ "verb": "add_pe_to_p",
+ }
+
+ lso_client.execute_playbook(
+ playbook_name="update_ibgp_mesh.yaml",
+ callback_route=callback_route,
+ inventory=subscription["router"]["router_fqdn"],
extra_vars=extra_vars,
)
--
GitLab