"""A modification workflow that migrates an IP trunk to a different endpoint. For a trunk that originally connected endpoints A and B, this workflow introduces a new endpoint C. The trunk is then configured to run from A to C. B is then no longer associated with this IP trunk. """ import copy import json import re from uuid import uuid4 from orchestrator import step, workflow from orchestrator.config.assignee import Assignee from orchestrator.forms import FormPage from orchestrator.forms.validators import Choice, Label, UniqueConstrainedList from orchestrator.targets import Target from orchestrator.types import FormGenerator, State, UUIDstr from orchestrator.utils.errors import ProcessFailureError from orchestrator.utils.json import json_dumps from orchestrator.workflow import StepList, conditional, done, init, inputstep from orchestrator.workflows.steps import resync, store_process_subscription, unsync from orchestrator.workflows.utils import wrap_modify_initial_input_form from pydantic import validator from pydantic_forms.core import ReadOnlyField from pynetbox.models.dcim import Interfaces from gso.products.product_blocks.iptrunk import IptrunkInterfaceBlock from gso.products.product_types.iptrunk import Iptrunk from gso.products.product_types.router import Router from gso.services import infoblox from gso.services.infoblox import DeletionError from gso.services.netbox_client import NetboxClient from gso.services.provisioning_proxy import execute_playbook, pp_interaction from gso.services.subscriptions import get_active_router_subscriptions from gso.utils.helpers import ( LAGMember, available_interfaces_choices, available_lags_choices, get_router_vendor, validate_interface_name_list, validate_tt_number, ) from gso.utils.shared_enums import Vendor from gso.utils.workflow_steps import set_isis_to_max def initial_input_form_generator(subscription_id: UUIDstr) -> FormGenerator: """Gather input from the operator on the new router that the IP trunk should connect to.""" subscription = Iptrunk.from_subscription(subscription_id) form_title = ( f"Subscription {subscription.iptrunk.geant_s_sid} " f" from {subscription.iptrunk.iptrunk_sides[0].iptrunk_side_node.router_fqdn}" f" to {subscription.iptrunk.iptrunk_sides[1].iptrunk_side_node.router_fqdn}" ) sides_dict = { str(side.iptrunk_side_node.subscription.subscription_id): side.iptrunk_side_node.subscription.description for side in subscription.iptrunk.iptrunk_sides } replaced_side_enum = Choice( "Select the side of the IP trunk to be replaced", zip(sides_dict.keys(), sides_dict.items(), strict=True), # type: ignore[arg-type] ) class IPTrunkMigrateForm(FormPage): class Config: title = form_title tt_number: str replace_side: replaced_side_enum # type: ignore[valid-type] warning_label: Label = "Are we moving to a different Site?" # type: ignore[assignment] migrate_to_different_site: bool = False restore_isis_metric: bool = True @validator("tt_number", allow_reuse=True, pre=True, always=True) def validate_tt_number(cls, tt_number: str) -> str: return validate_tt_number(tt_number) migrate_form_input = yield IPTrunkMigrateForm current_routers = [ side.iptrunk_side_node.subscription.subscription_id for side in subscription.iptrunk.iptrunk_sides ] routers = {} for router in get_active_router_subscriptions(includes=["subscription_id", "description"]): router_id = router["subscription_id"] if router_id not in current_routers: current_router_site = Router.from_subscription(router_id).router.router_site.subscription old_side_site = Router.from_subscription(migrate_form_input.replace_side).router.router_site if ( not migrate_form_input.migrate_to_different_site and current_router_site.subscription_id != old_side_site.owner_subscription_id ): # We want to stay on the same site, so all routers that are in different sites get skipped. continue # If migrate_to_different_site is true, we can add ALL routers to the result map routers[str(router_id)] = router["description"] new_router_enum = Choice("Select a new router", zip(routers.keys(), routers.items(), strict=True)) # type: ignore[arg-type] class NewSideIPTrunkRouterForm(FormPage): class Config: title = form_title new_node: new_router_enum # type: ignore[valid-type] new_side_iptrunk_router_input = yield NewSideIPTrunkRouterForm new_router = new_side_iptrunk_router_input.new_node side_a_ae_iface = available_lags_choices(new_router) or str new_side_is_nokia = get_router_vendor(new_router) == Vendor.NOKIA if new_side_is_nokia: class NokiaLAGMember(LAGMember): interface_name: available_interfaces_choices( # type: ignore[valid-type] new_router, subscription.iptrunk.iptrunk_speed, ) class NokiaAeMembers(UniqueConstrainedList[NokiaLAGMember]): min_items = len(subscription.iptrunk.iptrunk_sides[0].iptrunk_side_ae_members) max_items = len(subscription.iptrunk.iptrunk_sides[1].iptrunk_side_ae_members) ae_members = NokiaAeMembers else: class JuniperLagMember(UniqueConstrainedList[LAGMember]): min_items = len(subscription.iptrunk.iptrunk_sides[0].iptrunk_side_ae_members) max_items = len(subscription.iptrunk.iptrunk_sides[1].iptrunk_side_ae_members) ae_members = JuniperLagMember # type: ignore[assignment] replace_index = ( 0 if str(subscription.iptrunk.iptrunk_sides[0].iptrunk_side_node.subscription.subscription_id) == migrate_form_input.replace_side else 1 ) existing_lag_ae_members = [ { "interface_name": iface.interface_name, "interface_description": iface.interface_description, } for iface in subscription.iptrunk.iptrunk_sides[replace_index].iptrunk_side_ae_members ] class NewSideIPTrunkForm(FormPage): class Config: title = form_title new_lag_interface: side_a_ae_iface # type: ignore[valid-type] existing_lag_interface: list[LAGMember] = ReadOnlyField(existing_lag_ae_members) new_lag_member_interfaces: ae_members # type: ignore[valid-type] @validator("new_lag_interface", allow_reuse=True, pre=True, always=True) def lag_interface_proper_name(cls, new_lag_interface: str) -> str: if get_router_vendor(new_router) == Vendor.JUNIPER: juniper_lag_re = re.compile("^ae\\d{1,2}$") if not juniper_lag_re.match(new_lag_interface): msg = "Invalid LAG name, please try again." raise ValueError(msg) return new_lag_interface @validator("new_lag_member_interfaces", allow_reuse=True, pre=True, always=True) def is_interface_names_valid_juniper(cls, new_lag_member_interfaces: list[LAGMember]) -> list[LAGMember]: vendor = get_router_vendor(new_router) return validate_interface_name_list(new_lag_member_interfaces, vendor) new_side_input = yield NewSideIPTrunkForm return ( migrate_form_input.dict() | new_side_iptrunk_router_input.dict() | new_side_input.dict() | {"replace_index": replace_index} ) @step("Netbox: Reserve new interfaces") def netbox_reserve_interfaces( subscription: Iptrunk, new_node: UUIDstr, new_lag_interface: str, new_lag_member_interfaces: list[dict] ) -> State: """Reserve new interfaces in Netbox, only when the new side's router is a NOKIA router.""" new_side = Router.from_subscription(new_node).router nbclient = NetboxClient() # Create :term:`LAG` interfaces lag_interface: Interfaces = nbclient.create_interface( iface_name=new_lag_interface, interface_type="lag", device_name=new_side.router_fqdn, description=str(subscription.subscription_id), enabled=True, ) # Attach physical interfaces to :term:`LAG` # Reserve interfaces for interface in new_lag_member_interfaces: nbclient.attach_interface_to_lag( device_name=new_side.router_fqdn, lag_name=lag_interface.name, iface_name=interface["interface_name"], description=str(subscription.subscription_id), ) nbclient.reserve_interface( device_name=new_side.router_fqdn, iface_name=interface["interface_name"], ) return {"subscription": subscription} @step("Calculate old side data") def calculate_old_side_data(subscription: Iptrunk, replace_index: int) -> State: """Store subscription information of the old side in the state of the workflow for later use.""" old_subscription = copy.deepcopy(subscription) old_side_data = { "iptrunk_side_node": old_subscription.iptrunk.iptrunk_sides[replace_index].iptrunk_side_node, "iptrunk_side_ae_iface": old_subscription.iptrunk.iptrunk_sides[replace_index].iptrunk_side_ae_iface, "iptrunk_side_ae_members": old_subscription.iptrunk.iptrunk_sides[replace_index].iptrunk_side_ae_members, } return {"old_side_data": old_side_data} @step("[DRY RUN] Disable configuration on old router") def disable_old_config_dry( subscription: Iptrunk, callback_route: str, new_node: Router, new_lag_interface: str, new_lag_member_interfaces: list[dict], replace_index: int, process_id: UUIDstr, tt_number: str, ) -> State: """Perform a dry run of disabling the old configuration on the routers.""" extra_vars = { "wfo_trunk_json": json.loads(json_dumps(subscription)), "new_node": json.loads(json_dumps(new_node)), "new_lag_interface": new_lag_interface, "new_lag_member_interfaces": new_lag_member_interfaces, "replace_index": replace_index, "verb": "deactivate", "config_object": "deactivate", "dry_run": True, "commit_comment": f"GSO_PROCESS_ID: {process_id} - TT_NUMBER: {tt_number} " f"- Deploy config for {subscription.iptrunk.geant_s_sid}", } execute_playbook( playbook_name="iptrunks_migration.yaml", callback_route=callback_route, inventory=f"{subscription.iptrunk.iptrunk_sides[0].iptrunk_side_node.router_fqdn}\n" f"{subscription.iptrunk.iptrunk_sides[1].iptrunk_side_node.router_fqdn}\n" f"{new_node.router.router_fqdn}\n", extra_vars=extra_vars, ) return {"subscription": subscription} @step("[REAL] Disable configuration on old router") def disable_old_config_real( subscription: Iptrunk, callback_route: str, new_node: Router, new_lag_interface: str, new_lag_member_interfaces: list[dict], replace_index: int, process_id: UUIDstr, tt_number: str, ) -> State: """Disable old configuration on the routers.""" extra_vars = { "wfo_trunk_json": json.loads(json_dumps(subscription)), "new_node": json.loads(json_dumps(new_node)), "new_lag_interface": new_lag_interface, "new_lag_member_interfaces": new_lag_member_interfaces, "replace_index": replace_index, "verb": "deactivate", "config_object": "deactivate", "dry_run": False, "commit_comment": f"GSO_PROCESS_ID: {process_id} - TT_NUMBER: {tt_number} " f"- Deploy config for {subscription.iptrunk.geant_s_sid}", } execute_playbook( playbook_name="iptrunks_migration.yaml", callback_route=callback_route, inventory=f"{subscription.iptrunk.iptrunk_sides[0].iptrunk_side_node.router_fqdn}\n" f"{subscription.iptrunk.iptrunk_sides[1].iptrunk_side_node.router_fqdn}\n" f"{new_node.router.router_fqdn}\n", extra_vars=extra_vars, ) return { "subscription": subscription, } @step("[DRY RUN] Deploy configuration on new router") def deploy_new_config_dry( subscription: Iptrunk, callback_route: str, new_node: Router, new_lag_interface: str, new_lag_member_interfaces: list[dict], replace_index: int, process_id: UUIDstr, tt_number: str, ) -> State: """Perform a dry run of deploying configuration on the new router.""" extra_vars = { "wfo_trunk_json": json.loads(json_dumps(subscription)), "new_node": json.loads(json_dumps(new_node)), "new_lag_interface": new_lag_interface, "new_lag_member_interfaces": new_lag_member_interfaces, "replace_index": replace_index, "verb": "deploy", "config_object": "trunk_interface", "dry_run": True, "commit_comment": f"GSO_PROCESS_ID: {process_id} - TT_NUMBER: {tt_number} " f"- Deploy config for {subscription.iptrunk.geant_s_sid}", } execute_playbook( playbook_name="iptrunks_migration.yaml", callback_route=callback_route, inventory=f"{subscription.iptrunk.iptrunk_sides[0].iptrunk_side_node.router_fqdn}\n" f"{subscription.iptrunk.iptrunk_sides[1].iptrunk_side_node.router_fqdn}\n" f"{new_node.router.router_fqdn}\n", extra_vars=extra_vars, ) return {"subscription": subscription} @step("Deploy configuration on new router") def deploy_new_config_real( subscription: Iptrunk, callback_route: str, new_node: Router, new_lag_interface: str, new_lag_member_interfaces: list[dict], replace_index: int, process_id: UUIDstr, tt_number: str, ) -> State: """Deploy configuration on the new router.""" extra_vars = { "wfo_trunk_json": json.loads(json_dumps(subscription)), "new_node": json.loads(json_dumps(new_node)), "new_lag_interface": new_lag_interface, "new_lag_member_interfaces": new_lag_member_interfaces, "replace_index": replace_index, "verb": "deploy", "config_object": "trunk_interface", "dry_run": False, "commit_comment": f"GSO_PROCESS_ID: {process_id} - TT_NUMBER: {tt_number} " f"- Deploy config for {subscription.iptrunk.geant_s_sid}", } execute_playbook( playbook_name="iptrunks_migration.yaml", callback_route=callback_route, inventory=f"{subscription.iptrunk.iptrunk_sides[0].iptrunk_side_node.router_fqdn}\n" f"{subscription.iptrunk.iptrunk_sides[1].iptrunk_side_node.router_fqdn}\n" f"{new_node.router.router_fqdn}\n", extra_vars=extra_vars, ) return {"subscription": subscription} @inputstep("Wait for confirmation", assignee=Assignee.SYSTEM) def confirm_continue_move_fiber() -> FormGenerator: """Wait for confirmation from an operator that the physical fiber has been moved.""" class ProvisioningResultPage(FormPage): class Config: title = "Please confirm before continuing" info_label: Label = "New trunk interface has been deployed, wait for the physical connection to be moved." # type: ignore[assignment] yield ProvisioningResultPage return {} @step("Deploy ISIS configuration on new router") def deploy_new_isis( subscription: Iptrunk, callback_route: str, new_node: Router, new_lag_interface: str, new_lag_member_interfaces: list[dict], replace_index: int, process_id: UUIDstr, tt_number: str, ) -> State: """Deploy :term:`ISIS` configuration.""" extra_vars = { "wfo_trunk_json": json.loads(json_dumps(subscription)), "new_node": json.loads(json_dumps(new_node)), "new_lag_interface": new_lag_interface, "new_lag_member_interfaces": new_lag_member_interfaces, "replace_index": replace_index, "verb": "deploy", "config_object": "isis_interface", "dry_run": False, "commit_comment": f"GSO_PROCESS_ID: {process_id} - TT_NUMBER: {tt_number} " f"- Deploy config for {subscription.iptrunk.geant_s_sid}", } execute_playbook( playbook_name="iptrunks_migration.yaml", callback_route=callback_route, inventory=f"{subscription.iptrunk.iptrunk_sides[0].iptrunk_side_node.router_fqdn}\n" f"{subscription.iptrunk.iptrunk_sides[1].iptrunk_side_node.router_fqdn}\n" f"{new_node.router.router_fqdn}\n", extra_vars=extra_vars, ) return {"subscription": subscription} @inputstep("Wait for confirmation", assignee=Assignee.SYSTEM) def confirm_continue_restore_isis() -> FormGenerator: """Wait for an operator to confirm that the old :term:`ISIS` metric should be restored.""" class ProvisioningResultPage(FormPage): class Config: title = "Please confirm before continuing" info_label: Label = "ISIS config has been deployed, confirm if you want to restore the old metric." # type: ignore[assignment] yield ProvisioningResultPage return {} @step("Restore ISIS metric to original value") def restore_isis_metric( subscription: Iptrunk, process_id: UUIDstr, callback_route: str, tt_number: str, old_isis_metric: int, ) -> State: """Restore the :term:`ISIS` metric to its original value.""" subscription.iptrunk.iptrunk_isis_metric = old_isis_metric extra_vars = { "wfo_trunk_json": json.loads(json_dumps(subscription)), "dry_run": False, "verb": "deploy", "config_object": "isis_interface", "commit_comment": f"GSO_PROCESS_ID: {process_id} - TT_NUMBER: {tt_number} - Deploy config for " f"{subscription.iptrunk.geant_s_sid}", } execute_playbook( playbook_name="iptrunks.yaml", callback_route=callback_route, inventory=f"{subscription.iptrunk.iptrunk_sides[0].iptrunk_side_node.router_fqdn}\n" f"{subscription.iptrunk.iptrunk_sides[1].iptrunk_side_node.router_fqdn}\n", extra_vars=extra_vars, ) return {"subscription": subscription} @step("[DRY RUN] Delete configuration on old router") def delete_old_config_dry( subscription: Iptrunk, callback_route: str, new_node: Router, new_lag_interface: str, new_lag_member_interfaces: list[dict], replace_index: int, process_id: UUIDstr, tt_number: str, ) -> State: """Perform a dry run of deleting the old configuration.""" extra_vars = { "wfo_trunk_json": json.loads(json_dumps(subscription)), "new_node": json.loads(json_dumps(new_node)), "new_lag_interface": new_lag_interface, "new_lag_member_interfaces": new_lag_member_interfaces, "replace_index": replace_index, "verb": "delete", "config_object": "delete", "dry_run": True, "commit_comment": f"GSO_PROCESS_ID: {process_id} - TT_NUMBER: {tt_number} " f"- Deploy config for {subscription.iptrunk.geant_s_sid}", } execute_playbook( playbook_name="iptrunks_migration.yaml", callback_route=callback_route, inventory=f"{subscription.iptrunk.iptrunk_sides[0].iptrunk_side_node.router_fqdn}\n" f"{subscription.iptrunk.iptrunk_sides[1].iptrunk_side_node.router_fqdn}\n" f"{new_node.router.router_fqdn}\n", extra_vars=extra_vars, ) return {"subscription": subscription} @step("Delete configuration on old router") def delete_old_config_real( subscription: Iptrunk, callback_route: str, new_node: Router, new_lag_interface: str, new_lag_member_interfaces: list[dict], replace_index: int, process_id: UUIDstr, tt_number: str, ) -> State: """Delete old configuration from the routers.""" extra_vars = { "wfo_trunk_json": json.loads(json_dumps(subscription)), "new_node": json.loads(json_dumps(new_node)), "new_lag_interface": new_lag_interface, "new_lag_member_interfaces": new_lag_member_interfaces, "replace_index": replace_index, "verb": "delete", "config_object": "delete", "dry_run": False, "commit_comment": f"GSO_PROCESS_ID: {process_id} - TT_NUMBER: {tt_number} " f"- Deploy config for {subscription.iptrunk.geant_s_sid}", } execute_playbook( playbook_name="iptrunks_migration.yaml", callback_route=callback_route, inventory=f"{subscription.iptrunk.iptrunk_sides[0].iptrunk_side_node.router_fqdn}\n" f"{subscription.iptrunk.iptrunk_sides[1].iptrunk_side_node.router_fqdn}\n" f"{new_node.router.router_fqdn}\n", extra_vars=extra_vars, ) return {"subscription": subscription} @step("Update IP records in IPAM") def update_ipam(subscription: Iptrunk, old_side_data: dict, new_node: Router, new_lag_interface: str) -> State: """Update :term:`IPAM` resources. Move the DNS record pointing to the old side of the trunk, to the new side. """ old_fqdn = f"{old_side_data['iptrunk_side_ae_iface']}.{old_side_data['iptrunk_side_node']['router_fqdn']}" trunk_v4 = infoblox.find_host_by_fqdn(old_fqdn) trunk_v6 = infoblox.find_v6_host_by_fqdn(old_fqdn) # Out with the old try: infoblox.delete_host_by_fqdn(old_fqdn) except DeletionError as e: msg = "Failed to delete record from Infoblox." raise ProcessFailureError(msg) from e # And in with the new new_fqdn = f"{new_lag_interface}.{new_node.router.router_fqdn}" comment = str(subscription.subscription_id) infoblox.create_host_by_ip(new_fqdn, trunk_v4.ipv4addr, trunk_v6.ipv6addr, service_type="TRUNK", comment=comment) return {"subscription": subscription} @step("Update subscription model") def update_subscription_model( subscription: Iptrunk, replace_index: int, new_node: UUIDstr, new_lag_interface: str, new_lag_member_interfaces: list[dict], ) -> State: """Update the subscription model in the database.""" # Deep copy of subscription data subscription.iptrunk.iptrunk_sides[replace_index].iptrunk_side_node = Router.from_subscription(new_node).router subscription.iptrunk.iptrunk_sides[replace_index].iptrunk_side_ae_iface = new_lag_interface subscription.iptrunk.iptrunk_sides[replace_index].iptrunk_side_ae_members.clear() # And update the list to only include the new member interfaces for member in new_lag_member_interfaces: subscription.iptrunk.iptrunk_sides[0].iptrunk_side_ae_members.append( IptrunkInterfaceBlock.new(subscription_id=uuid4(), **member), ) return {"subscription": subscription} @step("Netbox: Remove old LAG interface") def netbox_remove_old_interfaces(old_side_data: dict) -> State: """Remove the old :term:`LAG` interface from Netbox, only relevant if the old side is a Nokia router.""" nbclient = NetboxClient() for iface in old_side_data["iptrunk_side_ae_members"]: nbclient.free_interface( old_side_data["iptrunk_side_node"]["router_fqdn"], iface["interface_name"], ) nbclient.delete_interface( old_side_data["iptrunk_side_node"]["router_fqdn"], old_side_data["iptrunk_side_ae_iface"], ) return {} @step("Netbox: Allocate new LAG member interfaces") def netbox_allocate_new_interfaces(subscription: Iptrunk, replace_index: int) -> State: """Allocate the new :term:`LAG` interface in Netbox. Only relevant if the new router is a Nokia.""" nbclient = NetboxClient() new_side = subscription.iptrunk.iptrunk_sides[replace_index] for interface in new_side.iptrunk_side_ae_members: nbclient.allocate_interface( device_name=new_side.iptrunk_side_node.router_fqdn, iface_name=interface.interface_name, ) return {"subscription": subscription} @workflow( "Migrate an IP Trunk", initial_input_form=wrap_modify_initial_input_form(initial_input_form_generator), target=Target.MODIFY, ) def migrate_iptrunk() -> StepList: """Migrate an IP trunk. * Reserve new interfaces in Netbox * Set the :term:`ISIS` metric of the current trunk to an arbitrarily high value to drain all traffic * Disable - but do not delete - the old configuration on the routers, first as a dry run * Deploy the new configuration on the routers, first as a dry run * Wait for operator confirmation that the physical fiber has been moved before continuing * Deploy a new :term:`ISIS` interface between routers A and C * Wait for operator confirmation that :term:`ISIS` is behaving as expected * Restore the old :term:`ISIS` metric on the new trunk * Delete the old, disabled configuration on the routers, first as a dry run * Reflect the changes made in :term:`IPAM` * Update the subscription model in the database * Update the reserved interfaces in Netbox """ new_side_is_nokia = conditional(lambda state: get_router_vendor(state["new_node"]) == Vendor.NOKIA) old_side_is_nokia = conditional( lambda state: get_router_vendor(state["old_side_data"]["iptrunk_side_node"]["owner_subscription_id"]) == Vendor.NOKIA ) should_restore_isis_metric = conditional(lambda state: state["restore_isis_metric"]) return ( init >> store_process_subscription(Target.MODIFY) >> unsync >> new_side_is_nokia(netbox_reserve_interfaces) >> calculate_old_side_data >> pp_interaction(set_isis_to_max) >> pp_interaction(disable_old_config_dry) >> pp_interaction(disable_old_config_real) >> pp_interaction(deploy_new_config_dry) >> pp_interaction(deploy_new_config_real) >> confirm_continue_move_fiber >> pp_interaction(deploy_new_isis) >> should_restore_isis_metric(confirm_continue_restore_isis) >> should_restore_isis_metric(pp_interaction(restore_isis_metric)) >> pp_interaction(delete_old_config_dry) >> pp_interaction(delete_old_config_real) >> update_ipam >> update_subscription_model >> old_side_is_nokia(netbox_remove_old_interfaces) >> new_side_is_nokia(netbox_allocate_new_interfaces) >> resync >> done )