From 29d1d7773fc3b786858be80c97ca216ec6ae51b5 Mon Sep 17 00:00:00 2001 From: Mohammad Torkashvand <mohammad.torkashvand@geant.org> Date: Tue, 29 Aug 2023 10:09:44 +0200 Subject: [PATCH] added iptrunk import api --- .gitlab-ci.yml | 2 +- gso/api/__init__.py | 7 + gso/api/api_v1/api.py | 10 - gso/api/api_v1/endpoints/__init__.py | 0 gso/api/v1/__init__.py | 7 + gso/api/{api_v1/endpoints => v1}/imports.py | 79 ++++---- gso/main.py | 2 +- gso/products/__init__.py | 2 + gso/products/product_blocks/__init__.py | 16 +- gso/repository.py | 61 ++++++ gso/{api/api_v1 => schemas}/__init__.py | 0 gso/schemas/enums.py | 11 + gso/schemas/imports.py | 123 +++++++++++ gso/services/crm.py | 6 +- gso/services/subscriptions.py | 30 --- gso/workflows/__init__.py | 1 + gso/workflows/iptrunk/create_iptrunk.py | 12 +- gso/workflows/router/create_router.py | 13 +- gso/workflows/tasks/import_iptrunk.py | 103 ++++++++++ gso/workflows/tasks/import_router.py | 26 ++- gso/workflows/tasks/import_site.py | 12 +- requirements.txt | 1 - test/conftest.py | 146 ++++++++++++++ test/test_imports.py | 8 +- test/test_imports_iptrunk.py | 213 ++++++++++++++++++++ 25 files changed, 757 insertions(+), 134 deletions(-) delete mode 100644 gso/api/api_v1/api.py delete mode 100644 gso/api/api_v1/endpoints/__init__.py create mode 100644 gso/api/v1/__init__.py rename gso/api/{api_v1/endpoints => v1}/imports.py (55%) create mode 100644 gso/repository.py rename gso/{api/api_v1 => schemas}/__init__.py (100%) create mode 100644 gso/schemas/enums.py create mode 100644 gso/schemas/imports.py delete mode 100644 gso/services/subscriptions.py create mode 100644 gso/workflows/tasks/import_iptrunk.py create mode 100644 test/test_imports_iptrunk.py diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 4aa7c821..e93f467b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -11,7 +11,7 @@ run-tox-pipeline: stage: tox tags: - docker-executor - image: python:3.10 + image: python:3.11 services: - postgres:15.4 diff --git a/gso/api/__init__.py b/gso/api/__init__.py index e69de29b..f30090d3 100644 --- a/gso/api/__init__.py +++ b/gso/api/__init__.py @@ -0,0 +1,7 @@ +from fastapi import APIRouter + +from gso.api.v1 import router as router_v1 + +router = APIRouter() + +router.include_router(router_v1, prefix="/v1") diff --git a/gso/api/api_v1/api.py b/gso/api/api_v1/api.py deleted file mode 100644 index b7ba2d52..00000000 --- a/gso/api/api_v1/api.py +++ /dev/null @@ -1,10 +0,0 @@ -"""Module that implements process related API endpoints.""" - -from fastapi.param_functions import Depends -from fastapi.routing import APIRouter -from orchestrator.security import opa_security_default - -from gso.api.api_v1.endpoints import imports - -api_router = APIRouter() -api_router.include_router(imports.router, prefix="/imports", dependencies=[Depends(opa_security_default)]) diff --git a/gso/api/api_v1/endpoints/__init__.py b/gso/api/api_v1/endpoints/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/gso/api/v1/__init__.py b/gso/api/v1/__init__.py new file mode 100644 index 00000000..89fd2c8e --- /dev/null +++ b/gso/api/v1/__init__.py @@ -0,0 +1,7 @@ +from fastapi import APIRouter + +from gso.api.v1.imports import router as imports_router + +router = APIRouter() + +router.include_router(imports_router) diff --git a/gso/api/api_v1/endpoints/imports.py b/gso/api/v1/imports.py similarity index 55% rename from gso/api/api_v1/endpoints/imports.py rename to gso/api/v1/imports.py index c83b0919..ae2b7c7d 100644 --- a/gso/api/api_v1/endpoints/imports.py +++ b/gso/api/v1/imports.py @@ -1,20 +1,18 @@ -import ipaddress -from typing import Any, Dict, Optional +from typing import Any, Dict from uuid import UUID -from fastapi import HTTPException, status +from fastapi import Depends, HTTPException, status from fastapi.routing import APIRouter +from orchestrator.security import opa_security_default from orchestrator.services import processes, subscriptions -from pydantic import BaseModel from sqlalchemy.exc import MultipleResultsFound -from gso.products.product_blocks.router import RouterRole, RouterVendor -from gso.products.product_blocks.site import SiteTier +from gso.schemas.imports import ImportResponseModel, IptrunkImportModel, RouterImportModel, SiteImportModel -router = APIRouter() +router = APIRouter(prefix="/imports", tags=["Imports"], dependencies=[Depends(opa_security_default)]) -def start_process(process_name: str, data: dict) -> UUID: +def _start_process(process_name: str, data: dict) -> UUID: """Start a process and handle common exceptions.""" pid: UUID = processes.start_process(process_name, [data]) @@ -31,27 +29,13 @@ def start_process(process_name: str, data: dict) -> UUID: return pid -class SiteImport(BaseModel): - site_name: str - site_city: str - site_country: str - site_country_code: str - site_latitude: float - site_longitude: float - site_bgp_community_id: int - site_internal_id: int - site_tier: SiteTier - site_ts_address: str - customer: str - - -@router.post("/sites", status_code=status.HTTP_201_CREATED, tags=["Import"]) -def import_site(site: SiteImport) -> Dict[str, Any]: - """Import site by running the import_site workflow. +@router.post("/sites", status_code=status.HTTP_201_CREATED, response_model=ImportResponseModel) +def import_site(site: SiteImportModel) -> Dict[str, Any]: + """Import a site by running the import_site workflow. Args: ---- - site (SiteImport): The site information to be imported. + site (SiteImportModel): The site information to be imported. Returns: ------- @@ -70,27 +54,11 @@ def import_site(site: SiteImport) -> Dict[str, Any]: except MultipleResultsFound: raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail="Multiple subscriptions found.") - pid = start_process("import_site", site.dict()) + pid = _start_process("import_site", site.dict()) return {"detail": "Site added successfully.", "pid": pid} -class RouterImportModel(BaseModel): - customer: str - router_site: str - hostname: str - ts_port: int - router_vendor: RouterVendor - router_role: RouterRole - is_ias_connected: bool - router_lo_ipv4_address: ipaddress.IPv4Address - router_lo_ipv6_address: ipaddress.IPv6Address - router_lo_iso_address: str - router_si_ipv4_network: Optional[ipaddress.IPv4Network] = None - router_ias_lt_ipv4_network: Optional[ipaddress.IPv4Network] = None - router_ias_lt_ipv6_network: Optional[ipaddress.IPv6Network] = None - - -@router.post("/routers", status_code=status.HTTP_201_CREATED, tags=["Import"]) +@router.post("/routers", status_code=status.HTTP_201_CREATED, response_model=ImportResponseModel) def import_router(router_data: RouterImportModel) -> Dict[str, Any]: """Import a router by running the import_router workflow. @@ -107,5 +75,26 @@ def import_router(router_data: RouterImportModel) -> Dict[str, Any]: HTTPException: If there's an error in the process. """ - pid = start_process("import_router", router_data.dict()) + pid = _start_process("import_router", router_data.dict()) return {"detail": "Router added successfully", "pid": pid} + + +@router.post("/iptrunks", status_code=status.HTTP_201_CREATED, response_model=ImportResponseModel) +def import_iptrunk(iptrunk_data: IptrunkImportModel) -> Dict[str, Any]: + """Import an iptrunk by running the import_iptrunk workflow. + + Args: + ---- + iptrunk_data (IptrunkImportModel): The iptrunk information to be imported. + + Returns: + ------- + dict: A dictionary containing the process id of the started process and detail message. + + Raises: + ------ + HTTPException: If there's an error in the process. + """ + + pid = _start_process("import_iptrunk", iptrunk_data.dict()) + return {"detail": "Iptrunk added successfully", "pid": pid} diff --git a/gso/main.py b/gso/main.py index 9c8223bb..e05aac88 100644 --- a/gso/main.py +++ b/gso/main.py @@ -7,7 +7,7 @@ from orchestrator.settings import AppSettings import gso.products # noqa: F401 import gso.workflows # noqa: F401 from gso import load_gso_cli -from gso.api.api_v1.api import api_router +from gso.api import router as api_router def init_gso_app(settings: AppSettings) -> OrchestratorCore: diff --git a/gso/products/__init__.py b/gso/products/__init__.py index f0f4872e..16da3f77 100644 --- a/gso/products/__init__.py +++ b/gso/products/__init__.py @@ -12,3 +12,5 @@ SUBSCRIPTION_MODEL_REGISTRY.update( "IP trunk": Iptrunk, } ) + +__all__ = ["Site", "Iptrunk", "Router"] diff --git a/gso/products/product_blocks/__init__.py b/gso/products/product_blocks/__init__.py index 0dcc0060..d4c0aa27 100644 --- a/gso/products/product_blocks/__init__.py +++ b/gso/products/product_blocks/__init__.py @@ -3,20 +3,16 @@ In this file, some enumerators may be declared that are available for use across all subscriptions. """ -from enum import Enum +from enum import StrEnum -class PhyPortCapacity(Enum): +class PhyPortCapacity(StrEnum): """Physical port capacity enumerator. An enumerator that has the different possible capacities of ports that are available to use in subscriptions. """ - ONE = "1G" - """1Gbps""" - TEN = "10G" - """10Gbps""" - HUNDRED = "100G" - """100Gbps""" - FOUR_HUNDRED = "400G" - """400Gbps""" + ONE_GIGABIT_PER_SECOND = "1G" + TEN_GIGABIT_PER_SECOND = "10G" + HUNDRED_GIGABIT_PER_SECOND = "100G" + FOUR_HUNDRED_GIGABIT_PER_SECOND = "400G" diff --git a/gso/repository.py b/gso/repository.py new file mode 100644 index 00000000..68d6807b --- /dev/null +++ b/gso/repository.py @@ -0,0 +1,61 @@ +from uuid import UUID + +from asyncio_redis import Subscription +from orchestrator.db import ( + ProductTable, + ResourceTypeTable, + SubscriptionInstanceTable, + SubscriptionInstanceValueTable, + SubscriptionTable, +) + +from gso.schemas.enums import ProductType, SubscriptionStatus + + +def all_active_subscriptions( + product_type: str, + fields: list[str], +) -> list[Subscription]: + dynamic_fields = [getattr(SubscriptionTable, field) for field in fields] + + return ( + SubscriptionTable.query.join(ProductTable) + .filter( + ProductTable.product_type == product_type, + SubscriptionTable.status == SubscriptionStatus.ACTIVE, + ) + .with_entities(*dynamic_fields) + .all() + ) + + +def all_active_site_subscriptions(fields: list[str]) -> list[Subscription]: + return all_active_subscriptions(ProductType.SITE, fields) + + +def site_product_id() -> UUID: + return ProductTable.query.filter_by(name=ProductType.SITE).first().product_id + + +def active_site_subscription_by_name(site_name: str) -> Subscription: + return ( + SubscriptionTable.query.join( + ProductTable, SubscriptionInstanceTable, SubscriptionInstanceValueTable, ResourceTypeTable + ) + .filter(SubscriptionInstanceValueTable.value == site_name) + .filter(ResourceTypeTable.resource_type == "site_name") + .filter(SubscriptionTable.status == SubscriptionStatus.ACTIVE) + .first() + ) + + +def iptrunk_product_id() -> UUID: + return ProductTable.query.filter_by(name=ProductType.IP_TRUNK).first().product_id + + +def all_active_router_subscriptions(fields: list[str]) -> list[Subscription]: + return all_active_subscriptions(product_type=ProductType.ROUTER, fields=fields) + + +def router_product_id() -> UUID: + return ProductTable.query.filter_by(name=ProductType.ROUTER).first().product_id diff --git a/gso/api/api_v1/__init__.py b/gso/schemas/__init__.py similarity index 100% rename from gso/api/api_v1/__init__.py rename to gso/schemas/__init__.py diff --git a/gso/schemas/enums.py b/gso/schemas/enums.py new file mode 100644 index 00000000..248e87eb --- /dev/null +++ b/gso/schemas/enums.py @@ -0,0 +1,11 @@ +from enum import StrEnum + + +class ProductType(StrEnum): + SITE = "Site" + ROUTER = "Router" + IP_TRUNK = "IP trunk" + + +class SubscriptionStatus(StrEnum): + ACTIVE = "active" diff --git a/gso/schemas/imports.py b/gso/schemas/imports.py new file mode 100644 index 00000000..ae8a84cb --- /dev/null +++ b/gso/schemas/imports.py @@ -0,0 +1,123 @@ +import ipaddress +from typing import Any +from uuid import UUID + +from pydantic import BaseModel, root_validator, validator + +from gso import repository +from gso.products.product_blocks import PhyPortCapacity +from gso.products.product_blocks.iptrunk import IptrunkType +from gso.products.product_blocks.router import RouterRole, RouterVendor +from gso.products.product_blocks.site import SiteTier +from gso.services.crm import CustomerNotFoundError, get_customer_id_by_name + + +class ImportResponseModel(BaseModel): + pid: UUID + detail: str + + +class SiteImportModel(BaseModel): + site_name: str + site_city: str + site_country: str + site_country_code: str + site_latitude: float + site_longitude: float + site_bgp_community_id: int + site_internal_id: int + site_tier: SiteTier + site_ts_address: str + customer: str + + +class RouterImportModel(BaseModel): + customer: str + router_site: str + hostname: str + ts_port: int + router_vendor: RouterVendor + router_role: RouterRole + is_ias_connected: bool + router_lo_ipv4_address: ipaddress.IPv4Address + router_lo_ipv6_address: ipaddress.IPv6Address + router_lo_iso_address: str + router_si_ipv4_network: ipaddress.IPv4Network | None = None + router_ias_lt_ipv4_network: ipaddress.IPv4Network | None = None + router_ias_lt_ipv6_network: ipaddress.IPv6Network | None = None + + +class IptrunkImportModel(BaseModel): + customer: str + geant_s_sid: str + iptrunk_type: IptrunkType + iptrunk_description: str + iptrunk_speed: PhyPortCapacity + iptrunk_minimum_links: int + iptrunk_sideA_node_id: str + iptrunk_sideA_ae_iface: str + iptrunk_sideA_ae_geant_a_sid: str + iptrunk_sideA_ae_members: list[str] + iptrunk_sideA_ae_members_descriptions: list[str] + iptrunk_sideB_node_id: str + iptrunk_sideB_ae_iface: str + iptrunk_sideB_ae_geant_a_sid: str + iptrunk_sideB_ae_members: list[str] + iptrunk_sideB_ae_members_descriptions: list[str] + + iptrunk_ipv4_network: ipaddress.IPv4Network + iptrunk_ipv6_network: ipaddress.IPv6Network + + @classmethod + def _get_active_routers(cls) -> set[str]: + return {str(router_id) for router_id in repository.all_active_router_subscriptions(fields=["subscription_id"])} + + @validator("customer") + def check_if_customer_exists(cls, value: str) -> str: + try: + get_customer_id_by_name(value) + except CustomerNotFoundError: + raise ValueError(f"Customer {value} not found") + + return value + + @validator("iptrunk_sideA_node_id", "iptrunk_sideB_node_id") + def check_if_router_side_is_available(cls, value: str) -> str: + if value not in cls._get_active_routers(): + raise ValueError("Router not found") + + return value + + @validator("iptrunk_sideA_ae_members", "iptrunk_sideB_ae_members") + def check_side_uniqueness(cls, value: list[str]) -> list[str]: + if len(value) != len(set(value)): + raise ValueError("Items must be unique") + + return value + + @root_validator + def check_members(cls, values: dict[str, Any]) -> dict[str, Any]: + min_links = values["iptrunk_minimum_links"] + side_a_members = values.get("iptrunk_sideA_ae_members", []) + side_a_descriptions = values.get("iptrunk_sideA_ae_members_descriptions", []) + side_b_members = values.get("iptrunk_sideB_ae_members", []) + side_b_descriptions = values.get("iptrunk_sideB_ae_members_descriptions", []) + + len_a = len(side_a_members) + len_a_desc = len(side_a_descriptions) + len_b = len(side_b_members) + len_b_desc = len(side_b_descriptions) + + if len_a < min_links: + raise ValueError(f"Side A members should be at least {min_links} (iptrunk_minimum_links)") + + if len_a != len_a_desc: + raise ValueError("Mismatch in Side A members and their descriptions") + + if len_a != len_b: + raise ValueError("Mismatch between Side A and B members") + + if len_a != len_b_desc: + raise ValueError("Mismatch in Side B members and their descriptions") + + return values diff --git a/gso/services/crm.py b/gso/services/crm.py index 92fb150d..7e4cc9b8 100644 --- a/gso/services/crm.py +++ b/gso/services/crm.py @@ -11,7 +11,7 @@ def all_customers() -> list[dict]: return [ { "id": "8f0df561-ce9d-4d9c-89a8-7953d3ffc961", - "name": "Geant", + "name": "GÉANT", }, ] @@ -22,3 +22,7 @@ def get_customer_by_name(name: str) -> Dict[str, Any]: return customer raise CustomerNotFoundError(f"Customer {name} not found") + + +def get_customer_id_by_name(name: str) -> str: + return get_customer_by_name(name)["id"] diff --git a/gso/services/subscriptions.py b/gso/services/subscriptions.py deleted file mode 100644 index 3cfbd16c..00000000 --- a/gso/services/subscriptions.py +++ /dev/null @@ -1,30 +0,0 @@ -from orchestrator.db import ( - ProductTable, - ResourceTypeTable, - SubscriptionInstanceTable, - SubscriptionInstanceValueTable, - SubscriptionTable, -) - -from gso.products.product_types.site import Site - - -def get_site_by_name(site_name: str) -> Site: - """Get a site by its name. - - Args: - ---- - site_name (str): The name of the site. - """ - subscription = ( - SubscriptionTable.query.join( - ProductTable, SubscriptionInstanceTable, SubscriptionInstanceValueTable, ResourceTypeTable - ) - .filter(SubscriptionInstanceValueTable.value == site_name) - .filter(ResourceTypeTable.resource_type == "site_name") - .filter(SubscriptionTable.status == "active") - .first() - ) - if not subscription: - raise ValueError(f"Site with name {site_name} not found.") - return Site.from_subscription(subscription.subscription_id) diff --git a/gso/workflows/__init__.py b/gso/workflows/__init__.py index afd2d56a..b578903d 100644 --- a/gso/workflows/__init__.py +++ b/gso/workflows/__init__.py @@ -10,3 +10,4 @@ LazyWorkflowInstance("gso.workflows.iptrunk.modify_isis_metric", "modify_isis_me LazyWorkflowInstance("gso.workflows.site.create_site", "create_site") LazyWorkflowInstance("gso.workflows.tasks.import_site", "import_site") LazyWorkflowInstance("gso.workflows.tasks.import_router", "import_router") +LazyWorkflowInstance("gso.workflows.tasks.import_iptrunk", "import_iptrunk") diff --git a/gso/workflows/iptrunk/create_iptrunk.py b/gso/workflows/iptrunk/create_iptrunk.py index 0f25f16e..5e40775a 100644 --- a/gso/workflows/iptrunk/create_iptrunk.py +++ b/gso/workflows/iptrunk/create_iptrunk.py @@ -1,4 +1,3 @@ -from orchestrator.db.models import ProductTable, SubscriptionTable from orchestrator.forms import FormPage from orchestrator.forms.validators import Choice, UniqueConstrainedList from orchestrator.targets import Target @@ -7,6 +6,7 @@ from orchestrator.workflow import StepList, done, init, step, workflow from orchestrator.workflows.steps import resync, set_status, store_process_subscription from orchestrator.workflows.utils import wrap_create_initial_input_form +from gso import repository from gso.products.product_blocks import PhyPortCapacity from gso.products.product_blocks.iptrunk import IptrunkType from gso.products.product_types.iptrunk import IptrunkInactive, IptrunkProvisioning @@ -21,14 +21,8 @@ def initial_input_form_generator(product_name: str) -> FormGenerator: # * interface names must be validated routers = {} - for router_id, router_description in ( - SubscriptionTable.query.join(ProductTable) - .filter( - ProductTable.product_type == "Router", - SubscriptionTable.status == "active", - ) - .with_entities(SubscriptionTable.subscription_id, SubscriptionTable.description) - .all() + for router_id, router_description in repository.all_active_router_subscriptions( + fields=["subscription_id", "description"] ): routers[str(router_id)] = router_description diff --git a/gso/workflows/router/create_router.py b/gso/workflows/router/create_router.py index 8d932545..44793629 100644 --- a/gso/workflows/router/create_router.py +++ b/gso/workflows/router/create_router.py @@ -1,8 +1,6 @@ import ipaddress import re -from orchestrator.db.models import ProductTable, SubscriptionTable - # noinspection PyProtectedMember from orchestrator.forms import FormPage from orchestrator.forms.validators import Choice @@ -12,6 +10,7 @@ from orchestrator.workflow import StepList, done, init, step, workflow from orchestrator.workflows.steps import resync, set_status, store_process_subscription from orchestrator.workflows.utils import wrap_create_initial_input_form +from gso import repository from gso.products.product_blocks import router as router_pb from gso.products.product_types import router from gso.products.product_types.router import RouterInactive, RouterProvisioning @@ -23,14 +22,8 @@ from gso.workflows.utils import customer_selector def site_selector() -> Choice: site_subscriptions = {} - for site_id, site_description in ( - SubscriptionTable.query.join(ProductTable) - .filter( - ProductTable.product_type == "Site", - SubscriptionTable.status == "active", - ) - .with_entities(SubscriptionTable.subscription_id, SubscriptionTable.description) - .all() + for site_id, site_description in repository.all_active_site_subscriptions( + fields=["subscription_id", "description"] ): site_subscriptions[str(site_id)] = site_description diff --git a/gso/workflows/tasks/import_iptrunk.py b/gso/workflows/tasks/import_iptrunk.py new file mode 100644 index 00000000..ab1642ca --- /dev/null +++ b/gso/workflows/tasks/import_iptrunk.py @@ -0,0 +1,103 @@ +import ipaddress + +from orchestrator import workflow +from orchestrator.forms import FormPage +from orchestrator.forms.validators import Choice, UniqueConstrainedList +from orchestrator.targets import Target +from orchestrator.types import FormGenerator, State, SubscriptionLifecycle +from orchestrator.workflow import StepList, done, init, step +from orchestrator.workflows.steps import resync, set_status, store_process_subscription + +from gso import repository +from gso.products.product_blocks import PhyPortCapacity +from gso.products.product_blocks.iptrunk import IptrunkType +from gso.products.product_types.iptrunk import IptrunkInactive, IptrunkProvisioning +from gso.services.crm import get_customer_id_by_name +from gso.workflows.iptrunk.create_iptrunk import initialize_subscription + + +def _generate_routers() -> dict[str, str]: + """Generate a dictionary of router IDs and descriptions.""" + routers = {} + for router_id, router_description in repository.all_active_router_subscriptions( + fields=["subscription_id", "description"] + ): + routers[str(router_id)] = router_description + return routers + + +def initial_input_form_generator() -> FormGenerator: + routers = _generate_routers() + RouterEnum = Choice("Select a router", zip(routers.keys(), routers.items())) # type: ignore + + class CreateIptrunkForm(FormPage): + class Config: + title = "Import Iptrunk" + + customer: str + geant_s_sid: str + iptrunk_description: str + iptrunk_type: IptrunkType + iptrunk_speed: PhyPortCapacity + iptrunk_minimum_links: int + + iptrunk_sideA_node_id: RouterEnum # type: ignore + iptrunk_sideA_ae_iface: str + iptrunk_sideA_ae_geant_a_sid: str + iptrunk_sideA_ae_members: UniqueConstrainedList[str] + iptrunk_sideA_ae_members_descriptions: UniqueConstrainedList[str] + + iptrunk_sideB_node_id: RouterEnum # type: ignore + iptrunk_sideB_ae_iface: str + iptrunk_sideB_ae_geant_a_sid: str + iptrunk_sideB_ae_members: UniqueConstrainedList[str] + iptrunk_sideB_ae_members_descriptions: UniqueConstrainedList[str] + + iptrunk_ipv4_network: ipaddress.IPv4Network + iptrunk_ipv6_network: ipaddress.IPv6Network + + initial_user_input = yield CreateIptrunkForm + + return initial_user_input.dict() + + +@step("Create a new subscription") +def create_subscription(customer: str) -> State: + customer_id = get_customer_id_by_name(customer) + product_id = repository.iptrunk_product_id() + subscription = IptrunkInactive.from_product_id(product_id, customer_id) + + return { + "subscription": subscription, + "subscription_id": subscription.subscription_id, + } + + +@step("Update IPAM Stub for Subscription") +def update_ipam_stub_for_subscription( + subscription: IptrunkProvisioning, + iptrunk_ipv4_network: ipaddress.IPv4Network, + iptrunk_ipv6_network: ipaddress.IPv6Network, +) -> State: + subscription.iptrunk.iptrunk_ipv4_network = iptrunk_ipv4_network + subscription.iptrunk.iptrunk_ipv6_network = iptrunk_ipv6_network + + return {"subscription": subscription} + + +@workflow( + "Import iptrunk", + initial_input_form=initial_input_form_generator, + target=Target.SYSTEM, +) +def import_iptrunk() -> StepList: + return ( + init + >> create_subscription + >> store_process_subscription(Target.CREATE) + >> initialize_subscription + >> update_ipam_stub_for_subscription + >> set_status(SubscriptionLifecycle.ACTIVE) + >> resync + >> done + ) diff --git a/gso/workflows/tasks/import_router.py b/gso/workflows/tasks/import_router.py index 30f56a53..57337162 100644 --- a/gso/workflows/tasks/import_router.py +++ b/gso/workflows/tasks/import_router.py @@ -3,25 +3,39 @@ from typing import Optional from uuid import UUID from orchestrator import workflow -from orchestrator.db import ProductTable from orchestrator.forms import FormPage from orchestrator.targets import Target from orchestrator.types import FormGenerator, State, SubscriptionLifecycle from orchestrator.workflow import StepList, done, init, step from orchestrator.workflows.steps import resync, set_status, store_process_subscription +from gso import repository from gso.products.product_blocks import router as router_pb from gso.products.product_blocks.router import RouterRole, RouterVendor from gso.products.product_types import router from gso.products.product_types.router import RouterInactive -from gso.services.crm import get_customer_by_name -from gso.services.subscriptions import get_site_by_name +from gso.products.product_types.site import Site +from gso.services.crm import get_customer_id_by_name + + +def _get_site_by_name(site_name: str) -> Site: + """Get a site by its name. + + Args: + ---- + site_name (str): The name of the site. + """ + subscription = repository.active_site_subscription_by_name(site_name) + if not subscription: + raise ValueError(f"Site with name {site_name} not found.") + + return Site.from_subscription(subscription.subscription_id) @step("Create subscription") def create_subscription(customer: str) -> State: - customer_id: UUID = get_customer_by_name(customer)["id"] - product_id: UUID = ProductTable.query.filter_by(name="Router").first().product_id + customer_id: str = get_customer_id_by_name(customer) + product_id: UUID = repository.router_product_id() subscription = RouterInactive.from_product_id(product_id, customer_id) return { @@ -72,7 +86,7 @@ def initialize_subscription( ) -> State: subscription.router.router_ts_port = ts_port subscription.router.router_vendor = router_vendor - subscription.router.router_site = get_site_by_name(router_site).site + subscription.router.router_site = _get_site_by_name(router_site).site fqdn = ( f"{hostname}.{subscription.router.router_site.site_name.lower()}." f"{subscription.router.router_site.site_country_code.lower()}" diff --git a/gso/workflows/tasks/import_site.py b/gso/workflows/tasks/import_site.py index 6402ae2a..14b69d42 100644 --- a/gso/workflows/tasks/import_site.py +++ b/gso/workflows/tasks/import_site.py @@ -1,23 +1,23 @@ from uuid import UUID -from orchestrator.db.models import ProductTable from orchestrator.forms import FormPage from orchestrator.targets import Target from orchestrator.types import FormGenerator, State, SubscriptionLifecycle from orchestrator.workflow import StepList, done, init, step, workflow from orchestrator.workflows.steps import resync, set_status, store_process_subscription +from gso import repository from gso.products.product_blocks.site import SiteTier -from gso.products.product_types import site -from gso.services.crm import get_customer_by_name +from gso.products.product_types.site import SiteInactive +from gso.services.crm import get_customer_id_by_name from gso.workflows.site.create_site import initialize_subscription @step("Create subscription") def create_subscription(customer: str) -> State: - customer_id: UUID = get_customer_by_name(customer)["id"] - product_id: UUID = ProductTable.query.filter_by(product_type="Site").first().product_id - subscription = site.SiteInactive.from_product_id(product_id, customer_id) + customer_id: str = get_customer_id_by_name(customer) + product_id: UUID = repository.site_product_id() + subscription = SiteInactive.from_product_id(product_id, customer_id) return { "subscription": subscription, diff --git a/requirements.txt b/requirements.txt index 91809af8..5806a685 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,5 +12,4 @@ mypy ruff sphinx sphinx-rtd-theme -requests typer \ No newline at end of file diff --git a/test/conftest.py b/test/conftest.py index 7fdf3345..7914bdc3 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -1,4 +1,5 @@ import contextlib +import ipaddress import json import os import socket @@ -9,15 +10,51 @@ import orchestrator import pytest from alembic import command from alembic.config import Config +from faker import Faker +from faker.providers import BaseProvider from orchestrator import app_settings from orchestrator.db import Database, db from orchestrator.db.database import ENGINE_ARGUMENTS, SESSION_ARGUMENTS, BaseModel +from orchestrator.domain import SubscriptionModel +from orchestrator.types import SubscriptionLifecycle, UUIDstr from sqlalchemy import create_engine from sqlalchemy.engine import make_url from sqlalchemy.orm import scoped_session, sessionmaker from starlette.testclient import TestClient +from gso import repository from gso.main import init_gso_app +from gso.products.product_blocks.router import RouterRole, RouterVendor +from gso.products.product_blocks.site import SiteTier +from gso.products.product_types.router import RouterInactive +from gso.products.product_types.site import Site, SiteInactive + +CUSTOMER_ID: UUIDstr = "2f47f65a-0911-e511-80d0-005056956c1a" + + +class FakerProvider(BaseProvider): + def ipv4_network(self): + ipv4 = self.generator.ipv4() + interface = ipaddress.IPv4Interface(ipv4 + "/24") + network = interface.network.network_address + + return ipaddress.IPv4Network(str(network) + "/24") + + def ipv6_network(self): + ipv6 = self.generator.ipv6() + interface = ipaddress.IPv6Interface(ipv6 + "/64") + network = interface.network.network_address + + return ipaddress.IPv6Network(str(network) + "/64") + + +faker = Faker() +faker.add_provider(FakerProvider) + + +@pytest.fixture(scope="session") +def fake() -> Faker: + return faker @pytest.fixture(scope="session") @@ -212,3 +249,112 @@ def fastapi_app(database, db_uri): @pytest.fixture(scope="session") def test_client(fastapi_app): return TestClient(fastapi_app) + + +@pytest.fixture +def site_subscription_factory(fake): + def subscription_create( + description=None, + start_date="2023-05-24T00:00:00+00:00", + site_name=None, + site_city=None, + site_country=None, + site_country_code=None, + site_latitude=None, + site_longitude=None, + site_bgp_community_id=None, + site_internal_id=None, + site_tier=SiteTier.TIER1, + site_ts_address=None, + ) -> UUIDstr: + description = description or "Site Subscription" + site_name = site_name or fake.name() + site_city = site_city or fake.city() + site_country = site_country or fake.country() + site_country_code = site_country_code or fake.country_code() + site_latitude = site_latitude or float(fake.latitude()) + site_longitude = site_longitude or float(fake.longitude()) + site_bgp_community_id = site_bgp_community_id or fake.pyint() + site_internal_id = site_internal_id or fake.pyint() + site_ts_address = site_ts_address or fake.ipv4() + + product_id = repository.site_product_id() + site_subscription = SiteInactive.from_product_id(product_id, customer_id=CUSTOMER_ID, insync=True) + site_subscription.site.site_city = site_city + site_subscription.site.site_name = site_name + site_subscription.site.site_country = site_country + site_subscription.site.site_country_code = site_country_code + site_subscription.site.site_latitude = site_latitude + site_subscription.site.site_longitude = site_longitude + site_subscription.site.site_bgp_community_id = site_bgp_community_id + site_subscription.site.site_internal_id = site_internal_id + site_subscription.site.site_tier = site_tier + site_subscription.site.site_ts_address = site_ts_address + + site_subscription = SubscriptionModel.from_other_lifecycle(site_subscription, SubscriptionLifecycle.ACTIVE) + site_subscription.description = description + site_subscription.start_date = start_date + site_subscription.save() + db.session.commit() + + return str(site_subscription.subscription_id) + + return subscription_create + + +@pytest.fixture +def router_subscription_factory(site_subscription_factory, fake): + def subscription_create( + description=None, + start_date="2023-05-24T00:00:00+00:00", + router_fqdn=None, + router_ts_port=None, + router_access_via_ts=None, + router_lo_ipv4_address=None, + router_lo_ipv6_address=None, + router_lo_iso_address=None, + router_si_ipv4_network=None, + router_ias_lt_ipv4_network=None, + router_ias_lt_ipv6_network=None, + router_vendor=RouterVendor.NOKIA, + router_role=RouterRole.PE, + router_site=None, + router_is_ias_connected=True, + ) -> UUIDstr: + description = description or fake.text(max_nb_chars=30) + router_fqdn = router_fqdn or fake.domain_name() + router_ts_port = router_ts_port or fake.random_int(min=1, max=65535) + router_access_via_ts = router_access_via_ts or fake.boolean() + router_lo_ipv4_address = router_lo_ipv4_address or ipaddress.IPv4Address(fake.ipv4()) + router_lo_ipv6_address = router_lo_ipv6_address or ipaddress.IPv6Address(fake.ipv6()) + router_lo_iso_address = router_lo_iso_address or fake.word() + router_si_ipv4_network = router_si_ipv4_network or fake.ipv4_network() + router_ias_lt_ipv4_network = router_ias_lt_ipv4_network or fake.ipv4_network() + router_ias_lt_ipv6_network = router_ias_lt_ipv6_network or fake.ipv6_network() + router_site = router_site or site_subscription_factory() + + product_id = repository.router_product_id() + router_subscription = RouterInactive.from_product_id(product_id, customer_id=CUSTOMER_ID, insync=True) + router_subscription.router.router_fqdn = router_fqdn + router_subscription.router.router_ts_port = router_ts_port + router_subscription.router.router_access_via_ts = router_access_via_ts + router_subscription.router.router_lo_ipv4_address = router_lo_ipv4_address + router_subscription.router.router_lo_ipv6_address = router_lo_ipv6_address + router_subscription.router.router_lo_iso_address = router_lo_iso_address + router_subscription.router.router_si_ipv4_network = router_si_ipv4_network + router_subscription.router.router_ias_lt_ipv4_network = router_ias_lt_ipv4_network + router_subscription.router.router_ias_lt_ipv6_network = router_ias_lt_ipv6_network + router_subscription.router.router_vendor = router_vendor + router_subscription.router.router_role = router_role + router_subscription.router.router_site = Site.from_subscription(router_site).site + router_subscription.router.router_is_ias_connected = router_is_ias_connected + + router_subscription = SubscriptionModel.from_other_lifecycle(router_subscription, SubscriptionLifecycle.ACTIVE) + router_subscription.description = description + router_subscription.start_date = start_date + router_subscription.save() + db.session.commit() + + return str(router_subscription.subscription_id) + + return subscription_create diff --git a/test/test_imports.py b/test/test_imports.py index e89341aa..9857e409 100644 --- a/test/test_imports.py +++ b/test/test_imports.py @@ -12,8 +12,8 @@ class TestImportEndpoints: def setup(self, test_client): self.faker = Faker() self.client = test_client - self.site_import_endpoint = "/api/imports/sites" - self.router_import_endpoint = "/api/imports/routers" + self.site_import_endpoint = "/api/v1/imports/sites" + self.router_import_endpoint = "/api/v1/imports/routers" self.site_data = { "site_name": self.faker.name(), "site_city": self.faker.city(), @@ -25,7 +25,7 @@ class TestImportEndpoints: "site_internal_id": self.faker.pyint(), "site_tier": SiteTier.TIER1, "site_ts_address": self.faker.ipv4(), - "customer": "Geant", + "customer": "GÉANT", } self.router_data = { "hostname": "127.0.0.1", @@ -33,7 +33,7 @@ class TestImportEndpoints: "router_vendor": RouterVendor.JUNIPER, "router_site": self.site_data["site_name"], "ts_port": 1234, - "customer": "Geant", + "customer": "GÉANT", "is_ias_connected": True, "router_lo_ipv4_address": self.faker.ipv4(), "router_lo_ipv6_address": self.faker.ipv6(), diff --git a/test/test_imports_iptrunk.py b/test/test_imports_iptrunk.py new file mode 100644 index 00000000..51111730 --- /dev/null +++ b/test/test_imports_iptrunk.py @@ -0,0 +1,213 @@ +from unittest.mock import patch +from uuid import uuid4 + +import pytest +from orchestrator.services import subscriptions + +from gso.products.product_blocks import PhyPortCapacity +from gso.products.product_blocks.iptrunk import IptrunkType + +IPTRUNK_IMPORT_API_URL = "/api/v1/imports/iptrunks" + + +@pytest.fixture +def iptrunk_data(router_subscription_factory, fake): + router_side_a = router_subscription_factory() + router_side_b = router_subscription_factory() + return { + "customer": "GÉANT", + "geant_s_sid": fake.pystr(), + "iptrunk_type": IptrunkType.DARK_FIBER, + "iptrunk_description": fake.sentence(), + "iptrunk_speed": PhyPortCapacity.HUNDRED_GIGABIT_PER_SECOND, + "iptrunk_minimum_links": 5, + "iptrunk_sideA_node_id": router_side_a, + "iptrunk_sideA_ae_iface": fake.pystr(), + "iptrunk_sideA_ae_geant_a_sid": fake.pystr(), + "iptrunk_sideA_ae_members": [fake.pystr() for _ in range(5)], + "iptrunk_sideA_ae_members_descriptions": [fake.sentence() for _ in range(5)], + "iptrunk_sideB_node_id": router_side_b, + "iptrunk_sideB_ae_iface": fake.pystr(), + "iptrunk_sideB_ae_geant_a_sid": fake.pystr(), + "iptrunk_sideB_ae_members": [fake.pystr() for _ in range(5)], + "iptrunk_sideB_ae_members_descriptions": [fake.sentence() for _ in range(5)], + "iptrunk_ipv4_network": str(fake.ipv4_network()), + "iptrunk_ipv6_network": str(fake.ipv6_network()), + } + + +@pytest.fixture +def mock_routers(iptrunk_data): + first_call = [iptrunk_data["iptrunk_sideA_node_id"], iptrunk_data["iptrunk_sideB_node_id"], str(uuid4())] + side_effects = [ + first_call, + first_call, + [ + (iptrunk_data["iptrunk_sideA_node_id"], "iptrunk_sideA_node_id description"), + (iptrunk_data["iptrunk_sideB_node_id"], "iptrunk_sideB_node_id description"), + (str(uuid4()), "random description"), + ], + ] + with patch("gso.repository.all_active_router_subscriptions") as mock_all_active_router_subscriptions: + mock_all_active_router_subscriptions.side_effect = side_effects + yield mock_all_active_router_subscriptions + + +@patch("gso.api.v1.imports._start_process") +def test_import_iptrunk_successful_with_mocked_process(mock_start_process, test_client, iptrunk_data, mock_routers): + mock_start_process.return_value = "123e4567-e89b-12d3-a456-426655440000" + response = test_client.post(IPTRUNK_IMPORT_API_URL, json=iptrunk_data) + + assert response.status_code == 201 + assert response.json()["pid"] == "123e4567-e89b-12d3-a456-426655440000" + + +def test_import_iptrunk_successful_with_real_process(test_client, iptrunk_data, mock_routers): + response = test_client.post(IPTRUNK_IMPORT_API_URL, json=iptrunk_data) + assert response.status_code == 201 + + response = response.json() + assert "detail" in response + assert "pid" in response + + subscription = subscriptions.retrieve_subscription_by_subscription_instance_value( + resource_type="geant_s_sid", value=iptrunk_data["geant_s_sid"] + ) + assert subscription is not None + + +@patch("gso.api.v1.imports._start_process") +def test_import_iptrunk_invalid_customer(mock_start_process, test_client, iptrunk_data, mock_routers): + iptrunk_data["customer"] = "not_existing_customer" + mock_start_process.return_value = "123e4567-e89b-12d3-a456-426655440000" + response = test_client.post(IPTRUNK_IMPORT_API_URL, json=iptrunk_data) + + assert response.status_code == 422 + assert response.json() == { + "detail": [ + {"loc": ["body", "customer"], "msg": "Customer not_existing_customer not found", "type": "value_error"} + ] + } + + +@patch("gso.api.v1.imports._start_process") +def test_import_iptrunk_invalid_router_id_side_a_and_b(mock_start_process, test_client, iptrunk_data): + mock_start_process.return_value = "123e4567-e89b-12d3-a456-426655440000" + response = test_client.post(IPTRUNK_IMPORT_API_URL, json=iptrunk_data) + + assert response.status_code == 422 + assert response.json() == { + "detail": [ + {"loc": ["body", "iptrunk_sideA_node_id"], "msg": "Router not found", "type": "value_error"}, + {"loc": ["body", "iptrunk_sideB_node_id"], "msg": "Router not found", "type": "value_error"}, + ] + } + + +@patch("gso.api.v1.imports._start_process") +def test_import_iptrunk_non_unique_members_side_a(mock_start_process, test_client, iptrunk_data, mock_routers): + mock_start_process.return_value = "123e4567-e89b-12d3-a456-426655440000" + + iptrunk_data["iptrunk_sideA_ae_members"] = [5, 5, 5, 5, 5] + iptrunk_data["iptrunk_sideB_ae_members"] = [4, 4, 4, 5, 5] + + response = test_client.post(IPTRUNK_IMPORT_API_URL, json=iptrunk_data) + + assert response.status_code == 422 + assert response.json() == { + "detail": [ + {"loc": ["body", "iptrunk_sideA_ae_members"], "msg": "Items must be unique", "type": "value_error"}, + {"loc": ["body", "iptrunk_sideB_ae_members"], "msg": "Items must be unique", "type": "value_error"}, + { + "loc": ["body", "__root__"], + "msg": "Side A members should be at least 5 (iptrunk_minimum_links)", + "type": "value_error", + }, + ] + } + + +@patch("gso.api.v1.imports._start_process") +def test_iptrunk_import_fails_on_side_a_member_count_mismatch( + mock_start_process, test_client, iptrunk_data, mock_routers +): + mock_start_process.return_value = "123e4567-e89b-12d3-a456-426655440000" + + iptrunk_data["iptrunk_sideA_ae_members"].remove(iptrunk_data["iptrunk_sideA_ae_members"][0]) + + response = test_client.post(IPTRUNK_IMPORT_API_URL, json=iptrunk_data) + + assert response.status_code == 422 + assert response.json() == { + "detail": [ + { + "loc": ["body", "__root__"], + "msg": "Side A members should be at least 5 (iptrunk_minimum_links)", + "type": "value_error", + } + ] + } + + +@patch("gso.api.v1.imports._start_process") +def test_iptrunk_import_fails_on_side_a_member_description_mismatch( + mock_start_process, test_client, iptrunk_data, mock_routers +): + mock_start_process.return_value = "123e4567-e89b-12d3-a456-426655440000" + + iptrunk_data["iptrunk_sideA_ae_members_descriptions"].remove( + iptrunk_data["iptrunk_sideA_ae_members_descriptions"][0] + ) + + response = test_client.post(IPTRUNK_IMPORT_API_URL, json=iptrunk_data) + + assert response.status_code == 422 + assert response.json() == { + "detail": [ + { + "loc": ["body", "__root__"], + "msg": "Mismatch in Side A members and their descriptions", + "type": "value_error", + } + ] + } + + +@patch("gso.api.v1.imports._start_process") +def test_iptrunk_import_fails_on_side_a_and_b_members_mismatch( + mock_start_process, test_client, iptrunk_data, mock_routers +): + mock_start_process.return_value = "123e4567-e89b-12d3-a456-426655440000" + + iptrunk_data["iptrunk_sideB_ae_members"].remove(iptrunk_data["iptrunk_sideB_ae_members"][0]) + + response = test_client.post(IPTRUNK_IMPORT_API_URL, json=iptrunk_data) + + assert response.status_code == 422 + assert response.json() == { + "detail": [{"loc": ["body", "__root__"], "msg": "Mismatch between Side A and B members", "type": "value_error"}] + } + + +@patch("gso.api.v1.imports._start_process") +def test_iptrunk_import_fails_on_side_b_member_description_mismatch( + mock_start_process, test_client, iptrunk_data, mock_routers +): + mock_start_process.return_value = "123e4567-e89b-12d3-a456-426655440000" + + iptrunk_data["iptrunk_sideB_ae_members_descriptions"].remove( + iptrunk_data["iptrunk_sideB_ae_members_descriptions"][0] + ) + + response = test_client.post(IPTRUNK_IMPORT_API_URL, json=iptrunk_data) + + assert response.status_code == 422 + assert response.json() == { + "detail": [ + { + "loc": ["body", "__root__"], + "msg": "Mismatch in Side B members and their descriptions", + "type": "value_error", + } + ] + } -- GitLab