Skip to content
Snippets Groups Projects
Commit f38cecd1 authored by Mohammad Torkashvand's avatar Mohammad Torkashvand
Browse files

rebase with develop

parent 96361b9f
No related branches found
No related tags found
1 merge request!188upgrade to orchestrato-core v2
...@@ -16,7 +16,6 @@ from gso.middlewares import ModifyProcessEndpointResponse ...@@ -16,7 +16,6 @@ from gso.middlewares import ModifyProcessEndpointResponse
def init_gso_app() -> OrchestratorCore: def init_gso_app() -> OrchestratorCore:
"""Initialise the :term:`GSO` app.""" """Initialise the :term:`GSO` app."""
app = OrchestratorCore(base_settings=app_settings) app = OrchestratorCore(base_settings=app_settings)
# app.register_graphql() # TODO: uncomment this line when the GUI V2 is ready
app.include_router(api_router, prefix="/api") app.include_router(api_router, prefix="/api")
app.add_middleware(ModifyProcessEndpointResponse) app.add_middleware(ModifyProcessEndpointResponse)
return app return app
......
...@@ -6,14 +6,14 @@ import json ...@@ -6,14 +6,14 @@ import json
import time import time
from datetime import UTC, datetime from datetime import UTC, datetime
from pathlib import Path from pathlib import Path
from typing import Any, TypeVar from typing import Self, TypeVar
import typer import typer
import yaml import yaml
from orchestrator.db import db from orchestrator.db import db
from orchestrator.services.processes import start_process from orchestrator.services.processes import start_process
from orchestrator.types import SubscriptionLifecycle from orchestrator.types import SubscriptionLifecycle
from pydantic import BaseModel, ValidationError, root_validator, validator from pydantic import BaseModel, ValidationError, field_validator, model_validator
from sqlalchemy.exc import SQLAlchemyError from sqlalchemy.exc import SQLAlchemyError
from gso.db.models import PartnerTable from gso.db.models import PartnerTable
...@@ -28,7 +28,7 @@ from gso.services.subscriptions import ( ...@@ -28,7 +28,7 @@ from gso.services.subscriptions import (
get_subscriptions, get_subscriptions,
) )
from gso.utils.helpers import BaseSiteValidatorModel, LAGMember from gso.utils.helpers import BaseSiteValidatorModel, LAGMember
from gso.utils.shared_enums import PortNumber, Vendor from gso.utils.shared_enums import IPv4AddressType, IPv6AddressType, PortNumber, Vendor
app: typer.Typer = typer.Typer() app: typer.Typer = typer.Typer()
...@@ -58,8 +58,8 @@ class RouterImportModel(BaseModel): ...@@ -58,8 +58,8 @@ class RouterImportModel(BaseModel):
ts_port: int ts_port: int
router_vendor: Vendor router_vendor: Vendor
router_role: RouterRole router_role: RouterRole
router_lo_ipv4_address: ipaddress.IPv4Address router_lo_ipv4_address: IPv4AddressType
router_lo_ipv6_address: ipaddress.IPv6Address router_lo_ipv6_address: IPv6AddressType
router_lo_iso_address: str router_lo_iso_address: str
...@@ -112,7 +112,7 @@ class IptrunkImportModel(BaseModel): ...@@ -112,7 +112,7 @@ class IptrunkImportModel(BaseModel):
str(router["subscription_id"]) for router in get_active_router_subscriptions(includes=["subscription_id"]) str(router["subscription_id"]) for router in get_active_router_subscriptions(includes=["subscription_id"])
} }
@validator("partner") @field_validator("partner")
def check_if_partner_exists(cls, value: str) -> str: def check_if_partner_exists(cls, value: str) -> str:
"""Validate that the partner exists.""" """Validate that the partner exists."""
try: try:
...@@ -123,7 +123,7 @@ class IptrunkImportModel(BaseModel): ...@@ -123,7 +123,7 @@ class IptrunkImportModel(BaseModel):
return value return value
@validator("side_a_node_id", "side_b_node_id") @field_validator("side_a_node_id", "side_b_node_id")
def check_if_router_side_is_available(cls, value: str) -> str: def check_if_router_side_is_available(cls, value: str) -> str:
"""Both sides of the trunk must exist in :term:`GSO`.""" """Both sides of the trunk must exist in :term:`GSO`."""
if value not in cls._get_active_routers(): if value not in cls._get_active_routers():
...@@ -132,7 +132,7 @@ class IptrunkImportModel(BaseModel): ...@@ -132,7 +132,7 @@ class IptrunkImportModel(BaseModel):
return value return value
@validator("side_a_ae_members", "side_b_ae_members") @field_validator("side_a_ae_members", "side_b_ae_members")
def check_side_uniqueness(cls, value: list[str]) -> list[str]: def check_side_uniqueness(cls, value: list[str]) -> list[str]:
""":term:`LAG` members must be unique.""" """:term:`LAG` members must be unique."""
if len(value) != len(set(value)): if len(value) != len(set(value)):
...@@ -141,25 +141,21 @@ class IptrunkImportModel(BaseModel): ...@@ -141,25 +141,21 @@ class IptrunkImportModel(BaseModel):
return value return value
@root_validator @model_validator(mode="after")
def check_members(cls, values: dict[str, Any]) -> dict[str, Any]: def check_members(self) -> Self:
"""Amount of :term:`LAG` members has to match on side A and B, and meet the minimum requirement.""" """Amount of :term:`LAG` members has to match on side A and B, and meet the minimum requirement."""
min_links = values["iptrunk_minimum_links"] len_a = len(self.side_a_ae_members)
side_a_members = values.get("side_a_ae_members", []) len_b = len(self.side_b_ae_members)
side_b_members = values.get("side_b_ae_members", [])
len_a = len(side_a_members) if len_a < self.iptrunk_minimum_links:
len_b = len(side_b_members) msg = f"Side A members should be at least {self.iptrunk_minimum_links} (iptrunk_minimum_links)"
if len_a < min_links:
msg = f"Side A members should be at least {min_links} (iptrunk_minimum_links)"
raise ValueError(msg) raise ValueError(msg)
if len_a != len_b: if len_a != len_b:
msg = "Mismatch between Side A and B members" msg = "Mismatch between Side A and B members"
raise ValueError(msg) raise ValueError(msg)
return values return self
T = TypeVar( T = TypeVar(
......
...@@ -10,52 +10,15 @@ from orchestrator.migrations.helpers import create_workflow, delete_workflow ...@@ -10,52 +10,15 @@ from orchestrator.migrations.helpers import create_workflow, delete_workflow
# revision identifiers, used by Alembic. # revision identifiers, used by Alembic.
revision = '1ec810b289c0' revision = '1ec810b289c0'
down_revision = '393acfa175c0' down_revision = '32cad119b7c4'
branch_labels = None branch_labels = None
# TODO: check it carefuly # TODO: check it carefuly
depends_on = '048219045729' # in this revision, SURF has added a new columns to the workflow table like delted_at, so we need to add a dependency on the revision that added the columns to the workflow table. depends_on = '048219045729' # in this revision, SURF has added a new columns to the workflow table like delted_at, so we need to add a dependency on the revision that added the columns to the workflow table.
new_workflows = [
{
"name": "import_site",
"target": "SYSTEM",
"description": "Import a site without provisioning it.",
"product_type": "Site"
},
{
"name": "import_router",
"target": "SYSTEM",
"description": "Import a router without provisioning it.",
"product_type": "Router"
},
{
"name": "import_iptrunk",
"target": "SYSTEM",
"description": "Import an IP trunk without provisioning it.",
"product_type": "Iptrunk"
},
{
"name": "import_super_pop_switch",
"target": "SYSTEM",
"description": "Import a Super PoP switch without provisioning it.",
"product_type": "SuperPopSwitch"
},
{
"name": "import_office_router",
"target": "SYSTEM",
"description": "Import an office router without provisioning it.",
"product_type": "OfficeRouter"
},
]
def upgrade() -> None: def upgrade() -> None:
conn = op.get_bind() pass
for workflow in new_workflows:
create_workflow(conn, workflow)
def downgrade() -> None: def downgrade() -> None:
conn = op.get_bind() pass
for workflow in new_workflows:
delete_workflow(conn, workflow["name"])
...@@ -8,6 +8,7 @@ from orchestrator.domain.base import ProductBlockModel, T ...@@ -8,6 +8,7 @@ from orchestrator.domain.base import ProductBlockModel, T
from orchestrator.types import SubscriptionLifecycle, strEnum from orchestrator.types import SubscriptionLifecycle, strEnum
from pydantic import AfterValidator from pydantic import AfterValidator
from pydantic_forms.validators import validate_unique_list from pydantic_forms.validators import validate_unique_list
from typing_extensions import Doc
from gso.products.product_blocks.router import ( from gso.products.product_blocks.router import (
RouterBlock, RouterBlock,
...@@ -35,8 +36,15 @@ class IptrunkType(strEnum): ...@@ -35,8 +36,15 @@ class IptrunkType(strEnum):
LEASED = "Leased" LEASED = "Leased"
LAGMemberList = Annotated[list[T], AfterValidator(validate_unique_list), Len(min_length=0)] LAGMemberList = Annotated[
IptrunkSides = Annotated[list[T], AfterValidator(validate_unique_list), Len(min_length=2, max_length=2)] list[T], AfterValidator(validate_unique_list), Len(min_length=0), Doc("A list of :term:`LAG` member interfaces.")
]
IptrunkSides = Annotated[
list[T],
AfterValidator(validate_unique_list),
Len(min_length=2, max_length=2),
Doc("A list of IP trunk interfaces that make up one side of a link."),
]
class IptrunkInterfaceBlockInactive( class IptrunkInterfaceBlockInactive(
......
...@@ -8,6 +8,7 @@ from orchestrator.types import SubscriptionLifecycle ...@@ -8,6 +8,7 @@ from orchestrator.types import SubscriptionLifecycle
from pydantic import AfterValidator from pydantic import AfterValidator
from pydantic_forms.types import strEnum from pydantic_forms.types import strEnum
from pydantic_forms.validators import validate_unique_list from pydantic_forms.validators import validate_unique_list
from typing_extensions import Doc
from gso.products.product_blocks.lan_switch_interconnect import ( from gso.products.product_blocks.lan_switch_interconnect import (
LanSwitchInterconnectBlock, LanSwitchInterconnectBlock,
...@@ -25,7 +26,7 @@ class LayerPreference(strEnum): ...@@ -25,7 +26,7 @@ class LayerPreference(strEnum):
L3 = "L3" L3 = "L3"
PortList = Annotated[list[T], AfterValidator(validate_unique_list)] PortList = Annotated[list[T], AfterValidator(validate_unique_list), Doc("A list of unique ports.")]
class PopVlanPortBlockInactive( class PopVlanPortBlockInactive(
......
...@@ -3,10 +3,10 @@ ...@@ -3,10 +3,10 @@
import re import re
from typing import Annotated from typing import Annotated
from annotated_types import doc
from orchestrator.domain.base import ProductBlockModel from orchestrator.domain.base import ProductBlockModel
from orchestrator.types import SubscriptionLifecycle, strEnum from orchestrator.types import SubscriptionLifecycle, strEnum
from pydantic import AfterValidator, Field from pydantic import AfterValidator, Field
from typing_extensions import Doc
class SiteTier(strEnum): class SiteTier(strEnum):
...@@ -48,7 +48,7 @@ LatitudeCoordinate = Annotated[ ...@@ -48,7 +48,7 @@ LatitudeCoordinate = Annotated[
le=90, le=90,
), ),
AfterValidator(validate_latitude), AfterValidator(validate_latitude),
doc( Doc(
"A latitude coordinate, modeled as a string. " "A latitude coordinate, modeled as a string. "
"The coordinate must match the format conforming to the latitude range of -90 to +90 degrees. " "The coordinate must match the format conforming to the latitude range of -90 to +90 degrees. "
"It can be a floating-point number or an integer. Valid examples: 40.7128, -74.0060, 90, -90, 0." "It can be a floating-point number or an integer. Valid examples: 40.7128, -74.0060, 90, -90, 0."
...@@ -62,11 +62,11 @@ LongitudeCoordinate = Annotated[ ...@@ -62,11 +62,11 @@ LongitudeCoordinate = Annotated[
le=180, le=180,
), ),
AfterValidator(validate_longitude), AfterValidator(validate_longitude),
doc( Doc(
"A longitude coordinate, modeled as a string. " "A longitude coordinate, modeled as a string. "
"The coordinate must match the format conforming to the longitude " "The coordinate must match the format conforming to the longitude "
"range of -180 to +180 degrees. It can be a floating-point number or an integer. " "range of -180 to +180 degrees. It can be a floating-point number or an integer. "
"Valid examples: 40.7128, -74.0060, 180, -180, 0." "Valid examples: 40.7128, -74.0060, 180, -180, 0."
), ),
] ]
......
...@@ -273,7 +273,7 @@ def find_host_by_ip(ip_addr: IPv4AddressType | ipaddress.IPv6Address) -> objects ...@@ -273,7 +273,7 @@ def find_host_by_ip(ip_addr: IPv4AddressType | ipaddress.IPv6Address) -> objects
"""Find a host record in Infoblox by its associated IP address. """Find a host record in Infoblox by its associated IP address.
:param ip_addr: The IP address of a host that is searched for. :param ip_addr: The IP address of a host that is searched for.
:type ip_addr: FancyIPV4Address | ipaddress.IPv6Address :type ip_addr: IPV4AddressType | ipaddress.IPv6Address
""" """
conn, _ = _setup_connection() conn, _ = _setup_connection()
if ip_addr.version == 4: # noqa: PLR2004, the 4 in IPv4 is well-known and not a "magic value." if ip_addr.version == 4: # noqa: PLR2004, the 4 in IPv4 is well-known and not a "magic value."
...@@ -322,7 +322,7 @@ def delete_host_by_ip(ip_addr: IPv4AddressType | ipaddress.IPv6Address) -> None: ...@@ -322,7 +322,7 @@ def delete_host_by_ip(ip_addr: IPv4AddressType | ipaddress.IPv6Address) -> None:
:class:`DeletionError` if no record can be found in Infoblox. :class:`DeletionError` if no record can be found in Infoblox.
:param ip_addr: The IP address of the host record that should get deleted. :param ip_addr: The IP address of the host record that should get deleted.
:type ip_addr: FancyIPV4Address | ipaddress.IPv6Address :type ip_addr: IPV4AddressType | ipaddress.IPv6Address
""" """
host = find_host_by_ip(ip_addr) host = find_host_by_ip(ip_addr)
if host: if host:
......
...@@ -13,6 +13,7 @@ from typing import Annotated ...@@ -13,6 +13,7 @@ from typing import Annotated
from pydantic import Field from pydantic import Field
from pydantic_settings import BaseSettings from pydantic_settings import BaseSettings
from typing_extensions import Doc
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
...@@ -46,8 +47,8 @@ class InfoBloxParams(BaseSettings): ...@@ -46,8 +47,8 @@ class InfoBloxParams(BaseSettings):
password: str password: str
V4Netmask = Annotated[int, Field(ge=0, le=32)] V4Netmask = Annotated[int, Field(ge=0, le=32), Doc("A valid netmask for an IPv4 network or address.")]
V6Netmask = Annotated[int, Field(ge=0, le=128)] V6Netmask = Annotated[int, Field(ge=0, le=128), Doc("A valid netmask for an IPv6 network or address.")]
class V4NetworkParams(BaseSettings): class V4NetworkParams(BaseSettings):
......
...@@ -3,9 +3,9 @@ ...@@ -3,9 +3,9 @@
import ipaddress import ipaddress
from typing import Annotated from typing import Annotated
from annotated_types import doc
from pydantic import Field, PlainSerializer from pydantic import Field, PlainSerializer
from pydantic_forms.types import strEnum from pydantic_forms.types import strEnum
from typing_extensions import Doc
class Vendor(strEnum): class Vendor(strEnum):
...@@ -21,10 +21,10 @@ PortNumber = Annotated[ ...@@ -21,10 +21,10 @@ PortNumber = Annotated[
gt=0, gt=0,
le=49151, le=49151,
), ),
doc( Doc(
"Constrained integer for valid port numbers. The range from 49152 to 65535 is marked as ephemeral, " "Constrained integer for valid port numbers. The range from 49152 to 65535 is marked as ephemeral, "
"and can therefore not be selected for permanent allocation." "and can therefore not be selected for permanent allocation."
), ),
] ]
......
orchestrator-core==2.1.2 orchestrator-core==2.2.1
requests==2.31.0 requests==2.31.0
infoblox-client~=0.6.0 infoblox-client~=0.6.0
pycountry==23.12.11 pycountry==23.12.11
......
...@@ -9,13 +9,13 @@ setup( ...@@ -9,13 +9,13 @@ setup(
url="https://gitlab.software.geant.org/goat/gap/geant-service-orchestrator", url="https://gitlab.software.geant.org/goat/gap/geant-service-orchestrator",
packages=find_packages(), packages=find_packages(),
install_requires=[ install_requires=[
"orchestrator-core==1.3.4", "orchestrator-core==2.2.1",
"requests==2.31.0", "requests==2.31.0",
"infoblox-client~=0.6.0", "infoblox-client~=0.6.0",
"pycountry==22.3.5", "pycountry==23.12.11",
"pynetbox==7.2.0", "pynetbox==7.3.3",
"celery-redbeat==2.1.1", "celery-redbeat==2.2.0",
"celery==5.3.4", "celery==5.3.6",
], ],
include_package_data=True, include_package_data=True,
) )
...@@ -206,19 +206,14 @@ def test_import_site_twice(mock_start_process, site_data, site_subscription_fact ...@@ -206,19 +206,14 @@ def test_import_site_twice(mock_start_process, site_data, site_subscription_fact
# Second identical import should print ValidationError to stdout # Second identical import should print ValidationError to stdout
import_sites(site_import_data["path"]) import_sites(site_import_data["path"])
out, _ = capfd.readouterr() captured_output, _ = capfd.readouterr()
assert (
"""Validation error: 4 validation errors for SiteImportModel assert "Validation error: 4 validation errors for SiteImportModel" in captured_output
site_bgp_community_id assert "Value error, site_bgp_community_id must be unique [type=value_error, input_value=" in captured_output
site_bgp_community_id must be unique (type=value_error) assert "Value error, site_internal_id must be unique [type=value_error, input_value=" in captured_output
site_internal_id assert "Value error, site_ts_address must be unique [type=value_error, input_value=" in captured_output
site_internal_id must be unique (type=value_error) assert "Value error, site_name must be unique [type=value_error, input_value=" in captured_output
site_ts_address
site_ts_address must be unique (type=value_error)
site_name
site_name must be unique (type=value_error)"""
in out
)
assert mock_start_process.call_count == 0 assert mock_start_process.call_count == 0
...@@ -229,15 +224,19 @@ def test_import_site_with_invalid_data(mock_start_process, site_data, capfd): ...@@ -229,15 +224,19 @@ def test_import_site_with_invalid_data(mock_start_process, site_data, capfd):
import_sites(incorrect_site_data["path"]) import_sites(incorrect_site_data["path"])
out, _ = capfd.readouterr() captured_output, _ = capfd.readouterr()
assert "Validation error: 2 validation errors for SiteImportModel" in captured_output
assert (
"""site_latitude
Input should be a valid number [type=float_type, input_value=None, input_type=NoneType]"""
in captured_output
)
assert ( assert (
"""Validation error: 2 validation errors for SiteImportModel """site_longitude
site_latitude Input should be a valid number, unable to parse string as a number [type=float_parsing, input_value='broken',"""
none is not an allowed value (type=type_error.none.not_allowed) in captured_output
site_longitude
value is not a valid float (type=type_error.float)"""
in out
) )
assert mock_start_process.call_count == 0 assert mock_start_process.call_count == 0
...@@ -252,13 +251,13 @@ def test_import_router_with_invalid_data(mock_start_process, router_data, capfd) ...@@ -252,13 +251,13 @@ def test_import_router_with_invalid_data(mock_start_process, router_data, capfd)
broken_data = router_data(hostname="", router_lo_ipv6_address="Not an IP address") broken_data = router_data(hostname="", router_lo_ipv6_address="Not an IP address")
import_routers(broken_data["path"]) import_routers(broken_data["path"])
out, _ = capfd.readouterr() captured_output, _ = capfd.readouterr()
# The extra space at the end of the next line is required, and not dangling by accident. # The extra space at the end of the next line is required, and not dangling by accident.
assert "Validation error: 1 validation error for RouterImportModel" in out
assert ( assert (
"""router_lo_ipv6_address """Validation error: 1 validation error for RouterImportModel
value is not a valid IPv6 address (type=value_error.ipv6address)""" router_lo_ipv6_address
in out Input is not a valid IPv6 address [type=ip_v6_address, input_value='Not an IP address', input_type=str]"""
in captured_output
) )
assert mock_start_process.call_count == 0 assert mock_start_process.call_count == 0
...@@ -274,14 +273,15 @@ def test_import_iptrunk_invalid_router_id_side_a_and_b(mock_start_process, iptru ...@@ -274,14 +273,15 @@ def test_import_iptrunk_invalid_router_id_side_a_and_b(mock_start_process, iptru
broken_data = iptrunk_data(side_a_node="Doesn't exist", side_b_node="Also doesn't exist") broken_data = iptrunk_data(side_a_node="Doesn't exist", side_b_node="Also doesn't exist")
import_iptrunks(broken_data["path"]) import_iptrunks(broken_data["path"])
out, _ = capfd.readouterr() captured_output, _ = capfd.readouterr()
assert ( assert (
"""Validation error: 2 validation errors for IptrunkImportModel """Validation error: 2 validation errors for IptrunkImportModel
side_a_node_id side_a_node_id
Router not found (type=value_error) Value error, Router not found [type=value_error, input_value='', input_type=str]
For further information visit https://errors.pydantic.dev/2.5/v/value_error
side_b_node_id side_b_node_id
Router not found (type=value_error)""" Value error, Router not found [type=value_error, input_value='', input_type=str]"""
in out in captured_output
) )
assert mock_start_process.call_count == 0 assert mock_start_process.call_count == 0
...@@ -294,14 +294,20 @@ def test_import_iptrunk_non_unique_members_side_a_and_b(mock_start_process, iptr ...@@ -294,14 +294,20 @@ def test_import_iptrunk_non_unique_members_side_a_and_b(mock_start_process, iptr
broken_data = iptrunk_data(side_a_members=side_a_members, side_b_members=side_b_members) broken_data = iptrunk_data(side_a_members=side_a_members, side_b_members=side_b_members)
import_iptrunks(broken_data["path"]) import_iptrunks(broken_data["path"])
out, _ = capfd.readouterr() captured_output, _ = capfd.readouterr()
assert SubscriptionTable.query.count() == 4
assert len(get_active_iptrunk_subscriptions()) == 0
assert ( assert (
"""Validation error: 3 validation errors for IptrunkImportModel """Validation error: 2 validation errors for IptrunkImportModel
side_a_ae_members side_a_ae_members
Items must be unique (type=value_error) Value error, Items must be unique [type=value_error, input_value=[{'interface_name':"""
in captured_output
)
assert (
"""
side_b_ae_members side_b_ae_members
Items must be unique (type=value_error)""" Value error, Items must be unique [type=value_error, input_value=[{'interface_name':"""
in out in captured_output
) )
assert mock_start_process.call_count == 0 assert mock_start_process.call_count == 0
...@@ -317,12 +323,11 @@ def test_import_iptrunk_side_a_member_count_mismatch(mock_start_process, iptrunk ...@@ -317,12 +323,11 @@ def test_import_iptrunk_side_a_member_count_mismatch(mock_start_process, iptrunk
broken_data = iptrunk_data(side_a_members=side_a_members, side_b_members=side_b_members) broken_data = iptrunk_data(side_a_members=side_a_members, side_b_members=side_b_members)
import_iptrunks(broken_data["path"]) import_iptrunks(broken_data["path"])
out, _ = capfd.readouterr() captured_output, _ = capfd.readouterr()
assert ( assert (
"""Validation error: 1 validation error for IptrunkImportModel """Validation error: 1 validation error for IptrunkImportModel
__root__ Value error, Mismatch between Side A and B members [type=value_error, input_value={'partner': 'GEANT',"""
Mismatch between Side A and B members (type=value_error)""" in captured_output
in out
) )
assert mock_start_process.call_count == 0 assert mock_start_process.call_count == 0
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment