Skip to content
Snippets Groups Projects
Commit 88606056 authored by geant-release-service's avatar geant-release-service
Browse files

Finished release 0.9.

parents 8f64c3dc 9d84da0a
Branches
Tags 0.9
No related merge requests found
Pipeline #85965 passed
Showing
with 240 additions and 74 deletions
# Changelog # Changelog
All notable changes to this project will be documented in this file. All notable changes to this project will be documented in this file.
## [0.9] - 2024-03-20
- `migrate_iptrunk` workflow includes Ansible trunk checks.
- `create_iptrunk` and `migrate_iptrunk` now update IPAM / DNS.
- lso result step title is now always the name of the provisioning step
## [0.8] - 2024-02-28 ## [0.8] - 2024-02-28
- Add two new workflows for "activating" `Router` and `Iptrunk` products. - Add two new workflows for "activating" `Router` and `Iptrunk` products.
......
# -- Project information ----------------------------------------------------- # -- Project information -----------------------------------------------------
project = "GÉANT Service Orchestrator" project = "GÉANT Service Orchestrator"
copyright = "2023, GÉANT Vereniging" copyright = "2023-2024, GÉANT Vereniging"
author = "GÉANT Orchestration and Automation Team" author = "GÉANT Orchestration and Automation Team"
# -- General configuration --------------------------------------------------- # -- General configuration ---------------------------------------------------
......
...@@ -15,6 +15,6 @@ Submodules ...@@ -15,6 +15,6 @@ Submodules
crm crm
infoblox infoblox
librenms_client librenms_client
lso_client
netbox_client netbox_client
provisioning_proxy
subscriptions subscriptions
``gso.services.lso_client``
===========================
.. automodule:: gso.services.lso_client
:members:
:show-inheritance:
``gso.services.provisioning_proxy``
===================================
.. automodule:: gso.services.provisioning_proxy
:members:
:show-inheritance:
...@@ -6,6 +6,7 @@ WFO ...@@ -6,6 +6,7 @@ WFO
Ansible Ansible
[Dd]eprovision [Dd]eprovision
API API
DNS
dry_run dry_run
Dark_fiber Dark_fiber
[A|a]ddress [A|a]ddress
......
...@@ -14,7 +14,7 @@ from gso.products.product_blocks.iptrunk import IptrunkType, PhyPortCapacity ...@@ -14,7 +14,7 @@ from gso.products.product_blocks.iptrunk import IptrunkType, PhyPortCapacity
from gso.products.product_blocks.router import RouterRole from gso.products.product_blocks.router import RouterRole
from gso.products.product_blocks.site import SiteTier from gso.products.product_blocks.site import SiteTier
from gso.services import subscriptions from gso.services import subscriptions
from gso.services.crm import CustomerNotFoundError, get_customer_by_name from gso.services.partners import PartnerNotFoundError, get_partner_by_name
from gso.utils.helpers import BaseSiteValidatorModel, LAGMember from gso.utils.helpers import BaseSiteValidatorModel, LAGMember
from gso.utils.shared_enums import PortNumber, Vendor from gso.utils.shared_enums import PortNumber, Vendor
...@@ -41,13 +41,13 @@ class SiteImportModel(BaseSiteValidatorModel): ...@@ -41,13 +41,13 @@ class SiteImportModel(BaseSiteValidatorModel):
site_internal_id: int site_internal_id: int
site_tier: SiteTier site_tier: SiteTier
site_ts_address: str site_ts_address: str
customer: str partner: str
class RouterImportModel(BaseModel): class RouterImportModel(BaseModel):
"""Required fields for importing an existing :class:`gso.product.product_types.router`.""" """Required fields for importing an existing :class:`gso.product.product_types.router`."""
customer: str partner: str
router_site: str router_site: str
hostname: str hostname: str
ts_port: int ts_port: int
...@@ -61,7 +61,7 @@ class RouterImportModel(BaseModel): ...@@ -61,7 +61,7 @@ class RouterImportModel(BaseModel):
class IptrunkImportModel(BaseModel): class IptrunkImportModel(BaseModel):
"""Required fields for importing an existing :class:`gso.products.product_types.iptrunk`.""" """Required fields for importing an existing :class:`gso.products.product_types.iptrunk`."""
customer: str partner: str
geant_s_sid: str geant_s_sid: str
iptrunk_type: IptrunkType iptrunk_type: IptrunkType
iptrunk_description: str iptrunk_description: str
...@@ -87,13 +87,13 @@ class IptrunkImportModel(BaseModel): ...@@ -87,13 +87,13 @@ class IptrunkImportModel(BaseModel):
for router in subscriptions.get_active_router_subscriptions(includes=["subscription_id"]) for router in subscriptions.get_active_router_subscriptions(includes=["subscription_id"])
} }
@validator("customer") @validator("partner")
def check_if_customer_exists(cls, value: str) -> str: def check_if_partner_exists(cls, value: str) -> str:
"""Validate that the customer exists.""" """Validate that the partner exists."""
try: try:
get_customer_by_name(value) get_partner_by_name(value)
except CustomerNotFoundError as e: except PartnerNotFoundError as e:
msg = f"Customer {value} not found" msg = f"partner {value} not found"
raise ValueError(msg) from e raise ValueError(msg) from e
return value return value
...@@ -140,7 +140,7 @@ class IptrunkImportModel(BaseModel): ...@@ -140,7 +140,7 @@ class IptrunkImportModel(BaseModel):
class SuperPopSwitchImportModel(BaseModel): class SuperPopSwitchImportModel(BaseModel):
"""Required fields for importing an existing :class:`gso.product.product_types.super_pop_switch`.""" """Required fields for importing an existing :class:`gso.product.product_types.super_pop_switch`."""
customer: str partner: str
super_pop_switch_site: str super_pop_switch_site: str
hostname: str hostname: str
super_pop_switch_ts_port: PortNumber super_pop_switch_ts_port: PortNumber
...@@ -150,7 +150,7 @@ class SuperPopSwitchImportModel(BaseModel): ...@@ -150,7 +150,7 @@ class SuperPopSwitchImportModel(BaseModel):
class OfficeRouterImportModel(BaseModel): class OfficeRouterImportModel(BaseModel):
"""Required fields for importing an existing :class:`gso.product.product_types.office_router`.""" """Required fields for importing an existing :class:`gso.product.product_types.office_router`."""
customer: str partner: str
office_router_site: str office_router_site: str
office_router_fqdn: str office_router_fqdn: str
office_router_ts_port: PortNumber office_router_ts_port: PortNumber
......
""":term:`CLI` command for importing data to coreDB.""" """:term:`CLI` command for importing data to coreDB."""
import csv
import ipaddress import ipaddress
import json import json
from datetime import UTC, datetime
from pathlib import Path from pathlib import Path
from typing import TypeVar from typing import TypeVar
import typer import typer
import yaml import yaml
from orchestrator.db import db
from pydantic import ValidationError from pydantic import ValidationError
from sqlalchemy.exc import SQLAlchemyError
from gso.api.v1.imports import ( from gso.api.v1.imports import (
IptrunkImportModel, IptrunkImportModel,
...@@ -21,6 +25,7 @@ from gso.api.v1.imports import ( ...@@ -21,6 +25,7 @@ from gso.api.v1.imports import (
import_site, import_site,
import_super_pop_switch, import_super_pop_switch,
) )
from gso.db.models import PartnerTable
from gso.services.subscriptions import get_active_subscriptions_by_field_and_value from gso.services.subscriptions import get_active_subscriptions_by_field_and_value
app: typer.Typer = typer.Typer() app: typer.Typer = typer.Typer()
...@@ -63,7 +68,7 @@ def generic_import_data( ...@@ -63,7 +68,7 @@ def generic_import_data(
successfully_imported_data = [] successfully_imported_data = []
data = read_data(filepath) data = read_data(filepath)
for details in data: for details in data:
details["customer"] = "GÉANT" details["partner"] = "GEANT"
typer.echo(f"Importing {name_key}: {details[name_key]}") typer.echo(f"Importing {name_key}: {details[name_key]}")
try: try:
initial_data = import_model(**details) initial_data = import_model(**details)
...@@ -163,7 +168,7 @@ def import_iptrunks(filepath: str = common_filepath_option) -> None: ...@@ -163,7 +168,7 @@ def import_iptrunks(filepath: str = common_filepath_option) -> None:
) )
try: try:
initial_data = IptrunkImportModel( initial_data = IptrunkImportModel(
customer="GÉANT", partner="GEANT",
geant_s_sid=trunk["id"], geant_s_sid=trunk["id"],
iptrunk_type=trunk["config"]["common"]["type"], iptrunk_type=trunk["config"]["common"]["type"],
iptrunk_description=trunk["config"]["common"].get("description", ""), iptrunk_description=trunk["config"]["common"].get("description", ""),
...@@ -197,3 +202,34 @@ def import_iptrunks(filepath: str = common_filepath_option) -> None: ...@@ -197,3 +202,34 @@ def import_iptrunks(filepath: str = common_filepath_option) -> None:
typer.echo("Successfully imported IP Trunks:") typer.echo("Successfully imported IP Trunks:")
for item in successfully_imported_data: for item in successfully_imported_data:
typer.echo(f"- {item}") typer.echo(f"- {item}")
def import_partners_from_csv(file_path: Path) -> list[dict]:
"""Read partners from a CSV file."""
with Path.open(file_path, encoding="utf-8") as csv_file:
csv_reader = csv.DictReader(csv_file)
return list(csv_reader)
@app.command()
def import_partners(file_path: str = typer.Argument(..., help="Path to the CSV file containing partners")) -> None:
"""Import partners from a CSV file into the database."""
typer.echo(f"Importing partners from {file_path} ...")
partners = import_partners_from_csv(Path(file_path))
try:
for partner in partners:
if partner.get("created_at"):
partner["created_at"] = datetime.strptime(partner["created_at"], "%Y-%m-%d").replace(tzinfo=UTC)
new_partner = PartnerTable(**partner)
db.session.add(new_partner)
db.session.commit()
typer.echo(f"Successfully imported {len(partners)} partners.")
except SQLAlchemyError as e:
db.session.rollback()
typer.echo(f"Failed to import partners: {e}")
finally:
db.session.close()
...@@ -14,7 +14,7 @@ def netbox_initial_setup() -> None: ...@@ -14,7 +14,7 @@ def netbox_initial_setup() -> None:
"""Set up NetBox for the first time. """Set up NetBox for the first time.
It includes: It includes:
- Creating a default site (GÉANT) - Creating a default site (GEANT)
- Creating device roles (Router) - Creating device roles (Router)
""" """
typer.echo("Initial setup of NetBox ...") typer.echo("Initial setup of NetBox ...")
...@@ -22,7 +22,7 @@ def netbox_initial_setup() -> None: ...@@ -22,7 +22,7 @@ def netbox_initial_setup() -> None:
nbclient = NetboxClient() nbclient = NetboxClient()
typer.echo("Creating GÉANT site ...") typer.echo("Creating GEANT site ...")
try: try:
nbclient.create_device_site(DEFAULT_SITE["name"], DEFAULT_SITE["slug"]) nbclient.create_device_site(DEFAULT_SITE["name"], DEFAULT_SITE["slug"])
typer.echo("Site created successfully.") typer.echo("Site created successfully.")
......
"""Initializes the GSO database module with model definitions and utilities."""
"""Database model definitions and table mappings for the GSO system."""
import enum
import structlog
from orchestrator.db import UtcTimestamp
from orchestrator.db.database import BaseModel
from sqlalchemy import (
Enum,
String,
text,
)
from sqlalchemy.dialects.postgresql import ARRAY
from sqlalchemy.orm import mapped_column
logger = structlog.get_logger(__name__)
class PartnerType(str, enum.Enum):
"""Defining different types of partners in the GSO system."""
NREN = "NREN"
RE_PEER = "RE_PEER"
PUBLIC_PEER = "PUBLIC_PEER"
PRIVATE_PEER = "PRIVATE_PEER"
UPSTREAM = "UPSTREAM"
GEANT = "GEANT"
class PartnerTable(BaseModel):
"""Database table for the partners in the GSO system."""
__tablename__ = "partners"
partner_id = mapped_column(String, server_default=text("uuid_generate_v4"), primary_key=True)
name = mapped_column(String, unique=True)
email = mapped_column(String, unique=True, nullable=True)
as_number = mapped_column(
String, unique=True
) # the as_number and as_set are mutually exclusive. if you give me one I don't need the other
as_set = mapped_column(String)
route_set = mapped_column(String, nullable=True)
black_listed_as_sets = mapped_column(ARRAY(String), nullable=True)
additional_routers = mapped_column(ARRAY(String), nullable=True)
additional_bgp_speakers = mapped_column(ARRAY(String), nullable=True)
partner_type = mapped_column(Enum(PartnerType), nullable=False)
created_at = mapped_column(UtcTimestamp, server_default=text("current_timestamp"), nullable=False)
updated_at = mapped_column(
UtcTimestamp, server_default=text("current_timestamp"), nullable=False, onupdate=text("current_timestamp")
)
...@@ -5,6 +5,8 @@ from orchestrator.db.database import BaseModel ...@@ -5,6 +5,8 @@ from orchestrator.db.database import BaseModel
from orchestrator.settings import app_settings from orchestrator.settings import app_settings
from sqlalchemy import engine_from_config, pool, text from sqlalchemy import engine_from_config, pool, text
from gso.db.models import PartnerTable # noqa: F401
# this is the Alembic Config object, which provides # this is the Alembic Config object, which provides
# access to the values within the .ini file in use. # access to the values within the .ini file in use.
config = context.config config = context.config
......
...@@ -10,9 +10,9 @@ from alembic import op ...@@ -10,9 +10,9 @@ from alembic import op
# revision identifiers, used by Alembic. # revision identifiers, used by Alembic.
revision = 'e8378fbcfbf3' revision = 'e8378fbcfbf3'
down_revision = None down_revision = 'da5c9f4cce1c'
branch_labels = ('data',) branch_labels = ('data',)
depends_on = 'da5c9f4cce1c' depends_on = None
def upgrade() -> None: def upgrade() -> None:
......
...@@ -10,7 +10,7 @@ from alembic import op ...@@ -10,7 +10,7 @@ from alembic import op
# revision identifiers, used by Alembic. # revision identifiers, used by Alembic.
revision = '1c3c90ea1d8c' revision = '1c3c90ea1d8c'
down_revision = '113a81d2a40a' down_revision = '5bea5647f61d'
branch_labels = None branch_labels = None
depends_on = None depends_on = None
......
"""Added Partner table.
Revision ID: eaed66b04913
Revises: 6e4952687205
Create Date: 2024-03-18 15:03:32.721760
"""
import sqlalchemy as sa
from alembic import op
from orchestrator.db import UtcTimestamp
from sqlalchemy.dialects import postgresql
# revision identifiers, used by Alembic.
revision = 'eaed66b04913'
down_revision = '6e4952687205'
branch_labels = None
depends_on = None
def upgrade() -> None:
op.create_table('partners',
sa.Column('partner_id', sa.String(), server_default=sa.text('uuid_generate_v4()'), nullable=False),
sa.Column('name', sa.String(), nullable=False),
sa.Column('email', sa.String(), nullable=True),
sa.Column('as_number', sa.String(), nullable=True),
sa.Column('as_set', sa.String(), nullable=True),
sa.Column('route_set', sa.String(), nullable=True),
sa.Column('black_listed_as_sets', postgresql.ARRAY(sa.String()), nullable=True),
sa.Column('additional_routers', postgresql.ARRAY(sa.String()), nullable=True),
sa.Column('additional_bgp_speakers', postgresql.ARRAY(sa.String()), nullable=True),
sa.Column('partner_type', sa.Enum('NREN', 'RE_PEER', 'PUBLIC_PEER', 'PRIVATE_PEER', 'UPSTREAM', 'GEANT', name='partnertype'), nullable=False),
sa.Column('created_at', UtcTimestamp(timezone=True), server_default=sa.text('current_timestamp'), nullable=False),
sa.Column('updated_at', UtcTimestamp(timezone=True), server_default=sa.text('current_timestamp'), nullable=False, onupdate=sa.text('current_timestamp')),
sa.PrimaryKeyConstraint('partner_id'),
sa.UniqueConstraint('as_number'),
sa.UniqueConstraint('email'),
sa.UniqueConstraint('name'),
)
# Add GEANT as a partner
conn = op.get_bind()
conn.execute(
sa.text("""INSERT INTO partners (partner_id, name, partner_type) VALUES ('8f0df561-ce9d-4d9c-89a8-7953d3ffc961', 'GEANT', 'GEANT' )"""))
# Add foreign key constraint to existing subscriptions table
op.create_foreign_key(
'fk_subscriptions_customer_id_partners',
'subscriptions',
'partners', #
['customer_id'],
['partner_id']
)
# Make customer_id not nullable
op.alter_column('subscriptions', 'customer_id',
existing_type=sa.String(length=255),
nullable=False)
def downgrade() -> None:
# Drop foreign key constraint
op.drop_constraint('fk_subscriptions_customer_id_partners', 'subscriptions', type_='foreignkey')
# Make customer_id nullable again
op.alter_column('subscriptions', 'customer_id',
existing_type=sa.String(length=255),
nullable=True)
op.drop_table('partners')
\ No newline at end of file
{ {
"GENERAL": { "GENERAL": {
"public_hostname": "https://gap.geant.org" "public_hostname": "https://gap.geant.org",
"isis_high_metric": 999999
}, },
"NETBOX": { "NETBOX": {
"api": "https://127.0.0.1:8000", "api": "https://127.0.0.1:8000",
...@@ -15,37 +16,37 @@ ...@@ -15,37 +16,37 @@
"password": "robot-user-password" "password": "robot-user-password"
}, },
"LO": { "LO": {
"V4": {"containers": [], "networks": ["1.1.0.0/24"], "mask": 0}, "V4": {"containers": [], "networks": ["10.255.255.0/26"], "mask": 32},
"V6": {"containers": [], "networks": ["dead:beef::/64"], "mask": 0}, "V6": {"containers": [], "networks": ["dead:beef::/80"], "mask": 128},
"domain_name": ".lo", "domain_name": ".geant.net",
"dns_view": "default", "dns_view": "default",
"network_view": "default" "network_view": "default"
}, },
"TRUNK": { "TRUNK": {
"V4": {"containers": ["1.1.1.0/24"], "networks": [], "mask": 31}, "V4": {"containers": ["10.255.255.0/24", "10.255.254.0/24"], "networks": [], "mask": 31},
"V6": {"containers": ["dead:beef::/64"], "networks": [], "mask": 126}, "V6": {"containers": ["dead:beef::/64", "dead:beee::/64"], "networks": [], "mask": 126},
"domain_name": ".trunk", "domain_name": ".trunk",
"dns_view": "default", "dns_view": "default",
"network_view": "default" "network_view": "default"
}, },
"GEANT_IP": { "GEANT_IP": {
"V4": {"containers": ["1.1.2.0/24"], "networks": [], "mask": 31}, "V4": {"containers": ["10.255.255.0/24", "10.255.254.0/24"], "networks": [], "mask": 31},
"V6": {"containers": ["dead:beef::/64"], "networks": [], "mask": 126}, "V6": {"containers": ["dead:beef::/64", "dead:beee::/64"], "networks": [], "mask": 126},
"domain_name": ".geantip", "domain_name": ".geantip",
"dns_view": "default", "dns_view": "default",
"network_view": "default" "network_view": "default"
}, },
"SI": { "SI": {
"V4": {"containers": ["1.1.3.0/24"], "networks": [], "mask": 31}, "V4": {"containers": ["10.255.253.128/25"], "networks": [], "mask": 31},
"V6": {"containers": ["dead:beef::/64"], "networks": [], "mask": 126}, "V6": {"containers": [], "networks": [], "mask": 126},
"domain_name": ".si", "domain_name": ".geantip",
"dns_view": "default", "dns_view": "default",
"network_view": "default" "network_view": "default"
}, },
"LT_IAS": { "LT_IAS": {
"V4": {"containers": ["1.1.4.0/24"], "networks": [], "mask": 31}, "V4": {"containers": ["10.255.255.0/24"], "networks": [], "mask": 31},
"V6": {"containers": ["dead:beef::/64"], "networks": [], "mask": 126}, "V6": {"containers": ["dead:beef:cc::/48"], "networks": [], "mask": 126},
"domain_name": ".ltias", "domain_name": ".geantip",
"dns_view": "default", "dns_view": "default",
"network_view": "default" "network_view": "default"
} }
...@@ -90,5 +91,8 @@ ...@@ -90,5 +91,8 @@
"starttls_enabled": true, "starttls_enabled": true,
"smtp_username": "username", "smtp_username": "username",
"smtp_password": "password" "smtp_password": "password"
},
"SHAREPOINT": {
"checklist_site_url": "https://example.sharepoint.com/sites/example-site"
} }
} }
"""It is used to group the schema files together as a package."""
"""Partner schema module."""
from datetime import datetime
from uuid import uuid4
from pydantic import BaseModel, EmailStr, Field
from gso.db.models import PartnerType
class PartnerCreate(BaseModel):
"""Partner create schema."""
partner_id: str = Field(default_factory=lambda: str(uuid4()))
name: str
email: EmailStr | None = None
as_number: str | None = Field(None, unique=True)
as_set: str | None = None
route_set: str | None = None
black_listed_as_sets: list[str] | None = None
additional_routers: list[str] | None = None
additional_bgp_speakers: list[str] | None = None
partner_type: PartnerType
created_at: datetime = Field(default_factory=lambda: datetime.now().astimezone())
updated_at: datetime = Field(default_factory=lambda: datetime.now().astimezone())
class Config:
"""Pydantic model configuration."""
orm_mode = True
"""A module that returns the customers available in :term:`GSO`.
For the time being, it's hardcoded to only contain GÉANT as a customer, since this is needed for the deployment of phase
1.
"""
from typing import Any
class CustomerNotFoundError(Exception):
"""Exception raised when a customer is not found."""
def all_customers() -> list[dict]:
"""Hardcoded list of customers available in :term:`GSO`."""
return [
{
"id": "8f0df561-ce9d-4d9c-89a8-7953d3ffc961",
"name": "GÉANT",
},
]
def get_customer_by_name(name: str) -> dict[str, Any]:
"""Try to get a customer by their name."""
for customer in all_customers():
if customer["name"] == name:
return customer
msg = f"Customer {name} not found"
raise CustomerNotFoundError(msg)
...@@ -257,8 +257,8 @@ def create_host_by_ip( ...@@ -257,8 +257,8 @@ def create_host_by_ip(
raise AllocationError(msg) raise AllocationError(msg)
conn, oss = _setup_connection() conn, oss = _setup_connection()
ipv6_object = objects.IP.create(ip=ipv6_address, mac=NULL_MAC, configure_for_dhcp=False) ipv6_object = objects.IP.create(ip=str(ipv6_address), mac=NULL_MAC, configure_for_dhcp=False)
ipv4_object = objects.IP.create(ip=ipv4_address, mac=NULL_MAC, configure_for_dhcp=False) ipv4_object = objects.IP.create(ip=str(ipv4_address), mac=NULL_MAC, configure_for_dhcp=False)
dns_view = getattr(oss, service_type).dns_view dns_view = getattr(oss, service_type).dns_view
network_view = getattr(oss, service_type).network_view network_view = getattr(oss, service_type).network_view
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment