Finished release 0.9.

Changelog.md

 # Changelog
 
 All notable changes to this project will be documented in this file.
+## [0.9] - 2024-03-20
+- `migrate_iptrunk` workflow includes Ansible trunk checks.
+- `create_iptrunk` and `migrate_iptrunk` now update IPAM / DNS.
+- lso result step title is now always the name of the provisioning step
 
 ## [0.8] - 2024-02-28
 - Add two new workflows for "activating" `Router` and `Iptrunk` products.

docs/source/conf.py

 # -- Project information -----------------------------------------------------
 project = "GÉANT Service Orchestrator"
-copyright = "2023, GÉANT Vereniging"
+copyright = "2023-2024, GÉANT Vereniging"
 author = "GÉANT Orchestration and Automation Team"
 
 # -- General configuration ---------------------------------------------------

docs/source/module/services/index.rst

@@ -15,6 +15,6 @@ Submodules
    crm
    infoblox
    librenms_client
+   lso_client
    netbox_client
-   provisioning_proxy
    subscriptions

docs/source/module/services/lso_client.rst

+``gso.services.lso_client``
+===========================
+
+.. automodule:: gso.services.lso_client
+   :members:
+   :show-inheritance:

docs/source/module/services/provisioning_proxy.rst

-``gso.services.provisioning_proxy``
-===================================
-
-.. automodule:: gso.services.provisioning_proxy
-   :members:
-   :show-inheritance:

docs/vale/styles/config/vocabularies/geant-jargon/accept.txt

@@ -6,6 +6,7 @@ WFO
 Ansible
 [Dd]eprovision
 API
+DNS
 dry_run
 Dark_fiber
 [A|a]ddress

gso/api/v1/imports.py

@@ -14,7 +14,7 @@ from gso.products.product_blocks.iptrunk import IptrunkType, PhyPortCapacity
 from gso.products.product_blocks.router import RouterRole
 from gso.products.product_blocks.site import SiteTier
 from gso.services import subscriptions
-from gso.services.crm import CustomerNotFoundError, get_customer_by_name
+from gso.services.partners import PartnerNotFoundError, get_partner_by_name
 from gso.utils.helpers import BaseSiteValidatorModel, LAGMember
 from gso.utils.shared_enums import PortNumber, Vendor
 
@@ -41,13 +41,13 @@ class SiteImportModel(BaseSiteValidatorModel):
     site_internal_id: int
     site_tier: SiteTier
     site_ts_address: str
-    customer: str
+    partner: str
 
 
 class RouterImportModel(BaseModel):
     """Required fields for importing an existing :class:`gso.product.product_types.router`."""
 
-    customer: str
+    partner: str
     router_site: str
     hostname: str
     ts_port: int
@@ -61,7 +61,7 @@ class RouterImportModel(BaseModel):
 class IptrunkImportModel(BaseModel):
     """Required fields for importing an existing :class:`gso.products.product_types.iptrunk`."""
 
-    customer: str
+    partner: str
     geant_s_sid: str
     iptrunk_type: IptrunkType
     iptrunk_description: str
@@ -87,13 +87,13 @@ class IptrunkImportModel(BaseModel):
             for router in subscriptions.get_active_router_subscriptions(includes=["subscription_id"])
         }
 
-    @validator("customer")
-    def check_if_customer_exists(cls, value: str) -> str:
-        """Validate that the customer exists."""
+    @validator("partner")
+    def check_if_partner_exists(cls, value: str) -> str:
+        """Validate that the partner exists."""
         try:
-            get_customer_by_name(value)
-        except CustomerNotFoundError as e:
-            msg = f"Customer {value} not found"
+            get_partner_by_name(value)
+        except PartnerNotFoundError as e:
+            msg = f"partner {value} not found"
             raise ValueError(msg) from e
 
         return value
@@ -140,7 +140,7 @@ class IptrunkImportModel(BaseModel):
 class SuperPopSwitchImportModel(BaseModel):
     """Required fields for importing an existing :class:`gso.product.product_types.super_pop_switch`."""
 
-    customer: str
+    partner: str
     super_pop_switch_site: str
     hostname: str
     super_pop_switch_ts_port: PortNumber
@@ -150,7 +150,7 @@ class SuperPopSwitchImportModel(BaseModel):
 class OfficeRouterImportModel(BaseModel):
     """Required fields for importing an existing :class:`gso.product.product_types.office_router`."""
 
-    customer: str
+    partner: str
     office_router_site: str
     office_router_fqdn: str
     office_router_ts_port: PortNumber

gso/cli/imports.py

 """:term:`CLI` command for importing data to coreDB."""
 
+import csv
 import ipaddress
 import json
+from datetime import UTC, datetime
 from pathlib import Path
 from typing import TypeVar
 
 import typer
 import yaml
+from orchestrator.db import db
 from pydantic import ValidationError
+from sqlalchemy.exc import SQLAlchemyError
 
 from gso.api.v1.imports import (
     IptrunkImportModel,
@@ -21,6 +25,7 @@ from gso.api.v1.imports import (
     import_site,
     import_super_pop_switch,
 )
+from gso.db.models import PartnerTable
 from gso.services.subscriptions import get_active_subscriptions_by_field_and_value
 
 app: typer.Typer = typer.Typer()
@@ -63,7 +68,7 @@ def generic_import_data(
     successfully_imported_data = []
     data = read_data(filepath)
     for details in data:
-        details["customer"] = "GÉANT"
+        details["partner"] = "GEANT"
         typer.echo(f"Importing {name_key}: {details[name_key]}")
         try:
             initial_data = import_model(**details)
@@ -163,7 +168,7 @@ def import_iptrunks(filepath: str = common_filepath_option) -> None:
         )
         try:
             initial_data = IptrunkImportModel(
-                customer="GÉANT",
+                partner="GEANT",
                 geant_s_sid=trunk["id"],
                 iptrunk_type=trunk["config"]["common"]["type"],
                 iptrunk_description=trunk["config"]["common"].get("description", ""),
@@ -197,3 +202,34 @@ def import_iptrunks(filepath: str = common_filepath_option) -> None:
         typer.echo("Successfully imported IP Trunks:")
         for item in successfully_imported_data:
             typer.echo(f"- {item}")
+
+
+def import_partners_from_csv(file_path: Path) -> list[dict]:
+    """Read partners from a CSV file."""
+    with Path.open(file_path, encoding="utf-8") as csv_file:
+        csv_reader = csv.DictReader(csv_file)
+        return list(csv_reader)
+
+
+@app.command()
+def import_partners(file_path: str = typer.Argument(..., help="Path to the CSV file containing partners")) -> None:
+    """Import partners from a CSV file into the database."""
+    typer.echo(f"Importing partners from {file_path} ...")
+
+    partners = import_partners_from_csv(Path(file_path))
+
+    try:
+        for partner in partners:
+            if partner.get("created_at"):
+                partner["created_at"] = datetime.strptime(partner["created_at"], "%Y-%m-%d").replace(tzinfo=UTC)
+
+            new_partner = PartnerTable(**partner)
+            db.session.add(new_partner)
+
+        db.session.commit()
+        typer.echo(f"Successfully imported {len(partners)} partners.")
+    except SQLAlchemyError as e:
+        db.session.rollback()
+        typer.echo(f"Failed to import partners: {e}")
+    finally:
+        db.session.close()

gso/cli/netbox.py

@@ -14,7 +14,7 @@ def netbox_initial_setup() -> None:
     """Set up NetBox for the first time.
 
     It includes:
-    - Creating a default site (GÉANT)
+    - Creating a default site (GEANT)
     - Creating device roles (Router)
     """
     typer.echo("Initial setup of NetBox ...")
@@ -22,7 +22,7 @@ def netbox_initial_setup() -> None:
 
     nbclient = NetboxClient()
 
-    typer.echo("Creating GÉANT site ...")
+    typer.echo("Creating GEANT site ...")
     try:
         nbclient.create_device_site(DEFAULT_SITE["name"], DEFAULT_SITE["slug"])
         typer.echo("Site created successfully.")

gso/db/__init__.py

+"""Initializes the GSO database module with model definitions and utilities."""

gso/db/models.py

+"""Database model definitions and table mappings for the GSO system."""
+
+import enum
+
+import structlog
+from orchestrator.db import UtcTimestamp
+from orchestrator.db.database import BaseModel
+from sqlalchemy import (
+    Enum,
+    String,
+    text,
+)
+from sqlalchemy.dialects.postgresql import ARRAY
+from sqlalchemy.orm import mapped_column
+
+logger = structlog.get_logger(__name__)
+
+
+class PartnerType(str, enum.Enum):
+    """Defining different types of partners in the GSO system."""
+
+    NREN = "NREN"
+    RE_PEER = "RE_PEER"
+    PUBLIC_PEER = "PUBLIC_PEER"
+    PRIVATE_PEER = "PRIVATE_PEER"
+    UPSTREAM = "UPSTREAM"
+    GEANT = "GEANT"
+
+
+class PartnerTable(BaseModel):
+    """Database table for the partners in the GSO system."""
+
+    __tablename__ = "partners"
+
+    partner_id = mapped_column(String, server_default=text("uuid_generate_v4"), primary_key=True)
+    name = mapped_column(String, unique=True)
+    email = mapped_column(String, unique=True, nullable=True)
+    as_number = mapped_column(
+        String, unique=True
+    )  # the as_number and as_set are mutually exclusive. if you give me one I don't need the other
+    as_set = mapped_column(String)
+    route_set = mapped_column(String, nullable=True)
+    black_listed_as_sets = mapped_column(ARRAY(String), nullable=True)
+    additional_routers = mapped_column(ARRAY(String), nullable=True)
+    additional_bgp_speakers = mapped_column(ARRAY(String), nullable=True)
+
+    partner_type = mapped_column(Enum(PartnerType), nullable=False)
+    created_at = mapped_column(UtcTimestamp, server_default=text("current_timestamp"), nullable=False)
+    updated_at = mapped_column(
+        UtcTimestamp, server_default=text("current_timestamp"), nullable=False, onupdate=text("current_timestamp")
+    )

gso/migrations/env.py

@@ -5,6 +5,8 @@ from orchestrator.db.database import BaseModel
 from orchestrator.settings import app_settings
 from sqlalchemy import engine_from_config, pool, text
 
+from gso.db.models import PartnerTable  # noqa: F401
+
 # this is the Alembic Config object, which provides
 # access to the values within the .ini file in use.
 config = context.config

gso/migrations/versions/2023-11-21_e8378fbcfbf3_add_initial_products.py

@@ -10,9 +10,9 @@ from alembic import op
 
 # revision identifiers, used by Alembic.
 revision = 'e8378fbcfbf3'
-down_revision = None
+down_revision = 'da5c9f4cce1c'
 branch_labels = ('data',)
-depends_on = 'da5c9f4cce1c'
+depends_on = None
 
 
 def upgrade() -> None:

gso/migrations/versions/2024-02-14_1c3c90ea1d8c_add_super_pop_switch_product.py

@@ -10,7 +10,7 @@ from alembic import op
 
 # revision identifiers, used by Alembic.
 revision = '1c3c90ea1d8c'
-down_revision = '113a81d2a40a'
+down_revision = '5bea5647f61d'
 branch_labels = None
 depends_on = None
 

gso/migrations/versions/2024-03-18_eaed66b04913_added_partner_table.py

+"""Added Partner table.
+
+Revision ID: eaed66b04913
+Revises: 6e4952687205
+Create Date: 2024-03-18 15:03:32.721760
+
+"""
+import sqlalchemy as sa
+from alembic import op
+from orchestrator.db import UtcTimestamp
+from sqlalchemy.dialects import postgresql
+# revision identifiers, used by Alembic.
+revision = 'eaed66b04913'
+down_revision = '6e4952687205'
+branch_labels = None
+depends_on = None
+
+def upgrade() -> None:
+    op.create_table('partners',
+sa.Column('partner_id', sa.String(), server_default=sa.text('uuid_generate_v4()'), nullable=False),
+         sa.Column('name', sa.String(), nullable=False),
+         sa.Column('email', sa.String(), nullable=True),
+         sa.Column('as_number', sa.String(), nullable=True),
+         sa.Column('as_set', sa.String(), nullable=True),
+         sa.Column('route_set', sa.String(), nullable=True),
+         sa.Column('black_listed_as_sets', postgresql.ARRAY(sa.String()), nullable=True),
+         sa.Column('additional_routers', postgresql.ARRAY(sa.String()), nullable=True),
+         sa.Column('additional_bgp_speakers', postgresql.ARRAY(sa.String()), nullable=True),
+         sa.Column('partner_type', sa.Enum('NREN', 'RE_PEER', 'PUBLIC_PEER', 'PRIVATE_PEER', 'UPSTREAM', 'GEANT', name='partnertype'), nullable=False),
+         sa.Column('created_at', UtcTimestamp(timezone=True), server_default=sa.text('current_timestamp'), nullable=False),
+         sa.Column('updated_at', UtcTimestamp(timezone=True), server_default=sa.text('current_timestamp'), nullable=False, onupdate=sa.text('current_timestamp')),
+
+         sa.PrimaryKeyConstraint('partner_id'),
+         sa.UniqueConstraint('as_number'),
+         sa.UniqueConstraint('email'),
+         sa.UniqueConstraint('name'),
+    )
+
+    # Add GEANT as a partner
+    conn = op.get_bind()
+    conn.execute(
+        sa.text("""INSERT INTO partners (partner_id, name, partner_type) VALUES ('8f0df561-ce9d-4d9c-89a8-7953d3ffc961', 'GEANT', 'GEANT' )"""))
+
+    # Add foreign key constraint to existing subscriptions table
+    op.create_foreign_key(
+        'fk_subscriptions_customer_id_partners',
+        'subscriptions',
+        'partners',  #
+        ['customer_id'],
+        ['partner_id']
+    )
+
+    # Make customer_id not nullable
+    op.alter_column('subscriptions', 'customer_id',
+                    existing_type=sa.String(length=255),
+                    nullable=False)
+
+
+def downgrade() -> None:
+    # Drop foreign key constraint
+    op.drop_constraint('fk_subscriptions_customer_id_partners', 'subscriptions', type_='foreignkey')
+    # Make customer_id nullable again
+    op.alter_column('subscriptions', 'customer_id',
+                    existing_type=sa.String(length=255),
+                    nullable=True)
+
+    op.drop_table('partners')
\ No newline at end of file

gso/oss-params-example.json

 {
   "GENERAL": {
-    "public_hostname": "https://gap.geant.org"
+    "public_hostname": "https://gap.geant.org",
+    "isis_high_metric": 999999
   },
   "NETBOX": {
     "api": "https://127.0.0.1:8000",
@@ -15,37 +16,37 @@
       "password": "robot-user-password"
     },
     "LO": {
-      "V4": {"containers": [], "networks": ["1.1.0.0/24"], "mask": 0},
-      "V6": {"containers": [], "networks": ["dead:beef::/64"], "mask": 0},
-      "domain_name": ".lo",
+      "V4": {"containers": [], "networks": ["10.255.255.0/26"], "mask": 32},
+      "V6": {"containers": [], "networks": ["dead:beef::/80"], "mask": 128},
+      "domain_name": ".geant.net",
       "dns_view": "default",
       "network_view": "default"
     },
     "TRUNK": {
-      "V4": {"containers": ["1.1.1.0/24"], "networks": [], "mask": 31},
-      "V6": {"containers": ["dead:beef::/64"], "networks": [], "mask": 126},
+      "V4": {"containers": ["10.255.255.0/24", "10.255.254.0/24"], "networks": [], "mask": 31},
+      "V6": {"containers": ["dead:beef::/64", "dead:beee::/64"], "networks": [], "mask": 126},
       "domain_name": ".trunk",
       "dns_view": "default",
       "network_view": "default"
     },
     "GEANT_IP": {
-      "V4": {"containers": ["1.1.2.0/24"], "networks": [], "mask": 31},
-      "V6": {"containers": ["dead:beef::/64"], "networks": [], "mask": 126},
+      "V4": {"containers": ["10.255.255.0/24", "10.255.254.0/24"], "networks": [], "mask": 31},
+      "V6": {"containers": ["dead:beef::/64", "dead:beee::/64"], "networks": [], "mask": 126},
       "domain_name": ".geantip",
       "dns_view": "default",
       "network_view": "default"
     },
     "SI": {
-      "V4": {"containers": ["1.1.3.0/24"], "networks": [], "mask": 31},
-      "V6": {"containers": ["dead:beef::/64"], "networks": [], "mask": 126},
-      "domain_name": ".si",
+      "V4": {"containers": ["10.255.253.128/25"], "networks": [], "mask": 31},
+      "V6": {"containers": [], "networks": [], "mask": 126},
+      "domain_name": ".geantip",
       "dns_view": "default",
       "network_view": "default"
     },
     "LT_IAS": {
-      "V4": {"containers": ["1.1.4.0/24"], "networks": [], "mask": 31},
-      "V6": {"containers": ["dead:beef::/64"], "networks": [], "mask": 126},
-      "domain_name": ".ltias",
+      "V4": {"containers": ["10.255.255.0/24"], "networks": [], "mask": 31},
+      "V6": {"containers": ["dead:beef:cc::/48"], "networks": [], "mask": 126},
+      "domain_name": ".geantip",
       "dns_view": "default",
       "network_view": "default"
     }
@@ -90,5 +91,8 @@
     "starttls_enabled": true,
     "smtp_username": "username",
     "smtp_password": "password"
+  },
+  "SHAREPOINT": {
+    "checklist_site_url": "https://example.sharepoint.com/sites/example-site"
   }
 }

gso/schema/__init__.py

+"""It is used to group the schema files together as a package."""

gso/schema/partner.py

+"""Partner schema module."""
+
+from datetime import datetime
+from uuid import uuid4
+
+from pydantic import BaseModel, EmailStr, Field
+
+from gso.db.models import PartnerType
+
+
+class PartnerCreate(BaseModel):
+    """Partner create schema."""
+
+    partner_id: str = Field(default_factory=lambda: str(uuid4()))
+    name: str
+    email: EmailStr | None = None
+    as_number: str | None = Field(None, unique=True)
+    as_set: str | None = None
+    route_set: str | None = None
+    black_listed_as_sets: list[str] | None = None
+    additional_routers: list[str] | None = None
+    additional_bgp_speakers: list[str] | None = None
+    partner_type: PartnerType
+    created_at: datetime = Field(default_factory=lambda: datetime.now().astimezone())
+    updated_at: datetime = Field(default_factory=lambda: datetime.now().astimezone())
+
+    class Config:
+        """Pydantic model configuration."""
+
+        orm_mode = True

gso/services/crm.py

-"""A module that returns the customers available in :term:`GSO`.
-
-For the time being, it's hardcoded to only contain GÉANT as a customer, since this is needed for the deployment of phase
-1.
-"""
-
-from typing import Any
-
-
-class CustomerNotFoundError(Exception):
-    """Exception raised when a customer is not found."""
-
-
-def all_customers() -> list[dict]:
-    """Hardcoded list of customers available in :term:`GSO`."""
-    return [
-        {
-            "id": "8f0df561-ce9d-4d9c-89a8-7953d3ffc961",
-            "name": "GÉANT",
-        },
-    ]
-
-
-def get_customer_by_name(name: str) -> dict[str, Any]:
-    """Try to get a customer by their name."""
-    for customer in all_customers():
-        if customer["name"] == name:
-            return customer
-
-    msg = f"Customer {name} not found"
-    raise CustomerNotFoundError(msg)

gso/services/infoblox.py

@@ -257,8 +257,8 @@ def create_host_by_ip(
         raise AllocationError(msg)
 
     conn, oss = _setup_connection()
-    ipv6_object = objects.IP.create(ip=ipv6_address, mac=NULL_MAC, configure_for_dhcp=False)
-    ipv4_object = objects.IP.create(ip=ipv4_address, mac=NULL_MAC, configure_for_dhcp=False)
+    ipv6_object = objects.IP.create(ip=str(ipv6_address), mac=NULL_MAC, configure_for_dhcp=False)
+    ipv4_object = objects.IP.create(ip=str(ipv4_address), mac=NULL_MAC, configure_for_dhcp=False)
     dns_view = getattr(oss, service_type).dns_view
     network_view = getattr(oss, service_type).network_view