Skip to content
Snippets Groups Projects
Commit 61ae3f6e authored by Mohammad Torkashvand's avatar Mohammad Torkashvand
Browse files

added partner model

added import script for reading partners from csv files
parent f206ea66
No related branches found
No related tags found
No related merge requests found
Pipeline #85917 failed
This commit is part of merge request !176. Comments created here will be created in the context of that merge request.
Showing
with 309 additions and 80 deletions
example.csv 0 → 100644
name,email,as_number,as_set,route_set,black_listed_as_sets,additional_routers,additional_bgp_speakers,partner_type,created_at
Partner A,partnera@example.com,AS1234,,,"AS100,AS200",,"Router1,Router2",NREN,2024-03-14
Partner B,partnerb@example.com,AS5678,,,"AS300,AS400",,"Router3,Router4",RE_PEER,2024-03-15
Partner C,partnerc@example.com,AS91011,,,"AS500,AS600",,"Router5,Router6",PUBLIC_PEER,2024-03-16
......@@ -14,7 +14,7 @@ from gso.products.product_blocks.iptrunk import IptrunkType, PhyPortCapacity
from gso.products.product_blocks.router import RouterRole
from gso.products.product_blocks.site import SiteTier
from gso.services import subscriptions
from gso.services.crm import CustomerNotFoundError, get_customer_by_name
from gso.services.partners import PartnerNotFoundError, get_partner_by_name
from gso.utils.helpers import BaseSiteValidatorModel, LAGMember
from gso.utils.shared_enums import PortNumber, Vendor
......@@ -41,13 +41,13 @@ class SiteImportModel(BaseSiteValidatorModel):
site_internal_id: int
site_tier: SiteTier
site_ts_address: str
customer: str
partner: str
class RouterImportModel(BaseModel):
"""Required fields for importing an existing :class:`gso.product.product_types.router`."""
customer: str
partner: str
router_site: str
hostname: str
ts_port: int
......@@ -61,7 +61,7 @@ class RouterImportModel(BaseModel):
class IptrunkImportModel(BaseModel):
"""Required fields for importing an existing :class:`gso.products.product_types.iptrunk`."""
customer: str
partner: str
geant_s_sid: str
iptrunk_type: IptrunkType
iptrunk_description: str
......@@ -87,13 +87,13 @@ class IptrunkImportModel(BaseModel):
for router in subscriptions.get_active_router_subscriptions(includes=["subscription_id"])
}
@validator("customer")
def check_if_customer_exists(cls, value: str) -> str:
"""Validate that the customer exists."""
@validator("partner")
def check_if_partner_exists(cls, value: str) -> str:
"""Validate that the partner exists."""
try:
get_customer_by_name(value)
except CustomerNotFoundError as e:
msg = f"Customer {value} not found"
get_partner_by_name(value)
except PartnerNotFoundError as e:
msg = f"partner {value} not found"
raise ValueError(msg) from e
return value
......@@ -140,7 +140,7 @@ class IptrunkImportModel(BaseModel):
class SuperPopSwitchImportModel(BaseModel):
"""Required fields for importing an existing :class:`gso.product.product_types.super_pop_switch`."""
customer: str
partner: str
super_pop_switch_site: str
hostname: str
super_pop_switch_ts_port: PortNumber
......@@ -150,7 +150,7 @@ class SuperPopSwitchImportModel(BaseModel):
class OfficeRouterImportModel(BaseModel):
"""Required fields for importing an existing :class:`gso.product.product_types.office_router`."""
customer: str
partner: str
office_router_site: str
office_router_fqdn: str
office_router_ts_port: PortNumber
......
""":term:`CLI` command for importing data to coreDB."""
import csv
import ipaddress
import json
from datetime import datetime, UTC
from pathlib import Path
from typing import TypeVar
import typer
import yaml
from orchestrator.db import db
from pydantic import ValidationError
from sqlalchemy.exc import SQLAlchemyError
from gso.api.v1.imports import (
IptrunkImportModel,
......@@ -21,6 +25,7 @@ from gso.api.v1.imports import (
import_site,
import_super_pop_switch,
)
from gso.db.models import PartnerTable
from gso.services.subscriptions import get_active_subscriptions_by_field_and_value
app: typer.Typer = typer.Typer()
......@@ -63,7 +68,7 @@ def generic_import_data(
successfully_imported_data = []
data = read_data(filepath)
for details in data:
details["customer"] = "GEANT"
details["partner"] = "GEANT"
typer.echo(f"Importing {name_key}: {details[name_key]}")
try:
initial_data = import_model(**details)
......@@ -163,7 +168,7 @@ def import_iptrunks(filepath: str = common_filepath_option) -> None:
)
try:
initial_data = IptrunkImportModel(
customer="GEANT",
partner="GEANT",
geant_s_sid=trunk["id"],
iptrunk_type=trunk["config"]["common"]["type"],
iptrunk_description=trunk["config"]["common"].get("description", ""),
......@@ -197,3 +202,36 @@ def import_iptrunks(filepath: str = common_filepath_option) -> None:
typer.echo("Successfully imported IP Trunks:")
for item in successfully_imported_data:
typer.echo(f"- {item}")
def import_partners_from_csv(file_path: Path) -> list[dict]:
"""Read partners from a CSV file."""
with Path.open(file_path, encoding="utf-8") as csv_file:
csv_reader = csv.DictReader(csv_file)
return list(csv_reader)
@app.command()
def import_partners(file_path: str = typer.Argument(..., help="Path to the CSV file containing partners")):
"""Import partners from a CSV file into the database."""
typer.echo(f"Importing partners from {file_path} ...")
partners = import_partners_from_csv(file_path)
try:
for partner in partners:
if partner.get("created_at"):
partner["created_at"] = datetime.strptime(partner["created_at"], "%Y-%m-%d").replace(
tzinfo=UTC
)
new_partner = PartnerTable(**partner)
db.session.add(new_partner)
db.session.commit()
typer.echo(f"Successfully imported {len(partners)} partners.")
except SQLAlchemyError as e:
db.session.rollback()
typer.echo(f"Failed to import partners: {e}")
finally:
db.session.close()
"""Initializes the GSO database module with model definitions and utilities."""
"""Database model definitions and table mappings for the GSO system."""
import enum
import structlog
from orchestrator.db import UtcTimestamp
from orchestrator.db.database import BaseModel
from sqlalchemy import (
Enum,
String,
text,
)
from sqlalchemy.dialects.postgresql import ARRAY
from sqlalchemy.orm import mapped_column
logger = structlog.get_logger(__name__)
class PartnerType(str, enum.Enum):
"""Enum defining different types of partners in the GSO system."""
NREN = "NREN"
RE_PEER = "RE_PEER"
PUBLIC_PEER = "PUBLIC_PEER"
PRIVATE_PEER = "PRIVATE_PEER"
UPSTREAM = "UPSTREAM"
GEANT = "GEANT"
class PartnerTable(BaseModel):
"""Database table for the partners in the GSO system."""
__tablename__ = "partners"
partner_id = mapped_column(String, server_default=text("uuid_generate_v4"), primary_key=True)
name = mapped_column(String, unique=True)
email = mapped_column(String, unique=True, nullable=True)
as_number = mapped_column(
String, unique=True
) # the as_number and as_set are mutually exclusive. if you give me one I don't need the other
as_set = mapped_column(String)
route_set = mapped_column(String, nullable=True)
black_listed_as_sets = mapped_column(ARRAY(String), nullable=True)
additional_routers = mapped_column(ARRAY(String), nullable=True)
additional_bgp_speakers = mapped_column(ARRAY(String), nullable=True)
partner_type = mapped_column(Enum(PartnerType), nullable=False)
created_at = mapped_column(UtcTimestamp, server_default=text("current_timestamp"), nullable=False)
updated_at = mapped_column(
UtcTimestamp, server_default=text("current_timestamp"), nullable=False, onupdate=text("current_timestamp")
)
......@@ -5,6 +5,8 @@ from orchestrator.db.database import BaseModel
from orchestrator.settings import app_settings
from sqlalchemy import engine_from_config, pool, text
from gso.db.models import PartnerTable # noqa: F401
# this is the Alembic Config object, which provides
# access to the values within the .ini file in use.
config = context.config
......
......@@ -10,9 +10,9 @@ from alembic import op
# revision identifiers, used by Alembic.
revision = 'e8378fbcfbf3'
down_revision = None
down_revision = 'da5c9f4cce1c'
branch_labels = ('data',)
depends_on = 'da5c9f4cce1c'
depends_on = None
def upgrade() -> None:
......
......@@ -10,7 +10,7 @@ from alembic import op
# revision identifiers, used by Alembic.
revision = '1c3c90ea1d8c'
down_revision = '113a81d2a40a'
down_revision = '5bea5647f61d'
branch_labels = None
depends_on = None
......
"""Added Partner table.
Revision ID: d6800280b31a
Revises: 6e4952687205
Create Date: 2024-03-14 16:38:55.948838
"""
import sqlalchemy as sa
from alembic import op
from orchestrator.db import UtcTimestamp
from sqlalchemy.dialects import postgresql
# revision identifiers, used by Alembic.
revision = 'd6800280b31a'
down_revision = '6e4952687205'
branch_labels = None
depends_on = None
def upgrade() -> None:
op.create_table('partners',
sa.Column('partner_id', sa.String(), server_default=sa.text('uuid_generate_v4()'), nullable=False),
sa.Column('name', sa.String(), nullable=False),
sa.Column('email', sa.String(), nullable=True),
sa.Column('as_number', sa.String(), nullable=True),
sa.Column('as_set', sa.String(), nullable=True),
sa.Column('route_set', sa.String(), nullable=True),
sa.Column('black_listed_as_sets', postgresql.ARRAY(sa.String()), nullable=True),
sa.Column('additional_routers', postgresql.ARRAY(sa.String()), nullable=True),
sa.Column('additional_bgp_speakers', postgresql.ARRAY(sa.String()), nullable=True),
sa.Column('partner_type', sa.Enum('NREN', 'RE_PEER', 'PUBLIC_PEER', 'PRIVATE_PEER', 'UPSTREAM', 'GEANT', name='partnertype'), nullable=False),
sa.Column('created_at', UtcTimestamp(timezone=True), server_default=sa.text('current_timestamp'), nullable=False),
sa.Column('updated_at', UtcTimestamp(timezone=True), server_default=sa.text('current_timestamp'), nullable=False, onupdate=sa.text('current_timestamp')),
sa.PrimaryKeyConstraint('partner_id'),
sa.UniqueConstraint('as_number'),
sa.UniqueConstraint('email'),
sa.UniqueConstraint('name'),
)
# Add GEANT as a partner
conn = op.get_bind()
conn.execute(
sa.text("""INSERT INTO partners (partner_id, name, partner_type) VALUES ('8f0df561-ce9d-4d9c-89a8-7953d3ffc961', 'GEANT', 'GEANT' )"""))
# Add foreign key constraint to existing subscriptions table
op.create_foreign_key(
'fk_subscriptions_customer_id_partners',
'subscriptions',
'partners', #
['customer_id'],
['partner_id']
)
# Make customer_id not nullable
op.alter_column('subscriptions', 'customer_id',
existing_type=sa.String(length=255),
nullable=False)
def downgrade() -> None:
# Drop foreign key constraint
op.drop_constraint('fk_subscriptions_customer_id_partners', 'subscriptions', type_='foreignkey')
# Make customer_id nullable again
op.alter_column('subscriptions', 'customer_id',
existing_type=sa.String(length=255),
nullable=True)
op.drop_table('partners')
"""Partner schema module."""
"""Partner schema module."""
from datetime import datetime
from uuid import uuid4
from pydantic import BaseModel, EmailStr, Field
from gso.db.models import PartnerType
class PartnerCreate(BaseModel):
"""Partner create schema."""
partner_id: str = Field(default_factory=lambda: str(uuid4()))
name: str
email: EmailStr | None = None
as_number: str | None = Field(None, unique=True)
as_set: str | None = None
route_set: str | None = None
black_listed_as_sets: list[str] | None = None
additional_routers: list[str] | None = None
additional_bgp_speakers: list[str] | None = None
partner_type: PartnerType
created_at: datetime = Field(default_factory=lambda: datetime.now().astimezone())
updated_at: datetime = Field(default_factory=lambda: datetime.now().astimezone())
class Config:
"""Pydantic model configuration."""
orm_mode = True
"""A module that returns the customers available in :term:`GSO`.
For the time being, it's hardcoded to only contain GEANT as a customer, since this is needed for the deployment of phase
1.
"""
from typing import Any
class CustomerNotFoundError(Exception):
"""Exception raised when a customer is not found."""
def all_customers() -> list[dict]:
"""Hardcoded list of customers available in :term:`GSO`."""
return [
{
"id": "8f0df561-ce9d-4d9c-89a8-7953d3ffc961",
"name": "GEANT",
},
]
def get_customer_by_name(name: str) -> dict[str, Any]:
"""Try to get a customer by their name."""
for customer in all_customers():
if customer["name"] == name:
return customer
msg = f"Customer {name} not found"
raise CustomerNotFoundError(msg)
"""A module that returns the partners available in :term:`GSO`."""
from typing import Any
from orchestrator.db import db
from sqlalchemy.exc import NoResultFound, SQLAlchemyError
from gso.db.models import PartnerTable
from gso.schema.partner import PartnerCreate
class PartnerNotFoundError(Exception):
"""Exception raised when a partner is not found."""
def get_all_partners() -> list[dict]:
"""Fetch all partners from the database and serialize them to JSON."""
partners = PartnerTable.query.all()
return [partner.__json__() for partner in partners]
def get_partner_by_name(name: str) -> dict[str, Any]:
"""Try to get a partner by their name."""
try:
partner = PartnerTable.query.filter(PartnerTable.name == name).one()
return partner.__json__()
except NoResultFound as e:
msg = f"partner {name} not found"
raise PartnerNotFoundError(msg) from e
def create_partner(
partner_data: PartnerCreate,
) -> dict:
"""Create a new partner and add it to the database using Pydantic schema for validation.
:param partner_data: Partner data validated by Pydantic schema.
:return: JSON representation of the created partner.
"""
try:
new_partner = PartnerTable(**partner_data.dict())
db.session.add(new_partner)
db.session.commit()
return new_partner.__json__()
except SQLAlchemyError:
db.session.rollback()
raise
def delete_partner_by_name(name: str) -> None:
"""Delete a partner by their name."""
try:
partner = PartnerTable.query.filter(PartnerTable.name == name).one()
db.session.delete(partner)
db.session.commit()
except NoResultFound as e:
msg = f"partner {name} not found"
raise PartnerNotFoundError(msg) from e
except SQLAlchemyError:
db.session.rollback()
raise
......@@ -25,8 +25,8 @@ from gso.products.product_blocks.iptrunk import (
from gso.products.product_types.iptrunk import IptrunkInactive, IptrunkProvisioning
from gso.products.product_types.router import Router
from gso.services import infoblox, subscriptions
from gso.services.crm import get_customer_by_name
from gso.services.netbox_client import NetboxClient
from gso.services.partners import get_partner_by_name
from gso.services.provisioning_proxy import execute_playbook, pp_interaction
from gso.settings import load_oss_params
from gso.utils.helpers import (
......@@ -56,7 +56,7 @@ def initial_input_form_generator(product_name: str) -> FormGenerator:
title = product_name
tt_number: str
customer: str = ReadOnlyField("GEANT")
partner: str = ReadOnlyField("GEANT")
geant_s_sid: str
iptrunk_description: str
iptrunk_type: IptrunkType
......@@ -179,9 +179,9 @@ def initial_input_form_generator(product_name: str) -> FormGenerator:
@step("Create subscription")
def create_subscription(product: UUIDstr, customer: str) -> State:
def create_subscription(product: UUIDstr, partner: str) -> State:
"""Create a new subscription object in the database."""
subscription = IptrunkInactive.from_product_id(product, get_customer_by_name(customer)["id"])
subscription = IptrunkInactive.from_product_id(product, get_partner_by_name(partner)["id"])
return {
"subscription": subscription,
......
......@@ -17,8 +17,8 @@ from gso.products.product_blocks.router import RouterRole
from gso.products.product_types.router import RouterInactive, RouterProvisioning
from gso.products.product_types.site import Site
from gso.services import infoblox, subscriptions
from gso.services.crm import get_customer_by_name
from gso.services.netbox_client import NetboxClient
from gso.services.partners import get_partner_by_name
from gso.services.provisioning_proxy import pp_interaction
from gso.settings import load_oss_params
from gso.utils.helpers import generate_fqdn, iso_from_ipv4
......@@ -43,7 +43,7 @@ def initial_input_form_generator(product_name: str) -> FormGenerator:
title = product_name
tt_number: str
customer: str = ReadOnlyField("GEANT")
partner: str = ReadOnlyField("GEANT")
vendor: Vendor
router_site: _site_selector() # type: ignore[valid-type]
hostname: str
......@@ -71,9 +71,9 @@ def initial_input_form_generator(product_name: str) -> FormGenerator:
@step("Create subscription")
def create_subscription(product: UUIDstr, customer: str) -> State:
def create_subscription(product: UUIDstr, partner: str) -> State:
"""Create a new subscription object."""
subscription = RouterInactive.from_product_id(product, get_customer_by_name(customer)["id"])
subscription = RouterInactive.from_product_id(product, get_partner_by_name(partner)["id"])
return {
"subscription": subscription,
......
......@@ -11,7 +11,7 @@ from pydantic_forms.core import ReadOnlyField
from gso.products.product_blocks import site as site_pb
from gso.products.product_blocks.site import LatitudeCoordinate, LongitudeCoordinate
from gso.products.product_types import site
from gso.services.crm import get_customer_by_name
from gso.services.partners import get_partner_by_name
from gso.utils.helpers import BaseSiteValidatorModel
......@@ -22,7 +22,7 @@ def initial_input_form_generator(product_name: str) -> FormGenerator:
class Config:
title = product_name
customer: str = ReadOnlyField("GEANT")
partner: str = ReadOnlyField("GEANT")
site_name: str
site_city: str
site_country: str
......@@ -40,9 +40,9 @@ def initial_input_form_generator(product_name: str) -> FormGenerator:
@step("Create subscription")
def create_subscription(product: UUIDstr, customer: str) -> State:
def create_subscription(product: UUIDstr, partner: str) -> State:
"""Create a new subscription object in the service database."""
subscription = site.SiteInactive.from_product_id(product, get_customer_by_name(customer)["id"])
subscription = site.SiteInactive.from_product_id(product, get_partner_by_name(partner)["id"])
return {
"subscription": subscription,
......
......@@ -16,7 +16,7 @@ from gso.products.product_blocks.iptrunk import IptrunkInterfaceBlockInactive, I
from gso.products.product_types.iptrunk import IptrunkInactive, IptrunkProvisioning
from gso.products.product_types.router import Router
from gso.services import subscriptions
from gso.services.crm import get_customer_by_name
from gso.services.partners import get_partner_by_name
from gso.utils.helpers import LAGMember
......@@ -38,7 +38,7 @@ def initial_input_form_generator() -> FormGenerator:
class Config:
title = "Import Iptrunk"
customer: str
partner: str
geant_s_sid: str
iptrunk_description: str
iptrunk_type: IptrunkType
......@@ -65,11 +65,11 @@ def initial_input_form_generator() -> FormGenerator:
@step("Create a new subscription")
def create_subscription(customer: str) -> State:
def create_subscription(partner: str) -> State:
"""Create a new subscription in the service database."""
customer_id = get_customer_by_name(customer)["id"]
partner_id = get_partner_by_name(partner)["id"]
product_id = subscriptions.get_product_id_by_name(ProductType.IP_TRUNK)
subscription = IptrunkInactive.from_product_id(product_id, customer_id)
subscription = IptrunkInactive.from_product_id(product_id, partner_id)
return {
"subscription": subscription,
......
......@@ -13,17 +13,17 @@ from gso.products import ProductType
from gso.products.product_types import office_router
from gso.products.product_types.office_router import OfficeRouterInactive
from gso.services import subscriptions
from gso.services.crm import get_customer_by_name
from gso.services.partners import get_partner_by_name
from gso.services.subscriptions import get_site_by_name
from gso.utils.shared_enums import PortNumber, Vendor
@step("Create subscription")
def create_subscription(customer: str) -> State:
def create_subscription(partner: str) -> State:
"""Create a new subscription object."""
customer_id = get_customer_by_name(customer)["id"]
partner_id = get_partner_by_name(partner)["id"]
product_id = subscriptions.get_product_id_by_name(ProductType.OFFICE_ROUTER)
subscription = OfficeRouterInactive.from_product_id(product_id, customer_id)
subscription = OfficeRouterInactive.from_product_id(product_id, partner_id)
return {
"subscription": subscription,
......@@ -38,7 +38,7 @@ def initial_input_form_generator() -> FormGenerator:
class Config:
title = "Import an office router"
customer: str
partner: str
office_router_site: str
office_router_fqdn: str
office_router_ts_port: PortNumber
......
......@@ -15,18 +15,18 @@ from gso.products.product_blocks.router import RouterRole
from gso.products.product_types import router
from gso.products.product_types.router import RouterInactive
from gso.services import subscriptions
from gso.services.crm import get_customer_by_name
from gso.services.partners import get_partner_by_name
from gso.services.subscriptions import get_site_by_name
from gso.utils.helpers import generate_fqdn
from gso.utils.shared_enums import PortNumber, Vendor
@step("Create subscription")
def create_subscription(customer: str) -> State:
def create_subscription(partner: str) -> State:
"""Create a new subscription object."""
customer_id = get_customer_by_name(customer)["id"]
partner_id = get_partner_by_name(partner)["id"]
product_id = subscriptions.get_product_id_by_name(ProductType.ROUTER)
subscription = RouterInactive.from_product_id(product_id, customer_id)
subscription = RouterInactive.from_product_id(product_id, partner_id)
return {
"subscription": subscription,
......@@ -41,7 +41,7 @@ def initial_input_form_generator() -> FormGenerator:
class Config:
title = "Import Router"
customer: str
partner: str
router_site: str
hostname: str
ts_port: int
......
......@@ -12,19 +12,19 @@ from gso.products import ProductType
from gso.products.product_blocks.site import SiteTier
from gso.products.product_types.site import SiteInactive
from gso.services import subscriptions
from gso.services.crm import get_customer_by_name
from gso.services.partners import get_partner_by_name
from gso.workflows.site.create_site import initialize_subscription
@step("Create subscription")
def create_subscription(customer: str) -> State:
def create_subscription(partner: str) -> State:
"""Create a new subscription object in the service database.
FIXME: all attributes passed by the input form appear to be unused
"""
customer_id = get_customer_by_name(customer)["id"]
partner_id = get_partner_by_name(partner)["id"]
product_id: UUID = subscriptions.get_product_id_by_name(ProductType.SITE)
subscription = SiteInactive.from_product_id(product_id, customer_id)
subscription = SiteInactive.from_product_id(product_id, partner_id)
return {
"subscription": subscription,
......@@ -49,7 +49,7 @@ def generate_initial_input_form() -> FormGenerator:
site_internal_id: int
site_tier: SiteTier
site_ts_address: str
customer: str
partner: str
user_input = yield ImportSite
return user_input.dict()
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment