diff --git a/data/routers.json b/data/routers.json new file mode 100644 index 0000000000000000000000000000000000000000..7e1be59aefeef98f5d641f1f1af63c500ea5885f --- /dev/null +++ b/data/routers.json @@ -0,0 +1,25 @@ +[{ + "customer": "GÉANT", + "router_site": "AMS", + "hostname": "rt1.ams.nl.lab.office", + "ts_port": 22111, + "router_vendor": "juniper", + "router_role": "p", + "router_lo_ipv4_address": "62.40.119.2", + "router_lo_ipv6_address": "2001:798:1ab::2", + "router_lo_iso_address": "49.51e5.0001.0620.4011.9002.00", + "is_ias_connected": false +}, + { + "customer": "GÉANT", + "router_site": "AMS", + "hostname": "rt1.ath.gr.lab.office", + "ts_port": 22111, + "router_vendor": "juniper", + "router_role": "p", + "router_lo_ipv4_address": "62.40.119.1", + "router_lo_ipv6_address": "2001:798:1ab::1", + "router_lo_iso_address": "49.51e5.0001.0620.4011.9001.00", + "is_ias_connected": false +} +] diff --git a/data/sites.json b/data/sites.json new file mode 100644 index 0000000000000000000000000000000000000000..071e7984e2997165c58042f05551c554f5c70ffa --- /dev/null +++ b/data/sites.json @@ -0,0 +1,16 @@ +[ +{ + "site_name": "AMS2", + "site_city": "Amsterdam", + "site_country": "The Netherlands", + "site_country_code": "NL", + "site_latitude": 0, + "site_longitude": 0, + "site_bgp_community_id": 4, + "site_internal_id": 4, + "site_tier": "1", + "site_ts_address": "0.1.1.1", + "customer": "GÉANT" +} +] +//Missing: City, Country Code, Tier \ No newline at end of file diff --git a/data/sites.yaml b/data/sites.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d10d8b6d8d4034b396fc92cf1c9272cfb1014d18 --- /dev/null +++ b/data/sites.yaml @@ -0,0 +1,11 @@ +- site_name: AMS2 + site_city: Amsterdam + site_country: The Netherlands + site_country_code: NL + site_latitude: 0 + site_longitude: 0 + site_bgp_community_id: 4 + site_internal_id: 4 + site_tier: '1' + site_ts_address: 0.1.1.1 + customer: GÉANT \ No newline at end of file diff --git a/data/trunks.json b/data/trunks.json new file mode 100644 index 0000000000000000000000000000000000000000..1acc4b7198f21a4b0be775173fbc5c77f45f311f --- /dev/null +++ b/data/trunks.json @@ -0,0 +1,28 @@ +[{ + "customer": "GÉANT", + "geant_s_sid": "12", + "iptrunk_type": "Dark_fiber", + "iptrunk_description": "Description", + "iptrunk_speed": "100G", + "iptrunk_minimum_links": 1, + "side_a_node_id": "", + "side_a_ae_iface": "string", + "side_a_ae_geant_a_sid": "string", + "side_a_ae_members": [ + { + "interface_name": "string", + "interface_description": "string" + } + ], + "side_b_node_id": "string", + "side_b_ae_iface": "string", + "side_b_ae_geant_a_sid": "string", + "side_b_ae_members": [ + { + "interface_name": "string", + "interface_description": "string" + } + ], + "iptrunk_ipv4_network": "string", + "iptrunk_ipv6_network": "string" // calculated from ipv6 +}] diff --git a/data/trunks.yaml b/data/trunks.yaml new file mode 100644 index 0000000000000000000000000000000000000000..194fb83b4669801bdf036551fe8e3b98d67246ce --- /dev/null +++ b/data/trunks.yaml @@ -0,0 +1,25 @@ + - id: LGS-00001 # Comes from SM identifies the Trunk + config: + common: + link_speed: 100g + minimum_links: 1 + isis_metric: 500 + type: "Leased" + nodeA: + name: rt1.ath.gr.lab.office.geant.net + ae_name: ae0 + port_sid: LGA-00001 # Comes from SM - identifies the ae + members: + - interface_name: et-0/0/2 + interface_description: et-0/0/2 + ipv4_address: 62.40.98.0/31 + ipv6_address: 2001:798:cc::1/126 + nodeB: + name: rt1.ams.nl.lab.office.geant.net + ae_name: ae0 + port_sid: LGA-00002 # Comes from SM - identifies the ae + members: + - interface_name: et-9/0/2 + interface_description: et-9/0/2 + ipv4_address: 62.40.98.1/31 + ipv6_address: 2001:798:cc::2/126 diff --git a/gso/__init__.py b/gso/__init__.py index c92a6db2c3fb4820db8d3cb6f7a4e2ae6c3e7146..e400cf379de6d6a60bc3e2841e11b96d8636ce21 100644 --- a/gso/__init__.py +++ b/gso/__init__.py @@ -2,13 +2,13 @@ import typer from orchestrator import OrchestratorCore, app_settings -from orchestrator.cli.main import app as cli_app # noinspection PyUnresolvedReferences import gso.products import gso.workflows # noqa: F401 from gso.api import router as api_router -from gso.cli import netbox +from gso.cli import imports +from orchestrator.cli.main import app as cli_app def init_gso_app() -> OrchestratorCore: @@ -25,8 +25,8 @@ def init_worker_app() -> OrchestratorCore: def init_cli_app() -> typer.Typer: """Initialise :term:`GSO` as a CLI application.""" - from gso.cli import import_sites # noqa: PLC0415 + from gso.cli import import_sites, netbox # noqa: PLC0415 - cli_app.add_typer(import_sites.app, name="import_sites") + cli_app.add_typer(imports.app, name="import-cli") cli_app.add_typer(netbox.app, name="netbox-cli") return cli_app() diff --git a/gso/cli/import_sites.py b/gso/cli/import_sites.py deleted file mode 100644 index 21182766412e919bf1df1413ab42c4f19c00cae6..0000000000000000000000000000000000000000 --- a/gso/cli/import_sites.py +++ /dev/null @@ -1,12 +0,0 @@ -""":term:`CLI` command for importing sites.""" - -import typer - -app: typer.Typer = typer.Typer() - - -@app.command() -def import_sites() -> None: - """Import sites from a source.""" - # TODO: Implement this CLI command to import sites from a source. - typer.echo("Importing sites...") diff --git a/gso/cli/imports.py b/gso/cli/imports.py new file mode 100644 index 0000000000000000000000000000000000000000..335d0f5ebf72257173e3661e415279eab7bfb9e3 --- /dev/null +++ b/gso/cli/imports.py @@ -0,0 +1,161 @@ +""":term:`CLI` command for importing data to coreDB.""" + +import ipaddress +import json +import os +from typing import TypeVar, Type, Any + +import typer +import yaml +from pydantic import ValidationError + +from gso.api.v1.imports import ( + SiteImportModel, + import_site, + RouterImportModel, + import_router, + IptrunkImportModel, + import_iptrunk, +) +from gso.services.subscriptions import get_active_subscriptions_by_field_and_value + +app: typer.Typer = typer.Typer() + +T = TypeVar("T", SiteImportModel, RouterImportModel, IptrunkImportModel) + + +def read_data(filepath: str) -> dict | None: + """Read data from a JSON or YAML file.""" + + typer.echo(f"Starting import from {filepath}") + _, file_extension = os.path.splitext(filepath) + + with open(filepath, "r") as f: + if file_extension.lower() == ".json": + return json.load(f) + elif file_extension.lower() in (".yaml", ".yml"): + return yaml.safe_load(f) + else: + typer.echo(f"Unsupported file format: {file_extension}") + return + + +def generic_import_data(filepath: str, import_model: Type[T], import_function: callable, name_key: str): + """Generic function to import data from a JSON or YAML file.""" + + successfully_imported_data = [] + data = read_data(filepath) + for details in data: + details["customer"] = "GÉANT" + typer.echo(f"Importing {name_key}: {details[name_key]}") + try: + initial_data = import_model(**details) + import_function(initial_data) + successfully_imported_data.append(getattr(initial_data, name_key)) + typer.echo(f"Successfully imported {name_key}: {getattr(initial_data, name_key)}") + except ValidationError as e: + typer.echo(f"Validation error: {e}") + + if successfully_imported_data: + typer.echo(f"Successfully imported {name_key}s:") + for item in successfully_imported_data: + typer.echo(f"- {item}") + + +@app.command() +def import_sites( + filepath: str = typer.Option(help="Path to the file containing the sites to import.", default="gso/sites.json") +): + """ + Import sites into GSO. + """ + # Use the import_data function to handle common import logic + generic_import_data(filepath, SiteImportModel, import_site, "site_name") + + +@app.command() +def import_routers( + filepath: str = typer.Option( + help="Path to the file containing the routers to import.", default="gso/routers.json", show_default=True + ) +): + """ + Import routers into GSO. + """ + # Use the import_data function to handle common import logic + generic_import_data(filepath, RouterImportModel, import_router, "hostname") + + +def get_router_subscription_id(node_name: str) -> str | None: + """Get the subscription id for a router by its node name.""" + + subscriptions = get_active_subscriptions_by_field_and_value("router_fqdn", node_name) + if subscriptions: + return str(subscriptions[0].subscription_id) + else: + return None + + +@app.command() +def import_iptrunks( + filepath: str = typer.Option( + help="Path to the file containing the routers to import.", default="gso/trunks.json", show_default=True + ) +): + """ + Import IP trunks into GSO. + """ + successfully_imported_data = [] + data = read_data(filepath) + for trunk in data: + ipv4_network_a = ipaddress.ip_network(trunk["config"]["nodeA"]["ipv4_address"], strict=False) + ipv4_network_b = ipaddress.ip_network(trunk["config"]["nodeB"]["ipv4_address"], strict=False) + ipv6_network_a = ipaddress.ip_network(trunk["config"]["nodeA"]["ipv6_address"], strict=False) + ipv6_network_b = ipaddress.ip_network(trunk["config"]["nodeB"]["ipv6_address"], strict=False) + # Check if IPv4 networks are equal + if ipv4_network_a == ipv4_network_b: + iptrunk_ipv4_network = ipv4_network_a + else: + # Handle the case where IPv4 networks are different + typer.echo(f"Error: IPv4 networks are different for trunk {trunk['id']}.") + continue + # Check if IPv6 networks are the same + if ipv6_network_a == ipv6_network_b: + iptrunk_ipv6_network = ipv6_network_a + else: + # Handle the case where IPv6 networks are different + typer.echo(f"Error: IPv6 networks are different for trunk {trunk['id']}.") + continue + + typer.echo( + f'Importing IP Trunk: {get_active_subscriptions_by_field_and_value("router_fqdn", trunk["config"]["nodeA"]["name"])}' + ) + try: + initial_data = IptrunkImportModel( + customer="GÉANT", + geant_s_sid=trunk["id"], + iptrunk_type=trunk["config"]["common"]["type"], + iptrunk_description=" ", + iptrunk_speed=trunk["config"]["common"]["link_speed"], + iptrunk_minimum_links=trunk["config"]["common"]["minimum_links"], + side_a_node_id=get_router_subscription_id(trunk["config"]["nodeA"]["name"]), + side_a_ae_iface=trunk["config"]["nodeA"]["ae_name"], + side_a_ae_geant_a_sid=trunk["config"]["nodeA"]["port_sid"], + side_a_ae_members=trunk["config"]["nodeA"]["members"], + side_b_node_id=get_router_subscription_id(trunk["config"]["nodeB"]["name"]), + side_b_ae_iface=trunk["config"]["nodeB"]["ae_name"], + side_b_ae_geant_a_sid=trunk["config"]["nodeB"]["port_sid"], + side_b_ae_members=trunk["config"]["nodeB"]["members"], + iptrunk_ipv4_network=iptrunk_ipv4_network, + iptrunk_ipv6_network=iptrunk_ipv6_network, + ) + import_iptrunk(initial_data) + successfully_imported_data.append(trunk["id"]) + typer.echo(f"Successfully imported IP Trunk: {trunk['id']}") + except ValidationError as e: + typer.echo(f"Validation error: {e}") + + if successfully_imported_data: + typer.echo(f"Successfully imported IP Trunks:") + for item in successfully_imported_data: + typer.echo(f"- {item}") diff --git a/gso/workflows/tasks/import_router.py b/gso/workflows/tasks/import_router.py index 3d00a6099f85a886685c29319d6bd6f7d7ddfc21..f2498d8adbbfcb469dd73af43851b0426f6e8922 100644 --- a/gso/workflows/tasks/import_router.py +++ b/gso/workflows/tasks/import_router.py @@ -25,12 +25,12 @@ def _get_site_by_name(site_name: str) -> Site: :param site_name: The name of the site. :type site_name: str """ - subscription = subscriptions.get_active_subscriptions_by_field_and_value("site_name", site_name)[0] + subscription = subscriptions.get_active_subscriptions_by_field_and_value("site_name", site_name) if not subscription: msg = f"Site with name {site_name} not found." raise ValueError(msg) - return Site.from_subscription(subscription.subscription_id) + return Site.from_subscription(subscription[0].subscription_id) @step("Create subscription")