Implement functionality to generate Ansible inventory from WFO API

.gitignore

+__pycache__
+.coverage
+.idea
+.vscode
+.tox
+coverage.xml
+*.egg-info
+
+docs/build
+docs/source/_static/openapi.json
+docs/vale/styles/*
+!docs/vale/styles/Vocab/
+venv/
+.venv/
+
+# Ignore files generated by apidoc
+docs/source/lso.rst
+docs/source/lso.*.rst
+docs/source/modules.rst
+
+.env

.gitlab-ci.yml

+---
+stages:
+  - tox
+
+####################################  tox   -   Testing and linting
+run-tox-pipeline:
+  stage: tox
+  tags:
+    - docker-executor
+  image: python:3.10
+
+  # Change pip's cache directory to be inside the project directory since we can
+  # only cache local items.
+  variables:
+    PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"
+
+  # Pip's cache doesn't store the python packages
+  # https://pip.pypa.io/en/stable/topics/caching/
+  #
+  # If you want to also cache the installed packages, you have to install
+  # them in a virtualenv and cache it as well.
+  cache:
+    paths:
+      - .cache/pip
+
+  before_script:
+    - pip install virtualenv
+    - virtualenv venv
+    - . venv/bin/activate
+
+  script:
+    - pip install tox
+    - tox
+
+  artifacts:
+    paths:
+      - htmlcov

Jenkinsfile

+// https://gitlab.geant.net/live-projects/jenkins-pipeline/-/tree/master/vars
+library 'SWDPipeline'
+
+// Parameters:
+// name (must match the name of the project in GitLab/SWD release jenkins)
+String name = 'ansible-inventory-generator'
+
+// emails of people to always notify on build status changes
+List<String> extraRecipients = ['erik.reid@geant.org']
+
+// python versions (docker tags) to test against, must be explicit versions
+List<String> pythonTestVersions = ['3.10']
+
+// Environment variables you want to pass
+Map<String, String> appEnvironmentVariables = [
+    'SKIP_ALL_TESTS': '1',
+    // add more as needed
+]
+
+SimplePythonBuild(name, extraRecipients, pythonTestVersions, appEnvironmentVariables)

README.md

@@ -2,8 +2,21 @@ Ansible Inventory Generator
 Overview
 
 This project aims to automate the generation of Ansible inventory files by pulling data from the WFO API.
+
 Features
 
     Generate Ansible inventory from WFO API.
     Save processed data to appropriate inventory file(s).
-    Ensure no alteration to existing inventory files in case of process failure.
+    Ensure no alteration to existing inventory files in case of process failure.
\ No newline at end of file
+
+Once installed, you can use the `ansible_inventory_generator` command to generate the Ansible inventory from the API:
+
+```bash
+pip install ansible_inventory_generator
+
+export api_url=http://127.0.0.1:8080/api/v1/subscriptions/routers
+export host_vars_dir=/path/to/base/hostvars/dir
+export vars_file_name=wfo_vars.yaml
+export hosts_file_dir=/path/to/base/hosts/dir
+
+ansible_inventory_generator
\ No newline at end of file

ansible_inventory_generator/app.py

+import shutil
+import sys
+import tempfile
+from contextlib import contextmanager
+from pathlib import Path
+from typing import Any, Generator
+
+import requests
+import typer
+import yaml
+from yaml import ScalarNode
+from yaml.representer import BaseRepresenter
+
+from ansible_inventory_generator.config import load_settings
+
+API_TIMEOUT_SEC = 60
+
+app = typer.Typer()
+
+
+def represent_none(self: BaseRepresenter, _: Any) -> ScalarNode:
+    """Represent None as an empty string in YAML."""
+    return self.represent_scalar("tag:yaml.org,2002:null", "")
+
+
+yaml.add_representer(type(None), represent_none)
+
+
+def write_host_vars(host_name: str, router_data: dict, host_vars_dir: Path) -> None:
+    """Write host variables to a file."""
+    settings = load_settings()
+    host_dir = host_vars_dir / host_name
+    host_dir.mkdir(exist_ok=True)
+
+    vars_file_path = host_dir / settings.vars_file_name
+    with vars_file_path.open("w") as vars_file:
+        yaml.dump(router_data, vars_file, default_flow_style=False)
+
+
+def write_hosts_file(groups: dict, hosts_file: Path) -> None:
+    """Write hosts data to a file."""
+    hosts_data = {
+        "all": {
+            "children": {
+                "routers": {
+                    "children": {group: {"hosts": {host: None for host in hosts}} for group, hosts in groups.items()}
+                }
+            }
+        }
+    }
+
+    with hosts_file.open("w") as file:
+        yaml.dump(hosts_data, file, default_flow_style=False)
+
+
+def generate_host_vars_and_hosts_file(router_subscriptions: list, temp_dir: Path) -> None:
+    """Process router subscriptions data."""
+    groups: dict[str, list[str]] = {}
+    host_vars_dir = temp_dir / "hostvars"
+    host_vars_dir.mkdir()
+
+    for router_subscription in router_subscriptions:
+        router = router_subscription.get("router", {})
+        host_name = router.get("router_fqdn")
+        if not host_name:
+            continue
+
+        write_host_vars(host_name, router, host_vars_dir)
+
+        vendor_group = router.get("router_vendor", "").lower()
+        role_group = f"{router.get('router_role', '')}_routers"
+
+        groups.setdefault(vendor_group, []).append(host_name)
+        groups.setdefault(role_group, []).append(host_name)
+
+    hosts_file = temp_dir / "hosts.yaml"
+    write_hosts_file(groups, hosts_file)
+
+
+@contextmanager
+def safe_write(temp_dir: Path, old_vars_dir: Path, old_hosts_file: Path) -> Generator[Path, None, None]:
+    temp_dir.mkdir(exist_ok=True)
+
+    try:
+        yield temp_dir
+    except Exception as e:
+        shutil.rmtree(temp_dir)
+        typer.echo(f"Error: {e}")
+        sys.exit(1)
+    else:
+        if old_vars_dir.exists():
+            shutil.rmtree(old_vars_dir)
+
+        shutil.copytree(temp_dir, old_vars_dir)  # Copy new host vars dir
+        shutil.copy(temp_dir / "hosts.yaml", old_hosts_file)  # Copy new hosts file
+        shutil.rmtree(temp_dir)
+
+
+@app.command()
+def generate_inventory_from_api() -> None:
+    settings = load_settings()
+    """Generate Ansible inventory from API."""
+    response = requests.get(settings.api_url, timeout=API_TIMEOUT_SEC)
+    response.raise_for_status()
+    try:
+        router_subscriptions = response.json()
+    except ValueError:
+        typer.echo("Error: API did not return valid JSON.")
+        sys.exit(1)
+
+    temp_dir = Path(tempfile.mkdtemp())
+    old_host_vars_dir = Path(settings.host_vars_dir)
+    old_hosts_file = Path(settings.hosts_file_dir)
+
+    with safe_write(temp_dir, old_host_vars_dir, old_hosts_file) as temp_dir:
+        generate_host_vars_and_hosts_file(router_subscriptions, temp_dir)
+
+
+if __name__ == "__main__":
+    app()

ansible_inventory_generator/config.py

+from pydantic.v1 import BaseSettings
+
+
+class Settings(BaseSettings):
+    api_url: str
+    host_vars_dir: str
+    vars_file_name: str
+    hosts_file_dir: str
+
+    class Config:
+        env_file = "../.env"
+        env_file_encoding = "utf-8"
+
+
+def load_settings() -> Settings:
+    return Settings()  # type: ignore[call-arg]

example.env

+api_url=http://127.0.0.1:8080/api/v1/subscriptions/routers
+host_vars_dir=/path/to/base/hostvars/dir
+vars_file_name=wfo_vars.yaml
+hosts_file_dir=/path/to/base/hosts/dir

pyproject.toml

+[tool.isort]
+profile = "black"
+line_length = 120
+skip = ["venv", ".tox", "docs"]
+known_third_party = ["pydantic",]
+known_first_party = ["test", "docs"]
+
+[tool.black]
+line-length = 120
+target-version = ["py310"]
+exclude = '''
+(
+  /(
+    ansible_inventory_generator\.egg-info      # exclude a few common directories in the
+    | \.git                                   # root of the project
+    | \.*_cache
+    | \.tox
+    | venv
+    | docs
+  )/
+)
+'''
+
+[tool.mypy]
+exclude = [
+    "venv",
+    "test/*",
+    "docs"
+]
+ignore_missing_imports = true
+disallow_untyped_calls = true
+disallow_untyped_defs = true
+disallow_incomplete_defs = true
+disallow_untyped_decorators = true
+no_implicit_optional = true
+strict_optional = true
+namespace_packages = true
+warn_unused_ignores = true
+warn_redundant_casts = true
+warn_no_return = true
+warn_unreachable = true
+implicit_reexport = false
+strict_equality = true
+show_error_codes = true
+show_column_numbers = true
+# Suppress "note: By default the bodies of untyped functions are not checked"
+disable_error_code = "annotation-unchecked"
+# Forbid the use of a generic "type: ignore" without specifying the exact error that is ignored
+enable_error_code = "ignore-without-code"
+
+[tool.ruff]
+exclude = [
+    ".git",
+    ".*_cache",
+    ".tox",
+    "*.egg-info",
+    "__pycache__",
+    "htmlcov",
+    "venv",
+    "docs",
+]
+ignore = [
+    "C417",
+    "D100",
+    "D101",
+    "D102",
+    "D103",
+    "D104",
+    "D105",
+    "D106",
+    "D107",
+    "D202",
+    "D203",
+    "D213",
+    "E501",
+    "N806",
+    "B905",
+    "N805",
+    "B904",
+    "N803",
+    "N801",
+    "N815",
+    "N802",
+    "S101"
+]
+line-length = 120
+select = [
+    "B",
+    "C",
+    "D",
+    "E",
+    "F",
+    "I",
+    "N",
+    "RET",
+    "S",
+    "T",
+    "W",
+]
+target-version = "py310"
+
+[tool.ruff.flake8-tidy-imports]
+ban-relative-imports = "all"
+
+[tool.ruff.per-file-ignores]
+"test/*" = ["B033", "N816", "N802"]
+
+[tool.ruff.isort]
+known-third-party = ["pydantic",]
+known-first-party = ["test", "docs"]

requirements.txt

+requests==2.31.0
+pyyaml==6.0.1
+typer[all]==0.9.0
+pydantic==2.4.2
+python-dotenv==1.0.0
+
+# Test and linting dependencies
+pytest==7.4.2
+black==23.10.0
+isort==5.12.0
+flake8==6.1.0
+mypy==1.6.1
+ruff==0.1.1

setup.py

+from setuptools import find_packages, setup
+
+setup(
+    name="ansible_inventory_generator",
+    version="0.1",
+    author="GEANT",
+    author_email="swd@geant.org",
+    description="Ansible inventory generator for WFO",
+    url="https://gitlab.software.geant.org/goat/gap/ansible_inventory_generator",
+    packages=find_packages(),
+    install_requires=[
+        "requests== 2.31.0",
+        "pyyaml== 6.0.1",
+        "typer[all]==0.9.0",
+        "pydantic== 2.4.2",
+        "python-dotenv==1.0.0",
+    ],
+    entry_points={
+        "console_scripts": [
+            "ansible_inventory_generator=ansible_inventory_generator.app:generate_inventory_from_api",
+        ],
+    },
+)

tests/conftest.py

+import os
+
+import pytest
+
+
+def pytest_collection_modifyitems(config, items):
+    if bool(os.environ.get("SKIP_ALL_TESTS")):
+        for item in items:
+            item.add_marker(pytest.mark.skip(reason="Skipped due to SKIP_ALL_TESTS env variable"))

tests/test_inventory_generator.py

+import shutil
+import tempfile
+from pathlib import Path
+from unittest.mock import patch
+
+import pytest
+import yaml
+
+from ansible_inventory_generator.app import generate_inventory_from_api
+from ansible_inventory_generator.config import Settings
+
+
+@pytest.fixture(scope="module")
+def tmp_dir():
+    return Path(tempfile.mkdtemp())
+
+
+@pytest.fixture(scope="module")
+def mocked_subscription_api_data():
+    return [
+        {
+            "subscription_id": "dc20ffd2-2468-4da6-a34a-5beb5c58f551",
+            "start_date": 1695386639.226367,
+            "description": "Router amsterdam.nl.geant.net",
+            "status": "active",
+            "product_id": None,
+            "customer_id": "8f0df561-ce9d-4d9c-89a8-7953d3ffc961",
+            "insync": True,
+            "note": None,
+            "name": None,
+            "end_date": None,
+            "product": {
+                "product_id": "1b7c7c09-f48a-40a4-a5fe-172b79b846c2",
+                "name": "Router",
+                "description": "A GÉANT Router",
+                "product_type": "Router",
+                "status": "active",
+                "tag": "ROUTER",
+                "created_at": 1695379709.731743,
+                "end_date": None,
+            },
+            "customer_descriptions": [],
+            "tag": None,
+            "router": {
+                "name": "RouterBlock",
+                "subscription_instance_id": "165ef7d3-4ce9-4739-a2cf-2b67d199d210",
+                "owner_subscription_id": "dc20ffd2-2468-4da6-a34a-5beb5c58f551",
+                "label": None,
+                "router_fqdn": "amsterdam.nl.geant.net",
+                "router_ts_port": 1,
+                "router_access_via_ts": True,
+                "router_lo_ipv4_address": "62.40.96.107",
+                "router_lo_ipv6_address": "2001:798:aa:1::5e",
+                "router_lo_iso_address": "49.51e5.0001.0620.4009.6107.00",
+                "router_si_ipv4_network": "62.40.97.222/31",
+                "router_ias_lt_ipv4_network": "83.97.89.116/31",
+                "router_ias_lt_ipv6_network": "2001:798:1::2a4/126",
+                "router_vendor": "nokia",
+                "router_role": "p",
+                "router_site": {
+                    "name": "SiteBlock",
+                    "subscription_instance_id": "ea07efe6-5760-4db2-af89-51bc1ba83654",
+                    "owner_subscription_id": "f6911222-b103-4e02-9243-c98b7a5114d1",
+                    "label": None,
+                    "site_name": "Amsterdam",
+                    "site_city": "Amsterdam",
+                    "site_country": "Netherlands",
+                    "site_country_code": "nl",
+                    "site_latitude": "10",
+                    "site_longitude": "-10",
+                    "site_internal_id": 1,
+                    "site_bgp_community_id": 1,
+                    "site_tier": "1",
+                    "site_ts_address": "127.0.0.1",
+                    "in_use_by_ids": [
+                        "a577656e-8810-4ce2-8384-c7aabf437a2c",
+                        "0acd4d9c-56cf-4ab3-b9e4-2bce38197571",
+                    ],
+                },
+                "router_is_ias_connected": True,
+                "in_use_by_ids": ["f9bfeecb-9d9a-460a-b00b-7a2edb349baf", "ce87f170-5207-42ef-9fb5-782f7afeaa62"],
+            },
+        },
+        {
+            "subscription_id": "dc20ffd2-2468-4da6-a34a-5beb5c58f551",
+            "start_date": 1695386639.226367,
+            "description": "Router utrecht.nl.geant.net",
+            "status": "active",
+            "product_id": None,
+            "customer_id": "8f0df561-ce9d-4d9c-89a8-7953d3ffc961",
+            "insync": True,
+            "note": None,
+            "name": None,
+            "end_date": None,
+            "product": {
+                "product_id": "1b7c7c09-f48a-40a4-asdaa-172b79b846c2",
+                "name": "Router",
+                "description": "A GÉANT Router",
+                "product_type": "Router",
+                "status": "active",
+                "tag": "ROUTER",
+                "created_at": 1695379709.731743,
+                "end_date": None,
+            },
+            "customer_descriptions": [],
+            "tag": None,
+            "router": {
+                "name": "RouterBlock",
+                "subscription_instance_id": "asdq-4ce9-4739-a2cf-2b67d199d210",
+                "owner_subscription_id": "dc20ffd2-2468-4da6-a34a-5beb5c58f551",
+                "label": None,
+                "router_fqdn": "utrecht.nl.geant.net",
+                "router_ts_port": 1,
+                "router_access_via_ts": True,
+                "router_lo_ipv4_address": "62.40.96.107",
+                "router_lo_ipv6_address": "2001:798:aa:1::5e",
+                "router_lo_iso_address": "49.51e5.0001.0620.4009.6107.00",
+                "router_si_ipv4_network": "62.40.97.222/31",
+                "router_ias_lt_ipv4_network": "83.97.89.116/31",
+                "router_ias_lt_ipv6_network": "2001:798:1::2a4/126",
+                "router_vendor": "juniper",
+                "router_role": "amt",
+                "router_site": {
+                    "name": "SiteBlock",
+                    "subscription_instance_id": "asd-5760-4db2-af89-51bc1ba83654",
+                    "owner_subscription_id": "f6911222-b103-4e02-9243-c98b7a5114d1",
+                    "label": None,
+                    "site_name": "Amsterdam",
+                    "site_city": "Amsterdam",
+                    "site_country": "Netherlands",
+                    "site_country_code": "nl",
+                    "site_latitude": "10",
+                    "site_longitude": "-20",
+                    "site_internal_id": 1,
+                    "site_bgp_community_id": 1,
+                    "site_tier": "1",
+                    "site_ts_address": "127.0.0.1",
+                    "in_use_by_ids": [
+                        "a577656e-8810-4ce2-8384-ads",
+                        "0acd4d9c-56cf-4ab3-b9e4-asdasdasd",
+                    ],
+                },
+                "router_is_ias_connected": True,
+                "in_use_by_ids": ["f9bfeecb-9d9a-460a-b00b-7a2edb349baf", "ce87f170-5207-42ef-9fb5-782f7afeaa62"],
+            },
+        },
+    ]
+
+
+@pytest.fixture()
+def setup_test(tmp_dir, mocked_subscription_api_data):
+    with patch("ansible_inventory_generator.app.load_settings") as mock_load_settings, patch(
+        "requests.get"
+    ) as mock_get:
+        mock_load_settings.return_value = Settings(
+            api_url="http://test",
+            host_vars_dir=str(tmp_dir),
+            vars_file_name="wfo_vars.yaml",
+            hosts_file_dir=str(tmp_dir),
+        )
+
+        mock_get.return_value.json.return_value = mocked_subscription_api_data
+        mock_get.return_value.status_code = 200
+
+        yield
+
+        shutil.rmtree(tmp_dir)
+
+
+def test_generate_inventory_from_api_valid_json(setup_test, tmp_dir, mocked_subscription_api_data):
+    hosts_file = tmp_dir / "hosts.yaml"
+    amsterdam_host_vars = tmp_dir / "hostvars/amsterdam.nl.geant.net/wfo_vars.yaml"
+    utrecht_host_vars = tmp_dir / "hostvars/utrecht.nl.geant.net/wfo_vars.yaml"
+
+    assert not amsterdam_host_vars.exists()
+    assert not utrecht_host_vars.exists()
+    assert not hosts_file.exists()
+
+    generate_inventory_from_api()
+
+    assert amsterdam_host_vars.exists()
+    assert utrecht_host_vars.exists()
+    assert hosts_file.exists()
+
+    with hosts_file.open() as f:
+        content = yaml.safe_load(f)
+        expected_content = {
+            "all": {
+                "children": {
+                    "routers": {
+                        "children": {
+                            "amt_routers": {"hosts": {"utrecht.nl.geant.net": None}},
+                            "juniper": {"hosts": {"utrecht.nl.geant.net": None}},
+                            "nokia": {"hosts": {"amsterdam.nl.geant.net": None}},
+                            "p_routers": {"hosts": {"amsterdam.nl.geant.net": None}},
+                        }
+                    }
+                }
+            }
+        }
+
+        assert content == expected_content
+
+    with amsterdam_host_vars.open() as f:
+        content = yaml.safe_load(f)
+        expected_content = mocked_subscription_api_data[0]["router"]
+        assert content == expected_content
+
+    with utrecht_host_vars.open() as f:
+        content = yaml.safe_load(f)
+        expected_content = mocked_subscription_api_data[1]["router"]
+        assert content == expected_content
+
+
+def test_inventory_generation_handles_exceptions_and_preserves_file_integrity(
+    setup_test, tmp_dir, mocked_subscription_api_data
+):
+    paths = {
+        "hosts": tmp_dir / "hosts.yaml",
+        "ams_vars": tmp_dir / "hostvars/amsterdam.nl.geant.net/wfo_vars.yaml",
+        "utr_vars": tmp_dir / "hostvars/utrecht.nl.geant.net/wfo_vars.yaml",
+    }
+
+    file_contents = {
+        "hosts": "Sample host file",
+        "ams_vars": "ams.geant.org",
+        "utr_vars": "utrecht.geant.org",
+    }
+
+    paths["hosts"].parent.mkdir(parents=True, exist_ok=True)
+    paths["hosts"].write_text(file_contents["hosts"])
+
+    paths["ams_vars"].parent.mkdir(parents=True, exist_ok=True)
+    paths["ams_vars"].write_text(file_contents["ams_vars"])
+
+    paths["utr_vars"].parent.mkdir(parents=True, exist_ok=True)
+    paths["utr_vars"].write_text(file_contents["utr_vars"])
+
+    with patch("ansible_inventory_generator.app.generate_host_vars_and_hosts_file") as mocked_func:
+        mocked_func.side_effect = Exception("This is an intentional exception!")
+        with patch("sys.exit") as exit_mock:
+            generate_inventory_from_api()
+            exit_mock.assert_called_once_with(1)
+
+    for path_key, path in paths.items():
+        assert path.exists()
+        with path.open("r") as f:
+            content = f.read()
+            assert content == file_contents[path_key]

tox.ini

+[flake8]
+; Allow >> on newline (W503), and allow cls as first argument for pydantic validators (B902)
+ignore = B902,W503
+exclude = .git,.*_cache,.eggs,*.egg-info,__pycache__,venv,.tox,docs
+enable-extensions = G
+select = B,C,D,E,F,G,I,N,S,T,W,B902,B903,R
+max-line-length = 120
+ban-relative-imports = true
+
+[testenv]
+passenv = SKIP_ALL_TESTS
+deps =
+    coverage
+    flake8
+    black
+    mypy
+    ruff
+    isort
+    types-requests
+    types-PyYAML
+    -r requirements.txt
+
+commands =
+    isort -c .
+    ruff .
+    black --check .
+    mypy .
+    flake8
+    coverage erase
+    coverage run --source ansible_inventory_generator -m pytest {posargs}
+    coverage xml
+    coverage html
+    coverage report --fail-under 80