From 7bac37c48d9fbfb25d3902d1f0a6b61662e8ffe4 Mon Sep 17 00:00:00 2001 From: Mohammad Torkashvand <mohammad.torkashvand@geant.org> Date: Tue, 24 Oct 2023 10:53:25 +0200 Subject: [PATCH] Implement functionality to generate Ansible inventory from WFO API --- .gitignore | 21 ++ .gitlab-ci.yml | 37 ++++ Jenkinsfile | 20 ++ README.md | 15 +- ansible_inventory_generator/__init__.py | 0 ansible_inventory_generator/app.py | 120 ++++++++++++ ansible_inventory_generator/config.py | 16 ++ example.env | 4 + pyproject.toml | 110 +++++++++++ requirements.txt | 13 ++ setup.py | 23 +++ tests/__init__.py | 0 tests/conftest.py | 9 + tests/test_inventory_generator.py | 249 ++++++++++++++++++++++++ tox.ini | 33 ++++ 15 files changed, 669 insertions(+), 1 deletion(-) create mode 100644 .gitignore create mode 100644 .gitlab-ci.yml create mode 100644 Jenkinsfile create mode 100644 ansible_inventory_generator/__init__.py create mode 100644 ansible_inventory_generator/app.py create mode 100644 ansible_inventory_generator/config.py create mode 100644 example.env create mode 100644 pyproject.toml create mode 100644 requirements.txt create mode 100644 setup.py create mode 100644 tests/__init__.py create mode 100644 tests/conftest.py create mode 100644 tests/test_inventory_generator.py create mode 100644 tox.ini diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..5e778b5 --- /dev/null +++ b/.gitignore @@ -0,0 +1,21 @@ +__pycache__ +.coverage +.idea +.vscode +.tox +coverage.xml +*.egg-info + +docs/build +docs/source/_static/openapi.json +docs/vale/styles/* +!docs/vale/styles/Vocab/ +venv/ +.venv/ + +# Ignore files generated by apidoc +docs/source/lso.rst +docs/source/lso.*.rst +docs/source/modules.rst + +.env diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 0000000..5a0c546 --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,37 @@ +--- +stages: + - tox + +#################################### tox - Testing and linting +run-tox-pipeline: + stage: tox + tags: + - docker-executor + image: python:3.10 + + # Change pip's cache directory to be inside the project directory since we can + # only cache local items. + variables: + PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip" + + # Pip's cache doesn't store the python packages + # https://pip.pypa.io/en/stable/topics/caching/ + # + # If you want to also cache the installed packages, you have to install + # them in a virtualenv and cache it as well. + cache: + paths: + - .cache/pip + + before_script: + - pip install virtualenv + - virtualenv venv + - . venv/bin/activate + + script: + - pip install tox + - tox + + artifacts: + paths: + - htmlcov diff --git a/Jenkinsfile b/Jenkinsfile new file mode 100644 index 0000000..7ba07e1 --- /dev/null +++ b/Jenkinsfile @@ -0,0 +1,20 @@ +// https://gitlab.geant.net/live-projects/jenkins-pipeline/-/tree/master/vars +library 'SWDPipeline' + +// Parameters: +// name (must match the name of the project in GitLab/SWD release jenkins) +String name = 'ansible-inventory-generator' + +// emails of people to always notify on build status changes +List<String> extraRecipients = ['erik.reid@geant.org'] + +// python versions (docker tags) to test against, must be explicit versions +List<String> pythonTestVersions = ['3.10'] + +// Environment variables you want to pass +Map<String, String> appEnvironmentVariables = [ + 'SKIP_ALL_TESTS': '1', + // add more as needed +] + +SimplePythonBuild(name, extraRecipients, pythonTestVersions, appEnvironmentVariables) diff --git a/README.md b/README.md index 6f8457a..abb4fd1 100644 --- a/README.md +++ b/README.md @@ -2,8 +2,21 @@ Ansible Inventory Generator Overview This project aims to automate the generation of Ansible inventory files by pulling data from the WFO API. + Features Generate Ansible inventory from WFO API. Save processed data to appropriate inventory file(s). - Ensure no alteration to existing inventory files in case of process failure. \ No newline at end of file + Ensure no alteration to existing inventory files in case of process failure. + +Once installed, you can use the `ansible_inventory_generator` command to generate the Ansible inventory from the API: + +```bash +pip install ansible_inventory_generator + +export api_url=http://127.0.0.1:8080/api/v1/subscriptions/routers +export host_vars_dir=/path/to/base/hostvars/dir +export vars_file_name=wfo_vars.yaml +export hosts_file_dir=/path/to/base/hosts/dir + +ansible_inventory_generator \ No newline at end of file diff --git a/ansible_inventory_generator/__init__.py b/ansible_inventory_generator/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/ansible_inventory_generator/app.py b/ansible_inventory_generator/app.py new file mode 100644 index 0000000..7c4d1fb --- /dev/null +++ b/ansible_inventory_generator/app.py @@ -0,0 +1,120 @@ +import shutil +import sys +import tempfile +from contextlib import contextmanager +from pathlib import Path +from typing import Any, Generator + +import requests +import typer +import yaml +from yaml import ScalarNode +from yaml.representer import BaseRepresenter + +from ansible_inventory_generator.config import load_settings + +API_TIMEOUT_SEC = 60 + +app = typer.Typer() + + +def represent_none(self: BaseRepresenter, _: Any) -> ScalarNode: + """Represent None as an empty string in YAML.""" + return self.represent_scalar("tag:yaml.org,2002:null", "") + + +yaml.add_representer(type(None), represent_none) + + +def write_host_vars(host_name: str, router_data: dict, host_vars_dir: Path) -> None: + """Write host variables to a file.""" + settings = load_settings() + host_dir = host_vars_dir / host_name + host_dir.mkdir(exist_ok=True) + + vars_file_path = host_dir / settings.vars_file_name + with vars_file_path.open("w") as vars_file: + yaml.dump(router_data, vars_file, default_flow_style=False) + + +def write_hosts_file(groups: dict, hosts_file: Path) -> None: + """Write hosts data to a file.""" + hosts_data = { + "all": { + "children": { + "routers": { + "children": {group: {"hosts": {host: None for host in hosts}} for group, hosts in groups.items()} + } + } + } + } + + with hosts_file.open("w") as file: + yaml.dump(hosts_data, file, default_flow_style=False) + + +def generate_host_vars_and_hosts_file(router_subscriptions: list, temp_dir: Path) -> None: + """Process router subscriptions data.""" + groups: dict[str, list[str]] = {} + host_vars_dir = temp_dir / "hostvars" + host_vars_dir.mkdir() + + for router_subscription in router_subscriptions: + router = router_subscription.get("router", {}) + host_name = router.get("router_fqdn") + if not host_name: + continue + + write_host_vars(host_name, router, host_vars_dir) + + vendor_group = router.get("router_vendor", "").lower() + role_group = f"{router.get('router_role', '')}_routers" + + groups.setdefault(vendor_group, []).append(host_name) + groups.setdefault(role_group, []).append(host_name) + + hosts_file = temp_dir / "hosts.yaml" + write_hosts_file(groups, hosts_file) + + +@contextmanager +def safe_write(temp_dir: Path, old_vars_dir: Path, old_hosts_file: Path) -> Generator[Path, None, None]: + temp_dir.mkdir(exist_ok=True) + + try: + yield temp_dir + except Exception as e: + shutil.rmtree(temp_dir) + typer.echo(f"Error: {e}") + sys.exit(1) + else: + if old_vars_dir.exists(): + shutil.rmtree(old_vars_dir) + + shutil.copytree(temp_dir, old_vars_dir) # Copy new host vars dir + shutil.copy(temp_dir / "hosts.yaml", old_hosts_file) # Copy new hosts file + shutil.rmtree(temp_dir) + + +@app.command() +def generate_inventory_from_api() -> None: + settings = load_settings() + """Generate Ansible inventory from API.""" + response = requests.get(settings.api_url, timeout=API_TIMEOUT_SEC) + response.raise_for_status() + try: + router_subscriptions = response.json() + except ValueError: + typer.echo("Error: API did not return valid JSON.") + sys.exit(1) + + temp_dir = Path(tempfile.mkdtemp()) + old_host_vars_dir = Path(settings.host_vars_dir) + old_hosts_file = Path(settings.hosts_file_dir) + + with safe_write(temp_dir, old_host_vars_dir, old_hosts_file) as temp_dir: + generate_host_vars_and_hosts_file(router_subscriptions, temp_dir) + + +if __name__ == "__main__": + app() diff --git a/ansible_inventory_generator/config.py b/ansible_inventory_generator/config.py new file mode 100644 index 0000000..b94b8ce --- /dev/null +++ b/ansible_inventory_generator/config.py @@ -0,0 +1,16 @@ +from pydantic.v1 import BaseSettings + + +class Settings(BaseSettings): + api_url: str + host_vars_dir: str + vars_file_name: str + hosts_file_dir: str + + class Config: + env_file = "../.env" + env_file_encoding = "utf-8" + + +def load_settings() -> Settings: + return Settings() # type: ignore[call-arg] diff --git a/example.env b/example.env new file mode 100644 index 0000000..4654690 --- /dev/null +++ b/example.env @@ -0,0 +1,4 @@ +api_url=http://127.0.0.1:8080/api/v1/subscriptions/routers +host_vars_dir=/path/to/base/hostvars/dir +vars_file_name=wfo_vars.yaml +hosts_file_dir=/path/to/base/hosts/dir diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..a641239 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,110 @@ +[tool.isort] +profile = "black" +line_length = 120 +skip = ["venv", ".tox", "docs"] +known_third_party = ["pydantic",] +known_first_party = ["test", "docs"] + +[tool.black] +line-length = 120 +target-version = ["py310"] +exclude = ''' +( + /( + ansible_inventory_generator\.egg-info # exclude a few common directories in the + | \.git # root of the project + | \.*_cache + | \.tox + | venv + | docs + )/ +) +''' + +[tool.mypy] +exclude = [ + "venv", + "test/*", + "docs" +] +ignore_missing_imports = true +disallow_untyped_calls = true +disallow_untyped_defs = true +disallow_incomplete_defs = true +disallow_untyped_decorators = true +no_implicit_optional = true +strict_optional = true +namespace_packages = true +warn_unused_ignores = true +warn_redundant_casts = true +warn_no_return = true +warn_unreachable = true +implicit_reexport = false +strict_equality = true +show_error_codes = true +show_column_numbers = true +# Suppress "note: By default the bodies of untyped functions are not checked" +disable_error_code = "annotation-unchecked" +# Forbid the use of a generic "type: ignore" without specifying the exact error that is ignored +enable_error_code = "ignore-without-code" + +[tool.ruff] +exclude = [ + ".git", + ".*_cache", + ".tox", + "*.egg-info", + "__pycache__", + "htmlcov", + "venv", + "docs", +] +ignore = [ + "C417", + "D100", + "D101", + "D102", + "D103", + "D104", + "D105", + "D106", + "D107", + "D202", + "D203", + "D213", + "E501", + "N806", + "B905", + "N805", + "B904", + "N803", + "N801", + "N815", + "N802", + "S101" +] +line-length = 120 +select = [ + "B", + "C", + "D", + "E", + "F", + "I", + "N", + "RET", + "S", + "T", + "W", +] +target-version = "py310" + +[tool.ruff.flake8-tidy-imports] +ban-relative-imports = "all" + +[tool.ruff.per-file-ignores] +"test/*" = ["B033", "N816", "N802"] + +[tool.ruff.isort] +known-third-party = ["pydantic",] +known-first-party = ["test", "docs"] diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..b1d7c3b --- /dev/null +++ b/requirements.txt @@ -0,0 +1,13 @@ +requests==2.31.0 +pyyaml==6.0.1 +typer[all]==0.9.0 +pydantic==2.4.2 +python-dotenv==1.0.0 + +# Test and linting dependencies +pytest==7.4.2 +black==23.10.0 +isort==5.12.0 +flake8==6.1.0 +mypy==1.6.1 +ruff==0.1.1 diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..f676b0f --- /dev/null +++ b/setup.py @@ -0,0 +1,23 @@ +from setuptools import find_packages, setup + +setup( + name="ansible_inventory_generator", + version="0.1", + author="GEANT", + author_email="swd@geant.org", + description="Ansible inventory generator for WFO", + url="https://gitlab.software.geant.org/goat/gap/ansible_inventory_generator", + packages=find_packages(), + install_requires=[ + "requests== 2.31.0", + "pyyaml== 6.0.1", + "typer[all]==0.9.0", + "pydantic== 2.4.2", + "python-dotenv==1.0.0", + ], + entry_points={ + "console_scripts": [ + "ansible_inventory_generator=ansible_inventory_generator.app:generate_inventory_from_api", + ], + }, +) diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..a810ea1 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,9 @@ +import os + +import pytest + + +def pytest_collection_modifyitems(config, items): + if bool(os.environ.get("SKIP_ALL_TESTS")): + for item in items: + item.add_marker(pytest.mark.skip(reason="Skipped due to SKIP_ALL_TESTS env variable")) diff --git a/tests/test_inventory_generator.py b/tests/test_inventory_generator.py new file mode 100644 index 0000000..a309c7b --- /dev/null +++ b/tests/test_inventory_generator.py @@ -0,0 +1,249 @@ +import shutil +import tempfile +from pathlib import Path +from unittest.mock import patch + +import pytest +import yaml + +from ansible_inventory_generator.app import generate_inventory_from_api +from ansible_inventory_generator.config import Settings + + +@pytest.fixture(scope="module") +def tmp_dir(): + return Path(tempfile.mkdtemp()) + + +@pytest.fixture(scope="module") +def mocked_subscription_api_data(): + return [ + { + "subscription_id": "dc20ffd2-2468-4da6-a34a-5beb5c58f551", + "start_date": 1695386639.226367, + "description": "Router amsterdam.nl.geant.net", + "status": "active", + "product_id": None, + "customer_id": "8f0df561-ce9d-4d9c-89a8-7953d3ffc961", + "insync": True, + "note": None, + "name": None, + "end_date": None, + "product": { + "product_id": "1b7c7c09-f48a-40a4-a5fe-172b79b846c2", + "name": "Router", + "description": "A GÉANT Router", + "product_type": "Router", + "status": "active", + "tag": "ROUTER", + "created_at": 1695379709.731743, + "end_date": None, + }, + "customer_descriptions": [], + "tag": None, + "router": { + "name": "RouterBlock", + "subscription_instance_id": "165ef7d3-4ce9-4739-a2cf-2b67d199d210", + "owner_subscription_id": "dc20ffd2-2468-4da6-a34a-5beb5c58f551", + "label": None, + "router_fqdn": "amsterdam.nl.geant.net", + "router_ts_port": 1, + "router_access_via_ts": True, + "router_lo_ipv4_address": "62.40.96.107", + "router_lo_ipv6_address": "2001:798:aa:1::5e", + "router_lo_iso_address": "49.51e5.0001.0620.4009.6107.00", + "router_si_ipv4_network": "62.40.97.222/31", + "router_ias_lt_ipv4_network": "83.97.89.116/31", + "router_ias_lt_ipv6_network": "2001:798:1::2a4/126", + "router_vendor": "nokia", + "router_role": "p", + "router_site": { + "name": "SiteBlock", + "subscription_instance_id": "ea07efe6-5760-4db2-af89-51bc1ba83654", + "owner_subscription_id": "f6911222-b103-4e02-9243-c98b7a5114d1", + "label": None, + "site_name": "Amsterdam", + "site_city": "Amsterdam", + "site_country": "Netherlands", + "site_country_code": "nl", + "site_latitude": "10", + "site_longitude": "-10", + "site_internal_id": 1, + "site_bgp_community_id": 1, + "site_tier": "1", + "site_ts_address": "127.0.0.1", + "in_use_by_ids": [ + "a577656e-8810-4ce2-8384-c7aabf437a2c", + "0acd4d9c-56cf-4ab3-b9e4-2bce38197571", + ], + }, + "router_is_ias_connected": True, + "in_use_by_ids": ["f9bfeecb-9d9a-460a-b00b-7a2edb349baf", "ce87f170-5207-42ef-9fb5-782f7afeaa62"], + }, + }, + { + "subscription_id": "dc20ffd2-2468-4da6-a34a-5beb5c58f551", + "start_date": 1695386639.226367, + "description": "Router utrecht.nl.geant.net", + "status": "active", + "product_id": None, + "customer_id": "8f0df561-ce9d-4d9c-89a8-7953d3ffc961", + "insync": True, + "note": None, + "name": None, + "end_date": None, + "product": { + "product_id": "1b7c7c09-f48a-40a4-asdaa-172b79b846c2", + "name": "Router", + "description": "A GÉANT Router", + "product_type": "Router", + "status": "active", + "tag": "ROUTER", + "created_at": 1695379709.731743, + "end_date": None, + }, + "customer_descriptions": [], + "tag": None, + "router": { + "name": "RouterBlock", + "subscription_instance_id": "asdq-4ce9-4739-a2cf-2b67d199d210", + "owner_subscription_id": "dc20ffd2-2468-4da6-a34a-5beb5c58f551", + "label": None, + "router_fqdn": "utrecht.nl.geant.net", + "router_ts_port": 1, + "router_access_via_ts": True, + "router_lo_ipv4_address": "62.40.96.107", + "router_lo_ipv6_address": "2001:798:aa:1::5e", + "router_lo_iso_address": "49.51e5.0001.0620.4009.6107.00", + "router_si_ipv4_network": "62.40.97.222/31", + "router_ias_lt_ipv4_network": "83.97.89.116/31", + "router_ias_lt_ipv6_network": "2001:798:1::2a4/126", + "router_vendor": "juniper", + "router_role": "amt", + "router_site": { + "name": "SiteBlock", + "subscription_instance_id": "asd-5760-4db2-af89-51bc1ba83654", + "owner_subscription_id": "f6911222-b103-4e02-9243-c98b7a5114d1", + "label": None, + "site_name": "Amsterdam", + "site_city": "Amsterdam", + "site_country": "Netherlands", + "site_country_code": "nl", + "site_latitude": "10", + "site_longitude": "-20", + "site_internal_id": 1, + "site_bgp_community_id": 1, + "site_tier": "1", + "site_ts_address": "127.0.0.1", + "in_use_by_ids": [ + "a577656e-8810-4ce2-8384-ads", + "0acd4d9c-56cf-4ab3-b9e4-asdasdasd", + ], + }, + "router_is_ias_connected": True, + "in_use_by_ids": ["f9bfeecb-9d9a-460a-b00b-7a2edb349baf", "ce87f170-5207-42ef-9fb5-782f7afeaa62"], + }, + }, + ] + + +@pytest.fixture() +def setup_test(tmp_dir, mocked_subscription_api_data): + with patch("ansible_inventory_generator.app.load_settings") as mock_load_settings, patch( + "requests.get" + ) as mock_get: + mock_load_settings.return_value = Settings( + api_url="http://test", + host_vars_dir=str(tmp_dir), + vars_file_name="wfo_vars.yaml", + hosts_file_dir=str(tmp_dir), + ) + + mock_get.return_value.json.return_value = mocked_subscription_api_data + mock_get.return_value.status_code = 200 + + yield + + shutil.rmtree(tmp_dir) + + +def test_generate_inventory_from_api_valid_json(setup_test, tmp_dir, mocked_subscription_api_data): + hosts_file = tmp_dir / "hosts.yaml" + amsterdam_host_vars = tmp_dir / "hostvars/amsterdam.nl.geant.net/wfo_vars.yaml" + utrecht_host_vars = tmp_dir / "hostvars/utrecht.nl.geant.net/wfo_vars.yaml" + + assert not amsterdam_host_vars.exists() + assert not utrecht_host_vars.exists() + assert not hosts_file.exists() + + generate_inventory_from_api() + + assert amsterdam_host_vars.exists() + assert utrecht_host_vars.exists() + assert hosts_file.exists() + + with hosts_file.open() as f: + content = yaml.safe_load(f) + expected_content = { + "all": { + "children": { + "routers": { + "children": { + "amt_routers": {"hosts": {"utrecht.nl.geant.net": None}}, + "juniper": {"hosts": {"utrecht.nl.geant.net": None}}, + "nokia": {"hosts": {"amsterdam.nl.geant.net": None}}, + "p_routers": {"hosts": {"amsterdam.nl.geant.net": None}}, + } + } + } + } + } + + assert content == expected_content + + with amsterdam_host_vars.open() as f: + content = yaml.safe_load(f) + expected_content = mocked_subscription_api_data[0]["router"] + assert content == expected_content + + with utrecht_host_vars.open() as f: + content = yaml.safe_load(f) + expected_content = mocked_subscription_api_data[1]["router"] + assert content == expected_content + + +def test_inventory_generation_handles_exceptions_and_preserves_file_integrity( + setup_test, tmp_dir, mocked_subscription_api_data +): + paths = { + "hosts": tmp_dir / "hosts.yaml", + "ams_vars": tmp_dir / "hostvars/amsterdam.nl.geant.net/wfo_vars.yaml", + "utr_vars": tmp_dir / "hostvars/utrecht.nl.geant.net/wfo_vars.yaml", + } + + file_contents = { + "hosts": "Sample host file", + "ams_vars": "ams.geant.org", + "utr_vars": "utrecht.geant.org", + } + + paths["hosts"].parent.mkdir(parents=True, exist_ok=True) + paths["hosts"].write_text(file_contents["hosts"]) + + paths["ams_vars"].parent.mkdir(parents=True, exist_ok=True) + paths["ams_vars"].write_text(file_contents["ams_vars"]) + + paths["utr_vars"].parent.mkdir(parents=True, exist_ok=True) + paths["utr_vars"].write_text(file_contents["utr_vars"]) + + with patch("ansible_inventory_generator.app.generate_host_vars_and_hosts_file") as mocked_func: + mocked_func.side_effect = Exception("This is an intentional exception!") + with patch("sys.exit") as exit_mock: + generate_inventory_from_api() + exit_mock.assert_called_once_with(1) + + for path_key, path in paths.items(): + assert path.exists() + with path.open("r") as f: + content = f.read() + assert content == file_contents[path_key] diff --git a/tox.ini b/tox.ini new file mode 100644 index 0000000..f59a410 --- /dev/null +++ b/tox.ini @@ -0,0 +1,33 @@ +[flake8] +; Allow >> on newline (W503), and allow cls as first argument for pydantic validators (B902) +ignore = B902,W503 +exclude = .git,.*_cache,.eggs,*.egg-info,__pycache__,venv,.tox,docs +enable-extensions = G +select = B,C,D,E,F,G,I,N,S,T,W,B902,B903,R +max-line-length = 120 +ban-relative-imports = true + +[testenv] +passenv = SKIP_ALL_TESTS +deps = + coverage + flake8 + black + mypy + ruff + isort + types-requests + types-PyYAML + -r requirements.txt + +commands = + isort -c . + ruff . + black --check . + mypy . + flake8 + coverage erase + coverage run --source ansible_inventory_generator -m pytest {posargs} + coverage xml + coverage html + coverage report --fail-under 80 -- GitLab