Finished release 0.3.

.gitignore

@@ -16,6 +16,7 @@ docs/build
 docs/vale/styles/*
 !docs/vale/styles/config/
 !docs/vale/styles/custom/
+.DS_Store
 
 .idea
 .venv

Changelog.md

@@ -2,6 +2,9 @@
 
 All notable changes to this project will be documented in this file.
 
+## [0.3] - 2024-01-23
+- Fixed related to the Authentication and some small improvments.
+
 ## [0.2] - 2024-01-16
 - Initial release
 

docs/source/module/api/v1/index.rst

@@ -14,3 +14,4 @@ Submodules
 
    imports
    subscriptions
+   processes

docs/source/module/api/v1/processes.rst

+``gso.api.v1.processes``
+============================
+
+.. automodule:: gso.api.v1.processes
+   :members:
+   :show-inheritance:

docs/source/module/auth/index.rst

-``gso.products``
-================
+``gso.auth``
+============
 
 .. automodule:: gso.auth
    :members:

docs/source/module/services/index.rst

@@ -14,6 +14,7 @@ Submodules
 
    crm
    infoblox
+   librenms_client
    netbox_client
    provisioning_proxy
    subscriptions

docs/source/module/services/librenms_client.rst

+``gso.services.librenms_client``
+================================
+
+.. automodule:: gso.services.librenms_client
+   :members:
+   :show-inheritance:

docs/source/modules.rst

@@ -19,6 +19,7 @@ Subpackages
    :titlesonly:
 
    module/api/index
+   module/auth/index
    module/cli/index
    module/products/index
    module/schedules/index

docs/vale/styles/config/vocabularies/geant-jargon/accept.txt

@@ -14,3 +14,4 @@ Dark_fiber
 PHASE 1
 [Mm]odify
 AAI
+[M|m]iddleware
\ No newline at end of file

gso/__init__.py

@@ -10,12 +10,14 @@ from orchestrator.cli.main import app as cli_app
 import gso.products
 import gso.workflows  # noqa: F401
 from gso.api import router as api_router
+from gso.middlewares import ModifyProcessEndpointResponse
 
 
 def init_gso_app() -> OrchestratorCore:
     """Initialise the :term:`GSO` app."""
     app = OrchestratorCore(base_settings=app_settings)
     app.include_router(api_router, prefix="/api")
+    app.add_middleware(ModifyProcessEndpointResponse)
     return app
 
 

gso/api/v1/__init__.py

@@ -3,9 +3,11 @@
 from fastapi import APIRouter
 
 from gso.api.v1.imports import router as imports_router
+from gso.api.v1.processes import router as processes_router
 from gso.api.v1.subscriptions import router as subscriptions_router
 
 router = APIRouter()
 
 router.include_router(imports_router)
 router.include_router(subscriptions_router)
+router.include_router(processes_router)

gso/api/v1/processes.py

+"""Process related endpoints."""
+
+from typing import Any
+from uuid import UUID
+
+from fastapi import APIRouter, Depends, HTTPException, status
+from orchestrator.db import ProcessStepTable
+from orchestrator.schemas.base import OrchestratorBaseModel
+
+from gso.auth.security import opa_security_default
+
+router = APIRouter(prefix="/processes", tags=["Processes"], dependencies=[Depends(opa_security_default)])
+
+
+class CallBackResultsBaseModel(OrchestratorBaseModel):
+    """Base model for callback results."""
+
+    callback_results: dict
+
+
+@router.get(
+    "/steps/{step_id}/callback-results", status_code=status.HTTP_200_OK, response_model=CallBackResultsBaseModel
+)
+def callback_results(step_id: UUID) -> dict[str, Any]:
+    """Retrieve callback results for a specific process step.
+
+    :param step_id: The unique identifier of the process step.
+    :type step_id: UUID
+
+    :return: Dictionary containing callback results.
+    :rtype: dict[str, Any]
+
+    :raises HTTPException: 404 status code if the specified step_id is not found or if the 'callback_result' key
+    is not present in the state.
+    """
+    step = ProcessStepTable.query.filter(ProcessStepTable.step_id == step_id).first()
+
+    if not (step and step.state.get("callback_result", None)):
+        raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Callback result not found.")
+
+    return {"callback_results": step.state["callback_result"]}

gso/auth/oidc_policy_helper.py

@@ -30,6 +30,16 @@ logger = get_logger(__name__)
 
 HTTPX_SSL_CONTEXT = ssl.create_default_context()  # https://github.com/encode/httpx/issues/838
 
+_CALLBACK_STEP_API_URL_PATTERN = re.compile(
+    r"^/api/processes/([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12})"
+    r"/callback/([0-9a-zA-Z\-_]+)$"
+)
+
+
+def _is_callback_step_endpoint(request: Request) -> bool:
+    """Check if the request is a callback step API call."""
+    return re.match(_CALLBACK_STEP_API_URL_PATTERN, request.url.path) is not None
+
 
 class InvalidScopeValueError(ValueError):
     """Exception raised for invalid scope values in OIDC."""
@@ -212,14 +222,18 @@ class OIDCUser(HTTPBearer):
             return None
 
         async with AsyncClient(http1=True, verify=HTTPX_SSL_CONTEXT) as async_request:
-            await self.check_openid_config(async_request)
-
             if not token:
                 credentials = await super().__call__(request)
                 if not credentials:
                     return None
                 token = credentials.credentials
+            elif _is_callback_step_endpoint(request):
+                logger.debug(
+                    "callback step endpoint is called. verification will be done by endpoint itself.", url=request.url
+                )
+                return None
 
+            await self.check_openid_config(async_request)
             intercepted_token = await self.introspect_token(async_request, token)
 
             if "active" not in intercepted_token:
@@ -397,6 +411,9 @@ def opa_decision(
         if not (oauth2lib_settings.OAUTH2_ACTIVE and oauth2lib_settings.OAUTH2_AUTHORIZATION_ACTIVE):
             return None
 
+        if _is_callback_step_endpoint(request):
+            return None
+
         try:
             json = await request.json()
         # Silencing the Decode error or Type error when request.json() does not return anything sane.

gso/middlewares.py

+"""Custom middlewares for the GSO API."""
+
+import json
+import re
+from collections.abc import Callable
+from typing import Any
+
+from fastapi import Request
+from starlette.middleware.base import BaseHTTPMiddleware
+from starlette.responses import Response
+from starlette.status import HTTP_200_OK
+
+
+class ModifyProcessEndpointResponse(BaseHTTPMiddleware):
+    """Middleware to modify the response for Process details endpoint."""
+
+    async def dispatch(self, request: Request, call_next: Callable) -> Response:
+        """Middleware to modify the response for Process details endpoint.
+
+        :param request: The incoming HTTP request.
+        :type request: Request
+
+        :param call_next: The next middleware or endpoint in the stack.
+        :type call_next: Callable
+
+        :return: The modified HTTP response.
+        :rtype: Response
+        """
+        response = await call_next(request)
+        path_pattern = re.compile(
+            r"/api/processes/([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12})"
+        )
+
+        match = path_pattern.match(request.url.path)
+
+        if match and response.status_code == HTTP_200_OK:
+            # Modify the response body as needed
+            response_body = b""
+            async for chunk in response.body_iterator:
+                response_body += chunk
+            try:
+                json_content = json.loads(response_body)
+                await self._modify_response_body(json_content, request)
+                modified_response_body = json.dumps(json_content).encode()
+                headers = dict(response.headers)
+                headers["content-length"] = str(len(modified_response_body))
+                return Response(
+                    content=modified_response_body,
+                    status_code=response.status_code,
+                    headers=headers,
+                    media_type=response.media_type,
+                )
+
+            except json.JSONDecodeError:
+                pass
+
+        return response
+
+    @staticmethod
+    async def _get_token(request: Request) -> str:
+        """Get the token from the request headers.
+
+        :param request: The incoming HTTP request.
+        :type request: Request
+
+        :return: The token from the request headers in specific format.
+        :rtype: str
+        """
+        bearer_prefix = "Bearer "
+        authorization_header = request.headers.get("Authorization")
+        if authorization_header:
+            # Remove the "Bearer " prefix from the token
+            token = authorization_header.replace(bearer_prefix, "")
+            return f"?token={token}"
+        return ""
+
+    async def _modify_response_body(self, response_body: dict[str, Any], request: Request) -> None:
+        """Modify the response body as needed.
+
+        :param response_body: The response body in dictionary format.
+        :type response_body: dict[str, Any]
+        :param request: The incoming HTTP request.
+        :type request: Request
+
+        :return: None
+        """
+        max_output_length = 500
+        token = await self._get_token(request)
+        try:
+            for step in response_body["steps"]:
+                if step["state"].get("callback_result", None):
+                    callback_result = step["state"]["callback_result"]
+                    if callback_result and isinstance(callback_result, str):
+                        callback_result = json.loads(callback_result)
+                    if callback_result.get("output") and len(callback_result["output"]) > max_output_length:
+                        callback_result[
+                            "output"
+                        ] = f'{request.base_url}api/v1/processes/steps/{step["step_id"]}/callback-results{token}'
+                    step["state"]["callback_result"] = callback_result
+        except (AttributeError, KeyError, TypeError):
+            pass

gso/migrations/versions/2023-12-18_bacd55c26106_add_ibgp_mesh_workflow.py

+"""Add iBGP mesh workflow.
+
+Revision ID: bacd55c26106
+Revises: f0764c6f392c
+Create Date: 2023-12-18 17:58:29.581963
+
+"""
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = 'bacd55c26106'
+down_revision = 'f0764c6f392c'
+branch_labels = None
+depends_on = None
+
+
+from orchestrator.migrations.helpers import create_workflow, delete_workflow
+
+new_workflows = [
+    {
+        "name": "update_ibgp_mesh",
+        "target": "MODIFY",
+        "description": "Update iBGP mesh",
+        "product_type": "Router"
+    }
+]
+
+
+def upgrade() -> None:
+    conn = op.get_bind()
+    for workflow in new_workflows:
+        create_workflow(conn, workflow)
+
+
+def downgrade() -> None:
+    conn = op.get_bind()
+    for workflow in new_workflows:
+        delete_workflow(conn, workflow["name"])

gso/oss-params-example.json

@@ -45,6 +45,25 @@
       "dns_view": "default"
     }
   },
+  "MONITORING": {
+    "LIBRENMS": {
+      "base_url": "https://librenms/api/v0",
+      "token": "<token>"
+    },
+    "SNMP": {
+      "v2c": {
+        "community": "secret-community"
+      },
+      "v3": {
+        "authlevel": "AuthPriv",
+        "authname": "librenms",
+        "authpass": "<password1>",
+        "authalgo": "sha",
+        "cryptopass": "<password2>",
+        "cryptoalgo": "aes"
+      }
+    }
+  },
   "PROVISIONING_PROXY": {
     "scheme": "https",
     "api_base": "localhost:44444",

gso/services/crm.py

@@ -6,8 +6,6 @@ For the time being, it's hardcoded to only contain GÉANT as a customer, since t
 
 from typing import Any
 
-from pydantic_forms.validators import Choice
-
 
 class CustomerNotFoundError(Exception):
     """Exception raised when a customer is not found."""
@@ -31,12 +29,3 @@ def get_customer_by_name(name: str) -> dict[str, Any]:
 
     msg = f"Customer {name} not found"
     raise CustomerNotFoundError(msg)
-
-
-def customer_selector() -> Choice:
-    """GUI input field for selecting a customer."""
-    customers = {}
-    for customer in all_customers():
-        customers[customer["id"]] = customer["name"]
-
-    return Choice("Select a customer", zip(customers.keys(), customers.items(), strict=True))  # type: ignore[arg-type]

gso/services/librenms_client.py

+"""The LibreNMS module interacts with the inventory management system of :term:`GAP`."""
+
+import logging
+from http import HTTPStatus
+from importlib import metadata
+from typing import Any
+
+import requests
+from requests import HTTPError
+
+from gso.settings import load_oss_params
+from gso.utils.helpers import SNMPVersion
+
+logger = logging.getLogger(__name__)
+
+
+class LibreNMSClient:
+    """The client for LibreNMS that interacts with the inventory management system."""
+
+    def __init__(self) -> None:
+        """Initialise a new LibreNMS client with an authentication token."""
+        config = load_oss_params().MONITORING
+        token = config.LIBRENMS.token
+
+        self.base_url = config.LIBRENMS.base_url
+        self.snmp_config = config.SNMP
+
+        self.headers = {
+            "User-Agent": f"geant-service-orchestrator/{metadata.version('geant-service-orchestrator')}",
+            "Accept": "application/json",
+            "Content-Type": "application/json",
+            "X-Auth-Token": token,
+        }
+
+    def get_device(self, fqdn: str) -> dict[str, Any]:
+        """Get an existing device from LibreNMS.
+
+        :param str fqdn: The :term:`FQDN` of a device that is retrieved.
+        :return dict[str, Any]: A :term:`JSON` formatted list of devices that match the queried :term:`FQDN`.
+        :raises HTTPError: Raises an HTTP error 404 when the device is not found
+        """
+        response = requests.get(f"{self.base_url}/devices/{fqdn}", headers=self.headers, timeout=(0.5, 75))
+        response.raise_for_status()
+
+        return response.json()
+
+    def device_exists(self, fqdn: str) -> bool:
+        """Check whether a device exists in LibreNMS.
+
+        :param str fqdn: The hostname that should be checked for.
+        :return bool: Whether the device exists or not.
+        """
+        try:
+            device = self.get_device(fqdn)
+        except HTTPError as e:
+            if e.response.status_code == HTTPStatus.NOT_FOUND:
+                return False
+            raise
+
+        return device["status"] == "ok"
+
+    def add_device(self, fqdn: str, snmp_version: SNMPVersion) -> dict[str, Any]:
+        """Add a new device to LibreNMS.
+
+        :param str fqdn: The hostname of the newly added device.
+        :param SNMPVersion snmp_version: The SNMP version of the new device, which decides the authentication parameters
+                                         that LibreNMS should use to poll the device.
+        """
+        device_data = {
+            "display": fqdn,
+            "hostname": fqdn,
+            "sysName": fqdn,
+            "snmpver": snmp_version.value,
+        }
+        device_data.update(getattr(self.snmp_config, snmp_version))
+
+        device = requests.post(f"{self.base_url}/devices", headers=self.headers, json=device_data, timeout=(0.5, 75))
+        device.raise_for_status()
+
+        return device.json()
+
+    def remove_device(self, fqdn: str) -> dict[str, Any]:
+        """Remove a device from LibreNMS.
+
+        :param str fqdn: The :term:`FQDN` of the hostname that should get deleted.
+        :return dict[str, Any]: A JSON representation of the device that got removed.
+        :raises HTTPError: Raises an exception if the request did not succeed.
+        """
+        device = requests.delete(f"{self.base_url}/devices/{fqdn}", headers=self.headers, timeout=(0.5, 75))
+        device.raise_for_status()
+
+        return device.json()
+
+    def validate_device(self, fqdn: str) -> list[str]:
+        """Validate a device in LibreNMS by fetching the record match the queried :term:`FQDN` against its hostname.
+
+        :param str fqdn: The :term:`FQDN` of the host that is validated.
+        :return list[str]: A list of errors, if empty the device is successfully validated.
+        """
+        errors = []
+        try:
+            device = self.get_device(fqdn)
+
+            if device["devices"][0]["hostname"] != fqdn:
+                errors += ["Device hostname in LibreNMS does not match FQDN."]
+        except HTTPError as e:
+            if e.response.status_code == HTTPStatus.NOT_FOUND:
+                errors += ["Device does not exist in LibreNMS."]
+            else:
+                raise
+
+        return errors

gso/services/netbox_client.py

@@ -300,7 +300,7 @@ class NetboxClient:
         ]
 
         # Generate all feasible LAGs
-        all_feasible_lags = [f"LAG-{i}" for i in FEASIBLE_IP_TRUNK_LAG_RANGE]
+        all_feasible_lags = [f"lag-{i}" for i in FEASIBLE_IP_TRUNK_LAG_RANGE]
 
         # Return available LAGs not assigned to the device
         return [lag for lag in all_feasible_lags if lag not in lag_interface_names]

gso/services/subscriptions.py

@@ -15,7 +15,9 @@ from orchestrator.db import (
     SubscriptionInstanceValueTable,
     SubscriptionTable,
 )
+from orchestrator.services.subscriptions import query_in_use_by_subscriptions
 from orchestrator.types import SubscriptionLifecycle
+from pydantic_forms.types import UUIDstr
 
 from gso.products import ProductType
 
@@ -85,6 +87,43 @@ def get_active_router_subscriptions(
     return get_active_subscriptions(product_type="Router", includes=includes)
 
 
+def get_active_iptrunk_subscriptions(
+    includes: list[str] | None = None,
+) -> list[SubscriptionType]:
+    """Retrieve active subscriptions specifically for IP trunks.
+
+    :param includes: The fields to be included in the returned Subscription objects.
+    :type includes: list[str]
+
+    :return: A list of Subscription objects for IP trunks.
+    :rtype: list[Subscription]
+    """
+    return get_active_subscriptions(product_type="Iptrunk", includes=includes)
+
+
+def get_active_trunks_that_terminate_on_router(subscription_id: UUIDstr) -> list[SubscriptionTable]:
+    """Get all IP trunk subscriptions that are active, and terminate on the given ``subscription_id`` of a Router.
+
+    Given a ``subscription_id`` of a Router subscription, this method gives a list of all active IP trunk subscriptions
+    that terminate on this Router.
+
+    :param subscription_id: Subscription ID of a Router
+    :type subscription_id: UUIDstr
+
+    :return: A list of IP trunk subscriptions
+    :rtype: list[SubscriptionTable]
+    """
+    return (
+        query_in_use_by_subscriptions(UUID(subscription_id))
+        .join(ProductTable)
+        .filter(
+            ProductTable.product_type == "Iptrunk",
+            SubscriptionTable.status == "active",
+        )
+        .all()
+    )
+
+
 def get_product_id_by_name(product_name: ProductType) -> UUID:
     """Retrieve the :term:`UUID` of a product by its name.