Merge branch 'release/0.1'

mapping_provider/config.py

+import os
+
+from pydantic import BaseModel, Field, HttpUrl
+
+
+class InfluxConnectionParams(BaseModel):
+    hostname: str
+    port: int = 8086
+    ssl: bool = True
+    username: str
+    password: str
+    database: str
+    measurement: str
+
+
+class RMQConnectionParams(BaseModel):
+    brokers: list[str]
+    username: str
+    password: str
+    vhost: str
+
+
+class SentryConfig(BaseModel):
+    dsn: str
+    environment: str
+    level: str = Field(
+        default='error',
+        pattern="^(debug|info|warning|error)$")
+
+    # @field_validator('level')
+    # @classmethod
+    # def validate_level(cls, v):
+    #     if v not in ['debug', 'info', 'warning', 'error']:
+    #         raise ValueError('level must be one of: debug, info, warning, error')
+    #     return v
+
+class Configuration(BaseModel):
+    sentry: SentryConfig | None = None
+    inventory: HttpUrl
+    reporting: HttpUrl
+    rmq: RMQConnectionParams | None = None
+    correlator_exchange: str = 'dashboard.alarms.broadcast'
+    brian: InfluxConnectionParams
+
+
+def load() -> Configuration:
+    """
+    Loads, validates and returns configuration parameters from
+    the file named in the environment variable 'SETTINGS_FILENAME'.
+
+    :return:
+    """
+    assert 'SETTINGS_FILENAME' in os.environ
+    with open(os.environ['SETTINGS_FILENAME']) as f:
+        return Configuration.model_validate_json(f.read())

mapping_provider/environment.py

+"""
+Environment setup
+===================
+
+.. autofunction:: mapping_provider.environment.setup_logging
+
+.. autofunction:: mapping_provider.environment.setup_sentry
+
+"""
+import json
+import logging
+import logging.config
+import os
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from mapping_provider.config import SentryConfig
+
+import sentry_sdk
+from sentry_sdk.integrations.logging import LoggingIntegration
+
+LOGGING_DEFAULT_CONFIG = {
+    'version': 1,
+    'disable_existing_loggers': False,
+    'formatters': {
+        'simple': {
+            'format': '%(asctime)s - %(name)s '
+                      '(%(lineno)d) - %(levelname)s - %(message)s'
+        }
+    },
+
+    'handlers': {
+        'console': {
+            'class': 'logging.StreamHandler',
+            'level': 'DEBUG',
+            'formatter': 'simple',
+            'stream': 'ext://sys.stdout'
+        }
+
+    },
+
+    'loggers': {
+        'mapping_provider': {
+            'level': 'DEBUG',
+            'handlers': ['console'],
+            'propagate': False
+        }
+    },
+
+    'root': {
+        'level': 'WARNING',
+        'handlers': ['console']
+    }
+}
+
+
+def setup_logging() -> None:
+    """
+    Sets up logging using the configured filename.
+
+    if LOGGING_CONFIG is defined in the environment, use this for
+    the filename, otherwise use LOGGING_DEFAULT_CONFIG
+    """
+    logging_config = LOGGING_DEFAULT_CONFIG
+    if 'LOGGING_CONFIG' in os.environ:
+        filename = os.environ['LOGGING_CONFIG']
+        with open(filename) as f:
+            logging_config = json.load(f)
+
+    logging.config.dictConfig(logging_config)
+
+
+def setup_sentry(sentry_config_params: 'SentryConfig') -> None:
+    """
+    Sets up the sentry instrumentation based on the Configuration.sentry params.
+    """
+    match str(sentry_config_params.level):
+        case 'debug':
+            level = logging.DEBUG
+        case 'info':
+            level = logging.INFO
+        case 'warning':
+            level = logging.WARNING
+        case 'error':
+            level = logging.ERROR
+
+    sentry_sdk.init(
+        dsn=sentry_config_params.dsn,
+        environment=sentry_config_params.environment,
+        integrations=[
+            LoggingIntegration(
+                level=logging.INFO,  # Capture info and above as breadcrumbs
+                event_level=level  # Send records as events
+            ),
+        ],
+    )

pyproject.toml

+[build-system]
+requires = ["setuptools>=61.0", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "mapping-provider"
+version = "0.1"
+description = "A webservice providing information for GEANT maps."
+authors = [
+    {name = "GÉANT", email = "swd@geant.org"}
+]
+license = "MIT"
+readme = "README.md"
+requires-python = ">=3.10"
+classifiers = [
+    "Programming Language :: Python :: 3",
+    "Framework :: FastAPI",
+    "Operating System :: OS Independent",
+]
+dependencies = [
+    "fastapi",
+    "requests",
+    "jsonschema",
+    "sentry_sdk",
+    "pika",
+    "influxdb"
+]
+
+[tool.setuptools]
+include-package-data = true
+
+[tool.setuptools.packages.find]
+exclude = ["test"]
+
+
 [tool.ruff]
 line-length = 120
 target-version = "py313"
-select = ["E", "F", "I", "B", "UP", "N"]
-fixable = ["ALL"]
+lint.select = ["E", "F", "I", "B", "UP", "N"]
+lint.fixable = ["ALL"]
 exclude = ["tests", "docs", "build"]
 
 [tool.mypy]
 python_version = "3.13"
 strict = true
 warn_unused_ignores = true
-warn_return_any = true
\ No newline at end of file
+warn_return_any = true
+
+[[tool.mypy.overrides]]
+module = ["influxdb"]
+ignore_missing_imports = true
+
+[tool.coverage.run]
+source = ["mapping_provider"]
+omit = [
+    "mapping_provider/backends/rmq/*",
+    "test/*"
+]

requirements.txt

 fastapi
-uvicorn[standard]
+requests
+jsonschema
+sentry_sdk
+pika
+influxdb
 
-sphinx
-sphinx-rtd-theme
-sphinxcontrib-plantuml
-sphinxcontrib-drawio
-sphinxcontrib-openapi
+httpx  # required for fastapi TestClient
+pytest
+responses
 
 ruff
 mypy
-tox
 pre-commit

setup.py

-from setuptools import find_packages, setup
-
-setup(
-    name="mapping-provider",
-    version="0.1",
-    description="A FastAPI app to provide mapping services for GEANT maps.",
-    author="GÉANT",
-    author_email="info@geant.org",
-    license="MIT",
-    packages=find_packages(where="mapping_provider"),
-    package_dir={"": "mapping_provider"},
-    include_package_data=True,
-    python_requires=">=3.10",
-    install_requires=[
-        "fastapi",
-        "uvicorn[standard]"
-    ],
-    long_description=open("README.md", encoding="utf-8").read(),
-    long_description_content_type="text/markdown",
-    classifiers=[
-        "Programming Language :: Python :: 3",
-        "Framework :: FastAPI",
-        "License :: OSI Approved :: MIT License",
-        "Operating System :: OS Independent",
-    ],
-)
\ No newline at end of file

test/common.py

+import json
+import os
+
+DATA_DIR = os.path.join(os.path.dirname(__file__), 'data')
+
+
+def load_test_data(filename: str) -> dict:
+    with open(os.path.join(DATA_DIR, filename)) as f:
+        return json.load(f)
\ No newline at end of file

test/conftest.py

+import json
+import os
+import tempfile
+from unittest.mock import patch
+
+import pytest
+from fastapi.testclient import TestClient
+
+from mapping_provider import create_app
+from mapping_provider.backends import brian, cache, correlator, inventory
+
+from .common import load_test_data
+
+
+@pytest.fixture
+def dummy_config():
+    return {
+        'sentry': {
+            'dsn': 'https://token@hostname.geant.org:1111/a/b',
+            'environment': 'unit tests'
+        },
+        'inventory': 'https://inventory.bogus.domain',
+        'reporting': 'http://reporting.another-bogus.domain',
+        # no rmq param to skip correlator thread
+        # 'rmq': {
+        #     'brokers': [
+        #         'test-noc-alarms01.geant.org',
+        #         'test-noc-alarms02.geant.org',
+        #         'test-noc-alarms03.geant.org'
+        #     ],
+        #     'username': 'guest',
+        #     'password': 'guest',
+        #     'vhost': '/'
+        # },
+        'brian': {
+            'hostname': 'bogus hostname',
+            'username': 'bogus username',
+            'password': 'bogus password',
+            'database': 'bogus database name',
+            'measurement': 'bogus measurement'
+        }
+    }
+
+
+@pytest.fixture
+def dummy_config_filename(dummy_config):
+    with tempfile.NamedTemporaryFile(delete=False) as f:
+        f.write(json.dumps(dummy_config).encode('utf-8'))
+        f.flush()
+        yield f.name
+
+
+@pytest.fixture
+def client(dummy_config_filename):
+    os.environ['SETTINGS_FILENAME'] = dummy_config_filename
+
+    with tempfile.TemporaryDirectory() as tmp_dir:
+
+        # there's no rmq in the test config data, so cache won't be initialized
+        cache.init(tmp_dir)
+        cache.set(inventory.REPORTING_SCID_CURRENT_CACHE_FILENAME, load_test_data('scid-current.json'))
+        cache.set(inventory.INPROV_EQUIPMENT_CACHE_FILENAME, load_test_data('inprov-equipment.json'))
+        cache.set(correlator.CACHED_CORRELATOR_STATE_FILENAME, load_test_data('correlator-state.json'))
+        cache.set(brian.CACHED_BRIAN_SCID_RATES_FILENAME, load_test_data('brian-scid-rates.json'))
+ 
+        with patch('sentry_sdk.init') as _mock_sentry_init:
+            yield TestClient(create_app())
+
+
+@pytest.fixture(autouse=True)
+def run_around_tests():
+    assert cache._cache_dir is None  # test env sanity check
+    yield
+
+    # make sure cache is set to unused before the next test
+    cache._cache_dir = None

test/test_correlator.py

+import tempfile
+
+from mapping_provider.backends import cache, correlator
+
+
+def test_handle_correlator_state_broadcast():
+    """
+    tmp bogus test - just to have something
+    """
+    with tempfile.TemporaryDirectory() as tmp_dir:
+        cache.init(tmp_dir)
+        correlator.handle_correlator_state_broadcast(
+            message={'alarms': [], 'endpoints': []})
+        
+        cached_data = cache.get(correlator.CACHED_CORRELATOR_STATE_FILENAME)
+        assert cached_data == {'alarms': [], 'endpoints': []}

test/test_inventory.py

+import tempfile
+
+import responses
+
+from mapping_provider.backends import cache, inventory
+
+from .common import load_test_data
+
+
+@responses.activate
+def test_inventory_service_download():
+    """
+    tmp bogus test - just to have something
+    """
+
+    inventory_base_uri = 'https://dummy-hostname.dummy.domain'
+    reporting_base_uri = 'https://another-dummy-hostname.dummy.domain'
+
+    responses.add(
+        method=responses.GET,
+        url=f'{reporting_base_uri}/scid/current',
+        json=load_test_data('scid-current.json')
+    )
+    responses.add(
+        method=responses.GET,
+        url=f'{inventory_base_uri}/map/equipment',
+        json=load_test_data('inprov-equipment.json')
+    )
+    
+    with tempfile.TemporaryDirectory() as tmp_dir:
+
+        cache.init(tmp_dir)
+        inventory._load_all_inventory(
+            inventory_base_uri=inventory_base_uri,
+            reporting_base_uri=reporting_base_uri)
+
+        cached_data = cache.get(inventory.INPROV_EQUIPMENT_CACHE_FILENAME)
+        assert cached_data == load_test_data('inprov-equipment.json')
+
+        cached_data = cache.get(inventory.REPORTING_SCID_CURRENT_CACHE_FILENAME)
+        assert cached_data == load_test_data('scid-current.json')

test/test_map_endpoints.py

+import re
+
+import pytest
+import responses
+
+from mapping_provider.api.map import EquipmentList, PopList
+from mapping_provider.backends.services import ServiceList
+
+from .common import load_test_data
+
+
+@responses.activate
+def test_get_pops(client):
+
+    responses.add(
+        method=responses.GET,
+        url=re.compile(r'.*/map/pops$'),
+        json=load_test_data('inprov-pops.json')
+    )
+
+    rv = client.get("/map/pops")
+    assert rv.status_code == 200
+    pop_list = PopList.model_validate(rv.json())
+    assert pop_list.pops, 'test data should not be empty'
+
+
+@responses.activate
+def test_get_equipment(client):
+
+    responses.add(
+        method=responses.GET,
+        url=re.compile(r'.*/map/equipment$'),
+        json=load_test_data('inprov-equipment.json')
+    )
+
+    rv = client.get("/map/equipment")
+    assert rv.status_code == 200
+    equipment_list = EquipmentList.model_validate(rv.json())
+    assert equipment_list.equipment, 'test data should not be empty'
+
+
+
+
+@responses.activate
+@pytest.mark.parametrize('service_type', [
+    'IP PEERING - R&E',
+    'GEANT SPECTRUM SERVICE',
+    'L3-VPN',
+    'OOB IP LINK',
+    'GEANT OPEN CROSS CONNECT',
+    'GEANT - GBS',
+    'GWS - INDIRECT',
+    'GEANT IP',
+    'POP LAN LINK',
+    'IP PEERING - NON R&E (PUBLIC)',
+    'IP TRUNK',
+    'GEANT PLUS',
+    'L2SERVICES',
+    'ETHERNET',
+    'EUMETSAT TERRESTRIAL',
+    'IP PEERING - NON R&E (PRIVATE)',
+    'EXPRESS ROUTE',
+    'CBL1',
+    'GWS - UPSTREAM',
+    'GEANT PEERING',
+    'SERVER LINK',
+    'GEANT MANAGED WAVELENGTH SERVICE',
+    'CORPORATE',
+    'EUMETSAT GRE'])
+def test_get_services(client, service_type):
+    rv = client.get(f"/map/services/{service_type}")
+    assert rv.status_code == 200
+    service_list = ServiceList.model_validate(rv.json())
+    assert service_list.services, 'test data should not be empty'
+    assert all(s.type == service_type for s in service_list.services)
+
+
+def test_get_unknown_service_type(client):
+    rv = client.get("/map/services/BOGUS_SERVICE_TYPE")
+    assert rv.status_code == 404
+
+
+def test_get_all_services(client):
+    rv = client.get("/map/services")
+    assert rv.status_code == 200
+    service_list = ServiceList.model_validate(rv.json())
+    assert service_list.services, 'test data should not be empty'
+
+@responses.activate
+def test_get_trunks(client):
+    rv = client.get("/map/trunks")
+    assert rv.status_code == 200
+    service_list = ServiceList.model_validate(rv.json())
+    assert service_list.services, 'test data should not be empty'
+    assert all(s.type == 'IP TRUNK' for s in service_list.services)
+
+

test/test_utilization.py

+import tempfile
+from unittest.mock import MagicMock, patch
+
+import jsonschema
+
+from mapping_provider.backends import brian, cache
+from mapping_provider.config import InfluxConnectionParams
+
+from .common import load_test_data
+
+
+def test_utilization():
+
+    with tempfile.TemporaryDirectory() as tmpdir:
+        cache.init(tmpdir)
+
+        with patch('mapping_provider.backends.brian.InfluxDBClient') as mocked_influx:
+        # with patch('influxdb.InfluxDBClient') as mocked_influx_client:
+
+            mocked_client_instance = MagicMock()
+            mocked_influx.return_value = mocked_client_instance
+
+            mocked_query = MagicMock()
+            mocked_query.return_value.raw = load_test_data('influx-scid-rates-query-result.json')
+
+            mocked_client_instance.query = mocked_query
+
+            brian.load_scid_rates(InfluxConnectionParams(
+                hostname='bogus hostname',
+                username='bogus username',
+                password='bogus password',
+                database='bogus database name',
+                measurement='bogus measurement'))
+
+            cached_scid_rates = cache.get(brian.CACHED_BRIAN_SCID_RATES_FILENAME)
+            assert cached_scid_rates, "test data is not empty"
+            jsonschema.validate(cached_scid_rates, brian.CACHED_SCID_RATES_SCHEMA)

test/test_version.py

+from mapping_provider.api.common import Version
+
+
+def test_version(client):
+    rv = client.get("/version")
+    assert rv.status_code == 200
+    assert rv.json()
+    Version.model_validate(rv.json())

tox.ini

-[tox]
-envlist = lint, typecheck, docs
 
-[testenv:lint]
-description = Lint code with Ruff
-deps = ruff
-commands = ruff check mapping_provider
-
-[testenv:typecheck]
-description = Type-check code with mypy
-deps = mypy
-commands = mypy mapping_provider
-
-[testenv:docs]
-description = Build docs
+[testenv]
 deps =
-    sphinx
-    sphinx-rtd-theme
-    sphinxcontrib-plantuml
-    sphinxcontrib-drawio
-    sphinxcontrib-openapi
-commands = sphinx-build -b html docs/source docs/build
+  pytest
+  pytest-cov
+  httpx  # required for fastapi TestClient
+  responses
+  ruff
+  mypy
+  types-jsonschema
+  types-requests
+  types-pika
+  cyclonedx-py
+  sphinx
+  sphinx-rtd-theme
+  sphinxcontrib-plantuml
+  sphinxcontrib-drawio
+  sphinxcontrib-openapi
+commands =
+  coverage erase
+  pytest --cov mapping_provider --cov-fail-under=80 --cov-report html --cov-report xml --cov-report term -p no:checkdocs
+  ruff check mapping_provider test
+  mypy mapping_provider
+  cyclonedx-py environment --output-format json -o bom.json
+  sphinx-build -b html docs/source docs/build
 
+# [tox]
+# envlist = coverage, lint, typecheck, docs
+# 
+# [testenv:coverage]
+# description = Run unit tests and save coverage
+# deps =
+#   pytest
+#   pytest-cov
+#   httpx  # required for fastapi TestClient
+#   responses
+# commands =
+#   coverage erase
+#   pytest --cov mapping_provider --cov-fail-under=80 --cov-report html --cov-report xml --cov-report term -p no:checkdocs
+# 
+# [testenv:lint]
+# description = Lint code with Ruff
+# deps = ruff
+# commands = ruff check mapping_provider test
+# 
+# [testenv:typecheck]
+# description = Type-check code with mypy
+# deps =
+#     mypy
+#     types-jsonschema
+#     types-requests
+#     types-pika
+# commands = mypy mapping_provider
+# 
+# [testenv:sbom]
+# description = Create SBOM for dependency analysis
+# deps = cyclonedx-py
+# commands = cyclonedx-py environment --output-format json -o bom.json
+# 
+# [testenv:docs]
+# description = Build docs
+# deps =
+#     sphinx
+#     sphinx-rtd-theme
+#     sphinxcontrib-plantuml
+#     sphinxcontrib-drawio
+#     sphinxcontrib-openapi
+# commands = sphinx-build -b html docs/source docs/build