Merge branch 'router-info-endpoint' into 'develop'

Add classifier endpoint for router info

See merge request !44

inventory_provider/routes/classifier.py

@@ -38,19 +38,34 @@ These endpoints are intended for use by Dashboard V3.
 --------------------------------
 
 .. autofunction:: inventory_provider.routes.classifier.get_mtc_interface_info
+
+
+/classifier/router-info
+--------------------------------
+
+.. autofunction:: inventory_provider.routes.classifier.get_all_routers
+
+
+/classifier/router-info/<equimpent-name>
+--------------------------------
+
+.. autofunction:: inventory_provider.routes.classifier.get_router_info
+
 """
+
+import functools
 import ipaddress
 import json
 import logging
 import re
 from functools import lru_cache
-from typing import Optional
+from typing import Iterable, List, Optional
 
-from flask import Blueprint, Response, request
+from flask import Blueprint, Response, jsonify, request
 from redis import Redis
 
 from inventory_provider.routes import common
-from inventory_provider.routes.common import _ignore_cache_or_retrieve
+from inventory_provider.routes.common import _ignore_cache_or_retrieve, cache_result
 
 routes = Blueprint("inventory-data-classifier-support-routes", __name__)
 
@@ -137,22 +152,39 @@ def after_request(resp):
 
 
 @lru_cache(256)
-def get_ims_equipment_name(equipment_name: str, r: Redis = None) -> str:
-    if not r:
-        r = common.get_current_redis()
+def get_ims_equipment_name(equipment_name: str, redis: Redis = None) -> str:
+    redis = redis or common.get_current_redis()
+    candidates = _get_equipment_names_candidates(equipment_name)
+    result = _get_candidate_or_none(candidates, redis)
+    return result or candidates[0]
+
+
+# We can't lru_cache this function like the above one, since if new equipment is
+# installed and queried before the inventory is updated, it may return None. Then that
+# result would be cached, and it would still not recognize that equipment, even after
+# the inventory is updated
+def get_ims_equipment_name_or_none(
+    equipment_name: str, redis: Redis = None
+) -> Optional[str]:
+    redis = redis or common.get_current_redis()
+    candidates = _get_equipment_names_candidates(equipment_name)
+    return _get_candidate_or_none(candidates, redis)
+
+
+def _get_candidate_or_none(candidates: Iterable[str], r: Redis) -> Optional[str]:
+    for c in candidates:
+        if r.exists(f"ims:location:{c}"):
+            return c
+    return None
+
+
+def _get_equipment_names_candidates(equipment_name) -> List[str]:
     ims_equipment_name = equipment_name.upper()
-    candidates = [
-        ims_equipment_name.split('.GEANT.')[0],
+    return [
+        ims_equipment_name.split(".GEANT.")[0],
         ims_equipment_name,
-        ims_equipment_name.split('.OFFICE.')[0]
+        ims_equipment_name.split(".OFFICE.")[0],
     ]
-    return_value = candidates[0]
-    loc_key = 'ims:location:{}'
-    for c in candidates:
-        if r.exists(loc_key.format(c)):
-            return_value = c
-            break
-    return return_value
 
 
 def get_ims_interface(interface: str) -> str:
@@ -1120,3 +1152,75 @@ def _get_coriant_info(
         r.set(cache_key, result.encode('utf-8'))
 
     return Response(result, mimetype="application/json")
+
+
+@routes.route("/router-info", methods=["GET"])
+@common.require_accepts_json
+def get_all_routers() -> Response:
+    redis = common.get_current_redis()
+    all_routers_raw = redis.get("netdash")
+    all_routers = json.loads(all_routers_raw) if all_routers_raw else {}
+    return jsonify(
+        [
+            {"hostname": hostname, "vendor": vendor}
+            for hostname, vendor in all_routers.items()
+        ]
+    )
+
+
+@routes.route("/router-info/<equipment_name>", methods=["GET"])
+@common.require_accepts_json
+def get_router_info(equipment_name: str) -> Response:
+    redis = common.get_current_redis()
+    ims_equipment_name = get_ims_equipment_name_or_none(equipment_name, redis)
+    if not ims_equipment_name:
+        return Response(
+            response=f"no router info available for {equipment_name}",
+            status=404,
+            mimetype="text/plain",
+        )
+
+    cache_key = f"classifier-cache:router:{ims_equipment_name}"
+
+    result = cache_result(
+        cache_key,
+        func=functools.partial(_get_router_info, ims_equipment_name, redis),
+        redis=redis,
+    )
+    return Response(result, mimetype="application/json")
+
+
+def _get_router_info(ims_source_equipment, redis):
+
+    def _format_service(service: dict):
+        keys = {"name", "status", "service_type", "sid"}
+        return {k: v for k, v in service.items() if k in keys}
+
+    related_services = {}
+    contacts = set()
+
+    vendor = "unknown"
+    all_routers = redis.get("netdash")
+
+    for name, v in (json.loads(all_routers) if all_routers else {}).items():
+        if name.startswith(ims_source_equipment.lower()):
+            vendor = v
+            break
+
+    for key in redis.scan_iter(f"ims:interface_services:{ims_source_equipment}:*"):
+        raw_services = redis.get(key.decode())
+        if not raw_services:
+            continue
+
+        services = json.loads(raw_services)
+        for service in services:
+            related_services.update({r["id"]: r for r in service["related-services"]})
+            contacts.update(service.get("contacts", []))
+    return {
+        "related-services": sorted(
+            map(_format_service, related_services.values()), key=lambda s: s["name"]
+        ),
+        "contacts": sorted(contacts),
+        "location": _location_from_equipment(ims_source_equipment, redis),
+        "vendor": vendor,
+    }

inventory_provider/routes/classifier_schema.py

@@ -752,3 +752,57 @@ TNMS_FIBERLINK_INFO_RESPONSE_SCHEMA = {
         'locations', 'ends', 'df_route', 'related-services', 'contacts'],
     'additionalProperties': False
 }
+
+ROUTER_INFO_RESPONSE_SCHEMA = {
+    "$schema": "https://json-schema.org/draft-07/schema#",
+    "definitions": {
+        "related-service-info": {
+            "type": "object",
+            "properties": {
+                "name": {"type": "string"},
+                "status": {"$ref": "#/definitions/status"},
+                "service_type": {"type": "string"},
+                "sid": {"type": "string"},
+            },
+            "required": ["name", "status", "service_type"],
+            "additionalProperties": False,
+        },
+        "status": {
+            "type": "string",
+            "enum": [
+                "planned",
+                "installed",
+                "operational",
+                "terminated",
+                "disposed",
+                "non-monitored",
+            ],
+        },
+        **_common_locations_schema_definitions,
+    },
+    "type": "object",
+    "properties": {
+        "related-services": {
+            "type": "array",
+            "items": {"$ref": "#/definitions/related-service-info"},
+        },
+        "contacts": {"type": "array", "items": {"type": "string"}},
+        "location": {"$ref": "#/definitions/location-endpoint"},
+        "vendor": {"type": "string"},
+    },
+    "required": ["related-services", "contacts", "location", "vendor"],
+    "additionalProperties": False,
+}
+ROUTER_INFO_ALL_ROUTERS_RESPONSE_SCHEMA = {
+    "$schema": "https://json-schema.org/draft-07/schema#",
+    "type": "array",
+    "items": {
+        "type": "object",
+        "properties": {
+            "hostname": {"type": "string"},
+            "vendor": {"type": "string", "enum": ["juniper", "nokia", "unknown"]},
+        },
+        "required": ["hostname", "vendor"],
+        "additionalProperties": False,
+    },
+}

inventory_provider/routes/common.py

@@ -8,6 +8,7 @@ import re
 import threading
 
 from distutils.util import strtobool
+from typing import Optional
 from lxml import etree
 import requests
 from flask import request, Response, current_app, g
@@ -28,10 +29,35 @@ def get_bool_request_arg(name, default=False):
     return value
 
 
+def cache_result(key, func, redis) -> Optional[str]:
+    """Get the result one way or the other, first check in the cache and return it
+    if it exists. If it doesn't, or ignore-cache is active, it will obtain the
+    result from ``func``, a callable that takes zero arguments and returns
+    the result. If this result is not None, the result will then be cached. The final
+    result will be a json encoded string (or None).
+
+    :param key: a redis key
+    :param func: a callable that takes zero arguments and returns a dict or other
+        json-encodable object
+    :param redis: a Redis instance
+
+    :return: a str with the json-encoded result data
+    """
+    result = _ignore_cache_or_retrieve(None, key, redis)
+    if result is not None:
+        return result
+    data = func()
+    if data is not None:
+        data = json.dumps(data)
+        # cache this data for the next call
+        redis.set(key, data.encode())
+    return data
+
+
 def _ignore_cache_or_retrieve(request_, cache_key, r):
     ignore_cache = get_bool_request_arg('ignore-cache', default=False)
     if ignore_cache:
-        result = False
+        result = None
         logger.debug('ignoring cache')
     else:
         result = r.get(cache_key)

test/conftest.py

@@ -260,6 +260,7 @@ def mocked_redis(mocker):
     mocker.patch(
         'inventory_provider.tasks.common.redis.StrictRedis',
         return_value=instance)
+    return instance
 
 
 @pytest.fixture

test/test_classifier_routes.py

@@ -2,11 +2,17 @@ import json
 import jsonschema
 import pytest
 
-from inventory_provider.routes.classifier_schema \
-    import JUNIPER_LINK_RESPONSE_SCHEMA, PEER_INFO_RESPONSE_SCHEMA, \
-    INFINERA_LAMBDA_INFO_RESPONSE_SCHEMA, CORIANT_INFO_RESPONSE_SCHEMA, \
-    INFINERA_FIBERLINK_INFO_RESPONSE_SCHEMA, \
-    MTC_INTERFACE_INFO_RESPONSE_SCHEMA, TNMS_FIBERLINK_INFO_RESPONSE_SCHEMA
+from inventory_provider.routes.classifier_schema import (
+    JUNIPER_LINK_RESPONSE_SCHEMA,
+    PEER_INFO_RESPONSE_SCHEMA,
+    INFINERA_LAMBDA_INFO_RESPONSE_SCHEMA,
+    CORIANT_INFO_RESPONSE_SCHEMA,
+    INFINERA_FIBERLINK_INFO_RESPONSE_SCHEMA,
+    MTC_INTERFACE_INFO_RESPONSE_SCHEMA,
+    ROUTER_INFO_ALL_ROUTERS_RESPONSE_SCHEMA,
+    ROUTER_INFO_RESPONSE_SCHEMA,
+    TNMS_FIBERLINK_INFO_RESPONSE_SCHEMA,
+)
 
 DEFAULT_REQUEST_HEADERS = {
     "Content-type": "application/json",
@@ -219,3 +225,42 @@ def test_tnms_fibre_info(client):
     assert rv.is_json
     response_data = json.loads(rv.data.decode('utf-8'))
     jsonschema.validate(response_data, TNMS_FIBERLINK_INFO_RESPONSE_SCHEMA)
+
+
+def test_router_info_all_routers(client):
+    rv = client.get("/classifier/router-info", headers=DEFAULT_REQUEST_HEADERS)
+    assert rv.status_code == 200
+    assert rv.is_json
+    result = rv.json
+
+    jsonschema.validate(result, ROUTER_INFO_ALL_ROUTERS_RESPONSE_SCHEMA)
+    assert len(result) > 0
+
+
+@pytest.mark.parametrize("router", ["mx1.ams.nl", "mx1.ams.nl.geant.net"])
+def test_router_info(client, router):
+    rv = client.get(
+        f"/classifier/router-info/{router}", headers=DEFAULT_REQUEST_HEADERS
+    )
+    assert rv.status_code == 200
+    assert rv.is_json
+    result = rv.json
+    jsonschema.validate(result, ROUTER_INFO_RESPONSE_SCHEMA)
+    assert len(result['related-services']) > 0
+    assert len(result['contacts']) > 0
+
+
+def test_router_info_caches_result(client, mocked_redis):
+    router = "mx1.ams.nl"
+    cache_key = f"classifier-cache:router:{router.upper()}"
+    assert not mocked_redis.exists(cache_key)
+    client.get(f"/classifier/router-info/{router}", headers=DEFAULT_REQUEST_HEADERS)
+    assert mocked_redis.exists(cache_key)
+
+
+def test_404_for_nonexistent_router(client):
+    router = "invalid.ams.nl"
+    rv = client.get(
+        f"/classifier/router-info/{router}", headers=DEFAULT_REQUEST_HEADERS
+    )
+    assert rv.status_code == 404