Finished feature DBOARD-445-poller-gws-indirect-endpoint.

inventory_provider/routes/msr.py

@@ -4,18 +4,13 @@ MSR Support Endpoints
 
 These endpoints are intended for use by MSR.
 
-.. contents:: :local:
-
-/msr/access-services
----------------------------------
-
-.. autofunction:: inventory_provider.routes.msr.access_services
 
+.. contents:: :local:
 
-/msr/gws-indirect
+/poller/access-services
 ---------------------------------
 
-.. autofunction:: inventory_provider.routes.msr.gws_indirect
+.. autofunction::inventory_provider.routes.msr.get_access_services
 
 
 /msr/bgp/logical-systems
@@ -68,36 +63,11 @@ from flask import Blueprint, Response, request
 
 from inventory_provider.routes import common
 from inventory_provider.routes.common import _ignore_cache_or_retrieve
+from inventory_provider.routes.poller import get_services
 
 routes = Blueprint('msr-query-routes', __name__)
 
 
-ACCESS_SERVICES_LIST_SCHEMA = {
-    '$schema': 'http://json-schema.org/draft-07/schema#',
-
-    'definitions': {
-        'service': {
-            'type': 'object',
-            'properties': {
-                'id': {'type': 'integer'},
-                'name': {'type': 'string'},
-                'customer': {'type': 'string'},
-                'speed': {'type': 'integer'},
-                'pop': {'type': 'string'},
-                'hostname': {'type': 'string'},
-                'interface': {'type': 'string'},
-            },
-            'required': [
-                'id', 'name', 'customer', 'speed',
-                'pop', 'hostname', 'interface'],
-            'additionalProperties': False
-        }
-    },
-
-    'type': 'array',
-    'items': {'$ref': '#/definitions/service'}
-}
-
 PEERING_GROUP_LIST_SCHEMA = {
     '$schema': 'http://json-schema.org/draft-07/schema#',
     'type': 'array',
@@ -141,111 +111,6 @@ def after_request(resp):
     return common.after_request(resp)
 
 
-@routes.route('/access-services', methods=['GET', 'POST'])
-@common.require_accepts_json
-def access_services():
-    """
-    Handler for `/msr/access-services`.
-
-    This method is in development, not yet used.
-
-    The response will be formatted according to the following schema:
-
-    .. asjson::
-       inventory_provider.routes.msr.ACCESS_SERVICES_LIST_SCHEMA
-
-    :return:
-    """
-    redis = common.get_current_redis()
-
-    def _services():
-        for k in redis.scan_iter('ims:services:access_services:*'):
-            service = redis.get(k.decode('utf-8')).decode('utf-8')
-            service = json.loads(service)
-
-            yield {
-                'id': service['id'],
-                'name': service['name'],
-                'customer': service['project'],
-                'speed': service['speed_value'],
-                'pop': service['here']['pop']['name'],
-                'hostname': common.ims_equipment_to_hostname(
-                    service['here']['equipment']),
-                'interface': service['here']['port']
-            }
-
-    cache_key = 'classifier-cache:msr:access-services'
-
-    result = _ignore_cache_or_retrieve(request, cache_key, redis)
-
-    if not result:
-        result = list(_services())
-
-        if not result:
-            return Response(
-                response='no access services found',
-                status=404,
-                mimetype='text/html')
-
-        # cache this data for the next call
-        result = json.dumps(result)
-        redis.set(cache_key, result.encode('utf-8'))
-
-    return Response(result, mimetype='application/json')
-
-
-@routes.route('/gws-indirect', methods=['GET', 'POST'])
-@common.require_accepts_json
-def gws_indirect():
-    """
-    Handler for `/msr/gws-indirect`.
-
-    This method is in development, not yet used.
-
-    The response will be formatted according to the following schema:
-
-    .. asjson::
-       inventory_provider.routes.msr.ACCESS_SERVICES_LIST_SCHEMA
-
-    :return:
-    """
-    redis = common.get_current_redis()
-
-    def _services():
-        for k in redis.scan_iter('ims:services:gws_indirect:*'):
-            service = redis.get(k.decode('utf-8')).decode('utf-8')
-            service = json.loads(service)
-            yield {
-                'id': service['id'],
-                'name': service['name'],
-                'customer': service['project'],
-                'speed': service['speed_value'],
-                'pop': service['here']['pop']['name'],
-                'hostname': common.ims_equipment_to_hostname(
-                    service['here']['equipment']),
-                'interface': service['here']['port']
-            }
-
-    cache_key = 'classifier-cache:msr:gws-indirect'
-
-    result = _ignore_cache_or_retrieve(request, cache_key, redis)
-
-    if not result:
-        result = list(_services())
-
-        if not result:
-            return Response(
-                response='no gws indirect services found',
-                status=404,
-                mimetype='text/html')
-
-        # cache this data for the next call
-        result = json.dumps(result)
-        redis.set(cache_key, result.encode('utf-8'))
-
-    return Response(result, mimetype='application/json')
-
-
 def _handle_peering_group_request(name, cache_key, group_key_base):
     """
     Common method for used by
@@ -449,3 +314,16 @@ def get_peering_routing_instances():
     return _handle_peering_group_list_request(
         cache_key='classifier-cache:msr:routing-instances',
         group_key_base='juniper-peerings:routing-instance')
+
+
+@routes.route('/access-services', methods=['GET', 'POST'])
+@common.require_accepts_json
+def get_access_services():
+    """
+    Handler for `/msr/access-services`
+
+    Same as `/poller/services/geant_ip`
+
+    cf. :meth:`inventory_provider.routes.poller.get_services`
+    """
+    return get_services(service_type='geant_ip')

inventory_provider/routes/poller.py

@@ -29,6 +29,19 @@ These endpoints are intended for use by BRIAN.
 
 .. autofunction:: inventory_provider.routes.poller.gws_direct
 
+
+/poller/gws/direct
+---------------------------------
+
+.. autofunction:: inventory_provider.routes.poller.gws_indirect
+
+
+/poller/services</type>
+---------------------------------
+
+.. autofunction:: inventory_provider.routes.poller.get_services
+
+
 """
 import json
 import logging
@@ -203,6 +216,33 @@ GWS_DIRECT_DATA_SCHEMA = {
 }
 
 
+SERVICES_LIST_SCHEMA = {
+    '$schema': 'http://json-schema.org/draft-07/schema#',
+
+    'definitions': {
+        'service': {
+            'type': 'object',
+            'properties': {
+                'id': {'type': 'integer'},
+                'name': {'type': 'string'},
+                'customer': {'type': 'string'},
+                'speed': {'type': 'integer'},
+                'pop': {'type': 'string'},
+                'hostname': {'type': 'string'},
+                'interface': {'type': 'string'},
+            },
+            'required': [
+                'id', 'name', 'customer', 'speed',
+                'pop', 'hostname', 'interface'],
+            'additionalProperties': False
+        }
+    },
+
+    'type': 'array',
+    'items': {'$ref': '#/definitions/service'}
+}
+
+
 @routes.after_request
 def after_request(resp):
     return common.after_request(resp)
@@ -623,7 +663,7 @@ def gws_direct():
        inventory_provider.routes.poller.GWS_DIRECT_DATA_SCHEMA
 
     WARNING: interface tags in the `gws-direct` section of the config data
-    should be unique for each nren/isp/hostname combination.  i.e. if there
+    should be unique for each nren/isp combination.  i.e. if there
     are multiple community strings in use for a particular host, then please
     keep the interface tags unique.
 
@@ -662,3 +702,83 @@ def gws_direct():
         r.set(cache_key, result.encode('utf-8'))
 
     return Response(result, mimetype="application/json")
+
+
+@routes.route('/services', methods=['GET', 'POST'])
+@routes.route('/services/<service_type>', methods=['GET', 'POST'])
+@common.require_accepts_json
+def get_services(service_type=None):
+    """
+    Handler for `/poller/services/type`.
+
+    The response will be formatted according to the following schema:
+
+    .. asjson::
+       inventory_provider.routes.poller.SERVICES_LIST_SCHEMA
+
+    :return:
+    """
+    def _services():
+        key_pattern = f'ims:services:{service_type}:*' \
+            if service_type else 'ims:services:*'
+
+        for doc in common.load_json_docs(
+                config_params=current_app.config['INVENTORY_PROVIDER_CONFIG'],
+                key_pattern=key_pattern,
+                num_threads=20):
+            yield doc['value']
+
+    def _operational(s):
+        return s['status'].lower() == 'operational'
+
+    def _format_services(s):
+        return {
+            'id': s['id'],
+            'name': s['name'],
+            'customer': s['project'],
+            'speed': s['speed_value'],
+            'pop': s['here']['pop']['name'],
+            'hostname': common.ims_equipment_to_hostname(
+                s['here']['equipment']),
+            'interface': s['here']['port']
+        }
+
+    cache_key = f'classifier-cache:poller:services:{service_type}' \
+        if service_type else 'classifier-cache:poller:services:all'
+
+    redis = common.get_current_redis()
+    result = _ignore_cache_or_retrieve(request, cache_key, redis)
+
+    if not result:
+        result = _services()
+        result = filter(_operational, result)
+        result = map(_format_services, result)
+        result = list(result)
+
+        if not result:
+            message = f'no {service_type} services found' \
+                    if service_type else 'no services found'
+            return Response(
+                response=message,
+                status=404,
+                mimetype='text/html')
+
+        # cache this data for the next call
+        result = json.dumps(result)
+        redis.set(cache_key, result.encode('utf-8'))
+
+    return Response(result, mimetype='application/json')
+
+
+@routes.route("/gws/indirect", methods=['GET', 'POST'])
+@common.require_accepts_json
+def gws_indirect():
+    """
+    Handler for `/poller/gws/indirect`
+
+    Same as `/poller/services/gws_indirect`
+
+    cf. :meth:`inventory_provider.routes.poller.get_services`
+    :return:
+    """
+    return get_services(service_type='geant_ip')

inventory_provider/tasks/worker.py

@@ -829,7 +829,8 @@ def update_circuit_hierarchy_and_port_id_services(self, use_current=False):
                     },
                     'speed_value': v['calculated-speed'],
                     'speed_unit': 'n/a',
-                    'status': v['status']
+                    'status': v['status'],
+                    'type': v['service_type']
                 }))
 
     rp.execute()

tox.ini

@@ -14,7 +14,7 @@ commands =
     coverage run --source inventory_provider --omit='inventory_provider/routes/ims*,inventory_provider/db/ims*,inventory_provider/tasks/ims*' -m py.test {posargs}
     coverage xml
     coverage html
-    coverage report --fail-under 75
+    coverage report --fail-under 70 
     # coverage report --fail-under 80
     flake8
     sphinx-build -M html docs/source docs/build