Finished feature POL1-228-remove-poller-service-category-handling.

inventory_provider/routes/poller.py

@@ -160,47 +160,3 @@ def poller_interface_oids(hostname=None):
         r.set(cache_key, result.encode('utf-8'))
 
     return Response(result, mimetype="application/json")
-
-
-@routes.route('/services/<category>', methods=['GET', 'POST'])
-@common.require_accepts_json
-def service_category_interfaces(category):
-
-    def _interfaces(cat):
-        config_params = current_app.config['INVENTORY_PROVIDER_CONFIG']
-        for doc in common.load_json_docs(
-                config_params=config_params,
-                key_pattern=f'interface-services:{cat}:*'):
-            cached_ifc = doc['value']
-            basic_ifc_info = dict()
-            for k in ['description', 'interface', 'router']:
-                basic_ifc_info[k] = cached_ifc[k]
-            if not cached_ifc['users']:
-                yield basic_ifc_info
-            else:
-                for user in cached_ifc['users']:
-                    ifc = {'user': user}
-                    ifc.update(basic_ifc_info)
-                    yield ifc
-
-    category = category.lower()
-
-    r = common.get_current_redis()
-    cache_key = f'classifier-cache:poller:service:{category}'
-    result = r.get(cache_key)
-
-    if result:
-        result = result.decode('utf-8')
-    else:
-        result = list(_interfaces(category))
-        if not result:
-            return Response(
-                response=f'no info available for service category {category}',
-                status=404,
-                mimetype="text/html")
-
-        result = json.dumps(result)
-        # cache this data for the next call
-        r.set(cache_key, result.encode('utf-8'))
-
-    return Response(result, mimetype="application/json")

inventory_provider/tasks/worker.py

-import enum
 import json
 import logging
 import os
@@ -673,7 +672,6 @@ def refresh_finalizer(self, pending_task_ids_json):
 
         _wait_for_tasks(task_ids, update_callback=self.log_info)
         _build_subnet_db(update_callback=self.log_info)
-        _build_service_category_interface_list(update_callback=self.log_info)
 
     except (jsonschema.ValidationError,
             json.JSONDecodeError,
@@ -687,70 +685,6 @@ def refresh_finalizer(self, pending_task_ids_json):
     self.log_info('latched current/next dbs')
 
 
-class PollerServiceCategory(str, enum.Enum):
-    MDVPN = 'mdvpn'
-    LHCONE_CUST = 'lhcone_cust'
-    LHCONE_PEER = 'lhcone_peer'
-    L2_CIRCUITS = 'l2_circuits'
-    AUTOMATED_L2_CIRCUITS = 'automated_l2_circuits'
-    IAS = 'ias'
-    RE_CUST = 're_cust'
-    RE_PEER = 're_peer'
-
-
-def _classify_interface(ifc):
-
-    if ifc['description'].startswith('SRV_MDVPN CUSTOMER'):
-        yield PollerServiceCategory.MDVPN
-
-    if 'LHCONE' in ifc['description'] \
-            and 'SRV_L3VPN CUSTOMER' in ifc['description']:
-        yield PollerServiceCategory.LHCONE_CUST
-
-    if 'LHCONE' in ifc['description'] \
-            and 'SRV_L3VPN RE' in ifc['description']:
-        yield PollerServiceCategory.LHCONE_PEER
-
-    if 'SRV_L2CIRCUIT' in ifc['description']:
-        yield PollerServiceCategory.L2_CIRCUITS
-
-    if 'SRV_GCS' in ifc['description']:
-        yield PollerServiceCategory.AUTOMATED_L2_CIRCUITS
-
-    if 'PHY CUSTOMER' in ifc['description'] \
-            or 'LAG CUSTOMER' in ifc['description'] \
-            or 'SRV_GLOBAL CUSTOMER' in ifc['description']:
-        yield PollerServiceCategory.RE_CUST
-
-    if 'SRV_GLOBAL RE_INTERCONNECT' in ifc['description']:
-        yield PollerServiceCategory.RE_PEER
-
-    if 'SRV_IAS CUSTOMER' in ifc['description']:
-        yield PollerServiceCategory.IAS
-
-
-@log_task_entry_and_exit
-def _build_service_category_interface_list(update_callback=lambda s: None):
-
-    update_callback('loading all known interfaces')
-    interfaces = data.build_service_interface_user_list(InventoryTask.config)
-    interfaces = list(interfaces)
-    update_callback(f'loaded {len(interfaces)} interfaces, '
-                    'saving by service category')
-
-    r = get_next_redis(InventoryTask.config)
-    rp = r.pipeline()
-
-    for ifc in interfaces:
-        for service_category in _classify_interface(ifc):
-            rp.set(
-                f'interface-services:{service_category.value}'
-                f':{ifc["router"]}:{ifc["interface"]}',
-                json.dumps(ifc))
-
-    rp.execute()
-
-
 def _build_subnet_db(update_callback=lambda s: None):
 
     r = get_next_redis(InventoryTask.config)

test/test_general_poller_routes.py

 import json
-
 import jsonschema
-import pytest
-
-from inventory_provider.tasks import worker
 
 DEFAULT_REQUEST_HEADERS = {
     "Content-type": "application/json",
     "Accept": ["application/json"]
 }
 
-
-SCHEMA_INTERFACE_LIST_SCHEMA = {
-    '$schema': 'http://json-schema.org/draft-07/schema#',
-
-    'definitions': {
-        'ifc-info': {
-            'type': 'object',
-            'properties': {
-                'description': {'type': 'string'},
-                'router': {'type': 'string'},
-                'interface': {'type': 'string'},
-                'user': {'type': 'string'}
-            },
-            'required': ['router', 'interface', 'description'],
-            'additionalProperties': False
-        },
-    },
-
-    'type': 'array',
-    'items': {'$ref': '#/definitions/ifc-info'}
-}
-
-
 INTERFACE_LIST_SCHEMA = {
     '$schema': 'http://json-schema.org/draft-07/schema#',
 
@@ -84,28 +57,6 @@ INTERFACE_LIST_SCHEMA = {
 }
 
 
-@pytest.mark.parametrize('category', ['mdvpn', 'lhcone', 'MDVpn', 'LHCONE'])
-def test_service_category(client, mocked_worker_module, category):
-    worker._build_service_category_interface_list()
-    rv = client.get(
-        f'/poller/services/{category}',
-        headers=DEFAULT_REQUEST_HEADERS)
-    assert rv.status_code == 200
-    assert rv.is_json
-    response_data = json.loads(rv.data.decode('utf-8'))
-    jsonschema.validate(response_data, SCHEMA_INTERFACE_LIST_SCHEMA)
-    assert response_data, 'expected a non-empty list'
-
-
-@pytest.mark.parametrize('category', ['mdvpn ', ' mdvpn', 'mdvpn1', 'aaa'])
-def test_service_category_not_found(client, mocked_worker_module, category):
-    worker._build_service_category_interface_list()
-    rv = client.get(
-        f'/poller/services/{category}',
-        headers=DEFAULT_REQUEST_HEADERS)
-    assert rv.status_code == 404
-
-
 def test_get_all_interfaces(client):
     rv = client.get(
         '/poller/interfaces',
@@ -116,4 +67,4 @@ def test_get_all_interfaces(client):
     jsonschema.validate(response_data, INTERFACE_LIST_SCHEMA)
     response_routers = {ifc['router'] for ifc in response_data}
     assert len(response_routers) > 1, \
-        'there should data from be lots of routers'
+        'there should data from be lots of routers'
\ No newline at end of file

test/test_worker_utils.py

@@ -5,7 +5,6 @@ import json
 import re
 
 import jsonschema
-import pytest
 
 from inventory_provider.tasks import worker
 from inventory_provider.tasks import common
@@ -21,69 +20,6 @@ def backend_db():
     }).db
 
 
-def test_build_interface_services(mocked_worker_module):
-    """
-    checks that valid interface service objects are created
-    :param mocked_worker_module: fixture
-    :return:
-    """
-    ifc_schema = {
-        '$schema': 'http://json-schema.org/draft-07/schema#',
-        'type': 'object',
-        'properties': {
-            'description': {'type': 'string'},
-            'router': {'type': 'string'},
-            'interface': {'type': 'string'},
-            'users': {
-                'type': 'array',
-                'items': {'type': 'string'}
-            }
-        },
-        'required': ['router', 'interface', 'description'],
-        'additionalProperties': False
-    }
-
-    db = backend_db()  # also forces initialization
-    worker._build_service_category_interface_list()
-
-    seen_types = set()
-    for k, v in db.items():
-        if not k.startswith('interface-services:'):
-            continue
-
-        print(v)
-        (_, type, router, ifc_name) = k.split(':')
-
-        ifc_info = json.loads(v)
-        jsonschema.validate(json.loads(v), ifc_schema)
-
-        assert ifc_info['router'] == router
-        assert ifc_info['interface'] == ifc_name
-
-        seen_types.add(type)
-
-    assert type in ('mdvpn',
-                    'lhcone_peer',
-                    're_peer',
-                    're_cust',
-                    'ias',
-                    'lhcone',
-                    'lhcone_cust',
-                    'l2_circuits',
-                    'automated_l2_circuits')
-
-    expected_seen_types = set(['mdvpn',
-                               'lhcone_peer',
-                               're_peer',
-                               're_cust',
-                               'ias',
-                               'lhcone',
-                               'lhcone_cust',
-                               'l2_circuits',
-                               'automated_l2_circuits'])
-    assert seen_types == expected_seen_types
-
-
 def test_build_subnet_db(mocked_worker_module):
     """
     checks that valid reverse subnet objects are created
@@ -125,73 +61,3 @@ def test_build_subnet_db(mocked_worker_module):
         assert value['name'] == address
 
     assert found_record
-
-
-# TODO: more tests
-# TODO: then share with neteng to make sure the test data is sensible
-# this data is just enough to be an example
-#   ... seems the method is not yet ready for testing
-@pytest.mark.parametrize('description,expected_categories', [
-    [
-        'SRV_MDVPN CUSTOMER SANET SRF9939441 | VPN-Proxy to NREN CE',
-        {
-            worker.PollerServiceCategory.MDVPN
-        }
-    ],
-    [
-        'SRV_L3VPN CUSTOMER ESNET LHCONE SRF9928635 | ASN293 | GEN-EEX',
-        {
-            worker.PollerServiceCategory.LHCONE_CUST
-        }
-    ],
-    [
-        'SRV_L3VPN RE KREONET LHCONE SRF17072 | ASN17579',
-        {
-            worker.PollerServiceCategory.LHCONE_PEER
-        }
-    ],
-    [
-        'SRV_IAS CUSTOMER ACONET SRF9947199 IASPS | ASN1853',
-        {
-            worker.PollerServiceCategory.IAS
-        }
-    ],
-    [
-        'SRV_GLOBAL CUSTOMER HBKU SRF18085 | ASN34945|',
-        {
-            worker.PollerServiceCategory.RE_CUST
-        }
-    ],
-    [
-        'SRV_GLOBAL RE_INTERCONNECT HBKU SRF18085 | ASN34945|',
-        {
-            worker.PollerServiceCategory.RE_PEER
-        }
-    ],
-    [
-        'SRV_MDVPN CUSTOMER LHCONE SRV_L3VPN CUSTOMER SRF18085 | ASN34945|',
-        {
-            worker.PollerServiceCategory.MDVPN,
-            worker.PollerServiceCategory.LHCONE_CUST
-        }
-    ]
-])
-def test_interface_classification(description, expected_categories):
-    categories = worker._classify_interface({'description': description})
-    assert set(list(categories)) == expected_categories
-
-
-# def _all_interfaces():
-#     import redis
-#     r = redis.StrictRedis(host='test-dashboard-storage01.geant.org')
-#     for k in r.scan_iter('netconf-interfaces:*'):
-#         k = k.decode('utf-8')
-#         ifc = json.loads(r.get(k).decode('utf-8'))
-#         fields = k.split(':')
-#         ifc['router'] = fields[1]
-#         yield ifc
-#
-#
-# def test_123():
-#     with open('interfaces.json', 'w') as f:
-#         f.write(json.dumps(list(_all_interfaces())))