Finished feature fix-cache-cleanup.

inventory_provider/tasks/worker.py

@@ -260,7 +260,17 @@ def clear_cached_classifier_responses(hostname):
     r = get_redis(InventoryTask.config)
     for k in r.keys('classifier:cache:%s:*' % hostname):
         r.delete(k)
-    # TODO: remove peer-cache (after adding hostname to query)
+
+    # TODO: very inefficient ... but logically simplest at this point
+    for k in r.keys('classifier:peer-cache:*'):
+        value = r.get(k.decode('utf-8'))
+        if not value:
+            # deleted in another thread
+            continue
+        value = json.loads(value.decode('utf-8'))
+        interfaces = value.get('interfaces', [])
+        if hostname in [i['interface']['router'] for i in interfaces]:
+            r.delete(k)
 
 
 def _refresh_peers(hostname, key_base, peers):

test/data/extract-cache-entries.py

+import json
+
+with open('router-info.json') as f:
+    db = json.loads(f.read())
+
+cache = dict([(k, db[k]) for k in db if k.startswith('classifier:')])
+
+with open('cached-entries.json', 'w') as f:
+    f.write(json.dumps(cache))

test/data/update-test-db.py

+import json
+import os
+import redis
+
+
+r = redis.StrictRedis(host="test-dashboard02.geant.org")
+
+d = {}
+for k in r.keys():
+    print(k)
+    d[k.decode('utf-8')] = r.get(k).decode('utf-8')
+
+data_filename = os.path.join(os.path.dirname(__file__), "router-info.json")
+
+with open(data_filename, "w") as f:
+    f.write(json.dumps(d))

test/per_router/conftest.py

 import glob
+import json
 import os
 import re
 
 from lxml import etree
+import pytest
 
 import inventory_provider
 
@@ -13,6 +15,58 @@ TEST_DATA_DIRNAME = os.path.realpath(os.path.join(
     "data"))
 
 
+class MockedRedis(object):
+
+    db = None
+
+    def __init__(self, *args, **kwargs):
+        if MockedRedis.db is None:
+            test_data_filename = os.path.join(
+                TEST_DATA_DIRNAME,
+                "router-info.json")
+            with open(test_data_filename) as f:
+                MockedRedis.db = json.loads(f.read())
+
+    def set(self, name, value):
+        MockedRedis.db[name] = value
+
+    def get(self, name):
+        value = MockedRedis.db.get(name, None)
+        if value is None:
+            return None
+        return value.encode('utf-8')
+
+    def keys(self, glob=None):
+        if not glob:
+            return list([k.encode("utf-8") for k in MockedRedis.db.keys()])
+        m = re.match(r'^([^*]+)\*$', glob)
+        assert m  # all expected global are like this
+        return list([
+            k.encode("utf-8") for k in MockedRedis.db.keys()
+            if k.startswith(m.group(1))])
+
+    def delete(self, key):
+        if isinstance(key, bytes):
+            key = key.decode('utf-8')
+        del MockedRedis.db[key]
+
+
+@pytest.fixture
+def classifier_cache_test_entries():
+    filename = os.path.join(
+        TEST_DATA_DIRNAME, 'classifier-cache-entries.json')
+    with open(filename) as f:
+        return json.loads(f.read())
+
+
+@pytest.fixture
+def mocked_redis(mocker):
+    mocker.patch(
+        'inventory_provider.tasks.common.redis.StrictRedis',
+        MockedRedis)
+    return MockedRedis()
+
+
 def pytest_generate_tests(metafunc):
 
     def _junosspace_hosts():

test/per_router/test_celery_worker.py

@@ -3,87 +3,26 @@ just checks that the worker methods call the right functions
 and some data ends up in the right place ... otherwise not very detailed
 """
 import json
-import os
-import re
 
 from lxml import etree
 import pytest
 from inventory_provider.tasks import worker
+from inventory_provider.tasks.common import get_redis
 
-import inventory_provider
-
-TEST_DATA_DIRNAME = os.path.realpath(os.path.join(
-    inventory_provider.__path__[0],
-    '..',
-    'test',
-    'data'))
-
-# class MockedRedis(object):
-#
-#     db = {}
-#
-#     def __init__(self, *args, **kwargs):
-#         pass
-#
-#     def set(self, name, value):
-#         assert isinstance(value, str)
-#         if name.startswith('netconf:'):
-#             etree.fromstring(value)
-#         elif name.startswith('snmp-interfaces:'):
-#             obj = json.loads(value)
-#             assert isinstance(obj, list)
-#         MockedRedis.db[name] = value
-#
-#     def get(self, key):
-#         return MockedRedis.db[key].encode('utf-8')
-
-
-class MockedRedis(object):
-
-    db = None
-
-    def __init__(self, *args, **kwargs):
-        if MockedRedis.db is None:
-            test_data_filename = os.path.join(
-                TEST_DATA_DIRNAME,
-                "router-info.json")
-            with open(test_data_filename) as f:
-                MockedRedis.db = json.loads(f.read())
-
-    def set(self, name, value):
-        MockedRedis.db[name] = value
-
-    def get(self, name):
-        value = MockedRedis.db.get(name, None)
-        if value is None:
-            return None
-        return value.encode('utf-8')
-
-    def keys(self, glob=None):
-        if not glob:
-            return list([k.encode("utf-8") for k in MockedRedis.db.keys()])
-        m = re.match(r'^([^*]+)\*$', glob)
-        assert m  # all expected global are like this
-        return list([
-            k.encode("utf-8") for k in MockedRedis.db.keys()
-            if k.startswith(m.group(1))])
-
-    def delete(self, key):
-        if isinstance(key, bytes):
-            key = key.decode('utf-8')
-        del MockedRedis.db[key]
+
+def backend_db():
+    return get_redis({
+        'redis': {'hostname': None, 'port': None}
+    }).db
 
 
 @pytest.fixture
 def mocked_worker_module(
-        mocker, data_config, cached_test_data, mocked_netifaces):
+        mocker, mocked_redis, data_config,
+        cached_test_data, mocked_netifaces):
 
     worker.InventoryTask.config = data_config
 
-    mocker.patch(
-        'inventory_provider.tasks.common.redis.StrictRedis',
-        MockedRedis)
-
     def _mocked_snmp_interfaces(hostname, community, _):
         return json.loads(cached_test_data['snmp-interfaces:' + hostname])
     mocker.patch(
@@ -96,31 +35,29 @@ def mocked_worker_module(
         'inventory_provider.juniper.load_config',
         _mocked_load_juniper_netconf_config)
 
-    MockedRedis()  # create an instances so db is initialized
-
 
 def test_netconf_refresh_config(mocked_worker_module, router):
-    del MockedRedis.db['netconf:' + router]
+    del backend_db()['netconf:' + router]
     worker.netconf_refresh_config(router)
-    assert MockedRedis.db['netconf:' + router]
+    assert backend_db()['netconf:' + router]
 
 
 def test_snmp_refresh_interfaces(mocked_worker_module, router):
-    del MockedRedis.db['snmp-interfaces:' + router]
+    del backend_db()['snmp-interfaces:' + router]
     worker.snmp_refresh_interfaces(router, 'fake-community')
-    assert MockedRedis.db['snmp-interfaces:' + router]
+    assert backend_db()['snmp-interfaces:' + router]
 
 
 def test_reload_router_config(mocked_worker_module, router, mocker):
     saved_data = {}
     for key in ('netconf:' + router, 'snmp-interfaces:' + router):
-        saved_data[key] = MockedRedis.db.pop(key)
-    assert 'netconf:' + router not in MockedRedis.db
-    assert 'snmp-interfaces:' + router not in MockedRedis.db
+        saved_data[key] = backend_db().pop(key)
+    assert 'netconf:' + router not in backend_db()
+    assert 'snmp-interfaces:' + router not in backend_db()
 
     def _mocked_netconf_refresh_config_apply(args):
         key = 'netconf:' + args[0]
-        MockedRedis.db[key] = saved_data[key]
+        backend_db()[key] = saved_data[key]
     mocker.patch(
         'inventory_provider.tasks.worker.netconf_refresh_config.apply',
         _mocked_netconf_refresh_config_apply)
@@ -128,7 +65,7 @@ def test_reload_router_config(mocked_worker_module, router, mocker):
     def _mocked_snmp_refresh_interfaces_apply(args):
         assert len(args) == 2
         key = 'snmp-interfaces:' + args[0]
-        MockedRedis.db[key] = saved_data[key]
+        backend_db()[key] = saved_data[key]
     mocker.patch(
         'inventory_provider.tasks.worker.snmp_refresh_interfaces.apply',
         _mocked_snmp_refresh_interfaces_apply)
@@ -140,5 +77,5 @@ def test_reload_router_config(mocked_worker_module, router, mocker):
         _mocked_update_status)
 
     worker.reload_router_config(router)
-    assert 'netconf:' + router in MockedRedis.db
-    assert 'snmp-interfaces:' + router in MockedRedis.db
+    assert 'netconf:' + router in backend_db()
+    assert 'snmp-interfaces:' + router in backend_db()

test/per_router/test_classifier_data.py

+from inventory_provider.tasks import worker
+from inventory_provider.tasks.common import get_redis
+
+
+def backend_db():
+    return get_redis({
+        'redis': {'hostname': None, 'port': None}
+    }).db
+
+
+def test_clear_classifier_cache(
+        router,
+        mocked_redis,
+        data_config,
+        classifier_cache_test_entries):
+    worker.InventoryTask.config = data_config
+    backend_db().update(classifier_cache_test_entries)
+    worker.clear_cached_classifier_responses(router)
+    for k in backend_db():
+        assert not k.startswith('classifier:cache:%s:' % router)

test/per_router/test_data_routes.py

 import json
-import os
-import re
-
 import pytest
 import jsonschema
 
-import inventory_provider
-
-TEST_DATA_DIRNAME = os.path.realpath(os.path.join(
-    inventory_provider.__path__[0],
-    "..",
-    "test",
-    "data"))
-
 DEFAULT_REQUEST_HEADERS = {
     "Content-type": "application/json",
     "Accept": ["application/json"]
 }
 
 
-class MockedRedis(object):
-
-    db = None
-
-    def __init__(self, *args, **kwargs):
-        if MockedRedis.db is None:
-            test_data_filename = os.path.join(
-                TEST_DATA_DIRNAME,
-                "router-info.json")
-            with open(test_data_filename) as f:
-                MockedRedis.db = json.loads(f.read())
-
-    def set(self, key, value):
-        MockedRedis.db[key] = value
-
-    def get(self, key):
-        return MockedRedis.db[key].encode('utf-8')
-
-    # def hget(self, key, field):
-    #     value = MockedRedis.db[key]
-    #     return value[field].encode('utf-8')
-    #     # return json.dumps(value[field]).encode('utf-8')
-    #
-    # def hgetall(self, key):
-    #     result = {}
-    #     for k, v in MockedRedis.db[key].items():
-    #         result[k.encode('utf-8')] \
-    #             = json.dumps(v).encode('utf-8')
-    #     return result
-    #
-    # def keys(self, *args, **kwargs):
-    #     return list([k.encode("utf-8") for k in MockedRedis.db.keys()])
-
-    def keys(self, glob=None):
-        if not glob:
-            return list([k.encode("utf-8") for k in MockedRedis.db.keys()])
-        m = re.match(r'^([^*]+)\*$', glob)
-        assert m  # all expected global are like this
-        return list([
-            k.encode("utf-8") for k in MockedRedis.db.keys()
-            if k.startswith(m.group(1))])
-
-
-@pytest.fixture
-def client_with_mocked_data(mocker, client):
-    mocker.patch(
-        'inventory_provider.routes.common.redis.StrictRedis',
-        MockedRedis)
-    return client
-
-
 def test_router_interfaces(router, client_with_mocked_data):
 
     interfaces_list_schema = {

test/per_router/test_poller_routes.py

 import json
-import os
-
-import pytest
 import jsonschema
 
-import inventory_provider
-
-TEST_DATA_DIRNAME = os.path.realpath(os.path.join(
-    inventory_provider.__path__[0],
-    "..",
-    "test",
-    "data"))
-
 DEFAULT_REQUEST_HEADERS = {
     "Content-type": "application/json",
     "Accept": ["application/json"]
 }
 
 
-class MockedRedis(object):
-
-    db = None
-
-    def __init__(self, *args, **kwargs):
-        if MockedRedis.db is None:
-            test_data_filename = os.path.join(
-                TEST_DATA_DIRNAME,
-                "router-info.json")
-            with open(test_data_filename) as f:
-                MockedRedis.db = json.loads(f.read())
-
-    def set(self, key, value):
-        MockedRedis.db[key] = value
-
-    def get(self, key):
-        value = MockedRedis.db.get(key)
-        if value is None:
-            return None
-        return value.encode('utf-8')
-
-    def keys(self, *args, **kwargs):
-        return list([k.encode("utf-8") for k in MockedRedis.db.keys()])
-
-
-@pytest.fixture
-def client_with_mocked_data(mocker, client):
-    mocker.patch(
-        'inventory_provider.routes.common.redis.StrictRedis',
-        MockedRedis)
-    return client
-
-
 def test_router_interfaces(router, client_with_mocked_data):
 
     interfaces_list_schema = {