update to use new config passing method

inventory_provider/config.py

@@ -104,6 +104,6 @@ def load(f):
     """
     config = json.loads(f.read())
     jsonschema.validate(config, CONFIG_SCHEMA)
-    with open(config["oid_list.conf"]) as f:
-        config["oids"] = _load_oids(f)
+    # with open(config["oid_list.conf"]) as f:
+    #     config["oids"] = _load_oids(f)
     return config

inventory_provider/tasks/worker.py

@@ -63,15 +63,17 @@ def _save_value_etree(key, xml_doc):
 
 
 class LoadConfig(bootsteps.Step):
-    assert os.path.isfile(os.environ['INVENTORY_PROVIDER_CONFIG_FILENAME']), (
-            'config file %r not found' %
-            os.environ['INVENTORY_PROVIDER_CONFIG_FILENAME'])
 
-    with open(os.environ['INVENTORY_PROVIDER_CONFIG_FILENAME']) as f:
-        logging.info(
-                "Initializing worker with config from: %r" %
+    def __init__(self):
+        assert os.path.isfile(os.environ['INVENTORY_PROVIDER_CONFIG_FILENAME']), (
+                'config file %r not found' %
                 os.environ['INVENTORY_PROVIDER_CONFIG_FILENAME'])
-        InventoryTask.config = config.load(f)
+
+        with open(os.environ['INVENTORY_PROVIDER_CONFIG_FILENAME']) as f:
+            logging.info(
+                    "Initializing worker with config from: %r" %
+                    os.environ['INVENTORY_PROVIDER_CONFIG_FILENAME'])
+            InventoryTask.config = config.load(f)
 
 
 app.steps['worker'].add(LoadConfig)

test/conftest.py

@@ -10,8 +10,8 @@ from lxml import etree
 import pytest
 
 import inventory_provider
-from inventory_provider import config
 from inventory_provider.tasks import worker
+from inventory_provider import config
 
 TEST_DATA_DIRNAME = os.path.realpath(os.path.join(
     inventory_provider.__path__[0],
@@ -20,49 +20,53 @@ TEST_DATA_DIRNAME = os.path.realpath(os.path.join(
     "data"))
 
 
-def data_config_filename(tmp_dir_name):
-    config = {
-        "ops-db": {
-            "hostname": "xxxxxxx.yyyyy.zzz",
-            "dbname": "xxxxxx",
-            "username": "xxxxxx",
-            "password": "xxxxxxxx"
-        },
-        "oid_list.conf": os.path.join(
-            tmp_dir_name,
-            "oid_list.conf"),
-        "ssh": {
-            "username": "uSeR-NaMe",
-            "private-key": "private-key-filename",
-            "known-hosts": "known-hosts=filename"
-        },
-        "redis": {
-            "hostname": "xxxxxx",
-            "port": 6379
-        },
-        "junosspace": {
-            "api": "bogus-url",
-            "username": "bogus-username",
-            "password": "bogus-password"
+@pytest.fixture
+def data_config_filename():
+
+    with tempfile.NamedTemporaryFile() as f:
+        config = {
+            "ops-db": {
+                "hostname": "xxxxxxx.yyyyy.zzz",
+                "dbname": "xxxxxx",
+                "username": "xxxxxx",
+                "password": "xxxxxxxx"
+            },
+            # TODO: remove oid_list.conf
+            "oid_list.conf": os.path.join(
+                '',
+                "oid_list.conf"),
+            "ssh": {
+                "username": "uSeR-NaMe",
+                "private-key": "private-key-filename",
+                "known-hosts": "known-hosts=filename"
+            },
+            "redis": {
+                "hostname": "xxxxxx",
+                "port": 6379
+            },
+            "junosspace": {
+                "api": "bogus-url",
+                "username": "bogus-username",
+                "password": "bogus-password"
+            }
         }
-    }
 
-    shutil.copyfile(
-        os.path.join(TEST_DATA_DIRNAME, 'oid_list.conf'),
-        config['oid_list.conf']
-    )
+        # shutil.copyfile(
+        #     os.path.join(TEST_DATA_DIRNAME, 'oid_list.conf'),
+        #     config['oid_list.conf']
+        # )
+        #
+        # filename = os.path.join(tmp_dir_name, "config.json")
+        # with open(filename, "w") as f:
+        f.write(json.dumps(config).encode('utf-8'))
+        f.flush()
+        yield f.name
 
-    filename = os.path.join(tmp_dir_name, "config.json")
-    with open(filename, "w") as f:
-        f.write(json.dumps(config))
 
-    return filename
-
-
-def _tmp_data_config():
-    with tempfile.TemporaryDirectory() as tmpdir:
-        with open(data_config_filename(tmpdir)) as f:
-            return config.load(f)
+@pytest.fixture
+def data_config(data_config_filename):
+    with open(data_config_filename) as f:
+        return config.load(f)
 
 
 TEST_DATA_DIRNAME = os.path.realpath(os.path.join(
@@ -117,9 +121,6 @@ class MockedRedis(object):
         pass
 
 
-@pytest.fixture
-def data_config():
-    return _tmp_data_config()
 
 
 @pytest.fixture
@@ -130,29 +131,12 @@ def cached_test_data():
 
 
 @pytest.fixture
-def app_config():
-    with tempfile.TemporaryDirectory() as tmpdir:
-
-        app_config_filename = os.path.join(tmpdir, "app.config")
-        with open(app_config_filename, "w") as f:
-            f.write("%s = '%s'\n" % (
-                "INVENTORY_PROVIDER_CONFIG_FILENAME",
-                data_config_filename(tmpdir)))
-            f.write('ENABLE_TESTING_ROUTES = True\n')
+def flask_config_filename():
 
-        yield app_config_filename
-
-
-@pytest.fixture
-def client(app_config, mocker):
-
-    mocker.patch(
-        'inventory_provider.tasks.common.redis.StrictRedis',
-        MockedRedis)
-
-    os.environ["SETTINGS_FILENAME"] = app_config
-    with inventory_provider.create_app().test_client() as c:
-        yield c
+    with tempfile.NamedTemporaryFile() as f:
+        f.write('ENABLE_TESTING_ROUTES = True\n'.encode('utf-8'))
+        f.flush()
+        yield f.name
 
 
 @pytest.fixture
@@ -163,8 +147,16 @@ def mocked_redis(mocker):
 
 
 @pytest.fixture
-def client_with_mocked_data(client, mocked_redis):
-    return client
+def client(flask_config_filename, data_config_filename, mocked_redis):
+    os.environ['FLASK_SETTINGS_FILENAME'] = flask_config_filename
+    os.environ['INVENTORY_PROVIDER_CONFIG_FILENAME'] = data_config_filename
+    with inventory_provider.create_app().test_client() as c:
+        yield c
+
+#
+# @pytest.fixture
+# def client(client, mocked_redis):
+#     return client
 
 
 NETIFACES_TEST_DATA_STRING = """{
@@ -194,15 +186,13 @@ def mocked_netifaces(mocker):
 
 @pytest.fixture
 def mocked_worker_module(
-        mocker, mocked_redis, data_config,
+        mocker, mocked_redis, data_config_filename,
         cached_test_data, mocked_netifaces):
 
-    os.environ['BROKER_HOSTNAME'] = 'a.b.c'
-    os.environ['BROKER_PORT'] = '123'
-    os.environ['CELERY_DB_INDEX'] = '456'
-    os.environ['BROKER_SCHEME'] = 'redis'
+    os.environ['INVENTORY_PROVIDER_CONFIG_FILENAME'] = data_config_filename
 
-    worker.InventoryTask.config = data_config
+    with open(data_config_filename) as f:
+        worker.InventoryTask.config = config.load(f)
 
     def _mocked_snmp_interfaces(hostname, community):
         return json.loads(cached_test_data['snmp-interfaces:' + hostname])

test/per_router/test_data_routes.py

@@ -8,7 +8,7 @@ DEFAULT_REQUEST_HEADERS = {
 }
 
 
-def test_router_interfaces(router, client_with_mocked_data):
+def test_router_interfaces(router, client):
 
     interfaces_list_schema = {
         "$schema": "http://json-schema.org/draft-07/schema#",
@@ -36,7 +36,7 @@ def test_router_interfaces(router, client_with_mocked_data):
         }
     }
 
-    rv = client_with_mocked_data.post(
+    rv = client.post(
         "/data/interfaces/" + router,
         headers=DEFAULT_REQUEST_HEADERS)
 
@@ -46,7 +46,7 @@ def test_router_interfaces(router, client_with_mocked_data):
     assert response  # at least shouldn't be empty
 
 
-def test_snmp_ids(router, client_with_mocked_data):
+def test_snmp_ids(router, client):
 
     snmp_id_list_schema = {
         "$schema": "http://json-schema.org/draft-07/schema#",
@@ -62,7 +62,7 @@ def test_snmp_ids(router, client_with_mocked_data):
         }
     }
 
-    rv = client_with_mocked_data.post(
+    rv = client.post(
         "/testing/snmp/" + router,
         headers=DEFAULT_REQUEST_HEADERS)
 
@@ -71,7 +71,7 @@ def test_snmp_ids(router, client_with_mocked_data):
     assert response  # at least shouldn't be empty
 
 
-def test_router_bgp_routes(router, client_with_mocked_data):
+def test_router_bgp_routes(router, client):
 
     ROUTERS_WITH_BGP_CONFIG = [
         "mx1.bud.hu.geant.net",
@@ -114,7 +114,7 @@ def test_router_bgp_routes(router, client_with_mocked_data):
         }
     }
 
-    rv = client_with_mocked_data.post(
+    rv = client.post(
         "/testing/bgp/" + router,
         headers=DEFAULT_REQUEST_HEADERS)
 

test/per_router/test_poller_routes.py

@@ -7,7 +7,7 @@ DEFAULT_REQUEST_HEADERS = {
 }
 
 
-def test_router_interfaces(router, client_with_mocked_data):
+def test_router_interfaces(router, client):
     interfaces_list_schema = {
         "$schema": "http://json-schema.org/draft-07/schema#",
 
@@ -56,7 +56,7 @@ def test_router_interfaces(router, client_with_mocked_data):
         }
     }
 
-    rv = client_with_mocked_data.post(
+    rv = client.post(
         "/poller/interfaces/" + router,
         headers=DEFAULT_REQUEST_HEADERS)
 

test/test_celery_worker_global.py

@@ -5,6 +5,7 @@ and some data ends up in the right place ... otherwise not very detailed
 import contextlib
 
 from inventory_provider.tasks import worker
+
 from inventory_provider.tasks.common import get_redis
 
 

test/test_classifier_routes.py

@@ -130,8 +130,8 @@ JUNIPER_LINK_METADATA = {
 }
 
 
-def test_juniper_link_info(client_with_mocked_data):
-    rv = client_with_mocked_data.get(
+def test_juniper_link_info(client):
+    rv = client.get(
         '/classifier/juniper-link-info/mx1.ams.nl.geant.net/ae15.1500',
         headers=DEFAULT_REQUEST_HEADERS)
     assert rv.status_code == 200
@@ -152,7 +152,7 @@ IX_PUBLIC_PEER_INFO_KEYS = {'ix-public-peer-info', 'interfaces'}
 ]
 )
 def test_peer_info(
-        client_with_mocked_data, peer_address, expected_response_keys):
+        client, peer_address, expected_response_keys):
     response_schema = {
         "$schema": "http://json-schema.org/draft-07/schema#",
         "type": "object",
@@ -257,7 +257,7 @@ def test_peer_info(
         "additionalProperties": False
     }
 
-    rv = client_with_mocked_data.get(
+    rv = client.get(
         '/classifier/peer-info/%s' % peer_address,
         headers=DEFAULT_REQUEST_HEADERS)
     assert rv.status_code == 200
@@ -268,15 +268,15 @@ def test_peer_info(
     assert set(response_data.keys()) == expected_response_keys
 
 
-def test_peer_invalid_address(client_with_mocked_data):
-    rv = client_with_mocked_data.get(
+def test_peer_invalid_address(client):
+    rv = client.get(
         '/classifier/peer-info/1.2.3.4.5',
         headers=DEFAULT_REQUEST_HEADERS)
     assert rv.status_code == 422
 
 
-def test_peer_not_found(client_with_mocked_data):
-    rv = client_with_mocked_data.get(
+def test_peer_not_found(client):
+    rv = client.get(
         '/classifier/peer-info/1.2.3.4',
         headers=DEFAULT_REQUEST_HEADERS)
     assert rv.status_code == 404

test/test_infinera_classifier.py

@@ -89,8 +89,8 @@ INFINERA_LINK_METADATA = {
 }
 
 
-def test_trap_metadata(client_with_mocked_data):
-    rv = client_with_mocked_data.get(
+def test_trap_metadata(client):
+    rv = client.get(
         '/classifier/infinera-lambda-info/'
         'LON2-DTNX10-1/1-A-2-1-4/gen-lon3_LHC_CERN-JANET_09013',
         headers=DEFAULT_REQUEST_HEADERS)

test/test_testing_routes.py

@@ -28,8 +28,8 @@ def test_juniper_addresses(client):
     assert len(response_data) > 0  # test data is not empty
 
 
-def test_get_equipment_location(client_with_mocked_data):
-    rv = client_with_mocked_data.get(
+def test_get_equipment_location(client):
+    rv = client.get(
         '/testing/opsdb/equipment-location',
         headers=DEFAULT_REQUEST_HEADERS)
     assert rv.status_code == 200
@@ -37,8 +37,8 @@ def test_get_equipment_location(client_with_mocked_data):
     # TODO: validate against schema
 
 
-def test_get_interface_info(client_with_mocked_data):
-    rv = client_with_mocked_data.get(
+def test_get_interface_info(client):
+    rv = client.get(
         '/testing/opsdb/interfaces',
         headers=DEFAULT_REQUEST_HEADERS)
     assert rv.status_code == 200
@@ -46,8 +46,8 @@ def test_get_interface_info(client_with_mocked_data):
     # TODO: validate against schema
 
 
-def test_get_interface_info_for_equipment(client_with_mocked_data):
-    rv = client_with_mocked_data.get(
+def test_get_interface_info_for_equipment(client):
+    rv = client.get(
         '/testing/opsdb/interfaces/mx1.ams.nl.geant.net',
         headers=DEFAULT_REQUEST_HEADERS)
     assert rv.status_code == 200
@@ -56,8 +56,8 @@ def test_get_interface_info_for_equipment(client_with_mocked_data):
 
 
 def test_get_interface_info_for_equipment_and_interface(
-            client_with_mocked_data):
-    rv = client_with_mocked_data.get(
+            client):
+    rv = client.get(
         '/testing/opsdb/interfaces/mx1.ams.nl.geant.net/ae3.0',
         headers=DEFAULT_REQUEST_HEADERS)
     assert rv.status_code == 200
@@ -65,8 +65,8 @@ def test_get_interface_info_for_equipment_and_interface(
     # TODO: validate against schema
 
 
-def test_get_children(client_with_mocked_data):
-    rv = client_with_mocked_data.get(
+def test_get_children(client):
+    rv = client.get(
         '/testing/opsdb/circuit-hierarchy/children/12363',
         headers=DEFAULT_REQUEST_HEADERS)
     assert rv.status_code == 200
@@ -74,8 +74,8 @@ def test_get_children(client_with_mocked_data):
     # TODO: validate against schema
 
 
-def test_get_parents(client_with_mocked_data):
-    rv = client_with_mocked_data.get(
+def test_get_parents(client):
+    rv = client.get(
         '/testing/opsdb/circuit-hierarchy/parents/11725',
         headers=DEFAULT_REQUEST_HEADERS)
     assert rv.status_code == 200