diff --git a/inventory_provider/__init__.py b/inventory_provider/__init__.py
index 1e10f25b7eada979f0ed5b97c4c47d5f4f77f8af..97cb00a8654a6ac2688ac120296ce6359bd8c5f9 100644
--- a/inventory_provider/__init__.py
+++ b/inventory_provider/__init__.py
@@ -16,14 +16,32 @@ def create_app():
:return: a new flask app instance
"""
- if "SETTINGS_FILENAME" not in os.environ:
- assert False, \
- "environment variable SETTINGS_FILENAME' must be defined"
+ required_env_vars = [
+ 'FLASK_SETTINGS_FILENAME', 'INVENTORY_PROVIDER_CONFIG_FILENAME']
+
+ assert all([n in os.environ for n in required_env_vars]), \
+ 'environment variables %r must be defined' % required_env_vars
+
+ assert os.path.isfile(os.environ['INVENTORY_PROVIDER_CONFIG_FILENAME']), (
+ 'config file %r not found %r' %
+ os.environ['INVENTORY_PROVIDER_CONFIG_FILENAME'])
+
+ from inventory_provider import config
+ with open(os.environ['INVENTORY_PROVIDER_CONFIG_FILENAME']) as f:
+ logging.info(
+ 'loading config from: %r'
+ % os.environ['INVENTORY_PROVIDER_CONFIG_FILENAME'])
+ inventory_provider_config = config.load(f)
app = Flask(__name__)
- app.secret_key = "super secret session key"
+ app.secret_key = 'super secret session key'
- environment.setup_logging()
+ logging.info(
+ 'initializing Flask with config from: %r' %
+ os.environ['FLASK_SETTINGS_FILENAME'])
+ app.config.from_envvar('FLASK_SETTINGS_FILENAME')
+
+ app.config['INVENTORY_PROVIDER_CONFIG'] = inventory_provider_config
from inventory_provider.routes import default
app.register_blueprint(default.routes, url_prefix='/')
@@ -40,30 +58,13 @@ def create_app():
from inventory_provider.routes import poller
app.register_blueprint(poller.routes, url_prefix='/poller')
- logging.info("initializing Flask with config from: %r"
- % os.environ["SETTINGS_FILENAME"])
- app.config.from_envvar("SETTINGS_FILENAME")
-
- assert "INVENTORY_PROVIDER_CONFIG_FILENAME" in app.config, (
- "INVENTORY_PROVIDER_CONFIG_FILENAME not defined in %s"
- % os.environ["SETTINGS_FILENAME"])
-
- assert os.path.isfile(app.config["INVENTORY_PROVIDER_CONFIG_FILENAME"]), (
- "config file '%s' not found" %
- app.config["INVENTORY_PROVIDER_CONFIG_FILENAME"])
-
if app.config.get('ENABLE_TESTING_ROUTES', False):
from inventory_provider.routes import testing
app.register_blueprint(testing.routes, url_prefix='/testing')
logging.warning('DANGER!!! testing routes enabled')
- from inventory_provider import config
- with open(app.config["INVENTORY_PROVIDER_CONFIG_FILENAME"]) as f:
- # test the config file can be loaded
- logging.info("loading config from: %r"
- % app.config["INVENTORY_PROVIDER_CONFIG_FILENAME"])
- app.config["INVENTORY_PROVIDER_CONFIG"] = config.load(f)
-
logging.info('Inventory Provider Flask app initialized')
+ environment.setup_logging()
+
return app
diff --git a/inventory_provider/config.py b/inventory_provider/config.py
index 73b04a86f5c10d5c7a24890508fb3a7902ec9783..6b6e648531120572399c55bfe693c217875c35f6 100644
--- a/inventory_provider/config.py
+++ b/inventory_provider/config.py
@@ -1,5 +1,4 @@
import json
-import re
import jsonschema
@@ -24,7 +23,6 @@ CONFIG_SCHEMA = {
"type": "object",
"properties": {
"ops-db": {"$ref": "#/definitions/database_credentials"},
- "oid_list.conf": {"type": "string"},
"ssh": {
"type": "object",
"properties": {
@@ -64,7 +62,6 @@ CONFIG_SCHEMA = {
{
"required": [
"ops-db",
- "oid_list.conf",
"ssh",
"redis",
"junosspace"]
@@ -72,7 +69,6 @@ CONFIG_SCHEMA = {
{
"required": [
"ops-db",
- "oid_list.conf",
"ssh",
"sentinel",
"junosspace"]
@@ -82,19 +78,6 @@ CONFIG_SCHEMA = {
}
-def _load_oids(config_file):
- """
- :param config_file: file-like object
- :return:
- """
- result = {}
- for line in config_file:
- m = re.match(r'^([^=]+)=(.*)\s*$', line)
- if m:
- result[m.group(1)] = m.group(2)
- return result
-
-
def load(f):
"""
loads, validates and returns configuration parameters
@@ -104,6 +87,4 @@ def load(f):
"""
config = json.loads(f.read())
jsonschema.validate(config, CONFIG_SCHEMA)
- with open(config["oid_list.conf"]) as f:
- config["oids"] = _load_oids(f)
return config
diff --git a/inventory_provider/tasks/config.py b/inventory_provider/tasks/config.py
index 2d955ea63a4bd678b87014d3375f3459922638ca..0b1879a95f3c7d40399d1c8fa36c2304ab06840a 100644
--- a/inventory_provider/tasks/config.py
+++ b/inventory_provider/tasks/config.py
@@ -1,36 +1,42 @@
import logging
import os
-
+from inventory_provider import config
import redis.sentinel
logger = logging.getLogger(__name__)
-broker_hostname = os.getenv('BROKER_HOSTNAME')
-assert broker_hostname is not None
-broker_port = int(os.getenv('BROKER_PORT'))
-broker_db_index = int(os.getenv('CELERY_DB_INDEX'))
+assert os.path.isfile(os.environ['INVENTORY_PROVIDER_CONFIG_FILENAME']), (
+ 'config file %r not found' %
+ os.environ['INVENTORY_PROVIDER_CONFIG_FILENAME'])
-broker_scheme = os.getenv('BROKER_SCHEME', 'redis')
-assert broker_scheme in ('redis', 'sentinel'), 'unsupported broker scheme'
+with open(os.environ['INVENTORY_PROVIDER_CONFIG_FILENAME']) as f:
+ logging.info(
+ 'loading config from: %r'
+ % os.environ['INVENTORY_PROVIDER_CONFIG_FILENAME'])
+ inventory_provider_config = config.load(f)
-if broker_scheme == 'sentinel':
- master_name = os.getenv('SENTINEL_MASTER_NAME')
- assert master_name is not None
+if 'sentinel' in inventory_provider_config:
+ params = inventory_provider_config['sentinel']
sentinel = redis.sentinel.Sentinel(
- [(broker_hostname, broker_port)],
- socket_timeout=0.1)
- master = sentinel.discover_master(master_name)
+ [(params['hostname'], int(params['port']))],
+ socket_timeout=0.5)
+ master = sentinel.discover_master(params['name'])
assert master
- broker_hostname = master[0]
- broker_port = master[1]
+ _broker_hostname = master[0]
+ _broker_port = master[1]
+else:
+ params = inventory_provider_config['redis']
+ _broker_hostname = params['hostname']
+ _broker_port = int(params['port'])
+
+_broker_db_index = 1 # TODO: this should be a config param
-if ':' in broker_hostname:
+if ':' in _broker_hostname:
# assume this means hostname is an ipv6 address
- broker_hostname = '[%s]' % broker_hostname
+ _broker_hostname = '[%s]' % _broker_hostname
-broker_url = 'redis://%s:%d/%d' % (
- broker_hostname, broker_port, broker_db_index)
-result_backend = broker_url
+broker_url = result_backend = 'redis://%s:%d/%d' % (
+ _broker_hostname, _broker_port, _broker_db_index)
logger.debug('broker_url: %r' % broker_url)
diff --git a/inventory_provider/tasks/worker.py b/inventory_provider/tasks/worker.py
index 8779a1d4b22e6bff65759a829cc4733010a5b9b3..b948cb82176bbe7f525d08e6a06748adc74e23d8 100644
--- a/inventory_provider/tasks/worker.py
+++ b/inventory_provider/tasks/worker.py
@@ -1,8 +1,9 @@
import json
import logging
+import os
import re
-from celery import bootsteps, Task, states
+from celery import Task, states
from celery.result import AsyncResult
from collections import defaultdict
@@ -30,7 +31,21 @@ class InventoryTask(Task):
config = None
def __init__(self):
- pass
+
+ if InventoryTask.config:
+ return
+
+ assert os.path.isfile(
+ os.environ['INVENTORY_PROVIDER_CONFIG_FILENAME']), (
+ 'config file %r not found' %
+ os.environ['INVENTORY_PROVIDER_CONFIG_FILENAME'])
+
+ with open(os.environ['INVENTORY_PROVIDER_CONFIG_FILENAME']) as f:
+ logging.info(
+ "Initializing worker with config from: %r" %
+ os.environ['INVENTORY_PROVIDER_CONFIG_FILENAME'])
+ InventoryTask.config = config.load(f)
+ logging.debug("loaded config: %r" % InventoryTask.config)
def update_state(self, **kwargs):
logger = logging.getLogger(__name__)
@@ -61,28 +76,6 @@ def _save_value_etree(key, xml_doc):
etree.tostring(xml_doc, encoding='unicode'))
-class WorkerArgs(bootsteps.Step):
- def __init__(self, worker, config_filename, **options):
- with open(config_filename) as f:
- logger = logging.getLogger(__name__)
- logger.info(
- "Initializing worker with config from: %r" % config_filename)
- InventoryTask.config = config.load(f)
-
-
-def worker_args(parser):
- parser.add_argument(
- "--config_filename",
- dest="config_filename",
- action='store',
- type=str,
- help="Configuration filename")
-
-
-app.user_options['worker'].add(worker_args)
-app.steps['worker'].add(WorkerArgs)
-
-
@app.task
def snmp_refresh_interfaces(hostname, community):
logger = logging.getLogger(__name__)
diff --git a/test/conftest.py b/test/conftest.py
index 42e78419efa01571b81b5a2af94a0c984bce1976..6be4a2b70c3ec77be38303555a210f12e522fd3f 100644
--- a/test/conftest.py
+++ b/test/conftest.py
@@ -3,15 +3,14 @@ import json
import netifaces
import os
import re
-import shutil
import tempfile
from lxml import etree
import pytest
import inventory_provider
-from inventory_provider import config
from inventory_provider.tasks import worker
+from inventory_provider import config
TEST_DATA_DIRNAME = os.path.realpath(os.path.join(
inventory_provider.__path__[0],
@@ -20,49 +19,42 @@ TEST_DATA_DIRNAME = os.path.realpath(os.path.join(
"data"))
-def data_config_filename(tmp_dir_name):
- config = {
- "ops-db": {
- "hostname": "xxxxxxx.yyyyy.zzz",
- "dbname": "xxxxxx",
- "username": "xxxxxx",
- "password": "xxxxxxxx"
- },
- "oid_list.conf": os.path.join(
- tmp_dir_name,
- "oid_list.conf"),
- "ssh": {
- "username": "uSeR-NaMe",
- "private-key": "private-key-filename",
- "known-hosts": "known-hosts=filename"
- },
- "redis": {
- "hostname": "xxxxxx",
- "port": 6379
- },
- "junosspace": {
- "api": "bogus-url",
- "username": "bogus-username",
- "password": "bogus-password"
+@pytest.fixture
+def data_config_filename():
+
+ with tempfile.NamedTemporaryFile() as f:
+ config = {
+ "ops-db": {
+ "hostname": "xxxxxxx.yyyyy.zzz",
+ "dbname": "xxxxxx",
+ "username": "xxxxxx",
+ "password": "xxxxxxxx"
+ },
+ "ssh": {
+ "username": "uSeR-NaMe",
+ "private-key": "private-key-filename",
+ "known-hosts": "known-hosts=filename"
+ },
+ "redis": {
+ "hostname": "xxxxxx",
+ "port": 6379
+ },
+ "junosspace": {
+ "api": "bogus-url",
+ "username": "bogus-username",
+ "password": "bogus-password"
+ }
}
- }
- shutil.copyfile(
- os.path.join(TEST_DATA_DIRNAME, 'oid_list.conf'),
- config['oid_list.conf']
- )
+ f.write(json.dumps(config).encode('utf-8'))
+ f.flush()
+ yield f.name
- filename = os.path.join(tmp_dir_name, "config.json")
- with open(filename, "w") as f:
- f.write(json.dumps(config))
- return filename
-
-
-def _tmp_data_config():
- with tempfile.TemporaryDirectory() as tmpdir:
- with open(data_config_filename(tmpdir)) as f:
- return config.load(f)
+@pytest.fixture
+def data_config(data_config_filename):
+ with open(data_config_filename) as f:
+ return config.load(f)
TEST_DATA_DIRNAME = os.path.realpath(os.path.join(
@@ -117,11 +109,6 @@ class MockedRedis(object):
pass
-@pytest.fixture
-def data_config():
- return _tmp_data_config()
-
-
@pytest.fixture
def cached_test_data():
filename = os.path.join(TEST_DATA_DIRNAME, "router-info.json")
@@ -130,29 +117,12 @@ def cached_test_data():
@pytest.fixture
-def app_config():
- with tempfile.TemporaryDirectory() as tmpdir:
+def flask_config_filename():
- app_config_filename = os.path.join(tmpdir, "app.config")
- with open(app_config_filename, "w") as f:
- f.write("%s = '%s'\n" % (
- "INVENTORY_PROVIDER_CONFIG_FILENAME",
- data_config_filename(tmpdir)))
- f.write('ENABLE_TESTING_ROUTES = True\n')
-
- yield app_config_filename
-
-
-@pytest.fixture
-def client(app_config, mocker):
-
- mocker.patch(
- 'inventory_provider.tasks.common.redis.StrictRedis',
- MockedRedis)
-
- os.environ["SETTINGS_FILENAME"] = app_config
- with inventory_provider.create_app().test_client() as c:
- yield c
+ with tempfile.NamedTemporaryFile() as f:
+ f.write('ENABLE_TESTING_ROUTES = True\n'.encode('utf-8'))
+ f.flush()
+ yield f.name
@pytest.fixture
@@ -163,8 +133,16 @@ def mocked_redis(mocker):
@pytest.fixture
-def client_with_mocked_data(client, mocked_redis):
- return client
+def client(flask_config_filename, data_config_filename, mocked_redis):
+ os.environ['FLASK_SETTINGS_FILENAME'] = flask_config_filename
+ os.environ['INVENTORY_PROVIDER_CONFIG_FILENAME'] = data_config_filename
+ with inventory_provider.create_app().test_client() as c:
+ yield c
+
+#
+# @pytest.fixture
+# def client(client, mocked_redis):
+# return client
NETIFACES_TEST_DATA_STRING = """{
@@ -194,15 +172,13 @@ def mocked_netifaces(mocker):
@pytest.fixture
def mocked_worker_module(
- mocker, mocked_redis, data_config,
+ mocker, mocked_redis, data_config_filename,
cached_test_data, mocked_netifaces):
- os.environ['BROKER_HOSTNAME'] = 'a.b.c'
- os.environ['BROKER_PORT'] = '123'
- os.environ['CELERY_DB_INDEX'] = '456'
- os.environ['BROKER_SCHEME'] = 'redis'
+ os.environ['INVENTORY_PROVIDER_CONFIG_FILENAME'] = data_config_filename
- worker.InventoryTask.config = data_config
+ with open(data_config_filename) as f:
+ worker.InventoryTask.config = config.load(f)
def _mocked_snmp_interfaces(hostname, community):
return json.loads(cached_test_data['snmp-interfaces:' + hostname])
diff --git a/test/per_router/test_data_routes.py b/test/per_router/test_data_routes.py
index fc4cc8cfbdc614dcba3ce557e9929f1ad0a7ce80..063aa2a3e13b49ff9df6fc3d07953edfbaa4806c 100644
--- a/test/per_router/test_data_routes.py
+++ b/test/per_router/test_data_routes.py
@@ -8,7 +8,7 @@ DEFAULT_REQUEST_HEADERS = {
}
-def test_router_interfaces(router, client_with_mocked_data):
+def test_router_interfaces(router, client):
interfaces_list_schema = {
"$schema": "http://json-schema.org/draft-07/schema#",
@@ -36,7 +36,7 @@ def test_router_interfaces(router, client_with_mocked_data):
}
}
- rv = client_with_mocked_data.post(
+ rv = client.post(
"/data/interfaces/" + router,
headers=DEFAULT_REQUEST_HEADERS)
@@ -46,7 +46,7 @@ def test_router_interfaces(router, client_with_mocked_data):
assert response # at least shouldn't be empty
-def test_snmp_ids(router, client_with_mocked_data):
+def test_snmp_ids(router, client):
snmp_id_list_schema = {
"$schema": "http://json-schema.org/draft-07/schema#",
@@ -62,7 +62,7 @@ def test_snmp_ids(router, client_with_mocked_data):
}
}
- rv = client_with_mocked_data.post(
+ rv = client.post(
"/testing/snmp/" + router,
headers=DEFAULT_REQUEST_HEADERS)
@@ -71,7 +71,7 @@ def test_snmp_ids(router, client_with_mocked_data):
assert response # at least shouldn't be empty
-def test_router_bgp_routes(router, client_with_mocked_data):
+def test_router_bgp_routes(router, client):
ROUTERS_WITH_BGP_CONFIG = [
"mx1.bud.hu.geant.net",
@@ -114,7 +114,7 @@ def test_router_bgp_routes(router, client_with_mocked_data):
}
}
- rv = client_with_mocked_data.post(
+ rv = client.post(
"/testing/bgp/" + router,
headers=DEFAULT_REQUEST_HEADERS)
diff --git a/test/per_router/test_poller_routes.py b/test/per_router/test_poller_routes.py
index 79fd947d0a9e9b8f5676c1431274469bab6a2d59..fe0a91cb585737d04762f886c1d17cd2066b9218 100644
--- a/test/per_router/test_poller_routes.py
+++ b/test/per_router/test_poller_routes.py
@@ -7,7 +7,7 @@ DEFAULT_REQUEST_HEADERS = {
}
-def test_router_interfaces(router, client_with_mocked_data):
+def test_router_interfaces(router, client):
interfaces_list_schema = {
"$schema": "http://json-schema.org/draft-07/schema#",
@@ -56,7 +56,7 @@ def test_router_interfaces(router, client_with_mocked_data):
}
}
- rv = client_with_mocked_data.post(
+ rv = client.post(
"/poller/interfaces/" + router,
headers=DEFAULT_REQUEST_HEADERS)
diff --git a/test/test_celery_worker_global.py b/test/test_celery_worker_global.py
index 3592cc3e9b2f149a82e92df633a80ae240793d79..40f0d9aef53d789a89bff9777bb1525253c103fd 100644
--- a/test/test_celery_worker_global.py
+++ b/test/test_celery_worker_global.py
@@ -5,6 +5,7 @@ and some data ends up in the right place ... otherwise not very detailed
import contextlib
from inventory_provider.tasks import worker
+
from inventory_provider.tasks.common import get_redis
diff --git a/test/test_classifier_routes.py b/test/test_classifier_routes.py
index 43104e20e4072bc78f1efdbb037ce809090e91d3..013aa6ce6ab7f7d817a3d1afb40f71ecaef1a182 100644
--- a/test/test_classifier_routes.py
+++ b/test/test_classifier_routes.py
@@ -130,8 +130,8 @@ JUNIPER_LINK_METADATA = {
}
-def test_juniper_link_info(client_with_mocked_data):
- rv = client_with_mocked_data.get(
+def test_juniper_link_info(client):
+ rv = client.get(
'/classifier/juniper-link-info/mx1.ams.nl.geant.net/ae15.1500',
headers=DEFAULT_REQUEST_HEADERS)
assert rv.status_code == 200
@@ -152,7 +152,7 @@ IX_PUBLIC_PEER_INFO_KEYS = {'ix-public-peer-info', 'interfaces'}
]
)
def test_peer_info(
- client_with_mocked_data, peer_address, expected_response_keys):
+ client, peer_address, expected_response_keys):
response_schema = {
"$schema": "http://json-schema.org/draft-07/schema#",
"type": "object",
@@ -257,7 +257,7 @@ def test_peer_info(
"additionalProperties": False
}
- rv = client_with_mocked_data.get(
+ rv = client.get(
'/classifier/peer-info/%s' % peer_address,
headers=DEFAULT_REQUEST_HEADERS)
assert rv.status_code == 200
@@ -268,15 +268,15 @@ def test_peer_info(
assert set(response_data.keys()) == expected_response_keys
-def test_peer_invalid_address(client_with_mocked_data):
- rv = client_with_mocked_data.get(
+def test_peer_invalid_address(client):
+ rv = client.get(
'/classifier/peer-info/1.2.3.4.5',
headers=DEFAULT_REQUEST_HEADERS)
assert rv.status_code == 422
-def test_peer_not_found(client_with_mocked_data):
- rv = client_with_mocked_data.get(
+def test_peer_not_found(client):
+ rv = client.get(
'/classifier/peer-info/1.2.3.4',
headers=DEFAULT_REQUEST_HEADERS)
assert rv.status_code == 404
diff --git a/test/test_infinera_classifier.py b/test/test_infinera_classifier.py
index 9d114f9eacf408f6fde53f0e45dc8b0fa37bf938..892bd06f2bda4e1a407e3fad631af793d206acf2 100644
--- a/test/test_infinera_classifier.py
+++ b/test/test_infinera_classifier.py
@@ -89,8 +89,8 @@ INFINERA_LINK_METADATA = {
}
-def test_trap_metadata(client_with_mocked_data):
- rv = client_with_mocked_data.get(
+def test_trap_metadata(client):
+ rv = client.get(
'/classifier/infinera-lambda-info/'
'LON2-DTNX10-1/1-A-2-1-4/gen-lon3_LHC_CERN-JANET_09013',
headers=DEFAULT_REQUEST_HEADERS)
diff --git a/test/test_testing_routes.py b/test/test_testing_routes.py
index cb57dd4121174254ca7796752cd2864ae82c5295..17a910a26342ff826ff556a75fe3481953f464e3 100644
--- a/test/test_testing_routes.py
+++ b/test/test_testing_routes.py
@@ -28,8 +28,8 @@ def test_juniper_addresses(client):
assert len(response_data) > 0 # test data is not empty
-def test_get_equipment_location(client_with_mocked_data):
- rv = client_with_mocked_data.get(
+def test_get_equipment_location(client):
+ rv = client.get(
'/testing/opsdb/equipment-location',
headers=DEFAULT_REQUEST_HEADERS)
assert rv.status_code == 200
@@ -37,8 +37,8 @@ def test_get_equipment_location(client_with_mocked_data):
# TODO: validate against schema
-def test_get_interface_info(client_with_mocked_data):
- rv = client_with_mocked_data.get(
+def test_get_interface_info(client):
+ rv = client.get(
'/testing/opsdb/interfaces',
headers=DEFAULT_REQUEST_HEADERS)
assert rv.status_code == 200
@@ -46,8 +46,8 @@ def test_get_interface_info(client_with_mocked_data):
# TODO: validate against schema
-def test_get_interface_info_for_equipment(client_with_mocked_data):
- rv = client_with_mocked_data.get(
+def test_get_interface_info_for_equipment(client):
+ rv = client.get(
'/testing/opsdb/interfaces/mx1.ams.nl.geant.net',
headers=DEFAULT_REQUEST_HEADERS)
assert rv.status_code == 200
@@ -56,8 +56,8 @@ def test_get_interface_info_for_equipment(client_with_mocked_data):
def test_get_interface_info_for_equipment_and_interface(
- client_with_mocked_data):
- rv = client_with_mocked_data.get(
+ client):
+ rv = client.get(
'/testing/opsdb/interfaces/mx1.ams.nl.geant.net/ae3.0',
headers=DEFAULT_REQUEST_HEADERS)
assert rv.status_code == 200
@@ -65,8 +65,8 @@ def test_get_interface_info_for_equipment_and_interface(
# TODO: validate against schema
-def test_get_children(client_with_mocked_data):
- rv = client_with_mocked_data.get(
+def test_get_children(client):
+ rv = client.get(
'/testing/opsdb/circuit-hierarchy/children/12363',
headers=DEFAULT_REQUEST_HEADERS)
assert rv.status_code == 200
@@ -74,8 +74,8 @@ def test_get_children(client_with_mocked_data):
# TODO: validate against schema
-def test_get_parents(client_with_mocked_data):
- rv = client_with_mocked_data.get(
+def test_get_parents(client):
+ rv = client.get(
'/testing/opsdb/circuit-hierarchy/parents/11725',
headers=DEFAULT_REQUEST_HEADERS)
assert rv.status_code == 200