diff --git a/README.md b/README.md
index 93ef44eea42beb5978bfc3bef875c0abf6f8551e..be345ee3961cf134eb1106d2e8c42663f46e70dc 100644
--- a/README.md
+++ b/README.md
@@ -1,10 +1,10 @@
-1. [Inventory Provider]
- 1. [Overview]
- 2. [Configuration]
- 3. [Running this module]
- 4. [Protocol specification]
- 5. [backend (Redis) storage schema]
+* [Inventory Provider](#inventory-provider)
+ * [Overview](#overview)
+ * [Configuration](#configuration)
+ * [Running this module](#running-this-module)
+ * [Protocol Specification](#protocol-specification)
+ * [Backend (Redis) Storage Schema](#backend-redis-storage-schema)
@@ -42,18 +42,19 @@ The following is an example:
```python
INVENTORY_PROVIDER_CONFIG_FILENAME = "/somepath/config.json"
+ENABLE_TESTING_ROUTES = True
```
-- `INVENTORY_PROVIDER_CONFIG_FILENAME`: run-time accessible filename
+- `INVENTORY_PROVIDER_CONFIG_FILENAME`: [REQUIRED] Run-time accessible filename
of a json file containing the server configuration parameters. This file
must be formatted according to the following json schema:
```json
{
"$schema": "http://json-schema.org/draft-07/schema#",
- "type": "object",
- "properties": {
- "alarms-db": {
+
+ "definitions": {
+ "database_credentials": {
"type": "object",
"properties": {
"hostname": {"type": "string"},
@@ -63,12 +64,19 @@ must be formatted according to the following json schema:
},
"required": ["hostname", "dbname", "username", "password"],
"additionalProperties": False
- },
+
+ }
+ },
+
+ "type": "object",
+ "properties": {
+ "alarms-db": {"$ref": "#/definitions/database_credentials"},
+ "ops-db": {"$ref": "#/definitions/database_credentials"},
"oid_list.conf": {"type": "string"},
- "routers_community.conf": {"type": "string"},
"ssh": {
"type": "object",
"properties": {
+ "username": {"type": "string"},
"private-key": {"type": "string"},
"known-hosts": {"type": "string"}
},
@@ -83,19 +91,57 @@ must be formatted according to the following json schema:
},
"required": ["hostname", "port"],
"additionalProperties": False
+ },
+ "junosspace": {
+ "api": {"type": "string"},
+ "username": {"type": "string"},
+ "password": {"type": "string"}
+ },
+ "infinera-dna": {
+ "type": "array",
+ "items": {
+ "type": "object",
+ "properties": {
+ "name": {"type": "string"},
+ "address": {"type": "string"}
+ },
+ "required": ["name", "address"],
+ "additionalProperties": False
+ }
+ },
+ "coriant-tnms": {
+ "type": "array",
+ "items": {
+ "type": "object",
+ "properties": {
+ "name": {"type": "string"},
+ "address": {"type": "string"}
+ },
+ "required": ["name", "address"],
+ "additionalProperties": False
+ }
}
},
"required": [
"alarms-db",
+ "ops-db",
"oid_list.conf",
- "routers_community.conf",
"ssh",
- "redis"],
+ "redis",
+ "junosspace",
+ "infinera-dna",
+ "coriant-tnms"],
"additionalProperties": False
}
```
+- `ENABLE_TESTING_ROUTES`: [OPTIONAL (default value: False)]
+Flat (can be any value that evaluates to True) to enable
+routes to special utilities used for testing.
+*This must never be enabled in a production environment.*
+
+
## Running this module
This module has been tested in the following execution environments:
@@ -111,10 +157,14 @@ $ flask run
- As an Apache/`mod_wsgi` service.
- Details of Apache and `mod_wsgi`
-configuration are beyond the scope of this document.
+ configuration are beyond the scope of this document.
+- As a `gunicorn` wsgi service.
+ - Details of `gunicorn` configuration are
+ beyond the scope of this document.
-## protocol specification
+
+## Protocol Specification
The following resources can be requested from the webservice.
@@ -253,32 +303,51 @@ Any non-empty responses are JSON formatted messages.
* /jobs/update
This resource updates the inventory network data for juniper devices.
+ The function completes asynchronously and a list of outstanding
+ task id's is returned so the caller can
+ use `/jobs/check-task-status` to determine when all jobs
+ are finished. The response will be formatted as follows:
-* /jobs/update-startup
+ ```json
+ {
+ "$schema": "http://json-schema.org/draft-07/schema#",
+ "type": "array",
+ "items": {"type": "string"}
+ }
+ ```
- This resource updates data that should only be refreshed
- in case of system restart.
-
-* /jobs/update-interfaces-to-services
+* /jobs/reload-router-config/<equipment-name>
- This resource updates the information that lists all the services for
- interfaces found in the external inventory system
-
-* /jobs/update-service-hierarchy
+ This resource updates the inventory network data for
+ the identified juniper device. This function completes
+ asynchronously and returns the same value as
+ `/jobs/update`, except the return contains exactly
+ one task id.
- This resource updates all the information showing which services are related
- to one-another
-
-* /jobs/update-equipment-locations
+* /jobs/check-task-status/<task-id>"
+
+ This resource returns the current status of
+ an asynchronous task started by `/jobs/update`
+ or `jobs/reload-router-config`. The return value
+ will be formatted as follows:
+
+ ```json
+ {
+ "$schema": "http://json-schema.org/draft-07/schema#",
+ "type": "object",
+ "properties": {
+ "id": {"type": "string"},
+ "status": {"type": "string"},
+ "exception": {"type": "boolean"},
+ "ready": {"type": "boolean"},
+ "success": {"type": "boolean"},
+ "result": {"type": "object"}
+ },
+ "required": ["id", "status", "exception", "ready", "success"],
+ "additionalProperties": False
+ }
+ ```
- This resource loads the location information for all the equipment from the
- external inventory system
-
-* /jobs/update-from-inventory-system
-
- This resource updates the inventory data from the external inventory system,
- it is a short cut for executing the above three jobs
-
* /jobs/update-interface-statuses
This resource updates the last known statuses of all interfaces extracted
@@ -356,15 +425,25 @@ Any non-empty responses are JSON formatted messages.
}
```
-## backend (Redis) storage schema
-`netconf:<hostname>`
+* `/testing/flushdb`
+
+ This method erases all data in the backend redis
+ data bases and is only available if the server was
+ started with the `ENABLE_TESTING_ROUTES` flag.
+
+
+## Backend (Redis) Storage Schema
+
+* `netconf:<hostname>`
+
* key example
* `netconf:mx1.ams.nl.geant.net`
* value format
* cf. validation in `inventory_provider.juniper.load_config`
-`snmp-interfaces:<hostname>`
+* `snmp-interfaces:<hostname>`
+
* key example:
* `snmp-interfaces:mx1.lon2.uk.geant.net`
* value schema
@@ -379,20 +458,20 @@ Any non-empty responses are JSON formatted messages.
"properties": {
"v4Address": {
"type": "string",
- "pattern": "^(\d+\.){3}\d+$"
+ "pattern": r'^(\d+\.){3}\d+$'
},
"v4Mask": {
"type": "string",
- "pattern": "^(\d+\.){3}\d+$"
+ "pattern": r'^(\d+\.){3}\d+$'
},
- "v4InterfaceName": {:"type", "string"},
- "index": {`
+ "v4InterfaceName": {"type": "string"},
+ "index": {
"type": "string",
- "pattern": "^\d+$"
+ "pattern": r'^\d+$'
}
},
"required": [
- "v4Address", "v4Mask", "v4InterfaceName", "index],
+ "v4Address", "v4Mask", "v4InterfaceName", "index"],
"additionalProperties": False
},
"v6ifc": {
@@ -400,36 +479,36 @@ Any non-empty responses are JSON formatted messages.
"properties": {
"v6Address": {
"type": "string",
- "pattern": "^[\d:]+$"
+ "pattern": r'^[a-f\d:]+$'
},
"v6Mask": {
"type": "string",
- "pattern": "^\d+$"
+ "pattern": r'^\d+$'
},
- "v6InterfaceName": {:"type", "string"},
- "index": {`
+ "v6InterfaceName": {"type": "string"},
+ "index": {
"type": "string",
- "pattern": "^\d+$"
+ "pattern": r'^\d+$'
}
},
"required": [
- "v6Address", "v6Mask", "v6InterfaceName", "index],
+ "v6Address", "v6Mask", "v6InterfaceName", "index"],
"additionalProperties": False
}
},
"type": "array",
"items": {
- "anyOf": {
- "$ref": "#/definitions/v4Ifc",
- "$ref": "#/definitions/v6Ifc"
- }
+ "anyOf": [
+ {"$ref": "#/definitions/v4ifc"},
+ {"$ref": "#/definitions/v6ifc"}
+ ]
}
}
```
-`opsdb:interface_services:<equipment name>:<interface name>`
+* `opsdb:interface_services:<equipment name>:<interface name>`
* key examples
* `opsdb:interface_services:mx1.ams.nl.geant.net:ae15.1103`
@@ -504,7 +583,7 @@ Any non-empty responses are JSON formatted messages.
```
-`opsdb:location:<equipment name>`
+* `opsdb:location:<equipment name>`
* key examples
* `opsdb:location:mx1.ams.nl.geant.net`
@@ -535,8 +614,8 @@ Any non-empty responses are JSON formatted messages.
}
```
-`opsdb:services:children:<circuit db id>`
-`opsdb:services:parents:<circuit db id>`
+* `opsdb:services:children:<circuit db id>`
+* `opsdb:services:parents:<circuit db id>`
* key examples
* `opsdb:services:children:12363`
@@ -588,7 +667,7 @@ Any non-empty responses are JSON formatted messages.
}
```
-`alarmsdb:interface_status:<equipment name>:<interface name>`
+* `alarmsdb:interface_status:<equipment name>:<interface name>`
* key examples
* `alarmsdb:interface_status:Lab node 1:1-1/3`
* `alarmsdb:interface_status:mx1.ams.nl.geant.net:ae15.1500`
@@ -599,7 +678,7 @@ Any non-empty responses are JSON formatted messages.
* `down`
-`ix_public_peer:<address>`
+* `ix_public_peer:<address>`
* key examples
* `ix_public_peer:193.203.0.203`
* `ix_public_peer:2001:07f8:00a0:0000:0000:5926:0000:0002`
@@ -637,7 +716,7 @@ Any non-empty responses are JSON formatted messages.
}
```
-`vpn_rr_peers/<address>`
+* `vpn_rr_peers/<address>`
* key examples
* `ix_public_peer:193.203.0.203`
* valid values:
diff --git a/inventory_provider/constants.py b/inventory_provider/constants.py
deleted file mode 100644
index 3925c65bb0a7e8410de907d77899e5c92cf6b882..0000000000000000000000000000000000000000
--- a/inventory_provider/constants.py
+++ /dev/null
@@ -1,4 +0,0 @@
-SNMP_LOGGER_NAME = "inventory_provider.snmp"
-JUNIPER_LOGGER_NAME = "inventory_provider.juniper"
-DATABASE_LOGGER_NAME = "inventory_provider.database"
-TASK_LOGGER_NAME = "inventory_provider.task"
diff --git a/inventory_provider/juniper.py b/inventory_provider/juniper.py
index 741ef352381f1b6213e9bfc936fcbe8ac0c3ee61..3f33c8d94470f088f627562aa513ff491137f293 100644
--- a/inventory_provider/juniper.py
+++ b/inventory_provider/juniper.py
@@ -8,8 +8,6 @@ import netifaces
import requests
from requests.auth import HTTPBasicAuth
-from inventory_provider.constants import JUNIPER_LOGGER_NAME
-
CONFIG_SCHEMA = """<?xml version="1.1" encoding="UTF-8" ?>
<xs:schema xmlns:xs="http://www.w3.org/2001/XMLSchema">
@@ -105,6 +103,11 @@ UNIT_SCHEMA = """<?xml version="1.1" encoding="UTF-8" ?>
</xs:schema>
""" # noqa: E501
+
+# elements 'use-nat' and 'fingerprint' were added between
+# junosspace versions 15.x and 17.x ... hopefully new versions
+# will also add new elements at the end of the sequence so
+# that the final xs:any below will suffice to allow validation
JUNOSSPACE_DEVICES_SCHEMA = """<?xml version="1.1" encoding="UTF-8" ?>
<xs:schema xmlns:xs="http://www.w3.org/2001/XMLSchema">
@@ -127,8 +130,7 @@ JUNOSSPACE_DEVICES_SCHEMA = """<?xml version="1.1" encoding="UTF-8" ?>
<xs:element name="domain-id" minOccurs="1" maxOccurs="1" type="xs:string" />
<xs:element name="domain-name" minOccurs="1" maxOccurs="1" type="xs:string" />
<xs:element name="config-status" minOccurs="1" maxOccurs="1" type="xs:string" />
- <xs:element name="use-nat" minOccurs="0" maxOccurs="1" type="xs:boolean" />
- <xs:element name="fingerprint" minOccurs="0" maxOccurs="1" type="xs:string" />
+ <xs:any processContents="lax" minOccurs="0" maxOccurs="unbounded" />
</xs:sequence>
<xs:attribute name="href" type="xs:string" />
<xs:attribute name="uri" type="xs:string" />
@@ -159,13 +161,13 @@ def _rpc(hostname, ssh):
def validate_netconf_config(config_doc):
- juniper_logger = logging.getLogger(JUNIPER_LOGGER_NAME)
+ logger = logging.getLogger(__name__)
def _validate(schema, doc):
if schema.validate(doc):
return
for e in schema.error_log:
- juniper_logger.error("%d.%d: %s" % (e.line, e.column, e.message))
+ logger.error("%d.%d: %s" % (e.line, e.column, e.message))
assert False
schema_doc = etree.XML(CONFIG_SCHEMA.encode('utf-8'))
@@ -190,8 +192,8 @@ def load_config(hostname, ssh_params):
:param ssh_params: 'ssh' config element(cf. config.py:CONFIG_SCHEMA)
:return:
"""
- juniper_logger = logging.getLogger(JUNIPER_LOGGER_NAME)
- juniper_logger.info("capturing netconf data for '%s'" % hostname)
+ logger = logging.getLogger(__name__)
+ logger.info("capturing netconf data for '%s'" % hostname)
config = _rpc(hostname, ssh_params).get_config()
validate_netconf_config(config)
return config
@@ -313,7 +315,7 @@ def load_routers_from_junosspace(config):
:param config: junosspace config element from app config
:return: list of dictionaries, each element of which describes a router
"""
- juniper_logger = logging.getLogger(JUNIPER_LOGGER_NAME)
+ logger = logging.getLogger(__name__)
request_url = config['api']
if not request_url.endswith('/'):
@@ -328,7 +330,7 @@ def load_routers_from_junosspace(config):
)
# TODO: use a proper exception type
if r.status_code != 200:
- juniper_logger.error("error response from %r" % request_url)
+ logger.error("error response from %r" % request_url)
assert False # TODO: use proper exception type
devices = etree.fromstring(r.text.encode('utf-8'))
@@ -336,7 +338,7 @@ def load_routers_from_junosspace(config):
schema = etree.XMLSchema(schema_doc)
if not schema.validate(devices):
for e in schema.error_log:
- juniper_logger.error("%d.%d: %s" % (e.line, e.column, e.message))
+ logger.error('%d.%d: %s' % (e.line, e.column, e.message))
assert False
for d in devices.xpath('//devices/device'):
@@ -346,6 +348,8 @@ def load_routers_from_junosspace(config):
if m:
hostname = m.group(1) + '.geant.net'
else:
+ logger.error(
+ 'unrecognized junosspace device name format :%s' % name)
hostname = None
yield {
"OSVersion": d.xpath('./OSVersion/text()')[0],
diff --git a/inventory_provider/logging_default_config.json b/inventory_provider/logging_default_config.json
index 2ab40c772446f65e022057395ab9a07cd0e91d51..ba3f1eb38510e597063d1d27ceddbea4c0286f10 100644
--- a/inventory_provider/logging_default_config.json
+++ b/inventory_provider/logging_default_config.json
@@ -46,27 +46,13 @@
"loggers": {
"inventory_provider": {
- "level": "DEBUG",
+ "level": "INFO",
"handlers": ["console", "syslog_handler"],
"propagate": false
},
- "inventory_provider.snmp": {
- "level": "INFO"
- },
- "inventory_provider.juniper": {
- "level": "INFO"
- },
- "inventory_provider.database": {
- "level": "INFO"
- },
- "inventory_provider.task": {
+ "inventory_provider.tasks": {
"level": "DEBUG"
- },
- "celery.app.trace": {
- "level": "INFO",
- "handlers": ["console", "syslog_handler"],
- "propagate": false
- }
+ }
},
"root": {
diff --git a/inventory_provider/routes/jobs.py b/inventory_provider/routes/jobs.py
index 99be6cff92fa5b59389b144c02823b4aed4d0038..02e778ab4b0bd8d4b85fc44ae79dd33c663701f8 100644
--- a/inventory_provider/routes/jobs.py
+++ b/inventory_provider/routes/jobs.py
@@ -1,28 +1,32 @@
from flask import Blueprint, Response, current_app, jsonify
from inventory_provider.tasks import worker
+from inventory_provider.routes import common
routes = Blueprint("inventory-data-job-routes", __name__)
@routes.route("/update", methods=['GET', 'POST'])
+@common.require_accepts_json
def update():
job_ids = worker.launch_refresh_cache_all(
current_app.config["INVENTORY_PROVIDER_CONFIG"])
return jsonify(job_ids)
-@routes.route("update-interface-statuses")
+@routes.route("update-interface-statuses", methods=['GET', 'POST'])
def update_interface_statuses():
worker.update_interface_statuses.delay()
return Response("OK")
-@routes.route("reload-router-config/<equipment_name>")
+@routes.route("reload-router-config/<equipment_name>", methods=['GET', 'POST'])
+@common.require_accepts_json
def reload_router_config(equipment_name):
result = worker.reload_router_config.delay(equipment_name)
return jsonify([result.id])
-@routes.route("check-task-status/<task_id>")
+@routes.route("check-task-status/<task_id>", methods=['GET', 'POST'])
+@common.require_accepts_json
def check_task_status(task_id):
return jsonify(worker.check_task_status(task_id))
diff --git a/inventory_provider/snmp.py b/inventory_provider/snmp.py
index 0c2c50461f886e5c719101de3622c3b1dad71c3b..a5a48b7067e75117756d779c87abdd1767718c48 100644
--- a/inventory_provider/snmp.py
+++ b/inventory_provider/snmp.py
@@ -6,8 +6,6 @@ from pysnmp.hlapi import nextCmd, SnmpEngine, CommunityData, \
from pysnmp.smi import builder, compiler
# from pysnmp.smi import view, rfc1902
-from inventory_provider.constants import SNMP_LOGGER_NAME
-
def _v6address_oid2str(dotted_decimal):
hex_params = []
@@ -30,7 +28,7 @@ def walk(agent_hostname, community, base_oid): # pragma: no cover
:return:
"""
- snmp_logger = logging.getLogger(SNMP_LOGGER_NAME)
+ logger = logging.getLogger(__name__)
mibBuilder = builder.MibBuilder()
# mibViewController = view.MibViewController(mibBuilder)
@@ -43,7 +41,7 @@ def walk(agent_hostname, community, base_oid): # pragma: no cover
'SNMP-COMMUNITY-MIB',
'RFC1213-MIB')
- snmp_logger.debug("walking %s: %s" % (agent_hostname, base_oid))
+ logger.debug("walking %s: %s" % (agent_hostname, base_oid))
for (engineErrorIndication,
pduErrorIndication,
@@ -76,7 +74,7 @@ def walk(agent_hostname, community, base_oid): # pragma: no cover
# for x in varBinds]
for oid, val in varBinds:
result = {"oid": "." + str(oid), "value": val.prettyPrint()}
- snmp_logger.debug(result)
+ logger.debug(result)
yield result
diff --git a/inventory_provider/tasks/worker.py b/inventory_provider/tasks/worker.py
index 3b7db454efc165a0ec5a5b20e2e8cb70f47ab5e2..87020db6963d57a858724b26865f3419b398fe21 100644
--- a/inventory_provider/tasks/worker.py
+++ b/inventory_provider/tasks/worker.py
@@ -11,7 +11,6 @@ from lxml import etree
from inventory_provider.tasks.app import app
from inventory_provider.tasks.common import get_redis
from inventory_provider import config
-from inventory_provider import constants
from inventory_provider import environment
from inventory_provider.db import db, opsdb, alarmsdb
from inventory_provider import snmp
@@ -34,8 +33,8 @@ class InventoryTask(Task):
pass
def update_state(self, **kwargs):
- task_logger = logging.getLogger(constants.TASK_LOGGER_NAME)
- task_logger.debug(json.dumps(
+ logger = logging.getLogger(__name__)
+ logger.debug(json.dumps(
{'state': kwargs['state'], 'meta': kwargs['meta']}
))
super().update_state(**kwargs)
@@ -65,8 +64,8 @@ def _save_value_etree(key, xml_doc):
class WorkerArgs(bootsteps.Step):
def __init__(self, worker, config_filename, **options):
with open(config_filename) as f:
- task_logger = logging.getLogger(constants.TASK_LOGGER_NAME)
- task_logger.info(
+ logger = logging.getLogger(__name__)
+ logger.info(
"Initializing worker with config from: %r" % config_filename)
InventoryTask.config = config.load(f)
@@ -86,8 +85,8 @@ app.steps['worker'].add(WorkerArgs)
@app.task
def snmp_refresh_interfaces(hostname, community):
- task_logger = logging.getLogger(constants.TASK_LOGGER_NAME)
- task_logger.debug(
+ logger = logging.getLogger(__name__)
+ logger.debug(
'>>> snmp_refresh_interfaces(%r, %r)' % (hostname, community))
_save_value_json(
@@ -97,26 +96,26 @@ def snmp_refresh_interfaces(hostname, community):
community,
InventoryTask.config)))
- task_logger.debug(
+ logger.debug(
'<<< snmp_refresh_interfaces(%r, %r)' % (hostname, community))
@app.task
def netconf_refresh_config(hostname):
- task_logger = logging.getLogger(constants.TASK_LOGGER_NAME)
- task_logger.debug('>>> netconf_refresh_config(%r)' % hostname)
+ logger = logging.getLogger(__name__)
+ logger.debug('>>> netconf_refresh_config(%r)' % hostname)
_save_value_etree(
'netconf:' + hostname,
juniper.load_config(hostname, InventoryTask.config["ssh"]))
- task_logger.debug('<<< netconf_refresh_config(%r)' % hostname)
+ logger.debug('<<< netconf_refresh_config(%r)' % hostname)
@app.task
def update_interfaces_to_services():
- task_logger = logging.getLogger(constants.TASK_LOGGER_NAME)
- task_logger.debug('>>> update_interfaces_to_services')
+ logger = logging.getLogger(__name__)
+ logger.debug('>>> update_interfaces_to_services')
interface_services = defaultdict(list)
with db.connection(InventoryTask.config["ops-db"]) as cx:
@@ -133,13 +132,13 @@ def update_interfaces_to_services():
'opsdb:interface_services:' + equipment_interface,
json.dumps(services))
- task_logger.debug('<<< update_interfaces_to_services')
+ logger.debug('<<< update_interfaces_to_services')
@app.task
def update_equipment_locations():
- task_logger = logging.getLogger(constants.TASK_LOGGER_NAME)
- task_logger.debug('>>> update_equipment_locations')
+ logger = logging.getLogger(__name__)
+ logger.debug('>>> update_equipment_locations')
r = get_redis(InventoryTask.config)
for key in r.scan_iter('opsdb:location:*'):
@@ -148,13 +147,13 @@ def update_equipment_locations():
for ld in opsdb.get_equipment_location_data(cx):
r.set('opsdb:location:%s' % ld['equipment_name'], json.dumps(ld))
- task_logger.debug('<<< update_equipment_locations')
+ logger.debug('<<< update_equipment_locations')
@app.task
def update_circuit_hierarchy():
- task_logger = logging.getLogger(constants.TASK_LOGGER_NAME)
- task_logger.debug('>>> update_circuit_hierarchy')
+ logger = logging.getLogger(__name__)
+ logger.debug('>>> update_circuit_hierarchy')
# TODO: integers are not JSON keys
with db.connection(InventoryTask.config["ops-db"]) as cx:
@@ -177,13 +176,13 @@ def update_circuit_hierarchy():
for cid, children in child_to_parents.items():
r.set('opsdb:services:children:%d' % cid, json.dumps(children))
- task_logger.debug('<<< update_circuit_hierarchy')
+ logger.debug('<<< update_circuit_hierarchy')
@app.task
def update_interface_statuses():
- task_logger = logging.getLogger(constants.TASK_LOGGER_NAME)
- task_logger.debug('>>> update_interface_statuses')
+ logger = logging.getLogger(__name__)
+ logger.debug('>>> update_interface_statuses')
with db.connection(InventoryTask.config["ops-db"]) as cx:
services = opsdb.get_circuits(cx)
@@ -198,13 +197,13 @@ def update_interface_statuses():
service["interface_name"])
_save_value(key, status)
- task_logger.debug('<<< update_interface_statuses')
+ logger.debug('<<< update_interface_statuses')
@app.task(base=InventoryTask, bind=True)
def update_junosspace_device_list(self):
- task_logger = logging.getLogger(constants.TASK_LOGGER_NAME)
- task_logger.debug('>>> update_junosspace_device_list')
+ logger = logging.getLogger(__name__)
+ logger.debug('>>> update_junosspace_device_list')
self.update_state(
state=states.STARTED,
@@ -232,7 +231,7 @@ def update_junosspace_device_list(self):
for k, v in routers.items():
r.set(k, v)
- task_logger.debug('<<< update_junosspace_device_list')
+ logger.debug('<<< update_junosspace_device_list')
return {
'task': 'update_junosspace_device_list',
@@ -255,8 +254,8 @@ def load_netconf_data(hostname):
def clear_cached_classifier_responses(hostname):
- task_logger = logging.getLogger(constants.TASK_LOGGER_NAME)
- task_logger.debug(
+ logger = logging.getLogger(__name__)
+ logger.debug(
'removing cached classifier responses for %r' % hostname)
r = get_redis(InventoryTask.config)
for k in r.keys('classifier:cache:%s:*' % hostname):
@@ -264,8 +263,8 @@ def clear_cached_classifier_responses(hostname):
def _refresh_peers(hostname, key_base, peers):
- task_logger = logging.getLogger(constants.TASK_LOGGER_NAME)
- task_logger.debug(
+ logger = logging.getLogger(__name__)
+ logger.debug(
'removing cached %s for %r' % (key_base, hostname))
r = get_redis(InventoryTask.config)
for k in r.keys(key_base + ':*'):
@@ -301,8 +300,8 @@ def refresh_vpn_rr_peers(hostname, netconf):
@app.task(base=InventoryTask, bind=True)
def reload_router_config(self, hostname):
- task_logger = logging.getLogger(constants.TASK_LOGGER_NAME)
- task_logger.debug('>>> reload_router_config')
+ logger = logging.getLogger(__name__)
+ logger.debug('>>> reload_router_config')
self.update_state(
state=states.STARTED,
@@ -346,7 +345,7 @@ def reload_router_config(self, hostname):
# TODO: move this out of else? (i.e. clear even if netconf fails?)
clear_cached_classifier_responses(hostname)
- task_logger.debug('<<< reload_router_config')
+ logger.debug('<<< reload_router_config')
return {
'task': 'reload_router_config',
@@ -379,7 +378,7 @@ def launch_refresh_cache_all(config):
:param config: config structure as defined in config.py
:return:
"""
- task_logger = logging.getLogger(constants.TASK_LOGGER_NAME)
+ logger = logging.getLogger(__name__)
# first batch of subtasks: refresh cached opsdb data
subtasks = [
@@ -399,7 +398,7 @@ def launch_refresh_cache_all(config):
update_interface_statuses.s()
]
for hostname in _derive_router_hostnames(config):
- task_logger.debug(
+ logger.debug(
'queueing router refresh jobs for %r' % hostname)
subtasks.append(reload_router_config.s(hostname))
@@ -409,7 +408,7 @@ def launch_refresh_cache_all(config):
def check_task_status(task_id):
r = AsyncResult(task_id, app=app)
result = {
- 'id': task_id,
+ 'id': r.id,
'status': r.status,
'exception': r.status in states.EXCEPTION_STATES,
'ready': r.status in states.READY_STATES,
diff --git a/test/test_job_routes.py b/test/test_job_routes.py
new file mode 100644
index 0000000000000000000000000000000000000000..427ceafd39d521d88eb848b450add5258745ba21
--- /dev/null
+++ b/test/test_job_routes.py
@@ -0,0 +1,125 @@
+import json
+import jsonschema
+
+DEFAULT_REQUEST_HEADERS = {
+ "Content-type": "application/json",
+ "Accept": ["application/json"]
+}
+
+TASK_ID_LIST_SCHEMA = {
+ "$schema": "http://json-schema.org/draft-07/schema#",
+ "type": "array",
+ "items": {"type": "string"}
+}
+
+TASK_STATUS_SCHEMA = {
+ "$schema": "http://json-schema.org/draft-07/schema#",
+ "type": "object",
+ "properties": {
+ "id": {"type": "string"},
+ "status": {"type": "string"},
+ "exception": {"type": "boolean"},
+ "ready": {"type": "boolean"},
+ "success": {"type": "boolean"},
+ "result": {"type": "object"}
+ },
+ "required": ["id", "status", "exception", "ready", "success"],
+ "additionalProperties": False
+}
+
+
+def test_job_update_all(client, mocker):
+ launch_refresh_cache_all = mocker.patch(
+ 'inventory_provider.tasks.worker.launch_refresh_cache_all')
+ launch_refresh_cache_all.return_value = ['abc', 'def', 'xyz@123#456']
+
+ rv = client.post(
+ 'jobs/update',
+ headers=DEFAULT_REQUEST_HEADERS)
+ assert rv.status_code == 200
+ task_id_list = json.loads(rv.data.decode('utf-8'))
+ jsonschema.validate(task_id_list, TASK_ID_LIST_SCHEMA)
+ assert len(task_id_list) == 3
+
+
+class MockedAsyncResult(object):
+ status = None
+ result = None
+
+ def __init__(self, id, app=None):
+ self.id = id
+
+
+def test_reload_router_config(client, mocker):
+ delay_result = mocker.patch(
+ 'inventory_provider.tasks.worker.reload_router_config.delay')
+ delay_result.return_value = MockedAsyncResult('bogus task id')
+
+ rv = client.post(
+ 'jobs/reload-router-config/ignored###123',
+ headers=DEFAULT_REQUEST_HEADERS)
+ assert rv.status_code == 200
+ task_id_list = json.loads(rv.data.decode('utf-8'))
+ jsonschema.validate(task_id_list, TASK_ID_LIST_SCHEMA)
+ assert task_id_list == ['bogus task id']
+
+
+def test_check_task_status_success(client, mocker):
+ mocker.patch(
+ 'inventory_provider.tasks.worker.AsyncResult', MockedAsyncResult)
+ MockedAsyncResult.status = 'SUCCESS' # celery.states.SUCCESS
+ MockedAsyncResult.result = {'abc': 1, 'def': 'aaabbb'}
+
+ rv = client.post(
+ 'jobs/check-task-status/abc',
+ headers=DEFAULT_REQUEST_HEADERS)
+ assert rv.status_code == 200
+ status = json.loads(rv.data.decode('utf-8'))
+ jsonschema.validate(status, TASK_STATUS_SCHEMA)
+ assert status['id'] == 'abc'
+ assert status['status'] == 'SUCCESS'
+ assert not status['exception']
+ assert status['ready']
+ assert status['success']
+ assert 'result' in status
+
+
+def test_check_task_status_custom_status(client, mocker):
+ mocker.patch(
+ 'inventory_provider.tasks.worker.AsyncResult', MockedAsyncResult)
+ MockedAsyncResult.status = 'custom'
+ MockedAsyncResult.result = None
+
+ rv = client.post(
+ 'jobs/check-task-status/xyz',
+ headers=DEFAULT_REQUEST_HEADERS)
+ assert rv.status_code == 200
+ status = json.loads(rv.data.decode('utf-8'))
+ jsonschema.validate(status, TASK_STATUS_SCHEMA)
+ assert status['id'] == 'xyz'
+ assert status['status'] == 'custom'
+ assert not status['exception']
+ assert not status['ready']
+ assert not status['success']
+ assert 'result' not in status
+
+
+def test_check_task_status_exception(client, mocker):
+ mocker.patch(
+ 'inventory_provider.tasks.worker.AsyncResult', MockedAsyncResult)
+ MockedAsyncResult.status = 'FAILURE' # celery.states.FAILURE
+ MockedAsyncResult.result = AssertionError('test error message')
+
+ rv = client.post(
+ 'jobs/check-task-status/123-xyz.ABC',
+ headers=DEFAULT_REQUEST_HEADERS)
+ assert rv.status_code == 200
+ status = json.loads(rv.data.decode('utf-8'))
+ jsonschema.validate(status, TASK_STATUS_SCHEMA)
+ assert status['id'] == '123-xyz.ABC'
+ assert status['status'] == 'FAILURE'
+ assert status['exception']
+ assert status['ready']
+ assert not status['success']
+ assert status['result']['error type'] == 'AssertionError'
+ assert status['result']['message'] == 'test error message'
diff --git a/test/test_snmp_handling.py b/test/test_snmp_handling.py
index f8138a68e6a090befa102cf3ab50eed7b656f325..08485456e629bc31821555240bf4c6d97137493f 100644
--- a/test/test_snmp_handling.py
+++ b/test/test_snmp_handling.py
@@ -26,20 +26,58 @@ def test_snmp_interfaces(mocker, data_config, snmp_walk_responses):
expected_result_schema = {
"$schema": "http://json-schema.org/draft-07/schema#",
+
+ "definitions": {
+ "v4ifc": {
+ "type": "object",
+ "properties": {
+ "v4Address": {
+ "type": "string",
+ "pattern": r'^(\d+\.){3}\d+$'
+ },
+ "v4Mask": {
+ "type": "string",
+ "pattern": r'^(\d+\.){3}\d+$'
+ },
+ "v4InterfaceName": {"type": "string"},
+ "index": {
+ "type": "string",
+ "pattern": r'^\d+$'
+ }
+ },
+ "required": [
+ "v4Address", "v4Mask", "v4InterfaceName", "index"],
+ "additionalProperties": False
+ },
+ "v6ifc": {
+ "type": "object",
+ "properties": {
+ "v6Address": {
+ "type": "string",
+ "pattern": r'^[a-f\d:]+$'
+ },
+ "v6Mask": {
+ "type": "string",
+ "pattern": r'^\d+$'
+ },
+ "v6InterfaceName": {"type": "string"},
+ "index": {
+ "type": "string",
+ "pattern": r'^\d+$'
+ }
+ },
+ "required": [
+ "v6Address", "v6Mask", "v6InterfaceName", "index"],
+ "additionalProperties": False
+ }
+ },
+
"type": "array",
"items": {
- "type": "object",
- "properties": {
- "v4Address": {"type": "string"},
- "v4Mask": {"type": "string"},
- "v4InterfaceName": {"type": "string"},
- "v6Address": {"type": "string"},
- "v6Mask": {"type": "string"},
- "v6InterfaceName": {"type": "string"},
- "index": {"type": "string"}
- },
- "required": ["index"],
- "additionalProperties": False
+ "anyOf": [
+ {"$ref": "#/definitions/v4ifc"},
+ {"$ref": "#/definitions/v6ifc"}
+ ]
}
}
@@ -57,13 +95,3 @@ def test_snmp_interfaces(mocker, data_config, snmp_walk_responses):
jsonschema.validate(interfaces, expected_result_schema)
assert interfaces, "interface list isn't empty"
- for ifc in interfaces:
- if 'v4Address' in ifc \
- and 'v4Mask' in ifc \
- and 'v4InterfaceName' in ifc:
- continue
- if 'v6Address' in ifc \
- and 'v6Mask' in ifc \
- and 'v6InterfaceName' in ifc:
- continue
- assert False, "address details not found in interface dict"