just use normal module-based logging

inventory_provider/constants.py

-SNMP_LOGGER_NAME = "inventory_provider.snmp"
-JUNIPER_LOGGER_NAME = "inventory_provider.juniper"
-DATABASE_LOGGER_NAME = "inventory_provider.database"
-TASK_LOGGER_NAME = "inventory_provider.task"

inventory_provider/juniper.py

@@ -8,8 +8,6 @@ import netifaces
 import requests
 from requests.auth import HTTPBasicAuth
 
-from inventory_provider.constants import JUNIPER_LOGGER_NAME
-
 CONFIG_SCHEMA = """<?xml version="1.1" encoding="UTF-8" ?>
 <xs:schema xmlns:xs="http://www.w3.org/2001/XMLSchema">
 
@@ -152,6 +150,7 @@ JUNOSSPACE_DEVICES_SCHEMA = """<?xml version="1.1" encoding="UTF-8" ?>
 </xs:schema>
 """  # noqa: E501
 
+
 def _rpc(hostname, ssh):
     dev = Device(
         host=hostname,
@@ -162,13 +161,13 @@ def _rpc(hostname, ssh):
 
 
 def validate_netconf_config(config_doc):
-    juniper_logger = logging.getLogger(JUNIPER_LOGGER_NAME)
+    logger = logging.getLogger(__name__)
 
     def _validate(schema, doc):
         if schema.validate(doc):
             return
         for e in schema.error_log:
-            juniper_logger.error("%d.%d: %s" % (e.line, e.column, e.message))
+            logger.error("%d.%d: %s" % (e.line, e.column, e.message))
         assert False
 
     schema_doc = etree.XML(CONFIG_SCHEMA.encode('utf-8'))
@@ -193,8 +192,8 @@ def load_config(hostname, ssh_params):
     :param ssh_params: 'ssh' config element(cf. config.py:CONFIG_SCHEMA)
     :return:
     """
-    juniper_logger = logging.getLogger(JUNIPER_LOGGER_NAME)
-    juniper_logger.info("capturing netconf data for '%s'" % hostname)
+    logger = logging.getLogger(__name__)
+    logger.info("capturing netconf data for '%s'" % hostname)
     config = _rpc(hostname, ssh_params).get_config()
     validate_netconf_config(config)
     return config
@@ -316,7 +315,7 @@ def load_routers_from_junosspace(config):
     :param config: junosspace config element from app config
     :return: list of dictionaries, each element of which describes a router
     """
-    juniper_logger = logging.getLogger(JUNIPER_LOGGER_NAME)
+    logger = logging.getLogger(__name__)
 
     request_url = config['api']
     if not request_url.endswith('/'):
@@ -331,7 +330,7 @@ def load_routers_from_junosspace(config):
     )
     # TODO: use a proper exception type
     if r.status_code != 200:
-        juniper_logger.error("error response from %r" % request_url)
+        logger.error("error response from %r" % request_url)
         assert False  # TODO: use proper exception type
 
     devices = etree.fromstring(r.text.encode('utf-8'))
@@ -339,7 +338,7 @@ def load_routers_from_junosspace(config):
     schema = etree.XMLSchema(schema_doc)
     if not schema.validate(devices):
         for e in schema.error_log:
-            juniper_logger.error('%d.%d: %s' % (e.line, e.column, e.message))
+            logger.error('%d.%d: %s' % (e.line, e.column, e.message))
         assert False
 
     for d in devices.xpath('//devices/device'):
@@ -349,7 +348,7 @@ def load_routers_from_junosspace(config):
         if m:
             hostname = m.group(1) + '.geant.net'
         else:
-            juniper_logger.error(
+            logger.error(
                 'unrecognized junosspace device name format :%s' % name)
             hostname = None
         yield {

inventory_provider/logging_default_config.json

@@ -46,27 +46,13 @@
 
     "loggers": {
         "inventory_provider": {
-            "level": "DEBUG",
+            "level": "INFO",
             "handlers": ["console", "syslog_handler"],
             "propagate": false
         },
-        "inventory_provider.snmp": {
-            "level": "INFO"
-        },
-        "inventory_provider.juniper": {
-            "level": "INFO"
-        },
-        "inventory_provider.database": {
-            "level": "INFO"
-        },
-        "inventory_provider.task": {
+        "inventory_provider.tasks": {
             "level": "DEBUG"
-        },
-        "celery.app.trace": {
-            "level": "INFO",
-            "handlers": ["console", "syslog_handler"],
-            "propagate": false
-          }
+        }
     },
 
     "root": {

inventory_provider/snmp.py

@@ -6,8 +6,6 @@ from pysnmp.hlapi import nextCmd, SnmpEngine, CommunityData, \
 from pysnmp.smi import builder, compiler
 # from pysnmp.smi import view, rfc1902
 
-from inventory_provider.constants import SNMP_LOGGER_NAME
-
 
 def _v6address_oid2str(dotted_decimal):
     hex_params = []
@@ -30,7 +28,7 @@ def walk(agent_hostname, community, base_oid):  # pragma: no cover
     :return:
     """
 
-    snmp_logger = logging.getLogger(SNMP_LOGGER_NAME)
+    logger = logging.getLogger(__name__)
 
     mibBuilder = builder.MibBuilder()
     # mibViewController = view.MibViewController(mibBuilder)
@@ -43,7 +41,7 @@ def walk(agent_hostname, community, base_oid):  # pragma: no cover
         'SNMP-COMMUNITY-MIB',
         'RFC1213-MIB')
 
-    snmp_logger.debug("walking %s: %s" % (agent_hostname, base_oid))
+    logger.debug("walking %s: %s" % (agent_hostname, base_oid))
 
     for (engineErrorIndication,
          pduErrorIndication,
@@ -76,7 +74,7 @@ def walk(agent_hostname, community, base_oid):  # pragma: no cover
         #     for x in varBinds]
         for oid, val in varBinds:
             result = {"oid": "." + str(oid), "value": val.prettyPrint()}
-            snmp_logger.debug(result)
+            logger.debug(result)
             yield result
 
 

inventory_provider/tasks/worker.py

@@ -11,7 +11,6 @@ from lxml import etree
 from inventory_provider.tasks.app import app
 from inventory_provider.tasks.common import get_redis
 from inventory_provider import config
-from inventory_provider import constants
 from inventory_provider import environment
 from inventory_provider.db import db, opsdb, alarmsdb
 from inventory_provider import snmp
@@ -34,8 +33,8 @@ class InventoryTask(Task):
         pass
 
     def update_state(self, **kwargs):
-        task_logger = logging.getLogger(constants.TASK_LOGGER_NAME)
-        task_logger.debug(json.dumps(
+        logger = logging.getLogger(__name__)
+        logger.debug(json.dumps(
             {'state': kwargs['state'], 'meta': kwargs['meta']}
         ))
         super().update_state(**kwargs)
@@ -65,8 +64,8 @@ def _save_value_etree(key, xml_doc):
 class WorkerArgs(bootsteps.Step):
     def __init__(self, worker, config_filename, **options):
         with open(config_filename) as f:
-            task_logger = logging.getLogger(constants.TASK_LOGGER_NAME)
-            task_logger.info(
+            logger = logging.getLogger(__name__)
+            logger.info(
                 "Initializing worker with config from: %r" % config_filename)
             InventoryTask.config = config.load(f)
 
@@ -86,8 +85,8 @@ app.steps['worker'].add(WorkerArgs)
 
 @app.task
 def snmp_refresh_interfaces(hostname, community):
-    task_logger = logging.getLogger(constants.TASK_LOGGER_NAME)
-    task_logger.debug(
+    logger = logging.getLogger(__name__)
+    logger.debug(
         '>>> snmp_refresh_interfaces(%r, %r)' % (hostname, community))
 
     _save_value_json(
@@ -97,26 +96,26 @@ def snmp_refresh_interfaces(hostname, community):
             community,
             InventoryTask.config)))
 
-    task_logger.debug(
+    logger.debug(
         '<<< snmp_refresh_interfaces(%r, %r)' % (hostname, community))
 
 
 @app.task
 def netconf_refresh_config(hostname):
-    task_logger = logging.getLogger(constants.TASK_LOGGER_NAME)
-    task_logger.debug('>>> netconf_refresh_config(%r)' % hostname)
+    logger = logging.getLogger(__name__)
+    logger.debug('>>> netconf_refresh_config(%r)' % hostname)
 
     _save_value_etree(
         'netconf:' + hostname,
         juniper.load_config(hostname, InventoryTask.config["ssh"]))
 
-    task_logger.debug('<<< netconf_refresh_config(%r)' % hostname)
+    logger.debug('<<< netconf_refresh_config(%r)' % hostname)
 
 
 @app.task
 def update_interfaces_to_services():
-    task_logger = logging.getLogger(constants.TASK_LOGGER_NAME)
-    task_logger.debug('>>> update_interfaces_to_services')
+    logger = logging.getLogger(__name__)
+    logger.debug('>>> update_interfaces_to_services')
 
     interface_services = defaultdict(list)
     with db.connection(InventoryTask.config["ops-db"]) as cx:
@@ -133,13 +132,13 @@ def update_interfaces_to_services():
             'opsdb:interface_services:' + equipment_interface,
             json.dumps(services))
 
-    task_logger.debug('<<< update_interfaces_to_services')
+    logger.debug('<<< update_interfaces_to_services')
 
 
 @app.task
 def update_equipment_locations():
-    task_logger = logging.getLogger(constants.TASK_LOGGER_NAME)
-    task_logger.debug('>>> update_equipment_locations')
+    logger = logging.getLogger(__name__)
+    logger.debug('>>> update_equipment_locations')
 
     r = get_redis(InventoryTask.config)
     for key in r.scan_iter('opsdb:location:*'):
@@ -148,13 +147,13 @@ def update_equipment_locations():
         for ld in opsdb.get_equipment_location_data(cx):
             r.set('opsdb:location:%s' % ld['equipment_name'], json.dumps(ld))
 
-    task_logger.debug('<<< update_equipment_locations')
+    logger.debug('<<< update_equipment_locations')
 
 
 @app.task
 def update_circuit_hierarchy():
-    task_logger = logging.getLogger(constants.TASK_LOGGER_NAME)
-    task_logger.debug('>>> update_circuit_hierarchy')
+    logger = logging.getLogger(__name__)
+    logger.debug('>>> update_circuit_hierarchy')
 
     # TODO: integers are not JSON keys
     with db.connection(InventoryTask.config["ops-db"]) as cx:
@@ -177,13 +176,13 @@ def update_circuit_hierarchy():
         for cid, children in child_to_parents.items():
             r.set('opsdb:services:children:%d' % cid, json.dumps(children))
 
-    task_logger.debug('<<< update_circuit_hierarchy')
+    logger.debug('<<< update_circuit_hierarchy')
 
 
 @app.task
 def update_interface_statuses():
-    task_logger = logging.getLogger(constants.TASK_LOGGER_NAME)
-    task_logger.debug('>>> update_interface_statuses')
+    logger = logging.getLogger(__name__)
+    logger.debug('>>> update_interface_statuses')
 
     with db.connection(InventoryTask.config["ops-db"]) as cx:
         services = opsdb.get_circuits(cx)
@@ -198,13 +197,13 @@ def update_interface_statuses():
                     service["interface_name"])
                 _save_value(key, status)
 
-    task_logger.debug('<<< update_interface_statuses')
+    logger.debug('<<< update_interface_statuses')
 
 
 @app.task(base=InventoryTask, bind=True)
 def update_junosspace_device_list(self):
-    task_logger = logging.getLogger(constants.TASK_LOGGER_NAME)
-    task_logger.debug('>>> update_junosspace_device_list')
+    logger = logging.getLogger(__name__)
+    logger.debug('>>> update_junosspace_device_list')
 
     self.update_state(
         state=states.STARTED,
@@ -232,7 +231,7 @@ def update_junosspace_device_list(self):
     for k, v in routers.items():
         r.set(k, v)
 
-    task_logger.debug('<<< update_junosspace_device_list')
+    logger.debug('<<< update_junosspace_device_list')
 
     return {
         'task': 'update_junosspace_device_list',
@@ -255,8 +254,8 @@ def load_netconf_data(hostname):
 
 
 def clear_cached_classifier_responses(hostname):
-    task_logger = logging.getLogger(constants.TASK_LOGGER_NAME)
-    task_logger.debug(
+    logger = logging.getLogger(__name__)
+    logger.debug(
         'removing cached classifier responses for %r' % hostname)
     r = get_redis(InventoryTask.config)
     for k in r.keys('classifier:cache:%s:*' % hostname):
@@ -264,8 +263,8 @@ def clear_cached_classifier_responses(hostname):
 
 
 def _refresh_peers(hostname, key_base, peers):
-    task_logger = logging.getLogger(constants.TASK_LOGGER_NAME)
-    task_logger.debug(
+    logger = logging.getLogger(__name__)
+    logger.debug(
         'removing cached %s for %r' % (key_base, hostname))
     r = get_redis(InventoryTask.config)
     for k in r.keys(key_base + ':*'):
@@ -301,8 +300,8 @@ def refresh_vpn_rr_peers(hostname, netconf):
 
 @app.task(base=InventoryTask, bind=True)
 def reload_router_config(self, hostname):
-    task_logger = logging.getLogger(constants.TASK_LOGGER_NAME)
-    task_logger.debug('>>> reload_router_config')
+    logger = logging.getLogger(__name__)
+    logger.debug('>>> reload_router_config')
 
     self.update_state(
         state=states.STARTED,
@@ -346,7 +345,7 @@ def reload_router_config(self, hostname):
             # TODO: move this out of else? (i.e. clear even if netconf fails?)
             clear_cached_classifier_responses(hostname)
 
-    task_logger.debug('<<< reload_router_config')
+    logger.debug('<<< reload_router_config')
 
     return {
         'task': 'reload_router_config',
@@ -379,7 +378,7 @@ def launch_refresh_cache_all(config):
     :param config: config structure as defined in config.py
     :return:
     """
-    task_logger = logging.getLogger(constants.TASK_LOGGER_NAME)
+    logger = logging.getLogger(__name__)
 
     # first batch of subtasks: refresh cached opsdb data
     subtasks = [
@@ -399,7 +398,7 @@ def launch_refresh_cache_all(config):
         update_interface_statuses.s()
     ]
     for hostname in _derive_router_hostnames(config):
-        task_logger.debug(
+        logger.debug(
             'queueing router refresh jobs for %r' % hostname)
         subtasks.append(reload_router_config.s(hostname))