improve and prettify logging for get-^Cterface-stats

brian_polling_manager/interface_stats/cli.py

@@ -3,7 +3,7 @@ import logging.config
 import os
 import socket
 from datetime import datetime
-from typing import Iterable, List
+from typing import Iterable, List, Sequence
 
 import click
 import jsonschema
@@ -22,23 +22,18 @@ logger = logging.getLogger(__file__)
 LOGGING_DEFAULT_CONFIG = {
     "version": 1,
     "disable_existing_loggers": False,
-    "formatters": {
-        "simple": {
-            "format": "%(asctime)s - %(name)s "
-            "(%(lineno)d) - %(levelname)s - %(message)s"
-        }
-    },
+    "formatters": {"simple": {"format": "%(asctime)s - %(levelname)s - %(message)s"}},
     "handlers": {
         "console": {
             "class": "logging.StreamHandler",
-            "level": "DEBUG",
+            "level": "INFO",
             "formatter": "simple",
             "stream": "ext://sys.stdout",
         },
     },
     "loggers": {
         "brian_polling_manager": {
-            "level": "DEBUG",
+            "level": "INFO",
             "handlers": ["console"],
             "propagate": False,
         }
@@ -62,6 +57,17 @@ def setup_logging():
     if LOGGING_CONFIG is defined in the environment, use this for
     the filename, otherwise use LOGGING_DEFAULT_CONFIG
     """
+
+    # demote ncclient logs
+    def changeLevel(record):
+        if record.levelno == logging.INFO:
+            record.levelno = logging.DEBUG
+            record.levelname = "DEBUG"
+        return record
+
+    logging.getLogger("ncclient.transport.ssh").addFilter(changeLevel)
+    logging.getLogger("ncclient.operations.rpc").addFilter(changeLevel)
+
     logging_config = LOGGING_DEFAULT_CONFIG
     if "LOGGING_CONFIG" in os.environ:
         filename = os.environ["LOGGING_CONFIG"]
@@ -141,31 +147,58 @@ def process_juniper_router(
     router_fqdn: str,
     all_influx_params: dict,
 ):
-    logger.info(f"processing Juniper router {router_fqdn}")
+    logger.info(f"Processing Juniper router {router_fqdn}")
 
     document = get_netconf(router_fqdn, vendor=Vendor.JUNIPER)
     timestamp = datetime.now()
 
     influx_params = all_influx_params["brian-counters"]
-    points = _juniper_brian_points(
-        router_fqdn=router_fqdn,
-        netconf_doc=document,
-        timestamp=timestamp,
-        measurement_name=influx_params["measurement"],
+    logger.info("Processing Brian points...")
+    points = list(
+        _juniper_brian_points(
+            router_fqdn=router_fqdn,
+            netconf_doc=document,
+            timestamp=timestamp,
+            measurement_name=influx_params["measurement"],
+        )
     )
+    _log_interface_points_sorted(points)
     write_points(points, influx_params=influx_params)
 
     influx_params = all_influx_params["error-counters"]
-    points = _juniper_error_points(
-        router_fqdn=router_fqdn,
-        netconf_doc=document,
-        timestamp=timestamp,
-        measurement_name=influx_params["measurement"],
+    logger.info("Processing Error points...")
+    points = list(
+        _juniper_error_points(
+            router_fqdn=router_fqdn,
+            netconf_doc=document,
+            timestamp=timestamp,
+            measurement_name=influx_params["measurement"],
+        )
     )
 
+    _log_interface_points_sorted(points, point_kind='error')
     write_points(points, influx_params=influx_params)
 
 
+def _log_interface_points_sorted(points: Sequence[dict], point_kind=""):
+    N_COLUMNS = 5
+    num_points = len(points)
+    point_kind = point_kind + " " if point_kind else ""
+    semicolon = ':' if num_points else ''
+    logger.info(
+        f"Found {point_kind}points for {num_points} interfaces{semicolon}"
+    )
+
+    if not points:
+        return
+
+    interfaces = sorted(p["tags"]["interface_name"] for p in points)
+    longest_ifc = max(len(i) for i in interfaces)
+    for n in range(len(interfaces) // N_COLUMNS + 1):
+        ifc_slice = interfaces[n * N_COLUMNS : (n + 1) * N_COLUMNS]
+        logger.info("    ".join(i.ljust(longest_ifc) for i in ifc_slice))
+
+
 def _juniper_brian_points(router_fqdn, netconf_doc, timestamp, measurement_name):
     interfaces = juniper.interface_counters(netconf_doc)
     yield from vendors.brian_points(