Skip to content
Snippets Groups Projects
Commit e3c1d611 authored by Bjarke Madsen's avatar Bjarke Madsen
Browse files

use daemon thread for get_interface_stats to timeout long-running checks, with...

use daemon thread for get_interface_stats to timeout long-running checks, with debug output when it happens
parent 98313840
No related branches found
No related tags found
No related merge requests found
import enum
import json
import logging.config
import logging
import sys
import threading
import traceback
from datetime import datetime
from logging import LogRecord
from typing import Any, Collection, Dict, Iterable, List
......@@ -15,9 +17,9 @@ from brian_polling_manager.inventory import (
load_inventory_json,
)
from .common import PointGroup, RouterProcessor
from .juniper import JuniperRouterProcessor
from .nokia import NokiaRouterProcessor
from brian_polling_manager.interface_stats.common import PointGroup, RouterProcessor
from brian_polling_manager.interface_stats.juniper import JuniperRouterProcessor
from brian_polling_manager.interface_stats.nokia import NokiaRouterProcessor
logger = logging.getLogger()
......@@ -177,7 +179,7 @@ def _log_interface_points_sorted(points: Collection[dict], point_group: PointGro
longest_ifc = max(len(i) for i in interfaces)
ifc_count = len(interfaces)
for n in range(ifc_count // N_COLUMNS + (ifc_count % N_COLUMNS > 0)):
ifc_slice = interfaces[n * N_COLUMNS : (n + 1) * N_COLUMNS]
ifc_slice = interfaces[n * N_COLUMNS: (n + 1) * N_COLUMNS]
logger.info(" ".join(i.ljust(longest_ifc) for i in ifc_slice))
......@@ -224,7 +226,7 @@ def main(
def validate_config(_unused_ctx, _unused_param, file):
# import here because this is the only place we use the config module, and we want
# to reuse the name `config` for other purposes elsewheres
from . import config
from brian_polling_manager.interface_stats import config
try:
return config.load(file)
......@@ -295,11 +297,24 @@ def cli(
error_counter = setup_logging(debug=verbose)
try:
main(
processor=processor,
interfaces=interfaces if interfaces else ALL_,
output=OutputMethod.from_string(output.lower()),
thread = threading.Thread(
target=main,
args=(processor, interfaces if interfaces else ALL_, OutputMethod.from_string(output.lower())),
)
thread.daemon = True
thread.start()
thread.join(timeout=120)
if thread.is_alive():
logger.error("Thread timed out")
frames = sys._current_frames()
for thread_id, frame in frames.items():
if thread_id == thread.ident:
logger.error("Thread stack trace:")
for line in traceback.format_stack(frame):
logger.error(line.strip())
logger.error(f"-- Thread stack trace end --")
raise click.exceptions.Exit(2)
except Exception:
logger.exception(
f"Error while processing {processor.name.capitalize()} router {router_fqdn}"
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment