Skip to content
Snippets Groups Projects
Commit e3c1d611 authored by Bjarke Madsen's avatar Bjarke Madsen
Browse files

use daemon thread for get_interface_stats to timeout long-running checks, with...

use daemon thread for get_interface_stats to timeout long-running checks, with debug output when it happens
parent 98313840
Branches
Tags
No related merge requests found
import enum import enum
import json import json
import logging.config import logging
import sys import sys
import threading
import traceback
from datetime import datetime from datetime import datetime
from logging import LogRecord from logging import LogRecord
from typing import Any, Collection, Dict, Iterable, List from typing import Any, Collection, Dict, Iterable, List
...@@ -15,9 +17,9 @@ from brian_polling_manager.inventory import ( ...@@ -15,9 +17,9 @@ from brian_polling_manager.inventory import (
load_inventory_json, load_inventory_json,
) )
from .common import PointGroup, RouterProcessor from brian_polling_manager.interface_stats.common import PointGroup, RouterProcessor
from .juniper import JuniperRouterProcessor from brian_polling_manager.interface_stats.juniper import JuniperRouterProcessor
from .nokia import NokiaRouterProcessor from brian_polling_manager.interface_stats.nokia import NokiaRouterProcessor
logger = logging.getLogger() logger = logging.getLogger()
...@@ -177,7 +179,7 @@ def _log_interface_points_sorted(points: Collection[dict], point_group: PointGro ...@@ -177,7 +179,7 @@ def _log_interface_points_sorted(points: Collection[dict], point_group: PointGro
longest_ifc = max(len(i) for i in interfaces) longest_ifc = max(len(i) for i in interfaces)
ifc_count = len(interfaces) ifc_count = len(interfaces)
for n in range(ifc_count // N_COLUMNS + (ifc_count % N_COLUMNS > 0)): for n in range(ifc_count // N_COLUMNS + (ifc_count % N_COLUMNS > 0)):
ifc_slice = interfaces[n * N_COLUMNS : (n + 1) * N_COLUMNS] ifc_slice = interfaces[n * N_COLUMNS: (n + 1) * N_COLUMNS]
logger.info(" ".join(i.ljust(longest_ifc) for i in ifc_slice)) logger.info(" ".join(i.ljust(longest_ifc) for i in ifc_slice))
...@@ -224,7 +226,7 @@ def main( ...@@ -224,7 +226,7 @@ def main(
def validate_config(_unused_ctx, _unused_param, file): def validate_config(_unused_ctx, _unused_param, file):
# import here because this is the only place we use the config module, and we want # import here because this is the only place we use the config module, and we want
# to reuse the name `config` for other purposes elsewheres # to reuse the name `config` for other purposes elsewheres
from . import config from brian_polling_manager.interface_stats import config
try: try:
return config.load(file) return config.load(file)
...@@ -295,11 +297,24 @@ def cli( ...@@ -295,11 +297,24 @@ def cli(
error_counter = setup_logging(debug=verbose) error_counter = setup_logging(debug=verbose)
try: try:
main( thread = threading.Thread(
processor=processor, target=main,
interfaces=interfaces if interfaces else ALL_, args=(processor, interfaces if interfaces else ALL_, OutputMethod.from_string(output.lower())),
output=OutputMethod.from_string(output.lower()),
) )
thread.daemon = True
thread.start()
thread.join(timeout=120)
if thread.is_alive():
logger.error("Thread timed out")
frames = sys._current_frames()
for thread_id, frame in frames.items():
if thread_id == thread.ident:
logger.error("Thread stack trace:")
for line in traceback.format_stack(frame):
logger.error(line.strip())
logger.error(f"-- Thread stack trace end --")
raise click.exceptions.Exit(2)
except Exception: except Exception:
logger.exception( logger.exception(
f"Error while processing {processor.name.capitalize()} router {router_fqdn}" f"Error while processing {processor.name.capitalize()} router {router_fqdn}"
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment