diff --git a/brian_dashboard_manager/grafana/provision.py b/brian_dashboard_manager/grafana/provision.py index 5f56a4a7b5809adcb5b228edd8d7378c25891cfd..69da36bf491509215fec9a4a55a6b3d7c281f86f 100644 --- a/brian_dashboard_manager/grafana/provision.py +++ b/brian_dashboard_manager/grafana/provision.py @@ -9,7 +9,7 @@ from concurrent.futures import Future from concurrent.futures import ThreadPoolExecutor from brian_dashboard_manager.config import DEFAULT_ORGANIZATIONS from brian_dashboard_manager.grafana.utils.request import AdminRequest, \ - TokenRequest + TokenRequest, MAX_THREADS from brian_dashboard_manager.services.api import fetch_services @@ -41,7 +41,6 @@ from brian_dashboard_manager.templating.render import ( render_simple_dashboard, ) -MAX_THREADS = 16 logger = logging.getLogger(__name__) DASHBOARDS = { @@ -810,7 +809,9 @@ def _provision_org(config, org, org_config): with ThreadPoolExecutor(max_workers=MAX_THREADS) as thread_executor: args = (thread_executor, config, org_config, ds_name, token_request) - managed_dashboards = itertools.chain( + + # call to list is needed to queue up the futures + managed_dashboards = list(itertools.chain( _provision_interfaces(*args), _provision_gws_indirect(*args), _provision_gws_direct(*args), @@ -819,7 +820,7 @@ def _provision_org(config, org, org_config): _provision_service_dashboards(*args), _provision_static_dashboards(*args), _get_ignored_dashboards(*args) - ) + )) managed_dashboard_uids = {} for dashboard in managed_dashboards: diff --git a/brian_dashboard_manager/grafana/utils/request.py b/brian_dashboard_manager/grafana/utils/request.py index 8baa5e95199a5843df0e26c996c16c9b522d4604..c126690c0bd83725df4c8bf3a96d52963fb1fc57 100644 --- a/brian_dashboard_manager/grafana/utils/request.py +++ b/brian_dashboard_manager/grafana/utils/request.py @@ -4,13 +4,15 @@ from requests.adapters import HTTPAdapter logger = logging.getLogger(__name__) +MAX_THREADS = 16 + class Request(requests.Session): def __init__(self, url, headers=None): super().__init__() # allow using up to 16 connections - adapter = HTTPAdapter(pool_maxsize=16) + adapter = HTTPAdapter(pool_maxsize=MAX_THREADS) self.mount(url, adapter) self.headers.update({ diff --git a/changelog.md b/changelog.md index 8841c4b5c2b76cb9d6cb4a54cab7fb366601f9af..7c46ac02a9ce99371366db09bd81f21a401bfbdd 100644 --- a/changelog.md +++ b/changelog.md @@ -2,6 +2,10 @@ All notable changes to this project will be documented in this file. +## [0.74] - 2024-11-05 +- Ensure futures are all started at the same time, enabling proper multithreading +- Synchronize thread count and urllib pool size to avoid dropped connections + ## [0.73] - 2024-11-05 - Set minimum interval per panel rather than globally, to avoid issues with panels using data from different sources with different intervals. - Set 15m minimum interval for GEANT LAMBDA / Managed Wavelength Service panels since the polling interval is 15m in the TNMS data source. diff --git a/setup.py b/setup.py index 4ba247892ad02a38f8c391af630947c4db834f72..98195d89f54f558ebaddf65595a6fc3e2d02327b 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import setup, find_packages setup( name='brian-dashboard-manager', - version="0.73", + version="0.74", author='GEANT', author_email='swd@geant.org', description='',