Skip to content
Snippets Groups Projects
Commit 0b973d28 authored by Bjarke Madsen's avatar Bjarke Madsen
Browse files

use global state variable for provisioning, now that we only use 1 worker process

parent a76e36d0
No related branches found
No related tags found
No related merge requests found
......@@ -29,11 +29,8 @@ If the value is a list, dashboard titles within the list should be excluded.
import json
import jsonschema
import tempfile
STATE_PATH = tempfile.gettempdir() + '/briandashboardmanager-state.json'
DEFAULT_ORGANIZATIONS = [
{
"name": "GÉANT Staff",
......
import json
import datetime
from flask import jsonify, Response
from concurrent.futures import ThreadPoolExecutor
from json.decoder import JSONDecodeError
from flask import Blueprint, current_app
from brian_dashboard_manager.routes import common
from brian_dashboard_manager.grafana.provision import provision_maybe
from brian_dashboard_manager.grafana.provision import provision
from brian_dashboard_manager import CONFIG_KEY
from brian_dashboard_manager.config import STATE_PATH
provision_state = {
'time': datetime.datetime.now(datetime.timezone.utc),
'provisioning': False
}
routes = Blueprint("update", __name__)
......@@ -27,40 +29,38 @@ def after_request(resp):
return common.after_request(resp)
def should_provision():
def provision_maybe():
"""
Check if we should provision by checking the state file.
Multiple workers can call this function at the same time,
so we need to make sure we don't provision twice while
the first provisioning is still running.
Check if we should provision in case of multiple requests hitting the endpoint.
We need to make sure we don't provision if another thread is still running.
:return: tuple of (bool, datetime) representing if we can provision
and the timestamp of the last provisioning, respectively.
"""
try:
with open(STATE_PATH, 'r+') as f:
try:
state = json.load(f)
except JSONDecodeError:
state = {}
global provision_state
now = datetime.datetime.now(datetime.timezone.utc)
timestamp = provision_state['time']
provisioning = provision_state['provisioning']
if provisioning and (now - timestamp).total_seconds() < 600: # lockout for 10 minutes at most
return False, timestamp
def write_timestamp(timestamp, provisioning):
provision_state['time'] = timestamp
provision_state['provisioning'] = provisioning
provisioning = state.get('provisioning', False)
timestamp = datetime.datetime.fromtimestamp(
state.get('timestamp', 1))
def _finish():
now = datetime.datetime.now(datetime.timezone.utc)
write_timestamp(now, False)
now = datetime.datetime.now()
if provisioning and (now - timestamp).total_seconds() > 3600:
# if we stay in provisioning state
# for over an hour, we probably restarted
# and the state file is out of sync.
provisioning = False
write_timestamp(now, True)
can_provision = not provisioning
return can_provision, timestamp
except FileNotFoundError:
with open(STATE_PATH, 'w') as f:
return True, None
executor = ThreadPoolExecutor(max_workers=1)
f = executor.submit(provision, current_app.config[CONFIG_KEY])
f.add_done_callback(lambda _: _finish())
return True, now
@routes.route('/', methods=['GET'])
......@@ -78,11 +78,10 @@ def update():
:return: json
"""
should, timestamp = should_provision()
should, timestamp = provision_maybe()
if should:
executor = ThreadPoolExecutor(max_workers=1)
executor.submit(provision_maybe, current_app.config[CONFIG_KEY])
return jsonify({'data': {'message': 'Provisioning dashboards!'}})
else:
message = f'Provision already in progress since {timestamp}'
seconds_ago = (datetime.datetime.now(datetime.timezone.utc) - timestamp).total_seconds()
message = f'Provision already in progress since {timestamp} ({seconds_ago:.2f} seconds ago).'
return Response(message, status=503)
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment