add retries when creating dashboards on error 500

brian_dashboard_manager/grafana/dashboard.py

@@ -10,6 +10,7 @@ from brian_dashboard_manager.grafana.utils.request import TokenRequest
 
 logger = logging.getLogger(__name__)
 
+NUM_RETRIES = 3
 
 def get_dashboard_definitions(dir=None):
     """
@@ -231,9 +232,17 @@ def create_dashboard(request: TokenRequest, dashboard: dict, folder_id=None):
     if folder_id:
         payload['folderId'] = folder_id
 
-    try:
-        r = request.post('api/dashboards/db', json=payload)
-        return r.json()
-    except HTTPError:
-        logger.exception(f'Error when provisioning dashboard {title}')
+    # retry up to NUM_RETRIES times
+    for _ in range(NUM_RETRIES):
+        try:
+            r = request.post('api/dashboards/db', json=payload)
+            return r.json()
+        except HTTPError as e:
+            message = ''
+            if e.response is not None and e.response.status_code == 412:
+                message = e.response.json()
+            logger.exception(f'Error when provisioning dashboard {title}: {message}')
+            if e.response is not None and e.response.status_code < 500:
+                # only retry on server side errors
+                break
     return None