Add exception logging in threaded function

brian_dashboard_manager/grafana/provision.py

@@ -756,74 +756,78 @@ def provision(config):
             return None
 
     for org in all_orgs:
-        org_id = org['id']
-
-        logger.info(
-            f'--- Provisioning org {org["name"]} (ID #{org_id}) ---')
-        org_config = _find_org_config(org)
-        if not org_config:
-            # message logged from _find_org_config
-            continue
-
-        token = create_api_token(request, org_id)
-        token_request = TokenRequest(token=token['key'], **config)
-        tokens.append((org_id, token['id']))
-        logger.debug(tokens)
-
-        all_original_dashboards = list_dashboards(token_request)
-        all_original_dashboard_uids = {
-            d['uid'] for d in all_original_dashboards}
-
-        datasource = _provision_datasource(config, token_request)
-        ds_name = datasource.get('name', 'PollerInfluxDB')
-
-        managed_dashboards = itertools.chain(
-            _provision_interfaces(
-                config, org_config, ds_name, token_request),
-            _provision_gws_indirect(
-                config, org_config, ds_name, token_request),
-            _provision_gws_direct(
-                config, org_config, ds_name, token_request),
-            _provision_eumetsat_multicast(
-                config, org_config, ds_name, token_request),
-            _provision_aggregates(
-                config, org_config, ds_name, token_request),
-            _provision_static_dashboards(
-                config, org_config, ds_name, token_request),
-            _get_ignored_dashboards(
-                config, org_config, token_request)
-        )
+        try:
+            org_id = org['id']
 
-        managed_dashboard_uids = set()
-        for dashboard in managed_dashboards:
-            if isinstance(dashboard, Future):
-                dashboard = dashboard.result()
-            if dashboard is None:
+            logger.info(
+                f'--- Provisioning org {org["name"]} (ID #{org_id}) ---')
+            org_config = _find_org_config(org)
+            if not org_config:
+                # message logged from _find_org_config
                 continue
-            managed_dashboard_uids.add(dashboard['uid'])
-
-        for uid in all_original_dashboard_uids - managed_dashboard_uids:
-            # delete unmanaged dashboards
-            logger.info(f'Deleting stale dashboard with UID {uid}')
-            delete_dashboard(token_request, {'uid': uid})
-
-        folders_to_keep = {
-            # General is a base folder present in Grafana
-            'General',
-            # other folders, created outside of the DASHBOARDS list
-            'GWS Indirect',
-            'GWS Direct',
-            'Aggregates',
-            'EUMETSAT Multicast'
-        }
-        folders_to_keep.update({dash['folder_name']
-                                for dash in DASHBOARDS.values()})
-
-        ignored_folders = config.get('ignored_folders', [])
-        folders_to_keep.update(ignored_folders)
-
-        delete_unknown_folders(token_request, folders_to_keep)
-        delete_api_token(request, token['id'], org_id=org_id)
+
+            token = create_api_token(request, org_id)
+            token_request = TokenRequest(token=token['key'], **config)
+            tokens.append((org_id, token['id']))
+            logger.debug(tokens)
+
+            all_original_dashboards = list_dashboards(token_request)
+            all_original_dashboard_uids = {
+                d['uid'] for d in all_original_dashboards}
+
+            datasource = _provision_datasource(config, token_request)
+            ds_name = datasource.get('name', 'PollerInfluxDB')
+
+            managed_dashboards = itertools.chain(
+                _provision_interfaces(
+                    config, org_config, ds_name, token_request),
+                _provision_gws_indirect(
+                    config, org_config, ds_name, token_request),
+                _provision_gws_direct(
+                    config, org_config, ds_name, token_request),
+                _provision_eumetsat_multicast(
+                    config, org_config, ds_name, token_request),
+                _provision_aggregates(
+                    config, org_config, ds_name, token_request),
+                _provision_static_dashboards(
+                    config, org_config, ds_name, token_request),
+                _get_ignored_dashboards(
+                    config, org_config, token_request)
+            )
+
+            managed_dashboard_uids = set()
+            for dashboard in managed_dashboards:
+                if isinstance(dashboard, Future):
+                    dashboard = dashboard.result()
+                if dashboard is None:
+                    continue
+                managed_dashboard_uids.add(dashboard['uid'])
+
+            for uid in all_original_dashboard_uids - managed_dashboard_uids:
+                # delete unmanaged dashboards
+                logger.info(f'Deleting stale dashboard with UID {uid}')
+                delete_dashboard(token_request, {'uid': uid})
+
+            folders_to_keep = {
+                # General is a base folder present in Grafana
+                'General',
+                # other folders, created outside of the DASHBOARDS list
+                'GWS Indirect',
+                'GWS Direct',
+                'Aggregates',
+                'EUMETSAT Multicast'
+            }
+            folders_to_keep.update({dash['folder_name']
+                                    for dash in DASHBOARDS.values()})
+
+            ignored_folders = config.get('ignored_folders', [])
+            folders_to_keep.update(ignored_folders)
+
+            delete_unknown_folders(token_request, folders_to_keep)
+            delete_api_token(request, token['id'], org_id=org_id)
+        except Exception:
+            logger.exception(f'Error when provisioning org {org["name"]}')
+            break
 
     logger.info(f'Time to complete: {time.time() - start}')