clear log keys in a separate thread

inventory_provider/tasks/monitor.py

@@ -114,14 +114,18 @@ def run():
         t['thread'].join()
 
 
-def clear_joblog(r):
+def clear_joblog(r, keys_read_event=None):
     """
     :param r: connection to a redis database
+    :param keys_read_event: optional event to signal after all keys are read
     :return:
     """
     rp = r.pipeline()
     for key in r.scan_iter('joblog:*'):
         rp.delete(key)
+    if keys_read_event:
+        assert isinstance(keys_read_event, threading.Event)  # sanity
+        keys_read_event.set()
     rp.execute()
 
 

inventory_provider/tasks/worker.py

 import json
 import logging
 import os
+import threading
 import time
 
 from redis.exceptions import RedisError
@@ -554,7 +555,18 @@ def launch_refresh_cache_all(config):
         _erase_next_db(config)
         update_latch_status(config, pending=True)
 
-        monitor.clear_joblog(get_current_redis(config))
+        # call monitor.clear_joblog in a thread, since
+        # deletion might be slow and can be done in parallel
+        def _clear_log_proc(wait_event):
+            monitor.clear_joblog(get_current_redis(config), wait_event)
+
+        keys_captured_event = threading.Event()
+        threading.Thread(
+            target=_clear_log_proc,
+            args=[keys_captured_event]).start()
+        if not keys_captured_event.wait(timeout=60.0):
+            # wait a reasonable time
+            logging.error('timed out waiting for log keys to be read')
 
         # first batch of subtasks: refresh cached opsdb data
         subtasks = [