refactor for delayed logrotate

api.py

@@ -123,6 +123,10 @@ def parseLog(lines,criteria):
             continue
         
         rowDate = datetime.strptime(clearDateString(row[1]), '%a %b %d %H:%M:%S %Y').strftime('%Y-%m-%d')
+        
+      # check for entries to be in range of date of rotated log  
+        if 'prev_date' in criteria and criteria['date_from'] != rowDate:
+            continue
          
         if ('date_from' in criteria and 'date_to' in criteria and criteria['date_from'] <= rowDate <= criteria['date_to']) or ('date_from' not in criteria and 'date_to' not in criteria and rowDate):
             
@@ -178,6 +182,28 @@ def parseReqURL(url):
    
    return result
 
+def filterParsedData(json_data,criteria):
+   # idp = {}
+   # reg_auth = {}
+   
+   # if 'idp' in criteria and criteria['idp'] in json_data['idp'].keys():
+   #    for entry in json_data[criteria['prev_date']]['idp']:
+   #       # if criteria['idp'] == entry.key():
+   #       if entry.key().find(criteria['idp']) >= 0:
+   #          idp[entry.key()] = entry
+   # # search substring
+   # if 'reg_auth' in criteria and criteria['reg_auth'] in json_data['reg_auth'].keys():
+   #    for entry in json_data[criteria['reg_auth']]:
+   #       if criteria['reg_auth'] == entry.key():
+   #             reg_auth[criteria['reg_auth']] = entry
+   
+   # json_data[criteria['prev_date']] = {
+   #       'request_uniq' : {'idp':len(idp),'reg_auth':len(reg_auth)},
+   #       'idp' : idp,
+   #       'reg_auth' : reg_auth
+   # }
+   return json_data
+
 ### Classes
 
 # /api/test
@@ -408,6 +434,8 @@ class WebData(Resource):
       criteria = {}
       criteria['date_from'] = criteria['date_to'] = e_p.DAY
       eccsLogRotated = True
+      useParsedFile = True
+      
       in_data = request.args
 
       if ('dateFrom' in in_data and isValidDate(in_data['dateFrom'])):
@@ -434,51 +462,59 @@ class WebData(Resource):
       if ('idp' in in_data):
          criteria['idp'] = in_data['idp']
 
-      # here I have to parse eccs-log file
       lines = []
       results = {}
       cur_date = criteria['date_from']
-      
+
       if eccsLogRotated == True:
          while cur_date <= criteria['date_to']:
             json_data = {}
+            criteria['prev_date'] = (datetime.strptime(cur_date, '%Y-%m-%d') - timedelta(days=1)).strftime('%Y-%m-%d')
             tmpDate = datetime.strptime(cur_date, '%Y-%m-%d').strftime('%Y%m%d')
             file_path = f"{e_p.ECCS_LOGSDIR}/eccs-uwsgi-req.log-{tmpDate}"
-            json_file_path = f"{e_p.ECCS_DIR}/parsed/eccs-uwsgi-req-json-{tmpDate}"
-            
-            try:
-               f = open(json_file_path)
-               json_data = json.load(f)
-            
-            except (ValueError, FileNotFoundError) as e:
-               #print(e)
-               pass
-            
+
+            if useParsedFile == True:
+               tmpDate = (datetime.strptime(cur_date, '%Y-%m-%d') - timedelta(days=1)).strftime('%Y%m%d')
+               json_file_path = f"{e_p.ECCS_DIR}/parsed/eccs-uwsgi-req-json-{tmpDate}"
+               try:
+                  f = open(json_file_path)
+                  json_data = json.load(f)
+
+               except (ValueError, FileNotFoundError) as e:
+                  #print(e)
+                  pass
+               
+               if len(json_data) > 0 and ('idp' in criteria or 'reg_auth' in criteria):
+                  json_data = filterParsedData(json_data, criteria)
+
             if len(json_data) == 0:
                try:
                   with open(file_path,"r",encoding="utf-8") as fo:
                      lines = fo.readlines()
+
                   json_data = parseLog(lines, criteria)
-                  storeParsedDay(json_file_path, json_data)
+
+                  if useParsedFile == True and 'idp' not in criteria and 'reg_auth' not in criteria:
+                     storeParsedDay(json_file_path, json_data)
 
                except FileNotFoundError as e:
                   #print(e)
                   pass
-               
+
             results.update(json_data)
-            
+
             cur_date = (datetime.strptime(cur_date, '%Y-%m-%d') + timedelta(days=1)).strftime('%Y-%m-%d')
       else:
             try:
                with open(file_path,"r",encoding="utf-8") as fo:
                   lines = fo.readlines()
-            
+
             except FileNotFoundError as e:
                print(e)
                results = {}
-            
+
             results = parseLog(lines, criteria)
-      
+
       return json.dumps(results)
 
 # /api/