refactor to work logrotate files

api.py

@@ -63,7 +63,8 @@ def getSimpleDict(aux):
 def isValidDate(date_text):
       try:
          date.fromisoformat(date_text)
-      except ValueError:
+      except ValueError as e:
+         print(e)
          return False
       return True
 
@@ -73,6 +74,13 @@ def clearDateString(str):
         str = str.replace(elem, '')
     return str
 
+def storeParsedDay(path, data):
+   try: 
+      with open(path, "w") as outfile:
+         outfile.write(json.dumps(data))
+   except FileNotFoundError as e:
+      print(e)
+
 # Log will be parsed using predefined format
 # %(addr)|[%(ctime)]|%(method)|%(uri)|%(uagent)|%(referer)
 # target result is array like: 
@@ -133,7 +141,7 @@ def parseLog(lines,criteria):
             if rowGET['idp']:
                 request_param['idp'] += 1
 
-                if ('idp' in criteria and criteria['idp'] == rowGET['idp']) or 'idp' not in criteria:
+                if ('idp' in criteria and criteria['idp'] in rowGET['idp']) or 'idp' not in criteria:
                     if rowGET['idp'] not in idp.keys():
                         idp[rowGET['idp']] = 0
                     idp[rowGET['idp']] += 1
@@ -141,7 +149,7 @@ def parseLog(lines,criteria):
             if rowGET['reg_auth']:
                 request_param['reg_auth'] += 1
                 
-                if ('reg_auth' in criteria and criteria['reg_auth'] == rowGET['reg_auth']) or 'reg_auth' not in criteria:
+                if ('reg_auth' in criteria and criteria['reg_auth'] in rowGET['reg_auth']) or 'reg_auth' not in criteria:
                     if rowGET['reg_auth'] not in reg_auth.keys():
                         reg_auth[rowGET['reg_auth']] = 0
                     reg_auth[rowGET['reg_auth']] += 1
@@ -154,7 +162,7 @@ def parseLog(lines,criteria):
                 'reg_auth' : reg_auth
             }
         
-    return json.dumps(result)
+    return result
 
 # Parse URL from log line. Used to get only idp and reg_auth. 
 def parseReqURL(url):
@@ -396,26 +404,31 @@ class WebData(Resource):
       list_feds = get_list_from_url(e_p.ECCS_LISTFEDSURL, e_p.ECCS_LISTFEDSFILE)
       regAuthDict = get_reg_auth_dict(list_feds)
 
-      file_path = f"{e_p.ECCS_OUTPUTDIR}/{e_p.ECCS_RESULTSLOG}"
+      file_path = f"{e_p.ECCS_LOGSDIR}/eccs-uwsgi-req.log" # will this name be moved to properties definer file ?
       criteria = {}
       criteria['date_from'] = criteria['date_to'] = e_p.DAY
-      eccsLogRotated = False
+      eccsLogRotated = True
       in_data = request.args
 
-      if ('date_from' in in_data and isValidDate(in_data['date_from'])):
-         criteria['date_from'] = in_data['date_from']
-         if ('date_to' not in in_data):
-            criteria['date_to'] = criteria['date_from'] + timedelta(days=30)
+      if ('dateFrom' in in_data and isValidDate(in_data['dateFrom'])):
+         criteria['date_from'] = in_data['dateFrom']
+         if ('dateTo' not in in_data):
+            criteria['date_to'] = (datetime.strptime(criteria['date_from'], '%Y-%m-%d') + timedelta(days=30)).strftime('%Y-%m-%d')
+            
+            if datetime.today().strftime('%Y-%m-%d') < criteria['date_to']:
+               diff = (datetime.strptime(criteria['date_to'], '%Y-%m-%d') - datetime.today()).days
+               criteria['date_from'] = (datetime.strptime(criteria['date_from'], '%Y-%m-%d') - timedelta(days=diff)).strftime('%Y-%m-%d')
+               criteria['date_to'] = datetime.today().strftime('%Y-%m-%d')
       
-      if ('date_to' in in_data and isValidDate(in_data['date_to'])):
-         criteria['date_to'] = in_data['date_to']
-         if ('date_from' not in in_data):
-            criteria['date_from'] = criteria['date_to'] - timedelta(days=30)
+      if ('dateTo' in in_data and isValidDate(in_data['dateTo'])):
+         criteria['date_to'] = in_data['dateTo']
+         if ('dateFrom' not in in_data):
+            criteria['date_from'] = (datetime.strptime(criteria['date_to'], '%Y-%m-%d') + timedelta(days=30)).strftime('%Y-%m-%d')
       
-      if ('request_source' in in_data and in_data['request_source'] == 'divided'):
+      if ('requestSource' in in_data and in_data['requestSource'] == 'divided'):
          criteria['request_source'] = 'divided'
          
-      if ('reg_auth' in in_data and in_data['reg_auth'] in regAuthDict):
+      if ('regAuth' in in_data and in_data['regAuth'] in regAuthDict):
          criteria['reg_auth'] = in_data['reg_auth']
      
       if ('idp' in in_data):
@@ -423,34 +436,50 @@ class WebData(Resource):
 
       # here I have to parse eccs-log file
       lines = []
-      results = []
+      results = {}
       cur_date = criteria['date_from']
       
       if eccsLogRotated == True:
          while cur_date <= criteria['date_to']:
-            file_path = f"{e_p.ECCS_OUTPUTDIR}/eccs_{cur_date}.log"
+            json_data = {}
+            tmpDate = datetime.strptime(cur_date, '%Y-%m-%d').strftime('%Y%m%d')
+            file_path = f"{e_p.ECCS_LOGSDIR}/eccs-uwsgi-req.log-{tmpDate}"
+            json_file_path = f"{e_p.ECCS_DIR}/parsed/eccs-uwsgi-req-json-{tmpDate}"
             
             try:
-               with open(file_path,"r",encoding="utf-8") as fo:
-                  lines = fo.readlines()
+               f = open(json_file_path)
+               json_data = json.load(f)
             
-            except FileNotFoundError as e:
-               results[cur_date] = []
+            except (ValueError, FileNotFoundError) as e:
+               #print(e)
+               pass
+            
+            if len(json_data) == 0:
+               try:
+                  with open(file_path,"r",encoding="utf-8") as fo:
+                     lines = fo.readlines()
+                  json_data = parseLog(lines, criteria)
+                  storeParsedDay(json_file_path, json_data)
+
+               except FileNotFoundError as e:
+                  #print(e)
+                  pass
                
-            results = parseLog(lines, criteria)
-
-            cur_date += timedelta(days=1)
+            results.update(json_data)
+            
+            cur_date = (datetime.strptime(cur_date, '%Y-%m-%d') + timedelta(days=1)).strftime('%Y-%m-%d')
       else:
             try:
                with open(file_path,"r",encoding="utf-8") as fo:
                   lines = fo.readlines()
             
             except FileNotFoundError as e:
-               results = []
+               print(e)
+               results = {}
             
             results = parseLog(lines, criteria)
       
-      return results
+      return json.dumps(results)
 
 # /api/
 class Help(Resource):