python parse log fixed

api.py

@@ -67,6 +67,12 @@ def isValidDate(date_text):
          return False
       return True
 
+def clearDateString(str):
+    del_char = {"[","]"}
+    for elem in del_char:
+        str = str.replace(elem, '')
+    return str
+
 # Log will be parsed using predefined format
 # %(addr)|[%(ctime)]|%(method)|%(uri)|%(uagent)|%(referer)
 # target result is array like: 
@@ -95,39 +101,60 @@ def isValidDate(date_text):
 # ],
 #]
 def parseLog(lines,criteria):
-   result = {}
-   request_count = {'web':0,'api':0}
-   request_param = {'idp':0,'reg_auth':0}
-   idp = {}
-   reg_auth = {}
+    result = {}
+    request_count = {'web':0,'api':0}
+    request_param = {'idp':0,'reg_auth':0}
+    idp = {}
+    reg_auth = {}
+    rowDateCheck = ''
    
-   for line in lines:
-      row = line.split('|')
-      rowDate = datetime.strptime(row[1], '%a %b %d %H:%M:%S %Y').date()
-      
-      if criteria['datefrom'] <= rowDate <= criteria['dateto']:
-         rowGET = parseReqURL(row[3])
-         rowRequestSource = ('web' if len(row[5]) > 0 else 'api')
-         request_count['web'] += (1 if rowRequestSource == 'web' else 0)
-         request_count['api'] += (1 if rowRequestSource == 'api' else 0)
-         
-         if rowGET['idp']:
-            request_param['idp'] += 1
-            if ('idp' in criteria and criteria['idp'] == rowGET['idp']) or 'idp' not in criteria:
-               idp.append(rowGET['idp'])
+    for line in lines:
+        row = line.split('|')
+        
+        if len(row) <= 1:
+            continue
+        
+        rowDate = datetime.strptime(clearDateString(row[1]), '%a %b %d %H:%M:%S %Y').strftime('%Y-%m-%d')
          
-         if rowGET['reg_auth']:
-            request_param['reg_auth'] += 1
-            if ('reg_auth' in criteria and criteria['reg_auth'] == rowGET['reg_auth']) or 'reg_auth' not in criteria:
-               reg_auth.append(rowGET['reg_auth'])
-
-   result[rowDate] = {
-      'request_count' : request_count,
-      'request_param' : request_param,
-      'idp' : idp,
-      'reg_auth' : reg_auth
-   }
-   return result
+        if ('date_from' in criteria and 'date_to' in criteria and criteria['date_from'] <= rowDate <= criteria['date_to']) or ('date_from' not in criteria and 'date_to' not in criteria and rowDate):
+            
+            if rowDateCheck != rowDate:
+                request_count = {'web':0,'api':0}
+                request_param = {'idp':0,'reg_auth':0}
+                idp = {}
+                reg_auth = {}
+                rowDateCheck = rowDate
+                        
+            rowGET = parseReqURL(row[3])
+            rowRequestSource = ('web' if len(row[5]) > 5 else 'api')
+            request_count['web'] += (1 if rowRequestSource == 'web' else 0)
+            request_count['api'] += (1 if rowRequestSource == 'api' else 0)
+            
+            if rowGET['idp']:
+                request_param['idp'] += 1
+
+                if ('idp' in criteria and criteria['idp'] == rowGET['idp']) or 'idp' not in criteria:
+                    if rowGET['idp'] not in idp.keys():
+                        idp[rowGET['idp']] = 0
+                    idp[rowGET['idp']] += 1
+                   
+            if rowGET['reg_auth']:
+                request_param['reg_auth'] += 1
+                
+                if ('reg_auth' in criteria and criteria['reg_auth'] == rowGET['reg_auth']) or 'reg_auth' not in criteria:
+                    if rowGET['reg_auth'] not in reg_auth.keys():
+                        reg_auth[rowGET['reg_auth']] = 0
+                    reg_auth[rowGET['reg_auth']] += 1
+
+            result[rowDate] = {
+                'request_count' : request_count,
+                'request_param' : request_param,
+                'request_uniq' : {'idp':len(idp),'reg_auth':len(reg_auth)},
+                'idp' : idp,
+                'reg_auth' : reg_auth
+            }
+        
+    return json.dumps(result)
 
 # Parse URL from log line. Used to get only idp and reg_auth. 
 def parseReqURL(url):
@@ -423,7 +450,7 @@ class WebData(Resource):
             
             results = parseLog(lines, criteria)
       
-      return jsonify(results)
+      return results
 
 # /api/
 class Help(Resource):