added web request handler

api.py

@@ -9,7 +9,8 @@ from flask import Flask, request, jsonify
 from flask_restful import Resource, Api
 from utils import get_logger, get_list_from_url, get_reg_auth_dict, generate_login_url
 from markupsafe import escape
-from datetime import timedelta, date
+from datetime import timedelta, datetime, date
+from urllib.parse import urlparse, parse_qs
 
 app = Flask(__name__)
 api = Api(app)
@@ -66,6 +67,82 @@ def isValidDate(date_text):
          return False
       return True
 
+# Log will be parsed using predefined format
+# %(addr)|[%(ctime)]|%(method)|%(uri)|%(uagent)|%(referer)
+# target result is array like: 
+# [
+# "YYYY-MM-DD" => [
+#     "request_count" => [
+#        web => 100,
+#        api => 200 
+#     ],
+#     "requested_param" => [
+#        idp => 100,
+#        reg_auth => 200 
+#     ],
+#     idp => [
+#        "xx" => [
+#           "api" => 1,
+#           "web" => 10
+#        ],
+#     ],
+#     reg_auth => [
+#        "xx" => [
+#           "api" => 1,
+#           "web" => 10
+#        ],
+#      ],
+# ],
+#]
+def parseLog(lines,criteria):
+   result = {}
+   request_count = {'web':0,'api':0}
+   request_param = {'idp':0,'reg_auth':0}
+   idp = {}
+   reg_auth = {}
+   
+   for line in lines:
+      row = line.split('|')
+      rowDate = datetime.strptime(row[1], '%a %b %d %H:%M:%S %Y').date()
+      
+      if criteria['datefrom'] <= rowDate <= criteria['dateto']:
+         rowGET = parseReqURL(row[3])
+         rowRequestSource = ('web' if len(row[5]) > 0 else 'api')
+         request_count['web'] += (1 if rowRequestSource == 'web' else 0)
+         request_count['api'] += (1 if rowRequestSource == 'api' else 0)
+         
+         if rowGET['idp']:
+            request_param['idp'] += 1
+            if ('idp' in criteria and criteria['idp'] == rowGET['idp']) or 'idp' not in criteria:
+               idp.append(rowGET['idp'])
+         
+         if rowGET['reg_auth']:
+            request_param['reg_auth'] += 1
+            if ('reg_auth' in criteria and criteria['reg_auth'] == rowGET['reg_auth']) or 'reg_auth' not in criteria:
+               reg_auth.append(rowGET['reg_auth'])
+
+   result[rowDate] = {
+      'request_count' : request_count,
+      'request_param' : request_param,
+      'idp' : idp,
+      'reg_auth' : reg_auth
+   }
+   return result
+
+# Parse URL from log line. Used to get only idp and reg_auth. 
+def parseReqURL(url):
+   result = {'idp':False,'reg_auth':False}
+   parsed_url = urlparse(url)
+   captured_value = parse_qs(parsed_url.query)
+
+   if 'idp' in captured_value:
+      result['idp'] = captured_value['idp'][0]
+   
+   if 'reg_auth' in captured_value:
+      result['reg_auth'] = captured_value['reg_auth'][0]
+   
+   return result
+
 ### Classes
 
 # /api/test
@@ -293,81 +370,60 @@ class WebData(Resource):
       regAuthDict = get_reg_auth_dict(list_feds)
 
       file_path = f"{e_p.ECCS_OUTPUTDIR}/{e_p.ECCS_RESULTSLOG}"
-      date_from = date_to = e_p.DAY
-      reg_auth = None
-      eccsDataTable = False
+      criteria = {}
+      criteria['date_from'] = criteria['date_to'] = e_p.DAY
+      eccsLogRotated = False
       in_data = request.args
 
       if ('date_from' in in_data and isValidDate(in_data['date_from'])):
-         date_from = in_data['date_from']
+         criteria['date_from'] = in_data['date_from']
          if ('date_to' not in in_data):
-            date_to = date_from + timedelta(days=30)
+            criteria['date_to'] = criteria['date_from'] + timedelta(days=30)
       
       if ('date_to' in in_data and isValidDate(in_data['date_to'])):
-         date_to = in_data['date_to']
+         criteria['date_to'] = in_data['date_to']
          if ('date_from' not in in_data):
-            date_from = date_to - timedelta(days=30)
+            criteria['date_from'] = criteria['date_to'] - timedelta(days=30)
       
       if ('request_source' in in_data and in_data['request_source'] == 'divided'):
-         request_source = 'divided'
+         criteria['request_source'] = 'divided'
          
       if ('reg_auth' in in_data and in_data['reg_auth'] in regAuthDict):
-         reg_auth = in_data['reg_auth']
-      
-      if ('status' in in_data and in_data['status'].upper() in ['OK','DISABLED','ERROR','UNKNOWN']):
-         status = in_data['status'].upper()
-      
+         criteria['reg_auth'] = in_data['reg_auth']
+     
       if ('idp' in in_data):
-         idp = in_data['idp']
+         criteria['idp'] = in_data['idp']
 
       # here I have to parse eccs-log file
-      # target result is array like: 
-      # "YYYY-MM-DD" => [
-      #     "request_count" => [
-      #        web => 100,
-      #        api => 200 
-      #     ],
-      #     "uniqIP" => [
-      #        web => 100,
-      #        api => 200 
-      #     ],
-      #     "requested_param" => [
-      #        idp => 100,
-      #        reg_auth => 200 
-      #     ],
-      #     idp => [
-      #        "xx" => [
-      #           "api" => 1,
-      #           "web" => 10
-      #        ],
-      #     ],
-      #     reg_auth => [
-      #        "xx" => [
-      #           "api" => 1,
-      #           "web" => 10
-      #        ],
-      #      ],
-      # ]
-      
-      
       lines = []
       results = []
-      cur_date = date_from
+      cur_date = criteria['date_from']
       
-      while cur_date <= date_to:
-         file_path = f"{e_p.ECCS_OUTPUTDIR}/eccs_{cur_date}.log"
-         
-         try:
-            with open(file_path,"r",encoding="utf-8") as fo:
-               lines = fo.readlines()
-         
-         except FileNotFoundError as e:
-            results[cur_date] = []
+      if eccsLogRotated == True:
+         while cur_date <= criteria['date_to']:
+            file_path = f"{e_p.ECCS_OUTPUTDIR}/eccs_{cur_date}.log"
             
-         
+            try:
+               with open(file_path,"r",encoding="utf-8") as fo:
+                  lines = fo.readlines()
+            
+            except FileNotFoundError as e:
+               results[cur_date] = []
+               
+            results = parseLog(lines, criteria)
+
+            cur_date += timedelta(days=1)
+      else:
+            try:
+               with open(file_path,"r",encoding="utf-8") as fo:
+                  lines = fo.readlines()
             
+            except FileNotFoundError as e:
+               results = []
             
-         cur_date += timedelta(days=1)      
+            results = parseLog(lines, criteria)
+      
+      return jsonify(results)
 
 # /api/
 class Help(Resource):