diff --git a/api.py b/api.py
index 021113207e2c2ebaf8a57fdb7014704de91c66d2..a2cbe2fdba9f60d2a0a6aff734e403967315cfc3 100755
--- a/api.py
+++ b/api.py
@@ -9,7 +9,8 @@ from flask import Flask, request, jsonify
from flask_restful import Resource, Api
from utils import get_logger, get_list_from_url, get_reg_auth_dict, generate_login_url
from markupsafe import escape
-from datetime import timedelta, date
+from datetime import timedelta, datetime, date
+from urllib.parse import urlparse, parse_qs
app = Flask(__name__)
api = Api(app)
@@ -66,6 +67,82 @@ def isValidDate(date_text):
return False
return True
+# Log will be parsed using predefined format
+# %(addr)|[%(ctime)]|%(method)|%(uri)|%(uagent)|%(referer)
+# target result is array like:
+# [
+# "YYYY-MM-DD" => [
+# "request_count" => [
+# web => 100,
+# api => 200
+# ],
+# "requested_param" => [
+# idp => 100,
+# reg_auth => 200
+# ],
+# idp => [
+# "xx" => [
+# "api" => 1,
+# "web" => 10
+# ],
+# ],
+# reg_auth => [
+# "xx" => [
+# "api" => 1,
+# "web" => 10
+# ],
+# ],
+# ],
+#]
+def parseLog(lines,criteria):
+ result = {}
+ request_count = {'web':0,'api':0}
+ request_param = {'idp':0,'reg_auth':0}
+ idp = {}
+ reg_auth = {}
+
+ for line in lines:
+ row = line.split('|')
+ rowDate = datetime.strptime(row[1], '%a %b %d %H:%M:%S %Y').date()
+
+ if criteria['datefrom'] <= rowDate <= criteria['dateto']:
+ rowGET = parseReqURL(row[3])
+ rowRequestSource = ('web' if len(row[5]) > 0 else 'api')
+ request_count['web'] += (1 if rowRequestSource == 'web' else 0)
+ request_count['api'] += (1 if rowRequestSource == 'api' else 0)
+
+ if rowGET['idp']:
+ request_param['idp'] += 1
+ if ('idp' in criteria and criteria['idp'] == rowGET['idp']) or 'idp' not in criteria:
+ idp.append(rowGET['idp'])
+
+ if rowGET['reg_auth']:
+ request_param['reg_auth'] += 1
+ if ('reg_auth' in criteria and criteria['reg_auth'] == rowGET['reg_auth']) or 'reg_auth' not in criteria:
+ reg_auth.append(rowGET['reg_auth'])
+
+ result[rowDate] = {
+ 'request_count' : request_count,
+ 'request_param' : request_param,
+ 'idp' : idp,
+ 'reg_auth' : reg_auth
+ }
+ return result
+
+# Parse URL from log line. Used to get only idp and reg_auth.
+def parseReqURL(url):
+ result = {'idp':False,'reg_auth':False}
+ parsed_url = urlparse(url)
+ captured_value = parse_qs(parsed_url.query)
+
+ if 'idp' in captured_value:
+ result['idp'] = captured_value['idp'][0]
+
+ if 'reg_auth' in captured_value:
+ result['reg_auth'] = captured_value['reg_auth'][0]
+
+ return result
+
### Classes
# /api/test
@@ -293,81 +370,60 @@ class WebData(Resource):
regAuthDict = get_reg_auth_dict(list_feds)
file_path = f"{e_p.ECCS_OUTPUTDIR}/{e_p.ECCS_RESULTSLOG}"
- date_from = date_to = e_p.DAY
- reg_auth = None
- eccsDataTable = False
+ criteria = {}
+ criteria['date_from'] = criteria['date_to'] = e_p.DAY
+ eccsLogRotated = False
in_data = request.args
if ('date_from' in in_data and isValidDate(in_data['date_from'])):
- date_from = in_data['date_from']
+ criteria['date_from'] = in_data['date_from']
if ('date_to' not in in_data):
- date_to = date_from + timedelta(days=30)
+ criteria['date_to'] = criteria['date_from'] + timedelta(days=30)
if ('date_to' in in_data and isValidDate(in_data['date_to'])):
- date_to = in_data['date_to']
+ criteria['date_to'] = in_data['date_to']
if ('date_from' not in in_data):
- date_from = date_to - timedelta(days=30)
+ criteria['date_from'] = criteria['date_to'] - timedelta(days=30)
if ('request_source' in in_data and in_data['request_source'] == 'divided'):
- request_source = 'divided'
+ criteria['request_source'] = 'divided'
if ('reg_auth' in in_data and in_data['reg_auth'] in regAuthDict):
- reg_auth = in_data['reg_auth']
-
- if ('status' in in_data and in_data['status'].upper() in ['OK','DISABLED','ERROR','UNKNOWN']):
- status = in_data['status'].upper()
-
+ criteria['reg_auth'] = in_data['reg_auth']
+
if ('idp' in in_data):
- idp = in_data['idp']
+ criteria['idp'] = in_data['idp']
# here I have to parse eccs-log file
- # target result is array like:
- # "YYYY-MM-DD" => [
- # "request_count" => [
- # web => 100,
- # api => 200
- # ],
- # "uniqIP" => [
- # web => 100,
- # api => 200
- # ],
- # "requested_param" => [
- # idp => 100,
- # reg_auth => 200
- # ],
- # idp => [
- # "xx" => [
- # "api" => 1,
- # "web" => 10
- # ],
- # ],
- # reg_auth => [
- # "xx" => [
- # "api" => 1,
- # "web" => 10
- # ],
- # ],
- # ]
-
-
lines = []
results = []
- cur_date = date_from
+ cur_date = criteria['date_from']
- while cur_date <= date_to:
- file_path = f"{e_p.ECCS_OUTPUTDIR}/eccs_{cur_date}.log"
-
- try:
- with open(file_path,"r",encoding="utf-8") as fo:
- lines = fo.readlines()
-
- except FileNotFoundError as e:
- results[cur_date] = []
+ if eccsLogRotated == True:
+ while cur_date <= criteria['date_to']:
+ file_path = f"{e_p.ECCS_OUTPUTDIR}/eccs_{cur_date}.log"
-
+ try:
+ with open(file_path,"r",encoding="utf-8") as fo:
+ lines = fo.readlines()
+
+ except FileNotFoundError as e:
+ results[cur_date] = []
+
+ results = parseLog(lines, criteria)
+
+ cur_date += timedelta(days=1)
+ else:
+ try:
+ with open(file_path,"r",encoding="utf-8") as fo:
+ lines = fo.readlines()
+ except FileNotFoundError as e:
+ results = []
- cur_date += timedelta(days=1)
+ results = parseLog(lines, criteria)
+
+ return jsonify(results)
# /api/
class Help(Resource):