Skip to content
Snippets Groups Projects
Commit 4a1e860a authored by Valentin Pocotilenco's avatar Valentin Pocotilenco
Browse files

refactoring and fixes

parent 4c1d04e7
Branches
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
api.py
+ 50
− 45
View file @ 4a1e860a
...@@ -10,7 +10,7 @@ from flask_restful import Resource, Api ...@@ -10,7 +10,7 @@ from flask_restful import Resource, Api
from utils import get_logger, get_list_from_url, get_reg_auth_dict, generate_login_url from utils import get_logger, get_list_from_url, get_reg_auth_dict, generate_login_url
from markupsafe import escape from markupsafe import escape
from datetime import timedelta, datetime, date from datetime import timedelta, datetime, date
from urllib.parse import urlparse, parse_qs from urllib.parse import urlparse, parse_qs, unquote
app = Flask(__name__) app = Flask(__name__)
api = Api(app) api = Api(app)
...@@ -125,7 +125,7 @@ def parseLog(lines,criteria): ...@@ -125,7 +125,7 @@ def parseLog(lines,criteria):
rowDate = datetime.strptime(clearDateString(row[1]), '%a %b %d %H:%M:%S %Y').strftime('%Y-%m-%d') rowDate = datetime.strptime(clearDateString(row[1]), '%a %b %d %H:%M:%S %Y').strftime('%Y-%m-%d')
# check for entries to be in range of date of rotated log # check for entries to be in range of date of rotated log
if 'prev_date' in criteria and criteria['date_from'] != rowDate: if 'cur_date' in criteria and criteria['cur_date'] != rowDate:
continue continue
if ('date_from' in criteria and 'date_to' in criteria and criteria['date_from'] <= rowDate <= criteria['date_to']) or ('date_from' not in criteria and 'date_to' not in criteria and rowDate): if ('date_from' in criteria and 'date_to' in criteria and criteria['date_from'] <= rowDate <= criteria['date_to']) or ('date_from' not in criteria and 'date_to' not in criteria and rowDate):
...@@ -138,25 +138,25 @@ def parseLog(lines,criteria): ...@@ -138,25 +138,25 @@ def parseLog(lines,criteria):
rowDateCheck = rowDate rowDateCheck = rowDate
rowGET = parseReqURL(row[3]) rowGET = parseReqURL(row[3])
rowRequestSource = ('web' if len(row[5]) > 5 else 'api')
request_count['web'] += (1 if rowRequestSource == 'web' else 0) if ('idp' not in criteria and 'reg_auth' not in criteria) or ('idp' in criteria and rowGET['idp'] and rowGET['idp'].find(criteria['idp']) >= 0) or ('reg_auth' in criteria and rowGET['reg_auth'] and rowGET['reg_auth'].find(criteria['reg_auth']) >= 0):
request_count['api'] += (1 if rowRequestSource == 'api' else 0) rowRequestSource = ('web' if len(row[5]) > 5 else 'api')
request_count['web'] += (1 if rowRequestSource == 'web' else 0)
request_count['api'] += (1 if rowRequestSource == 'api' else 0)
if rowGET['idp']: if rowGET['idp']:
request_param['idp'] += 1 if ('idp' in criteria and rowGET['idp'].find(criteria['idp']) >= 0) or 'idp' not in criteria:
if ('idp' in criteria and criteria['idp'] in rowGET['idp']) or 'idp' not in criteria:
if rowGET['idp'] not in idp.keys(): if rowGET['idp'] not in idp.keys():
idp[rowGET['idp']] = 0 idp[rowGET['idp']] = 0
idp[rowGET['idp']] += 1 idp[rowGET['idp']] += 1
request_param['idp'] += 1
if rowGET['reg_auth']: if rowGET['reg_auth']:
request_param['reg_auth'] += 1 if ('reg_auth' in criteria and rowGET['reg_auth'].find(criteria['reg_auth']) >= 0) or 'reg_auth' not in criteria:
if ('reg_auth' in criteria and criteria['reg_auth'] in rowGET['reg_auth']) or 'reg_auth' not in criteria:
if rowGET['reg_auth'] not in reg_auth.keys(): if rowGET['reg_auth'] not in reg_auth.keys():
reg_auth[rowGET['reg_auth']] = 0 reg_auth[rowGET['reg_auth']] = 0
reg_auth[rowGET['reg_auth']] += 1 reg_auth[rowGET['reg_auth']] += 1
request_param['reg_auth'] += 1
result[rowDate] = { result[rowDate] = {
'request_count' : request_count, 'request_count' : request_count,
...@@ -183,26 +183,34 @@ def parseReqURL(url): ...@@ -183,26 +183,34 @@ def parseReqURL(url):
return result return result
def filterParsedData(json_data,criteria): def filterParsedData(json_data,criteria):
# idp = {} idp = {}
# reg_auth = {} reg_auth = {}
out_data = {}
# if 'idp' in criteria and criteria['idp'] in json_data['idp'].keys(): if criteria['cur_date'] in json_data:
# for entry in json_data[criteria['prev_date']]['idp']: if 'idp' in criteria and len(criteria['idp']):
# # if criteria['idp'] == entry.key(): for idpEntry in json_data[criteria['cur_date']]['idp'].keys():
# if entry.key().find(criteria['idp']) >= 0: if idpEntry.find(criteria['idp']) >= 0:
# idp[entry.key()] = entry idp[criteria['idp']] = json_data[criteria['cur_date']]['idp'][criteria['idp']]
# # search substring else:
# if 'reg_auth' in criteria and criteria['reg_auth'] in json_data['reg_auth'].keys(): idp = json_data[criteria['cur_date']]['idp']
# for entry in json_data[criteria['reg_auth']]:
# if criteria['reg_auth'] == entry.key(): if 'reg_auth' in criteria and len(criteria['reg_auth']):
# reg_auth[criteria['reg_auth']] = entry for regAuthEntry in json_data[criteria['cur_date']]['reg_auth'].keys():
if regAuthEntry.find(criteria['reg_auth']) >= 0:
reg_auth[criteria['reg_auth']] = json_data[criteria['cur_date']]['reg_auth'][criteria['reg_auth']]
else:
reg_auth = json_data[criteria['cur_date']]['reg_auth']
# json_data[criteria['prev_date']] = { out_data[criteria['cur_date']] = {
# 'request_uniq' : {'idp':len(idp),'reg_auth':len(reg_auth)}, 'request_count' : json_data[criteria['cur_date']]['request_count'],
# 'idp' : idp, 'request_param' : json_data[criteria['cur_date']]['request_param'],
# 'reg_auth' : reg_auth 'request_uniq' : {'idp':len(idp),'reg_auth':len(reg_auth)},
# } 'idp' : idp,
return json_data 'reg_auth' : reg_auth,
}
return out_data
### Classes ### Classes
...@@ -427,14 +435,12 @@ class FedStats(Resource): ...@@ -427,14 +435,12 @@ class FedStats(Resource):
# /api/webdata # /api/webdata
class WebData(Resource): class WebData(Resource):
def get(self): def get(self):
list_feds = get_list_from_url(e_p.ECCS_LISTFEDSURL, e_p.ECCS_LISTFEDSFILE)
regAuthDict = get_reg_auth_dict(list_feds)
file_path = f"{e_p.ECCS_LOGSDIR}/eccs-uwsgi-req.log" # will this name be moved to properties definer file ? file_path = f"{e_p.ECCS_LOGSDIR}/eccs-uwsgi-req.log" # will this name be moved to properties definer file ?
criteria = {} criteria = {}
criteria['date_from'] = criteria['date_to'] = e_p.DAY criteria['date_from'] = criteria['date_to'] = e_p.DAY
# TBM to config
eccsLogRotated = True eccsLogRotated = True
useParsedFile = True useParsedFile = False
in_data = request.args in_data = request.args
...@@ -455,12 +461,12 @@ class WebData(Resource): ...@@ -455,12 +461,12 @@ class WebData(Resource):
if ('requestSource' in in_data and in_data['requestSource'] == 'divided'): if ('requestSource' in in_data and in_data['requestSource'] == 'divided'):
criteria['request_source'] = 'divided' criteria['request_source'] = 'divided'
if ('regAuth' in in_data and in_data['regAuth'] in regAuthDict): if 'regAuth' in in_data:
criteria['reg_auth'] = in_data['reg_auth'] criteria['reg_auth'] = unquote(in_data['regAuth'])
if ('idp' in in_data): if 'idp' in in_data:
criteria['idp'] = in_data['idp'] criteria['idp'] = unquote(in_data['idp'].strip())
lines = [] lines = []
results = {} results = {}
...@@ -469,12 +475,12 @@ class WebData(Resource): ...@@ -469,12 +475,12 @@ class WebData(Resource):
if eccsLogRotated == True: if eccsLogRotated == True:
while cur_date <= criteria['date_to']: while cur_date <= criteria['date_to']:
json_data = {} json_data = {}
criteria['prev_date'] = (datetime.strptime(cur_date, '%Y-%m-%d') - timedelta(days=1)).strftime('%Y-%m-%d') criteria['logfile_date'] = (datetime.strptime(cur_date, '%Y-%m-%d') + timedelta(days=1)).strftime('%Y-%m-%d')
tmpDate = datetime.strptime(cur_date, '%Y-%m-%d').strftime('%Y%m%d') criteria['cur_date'] = cur_date
tmpDate = datetime.strptime(criteria['logfile_date'], '%Y-%m-%d').strftime('%Y%m%d')
file_path = f"{e_p.ECCS_LOGSDIR}/eccs-uwsgi-req.log-{tmpDate}" file_path = f"{e_p.ECCS_LOGSDIR}/eccs-uwsgi-req.log-{tmpDate}"
if useParsedFile == True: if useParsedFile == True:
tmpDate = (datetime.strptime(cur_date, '%Y-%m-%d') - timedelta(days=1)).strftime('%Y%m%d')
json_file_path = f"{e_p.ECCS_DIR}/parsed/eccs-uwsgi-req-json-{tmpDate}" json_file_path = f"{e_p.ECCS_DIR}/parsed/eccs-uwsgi-req-json-{tmpDate}"
try: try:
f = open(json_file_path) f = open(json_file_path)
...@@ -498,11 +504,10 @@ class WebData(Resource): ...@@ -498,11 +504,10 @@ class WebData(Resource):
storeParsedDay(json_file_path, json_data) storeParsedDay(json_file_path, json_data)
except FileNotFoundError as e: except FileNotFoundError as e:
#print(e) # print(e)
pass pass
results.update(json_data) results.update(json_data)
cur_date = (datetime.strptime(cur_date, '%Y-%m-%d') + timedelta(days=1)).strftime('%Y-%m-%d') cur_date = (datetime.strptime(cur_date, '%Y-%m-%d') + timedelta(days=1)).strftime('%Y-%m-%d')
else: else:
try: try:
......
web/index.php
+ 0
− 1
View file @ 4a1e860a
...@@ -111,7 +111,6 @@ $data['check_result'] = htmlspecialchars($_GET["check_result"]); ...@@ -111,7 +111,6 @@ $data['check_result'] = htmlspecialchars($_GET["check_result"]);
var check_result = "<?php echo $data['check_result'] ?>"; var check_result = "<?php echo $data['check_result'] ?>";
</script> </script>
<script type="text/javascript" src="eccs.js" /></script> <script type="text/javascript" src="eccs.js" /></script>
<script type="text/javascript" src="dataX.js" /></script>
<script type="text/javascript" src="statistics.js" /></script> <script type="text/javascript" src="statistics.js" /></script>
</div> <!-- END eccs-central --> </div> <!-- END eccs-central -->
<div id="statisticsModal" class="modal"> <div id="statisticsModal" class="modal">
......
web/statistics.js
+ 64
− 20
View file @ 4a1e860a
var statChart = {}; var statChart = {};
var baseURL = window.location.protocol + "//" + window.location.host; var baseURL = window.location.protocol + "//" + window.location.host + "/eccs";
function getStatistics() { function getStatistics() {
showLoading();
let checkDate = $.datepicker.formatDate("yy-mm-dd", $('#datepicker').datepicker().datepicker('getDate')); let reqData = {}
reqData['dateFrom'] = $.datepicker.formatDate("yy-mm-dd", $('#datepicker').datepicker().datepicker('getDate'));
if ($("#eccstable thead th:last-child input").val() != '') {
reqData['regAuth'] = $("#eccstable thead th:last-child input").val();
}
if ($("#eccstable thead th:nth-last-child(2) input").val() != '') {
reqData['idp'] = $("#eccstable thead th:nth-last-child(2) input").val();
}
$.ajax({ $.ajax({
url: baseURL + "/dataX.php", url: baseURL + "/api/webdata",
type : 'POST', type : 'GET',
data : { contentType: "application/json",
dateFrom : checkDate data : reqData,
},
success: function(result) { success: function(result) {
drawChart(result) drawChart(result)
}, },
error: function (jqXHR,error,thrownError) { error: function (jqXHR,error,thrownError) {
showErr(); showMessage("Something happened. Can't retrieve data");
} }
}); });
} }
function showErr() { function showMessage(text) {
hideChart();
hideLoading();
$("#statisticsModal .chart-error span").text(text)
$("#statisticsModal .chart-error").css('display', 'block') $("#statisticsModal .chart-error").css('display', 'block')
}
function showLoading() {
hideChart();
hideMessage();
$("#statisticsModal .chart-loading").css('display', 'block')
}
function showChart() {
hideMessage();
hideLoading();
$("#statisticsModal .chart-container").css('display', 'block')
}
function hideMessage() {
$("#statisticsModal .chart-error").css('display', 'none')
}
function hideLoading() {
$("#statisticsModal .chart-loading").css('display', 'none')
}
function hideChart() {
$("#statisticsModal .chart-container").css('display', 'none') $("#statisticsModal .chart-container").css('display', 'none')
} }
...@@ -35,10 +67,13 @@ function drawChart(data) { ...@@ -35,10 +67,13 @@ function drawChart(data) {
try { try {
chartData = $.parseJSON(data) chartData = $.parseJSON(data)
} catch (err) { } catch (err) {
showErr() showMessage("Invalid data format received. Can't use to draw chart");
} }
chartData = $.parseJSON(data) if ($.isEmptyObject(chartData)) {
showMessage("No data for parameters specified. Please refine query string.");
return false;
}
$.each( chartData, function( index, value ) { $.each( chartData, function( index, value ) {
key.push(index) key.push(index)
...@@ -47,15 +82,21 @@ function drawChart(data) { ...@@ -47,15 +82,21 @@ function drawChart(data) {
data3.push(value.request_uniq.idp) data3.push(value.request_uniq.idp)
}); });
let dataSet = [
{ label: 'API', data: data1 },
{ label: 'WEB', data: data2 },
{ label: 'Unique IdP', data: data3 }
];
let options = { let options = {
type: 'line', type: 'line',
data: { data: {
labels: key, labels: key,
datasets: [ datasets: dataSet
{ label: 'API', data: data1 }, // { label: 'API', data: data1 },
{ label: 'WEB', data: data2 }, // { label: 'WEB', data: data2 },
{ label: 'Unique IdP', data: data3 } // { label: 'Unique IdP', data: data3 }
] // ]
}, },
options: { options: {
responsive: true, responsive: true,
...@@ -67,13 +108,16 @@ function drawChart(data) { ...@@ -67,13 +108,16 @@ function drawChart(data) {
} }
} }
var ctx = document.getElementById('statistics').getContext('2d'); var ctx = document.getElementById('statistics').getContext('2d');
showChart();
$("#statisticsModal .chart-error").css('display', 'none')
$("#statisticsModal .chart-container").css('display', 'block')
if ($.isEmptyObject(statChart)) { if ($.isEmptyObject(statChart)) {
statChart = new Chart(ctx, options); statChart = new Chart(ctx, options);
} else { } else {
statChart.data.datasets = dataSet;
statChart.data.labels = key;
statChart.update(); statChart.update();
} }
} }
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment