Merged 'web/index.php' changes

api.py

@@ -9,6 +9,8 @@ from flask import Flask, request, jsonify
 from flask_restful import Resource, Api
 from utils import get_logger, get_list_from_url, get_reg_auth_dict, generate_login_url
 from markupsafe import escape
+from datetime import timedelta, datetime, date
+from urllib.parse import urlparse, parse_qs, unquote
 
 app = Flask(__name__)
 api = Api(app)
@@ -58,6 +60,157 @@ def getSimpleDict(aux):
     }
     return simpleDict
 
+def isValidDate(date_text):
+      try:
+         date.fromisoformat(date_text)
+      except ValueError as e:
+         print(e)
+         return False
+      return True
+
+def clearDateString(str):
+    del_char = {"[","]"}
+    for elem in del_char:
+        str = str.replace(elem, '')
+    return str
+
+def storeParsedDay(path, data):
+   try: 
+      with open(path, "w") as outfile:
+         outfile.write(json.dumps(data))
+   except FileNotFoundError as e:
+      print(e)
+
+# Log will be parsed using predefined format
+# %(addr)|[%(ctime)]|%(method)|%(uri)|%(uagent)|%(referer)
+# target result is array like: 
+# [
+# "YYYY-MM-DD" => [
+#     "request_count" => [
+#        web => 100,
+#        api => 200 
+#     ],
+#     "requested_param" => [
+#        idp => 100,
+#        reg_auth => 200 
+#     ],
+#     idp => [
+#        "xx" => [
+#           "api" => 1,
+#           "web" => 10
+#        ],
+#     ],
+#     reg_auth => [
+#        "xx" => [
+#           "api" => 1,
+#           "web" => 10
+#        ],
+#      ],
+# ],
+#]
+def parseLog(lines,criteria):
+    result = {}
+    request_count = {'web':0,'api':0}
+    request_param = {'idp':0,'reg_auth':0}
+    idp = {}
+    reg_auth = {}
+    rowDateCheck = ''
+   
+    for line in lines:
+        row = line.split('|')
+        
+        if len(row) <= 1 or row[3].find("api/webdata") >= 0:
+            continue
+        
+        rowDate = datetime.strptime(clearDateString(row[1]), '%a %b %d %H:%M:%S %Y').strftime('%Y-%m-%d')
+        
+      # check for entries to be in range of date of rotated log  
+        if 'cur_date' in criteria and criteria['cur_date'] != rowDate:
+            continue
+         
+        if ('date_from' in criteria and 'date_to' in criteria and criteria['date_from'] <= rowDate <= criteria['date_to']) or ('date_from' not in criteria and 'date_to' not in criteria and rowDate):
+            
+            if rowDateCheck != rowDate:
+                request_count = {'web':0,'api':0}
+                request_param = {'idp':0,'reg_auth':0}
+                idp = {}
+                reg_auth = {}
+                rowDateCheck = rowDate
+                        
+            rowGET = parseReqURL(row[3])
+            
+            if ('idp' not in criteria and 'reg_auth' not in criteria) or ('idp' in criteria and rowGET['idp'] and rowGET['idp'].find(criteria['idp']) >= 0) or ('reg_auth' in criteria and rowGET['reg_auth'] and rowGET['reg_auth'].find(criteria['reg_auth']) >= 0):
+               rowRequestSource = ('web' if len(row[5]) > 5 else 'api')
+               request_count['web'] += (1 if rowRequestSource == 'web' else 0)
+               request_count['api'] += (1 if rowRequestSource == 'api' else 0)
+            
+            if rowGET['idp']:
+                if ('idp' in criteria and rowGET['idp'].find(criteria['idp']) >= 0) or 'idp' not in criteria:
+                    if rowGET['idp'] not in idp.keys():
+                        idp[rowGET['idp']] = 0
+                    idp[rowGET['idp']] += 1
+                    request_param['idp'] += 1
+                   
+            if rowGET['reg_auth']:
+                if ('reg_auth' in criteria and rowGET['reg_auth'].find(criteria['reg_auth']) >= 0) or 'reg_auth' not in criteria:
+                    if rowGET['reg_auth'] not in reg_auth.keys():
+                        reg_auth[rowGET['reg_auth']] = 0
+                    reg_auth[rowGET['reg_auth']] += 1
+                    request_param['reg_auth'] += 1
+
+            result[rowDate] = {
+                'request_count' : request_count,
+                'request_param' : request_param,
+                'request_uniq' : {'idp':len(idp),'reg_auth':len(reg_auth)},
+                'idp' : idp,
+                'reg_auth' : reg_auth
+            }
+        
+    return result
+
+# Parse URL from log line. Used to get only idp and reg_auth. 
+def parseReqURL(url):
+   result = {'idp':False,'reg_auth':False}
+   parsed_url = urlparse(url)
+   captured_value = parse_qs(parsed_url.query)
+
+   if 'idp' in captured_value:
+      result['idp'] = captured_value['idp'][0]
+   
+   if 'reg_auth' in captured_value:
+      result['reg_auth'] = captured_value['reg_auth'][0]
+   
+   return result
+
+def filterParsedData(json_data,criteria):
+   idp = {}
+   reg_auth = {}
+   out_data = {}
+   
+   if criteria['cur_date'] in json_data: 
+      if 'idp' in criteria and len(criteria['idp']): 
+         for idpEntry in json_data[criteria['cur_date']]['idp'].keys():
+            if idpEntry.find(criteria['idp']) >= 0:
+               idp[criteria['idp']] = json_data[criteria['cur_date']]['idp'][criteria['idp']]
+      else:
+         idp = json_data[criteria['cur_date']]['idp']
+
+      if 'reg_auth' in criteria and len(criteria['reg_auth']):
+         for regAuthEntry in json_data[criteria['cur_date']]['reg_auth'].keys():
+            if regAuthEntry.find(criteria['reg_auth']) >= 0:
+               reg_auth[criteria['reg_auth']] = json_data[criteria['cur_date']]['reg_auth'][criteria['reg_auth']]
+      else:
+         reg_auth = json_data[criteria['cur_date']]['reg_auth']
+   
+      out_data[criteria['cur_date']] = {
+            'request_count' : json_data[criteria['cur_date']]['request_count'],
+            'request_param' : json_data[criteria['cur_date']]['request_param'],
+            'request_uniq' : {'idp':len(idp),'reg_auth':len(reg_auth)},
+            'idp' : idp,
+            'reg_auth' : reg_auth,
+      }
+      
+   return out_data
 
 ### Classes
 
@@ -278,6 +431,97 @@ class FedStats(Resource):
                results.append(resultDict)
            return jsonify(results)
 
+        
+# /api/webdata
+class WebData(Resource):
+   def get(self):
+      file_path = f"{e_p.ECCS_LOGSDIR}/eccs-uwsgi-req.log" # will this name be moved to properties definer file ?
+      criteria = {}
+      criteria['date_from'] = criteria['date_to'] = e_p.DAY
+      # TBM to config
+      eccsLogRotated = True
+      useParsedFile = False
+      
+      in_data = request.args
+
+      if ('dateFrom' in in_data and isValidDate(in_data['dateFrom'])):
+         criteria['date_from'] = in_data['dateFrom']
+         if ('dateTo' not in in_data):
+            criteria['date_to'] = (datetime.strptime(criteria['date_from'], '%Y-%m-%d') + timedelta(days=30)).strftime('%Y-%m-%d')
+            
+            if datetime.today().strftime('%Y-%m-%d') < criteria['date_to']:
+               diff = (datetime.strptime(criteria['date_to'], '%Y-%m-%d') - datetime.today()).days
+               criteria['date_from'] = (datetime.strptime(criteria['date_from'], '%Y-%m-%d') - timedelta(days=diff)).strftime('%Y-%m-%d')
+               criteria['date_to'] = datetime.today().strftime('%Y-%m-%d')
+      
+      if ('dateTo' in in_data and isValidDate(in_data['dateTo'])):
+         criteria['date_to'] = in_data['dateTo']
+         if ('dateFrom' not in in_data):
+            criteria['date_from'] = (datetime.strptime(criteria['date_to'], '%Y-%m-%d') + timedelta(days=30)).strftime('%Y-%m-%d')
+      
+      if ('requestSource' in in_data and in_data['requestSource'] == 'divided'):
+         criteria['request_source'] = 'divided'
+
+      if 'regAuth' in in_data:
+         criteria['reg_auth'] = unquote(in_data['regAuth'])
+      
+      if 'idp' in in_data:
+         criteria['idp'] = unquote(in_data['idp'].strip())
+
+      lines = []
+      results = {}
+      cur_date = criteria['date_from']
+
+      if eccsLogRotated == True:
+         while cur_date <= criteria['date_to']:
+            json_data = {}
+            criteria['logfile_date'] = (datetime.strptime(cur_date, '%Y-%m-%d') + timedelta(days=1)).strftime('%Y-%m-%d')
+            criteria['cur_date'] = cur_date
+            tmpDate = datetime.strptime(criteria['logfile_date'], '%Y-%m-%d').strftime('%Y%m%d')
+            file_path = f"{e_p.ECCS_LOGSDIR}/eccs-uwsgi-req.log-{tmpDate}"
+
+            if useParsedFile == True:
+               json_file_path = f"{e_p.ECCS_DIR}/parsed/eccs-uwsgi-req-json-{tmpDate}"
+               try:
+                  f = open(json_file_path)
+                  json_data = json.load(f)
+
+               except (ValueError, FileNotFoundError) as e:
+                  #print(e)
+                  pass
+               
+               if len(json_data) > 0 and ('idp' in criteria or 'reg_auth' in criteria):
+                  json_data = filterParsedData(json_data, criteria)
+
+            if len(json_data) == 0:
+               try:
+                  with open(file_path,"r",encoding="utf-8") as fo:
+                     lines = fo.readlines()
+
+                  json_data = parseLog(lines, criteria)
+
+                  if useParsedFile == True and 'idp' not in criteria and 'reg_auth' not in criteria:
+                     storeParsedDay(json_file_path, json_data)
+
+               except FileNotFoundError as e:
+                  # print(e)
+                  pass
+            
+            results.update(json_data)
+            cur_date = (datetime.strptime(cur_date, '%Y-%m-%d') + timedelta(days=1)).strftime('%Y-%m-%d')
+      else:
+            try:
+               with open(file_path,"r",encoding="utf-8") as fo:
+                  lines = fo.readlines()
+
+            except FileNotFoundError as e:
+               # print(e)
+               results = {}
+
+            results = parseLog(lines, criteria)
+
+      return json.dumps(results)
+
 # /api/
 class Help(Resource):
     def get(self):
@@ -306,6 +550,7 @@ api.add_resource(Help, '/') # Route_1
 api.add_resource(Test, '/test') # Route_2
 api.add_resource(EccsResults, '/eccsresults') # Route_3
 api.add_resource(FedStats, '/fedstats') # Route_4
+api.add_resource(WebData, '/webdata') # Route_4
 
 if __name__ == '__main__':
 

web/index.php

@@ -43,6 +43,7 @@ $data[ 'check_result' ] = htmlspecialchars($_GET[ "check_result" ]);
 
     <link rel="stylesheet" type="text/css" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.15.1/css/all.min.css"/>
     <link rel="stylesheet" type="text/css" href="eccs.css" />
+    <link rel="stylesheet" type="text/css" href="statistics.css" />
     <script>
        $( function() {
           $( "#datepicker" ).datepicker({
@@ -80,6 +81,7 @@ $data[ 'check_result' ] = htmlspecialchars($_GET[ "check_result" ]);
                  <div class="boxCalendar">
                     <div id="calendarGo">
                        <button id="goButton" onclick="getPastResults()">Go</button>
+                       <button id="statisticsButton">Statistics</button>
                        <label id="lbl-datepicker" for="datepicker" class="strong">Select date:</label>
                        <input type="text" id="datepicker" />
                     </div> <!-- END calendarGo -->
@@ -111,7 +113,21 @@ $data[ 'check_result' ] = htmlspecialchars($_GET[ "check_result" ]);
               var check_result = "<?php echo $data[ 'check_result' ] ?>";
            </script>
            <script type="text/javascript" src="eccs.js"></script>
+           <script type="text/javascript" src="statistics.js" /></script>
          </div> <!-- END eccs-central -->
+         <div id="statisticsModal" class="modal">
+            <div class="modal-content">
+               <span class="close">&times;</span>
+               <div class="chart-error" style="display: none;">
+                  <i class="fa fa-exclamation-triangle"></i> Something happened. Can't retrieve data
+               </div>
+               <div class="chart-container">
+                  <canvas id="statistics"></canvas>
+               </div>
+               <div class="data-container">
+               </div>
+            </div>
+         </div>
       </div> <!-- END main_body -->
 <?php include 'footer.php'; ?>
 <?php include 'edugain-scripts-css.php'; ?>

web/statistics.css

+.modal {
+  display: none; 
+  position: fixed; 
+  z-index: 1;
+  left: 0;
+  top: 0;
+  width: 100%; 
+  height: 100%; 
+  overflow: auto; 
+  background-color: rgb(0,0,0); 
+  background-color: rgba(0,0,0,0.4); 
+}
+
+.modal-content {
+  background-color: #fefefe;
+  margin: 5% auto 0; 
+  padding: 20px;
+  border: 1px solid #888;
+  width:85vw;
+  min-height: 30vh;
+}
+
+.chart-container {
+  height: 30vh; 
+  width: 80vw; 
+}
+
+.data-container {
+  max-height:50vh;
+  width:80vw;
+  overflow-y: auto;
+  margin-top: 1rem;
+}
+
+.close {
+  color: #aaa;
+  float: right;
+  font-size: 28px;
+  font-weight: bold;
+}
+
+.close:hover,
+.close:focus {
+  color: black;
+  text-decoration: none;
+  cursor: pointer;
+}
+
+.collapsible {
+  background-color: #777;
+  color: white;
+  cursor: pointer;
+  padding: 0.5rem;
+  width: 100%;
+  border: none;
+  text-align: left;
+  outline: none;
+  font-size: 12px;
+}
+
+.active, .collapsible:hover {
+  background-color: #555;
+}
+
+.collapsible:after {
+  content: '\002B';
+  color: white;
+  font-weight: bold;
+  float: right;
+  margin-left: 5px;
+}
+
+.active:after {
+  content: "\2212";
+}
+
+.content {
+  padding: 0 18px;
+  max-height: 0;
+  overflow: hidden;
+  transition: max-height 0.2s ease-out;
+  background-color: #f1f1f1;
+}
\ No newline at end of file

web/statistics.js

+var statChart = {};
+var baseURL = window.location.protocol + "//" + window.location.host + "/eccs";
+
+$(document).ready(function() {
+	$(".close").on("click", function() {
+		$("#statisticsModal").hide()
+	});
+	
+	$("#statisticsButton").on("click", function() {
+		$("#statisticsModal").show()
+		getStatistics();
+	})
+});
+
+function getStatistics() {
+	showLoading();
+	let reqData = {}
+	reqData['dateFrom'] =  $.datepicker.formatDate("yy-mm-dd", $('#datepicker').datepicker().datepicker('getDate'));
+	if ($("#eccstable thead th:last-child input").val() != '') {
+		reqData['regAuth'] = $("#eccstable thead th:last-child input").val();
+	}
+	if ($("#eccstable thead th:nth-last-child(2) input").val() != '') {
+		reqData['idp'] = $("#eccstable thead th:nth-last-child(2) input").val();
+	}
+	$.ajax({
+		url: baseURL + "/api/webdata",
+		type : 'GET',
+		contentType: "application/json",
+		data : reqData,
+		success: function(result) {
+			try {
+				parsedData = $.parseJSON(result)
+			} catch (err) {
+				showMessage("Invalid data format received. Can't use to draw chart");
+			}
+		
+			if ($.isEmptyObject(parsedData)) {
+				showMessage("No data for parameters specified. Please refine query string.");
+				return false;
+			}
+			fillInfo(parsedData);
+		}, 
+		error: function () {
+			showMessage("Something happened. Can't retrieve data");
+		}
+	});
+}
+function showMessage(text) {
+	hideModalContent();
+	hideLoading();
+	$("#statisticsModal .chart-error span").text(text)
+	$("#statisticsModal .chart-error").css('display', 'block')
+}
+function hideMessage() {
+	$("#statisticsModal .chart-error").css('display', 'none')
+}
+function showLoading() {
+	hideModalContent();
+	hideMessage();
+	$("#statisticsModal .chart-loading").css('display', 'block')
+}
+function hideLoading() {
+	$("#statisticsModal .chart-loading").css('display', 'none')
+}
+function fillInfo(parsedData) {
+	let chartConfig = {};
+	let tabledata = {};
+	chartConfig["key"] = [];
+	chartConfig["data1"] = [];
+	chartConfig["data2"] = [];
+	chartConfig["data3"] = [];
+	$.each( parsedData, function( index, value ) {
+		if (value.request_count === undefined || value.request_uniq === undefined
+			|| value.request_count.api === undefined || value.request_count.web === undefined
+			|| value.request_uniq.idp === undefined) {
+			return true;
+		}
+		chartConfig["key"].push(index);
+		chartConfig["data1"].push(value.request_count.api);
+		chartConfig["data2"].push(value.request_count.web);
+		chartConfig["data3"].push(value.request_uniq.idp);
+		tabledata[index] = value.idp;
+	});
+	if (!chartConfig["key"].length) {
+		showMessage("No valid data for parameters specified. Please refine query string.");
+		return false;
+	}
+	drawChart(chartConfig);
+	fillTable(tabledata);
+	hideMessage();
+	hideLoading();
+	showModalContent();
+}
+function drawChart(chartData) {
+	let chartType = 'line';
+	let dataSet = [];
+	let key = [];
+	if (chartData["key"].length == 1) {
+		chartType = 'doughnut';
+		dataSet = [
+			{ label: chartData["key"][0], data: [chartData["data1"][0], chartData["data2"][0], chartData["data3"][0]] },
+		];
+		key = ['API', 'WEB', 'Unique IdP'];
+	} else {
+		dataSet = [
+			{ label: 'API', data: chartData["data1"] },
+			{ label: 'WEB', data: chartData["data2"]	},
+			{ label: 'Unique IdP', data: chartData["data3"] }
+		];
+		key = chartData["key"];
+	}
+	let options = {
+		type: chartType,
+		data: {
+			labels: key,
+			datasets: dataSet
+		},
+		options: {
+			responsive: true,
+			maintainAspectRatio: false,
+			interaction: {
+				mode: 'index',
+				intersect: false,
+			}
+		}
+	}
+	var ctx = document.getElementById('statistics').getContext('2d');
+	if (!$.isEmptyObject(statChart) && statChart.config.type != chartType) {
+		statChart.destroy();
+		statChart = {};
+	}
+	if ($.isEmptyObject(statChart)) {
+		statChart = new Chart(ctx, options);
+	} else {
+		statChart.data.datasets = dataSet;
+		statChart.data.labels = key;
+		statChart.update();
+	}
+}
+function fillTable(tabledata) {
+	let container = $(".data-container")
+	container.html("");
+	$.each(tabledata, function (index, value) {
+		let list = '<ul style="list-style-type: decimal;">';
+		if (Object.keys(value).length > 0) {
+			$.each(value, function(idp, count) {
+				list += "<li>"+idp+":<strong>"+count+"</strong></li>"
+			})
+		} else {
+			list += 'No data';
+		}
+		list += "</ul>"
+		let button  = document.createElement("button")
+		button.classList.add("collapsible");
+		button.textContent = index;
+		button.addEventListener("click", function() {
+			this.classList.toggle("active");
+			var content = this.nextElementSibling;
+			
+			if (content.style.maxHeight){
+				content.style.maxHeight = null;
+			} else {
+				content.style.maxHeight = content.scrollHeight + "px";
+			} 
+		});
+		let div  = document.createElement("div")
+		div.classList.add("content");
+		div.innerHTML=list; 
+		container.append(button)
+		container.append(div)
+	})
+}
+function hideModalContent() {
+	$("#statisticsModal .chart-container").css('display', 'none')
+	$("#statisticsModal .data-container").css('display', 'none')
+}
+function showModalContent() {
+	$("#statisticsModal .chart-container").css('display', 'block')
+	$("#statisticsModal .data-container").css('display', 'block')
+}