diff --git a/api.py b/api.py index ef06eea26e429ab15c0c35301c7334c8668288c4..43318f649ee9be1e29c1c8e8cada362a144571cc 100755 --- a/api.py +++ b/api.py @@ -123,6 +123,10 @@ def parseLog(lines,criteria): continue rowDate = datetime.strptime(clearDateString(row[1]), '%a %b %d %H:%M:%S %Y').strftime('%Y-%m-%d') + + # check for entries to be in range of date of rotated log + if 'prev_date' in criteria and criteria['date_from'] != rowDate: + continue if ('date_from' in criteria and 'date_to' in criteria and criteria['date_from'] <= rowDate <= criteria['date_to']) or ('date_from' not in criteria and 'date_to' not in criteria and rowDate): @@ -178,6 +182,28 @@ def parseReqURL(url): return result +def filterParsedData(json_data,criteria): + # idp = {} + # reg_auth = {} + + # if 'idp' in criteria and criteria['idp'] in json_data['idp'].keys(): + # for entry in json_data[criteria['prev_date']]['idp']: + # # if criteria['idp'] == entry.key(): + # if entry.key().find(criteria['idp']) >= 0: + # idp[entry.key()] = entry + # # search substring + # if 'reg_auth' in criteria and criteria['reg_auth'] in json_data['reg_auth'].keys(): + # for entry in json_data[criteria['reg_auth']]: + # if criteria['reg_auth'] == entry.key(): + # reg_auth[criteria['reg_auth']] = entry + + # json_data[criteria['prev_date']] = { + # 'request_uniq' : {'idp':len(idp),'reg_auth':len(reg_auth)}, + # 'idp' : idp, + # 'reg_auth' : reg_auth + # } + return json_data + ### Classes # /api/test @@ -408,6 +434,8 @@ class WebData(Resource): criteria = {} criteria['date_from'] = criteria['date_to'] = e_p.DAY eccsLogRotated = True + useParsedFile = True + in_data = request.args if ('dateFrom' in in_data and isValidDate(in_data['dateFrom'])): @@ -434,51 +462,59 @@ class WebData(Resource): if ('idp' in in_data): criteria['idp'] = in_data['idp'] - # here I have to parse eccs-log file lines = [] results = {} cur_date = criteria['date_from'] - + if eccsLogRotated == True: while cur_date <= criteria['date_to']: json_data = {} + criteria['prev_date'] = (datetime.strptime(cur_date, '%Y-%m-%d') - timedelta(days=1)).strftime('%Y-%m-%d') tmpDate = datetime.strptime(cur_date, '%Y-%m-%d').strftime('%Y%m%d') file_path = f"{e_p.ECCS_LOGSDIR}/eccs-uwsgi-req.log-{tmpDate}" - json_file_path = f"{e_p.ECCS_DIR}/parsed/eccs-uwsgi-req-json-{tmpDate}" - - try: - f = open(json_file_path) - json_data = json.load(f) - - except (ValueError, FileNotFoundError) as e: - #print(e) - pass - + + if useParsedFile == True: + tmpDate = (datetime.strptime(cur_date, '%Y-%m-%d') - timedelta(days=1)).strftime('%Y%m%d') + json_file_path = f"{e_p.ECCS_DIR}/parsed/eccs-uwsgi-req-json-{tmpDate}" + try: + f = open(json_file_path) + json_data = json.load(f) + + except (ValueError, FileNotFoundError) as e: + #print(e) + pass + + if len(json_data) > 0 and ('idp' in criteria or 'reg_auth' in criteria): + json_data = filterParsedData(json_data, criteria) + if len(json_data) == 0: try: with open(file_path,"r",encoding="utf-8") as fo: lines = fo.readlines() + json_data = parseLog(lines, criteria) - storeParsedDay(json_file_path, json_data) + + if useParsedFile == True and 'idp' not in criteria and 'reg_auth' not in criteria: + storeParsedDay(json_file_path, json_data) except FileNotFoundError as e: #print(e) pass - + results.update(json_data) - + cur_date = (datetime.strptime(cur_date, '%Y-%m-%d') + timedelta(days=1)).strftime('%Y-%m-%d') else: try: with open(file_path,"r",encoding="utf-8") as fo: lines = fo.readlines() - + except FileNotFoundError as e: print(e) results = {} - + results = parseLog(lines, criteria) - + return json.dumps(results) # /api/