diff --git a/api.py b/api.py index 4c5ebc3489bf4b1ad712b57e8563e4345615ebb0..ef06eea26e429ab15c0c35301c7334c8668288c4 100755 --- a/api.py +++ b/api.py @@ -63,7 +63,8 @@ def getSimpleDict(aux): def isValidDate(date_text): try: date.fromisoformat(date_text) - except ValueError: + except ValueError as e: + print(e) return False return True @@ -73,6 +74,13 @@ def clearDateString(str): str = str.replace(elem, '') return str +def storeParsedDay(path, data): + try: + with open(path, "w") as outfile: + outfile.write(json.dumps(data)) + except FileNotFoundError as e: + print(e) + # Log will be parsed using predefined format # %(addr)|[%(ctime)]|%(method)|%(uri)|%(uagent)|%(referer) # target result is array like: @@ -133,7 +141,7 @@ def parseLog(lines,criteria): if rowGET['idp']: request_param['idp'] += 1 - if ('idp' in criteria and criteria['idp'] == rowGET['idp']) or 'idp' not in criteria: + if ('idp' in criteria and criteria['idp'] in rowGET['idp']) or 'idp' not in criteria: if rowGET['idp'] not in idp.keys(): idp[rowGET['idp']] = 0 idp[rowGET['idp']] += 1 @@ -141,7 +149,7 @@ def parseLog(lines,criteria): if rowGET['reg_auth']: request_param['reg_auth'] += 1 - if ('reg_auth' in criteria and criteria['reg_auth'] == rowGET['reg_auth']) or 'reg_auth' not in criteria: + if ('reg_auth' in criteria and criteria['reg_auth'] in rowGET['reg_auth']) or 'reg_auth' not in criteria: if rowGET['reg_auth'] not in reg_auth.keys(): reg_auth[rowGET['reg_auth']] = 0 reg_auth[rowGET['reg_auth']] += 1 @@ -154,7 +162,7 @@ def parseLog(lines,criteria): 'reg_auth' : reg_auth } - return json.dumps(result) + return result # Parse URL from log line. Used to get only idp and reg_auth. def parseReqURL(url): @@ -396,26 +404,31 @@ class WebData(Resource): list_feds = get_list_from_url(e_p.ECCS_LISTFEDSURL, e_p.ECCS_LISTFEDSFILE) regAuthDict = get_reg_auth_dict(list_feds) - file_path = f"{e_p.ECCS_OUTPUTDIR}/{e_p.ECCS_RESULTSLOG}" + file_path = f"{e_p.ECCS_LOGSDIR}/eccs-uwsgi-req.log" # will this name be moved to properties definer file ? criteria = {} criteria['date_from'] = criteria['date_to'] = e_p.DAY - eccsLogRotated = False + eccsLogRotated = True in_data = request.args - if ('date_from' in in_data and isValidDate(in_data['date_from'])): - criteria['date_from'] = in_data['date_from'] - if ('date_to' not in in_data): - criteria['date_to'] = criteria['date_from'] + timedelta(days=30) + if ('dateFrom' in in_data and isValidDate(in_data['dateFrom'])): + criteria['date_from'] = in_data['dateFrom'] + if ('dateTo' not in in_data): + criteria['date_to'] = (datetime.strptime(criteria['date_from'], '%Y-%m-%d') + timedelta(days=30)).strftime('%Y-%m-%d') + + if datetime.today().strftime('%Y-%m-%d') < criteria['date_to']: + diff = (datetime.strptime(criteria['date_to'], '%Y-%m-%d') - datetime.today()).days + criteria['date_from'] = (datetime.strptime(criteria['date_from'], '%Y-%m-%d') - timedelta(days=diff)).strftime('%Y-%m-%d') + criteria['date_to'] = datetime.today().strftime('%Y-%m-%d') - if ('date_to' in in_data and isValidDate(in_data['date_to'])): - criteria['date_to'] = in_data['date_to'] - if ('date_from' not in in_data): - criteria['date_from'] = criteria['date_to'] - timedelta(days=30) + if ('dateTo' in in_data and isValidDate(in_data['dateTo'])): + criteria['date_to'] = in_data['dateTo'] + if ('dateFrom' not in in_data): + criteria['date_from'] = (datetime.strptime(criteria['date_to'], '%Y-%m-%d') + timedelta(days=30)).strftime('%Y-%m-%d') - if ('request_source' in in_data and in_data['request_source'] == 'divided'): + if ('requestSource' in in_data and in_data['requestSource'] == 'divided'): criteria['request_source'] = 'divided' - if ('reg_auth' in in_data and in_data['reg_auth'] in regAuthDict): + if ('regAuth' in in_data and in_data['regAuth'] in regAuthDict): criteria['reg_auth'] = in_data['reg_auth'] if ('idp' in in_data): @@ -423,34 +436,50 @@ class WebData(Resource): # here I have to parse eccs-log file lines = [] - results = [] + results = {} cur_date = criteria['date_from'] if eccsLogRotated == True: while cur_date <= criteria['date_to']: - file_path = f"{e_p.ECCS_OUTPUTDIR}/eccs_{cur_date}.log" + json_data = {} + tmpDate = datetime.strptime(cur_date, '%Y-%m-%d').strftime('%Y%m%d') + file_path = f"{e_p.ECCS_LOGSDIR}/eccs-uwsgi-req.log-{tmpDate}" + json_file_path = f"{e_p.ECCS_DIR}/parsed/eccs-uwsgi-req-json-{tmpDate}" try: - with open(file_path,"r",encoding="utf-8") as fo: - lines = fo.readlines() + f = open(json_file_path) + json_data = json.load(f) - except FileNotFoundError as e: - results[cur_date] = [] + except (ValueError, FileNotFoundError) as e: + #print(e) + pass + + if len(json_data) == 0: + try: + with open(file_path,"r",encoding="utf-8") as fo: + lines = fo.readlines() + json_data = parseLog(lines, criteria) + storeParsedDay(json_file_path, json_data) + + except FileNotFoundError as e: + #print(e) + pass - results = parseLog(lines, criteria) - - cur_date += timedelta(days=1) + results.update(json_data) + + cur_date = (datetime.strptime(cur_date, '%Y-%m-%d') + timedelta(days=1)).strftime('%Y-%m-%d') else: try: with open(file_path,"r",encoding="utf-8") as fo: lines = fo.readlines() except FileNotFoundError as e: - results = [] + print(e) + results = {} results = parseLog(lines, criteria) - return results + return json.dumps(results) # /api/ class Help(Resource):