Skip to content
Snippets Groups Projects
Commit f3129d99 authored by Valentin Pocotilenco's avatar Valentin Pocotilenco
Browse files

python parse log fixed

parent 3c5964b7
Branches
No related tags found
No related merge requests found
...@@ -67,6 +67,12 @@ def isValidDate(date_text): ...@@ -67,6 +67,12 @@ def isValidDate(date_text):
return False return False
return True return True
def clearDateString(str):
del_char = {"[","]"}
for elem in del_char:
str = str.replace(elem, '')
return str
# Log will be parsed using predefined format # Log will be parsed using predefined format
# %(addr)|[%(ctime)]|%(method)|%(uri)|%(uagent)|%(referer) # %(addr)|[%(ctime)]|%(method)|%(uri)|%(uagent)|%(referer)
# target result is array like: # target result is array like:
...@@ -95,39 +101,60 @@ def isValidDate(date_text): ...@@ -95,39 +101,60 @@ def isValidDate(date_text):
# ], # ],
#] #]
def parseLog(lines,criteria): def parseLog(lines,criteria):
result = {} result = {}
request_count = {'web':0,'api':0} request_count = {'web':0,'api':0}
request_param = {'idp':0,'reg_auth':0} request_param = {'idp':0,'reg_auth':0}
idp = {} idp = {}
reg_auth = {} reg_auth = {}
rowDateCheck = ''
for line in lines: for line in lines:
row = line.split('|') row = line.split('|')
rowDate = datetime.strptime(row[1], '%a %b %d %H:%M:%S %Y').date()
if len(row) <= 1:
if criteria['datefrom'] <= rowDate <= criteria['dateto']: continue
rowGET = parseReqURL(row[3])
rowRequestSource = ('web' if len(row[5]) > 0 else 'api') rowDate = datetime.strptime(clearDateString(row[1]), '%a %b %d %H:%M:%S %Y').strftime('%Y-%m-%d')
request_count['web'] += (1 if rowRequestSource == 'web' else 0)
request_count['api'] += (1 if rowRequestSource == 'api' else 0)
if rowGET['idp']:
request_param['idp'] += 1
if ('idp' in criteria and criteria['idp'] == rowGET['idp']) or 'idp' not in criteria:
idp.append(rowGET['idp'])
if rowGET['reg_auth']: if ('date_from' in criteria and 'date_to' in criteria and criteria['date_from'] <= rowDate <= criteria['date_to']) or ('date_from' not in criteria and 'date_to' not in criteria and rowDate):
request_param['reg_auth'] += 1
if ('reg_auth' in criteria and criteria['reg_auth'] == rowGET['reg_auth']) or 'reg_auth' not in criteria: if rowDateCheck != rowDate:
reg_auth.append(rowGET['reg_auth']) request_count = {'web':0,'api':0}
request_param = {'idp':0,'reg_auth':0}
result[rowDate] = { idp = {}
'request_count' : request_count, reg_auth = {}
'request_param' : request_param, rowDateCheck = rowDate
'idp' : idp,
'reg_auth' : reg_auth rowGET = parseReqURL(row[3])
} rowRequestSource = ('web' if len(row[5]) > 5 else 'api')
return result request_count['web'] += (1 if rowRequestSource == 'web' else 0)
request_count['api'] += (1 if rowRequestSource == 'api' else 0)
if rowGET['idp']:
request_param['idp'] += 1
if ('idp' in criteria and criteria['idp'] == rowGET['idp']) or 'idp' not in criteria:
if rowGET['idp'] not in idp.keys():
idp[rowGET['idp']] = 0
idp[rowGET['idp']] += 1
if rowGET['reg_auth']:
request_param['reg_auth'] += 1
if ('reg_auth' in criteria and criteria['reg_auth'] == rowGET['reg_auth']) or 'reg_auth' not in criteria:
if rowGET['reg_auth'] not in reg_auth.keys():
reg_auth[rowGET['reg_auth']] = 0
reg_auth[rowGET['reg_auth']] += 1
result[rowDate] = {
'request_count' : request_count,
'request_param' : request_param,
'request_uniq' : {'idp':len(idp),'reg_auth':len(reg_auth)},
'idp' : idp,
'reg_auth' : reg_auth
}
return json.dumps(result)
# Parse URL from log line. Used to get only idp and reg_auth. # Parse URL from log line. Used to get only idp and reg_auth.
def parseReqURL(url): def parseReqURL(url):
...@@ -423,7 +450,7 @@ class WebData(Resource): ...@@ -423,7 +450,7 @@ class WebData(Resource):
results = parseLog(lines, criteria) results = parseLog(lines, criteria)
return jsonify(results) return results
# /api/ # /api/
class Help(Resource): class Help(Resource):
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment