diff --git a/api.py b/api.py index b5d8c00a268e31e78b2b9081eeb478e0bbfaa2c4..ae765ab92cc45b1ce2db95bf36fdfd4a2761a7b3 100755 --- a/api.py +++ b/api.py @@ -1,14 +1,14 @@ -#!/usr/bin/env python3 +#!/usr/bin/env python3.8 -from flask import Flask, request +import logging +import re + +from eccs2properties import DAY +from flask import Flask, request, jsonify from flask_restful import Resource, Api from json import dumps, loads -from flask import jsonify -from pathlib import PurePath -import logging from logging.handlers import RotatingFileHandler -import re -import eccs2properties +from pathlib import PurePath app = Flask(__name__) @@ -52,8 +52,8 @@ def setup(): driver = webdriver.Chrome('chromedriver', chrome_options=chrome_options) # Configure timeouts - driver.set_page_load_timeout(45) - driver.set_script_timeout(45) + driver.set_page_load_timeout(30) + driver.set_script_timeout(30) return driver @@ -68,7 +68,7 @@ class Checks(Resource): def get(self): app.logger.info("Request 'Checks'") - file_path = "logs/eccs2checks_%s.log" % eccs2properties.day + file_path = "%s/eccs2checks_%s.log" % (ECCS2LOGSDIR,DAY) date = PurePath(file_path).parts[-1].split('_')[1].split('.')[0] pretty = 0 status = None @@ -76,8 +76,8 @@ class Checks(Resource): if 'date' in request.args: app.logger.info("'date' parameter inserted") - file_path = "logs/eccs2checks_"+request.args['date']+".log" date = request.args['date'] + file_path = "%s/eccs2checks_%s.log" % (ECCS2LOGSDIR,date) if 'pretty' in request.args: app.logger.info("'pretty' parameter inserted") pretty = request.args['pretty'] @@ -155,7 +155,7 @@ class EccsResults(Resource): def get(self): app.logger.info("Request 'EccsResults'") - file_path = "logs/eccs2_%s.log" % eccs2properties.day + file_path = "%s/eccs2_%s.log" % (ECCS2LOGSDIR,DAY) date = PurePath(file_path).parts[-1].split('_')[1].split('.')[0] pretty = 0 status = None diff --git a/eccs2.py b/eccs2.py index 1acbc8a0e156652531d51b99aaba074312a8f371..029befbd548e760d988f5da926fa35388b02a3e8 100755 --- a/eccs2.py +++ b/eccs2.py @@ -9,7 +9,7 @@ import re import requests from datetime import date -from eccs2properties import ECCS2LOGSPATH, ECCS2RESULTSLOG, ECCS2CHECKSLOG, ECCS2SELENIUMLOG, ECCS2SELENIUMLOGLEVEL, FEDS_BLACKLIST, IDPS_BLACKLIST +from eccs2properties import ECCS2LOGSDIR, ECCS2RESULTSLOG, ECCS2CHECKSLOG, ECCS2SELENIUMLOG, ECCS2SELENIUMLOGLEVEL, FEDS_BLACKLIST, IDPS_BLACKLIST, ECCS2SELENIUMPAGELOADTIMEOUT, ECCS2SELENIUMSCRIPTTIMEOUT from selenium import webdriver from selenium.webdriver.common.by import By from selenium.webdriver.common.keys import Keys @@ -18,7 +18,9 @@ from selenium.webdriver.remote.remote_connection import LOGGER from selenium.common.exceptions import NoSuchElementException from selenium.common.exceptions import TimeoutException from selenium.common.exceptions import WebDriverException +from selenium.common.exceptions import UnexpectedAlertPresentException from urllib3.exceptions import MaxRetryError +from urllib3.util import parse_url """ @@ -49,39 +51,47 @@ def checkIdP(sp,idp,logger,driver): status_code = requests.get(driver.current_url, verify=False).status_code except TimeoutException as e: - logger.info("%s;%s;999;TIMEOUT" % (idp['entityID'],sp)) - return "TIMEOUT" + logger.info("%s;%s;999;Timeout" % (idp['entityID'],sp)) + return "Timeout" except NoSuchElementException as e: - logger.info("%s;%s;888;NoSuchElement" % (idp['entityID'],sp)) + # The input of the bootstrap tables are provided by "eccs2" and "eccs2checks" log. + # If I don't return anything and I don't put anything in the logger + # I'll not write on the input files for the table + # and I can re-run the command that caused the exception from the "stdout.log" file. print("!!! NO SUCH ELEMENT EXCEPTION !!!") - raise e + import selenium.webdriver.support.ui as ui + wait = ui.WebDriverWait(driver,10) + wait.until(lambda driver: driver.find_element_by_id("idpSelectInput")) + driver.find_element_by_id("idpSelectInput").send_keys(idp['entityID'] + Keys.ENTER) + page_source = driver.page_source + status_code = requests.get(driver.current_url, verify=False).status_code + + except UnexpectedAlertPresentException as e: + logger.info("%s;%s;888;UnexpectedAlertPresent" % (idp['entityID'],sp)) + return "ERROR" except WebDriverException as e: - logger.info("%s;%s;777;ConnectionError" % (idp['entityID'],sp)) - print("!!! WEB DRIVER EXCEPTION !!!") - print(e.__str__()) - return "ERROR" + print("!!! WEB DRIVER EXCEPTION - RUN AGAIN THE COMMAND!!!") + return None except requests.exceptions.ConnectionError as e: - logger.info("%s;%s;000;ConnectionError" % (idp['entityID'],sp)) - return "ERROR" + logger.info("%s;%s;000;ConnectionError" % (idp['entityID'],sp)) + return "ERROR" except requests.exceptions.TooManyRedirects as e: - logger.info("%s;%s;111;TooManyRedirects" % (idp['entityID'],sp)) - return "ERROR" + logger.info("%s;%s;111;TooManyRedirects" % (idp['entityID'],sp)) + return "ERROR" except requests.exceptions.RequestException as e: - logger.info("%s;%s;222;ConnectionError" % (idp['entityID'],sp)) - print ("!!! REQUESTS EXCEPTION !!!") - print(e.__str__()) - return "ERROR" + print ("!!! REQUESTS EXCEPTION !!!") + print (e.__str__()) + return None except Exception as e: - logger.info("%s;%s;555;ConnectionError" % (idp['entityID'],sp)) print ("!!! EXCEPTION !!!") - print(e.__str__()) - return "ERROR" + print (e.__str__()) + return None pattern_metadata = "Unable.to.locate(\sissuer.in|).metadata(\sfor|)|no.metadata.found|profile.is.not.configured.for.relying.party|Cannot.locate.entity|fail.to.load.unknown.provider|does.not.recognise.the.service|unable.to.load.provider|Nous.n'avons.pas.pu.(charg|charger).le.fournisseur.de service|Metadata.not.found|application.you.have.accessed.is.not.registered.for.use.with.this.service|Message.did.not.meet.security.requirements" @@ -190,6 +200,9 @@ def checkIdp(idp,sps,eccs2log,eccs2checksLog,driver): result[0], sps[1], result[1])) + elif (result[0] == None or result[1] == None): + # Do nothing + return else: eccs2log.info("%s;%s;%s;%s;%s;%s;%s;%s;%s;%s" % ( idp['displayname'].split(';')[1].split('==')[0], @@ -206,8 +219,8 @@ def checkIdp(idp,sps,eccs2log,eccs2checksLog,driver): # MAIN if __name__=="__main__": - eccs2log = getLogger(ECCS2RESULTSLOG, ECCS2LOGSPATH, "INFO") - eccs2checksLog = getLogger(ECCS2CHECKSLOG, ECCS2LOGSPATH, "INFO") + eccs2log = getLogger(ECCS2RESULTSLOG, ECCS2LOGSDIR, "INFO") + eccs2checksLog = getLogger(ECCS2CHECKSLOG, ECCS2LOGSDIR, "INFO") sps = ["https://sp24-test.garr.it/secure", "https://attribute-viewer.aai.switch.ch/eds/"] @@ -230,20 +243,20 @@ if __name__=="__main__": chrome_options.add_argument('--start-maximized') chrome_options.add_argument('--disable-extensions') - LOGGER.setLevel(ECCS2SELENIUMLOGLEVEL) + #driver = webdriver.Chrome('chromedriver', options=chrome_options) - driver = webdriver.Chrome('chromedriver', options=chrome_options, service_args=['--log-level=%d' % ECCS2SELENIUMLOGLEVEL, '--log-path=%s' % ECCS2SELENIUMLOG]) - # Utility for DEBUG - #driver = webdriver.Chrome('chromedriver', options=chrome_options, service_args=['--verbose', '--log-path=%s' % ECCS2SELENIUMLOG]) + # For DEBUG only + #driver = webdriver.Chrome('chromedriver', options=chrome_options, service_args=['--log-path=%s/%s.log' % (ECCS2SELENIUMLOGDIR, parse_url(idp['entityID'])[2])]) + driver = webdriver.Chrome('chromedriver', options=chrome_options, service_args=['--verbose', '--log-path=%s/%s.log' % (ECCS2SELENIUMLOGDIR, parse_url(idp['entityID'])[2])]) - # Configure timeouts: 30 sec - driver.set_page_load_timeout(30) - driver.set_script_timeout(30) + # Configure timeouts + driver.set_page_load_timeout("%d" % ECCS2SELENIUMPAGELOADTIMEOUT) + driver.set_script_timeout("%d" % ECCS2SELENIUMSCRIPTTIMEOUT) checkIdp(idp,sps,eccs2log,eccs2checksLog,driver) #driver.delete_all_cookies() - driver.close() + #driver.close() driver.quit() # Kill process to release resources and to avoid zombies - this reaise an issue diff --git a/eccs2properties.py b/eccs2properties.py index aed9ebb8bf79869f8f6ede1dcf5f59e90e0e77fb..39d007cdde41b3dbd5efb3b98b77c223e4a37dfc 100644 --- a/eccs2properties.py +++ b/eccs2properties.py @@ -2,24 +2,24 @@ from datetime import date -day = date.today().isoformat() +DAY = date.today().isoformat() -ECCS2PATH = "/opt/eccs2" -ECCS2LOGSPATH = "%s/logs" % ECCS2PATH -ECCS2INPUTPATH = "%s/input" % ECCS2PATH +ECCS2DIR = "/opt/eccs2" +ECCS2LOGSDIR = "%s/logs" % ECCS2DIR +ECCS2INPUTDIR = "%s/input" % ECCS2DIR # Input ECCS2LISTIDPSURL = 'https://technical.edugain.org/api.php?action=list_eccs_idps&format=json' -ECCS2LISTIDPSFILE = "%s/list_eccs_idps.json" % ECCS2INPUTPATH +ECCS2LISTIDPSFILE = "%s/list_eccs_idps.json" % ECCS2INPUTDIR ECCS2LISTFEDSURL = 'https://technical.edugain.org/api.php?action=list_feds&opt=1&format=json' -ECCS2LISTFEDSFILE = "%s/list_fed.json" % ECCS2INPUTPATH +ECCS2LISTFEDSFILE = "%s/list_fed.json" % ECCS2INPUTDIR # Output -ECCS2RESULTSLOG = "eccs2_%s.log" % day -ECCS2CHECKSLOG = "eccs2checks_%s.log" % day -ECCS2SELENIUMLOG = "%s/selenium_chromedriver.log" % ECCS2LOGSPATH -ECCS2STDOUT = "%s/stdout.log" % ECCS2LOGSPATH -ECCS2STDERR = "%s/stderr.log" % ECCS2LOGSPATH +ECCS2RESULTSLOG = "eccs2_%s.log" % DAY +ECCS2CHECKSLOG = "eccs2checks_%s.log" % DAY +ECCS2SELENIUMLOGDIR = "%s/selenium-logs" % ECCS2DIR +ECCS2STDOUT = "%s/stdout.log" % ECCS2LOGSDIR +ECCS2STDERR = "%s/stderr.log" % ECCS2LOGSDIR # Selenium Log Levels # 0: All (Show all log messages) @@ -30,6 +30,9 @@ ECCS2STDERR = "%s/stderr.log" % ECCS2LOGSPATH # 5: Off (Show no log messages) ECCS2SELENIUMLOGLEVEL = 4 +ECCS2SELENIUMPAGELOADTIMEOUT = 30 +ECCS2SELENIUMSCRIPTTIMEOUT = 30 + # Number of processes to run in parallel ECCS2NUMPROCESSES = 15 diff --git a/runEccs2.py b/runEccs2.py index 476ffa8fd2c819e1948cf13dc0e1837befa2b93c..cf19f390ae897a32ad6230c5e10c8bdd51bd1917 100755 --- a/runEccs2.py +++ b/runEccs2.py @@ -9,7 +9,7 @@ import requests import sys import time -from eccs2properties import ECCS2STDOUT, ECCS2STDERR, ECCS2PATH, ECCS2NUMPROCESSES, ECCS2LISTIDPSURL, ECCS2LISTIDPSFILE, ECCS2LISTFEDSURL, ECCS2LISTFEDSFILE +from eccs2properties import ECCS2STDOUT, ECCS2STDERR, ECCS2DIR, ECCS2NUMPROCESSES, ECCS2LISTIDPSURL, ECCS2LISTIDPSFILE, ECCS2LISTFEDSURL, ECCS2LISTFEDSFILE from subprocess import Popen,PIPE # Returns a Dict on "{ nameFed:reg_auth }" @@ -138,7 +138,7 @@ if __name__=="__main__": idpJsonList = getIdpList(list_eccs_idps,regAuth) num_idps = len(idpJsonList) - cmd_list = [["%s/eccs2.py \'%s\'" % (ECCS2PATH, json.dumps(idp))] for idp in idpJsonList] + cmd_list = [["%s/eccs2.py \'%s\'" % (ECCS2DIR, json.dumps(idp))] for idp in idpJsonList] proc_list = [] count = 0 @@ -150,4 +150,4 @@ if __name__=="__main__": asyncio.run(main(proc_list,stdout_file,stderr_file)) end = time.time() - print("Time taken in hh:mm:ss - %s", str(datetime.timedelta(seconds=end - start))) + print("Time taken in hh:mm:ss - ", str(datetime.timedelta(seconds=end - start)))