From 5dacd1f9135cf37c19b6940652dedea4e228b3ea Mon Sep 17 00:00:00 2001 From: Marco Malavolti <marco.malavolti@gmail.com> Date: Wed, 24 Jun 2020 23:35:31 +0200 Subject: [PATCH] Refactorized eccs2checks output and WebDriver exceptions --- .gitignore | 3 - README.md | 19 ++++- api.py | 152 ++++++++++++++--------------------- cleanAndRunEccs2.sh | 7 ++ eccs2.py | 187 ++++++++++++++++++++++++-------------------- eccs2properties.py | 25 +++--- html/.gitignore | 4 + output/.gitignore | 4 + runEccs2.py | 16 ++-- utils.py | 15 +++- web/script.js | 10 +++ 11 files changed, 239 insertions(+), 203 deletions(-) delete mode 100644 .gitignore create mode 100755 cleanAndRunEccs2.sh create mode 100644 html/.gitignore create mode 100644 output/.gitignore diff --git a/.gitignore b/.gitignore deleted file mode 100644 index 0fc8ee9..0000000 --- a/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -selenium_chromedriver.log -*.swp -*.txt diff --git a/README.md b/README.md index a47b8d4..7add029 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,21 @@ -# HOWTO Install and Configure ECCS-2 +# EduGAIN Connectivity Check Service 2 -* `sudo apt install python3 python3-pip chromium chromium-l10n git libapache2-mod-wsgi python3-dev` +# Requirements Hardware + +* OS: Debian 9,10 (tested) +* HDD: 10 GB +* RAM: 4 GB +* CPU: >= 2 vCPU + +# Requirements Software + +* Apache Server + WSGI +* Python 3.8 +* Selenim + Chromium Web Brower + +# HOWTO Install and Configure + +* `sudo apt install chromium chromium-l10n git jq` * `python3 -m pip install --user --upgrade pip virtualenv` * `python3 -m venv eccs2venv` * `source eccs2venv/bin/activate` (`deactivate` to exit Virtualenv) diff --git a/api.py b/api.py index 3e17f09..75f5a53 100755 --- a/api.py +++ b/api.py @@ -3,58 +3,17 @@ import logging import re -from eccs2properties import DAY,ECCS2LOGSDIR +from eccs2properties import DAY,ECCS2LOGSDIR,ECCS2OUTPUTDIR from flask import Flask, request, jsonify from flask_restful import Resource, Api from json import dumps, loads from logging.handlers import RotatingFileHandler from pathlib import PurePath +from utils import getLogger, getDriver app = Flask(__name__) api = Api(app) -def getLogger(filename,log_level="DEBUG",path="./"): - - logger = logging.getLogger(filename) - ch = logging.FileHandler(path+filename,'w','utf-8') - - if (log_level == "DEBUG"): - logger.setLevel(logging.DEBUG) - ch.setLevel(logging.DEBUG) - elif (log_level == "INFO"): - logger.setLevel(logging.INFO) - ch.setLevel(logging.INFO) - elif (log_level == "WARN"): - logger.setLevel(logging.WARN) - ch.setLevel(logging.WARN) - elif (log_level == "ERROR"): - logger.setLevel(logging.ERROR) - ch.setLevel(logging.ERROR) - elif (log_level == "CRITICAL"): - logger.setLevel(logging.CRITICAL) - ch.setLevel(logging.CRITICAL) - - formatter = logging.Formatter('%(message)s') - ch.setFormatter(formatter) - logger.addHandler(ch) - - return logger - - -# Setup Chromium Webdriver -def setup(): - - chrome_options = webdriver.ChromeOptions() - chrome_options.add_argument('--headless') - chrome_options.add_argument('--no-sandbox') - - driver = webdriver.Chrome('chromedriver', chrome_options=chrome_options) - - # Configure timeouts - driver.set_page_load_timeout(30) - driver.set_script_timeout(30) - - return driver # /eccs2/test class Test(Resource): @@ -67,7 +26,7 @@ class Checks(Resource): def get(self): app.logger.info("Request 'Checks'") - file_path = "%s/eccs2checks_%s.log" % (ECCS2LOGSDIR,DAY) + file_path = "%s/eccs2checks_%s.log" % (ECCS2OUTPUTDIR,DAY) date = PurePath(file_path).parts[-1].split('_')[1].split('.')[0] pretty = 0 status = None @@ -76,7 +35,7 @@ class Checks(Resource): if 'date' in request.args: app.logger.info("'date' parameter inserted") date = request.args['date'] - file_path = "%s/eccs2checks_%s.log" % (ECCS2LOGSDIR,date) + file_path = "%s/eccs2checks_%s.log" % (ECCS2OUTPUTDIR,date) if 'pretty' in request.args: app.logger.info("'pretty' parameter inserted") pretty = request.args['pretty'] @@ -97,41 +56,49 @@ class Checks(Resource): check_idp = check[0] check_sp = check[1] - check_status = check[2].rstrip("\n\r") + status_code = check[2] + check_time = check[3] + check_status = check[4].rstrip("\n\r") if (idp and status): - app.logger.info("Checks for 'idp' and 'status'.") + app.logger.info("Search for 'idp':'%s' and 'status':'%s'." % (idp,status)) if (idp == check_idp and status == check_status): result.append( { 'sp' : check_sp, 'idp' : check_idp, + 'check_time': check_time, + 'status_code': status_code, 'status' : check_status, 'date': date } ) elif (idp): - #app.logger.info(re.search(".*."+idp+".*.", check_idp, re.IGNORECASE)) - #app.logger.info(check_idp)) - app.logger.info("Checks for Idp '%s'." % idp) + app.logger.info("Search for 'idp':'%s'" % idp) if (re.search(".*."+idp+".*.", check_idp, re.IGNORECASE)): result.append( { 'sp' : check_sp, 'idp' : check_idp, + 'check_time': check_time, + 'status_code': status_code, 'status' : check_status, 'date': date } ) elif (status): - app.logger.info("Check for the status '%s'." % status) + app.logger.info("Search for 'status':'%s'." % status) if (status == check_status): + result.append( { 'sp' : check_sp, + 'idp' : check_idp, + 'check_time': check_time, + 'status_code': status_code, + 'status' : check_status, + 'date': date + } ) + else: + app.logger.info("All checks.") result.append( { 'sp' : check_sp, 'idp' : check_idp, + 'check_time': check_time, + 'status_code': status_code, 'status' : check_status, 'date': date } ) - else: - app.logger.info("All checks.") - result.append( { 'sp' : check_sp, - 'idp' : check_idp, - 'status' : check_status, - 'date': date - } ) if (pretty): pp_json = dumps(result, indent=4, sort_keys=True) @@ -154,7 +121,7 @@ class EccsResults(Resource): def get(self): app.logger.info("Request 'EccsResults'") - file_path = "%s/eccs2_%s.log" % (ECCS2LOGSDIR,DAY) + file_path = "%s/eccs2_%s.log" % (ECCS2OUTPUTDIR,DAY) date = PurePath(file_path).parts[-1].split('_')[1].split('.')[0] pretty = 0 status = None @@ -163,7 +130,7 @@ class EccsResults(Resource): if 'date' in request.args: app.logger.info("'date' parameter inserted") date = request.args['date'] - file_path = "%s/eccs2_%s.log" % (ECCS2LOGSDIR,date) + file_path = "%s/eccs2_%s.log" % (ECCS2OUTPUTDIR,date) if 'pretty' in request.args: app.logger.info("'pretty' parameter inserted") pretty = request.args['pretty'] @@ -197,23 +164,23 @@ class EccsResults(Resource): # SP-status-2 check[13] check = line.split(";") - idp_displayname = check[0].rstrip("\n\r") - idp_entity_id = check[1].rstrip("\n\r") - idp_reg_auth = check[2].rstrip("\n\r") - idp_tech_ctcs = check[3].rstrip("\n\r") - idp_supp_ctcs = check[4].rstrip("\n\r") - idp_checks_status = check[5].rstrip("\n\r") - sp1_entity_id = check[6].rstrip("\n\r") - sp1_check_time = check[7].rstrip("\n\r") - sp1_status_code = check[8].rstrip("\n\r") - sp1_check_status = check[9].rstrip("\n\r") - sp2_entity_id = check[10].rstrip("\n\r") - sp2_check_time = check[11].rstrip("\n\r") - sp2_status_code = check[12].rstrip("\n\r") + idp_displayname = check[0] + idp_entity_id = check[1] + idp_reg_auth = check[2] + idp_tech_ctcs = check[3] + idp_supp_ctcs = check[4] + idp_checks_status = check[5] + sp1_entity_id = check[6] + sp1_check_time = check[7] + sp1_status_code = check[8] + sp1_check_status = check[9] + sp2_entity_id = check[10] + sp2_check_time = check[11] + sp2_status_code = check[12] sp2_check_status = check[13].rstrip("\n\r") if (idp and status): - app.logger.info("Results for the idp '%s' with status '%s'" % (idp, status)) + app.logger.info("eccsresults: check for 'idp':'%s' with 'status':'%s'" % (idp, status)) if (idp == idp_entity_id and status == idp_checks_status): result.append( { @@ -228,21 +195,21 @@ class EccsResults(Resource): 'sp1' : { 'entityID' : sp1_entity_id, 'checkTime' : sp1_check_time, - 'status' : sp1_check_status, - 'statusCode' : sp1_status_code + 'statusCode' : sp1_status_code, + 'status' : sp1_check_status }, 'sp2' : { 'entityID' : sp2_entity_id, 'checkTime' : sp2_check_time, - 'status' : sp2_check_status, - 'statusCode' : sp2_status_code + 'statusCode' : sp2_status_code, + 'status' : sp2_check_status }, 'status' : idp_checks_status } ) elif (idp): #app.logger.info(re.search(".*."+idp+".*.", idp_entity_id, re.IGNORECASE)) #app.logger.info(idp_entity_id)) - app.logger.info("Results for IdP '%s'." % idp) + app.logger.info("eccsresults: results for IdP:'%s'." % idp) if (re.search(".*."+idp+".*.", idp_entity_id, re.IGNORECASE)): result.append( { @@ -257,18 +224,19 @@ class EccsResults(Resource): 'sp1' : { 'entityID' : sp1_entity_id, 'checkTime' : sp1_check_time, - 'status' : sp1_check_status, - 'statusCode' : sp1_status_code + 'statusCode' : sp1_status_code, + 'status' : sp1_check_status }, 'sp2' : { 'entityID' : sp2_entity_id, 'checkTime' : sp2_check_time, - 'status' : sp2_check_status, - 'statusCode' : sp2_status_code + 'statusCode' : sp2_status_code, + 'status' : sp2_check_status }, 'status' : idp_checks_status } ) elif (status): + app.logger.info("eccsresults: Search for 'status':'%s'." % status) if (status == idp_checks_status): result.append( { @@ -283,14 +251,14 @@ class EccsResults(Resource): 'sp1' : { 'entityID' : sp1_entity_id, 'checkTime' : sp1_check_time, - 'status' : sp1_check_status, - 'statusCode' : sp1_status_code + 'statusCode' : sp1_status_code, + 'status' : sp1_check_status }, 'sp2' : { 'entityID' : sp2_entity_id, 'checkTime' : sp2_check_time, - 'status' : sp2_check_status, - 'statusCode' : sp2_status_code + 'statusCode' : sp2_status_code, + 'status' : sp2_check_status }, 'status' : idp_checks_status } ) @@ -308,14 +276,14 @@ class EccsResults(Resource): 'sp1' : { 'entityID' : sp1_entity_id, 'checkTime' : sp1_check_time, - 'status' : sp1_check_status, - 'statusCode' : sp1_status_code + 'statusCode' : sp1_status_code, + 'status' : sp1_check_status }, 'sp2' : { 'entityID' : sp2_entity_id, 'checkTime' : sp2_check_time, - 'status' : sp2_check_status, - 'statusCode' : sp2_status_code + 'statusCode' : sp2_status_code, + 'status' : sp2_check_status }, 'status' : idp_checks_status } ) @@ -339,5 +307,5 @@ api.add_resource(EccsResults, '/eccs/eccsresults') # Route_3 if __name__ == '__main__': app.config['JSON_AS_ASCII'] = False - app.logger = getLogger("eccs2api.log", "INFO", ECCS2LOGSDIR) + app.logger = getLogger("eccs2api.log", ECCS2LOGSDIR, "w", "INFO") app.run(port='5002') diff --git a/cleanAndRunEccs2.sh b/cleanAndRunEccs2.sh new file mode 100755 index 0000000..8c7dba6 --- /dev/null +++ b/cleanAndRunEccs2.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +# Remove old IdP and Fed List +rm -f /opt/eccs2/input/*.json + +# Run ECCS2 +/opt/eccs2/runEccs2.py diff --git a/eccs2.py b/eccs2.py index 2d562de..40908c6 100755 --- a/eccs2.py +++ b/eccs2.py @@ -7,7 +7,8 @@ import re import requests import time -from eccs2properties import ECCS2LOGSDIR, ECCS2RESULTSLOG, ECCS2CHECKSLOG, FEDS_BLACKLIST, IDPS_BLACKLIST, ECCS2SPS, ECCS2SELENIUMDEBUG +from eccs2properties import DAY, ECCS2HTMLDIR, ECCS2LOGSDIR, ECCS2OUTPUTDIR, ECCS2RESULTSLOG, ECCS2CHECKSLOG, FEDS_BLACKLIST, IDPS_BLACKLIST, ECCS2SPS, ECCS2SELENIUMDEBUG +from pathlib import Path from selenium.webdriver.common.by import By from selenium.webdriver.common.keys import Keys from selenium.webdriver.support.ui import Select, WebDriverWait @@ -17,13 +18,14 @@ from urllib3.exceptions import MaxRetryError from urllib3.util import parse_url from utils import getLogger, getIdPContacts, getDriver + """ This script use Selenium and Chromium to select the IdP to check from a Shibboleth SP with the Shibboleth Embedded Discovery Service installed and configured to answer to all eduGAIN IdPs. The SPs used to check an IdP will be SP24(IDEM) and Attribute Viewer (SWITCH). The check will be passed when both SPs will return the authentication page of the IdP checked. """ -def checkIdP(sp,idp,logger): +def checkIdP(sp,idp): # Chromedriver MUST be instanced here to avoid problems with SESSION # Disable SSL requests warning messages @@ -33,19 +35,23 @@ def checkIdP(sp,idp,logger): fqdn_idp = parse_url(idp['entityID'])[2] driver = getDriver(fqdn_idp,debugSelenium) + # Exception of WebDriver raises + if (driver == None): + return None + # Configure Blacklists federation_blacklist = FEDS_BLACKLIST entities_blacklist = IDPS_BLACKLIST if (idp['registrationAuthority'] in federation_blacklist): check_time = datetime.datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%S') + 'Z' - logger.info("%s;%s;%s;NULL;Federation excluded from checks" % (idp['entityID'],sp,check_time)) - return (sp,check_time,"NULL","DISABLED") + #logger.info("%s;%s;%s;NULL;Federation excluded from checks" % (idp['entityID'],sp,check_time)) + return (idp['entityID'],sp,check_time,"NULL","DISABLED") if (idp['entityID'] in entities_blacklist): check_time = datetime.datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%S') + 'Z' - logger.info("%s;%s;%s;NULL;IdP excluded from checks" % (idp['entityID'],sp,check_time)) - return (sp,check_time,"NULL","DISABLED") + #logger.info("%s;%s;%s;NULL;IdP excluded from checks" % (idp['entityID'],sp,check_time)) + return (idp['entityID'],sp,check_time,"NULL","DISABLED") # Open SP, select the IDP from the EDS and press 'Enter' to reach the IdP login page to check try: @@ -56,9 +62,16 @@ def checkIdP(sp,idp,logger): page_source = driver.page_source samlrequest_url = driver.current_url + # Put the code of the page into an HTML file + Path("%s/%s" % (ECCS2HTMLDIR,DAY)).mkdir(parents=True, exist_ok=True) + fqdn_idp = parse_url(idp['entityID'])[2] + fqdn_sp = parse_url(sp)[2] + with open("%s/%s/%s---%s.html" % (ECCS2HTMLDIR,DAY,fqdn_idp,fqdn_sp),"w") as html: + html.write(page_source) + except TimeoutException as e: - logger.info("%s;%s;%s;999;Timeout" % (idp['entityID'],sp,check_time)) - return (sp,check_time,"999","Timeout") + #logger.info("%s;%s;999;%s;Timeout" % (idp['entityID'],sp,check_time)) + return (idp['entityID'],sp,check_time,"999","Timeout") except NoSuchElementException as e: # The input of the bootstrap tables are provided by "eccs2" and "eccs2checks" log. @@ -69,8 +82,8 @@ def checkIdP(sp,idp,logger): return None except UnexpectedAlertPresentException as e: - logger.info("%s;%s;%s;888;UnexpectedAlertPresent" % (idp['entityID'],sp,check_time)) - return (sp,check_time,"888","ERROR") + #logger.info("%s;%s;888;%s;UnexpectedAlertPresent" % (idp['entityID'],sp,check_time)) + return (idp['entityID'],sp,check_time,"888","ERROR") except WebDriverException as e: print("!!! WEB DRIVER EXCEPTION - RUN AGAIN THE COMMAND!!!") @@ -87,6 +100,7 @@ def checkIdP(sp,idp,logger): finally: driver.quit() + pattern_metadata = "Unable.to.locate(\sissuer.in|).metadata(\sfor|)|no.metadata.found|profile.is.not.configured.for.relying.party|Cannot.locate.entity|fail.to.load.unknown.provider|does.not.recognise.the.service|unable.to.load.provider|Nous.n'avons.pas.pu.(charg|charger).le.fournisseur.de service|Metadata.not.found|application.you.have.accessed.is.not.registered.for.use.with.this.service|Message.did.not.meet.security.requirements" pattern_username = '<input[\s]+[^>]*((type=\s*[\'"](text|email)[\'"]|user)|(name=\s*[\'"](name)[\'"]))[^>]*>'; @@ -98,7 +112,7 @@ def checkIdP(sp,idp,logger): try: headers = {'User-Agent':'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.61 Safari/537.36'} - status_code = requests.get(samlrequest_url, headers=headers, verify=False, timeout=30).status_code + status_code = str(requests.get(samlrequest_url, headers=headers, verify=False, timeout=30).status_code) except requests.exceptions.ConnectionError as e: #print("!!! REQUESTS STATUS CODE CONNECTION ERROR EXCEPTION !!!") @@ -132,88 +146,93 @@ def checkIdP(sp,idp,logger): if(metadata_not_found): - logger.info("%s;%s;%s;%s;No-eduGAIN-Metadata" % (idp['entityID'],sp,status_code,check_time)) - return (sp,check_time,status_code,"No-eduGAIN-Metadata") + #logger.info("%s;%s;%s;%s;No-eduGAIN-Metadata" % (idp['entityID'],sp,status_code,check_time)) + return (idp['entityID'],sp,check_time,status_code,"No-eduGAIN-Metadata") elif not username_found or not password_found: - logger.info("%s;%s;%s;%s;Invalid-Form" % (idp['entityID'],sp,status_code,check_time)) - return (sp,check_time,status_code,"Invalid-Form") + #logger.info("%s;%s;%s;%s;Invalid-Form" % (idp['entityID'],sp,status_code,check_time)) + return (idp['entityID'],sp,check_time,status_code,"Invalid-Form") else: - logger.info("%s;%s;%s;%s;OK" % (idp['entityID'],sp,status_code,check_time)) - return (sp,check_time,status_code,"OK") + #logger.info("%s;%s;%s;%s;OK" % (idp['entityID'],sp,status_code,check_time)) + return (idp['entityID'],sp,check_time,status_code,"OK") -def check(idp,sps,eccs2log,eccs2checksLog): - result = [] +def check(idp,sps,eccs2log): + results = [] for sp in sps: - resultCheck = checkIdP(sp,idp,eccs2checksLog) - result.append(resultCheck) - - listTechContacts = getIdPContacts(idp,'technical') - listSuppContacts = getIdPContacts(idp,'support') - - strTechContacts = ','.join(listTechContacts) - strSuppContacts = ','.join(listSuppContacts) - - # If all checks are 'OK', than the IdP consuming correctly eduGAIN Metadata. - if (result[0][3] == result[1][3] == "OK"): - # IdP-DisplayName;IdP-entityID;IdP-RegAuth;IdP-tech-ctc-1,IdP-tech-ctc-2;IdP-supp-ctc-1,IdP-supp-ctc-2;Status;SP-entityID-1;SP-check-time-1;SP-status-code-1;SP-result-1;SP-entityID-2;SP-check-time-2;SP-status-code-2;SP-result-2 - eccs2log.info("%s;%s;%s;%s;%s;%s;%s;%s;%s;%s;%s;%s;%s;%s" % ( - idp['displayname'].replace("'","'").split(';')[1].split('==')[0], - idp['entityID'], - idp['registrationAuthority'], - strTechContacts, - strSuppContacts, - 'OK', - result[0][0], - result[0][1], - result[0][2], - result[0][3], - result[1][0], - result[1][1], - result[1][2], - result[1][3])) - elif (result[0][3] == result[1][3] == "DISABLED"): - eccs2log.info("%s;%s;%s;%s;%s;%s;%s;%s;%s;%s;%s;%s;%s;%s" % ( - idp['displayname'].replace("'","'").split(';')[1].split('==')[0], - idp['entityID'], - idp['registrationAuthority'], - strTechContacts, - strSuppContacts, - 'DISABLE', - result[0][0], - result[0][1], - result[0][2], - result[0][3], - result[1][0], - result[1][1], - result[1][2], - result[1][3])) - elif (result[0][3] == None or result[1][3] == None): - # Do nothing - return - else: - eccs2log.info("%s;%s;%s;%s;%s;%s;%s;%s;%s;%s;%s;%s;%s;%s" % ( - idp['displayname'].replace("'","'").split(';')[1].split('==')[0], - idp['entityID'], - idp['registrationAuthority'], - strTechContacts, - strSuppContacts, - 'ERROR', - result[0][0], - result[0][1], - result[0][2], - result[0][3], - result[1][0], - result[1][1], - result[1][2], - result[1][3])) + resultCheck = checkIdP(sp,idp) + # Se il checkIdP ha successo, aggiungo alla lista dei check + # altrimenti no. + if resultCheck is not None: + results.append(resultCheck) + + if len(results) == 2: + with open("%s/%s" % (ECCS2OUTPUTDIR,ECCS2CHECKSLOG), 'a') as f: + for elem in results: + f.write(";".join(elem)) + f.write("\n") + + listTechContacts = getIdPContacts(idp,'technical') + listSuppContacts = getIdPContacts(idp,'support') + + strTechContacts = ','.join(listTechContacts) + strSuppContacts = ','.join(listSuppContacts) + + # If all checks are 'OK', than the IdP consuming correctly eduGAIN Metadata. + if (results[0][4] == results[1][4] == "OK"): + # IdP-DisplayName;IdP-entityID;IdP-RegAuth;IdP-tech-ctc-1,IdP-tech-ctc-2;IdP-supp-ctc-1,IdP-supp-ctc-2;Status;SP-entityID-1;SP-check-time-1;SP-status-code-1;SP-result-1;SP-entityID-2;SP-check-time-2;SP-status-code-2;SP-result-2 + eccs2log.info("%s;%s;%s;%s;%s;%s;%s;%s;%s;%s;%s;%s;%s;%s" % ( + idp['displayname'].replace("'","'").split(';')[1].split('==')[0], + idp['entityID'], + idp['registrationAuthority'], + strTechContacts, + strSuppContacts, + 'OK', + results[0][1], # SP-entityID-1 + results[0][2], # SP-check-time-1 + results[0][3], # SP-status-code-1 + results[0][4], # SP-result-1 + results[1][1], # SP-entityID-2 + results[1][2], # SP-check-time-2 + results[1][3], # SP-status-code-2 + results[1][4])) # SP-result-2 + elif (results[0][4] == results[1][4] == "DISABLED"): + eccs2log.info("%s;%s;%s;%s;%s;%s;%s;%s;%s;%s;%s;%s;%s;%s" % ( + idp['displayname'].replace("'","'").split(';')[1].split('==')[0], + idp['entityID'], + idp['registrationAuthority'], + strTechContacts, + strSuppContacts, + 'DISABLE', + results[0][1], + results[0][2], + results[0][3], + results[0][4], + results[1][1], + results[1][2], + results[1][3], + results[1][4])) + else: + eccs2log.info("%s;%s;%s;%s;%s;%s;%s;%s;%s;%s;%s;%s;%s;%s" % ( + idp['displayname'].replace("'","'").split(';')[1].split('==')[0], + idp['entityID'], + idp['registrationAuthority'], + strTechContacts, + strSuppContacts, + 'ERROR', + results[0][1], + results[0][2], + results[0][3], + results[0][4], + results[1][1], + results[1][2], + results[1][3], + results[1][4])) # MAIN if __name__=="__main__": - eccs2log = getLogger(ECCS2RESULTSLOG, ECCS2LOGSDIR, 'a', "INFO") - eccs2checksLog = getLogger(ECCS2CHECKSLOG, ECCS2LOGSDIR, 'a', "INFO") + eccs2log = getLogger(ECCS2RESULTSLOG, ECCS2OUTPUTDIR, 'a', "INFO") sps = ECCS2SPS @@ -224,4 +243,4 @@ if __name__=="__main__": idp = json.loads(args.idpJson[0]) - check(idp,sps,eccs2log,eccs2checksLog) + check(idp,sps,eccs2log) diff --git a/eccs2properties.py b/eccs2properties.py index 67027d0..a76852b 100644 --- a/eccs2properties.py +++ b/eccs2properties.py @@ -5,31 +5,34 @@ from datetime import date DAY = date.today().isoformat() ECCS2DIR = "/opt/eccs2" -ECCS2LOGSDIR = "%s/logs" % ECCS2DIR -ECCS2INPUTDIR = "%s/input" % ECCS2DIR # Input +ECCS2INPUTDIR = "%s/input" % ECCS2DIR ECCS2LISTIDPSURL = 'https://technical.edugain.org/api.php?action=list_eccs_idps&format=json' ECCS2LISTIDPSFILE = "%s/list_eccs_idps.json" % ECCS2INPUTDIR ECCS2LISTFEDSURL = 'https://technical.edugain.org/api.php?action=list_feds&opt=1&format=json' ECCS2LISTFEDSFILE = "%s/list_fed.json" % ECCS2INPUTDIR # Output +ECCS2OUTPUTDIR = "%s/output" % ECCS2DIR ECCS2RESULTSLOG = "eccs2_%s.log" % DAY ECCS2CHECKSLOG = "eccs2checks_%s.log" % DAY -ECCS2SELENIUMLOGDIR = "%s/selenium-logs" % ECCS2DIR -ECCS2STDOUT = "%s/stdout.log" % ECCS2LOGSDIR -ECCS2STDERR = "%s/stderr.log" % ECCS2LOGSDIR +ECCS2HTMLDIR = "%s/html" % ECCS2DIR +ECCS2FAILEDCMD = "%s/failed-cmd.sh" % ECCS2LOGSDIR -# Selenium Timeouts (in seconds) -ECCS2SELENIUMPAGELOADTIMEOUT = 30 -ECCS2SELENIUMSCRIPTTIMEOUT = 30 +# Selenium +ECCS2SELENIUMDEBUG = False +ECCS2SELENIUMLOGDIR = "%s/selenium-logs" % ECCS2DIR +ECCS2SELENIUMPAGELOADTIMEOUT = 30 #seconds +ECCS2SELENIUMSCRIPTTIMEOUT = 30 #seconds -# Selenium Debug Enable/Disable -ECCS2SELENIUMDEBUG = True +# Logs +ECCS2LOGSDIR = "%s/logs" % ECCS2DIR +ECCS2STDOUT = "%s/stdout_%s.log" % (ECCS2LOGSDIR,DAY) +ECCS2STDERR = "%s/stderr_%s.log" % (ECCS2LOGSDIR,DAY) # Number of processes to run in parallel -ECCS2NUMPROCESSES = 20 +ECCS2NUMPROCESSES = 30 # The 2 SPs that will be used to test each IdP ECCS2SPS = ["https://sp24-test.garr.it/secure", "https://attribute-viewer.aai.switch.ch/eds/"] diff --git a/html/.gitignore b/html/.gitignore new file mode 100644 index 0000000..5e7d273 --- /dev/null +++ b/html/.gitignore @@ -0,0 +1,4 @@ +# Ignore everything in this directory +* +# Except this file +!.gitignore diff --git a/output/.gitignore b/output/.gitignore new file mode 100644 index 0000000..5e7d273 --- /dev/null +++ b/output/.gitignore @@ -0,0 +1,4 @@ +# Ignore everything in this directory +* +# Except this file +!.gitignore diff --git a/runEccs2.py b/runEccs2.py index 70221e0..4785482 100755 --- a/runEccs2.py +++ b/runEccs2.py @@ -7,12 +7,12 @@ import json import time from utils import getListFeds, getListEccsIdps, getRegAuthDict, getIdpList -from eccs2properties import ECCS2STDOUT, ECCS2STDERR, ECCS2DIR, ECCS2NUMPROCESSES, ECCS2LISTIDPSURL, ECCS2LISTIDPSFILE, ECCS2LISTFEDSURL, ECCS2LISTFEDSFILE +from eccs2properties import ECCS2FAILEDCMD, ECCS2STDOUT, ECCS2STDERR, ECCS2DIR, ECCS2NUMPROCESSES, ECCS2LISTIDPSURL, ECCS2LISTIDPSFILE, ECCS2LISTFEDSURL, ECCS2LISTFEDSFILE from subprocess import Popen,PIPE # Run Command -async def run(name,queue,stdout_file,stderr_file): +async def run(name,queue,stdout_file,stderr_file,cmd_file): while True: # Get a "cmd item" out of the queue. cmd = await queue.get() @@ -27,15 +27,16 @@ async def run(name,queue,stdout_file,stderr_file): stdout, stderr = await proc.communicate() if stdout: - stdout_file.write(f'-----\n[cmd-out]\n{cmd}\n\n[stdout]\n{stdout.decode()}') + stdout_file.write('-----\n[cmd-out]\n%s\n\n[stdout]\n%s' % (cmd,stdout.decode())) if stderr: - stderr_file.write(f'-----\n[cmd-err]\n{cmd}\n\n[stderr]\n{stderr.decode()}') + stderr_file.write('-----\n[cmd-err]\n%s\n\n[stderr]\n%s' % (cmd,stderr.decode())) + cmd_file.write(cmd) # Notify the queue that the "work cmd" has been processed. queue.task_done() -async def main(cmd_list,stdout_file,stderr_file): +async def main(cmd_list,stdout_file,stderr_file,cmd_file): # Create a queue that we will use to store our "workload". queue = asyncio.Queue() @@ -47,7 +48,7 @@ async def main(cmd_list,stdout_file,stderr_file): tasks = [] for i in range(ECCS2NUMPROCESSES): - task = asyncio.create_task(run("cmd-{%d}" % i, queue, stdout_file, stderr_file)) + task = asyncio.create_task(run("cmd-{%d}" % i, queue, stdout_file, stderr_file, cmd_file)) tasks.append(task) # Wait until the queue is fully processed. @@ -80,6 +81,7 @@ if __name__=="__main__": stdout_file = open(ECCS2STDOUT,"w+") stderr_file = open(ECCS2STDERR,"w+") + cmd_file = open(ECCS2FAILEDCMD,"w+") # Prepare input file for ECCS2 regAuthDict = getRegAuthDict(list_feds) @@ -97,7 +99,7 @@ if __name__=="__main__": proc_list.append(cmd) count = count + 1 - asyncio.run(main(proc_list,stdout_file,stderr_file)) + asyncio.run(main(proc_list,stdout_file,stderr_file,cmd_file)) end = time.time() print("Time taken in hh:mm:ss - ", str(datetime.timedelta(seconds=end - start))) diff --git a/utils.py b/utils.py index ad16349..c2b8188 100644 --- a/utils.py +++ b/utils.py @@ -7,6 +7,7 @@ import requests from eccs2properties import ECCS2SELENIUMLOGDIR, ECCS2SELENIUMPAGELOADTIMEOUT, ECCS2SELENIUMSCRIPTTIMEOUT from selenium import webdriver +from selenium.common.exceptions import WebDriverException # Returns a Dict of "{ nameFed:reg_auth }" @@ -63,6 +64,7 @@ def getListEccsIdps(url, dest_file): # Use logger to produce files consumed by ECCS-2 API def getLogger(filename, path, mode, log_level="DEBUG"): + logger = logging.getLogger(filename) ch = logging.FileHandler("%s/%s" % (path,filename), mode,'utf-8') @@ -119,10 +121,15 @@ def getDriver(fqdn_idp=None,debugSelenium=False): # For DEBUG only (By default ChromeDriver logs only warnings/errors to stderr. # When debugging issues, it is helpful to enable more verbose logging.) - if (debugSelenium and fqdn_idp): - driver = webdriver.Chrome('chromedriver', options=chrome_options, service_args=['--verbose', '--log-path=%s/%s.log' % (ECCS2SELENIUMLOGDIR, fqdn_idp)]) - else: - driver = webdriver.Chrome('chromedriver', options=chrome_options) + try: + if (debugSelenium and fqdn_idp): + driver = webdriver.Chrome('chromedriver', options=chrome_options, service_args=['--verbose', '--log-path=%s/%s.log' % (ECCS2SELENIUMLOGDIR, fqdn_idp)]) + else: + driver = webdriver.Chrome('chromedriver', options=chrome_options) + except WebDriverException as e: + print("!!! WEB DRIVER EXCEPTION - RUN AGAIN THE COMMAND!!!") + print (e.__str__()) + return None # Configure timeouts driver.set_page_load_timeout("%d" % ECCS2SELENIUMPAGELOADTIMEOUT) diff --git a/web/script.js b/web/script.js index 8d89553..8460a72 100644 --- a/web/script.js +++ b/web/script.js @@ -1,3 +1,9 @@ +// use URL constructor and return hostname +function getHostname(url) { + const urlNew = new URL(url); + return urlNew.hostname; +} + /* Formatting function for row details - modify as you need */ function format ( d ) { // `d` is the original data object for the row @@ -16,6 +22,7 @@ function format ( d ) { '<td>Check Time</td>'+ '<td>Status Code</td>'+ '<td>Result Check</td>'+ + '<td>Page Source</td>'+ '</tr>'+ '<tr>'+ '<td>SP1:</td>'+ @@ -23,6 +30,7 @@ function format ( d ) { '<td>'+d.sp1.checkTime+'</td>'+ '<td>'+d.sp1.statusCode+'</td>'+ '<td>'+d.sp1.status+'</td>'+ + '<td><a href="/eccs2html/'+d.date+'/'+getHostname(d.entityID)+'---'+getHostname(d.sp1.entityID)+'.html" target="_blank">Click to open</a></td>'+ '</tr>'+ '<tr>'+ '<td>SP2:</td>'+ @@ -30,9 +38,11 @@ function format ( d ) { '<td>'+d.sp2.checkTime+'</td>'+ '<td>'+d.sp2.statusCode+'</td>'+ '<td>'+d.sp2.status+'</td>'+ + '<td><a href="/eccs2html/'+d.date+'/'+getHostname(d.entityID)+'---'+getHostname(d.sp2.entityID)+'.html" target="_blank">Click to open</a></td>'+ '</tr>'+ '</table>'; } + $(document).ready(function() { var table = $('#eccstable').DataTable( { -- GitLab