From 5dacd1f9135cf37c19b6940652dedea4e228b3ea Mon Sep 17 00:00:00 2001
From: Marco Malavolti <marco.malavolti@gmail.com>
Date: Wed, 24 Jun 2020 23:35:31 +0200
Subject: [PATCH] Refactorized eccs2checks output and WebDriver exceptions

---
 .gitignore          |   3 -
 README.md           |  19 ++++-
 api.py              | 152 ++++++++++++++---------------------
 cleanAndRunEccs2.sh |   7 ++
 eccs2.py            | 187 ++++++++++++++++++++++++--------------------
 eccs2properties.py  |  25 +++---
 html/.gitignore     |   4 +
 output/.gitignore   |   4 +
 runEccs2.py         |  16 ++--
 utils.py            |  15 +++-
 web/script.js       |  10 +++
 11 files changed, 239 insertions(+), 203 deletions(-)
 delete mode 100644 .gitignore
 create mode 100755 cleanAndRunEccs2.sh
 create mode 100644 html/.gitignore
 create mode 100644 output/.gitignore

diff --git a/.gitignore b/.gitignore
deleted file mode 100644
index 0fc8ee9..0000000
--- a/.gitignore
+++ /dev/null
@@ -1,3 +0,0 @@
-selenium_chromedriver.log
-*.swp
-*.txt
diff --git a/README.md b/README.md
index a47b8d4..7add029 100644
--- a/README.md
+++ b/README.md
@@ -1,6 +1,21 @@
-# HOWTO Install and Configure ECCS-2
+# EduGAIN Connectivity Check Service 2
 
-* `sudo apt install python3 python3-pip chromium chromium-l10n git libapache2-mod-wsgi python3-dev`
+# Requirements Hardware
+
+* OS: Debian 9,10 (tested)
+* HDD: 10 GB
+* RAM: 4 GB
+* CPU: >= 2 vCPU
+
+# Requirements Software
+
+* Apache Server + WSGI
+* Python 3.8
+* Selenim + Chromium Web Brower
+
+# HOWTO Install and Configure
+
+* `sudo apt install chromium chromium-l10n git jq`
 * `python3 -m pip install --user --upgrade pip virtualenv`
 * `python3 -m venv eccs2venv`
 * `source eccs2venv/bin/activate`   (`deactivate` to exit Virtualenv)
diff --git a/api.py b/api.py
index 3e17f09..75f5a53 100755
--- a/api.py
+++ b/api.py
@@ -3,58 +3,17 @@
 import logging
 import re
 
-from eccs2properties import DAY,ECCS2LOGSDIR
+from eccs2properties import DAY,ECCS2LOGSDIR,ECCS2OUTPUTDIR
 from flask import Flask, request, jsonify
 from flask_restful import Resource, Api
 from json import dumps, loads
 from logging.handlers import RotatingFileHandler
 from pathlib import PurePath
+from utils import getLogger, getDriver
 
 app = Flask(__name__)
 api = Api(app)
 
-def getLogger(filename,log_level="DEBUG",path="./"):
-
-    logger = logging.getLogger(filename)
-    ch = logging.FileHandler(path+filename,'w','utf-8')
-
-    if (log_level == "DEBUG"):
-       logger.setLevel(logging.DEBUG)
-       ch.setLevel(logging.DEBUG)
-    elif (log_level == "INFO"):
-       logger.setLevel(logging.INFO)
-       ch.setLevel(logging.INFO)
-    elif (log_level == "WARN"):
-       logger.setLevel(logging.WARN)
-       ch.setLevel(logging.WARN)
-    elif (log_level == "ERROR"):
-       logger.setLevel(logging.ERROR)
-       ch.setLevel(logging.ERROR)
-    elif (log_level == "CRITICAL"):
-       logger.setLevel(logging.CRITICAL)
-       ch.setLevel(logging.CRITICAL)
-
-    formatter = logging.Formatter('%(message)s')
-    ch.setFormatter(formatter)
-    logger.addHandler(ch)
-
-    return logger
-
-
-# Setup Chromium Webdriver
-def setup():
-
-   chrome_options = webdriver.ChromeOptions()
-   chrome_options.add_argument('--headless')
-   chrome_options.add_argument('--no-sandbox')
-
-   driver = webdriver.Chrome('chromedriver', chrome_options=chrome_options)
-
-   # Configure timeouts
-   driver.set_page_load_timeout(30)
-   driver.set_script_timeout(30)
-
-   return driver
 
 # /eccs2/test
 class Test(Resource):
@@ -67,7 +26,7 @@ class Checks(Resource):
     def get(self):
        app.logger.info("Request 'Checks'")
 
-       file_path = "%s/eccs2checks_%s.log" % (ECCS2LOGSDIR,DAY) 
+       file_path = "%s/eccs2checks_%s.log" % (ECCS2OUTPUTDIR,DAY) 
        date = PurePath(file_path).parts[-1].split('_')[1].split('.')[0]
        pretty = 0
        status = None 
@@ -76,7 +35,7 @@ class Checks(Resource):
        if 'date' in request.args:
           app.logger.info("'date' parameter inserted")
           date = request.args['date']
-          file_path = "%s/eccs2checks_%s.log" % (ECCS2LOGSDIR,date)
+          file_path = "%s/eccs2checks_%s.log" % (ECCS2OUTPUTDIR,date)
        if 'pretty' in request.args:
           app.logger.info("'pretty' parameter inserted")
           pretty = request.args['pretty']
@@ -97,41 +56,49 @@ class Checks(Resource):
 
           check_idp = check[0]
           check_sp = check[1]
-          check_status = check[2].rstrip("\n\r")
+          status_code = check[2]
+          check_time = check[3]
+          check_status = check[4].rstrip("\n\r")
 
           if (idp and status):
-              app.logger.info("Checks for 'idp' and 'status'.")
+              app.logger.info("Search for 'idp':'%s' and 'status':'%s'." % (idp,status))
               if (idp == check_idp and status == check_status):
                  result.append( { 'sp' : check_sp,
                                   'idp' : check_idp,
+                                  'check_time': check_time,
+                                  'status_code': status_code,
                                   'status' : check_status,
                                   'date': date
                                 } )
           elif (idp):
-              #app.logger.info(re.search(".*."+idp+".*.", check_idp, re.IGNORECASE))
-              #app.logger.info(check_idp))
-              app.logger.info("Checks for Idp '%s'." % idp)
+              app.logger.info("Search for 'idp':'%s'" % idp)
               if (re.search(".*."+idp+".*.", check_idp, re.IGNORECASE)):
                  result.append( { 'sp' : check_sp,
                                   'idp' : check_idp,
+                                  'check_time': check_time,
+                                  'status_code': status_code,
                                   'status' : check_status,
                                   'date': date
                                 } )
           elif (status):
-              app.logger.info("Check for the status '%s'." % status)
+              app.logger.info("Search for 'status':'%s'." % status)
               if (status == check_status):
+                  result.append( { 'sp' : check_sp,
+                                   'idp' : check_idp,
+                                   'check_time': check_time,
+                                   'status_code': status_code,
+                                   'status' : check_status,
+                                   'date': date
+                                 } )
+          else:
+                 app.logger.info("All checks.")
                  result.append( { 'sp' : check_sp,
                                   'idp' : check_idp,
+                                  'check_time': check_time,
+                                  'status_code': status_code,
                                   'status' : check_status,
                                   'date': date
                                 } )
-          else:
-             app.logger.info("All checks.")
-             result.append( { 'sp' : check_sp,
-                              'idp' : check_idp,
-                              'status' : check_status,
-                              'date': date
-                            } )
 
        if (pretty):
           pp_json = dumps(result, indent=4, sort_keys=True)
@@ -154,7 +121,7 @@ class EccsResults(Resource):
     def get(self):
        app.logger.info("Request 'EccsResults'")
 
-       file_path = "%s/eccs2_%s.log" % (ECCS2LOGSDIR,DAY) 
+       file_path = "%s/eccs2_%s.log" % (ECCS2OUTPUTDIR,DAY)
        date = PurePath(file_path).parts[-1].split('_')[1].split('.')[0]
        pretty = 0
        status = None
@@ -163,7 +130,7 @@ class EccsResults(Resource):
        if 'date' in request.args:
           app.logger.info("'date' parameter inserted")
           date = request.args['date']
-          file_path = "%s/eccs2_%s.log" % (ECCS2LOGSDIR,date)
+          file_path = "%s/eccs2_%s.log" % (ECCS2OUTPUTDIR,date)
        if 'pretty' in request.args:
           app.logger.info("'pretty' parameter inserted")
           pretty = request.args['pretty']
@@ -197,23 +164,23 @@ class EccsResults(Resource):
           # SP-status-2                      check[13]
           check = line.split(";")
 
-          idp_displayname = check[0].rstrip("\n\r")
-          idp_entity_id = check[1].rstrip("\n\r")
-          idp_reg_auth = check[2].rstrip("\n\r")
-          idp_tech_ctcs = check[3].rstrip("\n\r")
-          idp_supp_ctcs = check[4].rstrip("\n\r")
-          idp_checks_status = check[5].rstrip("\n\r")
-          sp1_entity_id = check[6].rstrip("\n\r")
-          sp1_check_time = check[7].rstrip("\n\r")
-          sp1_status_code = check[8].rstrip("\n\r")
-          sp1_check_status = check[9].rstrip("\n\r")
-          sp2_entity_id = check[10].rstrip("\n\r")
-          sp2_check_time = check[11].rstrip("\n\r")
-          sp2_status_code = check[12].rstrip("\n\r")
+          idp_displayname = check[0]
+          idp_entity_id = check[1]
+          idp_reg_auth = check[2]
+          idp_tech_ctcs = check[3]
+          idp_supp_ctcs = check[4]
+          idp_checks_status = check[5]
+          sp1_entity_id = check[6]
+          sp1_check_time = check[7]
+          sp1_status_code = check[8]
+          sp1_check_status = check[9]
+          sp2_entity_id = check[10]
+          sp2_check_time = check[11]
+          sp2_status_code = check[12]
           sp2_check_status = check[13].rstrip("\n\r")
 
           if (idp and status):
-              app.logger.info("Results for the idp '%s' with status '%s'" % (idp, status))
+              app.logger.info("eccsresults: check for 'idp':'%s' with 'status':'%s'" % (idp, status))
               if (idp == idp_entity_id and status == idp_checks_status):
                  result.append( 
                     { 
@@ -228,21 +195,21 @@ class EccsResults(Resource):
                         'sp1' : {
                             'entityID' : sp1_entity_id,
                             'checkTime' : sp1_check_time,
-                            'status' : sp1_check_status,
-                            'statusCode' : sp1_status_code
+                            'statusCode' : sp1_status_code,
+                            'status' : sp1_check_status
                         },
                         'sp2' : {
                             'entityID' : sp2_entity_id,
                             'checkTime' : sp2_check_time,
-                            'status' : sp2_check_status,
-                            'statusCode' : sp2_status_code
+                            'statusCode' : sp2_status_code,
+                            'status' : sp2_check_status
                         },
                         'status' : idp_checks_status
                     } )
           elif (idp):
               #app.logger.info(re.search(".*."+idp+".*.", idp_entity_id, re.IGNORECASE))
               #app.logger.info(idp_entity_id))
-              app.logger.info("Results for IdP '%s'." % idp)
+              app.logger.info("eccsresults: results for IdP:'%s'." % idp)
               if (re.search(".*."+idp+".*.", idp_entity_id, re.IGNORECASE)):
                  result.append( 
                     { 
@@ -257,18 +224,19 @@ class EccsResults(Resource):
                         'sp1' : {
                             'entityID' : sp1_entity_id,
                             'checkTime' : sp1_check_time,
-                            'status' : sp1_check_status,
-                            'statusCode' : sp1_status_code
+                            'statusCode' : sp1_status_code,
+                            'status' : sp1_check_status
                         },
                         'sp2' : {
                             'entityID' : sp2_entity_id,
                             'checkTime' : sp2_check_time,
-                            'status' : sp2_check_status,
-                            'statusCode' : sp2_status_code
+                            'statusCode' : sp2_status_code,
+                            'status' : sp2_check_status
                         },
                         'status' : idp_checks_status
                     } )
           elif (status):
+              app.logger.info("eccsresults: Search for 'status':'%s'." % status)
               if (status == idp_checks_status):
                  result.append( 
                     { 
@@ -283,14 +251,14 @@ class EccsResults(Resource):
                         'sp1' : {
                            'entityID' : sp1_entity_id,
                            'checkTime' : sp1_check_time,
-                           'status' : sp1_check_status,
-                           'statusCode' : sp1_status_code
+                           'statusCode' : sp1_status_code,
+                           'status' : sp1_check_status
                         },
                         'sp2' : {
                            'entityID' : sp2_entity_id,
                            'checkTime' : sp2_check_time,
-                           'status' : sp2_check_status,
-                           'statusCode' : sp2_status_code
+                           'statusCode' : sp2_status_code,
+                           'status' : sp2_check_status
                         },
                         'status' : idp_checks_status
                     } )
@@ -308,14 +276,14 @@ class EccsResults(Resource):
                  'sp1' : {
                     'entityID' : sp1_entity_id,
                     'checkTime' : sp1_check_time,
-                    'status' : sp1_check_status,
-                    'statusCode' : sp1_status_code
+                    'statusCode' : sp1_status_code,
+                    'status' : sp1_check_status
                  },
                  'sp2' : {
                     'entityID' : sp2_entity_id,
                     'checkTime' : sp2_check_time,
-                    'status' : sp2_check_status,
-                    'statusCode' : sp2_status_code
+                    'statusCode' : sp2_status_code,
+                    'status' : sp2_check_status
                  },
                  'status' : idp_checks_status
              } )
@@ -339,5 +307,5 @@ api.add_resource(EccsResults, '/eccs/eccsresults') # Route_3
 if __name__ == '__main__':
    
    app.config['JSON_AS_ASCII'] = False
-   app.logger = getLogger("eccs2api.log", "INFO", ECCS2LOGSDIR)
+   app.logger = getLogger("eccs2api.log", ECCS2LOGSDIR, "w", "INFO")
    app.run(port='5002')
diff --git a/cleanAndRunEccs2.sh b/cleanAndRunEccs2.sh
new file mode 100755
index 0000000..8c7dba6
--- /dev/null
+++ b/cleanAndRunEccs2.sh
@@ -0,0 +1,7 @@
+#!/bin/bash
+
+# Remove old IdP and Fed List
+rm -f /opt/eccs2/input/*.json
+
+# Run ECCS2
+/opt/eccs2/runEccs2.py
diff --git a/eccs2.py b/eccs2.py
index 2d562de..40908c6 100755
--- a/eccs2.py
+++ b/eccs2.py
@@ -7,7 +7,8 @@ import re
 import requests
 import time
 
-from eccs2properties import ECCS2LOGSDIR, ECCS2RESULTSLOG, ECCS2CHECKSLOG, FEDS_BLACKLIST, IDPS_BLACKLIST, ECCS2SPS, ECCS2SELENIUMDEBUG
+from eccs2properties import DAY, ECCS2HTMLDIR, ECCS2LOGSDIR, ECCS2OUTPUTDIR, ECCS2RESULTSLOG, ECCS2CHECKSLOG, FEDS_BLACKLIST, IDPS_BLACKLIST, ECCS2SPS, ECCS2SELENIUMDEBUG
+from pathlib import Path
 from selenium.webdriver.common.by import By
 from selenium.webdriver.common.keys import Keys
 from selenium.webdriver.support.ui import Select, WebDriverWait
@@ -17,13 +18,14 @@ from urllib3.exceptions import MaxRetryError
 from urllib3.util import parse_url
 from utils import getLogger, getIdPContacts, getDriver
 
+
 """
   This script use Selenium and Chromium to select the IdP to check from a Shibboleth SP with the Shibboleth Embedded Discovery Service installed and configured to answer to all eduGAIN IdPs.
   The SPs used to check an IdP will be SP24(IDEM) and Attribute Viewer (SWITCH). 
   The check will be passed when both SPs will return the authentication page of the IdP checked.
 """
 
-def checkIdP(sp,idp,logger):
+def checkIdP(sp,idp):
    # Chromedriver MUST be instanced here to avoid problems with SESSION
 
    # Disable SSL requests warning messages
@@ -33,19 +35,23 @@ def checkIdP(sp,idp,logger):
    fqdn_idp = parse_url(idp['entityID'])[2]
    driver = getDriver(fqdn_idp,debugSelenium)
 
+   # Exception of WebDriver raises
+   if (driver == None):
+      return None
+
    # Configure Blacklists
    federation_blacklist = FEDS_BLACKLIST
    entities_blacklist = IDPS_BLACKLIST 
 
    if (idp['registrationAuthority'] in federation_blacklist):
       check_time = datetime.datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%S') + 'Z'
-      logger.info("%s;%s;%s;NULL;Federation excluded from checks" % (idp['entityID'],sp,check_time))
-      return (sp,check_time,"NULL","DISABLED")
+      #logger.info("%s;%s;%s;NULL;Federation excluded from checks" % (idp['entityID'],sp,check_time))
+      return (idp['entityID'],sp,check_time,"NULL","DISABLED")
 
    if (idp['entityID'] in entities_blacklist):
       check_time = datetime.datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%S') + 'Z'
-      logger.info("%s;%s;%s;NULL;IdP excluded from checks" % (idp['entityID'],sp,check_time))
-      return (sp,check_time,"NULL","DISABLED")
+      #logger.info("%s;%s;%s;NULL;IdP excluded from checks" % (idp['entityID'],sp,check_time))
+      return (idp['entityID'],sp,check_time,"NULL","DISABLED")
 
    # Open SP, select the IDP from the EDS and press 'Enter' to reach the IdP login page to check
    try:
@@ -56,9 +62,16 @@ def checkIdP(sp,idp,logger):
       page_source = driver.page_source
       samlrequest_url = driver.current_url
 
+      # Put the code of the page into an HTML file
+      Path("%s/%s" % (ECCS2HTMLDIR,DAY)).mkdir(parents=True, exist_ok=True)
+      fqdn_idp = parse_url(idp['entityID'])[2]
+      fqdn_sp = parse_url(sp)[2]
+      with open("%s/%s/%s---%s.html" % (ECCS2HTMLDIR,DAY,fqdn_idp,fqdn_sp),"w") as html:
+           html.write(page_source)
+
    except TimeoutException as e:
-     logger.info("%s;%s;%s;999;Timeout" % (idp['entityID'],sp,check_time))
-     return (sp,check_time,"999","Timeout")
+     #logger.info("%s;%s;999;%s;Timeout" % (idp['entityID'],sp,check_time))
+     return (idp['entityID'],sp,check_time,"999","Timeout")
 
    except NoSuchElementException as e:
      # The input of the bootstrap tables are provided by "eccs2" and "eccs2checks" log.
@@ -69,8 +82,8 @@ def checkIdP(sp,idp,logger):
      return None
 
    except UnexpectedAlertPresentException as e:
-     logger.info("%s;%s;%s;888;UnexpectedAlertPresent" % (idp['entityID'],sp,check_time))
-     return (sp,check_time,"888","ERROR")
+     #logger.info("%s;%s;888;%s;UnexpectedAlertPresent" % (idp['entityID'],sp,check_time))
+     return (idp['entityID'],sp,check_time,"888","ERROR")
 
    except WebDriverException as e:
      print("!!! WEB DRIVER EXCEPTION - RUN AGAIN THE COMMAND!!!")
@@ -87,6 +100,7 @@ def checkIdP(sp,idp,logger):
    finally:
      driver.quit()
 
+
    pattern_metadata = "Unable.to.locate(\sissuer.in|).metadata(\sfor|)|no.metadata.found|profile.is.not.configured.for.relying.party|Cannot.locate.entity|fail.to.load.unknown.provider|does.not.recognise.the.service|unable.to.load.provider|Nous.n'avons.pas.pu.(charg|charger).le.fournisseur.de service|Metadata.not.found|application.you.have.accessed.is.not.registered.for.use.with.this.service|Message.did.not.meet.security.requirements"
 
    pattern_username = '<input[\s]+[^>]*((type=\s*[\'"](text|email)[\'"]|user)|(name=\s*[\'"](name)[\'"]))[^>]*>';
@@ -98,7 +112,7 @@ def checkIdP(sp,idp,logger):
 
    try:
       headers = {'User-Agent':'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.61 Safari/537.36'}
-      status_code = requests.get(samlrequest_url, headers=headers, verify=False, timeout=30).status_code
+      status_code = str(requests.get(samlrequest_url, headers=headers, verify=False, timeout=30).status_code)
 
    except requests.exceptions.ConnectionError as e:
      #print("!!! REQUESTS STATUS CODE CONNECTION ERROR EXCEPTION !!!")
@@ -132,88 +146,93 @@ def checkIdP(sp,idp,logger):
 
 
    if(metadata_not_found):
-      logger.info("%s;%s;%s;%s;No-eduGAIN-Metadata" % (idp['entityID'],sp,status_code,check_time))
-      return (sp,check_time,status_code,"No-eduGAIN-Metadata")
+      #logger.info("%s;%s;%s;%s;No-eduGAIN-Metadata" % (idp['entityID'],sp,status_code,check_time))
+      return (idp['entityID'],sp,check_time,status_code,"No-eduGAIN-Metadata")
    elif not username_found or not password_found:
-      logger.info("%s;%s;%s;%s;Invalid-Form" % (idp['entityID'],sp,status_code,check_time))
-      return (sp,check_time,status_code,"Invalid-Form")
+      #logger.info("%s;%s;%s;%s;Invalid-Form" % (idp['entityID'],sp,status_code,check_time))
+      return (idp['entityID'],sp,check_time,status_code,"Invalid-Form")
    else:
-      logger.info("%s;%s;%s;%s;OK" % (idp['entityID'],sp,status_code,check_time))
-      return (sp,check_time,status_code,"OK")
+      #logger.info("%s;%s;%s;%s;OK" % (idp['entityID'],sp,status_code,check_time))
+      return (idp['entityID'],sp,check_time,status_code,"OK")
 
 
-def check(idp,sps,eccs2log,eccs2checksLog):
-      result = []
+def check(idp,sps,eccs2log):
+      results = []
       for sp in sps:
-         resultCheck = checkIdP(sp,idp,eccs2checksLog)
-         result.append(resultCheck)
-
-      listTechContacts = getIdPContacts(idp,'technical')
-      listSuppContacts = getIdPContacts(idp,'support')
-
-      strTechContacts = ','.join(listTechContacts)
-      strSuppContacts = ','.join(listSuppContacts)
-
-      # If all checks are 'OK', than the IdP consuming correctly eduGAIN Metadata.
-      if (result[0][3] == result[1][3] == "OK"):
-         # IdP-DisplayName;IdP-entityID;IdP-RegAuth;IdP-tech-ctc-1,IdP-tech-ctc-2;IdP-supp-ctc-1,IdP-supp-ctc-2;Status;SP-entityID-1;SP-check-time-1;SP-status-code-1;SP-result-1;SP-entityID-2;SP-check-time-2;SP-status-code-2;SP-result-2
-         eccs2log.info("%s;%s;%s;%s;%s;%s;%s;%s;%s;%s;%s;%s;%s;%s" % (
-             idp['displayname'].replace("&apos;","'").split(';')[1].split('==')[0],
-             idp['entityID'],
-             idp['registrationAuthority'],
-             strTechContacts,
-             strSuppContacts,
-             'OK',
-             result[0][0],
-             result[0][1],
-             result[0][2],
-             result[0][3],
-             result[1][0],
-             result[1][1],
-             result[1][2],
-             result[1][3]))
-      elif (result[0][3] == result[1][3] == "DISABLED"):
-         eccs2log.info("%s;%s;%s;%s;%s;%s;%s;%s;%s;%s;%s;%s;%s;%s" % (
-             idp['displayname'].replace("&apos;","'").split(';')[1].split('==')[0],
-             idp['entityID'],
-             idp['registrationAuthority'],
-             strTechContacts,
-             strSuppContacts,
-             'DISABLE',
-             result[0][0],
-             result[0][1],
-             result[0][2],
-             result[0][3],
-             result[1][0],
-             result[1][1],
-             result[1][2],
-             result[1][3]))
-      elif (result[0][3] == None or result[1][3] == None):
-          # Do nothing
-          return
-      else:
-         eccs2log.info("%s;%s;%s;%s;%s;%s;%s;%s;%s;%s;%s;%s;%s;%s" % (
-             idp['displayname'].replace("&apos;","'").split(';')[1].split('==')[0],
-             idp['entityID'],
-             idp['registrationAuthority'],
-             strTechContacts,
-             strSuppContacts,
-             'ERROR',
-             result[0][0],
-             result[0][1],
-             result[0][2],
-             result[0][3],
-             result[1][0],
-             result[1][1],
-             result[1][2],
-             result[1][3]))
+         resultCheck = checkIdP(sp,idp)
+         # Se il checkIdP ha successo, aggiungo alla lista dei check
+         # altrimenti no.
+         if resultCheck is not None:
+            results.append(resultCheck)
+
+      if len(results) == 2:
+         with open("%s/%s" % (ECCS2OUTPUTDIR,ECCS2CHECKSLOG), 'a') as f:
+              for elem in results:
+                  f.write(";".join(elem))
+                  f.write("\n")
+
+         listTechContacts = getIdPContacts(idp,'technical')
+         listSuppContacts = getIdPContacts(idp,'support')
+
+         strTechContacts = ','.join(listTechContacts)
+         strSuppContacts = ','.join(listSuppContacts)
+
+         # If all checks are 'OK', than the IdP consuming correctly eduGAIN Metadata.
+         if (results[0][4] == results[1][4] == "OK"):
+            # IdP-DisplayName;IdP-entityID;IdP-RegAuth;IdP-tech-ctc-1,IdP-tech-ctc-2;IdP-supp-ctc-1,IdP-supp-ctc-2;Status;SP-entityID-1;SP-check-time-1;SP-status-code-1;SP-result-1;SP-entityID-2;SP-check-time-2;SP-status-code-2;SP-result-2
+            eccs2log.info("%s;%s;%s;%s;%s;%s;%s;%s;%s;%s;%s;%s;%s;%s" % (
+                idp['displayname'].replace("&apos;","'").split(';')[1].split('==')[0],
+                idp['entityID'],
+                idp['registrationAuthority'],
+                strTechContacts,
+                strSuppContacts,
+                'OK',
+                results[0][1],  # SP-entityID-1 
+                results[0][2],  # SP-check-time-1
+                results[0][3],  # SP-status-code-1
+                results[0][4],  # SP-result-1
+                results[1][1],  # SP-entityID-2
+                results[1][2],  # SP-check-time-2
+                results[1][3],  # SP-status-code-2
+                results[1][4])) # SP-result-2
+         elif (results[0][4] == results[1][4] == "DISABLED"):
+            eccs2log.info("%s;%s;%s;%s;%s;%s;%s;%s;%s;%s;%s;%s;%s;%s" % (
+                idp['displayname'].replace("&apos;","'").split(';')[1].split('==')[0],
+                idp['entityID'],
+                idp['registrationAuthority'],
+                strTechContacts,
+                strSuppContacts,
+                'DISABLE',
+                results[0][1],
+                results[0][2],
+                results[0][3],
+                results[0][4],
+                results[1][1],
+                results[1][2],
+                results[1][3],
+                results[1][4]))
+         else:
+            eccs2log.info("%s;%s;%s;%s;%s;%s;%s;%s;%s;%s;%s;%s;%s;%s" % (
+                idp['displayname'].replace("&apos;","'").split(';')[1].split('==')[0],
+                idp['entityID'],
+                idp['registrationAuthority'],
+                strTechContacts,
+                strSuppContacts,
+                'ERROR',
+                results[0][1],
+                results[0][2],
+                results[0][3],
+                results[0][4],
+                results[1][1],
+                results[1][2],
+                results[1][3],
+                results[1][4]))
 
 
 # MAIN
 if __name__=="__main__":
 
-   eccs2log = getLogger(ECCS2RESULTSLOG, ECCS2LOGSDIR, 'a', "INFO")
-   eccs2checksLog = getLogger(ECCS2CHECKSLOG, ECCS2LOGSDIR, 'a', "INFO")
+   eccs2log = getLogger(ECCS2RESULTSLOG, ECCS2OUTPUTDIR, 'a', "INFO")
 
    sps = ECCS2SPS
 
@@ -224,4 +243,4 @@ if __name__=="__main__":
 
    idp = json.loads(args.idpJson[0])
 
-   check(idp,sps,eccs2log,eccs2checksLog)
+   check(idp,sps,eccs2log)
diff --git a/eccs2properties.py b/eccs2properties.py
index 67027d0..a76852b 100644
--- a/eccs2properties.py
+++ b/eccs2properties.py
@@ -5,31 +5,34 @@ from datetime import date
 DAY = date.today().isoformat()
 
 ECCS2DIR = "/opt/eccs2"
-ECCS2LOGSDIR = "%s/logs" % ECCS2DIR
-ECCS2INPUTDIR = "%s/input" % ECCS2DIR
 
 # Input
+ECCS2INPUTDIR = "%s/input" % ECCS2DIR
 ECCS2LISTIDPSURL = 'https://technical.edugain.org/api.php?action=list_eccs_idps&format=json'
 ECCS2LISTIDPSFILE = "%s/list_eccs_idps.json" % ECCS2INPUTDIR
 ECCS2LISTFEDSURL = 'https://technical.edugain.org/api.php?action=list_feds&opt=1&format=json' 
 ECCS2LISTFEDSFILE = "%s/list_fed.json" % ECCS2INPUTDIR
 
 # Output
+ECCS2OUTPUTDIR = "%s/output" % ECCS2DIR
 ECCS2RESULTSLOG = "eccs2_%s.log" % DAY
 ECCS2CHECKSLOG = "eccs2checks_%s.log" % DAY
-ECCS2SELENIUMLOGDIR = "%s/selenium-logs" % ECCS2DIR
-ECCS2STDOUT = "%s/stdout.log" % ECCS2LOGSDIR
-ECCS2STDERR = "%s/stderr.log" % ECCS2LOGSDIR
+ECCS2HTMLDIR = "%s/html" % ECCS2DIR
+ECCS2FAILEDCMD = "%s/failed-cmd.sh" % ECCS2LOGSDIR
 
-# Selenium Timeouts (in seconds)
-ECCS2SELENIUMPAGELOADTIMEOUT = 30
-ECCS2SELENIUMSCRIPTTIMEOUT = 30
+# Selenium
+ECCS2SELENIUMDEBUG = False
+ECCS2SELENIUMLOGDIR = "%s/selenium-logs" % ECCS2DIR
+ECCS2SELENIUMPAGELOADTIMEOUT = 30 #seconds
+ECCS2SELENIUMSCRIPTTIMEOUT = 30   #seconds
 
-# Selenium Debug Enable/Disable
-ECCS2SELENIUMDEBUG = True
+# Logs
+ECCS2LOGSDIR = "%s/logs" % ECCS2DIR
+ECCS2STDOUT = "%s/stdout_%s.log" % (ECCS2LOGSDIR,DAY)
+ECCS2STDERR = "%s/stderr_%s.log" % (ECCS2LOGSDIR,DAY)
 
 # Number of processes to run in parallel
-ECCS2NUMPROCESSES = 20
+ECCS2NUMPROCESSES = 30
 
 # The 2 SPs that will be used to test each IdP
 ECCS2SPS = ["https://sp24-test.garr.it/secure", "https://attribute-viewer.aai.switch.ch/eds/"]
diff --git a/html/.gitignore b/html/.gitignore
new file mode 100644
index 0000000..5e7d273
--- /dev/null
+++ b/html/.gitignore
@@ -0,0 +1,4 @@
+# Ignore everything in this directory
+*
+# Except this file
+!.gitignore
diff --git a/output/.gitignore b/output/.gitignore
new file mode 100644
index 0000000..5e7d273
--- /dev/null
+++ b/output/.gitignore
@@ -0,0 +1,4 @@
+# Ignore everything in this directory
+*
+# Except this file
+!.gitignore
diff --git a/runEccs2.py b/runEccs2.py
index 70221e0..4785482 100755
--- a/runEccs2.py
+++ b/runEccs2.py
@@ -7,12 +7,12 @@ import json
 import time
 from utils import getListFeds, getListEccsIdps, getRegAuthDict, getIdpList
 
-from eccs2properties import ECCS2STDOUT, ECCS2STDERR, ECCS2DIR, ECCS2NUMPROCESSES, ECCS2LISTIDPSURL, ECCS2LISTIDPSFILE, ECCS2LISTFEDSURL, ECCS2LISTFEDSFILE 
+from eccs2properties import ECCS2FAILEDCMD, ECCS2STDOUT, ECCS2STDERR, ECCS2DIR, ECCS2NUMPROCESSES, ECCS2LISTIDPSURL, ECCS2LISTIDPSFILE, ECCS2LISTFEDSURL, ECCS2LISTFEDSFILE 
 from subprocess import Popen,PIPE
 
 
 # Run Command
-async def run(name,queue,stdout_file,stderr_file):
+async def run(name,queue,stdout_file,stderr_file,cmd_file):
    while True:
       # Get a "cmd item" out of the queue.
       cmd = await queue.get()
@@ -27,15 +27,16 @@ async def run(name,queue,stdout_file,stderr_file):
       stdout, stderr = await proc.communicate()
 
       if stdout:
-         stdout_file.write(f'-----\n[cmd-out]\n{cmd}\n\n[stdout]\n{stdout.decode()}')
+         stdout_file.write('-----\n[cmd-out]\n%s\n\n[stdout]\n%s' % (cmd,stdout.decode()))
       if stderr:
-         stderr_file.write(f'-----\n[cmd-err]\n{cmd}\n\n[stderr]\n{stderr.decode()}')
+         stderr_file.write('-----\n[cmd-err]\n%s\n\n[stderr]\n%s' % (cmd,stderr.decode()))
+         cmd_file.write(cmd)
 
       # Notify the queue that the "work cmd" has been processed.
       queue.task_done()
 
 
-async def main(cmd_list,stdout_file,stderr_file):
+async def main(cmd_list,stdout_file,stderr_file,cmd_file):
     # Create a queue that we will use to store our "workload".
     queue = asyncio.Queue()
 
@@ -47,7 +48,7 @@ async def main(cmd_list,stdout_file,stderr_file):
     tasks = []
 
     for i in range(ECCS2NUMPROCESSES):
-        task = asyncio.create_task(run("cmd-{%d}" % i, queue, stdout_file, stderr_file))
+        task = asyncio.create_task(run("cmd-{%d}" % i, queue, stdout_file, stderr_file, cmd_file))
         tasks.append(task)
 
     # Wait until the queue is fully processed.
@@ -80,6 +81,7 @@ if __name__=="__main__":
 
    stdout_file = open(ECCS2STDOUT,"w+")
    stderr_file = open(ECCS2STDERR,"w+")
+   cmd_file = open(ECCS2FAILEDCMD,"w+")
 
    # Prepare input file for ECCS2
    regAuthDict = getRegAuthDict(list_feds)
@@ -97,7 +99,7 @@ if __name__=="__main__":
          proc_list.append(cmd)
          count = count + 1
  
-      asyncio.run(main(proc_list,stdout_file,stderr_file))
+      asyncio.run(main(proc_list,stdout_file,stderr_file,cmd_file))
 
    end = time.time()
    print("Time taken in hh:mm:ss - ", str(datetime.timedelta(seconds=end - start)))
diff --git a/utils.py b/utils.py
index ad16349..c2b8188 100644
--- a/utils.py
+++ b/utils.py
@@ -7,6 +7,7 @@ import requests
 
 from eccs2properties import ECCS2SELENIUMLOGDIR, ECCS2SELENIUMPAGELOADTIMEOUT, ECCS2SELENIUMSCRIPTTIMEOUT
 from selenium import webdriver
+from selenium.common.exceptions import WebDriverException
 
 
 # Returns a Dict of "{ nameFed:reg_auth }"
@@ -63,6 +64,7 @@ def getListEccsIdps(url, dest_file):
 
 # Use logger to produce files consumed by ECCS-2 API
 def getLogger(filename, path, mode, log_level="DEBUG"):
+
     logger = logging.getLogger(filename)
     ch = logging.FileHandler("%s/%s" % (path,filename), mode,'utf-8')
 
@@ -119,10 +121,15 @@ def getDriver(fqdn_idp=None,debugSelenium=False):
 
     # For DEBUG only (By default ChromeDriver logs only warnings/errors to stderr.
     # When debugging issues, it is helpful to enable more verbose logging.)
-    if (debugSelenium and fqdn_idp):
-       driver = webdriver.Chrome('chromedriver', options=chrome_options,  service_args=['--verbose', '--log-path=%s/%s.log' % (ECCS2SELENIUMLOGDIR, fqdn_idp)])
-    else:
-       driver = webdriver.Chrome('chromedriver', options=chrome_options)
+    try:
+       if (debugSelenium and fqdn_idp):
+          driver = webdriver.Chrome('chromedriver', options=chrome_options,  service_args=['--verbose', '--log-path=%s/%s.log' % (ECCS2SELENIUMLOGDIR, fqdn_idp)])
+       else:
+          driver = webdriver.Chrome('chromedriver', options=chrome_options)
+    except WebDriverException as e:
+       print("!!! WEB DRIVER EXCEPTION - RUN AGAIN THE COMMAND!!!")
+       print (e.__str__())
+       return None
 
     # Configure timeouts
     driver.set_page_load_timeout("%d" % ECCS2SELENIUMPAGELOADTIMEOUT)
diff --git a/web/script.js b/web/script.js
index 8d89553..8460a72 100644
--- a/web/script.js
+++ b/web/script.js
@@ -1,3 +1,9 @@
+// use URL constructor and return hostname
+function getHostname(url) {
+   const urlNew = new URL(url);
+   return urlNew.hostname;
+}
+
 /* Formatting function for row details - modify as you need */
 function format ( d ) {
     // `d` is the original data object for the row
@@ -16,6 +22,7 @@ function format ( d ) {
             '<td>Check Time</td>'+
             '<td>Status Code</td>'+
             '<td>Result Check</td>'+
+            '<td>Page Source</td>'+
         '</tr>'+
         '<tr>'+
             '<td>SP1:</td>'+
@@ -23,6 +30,7 @@ function format ( d ) {
             '<td>'+d.sp1.checkTime+'</td>'+
             '<td>'+d.sp1.statusCode+'</td>'+
             '<td>'+d.sp1.status+'</td>'+
+            '<td><a href="/eccs2html/'+d.date+'/'+getHostname(d.entityID)+'---'+getHostname(d.sp1.entityID)+'.html" target="_blank">Click to open</a></td>'+
         '</tr>'+
         '<tr>'+
             '<td>SP2:</td>'+
@@ -30,9 +38,11 @@ function format ( d ) {
             '<td>'+d.sp2.checkTime+'</td>'+
             '<td>'+d.sp2.statusCode+'</td>'+
             '<td>'+d.sp2.status+'</td>'+
+            '<td><a href="/eccs2html/'+d.date+'/'+getHostname(d.entityID)+'---'+getHostname(d.sp2.entityID)+'.html" target="_blank">Click to open</a></td>'+
         '</tr>'+
     '</table>';
 }
+
  
 $(document).ready(function() {
     var table = $('#eccstable').DataTable( {
-- 
GitLab