From a6650dd92262f437a98f56959f05f26770d4d67f Mon Sep 17 00:00:00 2001
From: Marco Malavolti <marco.malavolti@gmail.com>
Date: Fri, 23 Jul 2021 01:06:14 +0200
Subject: [PATCH] ECCS2 First Working release

---
 README.md           |  62 ++++++----
 api.py              |   8 +-
 cleanAndRunEccs2.sh |  58 ---------
 eccs2.py            | 293 +++++++++-----------------------------------
 eccs2properties.py  |  25 ++--
 requirements.txt    |   8 +-
 runEccs2.py         | 116 +++++++++++-------
 utils.py            | 246 ++++++++++++++++++++++++++++++++-----
 web/eccs2.js        | 144 +++++++++++++++++++++-
 9 files changed, 547 insertions(+), 413 deletions(-)

diff --git a/README.md b/README.md
index 28d8e34..c0d2c7f 100644
--- a/README.md
+++ b/README.md
@@ -8,12 +8,12 @@
 6. [Requirements Hardware](#requirements-hardware)
 7. [Requirements Software](#requirements-software)
 8. [HOWTO Install and Configure](#howto-install-and-configure)
-   * [Install Python 3.8.x](#install-python-38x)
+   * [Install Python 3.9.x](#install-python-39x)
      + [CentOS 7 requirements](#centos-7-requirements)
      + [Debian requirements](#debian-requirements)
-     + [Python 3.8](#python-38)
+     + [Python 3.9](#python-39)
 9. [Install the Chromedriver](#install-the-chromedriver)
-10. [Install Chromium needed by Selenium](#install-chromium-needed-by-selenium)
+10. [Install Google Chrome needed by Selenium](#install-google-chrome-needed-by-selenium)
 11. [ECCS2 Script](#eccs2-script)
     * [Install](#install)
     * [Configure](#configure)
@@ -84,12 +84,15 @@ The tool uses following status for IdPs:
 * HDD: 10 GB
 * RAM: 4 GB
 * CPU: >= 2 vCPU (suggested)
+* ARCH: 64 Bit
 
 # Requirements Software
 
 * Apache Server + WSGI
-* Python 3.8 (tested with v3.8.3,v3.8.5)
-* Selenim + Chromium Web Brower
+* Python 3.9 (tested with v3.9.6)
+* Selenim + Google Chrome Web Brower (tested with v91.0.4472.164)
+* Chromedriver (tested with v91.0.4472.101)
+* Git
 
 # HOWTO Install and Configure
 
@@ -97,7 +100,7 @@ The tool uses following status for IdPs:
 
 * `cd $HOME ; git clone https://github.com/malavolti/eccs2.git`
 
-## Install Python 3.8.x
+## Install Python 3.9.x
 
 ### CentOS 7 requirements
 
@@ -110,6 +113,9 @@ The tool uses following status for IdPs:
 3. Install needed packages to build python:
    * `sudo yum -y install openssl-devel bzip2-devel libffi-devel wget`
 
+4. Install Git:
+   * `sudo yum -y install git`
+
 ### Debian requirements
 
 1. Update the system packages:
@@ -118,45 +124,49 @@ The tool uses following status for IdPs:
 2. Install needed packages to build python:
    * `sudo apt install build-essential zlib1g-dev libncurses5-dev libgdbm-dev libnss3-dev libssl-dev libreadline-dev libffi-dev`
 
-### Python 3.8
+3. Install Git:
+   * `sudo apt install git`
+
+### Python 3.9
 
-1. Download the last version of Python 3.8.x from https://www.python.org/downloads/source/ into your home:
-   * `wget https://www.python.org/ftp/python/3.8.5/Python-3.8.5.tgz -O $HOME/eccs2/Python-3.8.5.tgz`
+1. Download the last version of Python 3.9.x from https://www.python.org/downloads/source/ into your home:
+   * `wget https://www.python.org/ftp/python/3.9.6/Python-3.9.6.tgz -O $HOME/eccs2/Python-3.9.6.tgz`
 
 2. Extract Python source package:
    * `cd $HOME/eccs2/`
-   * `tar xzf Python-3.8.5.tgz`
+   * `tar xzf Python-3.9.6.tgz`
 
 3. Build Python from the source package:
-   * `cd $HOME/eccs2/Python-3.8.5`
+   * `cd $HOME/eccs2/Python-3.9.6`
    * `./configure --prefix=$HOME/eccs2/python`
    * `make`
 
-4. Install Python 3.8.x under `$HOME/eccs2/python`:
+4. Install Python 3.9.x under `$HOME/eccs2/python`:
    * `make install`
-   * `$HOME/eccs2/python/bin/python3.8 --version`
+   * `$HOME/eccs2/python/bin/python3.9 --version`
  
    This will install python under your $HOME directory.
    
 5. Remove useless things:
-   * `rm -Rf $HOME/eccs2/Python-3.8.5 $HOME/eccs2/Python-3.8.5.tgz`
+   * `rm -Rf $HOME/eccs2/Python-3.9.9 $HOME/eccs2/Python-3.9.6.tgz`
 
-# Install Chromium needed by Selenium
+# Install Google Chrome needed by Selenium
 
-* Debian:
-  * `sudo apt install chromium git jq`
+* Debian (64 bit):
+  * `sudo wget https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb`
+  * `sudo apt install ./google-chrome-stable_current_amd64.deb`
 
-* CentOS:
-  * `sudo yum install -y epel-release`
-  * `sudo yum install -y chromium git jq`
+* CentOS (64 bit):
+  * `sudo wget https://dl.google.com/linux/direct/google-chrome-stable_current_x86_64.rpm`
+  * `sudo yum install ./google-chrome-stable_current_x86_64.rpm`
 
 # Install the Chromedriver
 
 1. Find out which version of Chromium you are using:
    * Debian 9 (stretch):
-     * `chromium -version` => Chromium 73.0.3683.75
+     * `google-chrome -version` => Google Chrome 91.0.4472.164
    * CentOS 7.8:
-     * `chromium-browser -version` => Chromium 83.0.4103.116
+     * `google-chrome -version` => Google Chrome 91.0.4472.164
 
 2. Take the Chrome version number, remove the last part, and append the result to URL "`https://chromedriver.storage.googleapis.com/LATEST_RELEASE_`". For example, with Chrome version 73.0.3683.75, you'd get a URL "`https://chromedriver.storage.googleapis.com/LATEST_RELEASE_73.0.3683`".
 
@@ -178,8 +188,8 @@ After the initial download, it is recommended that you occasionally go through t
 ## Install
 
 * `cd $HOME/eccs2`
-* `./python/bin/python3.8 -m pip install virtualenv`
-* `$HOME/eccs2/python/bin/virtualenv --python=$HOME/eccs2/python/bin/python3.8 eccs2venv`
+* `./python/bin/python3.9 -m pip install virtualenv`
+* `$HOME/eccs2/python/bin/virtualenv --python=$HOME/eccs2/python/bin/python3.9 eccs2venv`
 * `source eccs2venv/bin/activate`   (`deactivate` to exit Virtualenv)
   * `python -m pip install -r requirements.txt`
 
@@ -228,11 +238,11 @@ After the initial download, it is recommended that you occasionally go through t
   * `./runEccs2.py --idp <IDP-ENTITYID>` (to run check on a single IdP)
   * `./runEccs2.py --test` (to run a full check without effects)
   * `./runEccs2.py --idp <IDP-ENTITYID> --test` (to run check on a single IdP without effects)
+  * `./runEccs2.py --idp <IDP-ENTITYID> --replace` (to run check on a single IdP and replace, or add, a result)
 
-  The check will run a second time for those IdPs that failed the first execution of the script.
   If something prevent the good execution of the ECCS2's check, the `logs/failed-cmd.sh` file will be not empty at the end of the execution.
 
-  The "--test" parameter will not change the result of ECCS2, but will write the output on the `logs/stdout_idp_YYYY-MM-DD.log`,`logs/stderr_idp_YYYY-MM-DD.log` and `logs/failed-cmd-idp.sh` files.
+  The "--test" parameter will not change the result of ECCS2, but will write the output on the `logs/stdout_idp_YYYY-MM-DD.log`,`logs/stderr_idp_YYYY-MM-DD.log` and `logs/failed-cmd-idp.sh` files if the argument "--test" will be used.
 
 # ECCS2 API Server (uWSGI)
 
diff --git a/api.py b/api.py
index 58d0c65..32c4783 100755
--- a/api.py
+++ b/api.py
@@ -7,7 +7,7 @@ import re
 from eccs2properties import DAY, ECCS2LOGSDIR, ECCS2OUTPUTDIR, ECCS2LISTFEDSURL, ECCS2LISTFEDSFILE
 from flask import Flask, request, jsonify
 from flask_restful import Resource, Api
-from utils import getLogger, getListFeds, getRegAuthDict
+from utils import get_logger, get_list_feds, get_reg_auth_dict
 
 app = Flask(__name__)
 api = Api(app)
@@ -175,8 +175,8 @@ class EccsResults(Resource):
 # /api/fedstats
 class FedStats(Resource):
    def get(self):
-       list_feds = getListFeds(ECCS2LISTFEDSURL, ECCS2LISTFEDSFILE)
-       regAuthDict = getRegAuthDict(list_feds)
+       list_feds = get_list_feds(ECCS2LISTFEDSURL, ECCS2LISTFEDSFILE)
+       regAuthDict = get_reg_auth_dict(list_feds)
 
        file_path = "%s/eccs2_%s.log" % (ECCS2OUTPUTDIR,DAY)
        date = DAY
@@ -263,5 +263,5 @@ if __name__ == '__main__':
    # Useful only for API development Server
    #app.config['JSON_AS_ASCII'] = True
    #app.logger.removeHandler(default_handler)
-   #app.logger = getLogger("eccs2api.log", ECCS2LOGSDIR, "w", "INFO")
+   #app.logger = get_logger("eccs2api.log", ECCS2LOGSDIR, "w", "INFO")
    app.run(port='5002')
diff --git a/cleanAndRunEccs2.sh b/cleanAndRunEccs2.sh
index 58b981c..dbc494d 100755
--- a/cleanAndRunEccs2.sh
+++ b/cleanAndRunEccs2.sh
@@ -11,61 +11,3 @@ rm -f $BASEDIR/eccs2/input/*.json
 
 # Run ECCS2
 $BASEDIR/eccs2/runEccs2.py
-
-# Run Failed Command again
-bash $BASEDIR/eccs2/logs/failed-cmd.sh
-
-date=$(date '+%Y-%m-%d')
-file="$BASEDIR/eccs2/logs/failed-cmd.sh"
-prefix="$BASEDIR/eccs2/eccs2.py '"
-suffix="'"
-eccs2output="$BASEDIR/eccs2/output/eccs2_$date.log"
-declare -a eccs2cmdToRemoveArray
-
-# If the ECCS2 output contains the result of a failed command (failed-cmd.sh),
-# than remove the failed command from the failed-cmd.sh file
-if [ -s $eccsoutput ]; then
-   if [ -s $file ]; then
-      while IFS= read -r line
-      do
-         string=$line
-
-         #remove "prefix" from the command string at the beginning.
-         prefix_removed_string=${string/#$prefix}
-
-         #remove "suffix" from the command string at the end.
-         suffix_removed_string=${prefix_removed_string/%$suffix}
-
-         entityIDidp=$(echo "$suffix_removed_string" | jq '.entityID')
-
-         #remove start and end quotes from the entityIDidp to be able to use "grep"
-         entityIDidp="${entityIDidp:1}"
-         entityIDidp="${entityIDidp%?}"
-
-         result=$(grep $entityIDidp $eccs2output | wc -l)
-   
-         if [[ "$result" = 1 ]]; then
-            eccs2cmdToRemoveArray+=("$entityIDidp")
-         else
-            echo "The result for the IdP '$entityIDidp' has been found multiple times on $eccs2output. It is wrong."
-         fi
-
-      done <"$file"
-
-      # Remove IdP command that had success from "failed-cmd.sh"
-      for idpToRemove in ${eccs2cmdToRemoveArray[@]}
-      do
-         $(grep -v $idpToRemove $file > temp ; mv -f temp $file)
-      done
-
-      if [ -s $file ]; then
-         echo "$date - ECCS2 NOT OK: Some eduGAIN IdPs have remained unchecked. See the 'logs/failed-cmd.sh' and logs/stderr_$date.log files"
-      else
-         echo "$date - ECCS2 OK: All eduGAIN IdPs have been checked successfully"
-      fi
-   else
-      echo "$date - ECCS2 OK: All eduGAIN IdPs have been checked successfully"
-   fi
-else
-   echo "$date - Something went wrong and the ECCS2 check has not been executed"
-fi
diff --git a/eccs2.py b/eccs2.py
index 02c76d1..0777b84 100755
--- a/eccs2.py
+++ b/eccs2.py
@@ -1,212 +1,20 @@
 #!/usr/bin/env python3
 
 import argparse
-import datetime
 import json
-import re
-import requests
 import sys
+import utils
+import eccs2properties as e2p
 
-from eccs2properties import DAY, ECCS2HTMLDIR, ECCS2OUTPUTDIR, ECCS2RESULTSLOG, ECCS2SPS, ECCS2SELENIUMDEBUG,ROBOTS_USER_AGENT,ECCS2REQUESTSTIMEOUT, FEDS_DISABLED_DICT, IDPS_DISABLED_DICT, ECCS2SELENIUMPAGELOADTIMEOUT
 from pathlib import Path
-from selenium.common.exceptions import TimeoutException
-from urllib3.util import parse_url
-from utils import getLogger, getIdPContacts, getDriver
-
 
 """
 The check works with the wayfless url of two SP and successed if the IdP Login Page appears and contains the fields "username" and "password" for each of them.
-It is possible to disable the check by eccs2properties with *denylist or by "robots.txt" put on the SAMLRequest endpoint root web dir.
+It is possible to disable the check by eccs2properties IDP_DISABLE_DICT or by "robots.txt" put on the SAMLRequest endpoint root web dir.
 """
 
-# Returns the FQDN to use on the HTML page_source files
-def getIDPlabel(url_or_urn):
-    if url_or_urn.startswith('http'):
-       return parse_url(url_or_urn)[2]
-    else:
-       return url_or_urn.split(":")[-1] 
-
-def getIDPfqdn(samlrequest_url):
-    return getIDPlabel(samlrequest_url)
-
-# This function checks if an IdP recognized the SP by presenting its Login page with "username" and "password" fields.
-# It is possible to disable the check on eccs2properties with the *denylist or by "robots.txt" file into the SAMLRequest endpoint root web dir.
-# If the IdP Login page contains "username" and "password" fields the test is passed.
-def checkIdP(sp,idp,test):
-
-   # Disable SSL requests warning messages
-   requests.packages.urllib3.disable_warnings()
-
-   debug_selenium = ECCS2SELENIUMDEBUG
-   label_idp = getIDPlabel(idp['entityID'])
-   # WebDriver MUST be instanced here to avoid problems with SESSION
-   driver = getDriver(label_idp,debug_selenium)
-
-   # Exception of WebDriver raises
-   if (driver == None):
-      return None
-
-   # Configure Blacklists
-   federations_disabled_dict = FEDS_DISABLED_DICT
-   idps_disabled_dict = IDPS_DISABLED_DICT
-
-   fqdn_sp = parse_url(sp)[2]
-   wayfless_url = sp + idp['entityID']
-
-   robots = ""
-
-   if (idp['registrationAuthority'] in federations_disabled_dict.keys()):
-      check_time = datetime.datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%S') + 'Z'
-
-      if (test is not True):
-         with open("%s/%s/%s---%s.html" % (ECCS2HTMLDIR,DAY,label_idp,fqdn_sp),"w") as html:
-              html.write("%s" % federations_disabled_dict[idp['registrationAuthority']])
-      else:
-         print("%s" % federations_disabled_dict[idp['registrationAuthority']])
-
-      return (idp['entityID'],wayfless_url,check_time,"NULL","DISABLED")
-
-   if (idp['entityID'] in idps_disabled_dict.keys()):
-      check_time = datetime.datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%S') + 'Z'
-
-      if (test is not True):
-         with open("%s/%s/%s---%s.html" % (ECCS2HTMLDIR,DAY,label_idp,fqdn_sp),"w") as html:
-              html.write("%s" % idps_disabled_dict[idp['entityID']])
-      else:
-         print("%s" % idps_disabled_dict[idp['entityID']])
-
-      return (idp['entityID'],wayfless_url,check_time,"NULL","DISABLED")
-
-   # Open SP via wayfless_url and reach the IdP login page to check
-   try:
-      check_time = datetime.datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%S') + 'Z'
-      driver.get(wayfless_url)
-      page_source = driver.page_source
-      samlrequest_url = driver.current_url
-
-      if (test is not True):
-         # Put the page_source into an appropriate HTML file
-         with open("%s/%s/%s---%s.html" % (ECCS2HTMLDIR,DAY,label_idp,fqdn_sp),"w") as html:
-              html.write(page_source)
-      else:
-         print("\n[page_source of '%s' for sp '%s']\n%s" % (label_idp,fqdn_sp,page_source))
-
-   except TimeoutException as e:
-     if (test is not True):
-        # Put an empty string into the page_source file
-        with open("%s/%s/%s---%s.html" % (ECCS2HTMLDIR,DAY,label_idp,fqdn_sp),"w") as html:
-             html.write("<html><h1>The IdP Login page was not loaded within %d seconds.</h1></html>" % ECCS2SELENIUMPAGELOADTIMEOUT )
-     else:
-        print("\n[page_source of '%s' for sp '%s']\nNo source code" % (label_idp,fqdn_sp))
-     return (idp['entityID'],wayfless_url,check_time,"(failed)","Timeout")
-
-   except Exception as e:
-     print ("!!! EXCEPTION DRIVER !!!")
-     print (e.__str__())
-     print ("IdP: %s\nSP: %s" % (idp['entityID'],sp))
-     return None
-
-   finally:
-     driver.quit()
-
-   try:
-      headers = {
-         'User-Agent': '%s' % ROBOTS_USER_AGENT
-      }
-
-      fqdn_idp = getIDPfqdn(samlrequest_url)
-
-      robots = requests.get("https://%s/robots.txt" % fqdn_idp, headers=headers, verify=True, timeout=ECCS2REQUESTSTIMEOUT)
-
-      if (robots == ""):
-         robots  = requests.get("http://%s/robots.txt" % fqdn_idp, headers=headers, verify=False, timeout=ECCS2REQUESTSTIMEOUT)
-
-   # Catch SSL Exceptions and block the ECCS check
-   except (requests.exceptions.SSLError) as e:
-      check_time = datetime.datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%S') + 'Z'
-
-      if (test is not True):
-         with open("%s/%s/%s---%s.html" % (ECCS2HTMLDIR,DAY,label_idp,fqdn_sp),"w") as html:
-             html.write("<p>IdP excluded from check due the following SSL Error:<br/><br/>%s</p><p>Check it on SSL Labs: <a href='https://www.ssllabs.com/ssltest/analyze.html?d=%s'>Click Here</a></p>" % (e.__str__(),fqdn_idp))
-      else:
-          print("IdP excluded from check due the following SSL Error:\n\n%s\n\nCheck it on SSL Labs: https://www.ssllabs.com/ssltest/analyze.html?d=%s" % (e.__str__(),fqdn_idp))
-
-      return (idp['entityID'],wayfless_url,check_time,"(failed)","SSL-Error")
-
-   # Pass every other exceptions on /robots.txt file. Consider only SSL Exceptions.
-   except Exception as e:
-      #print("IdP '%s' HAD HAD A REQUEST ERROR: %s" % (fqdn_idp,e.__str__()))
-      robots = ""
-
-   if (robots):
-      check_time = datetime.datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%S') + 'Z'
-
-      p = re.compile('^User-agent:\sECCS\sDisallow:\s\/\s*$', re.MULTILINE)
-      m = p.search(robots.text)
-
-      if (m):
-         if (test is not True):
-            with open("%s/%s/%s---%s.html" % (ECCS2HTMLDIR,DAY,label_idp,fqdn_sp),"w") as html:
-                 html.write("IdP excluded from check by robots.txt")
-         else:
-            print("IdP excluded from check by robots.txt")
-
-         return (idp['entityID'],wayfless_url,check_time,"NULL","DISABLED")
-
-
-   pattern_metadata = "Unable.to.locate(\sissuer.in|).metadata(\sfor|)|no.metadata.found|profile.is.not.configured.for.relying.party|Cannot.locate.entity|fail.to.load.unknown.provider|does.not.recognise.the.service|unable.to.load.provider|Nous.n'avons.pas.pu.(charg|charger).le.fournisseur.de service|Metadata.not.found|application.you.have.accessed.is.not.registered.for.use.with.this.service|Message.did.not.meet.security.requirements"
-
-   pattern_username = '<input[\s]+[^>]*((type=\s*[\'"](text|email)[\'"]|user)|(name=\s*[\'"](name)[\'"]))[^>]*>';
-   pattern_password = '<input[\s]+[^>]*(type=\s*[\'"]password[\'"]|password)[^>]*>';
-
-   metadata_not_found = re.search(pattern_metadata,page_source, re.I)
-   username_found = re.search(pattern_username,page_source, re.I)
-   password_found = re.search(pattern_password,page_source, re.I)
-
-   try:
-      headers = {'User-Agent':'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.61 Safari/537.36'}
-      http_code = str(requests.get(samlrequest_url, headers=headers, verify=False, timeout=ECCS2REQUESTSTIMEOUT).status_code)
-
-   except requests.exceptions.ConnectionError as e:
-     print ("http-code: (failed) - ConnectionError for IdP '%s' with SP '%s'" % (idp['entityID'],sp))
-     #print("!!! REQUESTS HTTP CODE CONNECTION ERROR EXCEPTION !!!")
-     #print (e.__str__())
-     http_code = "(failed)"
-
-   except requests.exceptions.Timeout as e:
-     print ("http-code: (failed) - TimeoutError for IdP '%s' with SP '%s'" % (idp['entityID'],sp))
-     #print("!!! REQUESTS HTTP CODE TIMEOUT EXCEPTION !!!")
-     #print (e.__str__())
-     http_code = "(failed)"
-
-   except requests.exceptions.TooManyRedirects as e:
-     print ("http-code: (failed) - TooManyRedirectsError for IdP '%s' with SP '%s'" % (idp['entityID'],sp))
-     #print("!!! REQUESTS HTTP CODE TOO MANY REDIRECTS EXCEPTION !!!")
-     #print (e.__str__())
-     http_code = "(failed)"
-
-   except requests.exceptions.RequestException as e:
-     print ("http-code: (failed) - RequestException for IdP '%s' with SP '%s'" % (idp['entityID'],sp))
-     #print ("!!! REQUESTS EXCEPTION !!!")
-     print (e.__str__())
-     http_code = "(failed)"
-
-   except Exception as e:
-     print ("http-code: (failed) - OtherException for IdP '%s' with SP '%s'" % (idp['entityID'],sp))
-     #print ("!!! EXCEPTION REQUESTS !!!")
-     print (e.__str__())
-     http_code = "(failed)"
-
-   if(metadata_not_found):
-      return (idp['entityID'],wayfless_url,check_time,http_code,"No-eduGAIN-Metadata")
-   elif not username_found or not password_found:
-      return (idp['entityID'],wayfless_url,check_time,http_code,"Invalid-Form")
-   else:
-      return (idp['entityID'],wayfless_url,check_time,http_code,"OK")
-
-
 # Extract IdP DisplayName by fixing input string
-def getDisplayName(display_name):
+def get_display_name(display_name):
     display_name_equal_splitted = display_name.split('==')
     for elem in display_name_equal_splitted:
         if "en" in elem:
@@ -216,39 +24,40 @@ def getDisplayName(display_name):
               elem = elem.replace('"','\\"')
            return elem.split(';', 1)[1]
 
-
 # Append the result of the check on a file
-def storeECCS2result(idp,check_results,idp_status,test):
+def store_eccs_result(idp,sp,check_results,idp_status,test):
 
     # Build the contacts lists: technical/support
-    list_technical_contacts = getIdPContacts(idp,'technical')
-    list_support_contacts = getIdPContacts(idp,'support')
+    list_technical_contacts = utils.get_idp_contacts(idp,'technical')
+    list_support_contacts = utils.get_idp_contacts(idp,'support')
 
     str_technical_contacts = ','.join(list_technical_contacts)
     str_support_contacts = ','.join(list_support_contacts)
 
-    if (test is not True):
-       # IdP-DisplayName;IdP-entityID;IdP-RegAuth;IdP-tech-ctc-1,IdP-tech-ctc-2;IdP-supp-ctc-1,IdP-supp-ctc-2;IdP-ECCS-Status;SP-wayfless-url-1;SP-check-time-1;SP-http-code-1;SP-result-1;SP-wayfless-url-2;SP-check-time-2;SP-http-code-2;SP-result-2
-       with open("%s/%s" % (ECCS2OUTPUTDIR,ECCS2RESULTSLOG), 'a') as f:
-           f.write('{"displayName":"%s","entityID":"%s","registrationAuthority":"%s","contacts":{"technical":"%s","support":"%s"},"status":"%s","sp1":{"wayflessUrl":"%s","checkTime":"%s","httpCode":"%s","checkResult":"%s"},"sp2":{"wayflessUrl":"%s","checkTime":"%s","httpCode":"%s","checkResult":"%s"}}\n' % (
-                   getDisplayName(idp['displayname']),  # IdP-DisplayName
-                   idp['entityID'],                     # IdP-entityID
-                   idp['registrationAuthority'],        # IdP-RegAuth
-                   str_technical_contacts,              # IdP-TechCtcsList
-                   str_support_contacts,                # IdP-SuppCtcsList
-                   idp_status,                          # IdP-ECCS-Status
-                   check_results[0][1],                 # SP-wayfless-url-1
-                   check_results[0][2],                 # SP-check-time-1
-                   check_results[0][3],                 # SP-http-code-1
-                   check_results[0][4],                 # SP-check-result-1
-                   check_results[1][1],                 # SP-wayfless-url-2
-                   check_results[1][2],                 # SP-check-time-2
-                   check_results[1][3],                 # SP-http-code-2
-                   check_results[1][4]))                # SP-check-result-2
+    if (test):
+       sys.stdout.write("\nECCS2:")
+       sys.stdout.write('{"displayName":"%s","entityID":"%s","registrationAuthority":"%s","contacts":{"technical":"%s","support":"%s"},"status":"%s","sp1":{"wayflessUrl":"%s","checkTime":"%s","httpCode":"%s","checkResult":"%s"},"sp2":{"wayflessUrl":"%s","checkTime":"%s","httpCode":"%s","checkResult":"%s"}}\n' % (
+                   get_display_name(idp['displayname']),  # IdP-DisplayName
+                   idp['entityID'],                       # IdP-entityID
+                   idp['registrationAuthority'],          # IdP-RegAuth
+                   str_technical_contacts,                # IdP-TechCtcsList
+                   str_support_contacts,                  # IdP-SuppCtcsList
+                   idp_status,                            # IdP-ECCS-Status
+                   check_results[0][1],                   # SP-wayfless-url-1
+                   check_results[0][2],                   # SP-check-time-1
+                   check_results[0][3],                   # SP-http-code-1
+                   check_results[0][4],                   # SP-check-result-1
+                   check_results[1][1],                   # SP-wayfless-url-2
+                   check_results[1][2],                   # SP-check-time-2
+                   check_results[1][3],                   # SP-http-code-2
+                   check_results[1][4]))                  # SP-check-result-2
+
     else:
-       print("\nECCS2:")
-       print('{"displayName":"%s","entityID":"%s","registrationAuthority":"%s","contacts":{"technical":"%s","support":"%s"},"status":"%s","sp1":{"wayflessUrl":"%s","checkTime":"%s","httpCode":"%s","checkResult":"%s"},"sp2":{"wayflessUrl":"%s","checkTime":"%s","httpCode":"%s","checkResult":"%s"}}\n' % (
-                   getDisplayName(idp['displayname']),  # IdP-DisplayName
+       # IdP-DisplayName;IdP-entityID;IdP-RegAuth;IdP-tech-ctc-1,IdP-tech-ctc-2;IdP-supp-ctc-1,IdP-supp-ctc-2;IdP-ECCS-Status;SP-wayfless-url-1;SP-check-time-1;SP-http-code-1;SP-result-1;SP-wayfless-url-2;SP-check-time-2;SP-http-code-2;SP-result-2
+       with open(f"{e2p.ECCS2OUTPUTDIR}/{e2p.ECCS2RESULTSLOG}", 'a') as f:
+            try:
+               f.write('{"displayName":"%s","entityID":"%s","registrationAuthority":"%s","contacts":{"technical":"%s","support":"%s"},"status":"%s","sp1":{"wayflessUrl":"%s","checkTime":"%s","httpCode":"%s","checkResult":"%s"},"sp2":{"wayflessUrl":"%s","checkTime":"%s","httpCode":"%s","checkResult":"%s"}}\n' % (
+                   get_display_name(idp['displayname']),  # IdP-DisplayName
                    idp['entityID'],                     # IdP-entityID
                    idp['registrationAuthority'],        # IdP-RegAuth
                    str_technical_contacts,              # IdP-TechCtcsList
@@ -261,45 +70,55 @@ def storeECCS2result(idp,check_results,idp_status,test):
                    check_results[1][1],                 # SP-wayfless-url-2
                    check_results[1][2],                 # SP-check-time-2
                    check_results[1][3],                 # SP-http-code-2
-                   check_results[1][4]))                # SP-check-result-2
-
+                   check_results[1][4]                  # SP-check-result-2
+                   )
+               )
+            except IOError:
+               sys.stderr.write(f"Failed writing result on output file for {idp['entityID']} with {utils.get_label(sp)}.\n\nRun {e2p.ECCS2DIR}/runEccs2.py --idp {idp['entityID']} --replace\n")
+               sys.exit(1)
 
 # Check an IdP with 2 SPs.
-def check(idp,sps,test):
+def check(idp,test):
     check_results = []
-    for sp in sps:
-        result = checkIdP(sp,idp,test)
-        if result is not None:
+    for sp in e2p.ECCS2SPS:
+        result = utils.check_idp_response_selenium(sp,idp,test)
+        if (result):
            check_results.append(result)
-
-    if len(check_results) == 2:
+        else:
+           sys.stderr.write(f"\nCheck failed for {idp['entityID']} with {utils.get_label(sp)}.\n\nRun {e2p.ECCS2DIR}/runEccs2.py --idp {idp['entityID']} --replace\n")
+           sys.exit(1)
+ 
+    if (len(check_results) == len(e2p.ECCS2SPS)):
        check_result_sp1 = check_results[0][4]
        check_result_sp2 = check_results[1][4]
+       check_result_weberr1 = check_results[0][5]
+       check_result_weberr2 = check_results[1][5]
 
        # If all checks are 'OK', than the IdP consuming correctly eduGAIN Metadata.
        if (check_result_sp1 == check_result_sp2 == "OK"):
-          storeECCS2result(idp,check_results,'OK',test)
+          store_eccs_result(idp,sp,check_results,'OK',test)
 
        elif (check_result_sp1 == check_result_sp2 == "DISABLED"):
-            storeECCS2result(idp,check_results,'DISABLED',test)
+            store_eccs_result(idp,sp,check_results,'DISABLED',test)
 
        else:
-            storeECCS2result(idp,check_results,'ERROR',test)
-
+            store_eccs_result(idp,sp,check_results,'ERROR',test)
 
 # MAIN
 if __name__=="__main__":
 
-   sps = ECCS2SPS
-
    parser = argparse.ArgumentParser(description='Checks if the input IdP consumed correctly eduGAIN metadata by accessing two different SPs')
    parser.add_argument("idpJson", metavar="idpJson", nargs=1, help="An IdP in Json format")
    parser.add_argument("--test", action='store_true', help="Test the IdP without effects")
+   parser.add_argument("--replace", action='store_true', help="Check an IdP and replace the result")
 
    args = parser.parse_args()
 
    idp = json.loads(args.idpJson[0])
 
-   Path("%s/%s" % (ECCS2HTMLDIR,DAY)).mkdir(parents=True, exist_ok=True)   # Create dir needed to page_source content
+   Path(f"{e2p.ECCS2HTMLDIR}/{e2p.DAY}").mkdir(parents=True, exist_ok=True)   # Create dir needed to page_source content
+
+   if (args.replace and not args.test):
+      utils.delete_line_with_word(f"{e2p.ECCS2OUTPUTDIR}/{e2p.ECCS2RESULTSLOG}",idp['entityID'])
 
-   check(idp,sps,args.test)
+   check(idp,args.test)
diff --git a/eccs2properties.py b/eccs2properties.py
index f85f8fb..3dd995a 100644
--- a/eccs2properties.py
+++ b/eccs2properties.py
@@ -22,9 +22,9 @@ ECCS2HTMLDIR = "%s/html" % ECCS2DIR
 # Selenium
 ECCS2SELENIUMDEBUG = False
 ECCS2SELENIUMLOGDIR = "%s/selenium-logs" % ECCS2DIR
-ECCS2SELENIUMPAGELOADTIMEOUT = 60 #seconds
-ECCS2SELENIUMSCRIPTTIMEOUT = 60   #seconds
-ECCS2REQUESTSTIMEOUT = 60   #seconds
+ECCS2SELENIUMPAGELOADTIMEOUT = 30 #seconds
+ECCS2SELENIUMSCRIPTTIMEOUT = 30   #seconds
+ECCS2REQUESTSTIMEOUT = 15   #seconds
 
 # Logs
 ECCS2LOGSDIR = "%s/logs" % ECCS2DIR
@@ -36,14 +36,22 @@ ECCS2STDERRIDP = "%s/stderr_idp_%s.log" % (ECCS2LOGSDIR,DAY)
 ECCS2FAILEDCMDIDP = "%s/failed-cmd-idp.sh" % ECCS2LOGSDIR
 
 # Number of processes to run in parallel
-ECCS2NUMPROCESSES = 10
+ECCS2NUMPROCESSES = 35
 
 # The 2 SPs that will be used to test each IdP
-#ECCS2SPS = ["https://sp24-test.garr.it/Shibboleth.sso/Login?entityID=", "https://attribute-viewer.aai.switch.ch/Shibboleth.sso/Login?entityID="]
-ECCS2SPS = ["https://sp-demo.idem.garr.it/Shibboleth.sso/Login?entityID=", "https://attribute-viewer.aai.switch.ch/Shibboleth.sso/Login?entityID="]
+ECCS2SPS = [
+   "https://sp-demo.idem.garr.it/Shibboleth.sso/Login?entityID=",
+   "https://attribute-viewer.aai.switch.ch/interfederation-test/Shibboleth.sso/Login?entityID="
+]
 
 # ROBOTS.TXT
-ROBOTS_USER_AGENT = "ECCS/2.0 (+https://technical.edugain.org/eccs2)"
+ROBOTS_USER_AGENT = "ECCS/2.0 (+https://technical-test.edugain.org/eccs2)"
+
+# PATTERNS
+METADATAPATTERN = "Unable.to.locate(\sissuer.in|).metadata(\sfor|)|no.metadata.found|profile.is.not.configured.for.relying.party|Cannot.locate.entity|fail.to.load.unknown.provider|does.not.recognise.the.service|unable.to.load.provider|Nous.n'avons.pas.pu.(charg|charger).le.fournisseur.de service|Metadata.not.found|application.you.have.accessed.is.not.registered.for.use.with.this.service|Message.did.not.meet.security.requirements|Unsupported.Request|Not.Authorized"
+USERNAMEPATTERN = '<input[\s]+[^>]*((type=\s*[\'"](text|email)[\'"]|user)|(name=\s*[\'"](name)[\'"]))[^>]*>'
+PASSWORDPATTERN = '<input[\s]+[^>]*(type=\s*[\'"]password[\'"]|password)[^>]*>'
+REFUSEDPATTERN = '(^http)(.*\.png$)|(.*\.css$)|(.*\.js$)|(.*\.gif$)|(.*\.svg$)|(.*\.jpg$)'
 
 # { 'reg_auth':'reason' }
 FEDS_DISABLED_DICT = {
@@ -74,5 +82,6 @@ IDPS_DISABLED_DICT = {
    'https://sso.vu.lt/SSO/saml2/idp/metadata.php':'Disabled on 2018-11-02 because ECCS2 cannot check non-standard login page',
    #'https://ssl.education.lu/saml/saml2/idp/metadata.php':'Disabled on 2018-11-06 ECCS2 cannot check non-standard login page',
    'https://iif.iucc.ac.il/idp/saml2/idp/metadata.php':'Disabled on 2018-11-06 ECCS2 cannot check non-standard login page',
-   'https://my.atsu.edu/swPublicSSO/SAML/incommon':'Disabled on 2020-11-17 because ECCS2 cannot check non-standard login page'
+   'https://my.atsu.edu/swPublicSSO/SAML/incommon':'Disabled on 2020-11-17 because ECCS2 cannot check non-standard login page',
+   'https://edugain-proxy.igtf.net/simplesaml/saml2/idp/metadata.php':'Disabled on 2017-03-17 on request of federation operator'
 }
diff --git a/requirements.txt b/requirements.txt
index 1bb6041..8c7fe6c 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,5 +1,7 @@
-urllib3==1.25.9
+urllib3==1.26.6
 Flask==1.1.2
-Flask_RESTful==0.3.8
-requests==2.23.0
+Flask-RESTful==0.3.9
+requests==2.25.1
 selenium==3.141.0
+uWSGI==2.0.19.1
+Click==7.1.2
diff --git a/runEccs2.py b/runEccs2.py
index 7b8917c..20cf798 100755
--- a/runEccs2.py
+++ b/runEccs2.py
@@ -3,19 +3,23 @@
 import argparse
 import asyncio
 import datetime
-import eccs2properties
 import json
 import time
-from utils import getListFeds, getListEccsIdps, getRegAuthDict, getIdpList
 
-from eccs2properties import ECCS2FAILEDCMD, ECCS2FAILEDCMDIDP, ECCS2STDOUT, ECCS2STDERR, ECCS2STDOUTIDP, ECCS2STDERRIDP, ECCS2DIR, ECCS2NUMPROCESSES, ECCS2LISTIDPSURL, ECCS2LISTIDPSFILE, ECCS2LISTFEDSURL, ECCS2LISTFEDSFILE 
+import eccs2properties as e2p
+import utils
+
 from subprocess import PIPE
 
+#from utils import get_list_feds, get_list_eccs_idps, get_reg_auth_dict, get_idp_list, gen_output
+#from eccs2properties import ECCS2FAILEDCMD, ECCS2FAILEDCMDIDP, ECCS2STDOUT, ECCS2STDERR, ECCS2STDOUTIDP, ECCS2STDERRIDP, ECCS2DIR, ECCS2NUMPROCESSES, ECCS2LISTIDPSURL, ECCS2LISTIDPSFILE, ECCS2LISTFEDSURL, ECCS2LISTFEDSFILE, ECCS2OUTPUTDIR, ECCS2RESULTSLOG, ECCS2AUXDIR 
 
 # Run Command
+# https://docs.python.org/3/library/asyncio-queue.html#examples
+# https://docs.python.org/3/library/asyncio-subprocess.html
 async def run(name,queue,stdout_file,stderr_file,cmd_file):
    while True:
-      # Get a "cmd item" out of the queue.
+      # Get a "cmd item" out of the queue or wait for the next one
       cmd = await queue.get()
 
       # Elaborate "cmd" from shell.
@@ -27,11 +31,14 @@ async def run(name,queue,stdout_file,stderr_file,cmd_file):
 
       stdout, stderr = await proc.communicate()
 
+      print(f'[{name} exited with {proc.returncode}] - {cmd!r}')
+
       if stdout:
-         stdout_file.write('-----\n[cmd-out]\n%s\n\n[stdout]\n%s' % (cmd,stdout.decode()))
+         stdout_file.write(f'-----\n[cmd]\n{cmd}\n\n[stdout]\n{stdout.decode()}')
+      # If an error occurred, the failed command is put into the 'cmd_file' (failed-cmd.sh / failed-cmd-idp.sh) 
       if stderr:
-         stderr_file.write('-----\n[cmd-err]\n%s\n\n[stderr]\n%s' % (cmd,stderr.decode()))
-         cmd_file.write('%s\n' % cmd)
+         stderr_file.write(f'-----\n[cmd]\n{cmd}\n\n[stderr]\n{stderr.decode()}')
+         cmd_file.write(f'{cmd}\n')
 
       # Notify the queue that the "work cmd" has been processed.
       queue.task_done()
@@ -48,14 +55,12 @@ async def main(cmd_list,stdout_file,stderr_file,cmd_file):
     # Create worker tasks to process the queue concurrently.
     tasks = []
 
-    for i in range(ECCS2NUMPROCESSES):
-        task = asyncio.create_task(run("cmd-{%d}" % i, queue, stdout_file, stderr_file, cmd_file))
+    for i in range(e2p.ECCS2NUMPROCESSES):
+        task = asyncio.create_task(run(f"cmd-{i}", queue, stdout_file, stderr_file, cmd_file))
         tasks.append(task)
 
     # Wait until the queue is fully processed.
-    started_at = time.monotonic()
     await queue.join()
-    total_slept_for = time.monotonic() - started_at
 
     # Cancel our worker tasks.
     for task in tasks:
@@ -72,60 +77,79 @@ if __name__=="__main__":
 
    parser.add_argument("--idp", metavar="entityid", dest="idp_entityid", nargs=1, help="An IdP entityID")
    parser.add_argument("--test", action='store_true', dest="test", help="Test without effects")
+   parser.add_argument("--replace", action='store_true', help="Check an IdP and replace the result")
       
    args = parser.parse_args()
          
    start = time.time()
 
    # Setup list_feds
-   url = ECCS2LISTFEDSURL
-   dest_file = ECCS2LISTFEDSFILE
-   list_feds = getListFeds(url, dest_file)
+   url = e2p.ECCS2LISTFEDSURL
+   dest_file = e2p.ECCS2LISTFEDSFILE
+   list_feds = utils.get_list_feds(url, dest_file)
 
    # Setup list_eccs_idps
-   url = ECCS2LISTIDPSURL
-   dest_file = ECCS2LISTIDPSFILE
-   list_eccs_idps = getListEccsIdps(url, dest_file)
+   url = e2p.ECCS2LISTIDPSURL
+   dest_file = e2p.ECCS2LISTIDPSFILE
+   list_eccs_idps = utils.get_list_eccs_idps(url, dest_file)
 
    if (args.idp_entityid):
-      stdout_file = open(ECCS2STDOUTIDP,"w+")
-      stderr_file = open(ECCS2STDERRIDP,"w+")
-      cmd_file = open(ECCS2FAILEDCMDIDP,"w+")
-      idpJsonList = getIdpList(list_eccs_idps,idp_entityid=args.idp_entityid[0])
+      stdout_file = open(e2p.ECCS2STDOUTIDP,"w+")
+      stderr_file = open(e2p.ECCS2STDERRIDP,"w+")
+      cmd_file = open(e2p.ECCS2FAILEDCMDIDP,"w+")
+      idpJsonList = utils.get_idp_list(list_eccs_idps,idp_entityid=args.idp_entityid[0])
 
-      if (args.test is not True):
-         cmd = "%s/eccs2.py \'%s\'" % (ECCS2DIR,json.dumps(idpJsonList[0]))
-      else:
-         cmd = "%s/eccs2.py \'%s\' --test" % (ECCS2DIR,json.dumps(idpJsonList[0]))
+      if (args.test):
+         cmd = f"{e2p.ECCS2DIR}/eccs2.py '{json.dumps(idpJsonList[0])}' --test"
+      elif (args.replace):
+         cmd = f"{e2p.ECCS2DIR}/eccs2.py '{json.dumps(idpJsonList[0])}' --replace"
 
+      # List of only one command
       proc_list = [cmd]
+
+      # Run Command
       asyncio.run(main(proc_list,stdout_file,stderr_file,cmd_file))
 
+      # Close File
+      stdout_file.close()
+      stderr_file.close()
+      cmd_file.close()
+
    else:
-      stdout_file = open(ECCS2STDOUT,"w+")
-      stderr_file = open(ECCS2STDERR,"w+")
-      cmd_file = open(ECCS2FAILEDCMD,"w+")
+      stdout_file = open(e2p.ECCS2STDOUT,"w+")
+      stderr_file = open(e2p.ECCS2STDERR,"w+")
+      cmd_file = open(e2p.ECCS2FAILEDCMD,"w+")
 
       # Prepare input file for ECCS2
-      regAuthDict = getRegAuthDict(list_feds)
-
-      for name,regAuth in regAuthDict.items():
-          idpJsonList = getIdpList(list_eccs_idps,regAuth)
-
-          num_idps = len(idpJsonList)
-          if (args.test is not True):
-             cmd_list = [["%s/eccs2.py \'%s\'" % (ECCS2DIR, json.dumps(idp))] for idp in idpJsonList]
-          else:
-             cmd_list = [["%s/eccs2.py \'%s\' --test" % (ECCS2DIR, json.dumps(idp))] for idp in idpJsonList]
-
-          proc_list = []
-          count = 0
-          while (count < num_idps):
-                cmd = "".join(cmd_list.pop())
-                proc_list.append(cmd)
-                count = count + 1
+      regAuthDict = utils.get_reg_auth_dict(list_feds)
+
+      #for name,regAuth in regAuthDict.items():
+      # Load the idps belonging to a Federation into idpJsonList
+      idpJsonList = utils.get_idp_list(list_eccs_idps)
+      num_idps = len(idpJsonList)
+
+      # Construct the list of commands to exec
+      if (args.test):
+         cmd_list = [[f"{e2p.ECCS2DIR}/eccs2.py '{json.dumps(idp)}' --test"] for idp in idpJsonList]
+      elif (args.replace):
+         cmd_list = [[f"{e2p.ECCS2DIR}/eccs2.py '{json.dumps(idp)}' --replace"] for idp in idpJsonList]
+      else:
+         cmd_list = [[f"{e2p.ECCS2DIR}/eccs2.py '{json.dumps(idp)}'"] for idp in idpJsonList]
+
+      # String Convertion needed for Asyncio
+      proc_list = []
+      count = 0
+      while (count < num_idps):
+         cmd = "".join(cmd_list.pop())
+         proc_list.append(cmd)
+         count = count + 1
  
-          asyncio.run(main(proc_list,stdout_file,stderr_file,cmd_file))
+      asyncio.run(main(proc_list,stdout_file,stderr_file,cmd_file))
+
+      stdout_file.close()
+      stderr_file.close()
+      cmd_file.close()
 
    end = time.time()
+   #utils.gen_output(e2p.ECCS2AUXDIR,f"{e2p.ECCS2OUTPUTDIR}/{e2p.ECCS2RESULTSLOG}")
    print("Time taken in hh:mm:ss - ", str(datetime.timedelta(seconds=end - start)))
diff --git a/utils.py b/utils.py
index 6f95b39..e433d31 100644
--- a/utils.py
+++ b/utils.py
@@ -1,18 +1,42 @@
 #!/usr/bin/env python3
 
+import datetime
 import json
 import logging
 import pathlib
+import re
 import requests
 import sys
+import shutil
+import time
+
+import eccs2properties as e2p
 
-from eccs2properties import ECCS2SELENIUMLOGDIR, ECCS2SELENIUMPAGELOADTIMEOUT, ECCS2SELENIUMSCRIPTTIMEOUT, PATHCHROMEDRIVER
 from selenium import webdriver
-from selenium.common.exceptions import WebDriverException
+from selenium.common.exceptions import WebDriverException,TimeoutException
+from selenium.webdriver.common.by import By
+from selenium.webdriver.support.ui import WebDriverWait
+from selenium.webdriver.support import expected_conditions as EC
+from selenium.webdriver.chrome.options import Options
+from logging.handlers import RotatingFileHandler
+from urllib3.util import parse_url
+
+def sha1(idp_entity_id):
+    import hashlib
+    result = hashlib.sha1(idp_entity_id.encode())
+    return result.hexdigest()
+
+
+# Return a label useful for a filename
+def get_label(url_or_urn):
+    if url_or_urn.startswith('http'):
+       return parse_url(url_or_urn)[2]
+    else:
+       return parse_url(url_or_urn)[4].lstrip('/') 
 
 
 # Returns a Dict of "{ nameFed:reg_auth }"
-def getRegAuthDict(list_feds):
+def get_reg_auth_dict(list_feds):
     regAuth_dict = {}
 
     for key,value in list_feds.items():
@@ -25,7 +49,7 @@ def getRegAuthDict(list_feds):
 
 
 # Returns a list of IdP for a single federation
-def getIdpList(list_eccs_idps,reg_auth=None,idp_entityid=None):
+def get_idp_list(list_eccs_idps,reg_auth=None,idp_entityid=None):
     fed_idp_list = []
     for idp in list_eccs_idps:
        if (idp_entityid):
@@ -40,8 +64,10 @@ def getIdpList(list_eccs_idps,reg_auth=None,idp_entityid=None):
     return fed_idp_list
 
 
-# Returns a Python Dictionary
-def getListFeds(url, dest_file):
+# Download all eduGAIN Federations from URL, store them on a local file and returns a Python Dictionary
+def get_list_feds(url, dest_file):
+    from pathlib import Path
+
     # If file does not exists... download it into the dest_file
     path = pathlib.Path(dest_file)
     if(path.exists() == False):
@@ -54,7 +80,9 @@ def getListFeds(url, dest_file):
 
 
 # Download all eduGAIN IdPs from URL, store them on a local file and returns a Python List
-def getListEccsIdps(url, dest_file):
+def get_list_eccs_idps(url, dest_file):
+    from pathlib import Path
+
     # If file does not exists... download it into the dest_file
     path = pathlib.Path(dest_file)
     if(path.exists() == False):
@@ -67,10 +95,10 @@ def getListEccsIdps(url, dest_file):
 
 
 # Use logger to produce files consumed by ECCS-2 API
-def getLogger(filename, path, mode, log_level="DEBUG"):
+def get_logger(path, filename, mode="a", log_level="DEBUG"):
 
-    logger = logging.getLogger(filename)
-    ch = logging.FileHandler("%s/%s" % (path,filename), mode,'utf-8')
+    logger = logging.getLogger(__name__)
+    ch = logging.handlers.RotatingFileHandler(f"{path}/{filename}", mode, 0, 5, 'utf-8')
 
     if (log_level == "DEBUG"):
        logger.setLevel(logging.DEBUG)
@@ -88,7 +116,7 @@ def getLogger(filename, path, mode, log_level="DEBUG"):
        logger.setLevel(logging.CRITICAL)
        ch.setLevel(logging.CRITICAL)
 
-    formatter = logging.Formatter('%(message)s')
+    formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(module)s - %(message)s', datefmt='%d/%m/%Y %H:%M:%S')
     ch.setFormatter(formatter)
     logger.addHandler(ch)
 
@@ -96,7 +124,7 @@ def getLogger(filename, path, mode, log_level="DEBUG"):
 
 
 # Return a list of email address for a specific type of contact
-def getIdPContacts(idp,contactType):
+def get_idp_contacts(idp,contactType):
     ctcList = []
     for ctcType in idp['contacts']:
         if (ctcType == contactType):
@@ -110,34 +138,194 @@ def getIdPContacts(idp,contactType):
                   ctcList.append('missing email')
     return ctcList
 
+# Write the login page source code into its file
+def store_page_source(page_source,idp,sp,test):
+    if (test):
+        sys.stdout.write(f"{page_source}")
+        return True
+    else:
+       # Put the page_source into an appropriate HTML file
+       with open(f"{e2p.ECCS2HTMLDIR}/{e2p.DAY}/{sha1(['entityID'])}---{get_label(sp)}.html","w") as html:
+            try:
+               html.write(page_source)
+               return True
+            except IOError:
+               return False
 
-def getDriver(fqdn_idp=None,debugSelenium=False):
-    # Disable SSL requests warning messages
-    requests.packages.urllib3.disable_warnings()
+# Get the Google Chrom Selenium Driver
+def get_driver_selenium(idp=None,sp=None,debugSelenium=False):
 
     # Configure Web-driver
-    chrome_options = webdriver.ChromeOptions()
+    # https://peter.sh/experiments/chromium-command-line-switches/
+    chrome_options = Options()
+    chrome_options.page_load_strategy = 'eager'
+ 
+    chrome_options.add_argument('--start-in-incognito')
     chrome_options.add_argument('--headless')
     chrome_options.add_argument('--no-sandbox')
     chrome_options.add_argument('--disable-dev-shm-usage')
-    chrome_options.add_argument('--ignore-certificate-errors')
-    chrome_options.add_argument('--remote-debugging-port=9222')
-    #chrome_options.add_argument('--start-maximized')
+    chrome_options.add_argument('--disable-gpu')
+    chrome_options.add_argument('--disable-extensions')
+    chrome_options.add_argument('--disable-dinosaur-easter-egg')
+    chrome_options.add_argument('--disable-sync')
 
     # For DEBUG only (By default ChromeDriver logs only warnings/errors to stderr.
     # When debugging issues, it is helpful to enable more verbose logging.)
+    if (debugSelenium):
+       label_idp = get_label(idp['entityID'])
+       label_sp = get_label(sp)
+       sha1_idp = sha1(idp['entityID'])
+       try:
+          driver = webdriver.Chrome(e2p.PATHCHROMEDRIVER, options=chrome_options, service_args=['--verbose', f'--log-path={e2p.ECCS2SELENIUMLOGDIR}/{sha1_idp}_{label_idp}_{label_sp}.log'])
+       except WebDriverException:
+          time.sleep(3)
+          driver = webdriver.Chrome(e2p.PATHCHROMEDRIVER, options=chrome_options, service_args=['--verbose', f'--log-path={e2p.ECCS2SELENIUMLOGDIR}/{sha1_idp}_{label_idp}_{label_sp}.log'])
+    else:
+       try:
+          driver = webdriver.Chrome(e2p.PATHCHROMEDRIVER, options=chrome_options)
+       except WebDriverException:
+          time.sleep(3)
+          driver = webdriver.Chrome(e2p.PATHCHROMEDRIVER, options=chrome_options)
+    return driver
+
+def check_idp_response_selenium(sp,idp,test):
+
+    # Disable SSL requests warning messages
+    #requests.packages.urllib3.disable_warnings()
+
+    # Common variables
+    fqdn_idp = get_label(idp['Location'])
+    wayfless_url = f"{sp}{idp['entityID']}"
+    robots = ""
+    federations_disabled_dict = e2p.FEDS_DISABLED_DICT
+    idps_disabled_dict = e2p.IDPS_DISABLED_DICT
+    webdriver_error = 0 # No WebDriver Error
+
+    # SELENIUM CONSTANTS
+    http_code = "NULL"
+
+    # Handle Disabled Idps/Federations
+    if (idp['registrationAuthority'] in federations_disabled_dict.keys()):
+       check_time = datetime.datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%S') + 'Z'
+       page_source = federations_disabled_dict[idp['registrationAuthority']]
+       store_page_source(page_source,idp,sp,test)
+       return (idp['entityID'],wayfless_url,check_time,"NULL","DISABLED",webdriver_error)
+
+    if (idp['entityID'] in idps_disabled_dict.keys()):
+       check_time = datetime.datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%S') + 'Z'
+       page_source = idps_disabled_dict[idp['entityID']]
+       store_page_source(page_source,idp,sp,test)
+       return (idp['entityID'],wayfless_url,check_time,"NULL","DISABLED",webdriver_error)
+
+    # Robots + SSL Check
     try:
-       if (debugSelenium and fqdn_idp):
-          driver = webdriver.Chrome(PATHCHROMEDRIVER, options=chrome_options,  service_args=['--verbose', '--log-path=%s/%s.log' % (ECCS2SELENIUMLOGDIR, fqdn_idp)])
+       hdrs = {
+          'User-Agent': f'{e2p.ROBOTS_USER_AGENT}'
+       }
+       check_time = datetime.datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%S') + 'Z'
+       robots = requests.get(f"https://{fqdn_idp}/robots.txt", headers=hdrs, verify=True, timeout=e2p.ECCS2REQUESTSTIMEOUT)
+
+       if (robots == ""):
+          robots  = requests.get(f"http://{fqdn_idp}/robots.txt", headers=hdrs, verify=False, timeout=e2p.ECCS2REQUESTSTIMEOUT)
+
+    # Catch SSL Exceptions and block the ECCS check
+    except requests.exceptions.SSLError as e:
+       if (test): page_source = f"\nAn SSL Error occurred while opening https://{fqdn_idp}/robots.txt.\n\n{e}\n\nCheck it on SSL Labs: https://www.ssllabs.com/ssltest/analyze.html?d={fqdn_idp}"
+       else: page_source = f"<h1>SSL ERROR</h1><h2>An SSL error occurred for the server {fqdn_idp}:</h2><p>{e}</p><p>Check it on SSL Labs: <a href='https://www.ssllabs.com/ssltest/analyze.html?d={fqdn_idp}'>Click Here</a></p>"
+       store_page_source(page_source,idp,sp,test)
+       return (idp['entityID'],wayfless_url,check_time,"(failed)","SSL-Error",webdriver_error)
+
+    except requests.exceptions.ConnectionError as e:
+       if (test): page_source = f"\nA Connection error occurred while opening https://{fqdn_idp}/robots.txt.\n\n{e}"
+       else: page_source = f"<h1>CONNECTION ERROR:</h1><h2>A Connection error occurred while opening <a href='https://{fqdn_idp}/robots.txt'>https://{fqdn_idp}/robots.txt</a>:</h2><p>{e}</p>"
+       store_page_source(page_source,idp,sp,test)
+       return (idp['entityID'],wayfless_url,check_time,"(failed)","Connection-Error",webdriver_error)
+
+    except requests.exceptions.Timeout as e:
+       if (test): page_source = f"\nThe request timed out while trying to connect to the remote server '{fqdn_idp}':\n\n{e}"
+       else: page_source = f"<h1>CONNECTION TIMEOUT</h1><h2>The request timed out while opening <a href='https://{fqdn_idp}/robots.txt'>https://{fqdn_idp}/robots.txt</a>:</h2><p>{e}</p>"
+       store_page_source(page_source,idp,sp,test)
+       return (idp['entityID'],wayfless_url,check_time,"(failed)","Connection-Error",webdriver_error)
+
+    except requests.exceptions.TooManyRedirects as e:
+       if (test): page_source = f"\nToo many redirects occurred while opening: https://{fqdn_idp}/robots.txt.\n\n{e}"
+       else: page_source = f"<h1>TOO MANY REDIRECTS</h1><h2>Too many redirects occurred while opening: <a href='https://{fqdn_idp}/robots.txt'>https://{fqdn_idp}/robots.txt</a>:</h2><p>{e}</p>"
+       store_page_source(page_source,idp,sp,test)
+       return (idp['entityID'],wayfless_url,check_time,"(failed)","Connection-Error",webdriver_error)
+
+    if (robots):
+       check_time = datetime.datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%S') + 'Z'
+       p = re.compile('^User-Agent:\sECCS\sDisallow:\s\/\s*$', re.MULTILINE)
+       m = p.search(robots.text)
+
+       if (m):
+          page_source = "IdP excluded from check by robots.txt"
+          store_page_source(page_source,idp,sp,test)
+          return (idp['entityID'],wayfless_url,check_time,"NULL","DISABLED",webdriver_error)
+
+    try:
+       # WebDriver MUST be instanced here to avoid problems with SESSION
+       driver = get_driver_selenium(idp,sp,e2p.ECCS2SELENIUMDEBUG)
+
+       # Exception of WebDriver raises
+       if (driver == None):
+          sys.stderr.write(f"get_driver_selenium() returned None for IDP {idp['entityID']}(SHA1: {sha1(idp['entityID'])}) with SP {get_label(sp)}")
+          return None
+
+       driver.set_page_load_timeout(e2p.ECCS2SELENIUMPAGELOADTIMEOUT)
+       driver.set_script_timeout(e2p.ECCS2SELENIUMSCRIPTTIMEOUT)
+
+       check_time = datetime.datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%S') + 'Z'
+       driver.get(wayfless_url)
+
+       elem = WebDriverWait(driver, e2p.ECCS2SELENIUMPAGELOADTIMEOUT).until(
+          EC.presence_of_element_located((By.XPATH,'//form//input[@type="password"]'))
+       )
+       page_source = driver.page_source
+
+       if (test): pgsrc = f"\n[WAYFLESS URL]{wayfless_url} - OK"
+       else: pgsrc = page_source
+       stored = store_page_source(pgsrc,idp,sp,test)
+       if (stored):
+          return (idp['entityID'],wayfless_url,check_time,http_code,"OK",webdriver_error)
+
+    except TimeoutException as e:
+       page_source = driver.page_source
+       metadata_not_found = re.search(e2p.METADATAPATTERN,page_source, re.I)
+
+       if (metadata_not_found):
+          if (test): pgsrc = f"\n[PAGE_SOURCE]\n{page_source}\n[WAYFLESS URL]{wayfless_url} - METADATA NOT FOUND"
+          else: pgsrc = page_source
+          stored = store_page_source(pgsrc,idp,sp,test)
+          if (stored):
+             return (idp['entityID'],wayfless_url,check_time,http_code,"No-eduGAIN-Metadata",webdriver_error)
        else:
-          driver = webdriver.Chrome(PATHCHROMEDRIVER, options=chrome_options)
+          if (test): pgsrc = f"Timeout: No access form found in {e2p.ECCS2SELENIUMPAGELOADTIMEOUT} seconds"
+          else: pgsrc = page_source
+          stored = store_page_source(pgsrc,idp,sp,test)
+          if (stored):
+             return (idp['entityID'],wayfless_url,check_time,"(failed)","Timeout",webdriver_error)
+
     except WebDriverException as e:
-       sys.stderr.write("!!! WEB DRIVER EXCEPTION - RUN AGAIN THE COMMAND!!!")
-       sys.stderr.write(e.__str__())
-       return None
+           error = e.__dict__['msg'].split('(')[0].rstrip()
+           if (test): pgsrc = f"\nERROR: {error}"
+           else: pgsrc = f"<h1>ECCS2 CHECK FAILED</h1><h2>The IdP Login failed the check due the following error:</h2><p>{error}</p>" 
+           webdriver_error = 1
+           stored = store_page_source(pgsrc,idp,sp,test)
+           if (stored):
+              return (idp['entityID'],wayfless_url,check_time,"(failed)","ERROR",webdriver_error)
 
-    # Configure timeouts
-    driver.set_page_load_timeout("%d" % ECCS2SELENIUMPAGELOADTIMEOUT)
-    driver.set_script_timeout("%d" % ECCS2SELENIUMSCRIPTTIMEOUT)
+    finally:
+       driver.quit()
 
-    return driver
+def delete_line_with_word(filepath,word):
+    import os.path
+ 
+    if os.path.isfile(filepath):
+       with open(filepath, "r") as f:
+            lines = f.readlines()
+
+       with open(filepath, "w") as f:
+            for line in lines:
+                if word not in line:
+                   f.write(line)
diff --git a/web/eccs2.js b/web/eccs2.js
index d74b2a0..ee462de 100644
--- a/web/eccs2.js
+++ b/web/eccs2.js
@@ -3,6 +3,145 @@ var table;
 var url = "/eccs2/api/eccsresults?eccsdt=1";
 var infoCircle = '<a href="https://wiki.geant.org/display/eduGAIN/eduGAIN+Connectivity+Check+2#eduGAINConnectivityCheck2-Statusesandresults"><i class="fas fa-info-circle"></i></a>';
 
+/*
+ * Secure Hash Algorithm (SHA1)
+ * https://www.webtoolkit.info/javascript_sha1.html
+*/
+function SHA1(msg) {
+   function rotate_left(n,s) {
+      var t4 = ( n<<s ) | (n>>>(32-s));
+      return t4;
+   };
+   function lsb_hex(val) {
+      var str='';
+      var i;
+      var vh;
+      var vl;
+      for( i=0; i<=6; i+=2 ) {
+         vh = (val>>>(i*4+4))&0x0f;
+         vl = (val>>>(i*4))&0x0f;
+         str += vh.toString(16) + vl.toString(16);
+      }
+      return str;
+   };
+   function cvt_hex(val) {
+      var str='';
+      var i;
+      var v;
+      for( i=7; i>=0; i-- ) {
+         v = (val>>>(i*4))&0x0f;
+         str += v.toString(16);
+      }
+      return str;
+   };
+   function Utf8Encode(string) {
+      string = string.replace(/\r\n/g,'\n');
+      var utftext = '';
+      for (var n = 0; n < string.length; n++) {
+         var c = string.charCodeAt(n);
+         if (c < 128) {
+            utftext += String.fromCharCode(c);
+         }
+         else if((c > 127) && (c < 2048)) {
+            utftext += String.fromCharCode((c >> 6) | 192);
+            utftext += String.fromCharCode((c & 63) | 128);
+         }
+         else {
+            utftext += String.fromCharCode((c >> 12) | 224);
+            utftext += String.fromCharCode(((c >> 6) & 63) | 128);
+            utftext += String.fromCharCode((c & 63) | 128);
+         }
+      }
+      return utftext;
+   };
+   var blockstart;
+   var i, j;
+   var W = new Array(80);
+   var H0 = 0x67452301;
+   var H1 = 0xEFCDAB89;
+   var H2 = 0x98BADCFE;
+   var H3 = 0x10325476;
+   var H4 = 0xC3D2E1F0;
+   var A, B, C, D, E;
+   var temp;
+   msg = Utf8Encode(msg);
+   var msg_len = msg.length;
+   var word_array = new Array();
+   for( i=0; i<msg_len-3; i+=4 ) {
+      j = msg.charCodeAt(i)<<24 | msg.charCodeAt(i+1)<<16 |
+      msg.charCodeAt(i+2)<<8 | msg.charCodeAt(i+3);
+      word_array.push( j );
+   }
+   switch( msg_len % 4 ) {
+      case 0:
+         i = 0x080000000;
+         break;
+      case 1:
+         i = msg.charCodeAt(msg_len-1)<<24 | 0x0800000;
+         break;
+      case 2:
+         i = msg.charCodeAt(msg_len-2)<<24 | msg.charCodeAt(msg_len-1)<<16 | 0x08000;
+         break;
+      case 3:
+         i = msg.charCodeAt(msg_len-3)<<24 | msg.charCodeAt(msg_len-2)<<16 | msg.charCodeAt(msg_len-1)<<8 | 0x80;
+         break;
+   }
+   word_array.push( i );
+   while( (word_array.length % 16) != 14 ) word_array.push( 0 );
+   word_array.push( msg_len>>>29 );
+   word_array.push( (msg_len<<3)&0x0ffffffff );
+   for ( blockstart=0; blockstart<word_array.length; blockstart+=16 ) {
+      for( i=0; i<16; i++ ) W[i] = word_array[blockstart+i];
+      for( i=16; i<=79; i++ ) W[i] = rotate_left(W[i-3] ^ W[i-8] ^ W[i-14] ^ W[i-16], 1);
+      A = H0;
+      B = H1;
+      C = H2;
+      D = H3;
+      E = H4;
+      for( i= 0; i<=19; i++ ) {
+         temp = (rotate_left(A,5) + ((B&C) | (~B&D)) + E + W[i] + 0x5A827999) & 0x0ffffffff;
+         E = D;
+         D = C;
+         C = rotate_left(B,30);
+         B = A;
+         A = temp;
+      }
+      for( i=20; i<=39; i++ ) {
+         temp = (rotate_left(A,5) + (B ^ C ^ D) + E + W[i] + 0x6ED9EBA1) & 0x0ffffffff;
+         E = D;
+         D = C;
+         C = rotate_left(B,30);
+         B = A;
+         A = temp;
+      }
+      for( i=40; i<=59; i++ ) {
+         temp = (rotate_left(A,5) + ((B&C) | (B&D) | (C&D)) + E + W[i] + 0x8F1BBCDC) & 0x0ffffffff;
+         E = D;
+         D = C;
+         C = rotate_left(B,30);
+         B = A;
+         A = temp;
+      }
+      for( i=60; i<=79; i++ ) {
+         temp = (rotate_left(A,5) + (B ^ C ^ D) + E + W[i] + 0xCA62C1D6) & 0x0ffffffff;
+         E = D;
+         D = C;
+         C = rotate_left(B,30);
+         B = A;
+         A = temp;
+      }
+      H0 = (H0 + A) & 0x0ffffffff;
+      H1 = (H1 + B) & 0x0ffffffff;
+      H2 = (H2 + C) & 0x0ffffffff;
+      H3 = (H3 + D) & 0x0ffffffff;
+      H4 = (H4 + E) & 0x0ffffffff;
+   }
+   var temp = cvt_hex(H0) + cvt_hex(H1) + cvt_hex(H2) + cvt_hex(H3) + cvt_hex(H4);
+
+   return temp.toLowerCase();
+}
+
+
 // PHP Variables retrieved from eccs2.php
 // idp (entityID of the IdP)
 // date (date time of the check)
@@ -21,6 +160,7 @@ if (status) {
    url = url.concat("&status=" + status);
 }
 
+
 function getPastResults() {
    var checkDate = $.datepicker.formatDate("yy-mm-dd", $('#datepicker').datepicker().datepicker('getDate'));
     
@@ -99,7 +239,7 @@ function format ( d ) {
             '<td>'+d.sp1.checkTime+'</td>'+
             '<td>'+getCheckResult(d.sp1.checkResult)+'</td>'+
             '<td>'+d.sp1.httpCode+'</td>'+
-            '<td><a href="/eccs2html/'+d.date+'/'+getHostname(d.entityID)+'---'+getHostname(d.sp1.wayflessUrl)+'.html" target="_blank">Click to open</a></td>'+
+            '<td><a href="/eccs2html/'+d.date+'/'+SHA1(d.entityID)+'---'+getHostname(d.sp1.wayflessUrl)+'.html" target="_blank">Click to open</a></td>'+
             '<td><a href="'+d.sp1.wayflessUrl+'" target="_blank">Click to retry</a></td>'+
         '</tr>'+
         '<tr>'+
@@ -108,7 +248,7 @@ function format ( d ) {
             '<td>'+d.sp2.checkTime+'</td>'+
             '<td>'+getCheckResult(d.sp2.checkResult)+'</td>'+
             '<td>'+d.sp2.httpCode+'</td>'+
-            '<td><a href="/eccs2html/'+d.date+'/'+getHostname(d.entityID)+'---'+getHostname(d.sp2.wayflessUrl)+'.html" target="_blank">Click to open</a></td>'+
+            '<td><a href="/eccs2html/'+d.date+'/'+SHA1(d.entityID)+'---'+getHostname(d.sp2.wayflessUrl)+'.html" target="_blank">Click to open</a></td>'+
             '<td><a href="'+d.sp2.wayflessUrl+'" target="_blank">Click to retry</a></td>'+
         '</tr>'+
     '</table>';
-- 
GitLab