Added 'retryFailedChecks.py' script to flow

README.md

@@ -95,7 +95,7 @@ The tool uses following status for IdPs:
 # Requirements Software
 
 * Apache Server + WSGI
-* Python 3.9 (tested with v3.9.6)
+* Python 3.9 (tested with v3.9.1)
 * Selenim + Google Chrome Web Brower (tested with v91.0.4472.164)
 * Chromedriver (tested with v91.0.4472.101)
 * Git
@@ -136,14 +136,14 @@ The tool uses following status for IdPs:
 ### Python 3.9
 
 1. Download the last version of Python 3.9.x from https://www.python.org/downloads/source/ into your home:
-   * `wget https://www.python.org/ftp/python/3.9.6/Python-3.9.6.tgz -O $HOME/eccs2/Python-3.9.6.tgz`
+   * `wget https://www.python.org/ftp/python/3.9.1/Python-3.9.1.tgz -O $HOME/eccs2/Python-3.9.1.tgz`
 
 2. Extract Python source package:
    * `cd $HOME/eccs2/`
-   * `tar xzf Python-3.9.6.tgz`
+   * `tar xzf Python-3.9.1.tgz`
 
 3. Build Python from the source package:
-   * `cd $HOME/eccs2/Python-3.9.6`
+   * `cd $HOME/eccs2/Python-3.9.1`
    * `./configure --prefix=$HOME/eccs2/python`
    * `make`
 
@@ -154,7 +154,7 @@ The tool uses following status for IdPs:
    This will install python under your $HOME directory.
    
 5. Remove useless things:
-   * `rm -Rf $HOME/eccs2/Python-3.9.9 $HOME/eccs2/Python-3.9.6.tgz`
+   * `rm -Rf $HOME/eccs2/Python-3.9.1 $HOME/eccs2/Python-3.9.1.tgz`
 
 # Install Google Chrome needed by Selenium
 

cleanAndRunEccs2.sh

@@ -11,3 +11,6 @@ rm -f $BASEDIR/eccs2/input/*.json
 
 # Run ECCS2
 $BASEDIR/eccs2/runEccs2.py
+
+# Run again ECCS2 for those IdPs who failed check
+$BASEDIR/eccs2/retryFailedChecks.py

eccs2properties.py

@@ -3,37 +3,37 @@ from datetime import date
 
 DAY = date.today().isoformat()
 
-ECCS2DIR = "%s/eccs2" % os.environ['HOME']
-PATHCHROMEDRIVER = "%s/chromedriver" % ECCS2DIR
-ECCS2PYTHON = "%s/python/bin/python3" % ECCS2DIR
+ECCS2DIR = f"{os.environ['HOME']}/eccs2"
+PATHCHROMEDRIVER = f"{ECCS2DIR}/chromedriver"
+ECCS2PYTHON = f"{ECCS2DIR}/python/bin/python3"
 
 # Input
-ECCS2INPUTDIR = "%s/input" % ECCS2DIR
+ECCS2INPUTDIR = f"{ECCS2DIR}/input"
 ECCS2LISTIDPSURL = 'https://technical.edugain.org/api.php?action=list_eccs_idps&format=json'
-ECCS2LISTIDPSFILE = "%s/list_eccs_idps.json" % ECCS2INPUTDIR
+ECCS2LISTIDPSFILE = f"{ECCS2INPUTDIR}/list_eccs_idps.json"
 ECCS2LISTFEDSURL = 'https://technical.edugain.org/api.php?action=list_feds&opt=1&format=json' 
-ECCS2LISTFEDSFILE = "%s/list_fed.json" % ECCS2INPUTDIR
+ECCS2LISTFEDSFILE = f"{ECCS2INPUTDIR}/list_fed.json"
 
 # Output
-ECCS2OUTPUTDIR = "%s/output" % ECCS2DIR
-ECCS2RESULTSLOG = "eccs2_%s.log" % DAY
-ECCS2HTMLDIR = "%s/html" % ECCS2DIR
+ECCS2OUTPUTDIR = f"{ECCS2DIR}/output"
+ECCS2RESULTSLOG = f"eccs2_{DAY}.log"
+ECCS2HTMLDIR = f"{ECCS2DIR}/html"
 
 # Selenium
 ECCS2SELENIUMDEBUG = False
-ECCS2SELENIUMLOGDIR = "%s/selenium-logs" % ECCS2DIR
-ECCS2SELENIUMPAGELOADTIMEOUT = 30 #seconds
+ECCS2SELENIUMLOGDIR = f"{ECCS2DIR}/selenium-logs"
+ECCS2SELENIUMPAGELOADTIMEOUT = 30 #seconds (remind to change timeout seconds also on web/eccs2.js)
 ECCS2SELENIUMSCRIPTTIMEOUT = 30   #seconds
 ECCS2REQUESTSTIMEOUT = 15   #seconds
 
 # Logs
-ECCS2LOGSDIR = "%s/logs" % ECCS2DIR
-ECCS2STDOUT = "%s/stdout_%s.log" % (ECCS2LOGSDIR,DAY)
-ECCS2STDERR = "%s/stderr_%s.log" % (ECCS2LOGSDIR,DAY)
-ECCS2FAILEDCMD = "%s/failed-cmd.sh" % ECCS2LOGSDIR
-ECCS2STDOUTIDP = "%s/stdout_idp_%s.log" % (ECCS2LOGSDIR,DAY)
-ECCS2STDERRIDP = "%s/stderr_idp_%s.log" % (ECCS2LOGSDIR,DAY)
-ECCS2FAILEDCMDIDP = "%s/failed-cmd-idp.sh" % ECCS2LOGSDIR
+ECCS2LOGSDIR = f"{ECCS2DIR}/logs"
+ECCS2STDOUT = f"{ECCS2LOGSDIR}/stdout_{DAY}.log"
+ECCS2STDERR = f"{ECCS2LOGSDIR}/stderr_{DAY}.log"
+ECCS2FAILEDCMD = f"{ECCS2LOGSDIR}/failed-cmd.sh"
+ECCS2STDOUTIDP = f"{ECCS2LOGSDIR}/stdout_idp_{DAY}.log"
+ECCS2STDERRIDP = f"{ECCS2LOGSDIR}/stderr_idp_{DAY}.log"
+ECCS2FAILEDCMDIDP = f"{ECCS2LOGSDIR}/failed-cmd-idp.sh"
 
 # Number of processes to run in parallel
 ECCS2NUMPROCESSES = 35

retryFailedChecks.py

+#!/usr/bin/env python3
+
+import os
+import eccs2properties as e2p
+import utils
+
+def get_idp_entityID(line):
+    import json
+
+    line = line.lstrip(f"{e2p.ECCS2DIR}/eccs2.py '")
+    line = line.rstrip("\'\n")
+    json_line = json.loads(line)
+
+    return json_line['entityID']
+
+ 
+# MAIN
+if __name__=="__main__":
+
+   if (os.stat(e2p.ECCS2FAILEDCMD).st_size == 0):
+      print(f"{e2p.DAY} - ECCS2 OK: All eduGAIN IdPs have been checked successfully.")
+   else:
+      with open(e2p.ECCS2FAILEDCMD) as f:
+
+           # For each one, run ECCS2 check and remove its line
+           # from the "failed-cmd.sh".
+           for line in f:
+               idp = get_idp_entityID(line)
+               os.system(f'{line.rstrip()}')
+
+               with open(f'{e2p.ECCS2OUTPUTDIR}/eccs2_{e2p.DAY}.log') as o:
+                    for line in o:
+                        if (idp in line):
+                           utils.delete_line_with_word(e2p.ECCS2FAILEDCMD,idp)
+                           print(f"ECCS2 check retried successfully for {idp}")
+
+
+      if (os.stat(e2p.ECCS2FAILEDCMD).st_size == 0):
+         print(f"{e2p.DAY} - ECCS2 OK: All eduGAIN IdPs have been checked successfully.")
+      else:
+         print(f"{e2p.DAY} - Something went wrong. See the log files and failed-cmd.sh.")