diff --git a/README.md b/README.md index 9c4633d14507eb54b689efa1341323e0d7a944c1..0630b12fcb6b221b2dc6ec40eee103f83ad122ef 100644 --- a/README.md +++ b/README.md @@ -95,7 +95,7 @@ The tool uses following status for IdPs: # Requirements Software * Apache Server + WSGI -* Python 3.9 (tested with v3.9.6) +* Python 3.9 (tested with v3.9.1) * Selenim + Google Chrome Web Brower (tested with v91.0.4472.164) * Chromedriver (tested with v91.0.4472.101) * Git @@ -136,14 +136,14 @@ The tool uses following status for IdPs: ### Python 3.9 1. Download the last version of Python 3.9.x from https://www.python.org/downloads/source/ into your home: - * `wget https://www.python.org/ftp/python/3.9.6/Python-3.9.6.tgz -O $HOME/eccs2/Python-3.9.6.tgz` + * `wget https://www.python.org/ftp/python/3.9.1/Python-3.9.1.tgz -O $HOME/eccs2/Python-3.9.1.tgz` 2. Extract Python source package: * `cd $HOME/eccs2/` - * `tar xzf Python-3.9.6.tgz` + * `tar xzf Python-3.9.1.tgz` 3. Build Python from the source package: - * `cd $HOME/eccs2/Python-3.9.6` + * `cd $HOME/eccs2/Python-3.9.1` * `./configure --prefix=$HOME/eccs2/python` * `make` @@ -154,7 +154,7 @@ The tool uses following status for IdPs: This will install python under your $HOME directory. 5. Remove useless things: - * `rm -Rf $HOME/eccs2/Python-3.9.9 $HOME/eccs2/Python-3.9.6.tgz` + * `rm -Rf $HOME/eccs2/Python-3.9.1 $HOME/eccs2/Python-3.9.1.tgz` # Install Google Chrome needed by Selenium diff --git a/cleanAndRunEccs2.sh b/cleanAndRunEccs2.sh index dbc494d5edc1dc5bc8518f5a21d32bc702188a37..9242c5dc8065d27e94c4cac6c9855fe2a81eff0d 100755 --- a/cleanAndRunEccs2.sh +++ b/cleanAndRunEccs2.sh @@ -11,3 +11,6 @@ rm -f $BASEDIR/eccs2/input/*.json # Run ECCS2 $BASEDIR/eccs2/runEccs2.py + +# Run again ECCS2 for those IdPs who failed check +$BASEDIR/eccs2/retryFailedChecks.py diff --git a/eccs2properties.py b/eccs2properties.py index 7ed262a39464d7bc0ebc0d47c6b9bf4d3d415d02..cd2f719247580ac13cb853ec7792dbbfbaf70ae8 100644 --- a/eccs2properties.py +++ b/eccs2properties.py @@ -3,37 +3,37 @@ from datetime import date DAY = date.today().isoformat() -ECCS2DIR = "%s/eccs2" % os.environ['HOME'] -PATHCHROMEDRIVER = "%s/chromedriver" % ECCS2DIR -ECCS2PYTHON = "%s/python/bin/python3" % ECCS2DIR +ECCS2DIR = f"{os.environ['HOME']}/eccs2" +PATHCHROMEDRIVER = f"{ECCS2DIR}/chromedriver" +ECCS2PYTHON = f"{ECCS2DIR}/python/bin/python3" # Input -ECCS2INPUTDIR = "%s/input" % ECCS2DIR +ECCS2INPUTDIR = f"{ECCS2DIR}/input" ECCS2LISTIDPSURL = 'https://technical.edugain.org/api.php?action=list_eccs_idps&format=json' -ECCS2LISTIDPSFILE = "%s/list_eccs_idps.json" % ECCS2INPUTDIR +ECCS2LISTIDPSFILE = f"{ECCS2INPUTDIR}/list_eccs_idps.json" ECCS2LISTFEDSURL = 'https://technical.edugain.org/api.php?action=list_feds&opt=1&format=json' -ECCS2LISTFEDSFILE = "%s/list_fed.json" % ECCS2INPUTDIR +ECCS2LISTFEDSFILE = f"{ECCS2INPUTDIR}/list_fed.json" # Output -ECCS2OUTPUTDIR = "%s/output" % ECCS2DIR -ECCS2RESULTSLOG = "eccs2_%s.log" % DAY -ECCS2HTMLDIR = "%s/html" % ECCS2DIR +ECCS2OUTPUTDIR = f"{ECCS2DIR}/output" +ECCS2RESULTSLOG = f"eccs2_{DAY}.log" +ECCS2HTMLDIR = f"{ECCS2DIR}/html" # Selenium ECCS2SELENIUMDEBUG = False -ECCS2SELENIUMLOGDIR = "%s/selenium-logs" % ECCS2DIR -ECCS2SELENIUMPAGELOADTIMEOUT = 30 #seconds +ECCS2SELENIUMLOGDIR = f"{ECCS2DIR}/selenium-logs" +ECCS2SELENIUMPAGELOADTIMEOUT = 30 #seconds (remind to change timeout seconds also on web/eccs2.js) ECCS2SELENIUMSCRIPTTIMEOUT = 30 #seconds ECCS2REQUESTSTIMEOUT = 15 #seconds # Logs -ECCS2LOGSDIR = "%s/logs" % ECCS2DIR -ECCS2STDOUT = "%s/stdout_%s.log" % (ECCS2LOGSDIR,DAY) -ECCS2STDERR = "%s/stderr_%s.log" % (ECCS2LOGSDIR,DAY) -ECCS2FAILEDCMD = "%s/failed-cmd.sh" % ECCS2LOGSDIR -ECCS2STDOUTIDP = "%s/stdout_idp_%s.log" % (ECCS2LOGSDIR,DAY) -ECCS2STDERRIDP = "%s/stderr_idp_%s.log" % (ECCS2LOGSDIR,DAY) -ECCS2FAILEDCMDIDP = "%s/failed-cmd-idp.sh" % ECCS2LOGSDIR +ECCS2LOGSDIR = f"{ECCS2DIR}/logs" +ECCS2STDOUT = f"{ECCS2LOGSDIR}/stdout_{DAY}.log" +ECCS2STDERR = f"{ECCS2LOGSDIR}/stderr_{DAY}.log" +ECCS2FAILEDCMD = f"{ECCS2LOGSDIR}/failed-cmd.sh" +ECCS2STDOUTIDP = f"{ECCS2LOGSDIR}/stdout_idp_{DAY}.log" +ECCS2STDERRIDP = f"{ECCS2LOGSDIR}/stderr_idp_{DAY}.log" +ECCS2FAILEDCMDIDP = f"{ECCS2LOGSDIR}/failed-cmd-idp.sh" # Number of processes to run in parallel ECCS2NUMPROCESSES = 35 diff --git a/retryFailedChecks.py b/retryFailedChecks.py new file mode 100755 index 0000000000000000000000000000000000000000..482d0bc0dba72e2ec5ab458851c6d4935c2644b1 --- /dev/null +++ b/retryFailedChecks.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python3 + +import os +import eccs2properties as e2p +import utils + +def get_idp_entityID(line): + import json + + line = line.lstrip(f"{e2p.ECCS2DIR}/eccs2.py '") + line = line.rstrip("\'\n") + json_line = json.loads(line) + + return json_line['entityID'] + + +# MAIN +if __name__=="__main__": + + if (os.stat(e2p.ECCS2FAILEDCMD).st_size == 0): + print(f"{e2p.DAY} - ECCS2 OK: All eduGAIN IdPs have been checked successfully.") + else: + with open(e2p.ECCS2FAILEDCMD) as f: + + # For each one, run ECCS2 check and remove its line + # from the "failed-cmd.sh". + for line in f: + idp = get_idp_entityID(line) + os.system(f'{line.rstrip()}') + + with open(f'{e2p.ECCS2OUTPUTDIR}/eccs2_{e2p.DAY}.log') as o: + for line in o: + if (idp in line): + utils.delete_line_with_word(e2p.ECCS2FAILEDCMD,idp) + print(f"ECCS2 check retried successfully for {idp}") + + + if (os.stat(e2p.ECCS2FAILEDCMD).st_size == 0): + print(f"{e2p.DAY} - ECCS2 OK: All eduGAIN IdPs have been checked successfully.") + else: + print(f"{e2p.DAY} - Something went wrong. See the log files and failed-cmd.sh.")