From f0c5771609033e91e89228f44c59117d62a1c999 Mon Sep 17 00:00:00 2001
From: Marco Malavolti <marco.malavolti@gmail.com>
Date: Mon, 26 Jul 2021 18:05:24 +0200
Subject: [PATCH] Added 'retryFailedChecks.py' script to flow
---
README.md | 10 +++++-----
cleanAndRunEccs2.sh | 3 +++
eccs2properties.py | 36 ++++++++++++++++++------------------
retryFailedChecks.py | 41 +++++++++++++++++++++++++++++++++++++++++
4 files changed, 67 insertions(+), 23 deletions(-)
create mode 100755 retryFailedChecks.py
diff --git a/README.md b/README.md
index 9c4633d..0630b12 100644
--- a/README.md
+++ b/README.md
@@ -95,7 +95,7 @@ The tool uses following status for IdPs:
# Requirements Software
* Apache Server + WSGI
-* Python 3.9 (tested with v3.9.6)
+* Python 3.9 (tested with v3.9.1)
* Selenim + Google Chrome Web Brower (tested with v91.0.4472.164)
* Chromedriver (tested with v91.0.4472.101)
* Git
@@ -136,14 +136,14 @@ The tool uses following status for IdPs:
### Python 3.9
1. Download the last version of Python 3.9.x from https://www.python.org/downloads/source/ into your home:
- * `wget https://www.python.org/ftp/python/3.9.6/Python-3.9.6.tgz -O $HOME/eccs2/Python-3.9.6.tgz`
+ * `wget https://www.python.org/ftp/python/3.9.1/Python-3.9.1.tgz -O $HOME/eccs2/Python-3.9.1.tgz`
2. Extract Python source package:
* `cd $HOME/eccs2/`
- * `tar xzf Python-3.9.6.tgz`
+ * `tar xzf Python-3.9.1.tgz`
3. Build Python from the source package:
- * `cd $HOME/eccs2/Python-3.9.6`
+ * `cd $HOME/eccs2/Python-3.9.1`
* `./configure --prefix=$HOME/eccs2/python`
* `make`
@@ -154,7 +154,7 @@ The tool uses following status for IdPs:
This will install python under your $HOME directory.
5. Remove useless things:
- * `rm -Rf $HOME/eccs2/Python-3.9.9 $HOME/eccs2/Python-3.9.6.tgz`
+ * `rm -Rf $HOME/eccs2/Python-3.9.1 $HOME/eccs2/Python-3.9.1.tgz`
# Install Google Chrome needed by Selenium
diff --git a/cleanAndRunEccs2.sh b/cleanAndRunEccs2.sh
index dbc494d..9242c5d 100755
--- a/cleanAndRunEccs2.sh
+++ b/cleanAndRunEccs2.sh
@@ -11,3 +11,6 @@ rm -f $BASEDIR/eccs2/input/*.json
# Run ECCS2
$BASEDIR/eccs2/runEccs2.py
+
+# Run again ECCS2 for those IdPs who failed check
+$BASEDIR/eccs2/retryFailedChecks.py
diff --git a/eccs2properties.py b/eccs2properties.py
index 7ed262a..cd2f719 100644
--- a/eccs2properties.py
+++ b/eccs2properties.py
@@ -3,37 +3,37 @@ from datetime import date
DAY = date.today().isoformat()
-ECCS2DIR = "%s/eccs2" % os.environ['HOME']
-PATHCHROMEDRIVER = "%s/chromedriver" % ECCS2DIR
-ECCS2PYTHON = "%s/python/bin/python3" % ECCS2DIR
+ECCS2DIR = f"{os.environ['HOME']}/eccs2"
+PATHCHROMEDRIVER = f"{ECCS2DIR}/chromedriver"
+ECCS2PYTHON = f"{ECCS2DIR}/python/bin/python3"
# Input
-ECCS2INPUTDIR = "%s/input" % ECCS2DIR
+ECCS2INPUTDIR = f"{ECCS2DIR}/input"
ECCS2LISTIDPSURL = 'https://technical.edugain.org/api.php?action=list_eccs_idps&format=json'
-ECCS2LISTIDPSFILE = "%s/list_eccs_idps.json" % ECCS2INPUTDIR
+ECCS2LISTIDPSFILE = f"{ECCS2INPUTDIR}/list_eccs_idps.json"
ECCS2LISTFEDSURL = 'https://technical.edugain.org/api.php?action=list_feds&opt=1&format=json'
-ECCS2LISTFEDSFILE = "%s/list_fed.json" % ECCS2INPUTDIR
+ECCS2LISTFEDSFILE = f"{ECCS2INPUTDIR}/list_fed.json"
# Output
-ECCS2OUTPUTDIR = "%s/output" % ECCS2DIR
-ECCS2RESULTSLOG = "eccs2_%s.log" % DAY
-ECCS2HTMLDIR = "%s/html" % ECCS2DIR
+ECCS2OUTPUTDIR = f"{ECCS2DIR}/output"
+ECCS2RESULTSLOG = f"eccs2_{DAY}.log"
+ECCS2HTMLDIR = f"{ECCS2DIR}/html"
# Selenium
ECCS2SELENIUMDEBUG = False
-ECCS2SELENIUMLOGDIR = "%s/selenium-logs" % ECCS2DIR
-ECCS2SELENIUMPAGELOADTIMEOUT = 30 #seconds
+ECCS2SELENIUMLOGDIR = f"{ECCS2DIR}/selenium-logs"
+ECCS2SELENIUMPAGELOADTIMEOUT = 30 #seconds (remind to change timeout seconds also on web/eccs2.js)
ECCS2SELENIUMSCRIPTTIMEOUT = 30 #seconds
ECCS2REQUESTSTIMEOUT = 15 #seconds
# Logs
-ECCS2LOGSDIR = "%s/logs" % ECCS2DIR
-ECCS2STDOUT = "%s/stdout_%s.log" % (ECCS2LOGSDIR,DAY)
-ECCS2STDERR = "%s/stderr_%s.log" % (ECCS2LOGSDIR,DAY)
-ECCS2FAILEDCMD = "%s/failed-cmd.sh" % ECCS2LOGSDIR
-ECCS2STDOUTIDP = "%s/stdout_idp_%s.log" % (ECCS2LOGSDIR,DAY)
-ECCS2STDERRIDP = "%s/stderr_idp_%s.log" % (ECCS2LOGSDIR,DAY)
-ECCS2FAILEDCMDIDP = "%s/failed-cmd-idp.sh" % ECCS2LOGSDIR
+ECCS2LOGSDIR = f"{ECCS2DIR}/logs"
+ECCS2STDOUT = f"{ECCS2LOGSDIR}/stdout_{DAY}.log"
+ECCS2STDERR = f"{ECCS2LOGSDIR}/stderr_{DAY}.log"
+ECCS2FAILEDCMD = f"{ECCS2LOGSDIR}/failed-cmd.sh"
+ECCS2STDOUTIDP = f"{ECCS2LOGSDIR}/stdout_idp_{DAY}.log"
+ECCS2STDERRIDP = f"{ECCS2LOGSDIR}/stderr_idp_{DAY}.log"
+ECCS2FAILEDCMDIDP = f"{ECCS2LOGSDIR}/failed-cmd-idp.sh"
# Number of processes to run in parallel
ECCS2NUMPROCESSES = 35
diff --git a/retryFailedChecks.py b/retryFailedChecks.py
new file mode 100755
index 0000000..482d0bc
--- /dev/null
+++ b/retryFailedChecks.py
@@ -0,0 +1,41 @@
+#!/usr/bin/env python3
+
+import os
+import eccs2properties as e2p
+import utils
+
+def get_idp_entityID(line):
+ import json
+
+ line = line.lstrip(f"{e2p.ECCS2DIR}/eccs2.py '")
+ line = line.rstrip("\'\n")
+ json_line = json.loads(line)
+
+ return json_line['entityID']
+
+
+# MAIN
+if __name__=="__main__":
+
+ if (os.stat(e2p.ECCS2FAILEDCMD).st_size == 0):
+ print(f"{e2p.DAY} - ECCS2 OK: All eduGAIN IdPs have been checked successfully.")
+ else:
+ with open(e2p.ECCS2FAILEDCMD) as f:
+
+ # For each one, run ECCS2 check and remove its line
+ # from the "failed-cmd.sh".
+ for line in f:
+ idp = get_idp_entityID(line)
+ os.system(f'{line.rstrip()}')
+
+ with open(f'{e2p.ECCS2OUTPUTDIR}/eccs2_{e2p.DAY}.log') as o:
+ for line in o:
+ if (idp in line):
+ utils.delete_line_with_word(e2p.ECCS2FAILEDCMD,idp)
+ print(f"ECCS2 check retried successfully for {idp}")
+
+
+ if (os.stat(e2p.ECCS2FAILEDCMD).st_size == 0):
+ print(f"{e2p.DAY} - ECCS2 OK: All eduGAIN IdPs have been checked successfully.")
+ else:
+ print(f"{e2p.DAY} - Something went wrong. See the log files and failed-cmd.sh.")
--
GitLab