diff --git a/.gitignore b/.gitignore
deleted file mode 100644
index 0fc8ee934dcad5c6b7733ffe578539a282b93661..0000000000000000000000000000000000000000
--- a/.gitignore
+++ /dev/null
@@ -1,3 +0,0 @@
-selenium_chromedriver.log
-*.swp
-*.txt
diff --git a/README.md b/README.md
index a47b8d42734ae554857c25abaa2f2c3972aed98e..7add029f160ab225870af61642cecf696f6cbf8c 100644
--- a/README.md
+++ b/README.md
@@ -1,6 +1,21 @@
-# HOWTO Install and Configure ECCS-2
+# EduGAIN Connectivity Check Service 2
-* `sudo apt install python3 python3-pip chromium chromium-l10n git libapache2-mod-wsgi python3-dev`
+# Requirements Hardware
+
+* OS: Debian 9,10 (tested)
+* HDD: 10 GB
+* RAM: 4 GB
+* CPU: >= 2 vCPU
+
+# Requirements Software
+
+* Apache Server + WSGI
+* Python 3.8
+* Selenim + Chromium Web Brower
+
+# HOWTO Install and Configure
+
+* `sudo apt install chromium chromium-l10n git jq`
* `python3 -m pip install --user --upgrade pip virtualenv`
* `python3 -m venv eccs2venv`
* `source eccs2venv/bin/activate` (`deactivate` to exit Virtualenv)
diff --git a/api.py b/api.py
index 3e17f09bf49ca50269c046fc88e2b86257f8cf65..75f5a53516c914480327142837160fe17140c2db 100755
--- a/api.py
+++ b/api.py
@@ -3,58 +3,17 @@
import logging
import re
-from eccs2properties import DAY,ECCS2LOGSDIR
+from eccs2properties import DAY,ECCS2LOGSDIR,ECCS2OUTPUTDIR
from flask import Flask, request, jsonify
from flask_restful import Resource, Api
from json import dumps, loads
from logging.handlers import RotatingFileHandler
from pathlib import PurePath
+from utils import getLogger, getDriver
app = Flask(__name__)
api = Api(app)
-def getLogger(filename,log_level="DEBUG",path="./"):
-
- logger = logging.getLogger(filename)
- ch = logging.FileHandler(path+filename,'w','utf-8')
-
- if (log_level == "DEBUG"):
- logger.setLevel(logging.DEBUG)
- ch.setLevel(logging.DEBUG)
- elif (log_level == "INFO"):
- logger.setLevel(logging.INFO)
- ch.setLevel(logging.INFO)
- elif (log_level == "WARN"):
- logger.setLevel(logging.WARN)
- ch.setLevel(logging.WARN)
- elif (log_level == "ERROR"):
- logger.setLevel(logging.ERROR)
- ch.setLevel(logging.ERROR)
- elif (log_level == "CRITICAL"):
- logger.setLevel(logging.CRITICAL)
- ch.setLevel(logging.CRITICAL)
-
- formatter = logging.Formatter('%(message)s')
- ch.setFormatter(formatter)
- logger.addHandler(ch)
-
- return logger
-
-
-# Setup Chromium Webdriver
-def setup():
-
- chrome_options = webdriver.ChromeOptions()
- chrome_options.add_argument('--headless')
- chrome_options.add_argument('--no-sandbox')
-
- driver = webdriver.Chrome('chromedriver', chrome_options=chrome_options)
-
- # Configure timeouts
- driver.set_page_load_timeout(30)
- driver.set_script_timeout(30)
-
- return driver
# /eccs2/test
class Test(Resource):
@@ -67,7 +26,7 @@ class Checks(Resource):
def get(self):
app.logger.info("Request 'Checks'")
- file_path = "%s/eccs2checks_%s.log" % (ECCS2LOGSDIR,DAY)
+ file_path = "%s/eccs2checks_%s.log" % (ECCS2OUTPUTDIR,DAY)
date = PurePath(file_path).parts[-1].split('_')[1].split('.')[0]
pretty = 0
status = None
@@ -76,7 +35,7 @@ class Checks(Resource):
if 'date' in request.args:
app.logger.info("'date' parameter inserted")
date = request.args['date']
- file_path = "%s/eccs2checks_%s.log" % (ECCS2LOGSDIR,date)
+ file_path = "%s/eccs2checks_%s.log" % (ECCS2OUTPUTDIR,date)
if 'pretty' in request.args:
app.logger.info("'pretty' parameter inserted")
pretty = request.args['pretty']
@@ -97,41 +56,49 @@ class Checks(Resource):
check_idp = check[0]
check_sp = check[1]
- check_status = check[2].rstrip("\n\r")
+ status_code = check[2]
+ check_time = check[3]
+ check_status = check[4].rstrip("\n\r")
if (idp and status):
- app.logger.info("Checks for 'idp' and 'status'.")
+ app.logger.info("Search for 'idp':'%s' and 'status':'%s'." % (idp,status))
if (idp == check_idp and status == check_status):
result.append( { 'sp' : check_sp,
'idp' : check_idp,
+ 'check_time': check_time,
+ 'status_code': status_code,
'status' : check_status,
'date': date
} )
elif (idp):
- #app.logger.info(re.search(".*."+idp+".*.", check_idp, re.IGNORECASE))
- #app.logger.info(check_idp))
- app.logger.info("Checks for Idp '%s'." % idp)
+ app.logger.info("Search for 'idp':'%s'" % idp)
if (re.search(".*."+idp+".*.", check_idp, re.IGNORECASE)):
result.append( { 'sp' : check_sp,
'idp' : check_idp,
+ 'check_time': check_time,
+ 'status_code': status_code,
'status' : check_status,
'date': date
} )
elif (status):
- app.logger.info("Check for the status '%s'." % status)
+ app.logger.info("Search for 'status':'%s'." % status)
if (status == check_status):
+ result.append( { 'sp' : check_sp,
+ 'idp' : check_idp,
+ 'check_time': check_time,
+ 'status_code': status_code,
+ 'status' : check_status,
+ 'date': date
+ } )
+ else:
+ app.logger.info("All checks.")
result.append( { 'sp' : check_sp,
'idp' : check_idp,
+ 'check_time': check_time,
+ 'status_code': status_code,
'status' : check_status,
'date': date
} )
- else:
- app.logger.info("All checks.")
- result.append( { 'sp' : check_sp,
- 'idp' : check_idp,
- 'status' : check_status,
- 'date': date
- } )
if (pretty):
pp_json = dumps(result, indent=4, sort_keys=True)
@@ -154,7 +121,7 @@ class EccsResults(Resource):
def get(self):
app.logger.info("Request 'EccsResults'")
- file_path = "%s/eccs2_%s.log" % (ECCS2LOGSDIR,DAY)
+ file_path = "%s/eccs2_%s.log" % (ECCS2OUTPUTDIR,DAY)
date = PurePath(file_path).parts[-1].split('_')[1].split('.')[0]
pretty = 0
status = None
@@ -163,7 +130,7 @@ class EccsResults(Resource):
if 'date' in request.args:
app.logger.info("'date' parameter inserted")
date = request.args['date']
- file_path = "%s/eccs2_%s.log" % (ECCS2LOGSDIR,date)
+ file_path = "%s/eccs2_%s.log" % (ECCS2OUTPUTDIR,date)
if 'pretty' in request.args:
app.logger.info("'pretty' parameter inserted")
pretty = request.args['pretty']
@@ -197,23 +164,23 @@ class EccsResults(Resource):
# SP-status-2 check[13]
check = line.split(";")
- idp_displayname = check[0].rstrip("\n\r")
- idp_entity_id = check[1].rstrip("\n\r")
- idp_reg_auth = check[2].rstrip("\n\r")
- idp_tech_ctcs = check[3].rstrip("\n\r")
- idp_supp_ctcs = check[4].rstrip("\n\r")
- idp_checks_status = check[5].rstrip("\n\r")
- sp1_entity_id = check[6].rstrip("\n\r")
- sp1_check_time = check[7].rstrip("\n\r")
- sp1_status_code = check[8].rstrip("\n\r")
- sp1_check_status = check[9].rstrip("\n\r")
- sp2_entity_id = check[10].rstrip("\n\r")
- sp2_check_time = check[11].rstrip("\n\r")
- sp2_status_code = check[12].rstrip("\n\r")
+ idp_displayname = check[0]
+ idp_entity_id = check[1]
+ idp_reg_auth = check[2]
+ idp_tech_ctcs = check[3]
+ idp_supp_ctcs = check[4]
+ idp_checks_status = check[5]
+ sp1_entity_id = check[6]
+ sp1_check_time = check[7]
+ sp1_status_code = check[8]
+ sp1_check_status = check[9]
+ sp2_entity_id = check[10]
+ sp2_check_time = check[11]
+ sp2_status_code = check[12]
sp2_check_status = check[13].rstrip("\n\r")
if (idp and status):
- app.logger.info("Results for the idp '%s' with status '%s'" % (idp, status))
+ app.logger.info("eccsresults: check for 'idp':'%s' with 'status':'%s'" % (idp, status))
if (idp == idp_entity_id and status == idp_checks_status):
result.append(
{
@@ -228,21 +195,21 @@ class EccsResults(Resource):
'sp1' : {
'entityID' : sp1_entity_id,
'checkTime' : sp1_check_time,
- 'status' : sp1_check_status,
- 'statusCode' : sp1_status_code
+ 'statusCode' : sp1_status_code,
+ 'status' : sp1_check_status
},
'sp2' : {
'entityID' : sp2_entity_id,
'checkTime' : sp2_check_time,
- 'status' : sp2_check_status,
- 'statusCode' : sp2_status_code
+ 'statusCode' : sp2_status_code,
+ 'status' : sp2_check_status
},
'status' : idp_checks_status
} )
elif (idp):
#app.logger.info(re.search(".*."+idp+".*.", idp_entity_id, re.IGNORECASE))
#app.logger.info(idp_entity_id))
- app.logger.info("Results for IdP '%s'." % idp)
+ app.logger.info("eccsresults: results for IdP:'%s'." % idp)
if (re.search(".*."+idp+".*.", idp_entity_id, re.IGNORECASE)):
result.append(
{
@@ -257,18 +224,19 @@ class EccsResults(Resource):
'sp1' : {
'entityID' : sp1_entity_id,
'checkTime' : sp1_check_time,
- 'status' : sp1_check_status,
- 'statusCode' : sp1_status_code
+ 'statusCode' : sp1_status_code,
+ 'status' : sp1_check_status
},
'sp2' : {
'entityID' : sp2_entity_id,
'checkTime' : sp2_check_time,
- 'status' : sp2_check_status,
- 'statusCode' : sp2_status_code
+ 'statusCode' : sp2_status_code,
+ 'status' : sp2_check_status
},
'status' : idp_checks_status
} )
elif (status):
+ app.logger.info("eccsresults: Search for 'status':'%s'." % status)
if (status == idp_checks_status):
result.append(
{
@@ -283,14 +251,14 @@ class EccsResults(Resource):
'sp1' : {
'entityID' : sp1_entity_id,
'checkTime' : sp1_check_time,
- 'status' : sp1_check_status,
- 'statusCode' : sp1_status_code
+ 'statusCode' : sp1_status_code,
+ 'status' : sp1_check_status
},
'sp2' : {
'entityID' : sp2_entity_id,
'checkTime' : sp2_check_time,
- 'status' : sp2_check_status,
- 'statusCode' : sp2_status_code
+ 'statusCode' : sp2_status_code,
+ 'status' : sp2_check_status
},
'status' : idp_checks_status
} )
@@ -308,14 +276,14 @@ class EccsResults(Resource):
'sp1' : {
'entityID' : sp1_entity_id,
'checkTime' : sp1_check_time,
- 'status' : sp1_check_status,
- 'statusCode' : sp1_status_code
+ 'statusCode' : sp1_status_code,
+ 'status' : sp1_check_status
},
'sp2' : {
'entityID' : sp2_entity_id,
'checkTime' : sp2_check_time,
- 'status' : sp2_check_status,
- 'statusCode' : sp2_status_code
+ 'statusCode' : sp2_status_code,
+ 'status' : sp2_check_status
},
'status' : idp_checks_status
} )
@@ -339,5 +307,5 @@ api.add_resource(EccsResults, '/eccs/eccsresults') # Route_3
if __name__ == '__main__':
app.config['JSON_AS_ASCII'] = False
- app.logger = getLogger("eccs2api.log", "INFO", ECCS2LOGSDIR)
+ app.logger = getLogger("eccs2api.log", ECCS2LOGSDIR, "w", "INFO")
app.run(port='5002')
diff --git a/cleanAndRunEccs2.sh b/cleanAndRunEccs2.sh
new file mode 100755
index 0000000000000000000000000000000000000000..8c7dba6b748dd08aca41f3c2af922cb366a295c1
--- /dev/null
+++ b/cleanAndRunEccs2.sh
@@ -0,0 +1,7 @@
+#!/bin/bash
+
+# Remove old IdP and Fed List
+rm -f /opt/eccs2/input/*.json
+
+# Run ECCS2
+/opt/eccs2/runEccs2.py
diff --git a/eccs2.py b/eccs2.py
index 2d562de5bad07b496e21979e268512834e2ec31d..40908c68e5ea8d8ffce0d30d52e182b145b0279f 100755
--- a/eccs2.py
+++ b/eccs2.py
@@ -7,7 +7,8 @@ import re
import requests
import time
-from eccs2properties import ECCS2LOGSDIR, ECCS2RESULTSLOG, ECCS2CHECKSLOG, FEDS_BLACKLIST, IDPS_BLACKLIST, ECCS2SPS, ECCS2SELENIUMDEBUG
+from eccs2properties import DAY, ECCS2HTMLDIR, ECCS2LOGSDIR, ECCS2OUTPUTDIR, ECCS2RESULTSLOG, ECCS2CHECKSLOG, FEDS_BLACKLIST, IDPS_BLACKLIST, ECCS2SPS, ECCS2SELENIUMDEBUG
+from pathlib import Path
from selenium.webdriver.common.by import By
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support.ui import Select, WebDriverWait
@@ -17,13 +18,14 @@ from urllib3.exceptions import MaxRetryError
from urllib3.util import parse_url
from utils import getLogger, getIdPContacts, getDriver
+
"""
This script use Selenium and Chromium to select the IdP to check from a Shibboleth SP with the Shibboleth Embedded Discovery Service installed and configured to answer to all eduGAIN IdPs.
The SPs used to check an IdP will be SP24(IDEM) and Attribute Viewer (SWITCH).
The check will be passed when both SPs will return the authentication page of the IdP checked.
"""
-def checkIdP(sp,idp,logger):
+def checkIdP(sp,idp):
# Chromedriver MUST be instanced here to avoid problems with SESSION
# Disable SSL requests warning messages
@@ -33,19 +35,23 @@ def checkIdP(sp,idp,logger):
fqdn_idp = parse_url(idp['entityID'])[2]
driver = getDriver(fqdn_idp,debugSelenium)
+ # Exception of WebDriver raises
+ if (driver == None):
+ return None
+
# Configure Blacklists
federation_blacklist = FEDS_BLACKLIST
entities_blacklist = IDPS_BLACKLIST
if (idp['registrationAuthority'] in federation_blacklist):
check_time = datetime.datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%S') + 'Z'
- logger.info("%s;%s;%s;NULL;Federation excluded from checks" % (idp['entityID'],sp,check_time))
- return (sp,check_time,"NULL","DISABLED")
+ #logger.info("%s;%s;%s;NULL;Federation excluded from checks" % (idp['entityID'],sp,check_time))
+ return (idp['entityID'],sp,check_time,"NULL","DISABLED")
if (idp['entityID'] in entities_blacklist):
check_time = datetime.datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%S') + 'Z'
- logger.info("%s;%s;%s;NULL;IdP excluded from checks" % (idp['entityID'],sp,check_time))
- return (sp,check_time,"NULL","DISABLED")
+ #logger.info("%s;%s;%s;NULL;IdP excluded from checks" % (idp['entityID'],sp,check_time))
+ return (idp['entityID'],sp,check_time,"NULL","DISABLED")
# Open SP, select the IDP from the EDS and press 'Enter' to reach the IdP login page to check
try:
@@ -56,9 +62,16 @@ def checkIdP(sp,idp,logger):
page_source = driver.page_source
samlrequest_url = driver.current_url
+ # Put the code of the page into an HTML file
+ Path("%s/%s" % (ECCS2HTMLDIR,DAY)).mkdir(parents=True, exist_ok=True)
+ fqdn_idp = parse_url(idp['entityID'])[2]
+ fqdn_sp = parse_url(sp)[2]
+ with open("%s/%s/%s---%s.html" % (ECCS2HTMLDIR,DAY,fqdn_idp,fqdn_sp),"w") as html:
+ html.write(page_source)
+
except TimeoutException as e:
- logger.info("%s;%s;%s;999;Timeout" % (idp['entityID'],sp,check_time))
- return (sp,check_time,"999","Timeout")
+ #logger.info("%s;%s;999;%s;Timeout" % (idp['entityID'],sp,check_time))
+ return (idp['entityID'],sp,check_time,"999","Timeout")
except NoSuchElementException as e:
# The input of the bootstrap tables are provided by "eccs2" and "eccs2checks" log.
@@ -69,8 +82,8 @@ def checkIdP(sp,idp,logger):
return None
except UnexpectedAlertPresentException as e:
- logger.info("%s;%s;%s;888;UnexpectedAlertPresent" % (idp['entityID'],sp,check_time))
- return (sp,check_time,"888","ERROR")
+ #logger.info("%s;%s;888;%s;UnexpectedAlertPresent" % (idp['entityID'],sp,check_time))
+ return (idp['entityID'],sp,check_time,"888","ERROR")
except WebDriverException as e:
print("!!! WEB DRIVER EXCEPTION - RUN AGAIN THE COMMAND!!!")
@@ -87,6 +100,7 @@ def checkIdP(sp,idp,logger):
finally:
driver.quit()
+
pattern_metadata = "Unable.to.locate(\sissuer.in|).metadata(\sfor|)|no.metadata.found|profile.is.not.configured.for.relying.party|Cannot.locate.entity|fail.to.load.unknown.provider|does.not.recognise.the.service|unable.to.load.provider|Nous.n'avons.pas.pu.(charg|charger).le.fournisseur.de service|Metadata.not.found|application.you.have.accessed.is.not.registered.for.use.with.this.service|Message.did.not.meet.security.requirements"
pattern_username = '<input[\s]+[^>]*((type=\s*[\'"](text|email)[\'"]|user)|(name=\s*[\'"](name)[\'"]))[^>]*>';
@@ -98,7 +112,7 @@ def checkIdP(sp,idp,logger):
try:
headers = {'User-Agent':'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.61 Safari/537.36'}
- status_code = requests.get(samlrequest_url, headers=headers, verify=False, timeout=30).status_code
+ status_code = str(requests.get(samlrequest_url, headers=headers, verify=False, timeout=30).status_code)
except requests.exceptions.ConnectionError as e:
#print("!!! REQUESTS STATUS CODE CONNECTION ERROR EXCEPTION !!!")
@@ -132,88 +146,93 @@ def checkIdP(sp,idp,logger):
if(metadata_not_found):
- logger.info("%s;%s;%s;%s;No-eduGAIN-Metadata" % (idp['entityID'],sp,status_code,check_time))
- return (sp,check_time,status_code,"No-eduGAIN-Metadata")
+ #logger.info("%s;%s;%s;%s;No-eduGAIN-Metadata" % (idp['entityID'],sp,status_code,check_time))
+ return (idp['entityID'],sp,check_time,status_code,"No-eduGAIN-Metadata")
elif not username_found or not password_found:
- logger.info("%s;%s;%s;%s;Invalid-Form" % (idp['entityID'],sp,status_code,check_time))
- return (sp,check_time,status_code,"Invalid-Form")
+ #logger.info("%s;%s;%s;%s;Invalid-Form" % (idp['entityID'],sp,status_code,check_time))
+ return (idp['entityID'],sp,check_time,status_code,"Invalid-Form")
else:
- logger.info("%s;%s;%s;%s;OK" % (idp['entityID'],sp,status_code,check_time))
- return (sp,check_time,status_code,"OK")
+ #logger.info("%s;%s;%s;%s;OK" % (idp['entityID'],sp,status_code,check_time))
+ return (idp['entityID'],sp,check_time,status_code,"OK")
-def check(idp,sps,eccs2log,eccs2checksLog):
- result = []
+def check(idp,sps,eccs2log):
+ results = []
for sp in sps:
- resultCheck = checkIdP(sp,idp,eccs2checksLog)
- result.append(resultCheck)
-
- listTechContacts = getIdPContacts(idp,'technical')
- listSuppContacts = getIdPContacts(idp,'support')
-
- strTechContacts = ','.join(listTechContacts)
- strSuppContacts = ','.join(listSuppContacts)
-
- # If all checks are 'OK', than the IdP consuming correctly eduGAIN Metadata.
- if (result[0][3] == result[1][3] == "OK"):
- # IdP-DisplayName;IdP-entityID;IdP-RegAuth;IdP-tech-ctc-1,IdP-tech-ctc-2;IdP-supp-ctc-1,IdP-supp-ctc-2;Status;SP-entityID-1;SP-check-time-1;SP-status-code-1;SP-result-1;SP-entityID-2;SP-check-time-2;SP-status-code-2;SP-result-2
- eccs2log.info("%s;%s;%s;%s;%s;%s;%s;%s;%s;%s;%s;%s;%s;%s" % (
- idp['displayname'].replace("'","'").split(';')[1].split('==')[0],
- idp['entityID'],
- idp['registrationAuthority'],
- strTechContacts,
- strSuppContacts,
- 'OK',
- result[0][0],
- result[0][1],
- result[0][2],
- result[0][3],
- result[1][0],
- result[1][1],
- result[1][2],
- result[1][3]))
- elif (result[0][3] == result[1][3] == "DISABLED"):
- eccs2log.info("%s;%s;%s;%s;%s;%s;%s;%s;%s;%s;%s;%s;%s;%s" % (
- idp['displayname'].replace("'","'").split(';')[1].split('==')[0],
- idp['entityID'],
- idp['registrationAuthority'],
- strTechContacts,
- strSuppContacts,
- 'DISABLE',
- result[0][0],
- result[0][1],
- result[0][2],
- result[0][3],
- result[1][0],
- result[1][1],
- result[1][2],
- result[1][3]))
- elif (result[0][3] == None or result[1][3] == None):
- # Do nothing
- return
- else:
- eccs2log.info("%s;%s;%s;%s;%s;%s;%s;%s;%s;%s;%s;%s;%s;%s" % (
- idp['displayname'].replace("'","'").split(';')[1].split('==')[0],
- idp['entityID'],
- idp['registrationAuthority'],
- strTechContacts,
- strSuppContacts,
- 'ERROR',
- result[0][0],
- result[0][1],
- result[0][2],
- result[0][3],
- result[1][0],
- result[1][1],
- result[1][2],
- result[1][3]))
+ resultCheck = checkIdP(sp,idp)
+ # Se il checkIdP ha successo, aggiungo alla lista dei check
+ # altrimenti no.
+ if resultCheck is not None:
+ results.append(resultCheck)
+
+ if len(results) == 2:
+ with open("%s/%s" % (ECCS2OUTPUTDIR,ECCS2CHECKSLOG), 'a') as f:
+ for elem in results:
+ f.write(";".join(elem))
+ f.write("\n")
+
+ listTechContacts = getIdPContacts(idp,'technical')
+ listSuppContacts = getIdPContacts(idp,'support')
+
+ strTechContacts = ','.join(listTechContacts)
+ strSuppContacts = ','.join(listSuppContacts)
+
+ # If all checks are 'OK', than the IdP consuming correctly eduGAIN Metadata.
+ if (results[0][4] == results[1][4] == "OK"):
+ # IdP-DisplayName;IdP-entityID;IdP-RegAuth;IdP-tech-ctc-1,IdP-tech-ctc-2;IdP-supp-ctc-1,IdP-supp-ctc-2;Status;SP-entityID-1;SP-check-time-1;SP-status-code-1;SP-result-1;SP-entityID-2;SP-check-time-2;SP-status-code-2;SP-result-2
+ eccs2log.info("%s;%s;%s;%s;%s;%s;%s;%s;%s;%s;%s;%s;%s;%s" % (
+ idp['displayname'].replace("'","'").split(';')[1].split('==')[0],
+ idp['entityID'],
+ idp['registrationAuthority'],
+ strTechContacts,
+ strSuppContacts,
+ 'OK',
+ results[0][1], # SP-entityID-1
+ results[0][2], # SP-check-time-1
+ results[0][3], # SP-status-code-1
+ results[0][4], # SP-result-1
+ results[1][1], # SP-entityID-2
+ results[1][2], # SP-check-time-2
+ results[1][3], # SP-status-code-2
+ results[1][4])) # SP-result-2
+ elif (results[0][4] == results[1][4] == "DISABLED"):
+ eccs2log.info("%s;%s;%s;%s;%s;%s;%s;%s;%s;%s;%s;%s;%s;%s" % (
+ idp['displayname'].replace("'","'").split(';')[1].split('==')[0],
+ idp['entityID'],
+ idp['registrationAuthority'],
+ strTechContacts,
+ strSuppContacts,
+ 'DISABLE',
+ results[0][1],
+ results[0][2],
+ results[0][3],
+ results[0][4],
+ results[1][1],
+ results[1][2],
+ results[1][3],
+ results[1][4]))
+ else:
+ eccs2log.info("%s;%s;%s;%s;%s;%s;%s;%s;%s;%s;%s;%s;%s;%s" % (
+ idp['displayname'].replace("'","'").split(';')[1].split('==')[0],
+ idp['entityID'],
+ idp['registrationAuthority'],
+ strTechContacts,
+ strSuppContacts,
+ 'ERROR',
+ results[0][1],
+ results[0][2],
+ results[0][3],
+ results[0][4],
+ results[1][1],
+ results[1][2],
+ results[1][3],
+ results[1][4]))
# MAIN
if __name__=="__main__":
- eccs2log = getLogger(ECCS2RESULTSLOG, ECCS2LOGSDIR, 'a', "INFO")
- eccs2checksLog = getLogger(ECCS2CHECKSLOG, ECCS2LOGSDIR, 'a', "INFO")
+ eccs2log = getLogger(ECCS2RESULTSLOG, ECCS2OUTPUTDIR, 'a', "INFO")
sps = ECCS2SPS
@@ -224,4 +243,4 @@ if __name__=="__main__":
idp = json.loads(args.idpJson[0])
- check(idp,sps,eccs2log,eccs2checksLog)
+ check(idp,sps,eccs2log)
diff --git a/eccs2properties.py b/eccs2properties.py
index 67027d0629522a08047f8adbb33f6c911c34a1bd..a76852b7d17bb0f915142baec3855217557560d4 100644
--- a/eccs2properties.py
+++ b/eccs2properties.py
@@ -5,31 +5,34 @@ from datetime import date
DAY = date.today().isoformat()
ECCS2DIR = "/opt/eccs2"
-ECCS2LOGSDIR = "%s/logs" % ECCS2DIR
-ECCS2INPUTDIR = "%s/input" % ECCS2DIR
# Input
+ECCS2INPUTDIR = "%s/input" % ECCS2DIR
ECCS2LISTIDPSURL = 'https://technical.edugain.org/api.php?action=list_eccs_idps&format=json'
ECCS2LISTIDPSFILE = "%s/list_eccs_idps.json" % ECCS2INPUTDIR
ECCS2LISTFEDSURL = 'https://technical.edugain.org/api.php?action=list_feds&opt=1&format=json'
ECCS2LISTFEDSFILE = "%s/list_fed.json" % ECCS2INPUTDIR
# Output
+ECCS2OUTPUTDIR = "%s/output" % ECCS2DIR
ECCS2RESULTSLOG = "eccs2_%s.log" % DAY
ECCS2CHECKSLOG = "eccs2checks_%s.log" % DAY
-ECCS2SELENIUMLOGDIR = "%s/selenium-logs" % ECCS2DIR
-ECCS2STDOUT = "%s/stdout.log" % ECCS2LOGSDIR
-ECCS2STDERR = "%s/stderr.log" % ECCS2LOGSDIR
+ECCS2HTMLDIR = "%s/html" % ECCS2DIR
+ECCS2FAILEDCMD = "%s/failed-cmd.sh" % ECCS2LOGSDIR
-# Selenium Timeouts (in seconds)
-ECCS2SELENIUMPAGELOADTIMEOUT = 30
-ECCS2SELENIUMSCRIPTTIMEOUT = 30
+# Selenium
+ECCS2SELENIUMDEBUG = False
+ECCS2SELENIUMLOGDIR = "%s/selenium-logs" % ECCS2DIR
+ECCS2SELENIUMPAGELOADTIMEOUT = 30 #seconds
+ECCS2SELENIUMSCRIPTTIMEOUT = 30 #seconds
-# Selenium Debug Enable/Disable
-ECCS2SELENIUMDEBUG = True
+# Logs
+ECCS2LOGSDIR = "%s/logs" % ECCS2DIR
+ECCS2STDOUT = "%s/stdout_%s.log" % (ECCS2LOGSDIR,DAY)
+ECCS2STDERR = "%s/stderr_%s.log" % (ECCS2LOGSDIR,DAY)
# Number of processes to run in parallel
-ECCS2NUMPROCESSES = 20
+ECCS2NUMPROCESSES = 30
# The 2 SPs that will be used to test each IdP
ECCS2SPS = ["https://sp24-test.garr.it/secure", "https://attribute-viewer.aai.switch.ch/eds/"]
diff --git a/html/.gitignore b/html/.gitignore
new file mode 100644
index 0000000000000000000000000000000000000000..5e7d2734cfc60289debf74293817c0a8f572ff32
--- /dev/null
+++ b/html/.gitignore
@@ -0,0 +1,4 @@
+# Ignore everything in this directory
+*
+# Except this file
+!.gitignore
diff --git a/output/.gitignore b/output/.gitignore
new file mode 100644
index 0000000000000000000000000000000000000000..5e7d2734cfc60289debf74293817c0a8f572ff32
--- /dev/null
+++ b/output/.gitignore
@@ -0,0 +1,4 @@
+# Ignore everything in this directory
+*
+# Except this file
+!.gitignore
diff --git a/runEccs2.py b/runEccs2.py
index 70221e097eea204fe8afdafebfcbae0600a1d0e5..478548228bcc5cf98c888e427fd4ceb0464eacc6 100755
--- a/runEccs2.py
+++ b/runEccs2.py
@@ -7,12 +7,12 @@ import json
import time
from utils import getListFeds, getListEccsIdps, getRegAuthDict, getIdpList
-from eccs2properties import ECCS2STDOUT, ECCS2STDERR, ECCS2DIR, ECCS2NUMPROCESSES, ECCS2LISTIDPSURL, ECCS2LISTIDPSFILE, ECCS2LISTFEDSURL, ECCS2LISTFEDSFILE
+from eccs2properties import ECCS2FAILEDCMD, ECCS2STDOUT, ECCS2STDERR, ECCS2DIR, ECCS2NUMPROCESSES, ECCS2LISTIDPSURL, ECCS2LISTIDPSFILE, ECCS2LISTFEDSURL, ECCS2LISTFEDSFILE
from subprocess import Popen,PIPE
# Run Command
-async def run(name,queue,stdout_file,stderr_file):
+async def run(name,queue,stdout_file,stderr_file,cmd_file):
while True:
# Get a "cmd item" out of the queue.
cmd = await queue.get()
@@ -27,15 +27,16 @@ async def run(name,queue,stdout_file,stderr_file):
stdout, stderr = await proc.communicate()
if stdout:
- stdout_file.write(f'-----\n[cmd-out]\n{cmd}\n\n[stdout]\n{stdout.decode()}')
+ stdout_file.write('-----\n[cmd-out]\n%s\n\n[stdout]\n%s' % (cmd,stdout.decode()))
if stderr:
- stderr_file.write(f'-----\n[cmd-err]\n{cmd}\n\n[stderr]\n{stderr.decode()}')
+ stderr_file.write('-----\n[cmd-err]\n%s\n\n[stderr]\n%s' % (cmd,stderr.decode()))
+ cmd_file.write(cmd)
# Notify the queue that the "work cmd" has been processed.
queue.task_done()
-async def main(cmd_list,stdout_file,stderr_file):
+async def main(cmd_list,stdout_file,stderr_file,cmd_file):
# Create a queue that we will use to store our "workload".
queue = asyncio.Queue()
@@ -47,7 +48,7 @@ async def main(cmd_list,stdout_file,stderr_file):
tasks = []
for i in range(ECCS2NUMPROCESSES):
- task = asyncio.create_task(run("cmd-{%d}" % i, queue, stdout_file, stderr_file))
+ task = asyncio.create_task(run("cmd-{%d}" % i, queue, stdout_file, stderr_file, cmd_file))
tasks.append(task)
# Wait until the queue is fully processed.
@@ -80,6 +81,7 @@ if __name__=="__main__":
stdout_file = open(ECCS2STDOUT,"w+")
stderr_file = open(ECCS2STDERR,"w+")
+ cmd_file = open(ECCS2FAILEDCMD,"w+")
# Prepare input file for ECCS2
regAuthDict = getRegAuthDict(list_feds)
@@ -97,7 +99,7 @@ if __name__=="__main__":
proc_list.append(cmd)
count = count + 1
- asyncio.run(main(proc_list,stdout_file,stderr_file))
+ asyncio.run(main(proc_list,stdout_file,stderr_file,cmd_file))
end = time.time()
print("Time taken in hh:mm:ss - ", str(datetime.timedelta(seconds=end - start)))
diff --git a/utils.py b/utils.py
index ad16349ec179f3a588022bbbaf14f184abaf23a7..c2b818889cab781c1587f05c1cdb5282809ac468 100644
--- a/utils.py
+++ b/utils.py
@@ -7,6 +7,7 @@ import requests
from eccs2properties import ECCS2SELENIUMLOGDIR, ECCS2SELENIUMPAGELOADTIMEOUT, ECCS2SELENIUMSCRIPTTIMEOUT
from selenium import webdriver
+from selenium.common.exceptions import WebDriverException
# Returns a Dict of "{ nameFed:reg_auth }"
@@ -63,6 +64,7 @@ def getListEccsIdps(url, dest_file):
# Use logger to produce files consumed by ECCS-2 API
def getLogger(filename, path, mode, log_level="DEBUG"):
+
logger = logging.getLogger(filename)
ch = logging.FileHandler("%s/%s" % (path,filename), mode,'utf-8')
@@ -119,10 +121,15 @@ def getDriver(fqdn_idp=None,debugSelenium=False):
# For DEBUG only (By default ChromeDriver logs only warnings/errors to stderr.
# When debugging issues, it is helpful to enable more verbose logging.)
- if (debugSelenium and fqdn_idp):
- driver = webdriver.Chrome('chromedriver', options=chrome_options, service_args=['--verbose', '--log-path=%s/%s.log' % (ECCS2SELENIUMLOGDIR, fqdn_idp)])
- else:
- driver = webdriver.Chrome('chromedriver', options=chrome_options)
+ try:
+ if (debugSelenium and fqdn_idp):
+ driver = webdriver.Chrome('chromedriver', options=chrome_options, service_args=['--verbose', '--log-path=%s/%s.log' % (ECCS2SELENIUMLOGDIR, fqdn_idp)])
+ else:
+ driver = webdriver.Chrome('chromedriver', options=chrome_options)
+ except WebDriverException as e:
+ print("!!! WEB DRIVER EXCEPTION - RUN AGAIN THE COMMAND!!!")
+ print (e.__str__())
+ return None
# Configure timeouts
driver.set_page_load_timeout("%d" % ECCS2SELENIUMPAGELOADTIMEOUT)
diff --git a/web/script.js b/web/script.js
index 8d89553afd5c3f064e7f278a1c87d9d8885beaad..8460a7223efc5282f19234ae061f678e785fc2cb 100644
--- a/web/script.js
+++ b/web/script.js
@@ -1,3 +1,9 @@
+// use URL constructor and return hostname
+function getHostname(url) {
+ const urlNew = new URL(url);
+ return urlNew.hostname;
+}
+
/* Formatting function for row details - modify as you need */
function format ( d ) {
// `d` is the original data object for the row
@@ -16,6 +22,7 @@ function format ( d ) {
'<td>Check Time</td>'+
'<td>Status Code</td>'+
'<td>Result Check</td>'+
+ '<td>Page Source</td>'+
'</tr>'+
'<tr>'+
'<td>SP1:</td>'+
@@ -23,6 +30,7 @@ function format ( d ) {
'<td>'+d.sp1.checkTime+'</td>'+
'<td>'+d.sp1.statusCode+'</td>'+
'<td>'+d.sp1.status+'</td>'+
+ '<td><a href="/eccs2html/'+d.date+'/'+getHostname(d.entityID)+'---'+getHostname(d.sp1.entityID)+'.html" target="_blank">Click to open</a></td>'+
'</tr>'+
'<tr>'+
'<td>SP2:</td>'+
@@ -30,9 +38,11 @@ function format ( d ) {
'<td>'+d.sp2.checkTime+'</td>'+
'<td>'+d.sp2.statusCode+'</td>'+
'<td>'+d.sp2.status+'</td>'+
+ '<td><a href="/eccs2html/'+d.date+'/'+getHostname(d.entityID)+'---'+getHostname(d.sp2.entityID)+'.html" target="_blank">Click to open</a></td>'+
'</tr>'+
'</table>';
}
+
$(document).ready(function() {
var table = $('#eccstable').DataTable( {