diff --git a/README.md b/README.md
index b87f877f00122a6a20fd0453fb9283e029fd1ad7..a47b8d42734ae554857c25abaa2f2c3972aed98e 100644
--- a/README.md
+++ b/README.md
@@ -6,7 +6,9 @@
* `source eccs2venv/bin/activate` (`deactivate` to exit Virtualenv)
* `python3 -m pip install --upgrade wheel setuptools certifi selenium urllib3 flask flask-jsonpify flask-restful`
* `cd ~ ; git clone https://github.com/malavolti/eccs2.git`
- * `cd eccs2 ; ./eccs2.py`
+ * `cd eccs2`
+ * `cp eccs2properties.py.template eccs2properties.py` (and change it with your needs)
+ * `./runEccs2.py`
# API Development Server
diff --git a/eccs2.py b/eccs2.py
index 36b7df27c9e90b7c6299c6b213c02d846d330e3e..ea26ecc35e5f897f538ddfaa9012847354726e2d 100755
--- a/eccs2.py
+++ b/eccs2.py
@@ -4,18 +4,20 @@ import argparse
import json
import logging
import os
-import eccs2properties
import signal
import re
import requests
from datetime import date
+from eccs2properties import ECCS2LOGSPATH, ECCS2RESULTSLOG, ECCS2CHECKSLOG, ECCS2SELENIUMLOG, FEDS_BLACKLIST, IDPS_BLACKLIST
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import Select
from selenium.webdriver.common.keys import Keys
from selenium.common.exceptions import NoSuchElementException
from selenium.common.exceptions import TimeoutException
+from selenium.common.exceptions import WebDriverException
+from urllib3.exceptions import MaxRetryError
"""
@@ -27,8 +29,8 @@ from selenium.common.exceptions import TimeoutException
def checkIdP(sp,idp,logger,driver):
# Configure Blacklists
- federation_blacklist = eccs2properties.FEDS_BLACKLIST
- entities_blacklist = eccs2properties.IDPS_BLACKLIST
+ federation_blacklist = FEDS_BLACKLIST
+ entities_blacklist = IDPS_BLACKLIST
if (idp['registrationAuthority'] in federation_blacklist):
logger.info("%s;%s;NULL;Federation excluded from checks" % (idp['entityID'],sp))
@@ -46,20 +48,46 @@ def checkIdP(sp,idp,logger,driver):
except TimeoutException as e:
logger.info("%s;%s;999;TIMEOUT" % (idp['entityID'],sp))
return "TIMEOUT"
+
+ except WebDriverException as e:
+ print("!!! WEB DRIVER EXCEPTION !!!")
+ raise e
+
+ except Exception as e:
+ print ("!!! EXCEPTION !!!")
+ raise e
+
+
+ """
+ except MaxRetryError as e:
+ logger.info("%s;%s;111;MaxRetryError" % (idp['entityID'],sp))
+ return "MaxRetryError"
+
+ except ConnectionRefusedError as e:
+ logger.info("%s;%s;222;ConnectionRefusedError" % (idp['entityID'],sp))
+ return "ConnectionRefusedError"
+
+ except ConnectionError as e:
+ logger.info("%s;%s;333;ConnectionError" % (idp['entityID'],sp))
+ return "ConnectionError"
+
except NoSuchElementException as e:
print("!!! NO SUCH ELEMENT EXCEPTION !!!")
print(e.__str__())
pass
- except WebDriverException as e:
+ """
+
+ """
if "ConnectionRefusedError" in e.__str__():
logger.info("%s;%s;000;ConnectionError" % (idp['entityID'],sp))
- return "Connection-Error"
+ return "ConnectionRefusedError"
+ elif "Connection Refused" in e.__str__():
+ logger.info("%s;%s;000;ConnectionRefused" % (idp['entityID'],sp))
+ return "Connection-Refused"
else:
print("!!! UN-HANDLE WEB DRIVER EXCEPTION !!!")
raise e
- except:
- print("!!! UN-HANDLE OTHER EXCEPTION !!!")
- raise e
+ """
pattern_metadata = "Unable.to.locate(\sissuer.in|).metadata(\sfor|)|no.metadata.found|profile.is.not.configured.for.relying.party|Cannot.locate.entity|fail.to.load.unknown.provider|does.not.recognise.the.service|unable.to.load.provider|Nous.n'avons.pas.pu.(charg|charger).le.fournisseur.de service|Metadata.not.found|application.you.have.accessed.is.not.registered.for.use.with.this.service|Message.did.not.meet.security.requirements"
@@ -75,16 +103,9 @@ def checkIdP(sp,idp,logger,driver):
status_code = r.status_code
except requests.exceptions.ConnectionError as e:
- driver.delete_all_cookies()
- driver.close()
- driver.quit()
-
logger.info("%s;%s;000;ConnectionError" % (idp['entityID'],sp))
return "Connection-Error"
except requests.exceptions.RequestException as e:
- driver.delete_all_cookies()
- driver.close()
- driver.quit()
print("!!! UN-HANDLE REQUEST EXCEPTION !!!")
raise SystemExit(e)
@@ -103,10 +124,10 @@ def checkIdP(sp,idp,logger,driver):
# Use logger to produce files consumed by ECCS-2 API
-def getLogger(filename,log_level="DEBUG",path="./"):
+def getLogger(filename, path=".", log_level="DEBUG"):
logger = logging.getLogger(filename)
- ch = logging.FileHandler(path+filename,'a','utf-8')
+ ch = logging.FileHandler(path + '/' + filename,'a','utf-8')
if (log_level == "DEBUG"):
logger.setLevel(logging.DEBUG)
@@ -138,7 +159,10 @@ def getIdPContacts(idp,contactType):
for ctcType in idp['contacts']:
if (ctcType == contactType):
for ctc in idp['contacts'][contactType]:
- ctcList.append(ctc['emailOrPhone']['EmailAddress'][0])
+ if (ctc['emailOrPhone'].get('EmailAddress')):
+ ctcList.append(ctc['emailOrPhone']['EmailAddress'][0])
+ else:
+ ctcList.append('missing')
return ctcList
@@ -196,8 +220,8 @@ def checkIdp(idp,sps,eccs2log,eccs2checksLog,driver):
# MAIN
if __name__=="__main__":
- eccs2log = getLogger("logs/"+eccs2properties.ECCS2LOGPATH,"INFO")
- eccs2checksLog = getLogger("logs/"+eccs2properties.ECCS2CHECKSLOGPATH,"INFO")
+ eccs2log = getLogger(ECCS2RESULTSLOG, ECCS2LOGSPATH, "INFO")
+ eccs2checksLog = getLogger(ECCS2CHECKSLOG, ECCS2LOGSPATH, "INFO")
sps = ["https://sp24-test.garr.it/secure", "https://attribute-viewer.aai.switch.ch/eds/"]
@@ -220,9 +244,9 @@ if __name__=="__main__":
chrome_options.add_argument('--start-maximized')
chrome_options.add_argument('--disable-extensions')
- #driver = webdriver.Chrome('chromedriver', options=chrome_options, service_args=['--log-path=./selenium_chromedriver.log'])
+ driver = webdriver.Chrome('chromedriver', options=chrome_options, service_args=['--log-path=%s' % ECCS2SELENIUMLOG])
#driver = webdriver.Chrome('chromedriver', options=chrome_options, service_args=['--verbose', '--log-path=./selenium_chromedriver.log'])
- driver = webdriver.Chrome('chromedriver', options=chrome_options)
+ #driver = webdriver.Chrome('chromedriver', options=chrome_options)
# Configure timeouts: 30 sec
driver.set_page_load_timeout(30)
diff --git a/eccs2properties.py.template b/eccs2properties.py.template
new file mode 100644
index 0000000000000000000000000000000000000000..77401133e45261afa8d9b9589654bc0819431af1
--- /dev/null
+++ b/eccs2properties.py.template
@@ -0,0 +1,27 @@
+#!/usr/bin/env python3.8
+
+from datetime import date
+
+day = date.today().isoformat()
+
+ECCS2PATH = "/opt/eccs2"
+ECCS2LOGSPATH = "%s/logs" % ECCS2PATH
+ECCS2RESULTSLOG = "eccs2_%s.log" % day
+ECCS2CHECKSLOG = "eccs2checks_%s.log" % day
+ECCS2SELENIUMLOG = "%s/selenium_chromedriver.log" % ECCS2LOGSPATH
+ECCS2STDOUT = "%s/stdout.log" % ECCS2LOGSPATH
+ECCS2STDERR = "%s/stderr.log" % ECCS2LOGSPATH
+
+ECCS2NUMPROCESSES = 20
+
+FEDS_BLACKLIST = [
+ '<REGISTRATION-AUTHORITY-1>',
+ '<REGISTRATION-AUTHORITY-2>',
+ '<REGISTRATION-AUTHORITY-3>',
+]
+
+IDPS_BLACKLIST = [
+ '<ENTITYID-IDP-1>',
+ '<ENTITYID-IDP-2>',
+ '<ENTITYID-IDP-3>',
+]
diff --git a/runEccs2.py b/runEccs2.py
index 21a93ebbfdca2884324bb3e8c072d116e944632d..3bd0671a68ae4a869c483e1ed38ac88d2e47139c 100755
--- a/runEccs2.py
+++ b/runEccs2.py
@@ -8,9 +8,10 @@ import requests
import sys
import time
+from eccs2properties import ECCS2STDOUT, ECCS2STDERR, ECCS2PATH, ECCS2NUMPROCESSES
from subprocess import Popen,PIPE
-# returns a Dict on "{ nameFed:reg_auth }"
+# Returns a Dict on "{ nameFed:reg_auth }"
def getRegAuthDict(list_feds):
regAuth_dict = {}
@@ -23,7 +24,7 @@ def getRegAuthDict(list_feds):
return regAuth_dict
-# returns a list of IdP for a single federation
+# Returns a list of IdP for a single federation
def getIdpList(list_eccs_idps,reg_auth):
fed_idp_list = []
@@ -44,7 +45,7 @@ def getListFeds(url, filename):
# then open it and work with local file
with open("%s" % (filename), mode="r", encoding='utf-8') as f:
- return json.loads(f.read())
+ return json.loads(f.read().replace("'", "'"))
# Returns a Python List
@@ -57,16 +58,9 @@ def getListEccsIdps(url, filename):
# then open it and work with local file
with open("%s" % (filename), mode="r", encoding='utf-8') as f:
- return json.loads(f.read())
-
-# Prepare input file for ECCS2
-def genEccs2input(reg_auth_dict):
- for name,regAuth in reg_auth_dict.items():
- fed_idp_list = getIdpList(list_eccs_idps,regAuth)
- filename = "/tmp/data/inputEccs2/%s.txt" % name
- with open("%s" % (filename), mode="w+", encoding='utf-8') as f:
- f.write(','.join(str(idp) for idp in fed_idp_list))
+ return json.loads(f.read().replace("'", "'"))
+# Run Command
async def run(name,queue,stdout_file,stderr_file):
while True:
# Get a "cmd item" out of the queue.
@@ -82,9 +76,9 @@ async def run(name,queue,stdout_file,stderr_file):
stdout, stderr = await proc.communicate()
if stdout:
- stdout_file.write(f'-----\n[cmd-out]\n{cmd}\n[stdout]\n{stdout.decode()}')
+ stdout_file.write(f'-----\n[cmd-out]\n{cmd}\n\n[stdout]\n{stdout.decode()}')
if stderr:
- stderr_file.write(f'-----\n[cmd-err]\n{cmd}\n[stderr]\n{stderr.decode()}')
+ stderr_file.write(f'-----\n[cmd-err]\n{cmd}\n\n[stderr]\n{stderr.decode()}')
# Notify the queue that the "work cmd" has been processed.
queue.task_done()
@@ -102,7 +96,7 @@ async def main(cmd_list,stdout_file,stderr_file):
tasks = []
#for i in range(15): # !!!-WORKING-!!!
#for i in range(30): # !!!-WORSTE-!!!
- for i in range(10):
+ for i in range(ECCS2NUMPROCESSES):
task = asyncio.create_task(run("cmd-{%d}" % i, queue, stdout_file, stderr_file))
tasks.append(task)
@@ -134,18 +128,17 @@ if __name__=="__main__":
filename = "/tmp/data/list_eccs_idps.txt"
list_eccs_idps = getListEccsIdps(url, filename)
- stdout_file = open(eccs2properties.ECCS2STDOUT,"w+")
- stderr_file = open(eccs2properties.ECCS2STDERR,"w+")
+ stdout_file = open(ECCS2STDOUT,"w+")
+ stderr_file = open(ECCS2STDERR,"w+")
# Prepare input file for ECCS2
regAuthDict = getRegAuthDict(list_feds)
- #genEccs2input(regAuthDict)
for name,regAuth in regAuthDict.items():
idpJsonList = getIdpList(list_eccs_idps,regAuth)
num_idps = len(idpJsonList)
- cmd_list = [["%s/eccs2.py \'%s\'" % (eccs2properties.ECCS2PATH, json.dumps(idp))] for idp in idpJsonList]
+ cmd_list = [["%s/eccs2.py \'%s\'" % (ECCS2PATH, json.dumps(idp))] for idp in idpJsonList]
proc_list = []
count = 0