Add properties and other refactor

README.md

@@ -6,7 +6,9 @@
 * `source eccs2venv/bin/activate`   (`deactivate` to exit Virtualenv)
   * `python3 -m pip install --upgrade wheel setuptools certifi selenium urllib3 flask flask-jsonpify flask-restful`
   * `cd ~ ; git clone https://github.com/malavolti/eccs2.git`
-  * `cd eccs2 ; ./eccs2.py`
+  * `cd eccs2`
+  * `cp eccs2properties.py.template eccs2properties.py` (and change it with your needs)
+  * `./runEccs2.py`
 
 # API Development Server
 

eccs2.py

@@ -4,18 +4,20 @@ import argparse
 import json
 import logging
 import os
-import eccs2properties
 import signal
 import re
 import requests
 
 from datetime import date
+from eccs2properties import ECCS2LOGSPATH, ECCS2RESULTSLOG, ECCS2CHECKSLOG, ECCS2SELENIUMLOG, FEDS_BLACKLIST, IDPS_BLACKLIST
 from selenium import webdriver
 from selenium.webdriver.common.by import By
 from selenium.webdriver.support.ui import Select
 from selenium.webdriver.common.keys import Keys
 from selenium.common.exceptions import NoSuchElementException
 from selenium.common.exceptions import TimeoutException
+from selenium.common.exceptions import WebDriverException
+from urllib3.exceptions import MaxRetryError
 
 
 """
@@ -27,8 +29,8 @@ from selenium.common.exceptions import TimeoutException
 def checkIdP(sp,idp,logger,driver):
 
    # Configure Blacklists
-   federation_blacklist = eccs2properties.FEDS_BLACKLIST
-   entities_blacklist = eccs2properties.IDPS_BLACKLIST 
+   federation_blacklist = FEDS_BLACKLIST
+   entities_blacklist = IDPS_BLACKLIST 
 
    if (idp['registrationAuthority'] in federation_blacklist):
       logger.info("%s;%s;NULL;Federation excluded from checks" % (idp['entityID'],sp))
@@ -46,20 +48,46 @@ def checkIdP(sp,idp,logger,driver):
    except TimeoutException as e:
      logger.info("%s;%s;999;TIMEOUT" % (idp['entityID'],sp))
      return "TIMEOUT"
+
+   except WebDriverException as e:
+     print("!!! WEB DRIVER EXCEPTION !!!")
+     raise e
+
+   except Exception as e:
+     print ("!!! EXCEPTION !!!")
+     raise e
+
+
+   """
+   except MaxRetryError as e:
+     logger.info("%s;%s;111;MaxRetryError" % (idp['entityID'],sp))
+     return "MaxRetryError"
+
+   except ConnectionRefusedError as e:
+     logger.info("%s;%s;222;ConnectionRefusedError" % (idp['entityID'],sp))
+     return "ConnectionRefusedError"
+
+   except ConnectionError as e:
+     logger.info("%s;%s;333;ConnectionError" % (idp['entityID'],sp))
+     return "ConnectionError"
+
    except NoSuchElementException as e:
      print("!!! NO SUCH ELEMENT EXCEPTION !!!")
      print(e.__str__())
      pass
-   except WebDriverException as e:
+   """
+
+   """
      if "ConnectionRefusedError" in e.__str__():
         logger.info("%s;%s;000;ConnectionError" % (idp['entityID'],sp))
-        return "Connection-Error"
+        return "ConnectionRefusedError"
+     elif "Connection Refused" in e.__str__():
+        logger.info("%s;%s;000;ConnectionRefused" % (idp['entityID'],sp))
+        return "Connection-Refused"
      else:
         print("!!! UN-HANDLE WEB DRIVER EXCEPTION !!!")
         raise e
-   except:
-     print("!!! UN-HANDLE OTHER EXCEPTION !!!")
-     raise e
+   """
 
    pattern_metadata = "Unable.to.locate(\sissuer.in|).metadata(\sfor|)|no.metadata.found|profile.is.not.configured.for.relying.party|Cannot.locate.entity|fail.to.load.unknown.provider|does.not.recognise.the.service|unable.to.load.provider|Nous.n'avons.pas.pu.(charg|charger).le.fournisseur.de service|Metadata.not.found|application.you.have.accessed.is.not.registered.for.use.with.this.service|Message.did.not.meet.security.requirements"
 
@@ -75,16 +103,9 @@ def checkIdP(sp,idp,logger,driver):
       status_code = r.status_code
 
    except requests.exceptions.ConnectionError as e:
-      driver.delete_all_cookies()
-      driver.close()
-      driver.quit()
-
       logger.info("%s;%s;000;ConnectionError" % (idp['entityID'],sp))
       return "Connection-Error"
    except requests.exceptions.RequestException as e:
-      driver.delete_all_cookies()
-      driver.close()
-      driver.quit()
       print("!!! UN-HANDLE REQUEST EXCEPTION !!!")
       raise SystemExit(e)
 
@@ -103,10 +124,10 @@ def checkIdP(sp,idp,logger,driver):
 
 
 # Use logger to produce files consumed by ECCS-2 API
-def getLogger(filename,log_level="DEBUG",path="./"):
+def getLogger(filename, path=".", log_level="DEBUG"):
 
     logger = logging.getLogger(filename)
-    ch = logging.FileHandler(path+filename,'a','utf-8')
+    ch = logging.FileHandler(path + '/' + filename,'a','utf-8')
 
     if (log_level == "DEBUG"):
        logger.setLevel(logging.DEBUG)
@@ -138,7 +159,10 @@ def getIdPContacts(idp,contactType):
    for ctcType in idp['contacts']:
       if (ctcType == contactType):
          for ctc in idp['contacts'][contactType]:
-            ctcList.append(ctc['emailOrPhone']['EmailAddress'][0])
+            if (ctc['emailOrPhone'].get('EmailAddress')):
+               ctcList.append(ctc['emailOrPhone']['EmailAddress'][0])
+            else:
+               ctcList.append('missing')
 
    return ctcList
 
@@ -196,8 +220,8 @@ def checkIdp(idp,sps,eccs2log,eccs2checksLog,driver):
 # MAIN
 if __name__=="__main__":
 
-   eccs2log = getLogger("logs/"+eccs2properties.ECCS2LOGPATH,"INFO")
-   eccs2checksLog = getLogger("logs/"+eccs2properties.ECCS2CHECKSLOGPATH,"INFO")
+   eccs2log = getLogger(ECCS2RESULTSLOG, ECCS2LOGSPATH, "INFO")
+   eccs2checksLog = getLogger(ECCS2CHECKSLOG, ECCS2LOGSPATH, "INFO")
 
    sps = ["https://sp24-test.garr.it/secure", "https://attribute-viewer.aai.switch.ch/eds/"]
 
@@ -220,9 +244,9 @@ if __name__=="__main__":
    chrome_options.add_argument('--start-maximized')
    chrome_options.add_argument('--disable-extensions')
 
-   #driver = webdriver.Chrome('chromedriver', options=chrome_options,  service_args=['--log-path=./selenium_chromedriver.log'])
+   driver = webdriver.Chrome('chromedriver', options=chrome_options,  service_args=['--log-path=%s' % ECCS2SELENIUMLOG])
    #driver = webdriver.Chrome('chromedriver', options=chrome_options,  service_args=['--verbose', '--log-path=./selenium_chromedriver.log'])
-   driver = webdriver.Chrome('chromedriver', options=chrome_options)
+   #driver = webdriver.Chrome('chromedriver', options=chrome_options)
 
    # Configure timeouts: 30 sec
    driver.set_page_load_timeout(30)

eccs2properties.py.template

+#!/usr/bin/env python3.8
+
+from datetime import date
+
+day = date.today().isoformat()
+
+ECCS2PATH = "/opt/eccs2"
+ECCS2LOGSPATH = "%s/logs" % ECCS2PATH
+ECCS2RESULTSLOG = "eccs2_%s.log" % day
+ECCS2CHECKSLOG = "eccs2checks_%s.log" % day
+ECCS2SELENIUMLOG = "%s/selenium_chromedriver.log" % ECCS2LOGSPATH
+ECCS2STDOUT = "%s/stdout.log" % ECCS2LOGSPATH
+ECCS2STDERR = "%s/stderr.log" % ECCS2LOGSPATH
+
+ECCS2NUMPROCESSES = 20
+
+FEDS_BLACKLIST = [
+   '<REGISTRATION-AUTHORITY-1>',
+   '<REGISTRATION-AUTHORITY-2>',
+   '<REGISTRATION-AUTHORITY-3>',
+]
+
+IDPS_BLACKLIST = [
+   '<ENTITYID-IDP-1>',
+   '<ENTITYID-IDP-2>',
+   '<ENTITYID-IDP-3>',
+]

runEccs2.py

@@ -8,9 +8,10 @@ import requests
 import sys
 import time
 
+from eccs2properties import ECCS2STDOUT, ECCS2STDERR, ECCS2PATH, ECCS2NUMPROCESSES 
 from subprocess import Popen,PIPE
 
-# returns a Dict on "{ nameFed:reg_auth }"
+# Returns a Dict on "{ nameFed:reg_auth }"
 def getRegAuthDict(list_feds):
    regAuth_dict = {}
 
@@ -23,7 +24,7 @@ def getRegAuthDict(list_feds):
    return regAuth_dict
 
 
-# returns a list of IdP for a single federation
+# Returns a list of IdP for a single federation
 def getIdpList(list_eccs_idps,reg_auth):
 
    fed_idp_list = []
@@ -44,7 +45,7 @@ def getListFeds(url, filename):
 
    # then open it and work with local file
    with open("%s" % (filename), mode="r", encoding='utf-8') as f:
-      return json.loads(f.read())
+      return json.loads(f.read().replace("'", "'"))
 
 
 # Returns a Python List
@@ -57,16 +58,9 @@ def getListEccsIdps(url, filename):
 
    # then open it and work with local file
    with open("%s" % (filename), mode="r", encoding='utf-8') as f:
-      return json.loads(f.read())
-
-# Prepare input file for ECCS2
-def genEccs2input(reg_auth_dict):
-   for name,regAuth in reg_auth_dict.items():
-      fed_idp_list = getIdpList(list_eccs_idps,regAuth)
-      filename = "/tmp/data/inputEccs2/%s.txt" % name
-      with open("%s" % (filename), mode="w+", encoding='utf-8') as f:
-         f.write(','.join(str(idp) for idp in fed_idp_list))
+      return json.loads(f.read().replace("'", "'"))
 
+# Run Command
 async def run(name,queue,stdout_file,stderr_file):
    while True:
       # Get a "cmd item" out of the queue.
@@ -82,9 +76,9 @@ async def run(name,queue,stdout_file,stderr_file):
       stdout, stderr = await proc.communicate()
 
       if stdout:
-         stdout_file.write(f'-----\n[cmd-out]\n{cmd}\n[stdout]\n{stdout.decode()}')
+         stdout_file.write(f'-----\n[cmd-out]\n{cmd}\n\n[stdout]\n{stdout.decode()}')
       if stderr:
-         stderr_file.write(f'-----\n[cmd-err]\n{cmd}\n[stderr]\n{stderr.decode()}')
+         stderr_file.write(f'-----\n[cmd-err]\n{cmd}\n\n[stderr]\n{stderr.decode()}')
 
       # Notify the queue that the "work cmd" has been processed.
       queue.task_done()
@@ -102,7 +96,7 @@ async def main(cmd_list,stdout_file,stderr_file):
     tasks = []
     #for i in range(15): # !!!-WORKING-!!!
     #for i in range(30): # !!!-WORSTE-!!!
-    for i in range(10):
+    for i in range(ECCS2NUMPROCESSES):
         task = asyncio.create_task(run("cmd-{%d}" % i, queue, stdout_file, stderr_file))
         tasks.append(task)
 
@@ -134,18 +128,17 @@ if __name__=="__main__":
    filename = "/tmp/data/list_eccs_idps.txt"
    list_eccs_idps = getListEccsIdps(url, filename)
 
-   stdout_file = open(eccs2properties.ECCS2STDOUT,"w+")
-   stderr_file = open(eccs2properties.ECCS2STDERR,"w+")
+   stdout_file = open(ECCS2STDOUT,"w+")
+   stderr_file = open(ECCS2STDERR,"w+")
 
    # Prepare input file for ECCS2
    regAuthDict = getRegAuthDict(list_feds)
-   #genEccs2input(regAuthDict)
 
    for name,regAuth in regAuthDict.items():
       idpJsonList = getIdpList(list_eccs_idps,regAuth)
 
       num_idps = len(idpJsonList)
-      cmd_list = [["%s/eccs2.py \'%s\'" % (eccs2properties.ECCS2PATH, json.dumps(idp))] for idp in idpJsonList]
+      cmd_list = [["%s/eccs2.py \'%s\'" % (ECCS2PATH, json.dumps(idp))] for idp in idpJsonList]
 
       proc_list = []
       count = 0