Added multiprocessing execution

README.md

@@ -3,11 +3,17 @@
 * `sudo apt install python3 python3-pip chromium chromium-l10n git libapache2-mod-wsgi python3-dev`
 * `python3 -m pip install --user --upgrade pip virtualenv`
 * `python3 -m venv eccs2venv`
-* `source eccs2venv/bin/activate`   (`deactivate` di exit Virtualenv)
+* `source eccs2venv/bin/activate`   (`deactivate` to exit Virtualenv)
   * `python3 -m pip install --upgrade wheel setuptools certifi selenium urllib3 flask flask-jsonpify flask-restful`
   * `cd ~ ; git clone https://github.com/malavolti/eccs2.git`
   * `cd eccs2 ; ./eccs2.py`
 
+# API Development Server
+
+* `sudo apt install libapache2-mod-wsgi-py3 python3-dev`
+* `sudo a2enmod wsgi`
+* `cd ~/eccs2 ; ./api.py`
+
 # API
 
 * `/eccs/test` (Trivial Test)
@@ -23,8 +29,4 @@
     * 'Excluded'
 * /eccs/eccsresults (Return the results of the last check ready for ECCS Gui)
 
-# API Development Server
 
-* `sudo apt install libapache2-mod-wsgi-py3 python3-dev`
-* `sudo a2enmod wsgi`
-* `cd ~/eccs2 ; ./api.py`

api.py

@@ -8,6 +8,7 @@ from pathlib import PurePath
 import logging
 from logging.handlers import RotatingFileHandler
 import re
+import eccs2properties
 
 
 app = Flask(__name__)
@@ -67,7 +68,7 @@ class Checks(Resource):
     def get(self):
        app.logger.info("Request 'Checks'")
 
-       file_path = "logs/eccs2checks_2020-02-22.log"
+       file_path = "logs/eccs2checks_%s.log" % eccs2properties.day
        date = PurePath(file_path).parts[-1].split('_')[1].split('.')[0]
        pretty = 0
        status = None 
@@ -142,7 +143,6 @@ class Checks(Resource):
 
 # Build Email Addresses Link for ECCS2 Web Gui
 def buildEmailAddress(listContacts):
-
     listCtcs = listContacts.split(",")
     hrefList = []
 
@@ -155,7 +155,7 @@ class EccsResults(Resource):
     def get(self):
        app.logger.info("Request 'EccsResults'")
 
-       file_path = "logs/eccs2_2020-03-01.log"
+       file_path = "logs/eccs2_%s.log" % eccs2properties.day
        date = PurePath(file_path).parts[-1].split('_')[1].split('.')[0]
        pretty = 0
        status = None
@@ -305,10 +305,15 @@ class EccsResults(Resource):
        else:
           return jsonify(result)
 
+# Run check for a specific IDP
+# <idpdisc:DiscoveryResponse Location>?entityID=<IDP_ENITIYID>&target=<DESTINATION_RESOURCE_URL> (tutto url encoded)
+#class RunCheck(Resource):
+#    def get(self):
 
 api.add_resource(Test, '/eccs/test') # Route_1
 api.add_resource(Checks, '/eccs/checks') # Route_2
 api.add_resource(EccsResults, '/eccs/eccsresults') # Route_3
+#api.add_resource(RunCheck, '/eccs/runcheck') # Route_4
 
 if __name__ == '__main__':
    

eccs2.py

-#!/usr/bin/env python3
+#!/usr/bin/env python3.8
 
-from datetime import date
+import argparse
+import json
 import logging
+import time
+import os
+import eccs2properties
+import psutil
+import signal
+import re
+import requests
+
+from datetime import date
+from selenium import webdriver
+from selenium.webdriver.common.by import By
+from selenium.webdriver.support.ui import Select
+from selenium.webdriver.common.keys import Keys
+from selenium.common.exceptions import NoSuchElementException
+from selenium.common.exceptions import TimeoutException
 
 
 """
@@ -31,47 +47,61 @@ def getIdpListFromUrl():
 def getIdpListFromFile():
    import json
 
-   with open('list_eccs_idps-idem.txt','r',encoding='utf-8') as f:
+   #with open('list_eccs_idps-idem.txt','r',encoding='utf-8') as f:
+   with open('federation_idps.txt','r',encoding='utf-8') as f:
       json_data = json.loads(f.read())
       return json_data
 
 
 def checkIdP(sp,idp,logger):
-   from selenium import webdriver
-   from selenium.webdriver.common.by import By
-   from selenium.webdriver.support.ui import Select
-   from selenium.webdriver.common.keys import Keys
-   from selenium.common.exceptions import NoSuchElementException
-   from selenium.common.exceptions import TimeoutException
-   import re
+   # Disable SSL requests warning messages
+   requests.packages.urllib3.disable_warnings()
 
    # Configure Web-driver
    chrome_options = webdriver.ChromeOptions()
    chrome_options.add_argument('--headless')
    chrome_options.add_argument('--no-sandbox')
+   chrome_options.add_argument('--disable-dev-shm-usage')
+   chrome_options.add_argument('--ignore-certificate-errors')
 
-#   driver = webdriver.Chrome('chromedriver', chrome_options=chrome_options,  service_args=['--verbose', '--log-path=./selenium_chromedriver.log'])
-   driver = webdriver.Chrome('chromedriver', chrome_options=chrome_options)
+   driver = webdriver.Chrome('chromedriver', options=chrome_options,  service_args=['--log-path=./selenium_chromedriver.log'])
+   #driver = webdriver.Chrome('chromedriver', chrome_options=chrome_options,  service_args=['--verbose', '--log-path=./selenium_chromedriver.log'])
+   #driver = webdriver.Chrome('chromedriver', chrome_options=chrome_options)
 
-   # Configure timeouts: 45 sec
-   driver.set_page_load_timeout(45)
-   driver.set_script_timeout(45)
+   # Configure timeouts: 30 sec
+   driver.set_page_load_timeout(30)
+   driver.set_script_timeout(30)
 
    # Configure Blacklists
-   federation_blacklist = ['http://www.surfconext.nl/','https://www.wayf.dk','http://feide.no/']
-   entities_blacklist = ['https://idp.eie.gr/idp/shibboleth','https://gn-vho.grnet.gr/idp/shibboleth','https://wtc.tu-chemnitz.de/shibboleth','https://wtc.tu-chemnitz.de/shibboleth','https://idp.fraunhofer.de/idp/shibboleth','https://login.hs-owl.de/nidp/saml2/metadata','https://idp.dfn-cert.de/idp/shibboleth']
+   federation_blacklist = [
+           'http://www.surfconext.nl/',
+           'https://www.wayf.dk',
+           'http://feide.no/'
+           ]
+
+   entities_blacklist = [ 
+           'https://idp.eie.gr/idp/shibboleth',
+           'https://gn-vho.grnet.gr/idp/shibboleth',
+           'https://wtc.tu-chemnitz.de/shibboleth',
+           'https://wtc.tu-chemnitz.de/shibboleth',
+           'https://idp.fraunhofer.de/idp/shibboleth',
+           'https://login.hs-owl.de/nidp/saml2/metadata',
+           'https://idp.dfn-cert.de/idp/shibboleth'
+           ]
 
-   if (idp['entityID'] in entities_blacklist):
-      logger.info("%s;%s;IdP excluded from checks" % (idp['entityID'],sp))
+   if (idp['registrationAuthority'] in federation_blacklist):
+      logger.info("%s;%s;NULL;Federation excluded from checks" % (idp['entityID'],sp))
       driver.close()
       driver.quit()
       return "DISABLED"
-   if (idp['registrationAuthority'] in federation_blacklist):
-      logger.info("%s;%s;Federation excluded from checks" % (idp['entityID'],sp))
+
+   if (idp['entityID'] in entities_blacklist):
+      logger.info("%s;%s;NULL;IdP excluded from checks" % (idp['entityID'],sp))
       driver.close()
       driver.quit()
       return "DISABLED"
 
+
    # Open SP, select the IDP from the EDS and press 'Enter' to reach the IdP login page to check
    try:
       driver.get(sp)
@@ -79,14 +109,17 @@ def checkIdP(sp,idp,logger):
 
       driver.find_element_by_id("username")
       driver.find_element_by_id("password")
-
-   except NoSuchElementException as e:
-     pass
    except TimeoutException as e:
-     logger.info("%s;%s;TIMEOUT" % (idp['entityID'],sp))
+     driver.delete_all_cookies()
+     print("TIMEOUT - driver.current_url: %s" % (driver.current_url))
+     status_code = requests.get(driver.current_url, verify=False).status_code
+     logger.info("%s;%s;%s;TIMEOUT" % (idp['entityID'],sp,status_code))
      driver.close()
      driver.quit()
      return "TIMEOUT"
+   except NoSuchElementException as e:
+     driver.delete_all_cookies()
+     pass
 
    pattern_metadata = "Unable.to.locate(\sissuer.in|).metadata(\sfor|)|no.metadata.found|profile.is.not.configured.for.relying.party|Cannot.locate.entity|fail.to.load.unknown.provider|does.not.recognise.the.service|unable.to.load.provider|Nous.n'avons.pas.pu.(charg|charger).le.fournisseur.de service|Metadata.not.found|application.you.have.accessed.is.not.registered.for.use.with.this.service|Message.did.not.meet.security.requirements"
 
@@ -98,17 +131,26 @@ def checkIdP(sp,idp,logger):
    password_found = re.search(pattern_password,driver.page_source, re.I)
 
    if(metadata_not_found):
-      logger.info("%s;%s;No-eduGAIN-Metadata" % (idp['entityID'],sp))
+      #print("MD-NOT-FOUND - driver.current_url: %s" % (driver.current_url))
+      status_code = requests.get(driver.current_url, verify=False).status_code
+      logger.info("%s;%s;%s;No-eduGAIN-Metadata" % (idp['entityID'],sp,status_code))
+      driver.delete_all_cookies()
       driver.close()
       driver.quit()
       return "No-eduGAIN-Metadata"
-   elif not username_found and not password_found:
-      logger.info("%s;%s;Invalid-Form" % (idp['entityID'],sp))
+   elif not username_found or not password_found:
+      #print("INVALID-FORM - entityID: %s, sp: %s, driver.current_url: %s" % (idp['entityID'],sp,driver.current_url))
+      status_code = requests.get(driver.current_url, verify=False).status_code
+      logger.info("%s;%s;%s;Invalid-Form" % (idp['entityID'],sp,status_code))
+      driver.delete_all_cookies()
       driver.close()
       driver.quit()
       return "Invalid Form"
    else:
-      logger.info("%s;%s;OK" % (idp['entityID'],sp))
+      #print("MD-FOUND - driver.current_url: %s" % (driver.current_url))
+      status_code = requests.get(driver.current_url, verify=False).status_code
+      logger.info("%s;%s;%s;OK" % (idp['entityID'],sp,status_code))
+      driver.delete_all_cookies()
       driver.close()
       driver.quit()
       return "OK"
@@ -118,7 +160,7 @@ def checkIdP(sp,idp,logger):
 def getLogger(filename,log_level="DEBUG",path="./"):
 
     logger = logging.getLogger(filename)
-    ch = logging.FileHandler(path+filename,'w','utf-8')
+    ch = logging.FileHandler(path+filename,'a','utf-8')
 
     if (log_level == "DEBUG"):
        logger.setLevel(logging.DEBUG)
@@ -154,23 +196,11 @@ def getIdPContacts(idp,contactType):
 
    return ctcList
 
-# MAIN
-if __name__=="__main__":
-
-   day = date.today().isoformat() 
-
-   eccs2log = getLogger("logs/eccs2_"+day+".log","INFO")
-   eccs2checksLog = getLogger("logs/eccs2checks_"+day+".log","INFO")
-
-   sps = ["https://sp24-test.garr.it/secure", "https://attribute-viewer.aai.switch.ch/eds/"]
-
-   #listIdPs = getIdpListFromUrl()
-   listIdPs = getIdpListFromFile()
-
-   for idp in listIdPs:
+def checkIdp(idp,sps,eccs2log,eccs2checksLog):
       result = []
       for sp in sps:
-         result.append(checkIdP(sp,idp,eccs2checksLog))
+         resultCheck = checkIdP(sp,idp,eccs2checksLog)
+         result.append(resultCheck)
 
       listTechContacts = getIdPContacts(idp,'technical')
       listSuppContacts = getIdPContacts(idp,'support')
@@ -216,3 +246,21 @@ if __name__=="__main__":
              result[0],
              sps[1],
              result[1]))
+
+# MAIN
+if __name__=="__main__":
+
+   eccs2log = getLogger("logs/"+eccs2properties.ECCS2LOGPATH,"INFO")
+   eccs2checksLog = getLogger("logs/"+eccs2properties.ECCS2CHECKSLOGPATH,"INFO")
+
+   sps = ["https://sp24-test.garr.it/secure", "https://attribute-viewer.aai.switch.ch/eds/"]
+   #sps = ["https://attribute-viewer.aai.switch.ch/eds/", "https://attribute-viewer.aai.switch.ch/eds/"]
+
+   parser = argparse.ArgumentParser(description='Checks if the input IdP consumed correctly eduGAIN metadata by accessing two different SPs')
+   parser.add_argument("idpJson", metavar="idpJson", nargs=1, help="An IdP in Json format")
+
+   args = parser.parse_args()
+
+   idp = json.loads(args.idpJson[0])
+
+   checkIdp(idp,sps,eccs2log,eccs2checksLog)

runEccs2.py

+#!/usr/bin/env python3.8
+
+import asyncio
+import eccs2properties
+import json
+import sys
+import time
+
+from subprocess import Popen,PIPE
+
+def getIdPs():
+   import certifi
+   import urllib3
+   import json
+
+   manager = urllib3.PoolManager(
+               cert_reqs='CERT_REQUIRED',
+               ca_certs=certifi.where()
+             )
+
+   url = "https://technical.edugain.org/api.php?action=list_eccs_idps"
+   idp_json = manager.request('GET', url)
+
+   idp_dict = json.loads(idp_json.data.decode('utf-8'))
+
+   idp_list = []
+
+   #federation = input("Insert the registrationAuthority: ")
+   federation = "http://www.idem.garr.it/"
+
+   for idp in idp_dict:
+      if (idp['registrationAuthority'] == federation):
+         idp_list.append(idp)
+
+   return json.dumps(idp_list)
+
+
+def getIdpListFromFile():
+   import json
+
+   #with open('list_eccs_idps-idem.txt','r',encoding='utf-8') as f:
+   with open('federation_idps.txt','r',encoding='utf-8') as f:
+      json_data = json.loads(f.read())
+      return json_data
+
+
+async def run(name,queue,stdout_file,stderr_file):
+   while True:
+      # Get a "cmd item" out of the queue.
+      cmd = await queue.get()
+
+      # Elaborate "cmd" from shell.
+      proc = await asyncio.create_subprocess_shell(
+                   cmd,
+                   stdout=asyncio.subprocess.PIPE,
+                   stderr=asyncio.subprocess.PIPE
+             )
+
+      stdout, stderr = await proc.communicate()
+
+      if stdout:
+         stdout_file.write(f'[stdout]\n{stdout.decode()}')
+      if stderr:
+         stderr_file.write(f'[stderr]\n{stderr.decode()}\n\n[cmd]\n{cmd}')
+
+      # Notify the queue that the "work cmd" has been processed.
+      queue.task_done()
+
+
+async def main(cmd_list,stdout_file,stderr_file):
+    # Create a queue that we will use to store our "workload".
+    queue = asyncio.Queue()
+
+    # Put all commands into the queue.
+    for cmd in cmd_list:
+        queue.put_nowait(cmd)
+
+    # Create worker tasks to process the queue concurrently.
+    tasks = []
+    for i in range(30):
+        task = asyncio.create_task(run("cmd-{%d}" % i, queue, stdout_file, stderr_file))
+        tasks.append(task)
+
+    # Wait until the queue is fully processed.
+    started_at = time.monotonic()
+    await queue.join()
+    total_slept_for = time.monotonic() - started_at
+
+    # Cancel our worker tasks.
+    for task in tasks:
+        task.cancel()
+
+    # Wait until all worker tasks are cancelled.
+    await asyncio.gather(*tasks, return_exceptions=True)
+
+
+# MAIN
+if __name__=="__main__":
+
+ start = time.time()
+
+ '''
+ data = getIdPs()
+
+ f = open('federation_idps.txt', 'w')
+ f.write(data)
+ f.close()
+ '''
+ stdout_file = open(eccs2properties.ECCS2STDOUT,"w+")
+ stderr_file = open(eccs2properties.ECCS2STDERR,"w+")
+
+ idpJsonList = getIdpListFromFile()
+ num_idps = len(idpJsonList)
+ cmd_list = [["%s/eccs2.py \'%s\'" % (eccs2properties.ECCS2PATH, json.dumps(idp))] for idp in idpJsonList]
+
+ proc_list = []
+ count = 0
+ while (count < num_idps):
+       cmd = "".join(cmd_list.pop())
+       proc_list.append(cmd)
+       count = count + 1
+ 
+ asyncio.run(main(proc_list,stdout_file,stderr_file))
+
+ end = time.time()
+ print("Time taken in seconds - ", end - start)

web/index.php

@@ -2,7 +2,7 @@
 <html>
    <head>
  
-      <script type="text/javascript" src="https://code.jquery.com/jquery-3.3.1.js"></script>
+      <script type="text/javascript" src="https://code.jquery.com/jquery-3.4.1.js"></script>
       <script type="text/javascript" src="https://cdn.datatables.net/1.10.20/js/jquery.dataTables.min.js"></script>
       <link rel="stylesheet" type="text/css" href="https://cdn.datatables.net/1.10.20/css/jquery.dataTables.min.css"/>
 
@@ -14,7 +14,7 @@
   </head>
   <body>
     <div class="container">
-      <table id="example" class="display" style="width:100%">
+      <table id="eccstable" class="display" style="width:100%">
         <thead>
             <tr>
                 <th></th>
@@ -22,8 +22,7 @@
                 <th>EntityID</th>
                 <th>Registration Authority</th>
                 <th>Technical Contacts</th>
-                <th>Support Contacts</th>
-                <th>Date</th>
+                <th>Check Date</th>
                 <th>Status</th>
             </tr>
         </thead>

web/script.js

@@ -12,6 +12,11 @@ function format ( d ) {
             '<td>'+d.contacts.technical+'</td>'+
             '<td></td>'+
         '</tr>'+
+        '<tr>'+
+            '<td>Support Contacts:</td>'+
+            '<td>'+d.contacts.support+'</td>'+
+            '<td></td>'+
+        '</tr>'+
         '<tr>'+
             '<td>SP1:</td>'+
             '<td>'+d.sp1.entityID+'</td>'+
@@ -26,7 +31,7 @@ function format ( d ) {
 }
  
 $(document).ready(function() {
-    var table = $('#example').DataTable( {
+    var table = $('#eccstable').DataTable( {
         "ajax": { 
            "url": "data.json",
            "dataSrc": ""
@@ -46,11 +51,7 @@ $(document).ready(function() {
             { "data": "registrationAuthority" },
             { 
               "data": "contacts.technical",
-              "defaultContent": ''
-            },
-            { 
-              "data": "contacts.support",
-              "defaultContent": ''
+              "defaultContent": ""
             },
             { "data": "date" },
             { "data": "status" }
@@ -70,7 +71,7 @@ $(document).ready(function() {
     } );
      
     // Add event listener for opening and closing details
-    $('#example tbody').on('click', 'td.details-control', function () {
+    $('#eccstable tbody').on('click', 'td.details-control', function () {
         var tr = $(this).closest('tr');
         var row = table.row( tr );