diff --git a/README.md b/README.md
index 19ae16e49b22583d37f6abdcf27bf5f68b813f6e..b87f877f00122a6a20fd0453fb9283e029fd1ad7 100644
--- a/README.md
+++ b/README.md
@@ -3,11 +3,17 @@
* `sudo apt install python3 python3-pip chromium chromium-l10n git libapache2-mod-wsgi python3-dev`
* `python3 -m pip install --user --upgrade pip virtualenv`
* `python3 -m venv eccs2venv`
-* `source eccs2venv/bin/activate` (`deactivate` di exit Virtualenv)
+* `source eccs2venv/bin/activate` (`deactivate` to exit Virtualenv)
* `python3 -m pip install --upgrade wheel setuptools certifi selenium urllib3 flask flask-jsonpify flask-restful`
* `cd ~ ; git clone https://github.com/malavolti/eccs2.git`
* `cd eccs2 ; ./eccs2.py`
+# API Development Server
+
+* `sudo apt install libapache2-mod-wsgi-py3 python3-dev`
+* `sudo a2enmod wsgi`
+* `cd ~/eccs2 ; ./api.py`
+
# API
* `/eccs/test` (Trivial Test)
@@ -23,8 +29,4 @@
* 'Excluded'
* /eccs/eccsresults (Return the results of the last check ready for ECCS Gui)
-# API Development Server
-* `sudo apt install libapache2-mod-wsgi-py3 python3-dev`
-* `sudo a2enmod wsgi`
-* `cd ~/eccs2 ; ./api.py`
diff --git a/api.py b/api.py
index b0b3b619c3a4aec22ea7180dfcd1befb3ba6b0b5..b5d8c00a268e31e78b2b9081eeb478e0bbfaa2c4 100755
--- a/api.py
+++ b/api.py
@@ -8,6 +8,7 @@ from pathlib import PurePath
import logging
from logging.handlers import RotatingFileHandler
import re
+import eccs2properties
app = Flask(__name__)
@@ -67,7 +68,7 @@ class Checks(Resource):
def get(self):
app.logger.info("Request 'Checks'")
- file_path = "logs/eccs2checks_2020-02-22.log"
+ file_path = "logs/eccs2checks_%s.log" % eccs2properties.day
date = PurePath(file_path).parts[-1].split('_')[1].split('.')[0]
pretty = 0
status = None
@@ -142,7 +143,6 @@ class Checks(Resource):
# Build Email Addresses Link for ECCS2 Web Gui
def buildEmailAddress(listContacts):
-
listCtcs = listContacts.split(",")
hrefList = []
@@ -155,7 +155,7 @@ class EccsResults(Resource):
def get(self):
app.logger.info("Request 'EccsResults'")
- file_path = "logs/eccs2_2020-03-01.log"
+ file_path = "logs/eccs2_%s.log" % eccs2properties.day
date = PurePath(file_path).parts[-1].split('_')[1].split('.')[0]
pretty = 0
status = None
@@ -305,10 +305,15 @@ class EccsResults(Resource):
else:
return jsonify(result)
+# Run check for a specific IDP
+# <idpdisc:DiscoveryResponse Location>?entityID=<IDP_ENITIYID>&target=<DESTINATION_RESOURCE_URL> (tutto url encoded)
+#class RunCheck(Resource):
+# def get(self):
api.add_resource(Test, '/eccs/test') # Route_1
api.add_resource(Checks, '/eccs/checks') # Route_2
api.add_resource(EccsResults, '/eccs/eccsresults') # Route_3
+#api.add_resource(RunCheck, '/eccs/runcheck') # Route_4
if __name__ == '__main__':
diff --git a/eccs2.py b/eccs2.py
index 150211788cdf98aa0a908f4e9bf590917e32f851..cca7f7858f97af9a888848066b2cec9f70c3ddb9 100755
--- a/eccs2.py
+++ b/eccs2.py
@@ -1,7 +1,23 @@
-#!/usr/bin/env python3
+#!/usr/bin/env python3.8
-from datetime import date
+import argparse
+import json
import logging
+import time
+import os
+import eccs2properties
+import psutil
+import signal
+import re
+import requests
+
+from datetime import date
+from selenium import webdriver
+from selenium.webdriver.common.by import By
+from selenium.webdriver.support.ui import Select
+from selenium.webdriver.common.keys import Keys
+from selenium.common.exceptions import NoSuchElementException
+from selenium.common.exceptions import TimeoutException
"""
@@ -31,47 +47,61 @@ def getIdpListFromUrl():
def getIdpListFromFile():
import json
- with open('list_eccs_idps-idem.txt','r',encoding='utf-8') as f:
+ #with open('list_eccs_idps-idem.txt','r',encoding='utf-8') as f:
+ with open('federation_idps.txt','r',encoding='utf-8') as f:
json_data = json.loads(f.read())
return json_data
def checkIdP(sp,idp,logger):
- from selenium import webdriver
- from selenium.webdriver.common.by import By
- from selenium.webdriver.support.ui import Select
- from selenium.webdriver.common.keys import Keys
- from selenium.common.exceptions import NoSuchElementException
- from selenium.common.exceptions import TimeoutException
- import re
+ # Disable SSL requests warning messages
+ requests.packages.urllib3.disable_warnings()
# Configure Web-driver
chrome_options = webdriver.ChromeOptions()
chrome_options.add_argument('--headless')
chrome_options.add_argument('--no-sandbox')
+ chrome_options.add_argument('--disable-dev-shm-usage')
+ chrome_options.add_argument('--ignore-certificate-errors')
-# driver = webdriver.Chrome('chromedriver', chrome_options=chrome_options, service_args=['--verbose', '--log-path=./selenium_chromedriver.log'])
- driver = webdriver.Chrome('chromedriver', chrome_options=chrome_options)
+ driver = webdriver.Chrome('chromedriver', options=chrome_options, service_args=['--log-path=./selenium_chromedriver.log'])
+ #driver = webdriver.Chrome('chromedriver', chrome_options=chrome_options, service_args=['--verbose', '--log-path=./selenium_chromedriver.log'])
+ #driver = webdriver.Chrome('chromedriver', chrome_options=chrome_options)
- # Configure timeouts: 45 sec
- driver.set_page_load_timeout(45)
- driver.set_script_timeout(45)
+ # Configure timeouts: 30 sec
+ driver.set_page_load_timeout(30)
+ driver.set_script_timeout(30)
# Configure Blacklists
- federation_blacklist = ['http://www.surfconext.nl/','https://www.wayf.dk','http://feide.no/']
- entities_blacklist = ['https://idp.eie.gr/idp/shibboleth','https://gn-vho.grnet.gr/idp/shibboleth','https://wtc.tu-chemnitz.de/shibboleth','https://wtc.tu-chemnitz.de/shibboleth','https://idp.fraunhofer.de/idp/shibboleth','https://login.hs-owl.de/nidp/saml2/metadata','https://idp.dfn-cert.de/idp/shibboleth']
+ federation_blacklist = [
+ 'http://www.surfconext.nl/',
+ 'https://www.wayf.dk',
+ 'http://feide.no/'
+ ]
+
+ entities_blacklist = [
+ 'https://idp.eie.gr/idp/shibboleth',
+ 'https://gn-vho.grnet.gr/idp/shibboleth',
+ 'https://wtc.tu-chemnitz.de/shibboleth',
+ 'https://wtc.tu-chemnitz.de/shibboleth',
+ 'https://idp.fraunhofer.de/idp/shibboleth',
+ 'https://login.hs-owl.de/nidp/saml2/metadata',
+ 'https://idp.dfn-cert.de/idp/shibboleth'
+ ]
- if (idp['entityID'] in entities_blacklist):
- logger.info("%s;%s;IdP excluded from checks" % (idp['entityID'],sp))
+ if (idp['registrationAuthority'] in federation_blacklist):
+ logger.info("%s;%s;NULL;Federation excluded from checks" % (idp['entityID'],sp))
driver.close()
driver.quit()
return "DISABLED"
- if (idp['registrationAuthority'] in federation_blacklist):
- logger.info("%s;%s;Federation excluded from checks" % (idp['entityID'],sp))
+
+ if (idp['entityID'] in entities_blacklist):
+ logger.info("%s;%s;NULL;IdP excluded from checks" % (idp['entityID'],sp))
driver.close()
driver.quit()
return "DISABLED"
+
# Open SP, select the IDP from the EDS and press 'Enter' to reach the IdP login page to check
try:
driver.get(sp)
@@ -79,14 +109,17 @@ def checkIdP(sp,idp,logger):
driver.find_element_by_id("username")
driver.find_element_by_id("password")
-
- except NoSuchElementException as e:
- pass
except TimeoutException as e:
- logger.info("%s;%s;TIMEOUT" % (idp['entityID'],sp))
+ driver.delete_all_cookies()
+ print("TIMEOUT - driver.current_url: %s" % (driver.current_url))
+ status_code = requests.get(driver.current_url, verify=False).status_code
+ logger.info("%s;%s;%s;TIMEOUT" % (idp['entityID'],sp,status_code))
driver.close()
driver.quit()
return "TIMEOUT"
+ except NoSuchElementException as e:
+ driver.delete_all_cookies()
+ pass
pattern_metadata = "Unable.to.locate(\sissuer.in|).metadata(\sfor|)|no.metadata.found|profile.is.not.configured.for.relying.party|Cannot.locate.entity|fail.to.load.unknown.provider|does.not.recognise.the.service|unable.to.load.provider|Nous.n'avons.pas.pu.(charg|charger).le.fournisseur.de service|Metadata.not.found|application.you.have.accessed.is.not.registered.for.use.with.this.service|Message.did.not.meet.security.requirements"
@@ -98,17 +131,26 @@ def checkIdP(sp,idp,logger):
password_found = re.search(pattern_password,driver.page_source, re.I)
if(metadata_not_found):
- logger.info("%s;%s;No-eduGAIN-Metadata" % (idp['entityID'],sp))
+ #print("MD-NOT-FOUND - driver.current_url: %s" % (driver.current_url))
+ status_code = requests.get(driver.current_url, verify=False).status_code
+ logger.info("%s;%s;%s;No-eduGAIN-Metadata" % (idp['entityID'],sp,status_code))
+ driver.delete_all_cookies()
driver.close()
driver.quit()
return "No-eduGAIN-Metadata"
- elif not username_found and not password_found:
- logger.info("%s;%s;Invalid-Form" % (idp['entityID'],sp))
+ elif not username_found or not password_found:
+ #print("INVALID-FORM - entityID: %s, sp: %s, driver.current_url: %s" % (idp['entityID'],sp,driver.current_url))
+ status_code = requests.get(driver.current_url, verify=False).status_code
+ logger.info("%s;%s;%s;Invalid-Form" % (idp['entityID'],sp,status_code))
+ driver.delete_all_cookies()
driver.close()
driver.quit()
return "Invalid Form"
else:
- logger.info("%s;%s;OK" % (idp['entityID'],sp))
+ #print("MD-FOUND - driver.current_url: %s" % (driver.current_url))
+ status_code = requests.get(driver.current_url, verify=False).status_code
+ logger.info("%s;%s;%s;OK" % (idp['entityID'],sp,status_code))
+ driver.delete_all_cookies()
driver.close()
driver.quit()
return "OK"
@@ -118,7 +160,7 @@ def checkIdP(sp,idp,logger):
def getLogger(filename,log_level="DEBUG",path="./"):
logger = logging.getLogger(filename)
- ch = logging.FileHandler(path+filename,'w','utf-8')
+ ch = logging.FileHandler(path+filename,'a','utf-8')
if (log_level == "DEBUG"):
logger.setLevel(logging.DEBUG)
@@ -154,23 +196,11 @@ def getIdPContacts(idp,contactType):
return ctcList
-# MAIN
-if __name__=="__main__":
-
- day = date.today().isoformat()
-
- eccs2log = getLogger("logs/eccs2_"+day+".log","INFO")
- eccs2checksLog = getLogger("logs/eccs2checks_"+day+".log","INFO")
-
- sps = ["https://sp24-test.garr.it/secure", "https://attribute-viewer.aai.switch.ch/eds/"]
-
- #listIdPs = getIdpListFromUrl()
- listIdPs = getIdpListFromFile()
-
- for idp in listIdPs:
+def checkIdp(idp,sps,eccs2log,eccs2checksLog):
result = []
for sp in sps:
- result.append(checkIdP(sp,idp,eccs2checksLog))
+ resultCheck = checkIdP(sp,idp,eccs2checksLog)
+ result.append(resultCheck)
listTechContacts = getIdPContacts(idp,'technical')
listSuppContacts = getIdPContacts(idp,'support')
@@ -216,3 +246,21 @@ if __name__=="__main__":
result[0],
sps[1],
result[1]))
+
+# MAIN
+if __name__=="__main__":
+
+ eccs2log = getLogger("logs/"+eccs2properties.ECCS2LOGPATH,"INFO")
+ eccs2checksLog = getLogger("logs/"+eccs2properties.ECCS2CHECKSLOGPATH,"INFO")
+
+ sps = ["https://sp24-test.garr.it/secure", "https://attribute-viewer.aai.switch.ch/eds/"]
+ #sps = ["https://attribute-viewer.aai.switch.ch/eds/", "https://attribute-viewer.aai.switch.ch/eds/"]
+
+ parser = argparse.ArgumentParser(description='Checks if the input IdP consumed correctly eduGAIN metadata by accessing two different SPs')
+ parser.add_argument("idpJson", metavar="idpJson", nargs=1, help="An IdP in Json format")
+
+ args = parser.parse_args()
+
+ idp = json.loads(args.idpJson[0])
+
+ checkIdp(idp,sps,eccs2log,eccs2checksLog)
diff --git a/runEccs2.py b/runEccs2.py
new file mode 100755
index 0000000000000000000000000000000000000000..664f7dbc481dce29e68af6f81a46efd121764ebc
--- /dev/null
+++ b/runEccs2.py
@@ -0,0 +1,126 @@
+#!/usr/bin/env python3.8
+
+import asyncio
+import eccs2properties
+import json
+import sys
+import time
+
+from subprocess import Popen,PIPE
+
+def getIdPs():
+ import certifi
+ import urllib3
+ import json
+
+ manager = urllib3.PoolManager(
+ cert_reqs='CERT_REQUIRED',
+ ca_certs=certifi.where()
+ )
+
+ url = "https://technical.edugain.org/api.php?action=list_eccs_idps"
+ idp_json = manager.request('GET', url)
+
+ idp_dict = json.loads(idp_json.data.decode('utf-8'))
+
+ idp_list = []
+
+ #federation = input("Insert the registrationAuthority: ")
+ federation = "http://www.idem.garr.it/"
+
+ for idp in idp_dict:
+ if (idp['registrationAuthority'] == federation):
+ idp_list.append(idp)
+
+ return json.dumps(idp_list)
+
+
+def getIdpListFromFile():
+ import json
+
+ #with open('list_eccs_idps-idem.txt','r',encoding='utf-8') as f:
+ with open('federation_idps.txt','r',encoding='utf-8') as f:
+ json_data = json.loads(f.read())
+ return json_data
+
+
+async def run(name,queue,stdout_file,stderr_file):
+ while True:
+ # Get a "cmd item" out of the queue.
+ cmd = await queue.get()
+
+ # Elaborate "cmd" from shell.
+ proc = await asyncio.create_subprocess_shell(
+ cmd,
+ stdout=asyncio.subprocess.PIPE,
+ stderr=asyncio.subprocess.PIPE
+ )
+
+ stdout, stderr = await proc.communicate()
+
+ if stdout:
+ stdout_file.write(f'[stdout]\n{stdout.decode()}')
+ if stderr:
+ stderr_file.write(f'[stderr]\n{stderr.decode()}\n\n[cmd]\n{cmd}')
+
+ # Notify the queue that the "work cmd" has been processed.
+ queue.task_done()
+
+
+async def main(cmd_list,stdout_file,stderr_file):
+ # Create a queue that we will use to store our "workload".
+ queue = asyncio.Queue()
+
+ # Put all commands into the queue.
+ for cmd in cmd_list:
+ queue.put_nowait(cmd)
+
+ # Create worker tasks to process the queue concurrently.
+ tasks = []
+ for i in range(30):
+ task = asyncio.create_task(run("cmd-{%d}" % i, queue, stdout_file, stderr_file))
+ tasks.append(task)
+
+ # Wait until the queue is fully processed.
+ started_at = time.monotonic()
+ await queue.join()
+ total_slept_for = time.monotonic() - started_at
+
+ # Cancel our worker tasks.
+ for task in tasks:
+ task.cancel()
+
+ # Wait until all worker tasks are cancelled.
+ await asyncio.gather(*tasks, return_exceptions=True)
+
+
+# MAIN
+if __name__=="__main__":
+
+ start = time.time()
+
+ '''
+ data = getIdPs()
+
+ f = open('federation_idps.txt', 'w')
+ f.write(data)
+ f.close()
+ '''
+ stdout_file = open(eccs2properties.ECCS2STDOUT,"w+")
+ stderr_file = open(eccs2properties.ECCS2STDERR,"w+")
+
+ idpJsonList = getIdpListFromFile()
+ num_idps = len(idpJsonList)
+ cmd_list = [["%s/eccs2.py \'%s\'" % (eccs2properties.ECCS2PATH, json.dumps(idp))] for idp in idpJsonList]
+
+ proc_list = []
+ count = 0
+ while (count < num_idps):
+ cmd = "".join(cmd_list.pop())
+ proc_list.append(cmd)
+ count = count + 1
+
+ asyncio.run(main(proc_list,stdout_file,stderr_file))
+
+ end = time.time()
+ print("Time taken in seconds - ", end - start)
diff --git a/web/index.php b/web/index.php
index b1de390cc920a1e525cf3ecfe17092ad43b3ed13..8bead0e5d81cf2869df594c5a9a3d558ef83b639 100644
--- a/web/index.php
+++ b/web/index.php
@@ -2,7 +2,7 @@
<html>
<head>
- <script type="text/javascript" src="https://code.jquery.com/jquery-3.3.1.js"></script>
+ <script type="text/javascript" src="https://code.jquery.com/jquery-3.4.1.js"></script>
<script type="text/javascript" src="https://cdn.datatables.net/1.10.20/js/jquery.dataTables.min.js"></script>
<link rel="stylesheet" type="text/css" href="https://cdn.datatables.net/1.10.20/css/jquery.dataTables.min.css"/>
@@ -14,7 +14,7 @@
</head>
<body>
<div class="container">
- <table id="example" class="display" style="width:100%">
+ <table id="eccstable" class="display" style="width:100%">
<thead>
<tr>
<th></th>
@@ -22,8 +22,7 @@
<th>EntityID</th>
<th>Registration Authority</th>
<th>Technical Contacts</th>
- <th>Support Contacts</th>
- <th>Date</th>
+ <th>Check Date</th>
<th>Status</th>
</tr>
</thead>
diff --git a/web/script.js b/web/script.js
index ba653f25b3aee1291e3e60b9eed5bdffb58b3170..300d347742e42140665e75664d2c25033d11c679 100644
--- a/web/script.js
+++ b/web/script.js
@@ -12,6 +12,11 @@ function format ( d ) {
'<td>'+d.contacts.technical+'</td>'+
'<td></td>'+
'</tr>'+
+ '<tr>'+
+ '<td>Support Contacts:</td>'+
+ '<td>'+d.contacts.support+'</td>'+
+ '<td></td>'+
+ '</tr>'+
'<tr>'+
'<td>SP1:</td>'+
'<td>'+d.sp1.entityID+'</td>'+
@@ -26,7 +31,7 @@ function format ( d ) {
}
$(document).ready(function() {
- var table = $('#example').DataTable( {
+ var table = $('#eccstable').DataTable( {
"ajax": {
"url": "data.json",
"dataSrc": ""
@@ -46,11 +51,7 @@ $(document).ready(function() {
{ "data": "registrationAuthority" },
{
"data": "contacts.technical",
- "defaultContent": ''
- },
- {
- "data": "contacts.support",
- "defaultContent": ''
+ "defaultContent": ""
},
{ "data": "date" },
{ "data": "status" }
@@ -70,7 +71,7 @@ $(document).ready(function() {
} );
// Add event listener for opening and closing details
- $('#example tbody').on('click', 'td.details-control', function () {
+ $('#eccstable tbody').on('click', 'td.details-control', function () {
var tr = $(this).closest('tr');
var row = table.row( tr );